From 9aa468adda4387106143bf0e2565f1ca24221edc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:01:29 +0200 Subject: [PATCH 01/50] fix for typehint --- freqtrade/fiat_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 17882f51a..3d268fdbe 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -34,7 +34,7 @@ class CryptoFiat(object): self.price = 0.0 # Private attributes - self._expiration = 0 + self._expiration = 0.0 self.crypto_symbol = crypto_symbol.upper() self.fiat_symbol = fiat_symbol.upper() From 0d6dffdc7ecfa71f51816b46ef1c39239c9741b2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:09:03 +0200 Subject: [PATCH 02/50] fix typehinting --- freqtrade/fiat_convert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 3d268fdbe..88eb702c9 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -5,7 +5,7 @@ e.g BTC to USD import logging import time -from typing import Dict +from typing import Dict, List from coinmarketcap import Market from requests.exceptions import RequestException @@ -65,7 +65,7 @@ class CryptoToFiatConverter(object): This object is also a Singleton """ __instance = None - _coinmarketcap = None + _coinmarketcap: Market = None # Constants SUPPORTED_FIAT = [ @@ -87,7 +87,7 @@ class CryptoToFiatConverter(object): return CryptoToFiatConverter.__instance def __init__(self) -> None: - self._pairs = [] + self._pairs: List[CryptoFiat] = [] self._load_cryptomap() def _load_cryptomap(self) -> None: From 88755fcded808c2c094b6ee317affbcd1894bf7c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:09:20 +0200 Subject: [PATCH 03/50] fix typing --- freqtrade/indicator_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index 14519d7a2..50586578a 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -13,7 +13,7 @@ def went_down(series: Series) -> bool: return series < series.shift(1) -def ehlers_super_smoother(series: Series, smoothing: float = 6) -> type(Series): +def ehlers_super_smoother(series: Series, smoothing: float = 6) -> Series: magic = pi * sqrt(2) / smoothing a1 = exp(-magic) coeff2 = 2 * a1 * cos(magic) From 45909af7e003770d903460e40051d8b79d49655b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:38:09 +0200 Subject: [PATCH 04/50] type anotation fixes --- freqtrade/configuration.py | 6 +++--- freqtrade/misc.py | 2 +- freqtrade/optimize/__init__.py | 6 ++++-- freqtrade/persistence.py | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 43d0a0bf9..14c29e8fd 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -5,7 +5,7 @@ This module contains the configuration class import json import logging from argparse import Namespace -from typing import Dict, Any +from typing import Optional, Dict, Any from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match import ccxt @@ -23,7 +23,7 @@ class Configuration(object): """ def __init__(self, args: Namespace) -> None: self.args = args - self.config = None + self.config: Optional[Dict[str, Any]] = None def load_config(self) -> Dict[str, Any]: """ @@ -192,7 +192,7 @@ class Configuration(object): validate(conf, constants.CONF_SCHEMA) return conf except ValidationError as exception: - logger.fatal( + logger.critical( 'Invalid configuration. See config.json.example. Reason: %s', exception ) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 225fb32df..90a1db42b 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -83,7 +83,7 @@ def file_dump_json(filename, data, is_zip=False) -> None: json.dump(data, fp, default=str) -def format_ms_time(date: str) -> str: +def format_ms_time(date: int) -> str: """ convert MS date to readable format. : epoch-string in ms diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f6f1ba47a..58a4b07fe 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,8 +4,8 @@ import gzip import json import logging import os +from typing import Optional, List, Dict, Tuple, Any import arrow -from typing import Optional, List, Dict, Tuple from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history @@ -139,7 +139,9 @@ def download_pairs(datadir, pairs: List[str], def load_cached_data_for_updating(filename: str, tick_interval: str, - timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[list, int]: + timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[ + List[Any], + Optional[int]]: """ Load cached data and choose what part of the data should be updated """ diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2d497662e..c10599b3c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,7 +5,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional +from typing import Dict, Optional, Any import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, @@ -21,7 +21,7 @@ from sqlalchemy import inspect logger = logging.getLogger(__name__) _CONF = {} -_DECL_BASE = declarative_base() +_DECL_BASE = declarative_base() # type: Any def init(config: dict, engine: Optional[Engine] = None) -> None: From 48516e6e1e4a608ff3a0d898e0d6c286b2ecf1a0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:41:05 +0200 Subject: [PATCH 05/50] Add typehint --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..cf7db5901 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -17,7 +17,7 @@ class Arguments(object): Arguments Class. Manage the arguments received by the cli """ - def __init__(self, args: List[str], description: str): + def __init__(self, args: List[str], description: str) -> None: self.args = args self.parsed_arg = None self.parser = argparse.ArgumentParser(description=description) From 4733aad7ff30220d1a135f3fce1c813b523ff6c6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:54:37 +0200 Subject: [PATCH 06/50] mypy_typing --- freqtrade/rpc/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dd3eed001..df809cc2f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,7 +4,7 @@ This module contains class to define a RPC communications import logging from datetime import datetime, timedelta from decimal import Decimal -from typing import Tuple, Any +from typing import Dict, Tuple, Any, Optional import arrow import sqlalchemy as sql @@ -114,7 +114,7 @@ class RPC(object): self, timescale: int, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: today = datetime.utcnow().date() - profit_days = {} + profit_days: Dict[int, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): return True, '*Daily [n]:* `must be an integer greater than 0`' @@ -172,7 +172,7 @@ class RPC(object): durations = [] for trade in trades: - current_rate = None + current_rate: Optional[float] = None if not trade.open_rate: continue @@ -278,7 +278,7 @@ class RPC(object): value = fiat.convert_amount(total, 'BTC', symbol) return False, (output, total, symbol, value) - def rpc_start(self) -> (bool, str): + def rpc_start(self) -> Tuple[bool, str]: """ Handler for start. """ @@ -288,7 +288,7 @@ class RPC(object): self.freqtrade.state = State.RUNNING return False, '`Starting trader ...`' - def rpc_stop(self) -> (bool, str): + def rpc_stop(self) -> Tuple[bool, str]: """ Handler for stop. """ From 0d251cbfdd497279b93e134cb8fc4b5534273895 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:55:26 +0200 Subject: [PATCH 07/50] rpc type hints --- freqtrade/rpc/rpc_manager.py | 5 +++-- freqtrade/rpc/telegram.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 0299c793a..bedd8fdd4 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,6 +1,7 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ +from typing import Any, Optional, List import logging from freqtrade.rpc.telegram import Telegram @@ -21,8 +22,8 @@ class RPCManager(object): """ self.freqtrade = freqtrade - self.registered_modules = [] - self.telegram = None + self.registered_modules: List[str] = [] + self.telegram: Any = None self._init() def _init(self) -> None: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c640fc77b..c110b9627 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -18,7 +18,7 @@ from freqtrade.rpc.rpc import RPC logger = logging.getLogger(__name__) -def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: +def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Callable[..., Any]: """ Decorator to check if the message comes from the correct chat_id :param command_handler: Telegram CommandHandler @@ -65,7 +65,7 @@ class Telegram(RPC): """ super().__init__(freqtrade) - self._updater = None + self._updater: Updater = None self._config = freqtrade.config self._init() From 1352f135d058df6be20c404a9fe81202d7da5945 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:55:45 +0200 Subject: [PATCH 08/50] typing --- freqtrade/analyze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index dcb5376ce..5756b845c 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,7 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import get_ticker_history from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.strategy.resolver import StrategyResolver, IStrategy logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class Analyze(object): :param config: Bot configuration (use the one from Configuration()) """ self.config = config - self.strategy = StrategyResolver(self.config).strategy + self.strategy: IStrategy = StrategyResolver(self.config).strategy @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: From 4eb55acdbcea4cf53b8fc2405de2d6154d5a338d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:04:10 +0200 Subject: [PATCH 09/50] fix typing --- freqtrade/tests/test_configuration.py | 37 ++++++++++++++------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d27405d91..8ba8f8289 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,6 +6,7 @@ Unit test file for configuration.py import json from copy import deepcopy from unittest.mock import MagicMock +from argparse import Namespace import pytest from jsonschema import ValidationError @@ -37,7 +38,7 @@ def test_load_config_invalid_pair(default_conf) -> None: conf['exchange']['pair_whitelist'].append('ETH-BTC') with pytest.raises(ValidationError, match=r'.*does not match.*'): - configuration = Configuration([]) + configuration = Configuration(Namespace()) configuration._validate_config(conf) @@ -49,7 +50,7 @@ def test_load_config_missing_attributes(default_conf) -> None: conf.pop('exchange') with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): - configuration = Configuration([]) + configuration = Configuration(Namespace()) configuration._validate_config(conf) @@ -61,7 +62,7 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: read_data=json.dumps(default_conf) )) - configuration = Configuration([]) + configuration = Configuration(Namespace()) validated_conf = configuration._load_config_file('somefile') assert file_mock.call_count == 1 assert validated_conf.items() >= default_conf.items() @@ -79,7 +80,7 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: read_data=json.dumps(conf) )) - Configuration([])._load_config_file('somefile') + Configuration(Namespace())._load_config_file('somefile') assert file_mock.call_count == 1 assert log_has('Validating configuration ...', caplog.record_tuples) @@ -92,7 +93,7 @@ def test_load_config_file_exception(mocker, caplog) -> None: 'freqtrade.configuration.open', MagicMock(side_effect=FileNotFoundError('File not found')) ) - configuration = Configuration([]) + configuration = Configuration(Namespace()) with pytest.raises(SystemExit): configuration._load_config_file('somefile') @@ -128,13 +129,13 @@ def test_load_config_with_params(default_conf, mocker) -> None: read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path', '--dry-run-db', ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) validated_conf = configuration.load_config() @@ -174,12 +175,12 @@ def test_show_info(default_conf, mocker, caplog) -> None: read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--dry-run-db' ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) configuration.get_config() @@ -202,8 +203,8 @@ def test_show_info(default_conf, mocker, caplog) -> None: ) # Test the Dry run condition - configuration.config.update({'dry_run': False}) - configuration._load_common_config(configuration.config) + configuration.config.update({'dry_run': False}) # type: ignore + configuration._load_common_config(configuration.config) # type: ignore assert log_has( 'Dry run is disabled. (--dry_run_db ignored)', caplog.record_tuples @@ -218,13 +219,13 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', 'backtesting' ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) config = configuration.get_config() @@ -262,7 +263,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', @@ -275,7 +276,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--export', '/bar/foo' ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) config = configuration.get_config() @@ -326,14 +327,14 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: read_data=json.dumps(default_conf) )) - args = [ + arglist = [ 'hyperopt', '--epochs', '10', '--use-mongodb', '--spaces', 'all', ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) config = configuration.get_config() @@ -357,7 +358,7 @@ def test_check_exchange(default_conf) -> None: Test the configuration validator with a missing attribute """ conf = deepcopy(default_conf) - configuration = Configuration([]) + configuration = Configuration(Namespace()) # Test a valid exchange conf.get('exchange').update({'name': 'BITTREX'}) From 69006b8fe821948f191839762f9246cacaba9dc8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:08:11 +0200 Subject: [PATCH 10/50] flake8 --- freqtrade/persistence.py | 2 +- freqtrade/rpc/rpc_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c10599b3c..47a7ac4ab 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,7 +5,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional, Any +from typing import Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index bedd8fdd4..58e9bf2b9 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,7 +1,7 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ -from typing import Any, Optional, List +from typing import Any, List import logging from freqtrade.rpc.telegram import Telegram From 2976a50c58a1ad9e43a6f12fc9ff9ebc0c3f7f77 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:10:15 +0200 Subject: [PATCH 11/50] fix typing --- freqtrade/persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 47a7ac4ab..f9a7d1e3c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,7 +5,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional +from typing import Dict, Optional, Any import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, @@ -21,7 +21,7 @@ from sqlalchemy import inspect logger = logging.getLogger(__name__) _CONF = {} -_DECL_BASE = declarative_base() # type: Any +_DECL_BASE: Any = declarative_base() def init(config: dict, engine: Optional[Engine] = None) -> None: From c0cef7250d6bc394769b647b32405597cd5ce4e3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:22:46 +0200 Subject: [PATCH 12/50] typing - avoid variable reuse with differen ttype --- freqtrade/arguments.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index cf7db5901..2421abc9f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -19,7 +19,7 @@ class Arguments(object): def __init__(self, args: List[str], description: str) -> None: self.args = args - self.parsed_arg = None + self.parsed_arg: Optional[argparse.Namespace] = None self.parser = argparse.ArgumentParser(description=description) def _load_args(self) -> None: @@ -211,7 +211,7 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: str) -> Optional[Tuple[List, int, int]]: + def parse_timerange(text: str) -> Optional[Tuple[Tuple, Optional[int], Optional[int]]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange @@ -231,21 +231,21 @@ class Arguments(object): if match: # Regex has matched rvals = match.groups() index = 0 - start = None - stop = None + start: Optional[int] = None + stop: Optional[int] = None if stype[0]: - start = rvals[index] + starts = rvals[index] if stype[0] == 'date': - start = arrow.get(start, 'YYYYMMDD').timestamp + start = arrow.get(starts, 'YYYYMMDD').timestamp else: - start = int(start) + start = int(starts) index += 1 if stype[1]: - stop = rvals[index] + stops = rvals[index] if stype[1] == 'date': - stop = arrow.get(stop, 'YYYYMMDD').timestamp + stop = arrow.get(stops, 'YYYYMMDD').timestamp else: - stop = int(stop) + stop = int(stops) return stype, start, stop raise Exception('Incorrect syntax for timerange "%s"' % text) From f4f821e88eeeda2b56d03004fe72a2764f6e68dc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:44:18 +0200 Subject: [PATCH 13/50] add typehints --- freqtrade/strategy/resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 8f4972919..23380dad9 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -33,7 +33,7 @@ class StrategyResolver(object): # Verify the strategy is in the configuration, otherwise fallback to the default strategy strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY - self.strategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) + self.strategy: IStrategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) # Set attributes # Check if we need to override configuration @@ -61,7 +61,7 @@ class StrategyResolver(object): self.strategy.stoploss = float(self.strategy.stoploss) def _load_strategy( - self, strategy_name: str, extra_dir: Optional[str] = None) -> Optional[IStrategy]: + self, strategy_name: str, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import From cf34b84cf1c67a9bcc60f89b886e514d615d22f2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:59:22 +0200 Subject: [PATCH 14/50] add attributes with typehints --- freqtrade/strategy/interface.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dcf665a02..4ae358c6f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,7 +2,7 @@ IStrategy interface This module defines the interface to apply for strategies """ - +from typing import Dict from abc import ABC, abstractmethod from pandas import DataFrame @@ -16,9 +16,13 @@ class IStrategy(ABC): Attributes you can use: minimal_roi -> Dict: Minimal ROI designed for the strategy stoploss -> float: optimal stoploss designed for the strategy - ticker_interval -> int: value of the ticker interval to use for the strategy + ticker_interval -> str: value of the ticker interval to use for the strategy """ + minimal_roi: Dict + stoploss: float + ticker_interval: str + @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ From 3fb1dd02f1204996da2c5cfc326cd719b64839a0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:00:46 +0200 Subject: [PATCH 15/50] add typehints and type: ignores --- freqtrade/analyze.py | 6 +++--- freqtrade/exchange/__init__.py | 2 +- freqtrade/optimize/hyperopt.py | 8 ++++---- freqtrade/strategy/resolver.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 5756b845c..6334fd846 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -95,7 +95,7 @@ class Analyze(object): Return ticker interval to use :return: Ticker interval value to use """ - return self.strategy.ticker_interval + return self.strategy.ticker_interval # type: ignore def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ @@ -195,13 +195,13 @@ class Analyze(object): :return True if bot should sell at current rate """ current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: + if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: # type: ignore logger.debug('Stop loss hit.') return True # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in self.strategy.minimal_roi.items(): + for duration, threshold in self.strategy.minimal_roi.items(): # type: ignore if time_diff <= duration: return False if current_profit > threshold: diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 109e3f7b2..b186f3611 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -290,7 +290,7 @@ def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] = # chached data was already downloaded till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) - data = [] + data: List[Dict[Any, Any]] = [] while not since_ms or since_ms < till_time_ms: data_part = _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..b4f534d7c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,7 +14,7 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, Optional import numpy import talib.abstract as ta @@ -60,7 +60,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Configuration and data used by hyperopt - self.processed = None + self.processed: Optional[Dict[str, Any]] = None # Hyperopt Trials self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') @@ -344,7 +344,7 @@ class Hyperopt(Backtesting): """ Return the space to use during Hyperopt """ - spaces = {} + spaces: Dict = {} if self.has_space('buy'): spaces = {**spaces, **Hyperopt.indicator_space()} if self.has_space('roi'): @@ -503,7 +503,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.analyze.populate_indicators = Hyperopt.populate_indicators + self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) if self.config.get('mongodb'): diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 23380dad9..28465210c 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -101,7 +101,7 @@ class StrategyResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # type: ignore valid_strategies_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) From 41a47df93f81ef868f29f5cf6d9cc66d728e22bd Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:09:31 +0200 Subject: [PATCH 16/50] setup travis to check mypy --- .travis.yml | 3 ++- setup.cfg | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6554f2095..1cff5c04b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ addons: install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -- pip install --upgrade flake8 coveralls pytest-random-order +- pip install --upgrade flake8 coveralls pytest-random-order mypy - pip install -r requirements.txt - pip install -e . jobs: @@ -26,6 +26,7 @@ jobs: - cp config.json.example config.json - python freqtrade/main.py hyperopt -e 5 - script: flake8 freqtrade + - script: mypy freqtrade after_success: - coveralls notifications: diff --git a/setup.cfg b/setup.cfg index ba065a7c2..6ffad0824 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,3 +2,6 @@ #ignore = max-line-length = 100 max-complexity = 12 + +[mypy] +ignore_missing_imports = True From 633620a5e925f08e8b7c5e8f5e29317066aba45b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:15:18 +0200 Subject: [PATCH 17/50] exclude .mypy_cache --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e5ac932f8..219a9fb40 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,4 @@ target/ .vscode .pytest_cache/ +.mypy_cache/ From e28973c50a04dabd386342c9ff7b170616e2a6df Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:17:46 +0200 Subject: [PATCH 18/50] fix flake8 --- freqtrade/analyze.py | 3 ++- freqtrade/strategy/resolver.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 6334fd846..0a560c86e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -195,7 +195,8 @@ class Analyze(object): :return True if bot should sell at current rate """ current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: # type: ignore + if self.strategy.stoploss is not None \ + and current_profit < self.strategy.stoploss: # type: ignore logger.debug('Stop loss hit.') return True diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 28465210c..60427bcf4 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -33,7 +33,8 @@ class StrategyResolver(object): # Verify the strategy is in the configuration, otherwise fallback to the default strategy strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY - self.strategy: IStrategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) + self.strategy: IStrategy = self._load_strategy(strategy_name, + extra_dir=config.get('strategy_path')) # Set attributes # Check if we need to override configuration From 4a322abd4d0736502b78d68b5d1611cb9b402931 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:43:51 +0200 Subject: [PATCH 19/50] Typecheck improvements --- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/optimize/hyperopt.py | 5 +++-- freqtrade/rpc/rpc.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376730d0f..5a3b790c9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -78,7 +78,7 @@ class Backtesting(object): Generates and returns a text table for the given backtest data and the results dataframe :return: pretty printed table with tabulate as str """ - stake_currency = self.config.get('stake_currency') + stake_currency = str(self.config.get('stake_currency')) floatfmt = ('s', 'd', '.2f', '.8f', '.1f') tabular_data = [] @@ -168,7 +168,7 @@ class Backtesting(object): record = args.get('record', None) records = [] trades = [] - trade_count_lock = {} + trade_count_lock: Dict = {} for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -230,7 +230,7 @@ class Backtesting(object): else: logger.info('Using local backtesting data (using whitelist in given config) ...') - timerange = Arguments.parse_timerange(self.config.get('timerange')) + timerange = Arguments.parse_timerange(str(self.config.get('timerange'))) data = optimize.load_data( self.config['datadir'], pairs=pairs, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b4f534d7c..f6cbb270b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -494,9 +494,10 @@ class Hyperopt(Backtesting): ) def start(self) -> None: - timerange = Arguments.parse_timerange(self.config.get('timerange')) + timerange = Arguments.parse_timerange(self.config.get('timerange') if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) data = load_data( - datadir=self.config.get('datadir'), + datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, timerange=timerange diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index df809cc2f..3e39657f2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -2,7 +2,7 @@ This module contains class to define a RPC communications """ import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date from decimal import Decimal from typing import Dict, Tuple, Any, Optional @@ -114,7 +114,7 @@ class RPC(object): self, timescale: int, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: today = datetime.utcnow().date() - profit_days: Dict[int, Dict] = {} + profit_days: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): return True, '*Daily [n]:* `must be an integer greater than 0`' @@ -172,7 +172,7 @@ class RPC(object): durations = [] for trade in trades: - current_rate: Optional[float] = None + current_rate: float = 0.0 if not trade.open_rate: continue From 6106822d102ef696ef46831d1e5ef6bc8870a49e Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:44:41 +0200 Subject: [PATCH 20/50] typing --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7c955d423..50fe10667 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -33,7 +33,7 @@ class FreqtradeBot(object): This is from here the bot start its logic. """ - def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None): + def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None)-> None: """ Init all variables and object the bot need to work :param config: configuration dict, you can use the Configuration.get_config() @@ -93,7 +93,7 @@ class FreqtradeBot(object): persistence.cleanup() return True - def worker(self, old_state: None) -> State: + def worker(self, old_state: State = None) -> State: """ Trading routine that must be run at each loop :param old_state: the previous service state from the previous call From 6fc21e30e5388b67921a47fd389fc58764220dce Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:52:55 +0200 Subject: [PATCH 21/50] remove unused import --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3e39657f2..715d38f95 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,7 +4,7 @@ This module contains class to define a RPC communications import logging from datetime import datetime, timedelta, date from decimal import Decimal -from typing import Dict, Tuple, Any, Optional +from typing import Dict, Tuple, Any import arrow import sqlalchemy as sql From d9e951447fc5bc7a43ea1d00c08fe040df0f4763 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:54:22 +0200 Subject: [PATCH 22/50] remove _init function in backtesting (and according test) --- freqtrade/optimize/backtesting.py | 12 ------------ freqtrade/tests/optimize/test_backtesting.py | 17 ----------------- 2 files changed, 29 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5a3b790c9..0e77572a5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,18 +33,6 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.analyze = None - self.ticker_interval = None - self.tickerdata_to_dataframe = None - self.populate_buy_trend = None - self.populate_sell_trend = None - self._init() - - def _init(self) -> None: - """ - Init objects required for backtesting - :return: None - """ self.analyze = Analyze(self.config) self.ticker_interval = self.analyze.strategy.ticker_interval self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f17a0115e..5cc41845d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -286,23 +286,6 @@ def test_start(mocker, fee, default_conf, caplog) -> None: assert start_mock.call_count == 1 -def test_backtesting__init__(mocker, default_conf) -> None: - """ - Test Backtesting.__init__() method - """ - init_mock = MagicMock() - mocker.patch('freqtrade.optimize.backtesting.Backtesting._init', init_mock) - - backtesting = Backtesting(default_conf) - assert backtesting.config == default_conf - assert backtesting.analyze is None - assert backtesting.ticker_interval is None - assert backtesting.tickerdata_to_dataframe is None - assert backtesting.populate_buy_trend is None - assert backtesting.populate_sell_trend is None - assert init_mock.call_count == 1 - - def test_backtesting_init(mocker, default_conf) -> None: """ Test Backtesting._init() method From 32300f6d5fe46dce5b29f9a934f568e765942840 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:55:06 +0200 Subject: [PATCH 23/50] don't initialize with None where it's not necessary --- freqtrade/freqtradebot.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 50fe10667..41841e911 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -51,9 +51,9 @@ class FreqtradeBot(object): # Init objects self.config = config - self.analyze = None - self.fiat_converter = None - self.rpc = None + self.analyze = Analyze(self.config) + self.fiat_converter = CryptoToFiatConverter() + self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = None @@ -66,9 +66,6 @@ class FreqtradeBot(object): :return: None """ # Initialize all modules - self.analyze = Analyze(self.config) - self.fiat_converter = CryptoToFiatConverter() - self.rpc = RPCManager(self) persistence.init(self.config, db_url) exchange.init(self.config) From 0a595190a3e622915defb62153ec3886ce5c4c54 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:59:35 +0200 Subject: [PATCH 24/50] fix last typechecks --- freqtrade/arguments.py | 3 ++- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 2421abc9f..fc917394e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -211,7 +211,8 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: str) -> Optional[Tuple[Tuple, Optional[int], Optional[int]]]: + def parse_timerange(text: Optional[str]) -> Optional[Tuple[Tuple, + Optional[int], Optional[int]]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0e77572a5..78ab72988 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -219,7 +219,7 @@ class Backtesting(object): logger.info('Using local backtesting data (using whitelist in given config) ...') timerange = Arguments.parse_timerange(str(self.config.get('timerange'))) - data = optimize.load_data( + data = optimize.load_data( # type: ignore self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f6cbb270b..8b8e9576b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -494,9 +494,9 @@ class Hyperopt(Backtesting): ) def start(self) -> None: - timerange = Arguments.parse_timerange(self.config.get('timerange') if self.config.get( + timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( + data = load_data( # type: ignore datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, From 0007002c80822cba5085655a10ca22dd1678e251 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:07:54 +0200 Subject: [PATCH 25/50] fix test failure --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 78ab72988..ef69539db 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -218,7 +218,8 @@ class Backtesting(object): else: logger.info('Using local backtesting data (using whitelist in given config) ...') - timerange = Arguments.parse_timerange(str(self.config.get('timerange'))) + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) data = optimize.load_data( # type: ignore self.config['datadir'], pairs=pairs, From 884395415fc4e46221f0b0bac205744c8f9a4f16 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:10:15 +0200 Subject: [PATCH 26/50] remove type:ignore --- freqtrade/analyze.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 0a560c86e..5756b845c 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -95,7 +95,7 @@ class Analyze(object): Return ticker interval to use :return: Ticker interval value to use """ - return self.strategy.ticker_interval # type: ignore + return self.strategy.ticker_interval def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ @@ -195,14 +195,13 @@ class Analyze(object): :return True if bot should sell at current rate """ current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None \ - and current_profit < self.strategy.stoploss: # type: ignore + if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: logger.debug('Stop loss hit.') return True # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in self.strategy.minimal_roi.items(): # type: ignore + for duration, threshold in self.strategy.minimal_roi.items(): if time_diff <= duration: return False if current_profit > threshold: From 3447e4bb9729d8f2c43240cc27ba2947471e7464 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:13:17 +0200 Subject: [PATCH 27/50] comment on ignore hint --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8b8e9576b..2497d6752 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -496,7 +496,7 @@ class Hyperopt(Backtesting): def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( # type: ignore + data = load_data( # type: ignore # timerange will be refactored datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, From f88729f0e855ef167027f64b6b85f07c94169f2d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:14:28 +0200 Subject: [PATCH 28/50] add ignore comment --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ef69539db..a0aee78b1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -220,7 +220,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( # type: ignore + data = optimize.load_data( # type: ignore # timerange will be refactored self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, From a8bf5092e86aaa84b28815ae3670ab852c037761 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:18:57 +0200 Subject: [PATCH 29/50] add ignore explanation --- freqtrade/strategy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 60427bcf4..3fd39bca3 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -102,7 +102,7 @@ class StrategyResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints valid_strategies_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) From 6ca375a39783eea32aafe271c5672ea85215c11c Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 19:45:08 +0300 Subject: [PATCH 30/50] Extend timerange to accept unix timestamps. This gives greater granularity over backtest, parsing tickerfiles. Example runs using date and unix time below. /usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /Users/creslin/PycharmProjects/freqtrade/scripts/report_correlation.py --timerange=20180528-20180529 2018-06-02 18:44:58,829 - freqtrade.configuration - INFO - Log level set to INFO 2018-06-02 18:44:58,830 - freqtrade.configuration - INFO - Using max_open_trades: 200 ... 2018-06-02 18:44:58,831 - freqtrade.configuration - INFO - Parameter --timerange detected: 20180528-20180529 ... 2018-06-02 18:44:58,831 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... BasePair Pair Correlation BTC % Change Pair % USD Ch Pair % BTC Ch Gain % on BTC Start Stop BTC Volume 1 BTC_USDT ETC_USD 0.965 -2.942 -4.070 -1.163 -1.128585 05-28 00:00 05-29 00:00 335.19 0 BTC_USDT SNT_USD 0.943 -2.942 -5.857 -3.004 -2.915585 05-28 00:00 05-29 00:00 496.01 3 BTC_USDT DASH_USD 0.902 -2.942 -9.034 -6.277 -6.092432 05-28 00:00 05-29 00:00 751.41 2 BTC_USDT MTH_USD 0.954 -2.942 -9.290 -6.541 -6.348708 05-28 00:00 05-29 00:00 23.00 4 BTC_USDT TRX_USD 0.951 -2.942 -13.647 -11.029 -10.704957 05-28 00:00 05-29 00:00 14544.57 Process finished with exit code 0 /usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /Users/creslin/PycharmProjects/freqtrade/scripts/report_correlation.py --timerange=1527595200-1527618600 2018-06-02 18:47:40,382 - freqtrade.configuration - INFO - Log level set to INFO 2018-06-02 18:47:40,382 - freqtrade.configuration - INFO - Using max_open_trades: 200 ... 2018-06-02 18:47:40,383 - freqtrade.configuration - INFO - Parameter --timerange detected: 1527595200-1527618600 ... 2018-06-02 18:47:40,383 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... BasePair Pair Correlation BTC % Change Pair % USD Ch Pair % BTC Ch Gain % on BTC Start Stop BTC Volume 0 BTC_USDT SNT_USD 0.680 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 68866.30 1 BTC_USDT ETC_USD 0.857 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 227514.17 2 BTC_USDT MTH_USD 0.790 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 12103.96 3 BTC_USDT DASH_USD 0.862 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 72982.78 4 BTC_USDT TRX_USD 0.178 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 1258316.95 Process finished with exit code 0 --- freqtrade/arguments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..00869f974 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,6 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'timestamp')), + (r'^(\d{10})-$', ('timestamp', None)), + (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -237,6 +240,8 @@ class Arguments(object): start = rvals[index] if stype[0] == 'date': start = arrow.get(start, 'YYYYMMDD').timestamp + elif stype[0] == 'timestamp': + start = arrow.get(start).timestamp else: start = int(start) index += 1 @@ -244,6 +249,8 @@ class Arguments(object): stop = rvals[index] if stype[1] == 'date': stop = arrow.get(stop, 'YYYYMMDD').timestamp + elif stype[1] == 'timestamp': + stop = arrow.get(stop).timestamp else: stop = int(stop) return stype, start, stop From 9dbe5fdb852e1fc32bab6792bb37610fab508d04 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 19:49:23 +0300 Subject: [PATCH 31/50] Update back testing document to include example using Posix timestamps as timerange e.g --timerange=1527595200-1527618600 --- docs/backtesting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index df105bd77..8c4c4180d 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -83,6 +83,8 @@ The full timerange specification: - Use tickframes till 2018/01/31: `--timerange=-20180131` - Use tickframes since 2018/01/31: `--timerange=20180131-` - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` +- Use tickframes between POSIX timestamps 1527595200 1527618600: + `--timerange=1527595200-1527618600` **Update testdata directory** From 2791d543ea38cf15224c8bf8dc7ae5a7f8e0cbb0 Mon Sep 17 00:00:00 2001 From: Raymond Luo Date: Fri, 18 May 2018 19:02:38 +0800 Subject: [PATCH 32/50] Make backtesting report markdown shareable Small tweak to make the backtesting report markdown ready and much easier to share reports on many markdown publishing tools and editors that already support Markdown Extra with just a copy and paste Example: ![Example](https://i.imgur.com/HXlNkfm.png) --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376730d0f..ab45b7754 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -106,7 +106,7 @@ class Backtesting(object): len(results[results.profit_BTC > 0]), len(results[results.profit_BTC < 0]) ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, From 9537f17dd41ea0f5ba6e87f06bc7cc32e0d7ccad Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 20:06:29 +0200 Subject: [PATCH 33/50] Fix test --- freqtrade/tests/optimize/test_backtesting.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index bfb82f3ec..786568f98 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -374,16 +374,15 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - 'pair buy count avg profit % ' - 'total profit BTC avg duration profit loss\n' - '------- ----------- -------------- ' - '------------------ -------------- -------- ------\n' - 'ETH/BTC 2 15.00 ' - '0.60000000 20.0 2 0\n' - 'TOTAL 2 15.00 ' - '0.60000000 20.0 2 0' + '| pair | buy count | avg profit % | ' + 'total profit BTC | avg duration | profit | loss |\n' + '|:--------|------------:|---------------:|' + '-------------------:|---------------:|---------:|-------:|\n' + '| ETH/BTC | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |\n' + '| TOTAL | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |' ) - assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str From 43ba02afc63e106925b47d3ee9e25a87809f02ae Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 21:59:18 +0300 Subject: [PATCH 34/50] Per feed back, kept the stype as date. Use a tuple to keep as epoch int or process via arrow to timestamp. I'll look at the test file also. --- freqtrade/arguments.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 00869f974..b61324ccc 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,9 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^-(\d{10})$', (None, 'timestamp')), - (r'^(\d{10})-$', ('timestamp', None)), - (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), + (r'^-(\d{10})$', (None, 'date')), + (r'^(\d{10})-$', ('date', None)), + (r'^(\d{10})-(\d{10})$', ('date', 'date')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -239,18 +239,16 @@ class Arguments(object): if stype[0]: start = rvals[index] if stype[0] == 'date': - start = arrow.get(start, 'YYYYMMDD').timestamp - elif stype[0] == 'timestamp': - start = arrow.get(start).timestamp + start = int(start) if len(start) == 10 \ + else arrow.get(start, 'YYYYMMDD').timestamp else: start = int(start) index += 1 if stype[1]: stop = rvals[index] if stype[1] == 'date': - stop = arrow.get(stop, 'YYYYMMDD').timestamp - elif stype[1] == 'timestamp': - stop = arrow.get(stop).timestamp + stop = int(stop) if len(stop) == 10 \ + else arrow.get(stop, 'YYYYMMDD').timestamp else: stop = int(stop) return stype, start, stop From dc65753a641a4c0de2bf9052eddc3f26e7470e67 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 12:35:07 -0700 Subject: [PATCH 35/50] Fix the in-progress dot that does not show up during a Hyperopt run --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..3317a6c6f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -455,6 +455,7 @@ class Hyperopt(Backtesting): if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') + sys.stdout.flush() return { 'status': STATUS_FAIL, 'loss': float('inf') From 94e586c049211e2267726e3b0509ece14b95bef3 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 2 Jun 2018 22:46:54 +0300 Subject: [PATCH 36/50] Added unit test to check posix time arguments passed to timerange Here is the pass report: freqtrade_new creslin$ pytest freqtrade/tests/test_arguments.py ==================================================================== test session starts ===================================================================== platform darwin -- Python 3.6.5, pytest-3.6.0, py-1.5.3, pluggy-0.6.0 rootdir: /Users/creslin/PycharmProjects/freqtrade_new, inifile: plugins: mock-1.10.0, cov-2.5.1 collected 19 items freqtrade/tests/test_arguments.py ................... [100%] ================================================================= 19 passed in 2.37 seconds ================================================================== --- freqtrade/tests/test_arguments.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 279ace0dc..474aa2507 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -116,6 +116,12 @@ def test_parse_timerange_incorrect() -> None: timerange = Arguments.parse_timerange('20100522-20150730') assert timerange == (('date', 'date'), 1274486400, 1438214400) + # Added test for unix timestamp - BTC genesis date + assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') + assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + timerange = Arguments.parse_timerange('1231006505-1233360000') + assert timerange == (('date', 'date'), 1231006505, 1233360000) + with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') From 127cf5d6192ce3c277798c6982887f688f9c55ac Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 13:55:05 -0700 Subject: [PATCH 37/50] Backtesting: Add the Interval required when data is missing Change the message: "No data for pair ETH/BTC, use --refresh-pairs-cached to download the data" for: "No data for pair: "ETH/BTC", Interval: 5m. Use --refresh-pairs-cached to download the data" The message structure is unified with the download message: "Download the pair: "ETH/BTC", Interval: 5m" --- freqtrade/optimize/__init__.py | 6 ++++-- freqtrade/tests/optimize/test_optimize.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 68ba5622e..19e235a7a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -104,8 +104,10 @@ def load_data(datadir: str, result[pair] = pairdata else: logger.warning( - 'No data for pair %s, use --refresh-pairs-cached to download the data', - pair + 'No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, + ticker_interval ) return result diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 765d88cd5..349fa3be3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -105,7 +105,8 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: refresh_pairs=False, pairs=['MEME/BTC']) assert os.path.isfile(file) is False - assert log_has('No data for pair MEME/BTC, use --refresh-pairs-cached to download the data', + assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' + 'Use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set From fe8ff1b929ed361197e459d2ff8568c9d6c2f7ac Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 14:07:31 -0700 Subject: [PATCH 38/50] Fix stake_currency return by Hyperopt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hyperopt had BTC hard coded in the result. This commit will display the real stake_currency used. If you used `"stake_currency": "USDT",` in your config file. Before this commit you saw a message like: "2 trades. Avg profit 0.13%. Total profit 0.00002651 BTC (0.0027Σ%). Avg duration 142.5 mins." Now with the commit, we fix the wrong BTC currency: "2 trades. Avg profit 0.13%. Total profit 0.00002651 USDT (0.0027Σ%). Avg duration 142.5 mins." --- freqtrade/optimize/hyperopt.py | 6 +++--- freqtrade/tests/optimize/test_hyperopt.py | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..111da9b5c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -479,16 +479,16 @@ class Hyperopt(Backtesting): 'result': result_explanation, } - @staticmethod - def format_results(results: DataFrame) -> str: + def format_results(self, results: DataFrame) -> str: """ Return the format result in a string """ return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), + self.config['stake_currency'], results.profit_percent.sum(), results.duration.mean(), ) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f8fa66b2e..3edfe4393 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -389,10 +389,12 @@ def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: # test buy_strategy_generator def populate_buy_trend # test optimizer if 'ro_t1' in params -def test_format_results(): +def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() """ + + # Test with BTC as stake_currency trades = [ ('ETH/BTC', 2, 2, 123), ('LTC/BTC', 1, 1, 123), @@ -400,8 +402,21 @@ def test_format_results(): ] labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] df = pd.DataFrame.from_records(trades, columns=labels) - x = Hyperopt.format_results(df) - assert x.find(' 66.67%') + + result = _HYPEROPT.format_results(df) + assert result.find(' 66.67%') + assert result.find('Total profit 1.00000000 BTC') + assert result.find('2.0000Σ %') + + # Test with EUR as stake_currency + trades = [ + ('ETH/EUR', 2, 2, 123), + ('LTC/EUR', 1, 1, 123), + ('XPR/EUR', -1, -2, -246) + ] + df = pd.DataFrame.from_records(trades, columns=labels) + result = _HYPEROPT.format_results(df) + assert result.find('Total profit 1.00000000 EUR') def test_signal_handler(mocker, init_hyperopt): From acbfe91f131ffa5854436bff275d92d2fce2eb21 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 31 May 2018 23:44:17 -0700 Subject: [PATCH 39/50] Allow EUR / USD as stake_currency It will enable to trade with FIAT on exhanges like GDAX or Kraken. --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 7731ea610..82794774b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -31,7 +31,7 @@ CONF_SCHEMA = { 'properties': { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': {'type': 'number', 'minimum': 0.0005}, 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', 'CLP', 'CNY', 'CZK', 'DKK', From c9e49ed7b40a541eb9d2e385d586c73eb6f22b3c Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 31 May 2018 23:45:35 -0700 Subject: [PATCH 40/50] Sort ticker_history CCXT does not sort the ticker history from exchanges. Bittrex and Binance are sorted ASC (oldest first, newest last) when GDAX is sorted DESC (newest first, oldest last). Because of that the get_ticker_history() fall in a very long loop when the tickers are sorted DESC. Means it downloads more than needed. This commit enable exhanges like GDAX and unify the ticker_history list across all exchanges. --- freqtrade/exchange/__init__.py | 5 ++ freqtrade/tests/exchange/test_exchange.py | 72 +++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 109e3f7b2..7be323329 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -294,6 +294,11 @@ def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] = while not since_ms or since_ms < till_time_ms: data_part = _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data_part = sorted(data_part, key=lambda x: x[0]) + if not data_part: break diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index a506a8416..56812c75e 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -393,6 +393,78 @@ def test_get_ticker_history(default_conf, mocker): get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) +def test_get_ticker_history_sort(default_conf, mocker): + api_mock = MagicMock() + + # GDAX use-case (real data from GDAX) + # This ticker history is ordered DESC (newest first, oldest last) + tick = [ + [1527833100000, 0.07666, 0.07671, 0.07666, 0.07668, 16.65244264], + [1527832800000, 0.07662, 0.07666, 0.07662, 0.07666, 1.30051526], + [1527832500000, 0.07656, 0.07661, 0.07656, 0.07661, 12.034778840000001], + [1527832200000, 0.07658, 0.07658, 0.07655, 0.07656, 0.59780186], + [1527831900000, 0.07658, 0.07658, 0.07658, 0.07658, 1.76278136], + [1527831600000, 0.07658, 0.07658, 0.07658, 0.07658, 2.22646521], + [1527831300000, 0.07655, 0.07657, 0.07655, 0.07657, 1.1753], + [1527831000000, 0.07654, 0.07654, 0.07651, 0.07651, 0.8073060299999999], + [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], + [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] + ] + type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) + api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) + mocker.patch('freqtrade.exchange._API', api_mock) + + # Test the ticker history sort + ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + assert ticks[0][0] == 1527830400000 + assert ticks[0][1] == 0.07649 + assert ticks[0][2] == 0.07651 + assert ticks[0][3] == 0.07649 + assert ticks[0][4] == 0.07651 + assert ticks[0][5] == 2.5734867 + + assert ticks[9][0] == 1527833100000 + assert ticks[9][1] == 0.07666 + assert ticks[9][2] == 0.07671 + assert ticks[9][3] == 0.07666 + assert ticks[9][4] == 0.07668 + assert ticks[9][5] == 16.65244264 + + # Bittrex use-case (real data from Bittrex) + # This ticker history is ordered ASC (oldest first, newest last) + tick = [ + [1527827700000, 0.07659999, 0.0766, 0.07627, 0.07657998, 1.85216924], + [1527828000000, 0.07657995, 0.07657995, 0.0763, 0.0763, 26.04051037], + [1527828300000, 0.0763, 0.07659998, 0.0763, 0.0764, 10.36434124], + [1527828600000, 0.0764, 0.0766, 0.0764, 0.0766, 5.71044773], + [1527828900000, 0.0764, 0.07666998, 0.0764, 0.07666998, 47.48888565], + [1527829200000, 0.0765, 0.07672999, 0.0765, 0.07672999, 3.37640326], + [1527829500000, 0.0766, 0.07675, 0.0765, 0.07675, 8.36203831], + [1527829800000, 0.07675, 0.07677999, 0.07620002, 0.076695, 119.22963884], + [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], + [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] + ] + type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) + api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) + mocker.patch('freqtrade.exchange._API', api_mock) + + # Test the ticker history sort + ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + assert ticks[0][0] == 1527827700000 + assert ticks[0][1] == 0.07659999 + assert ticks[0][2] == 0.0766 + assert ticks[0][3] == 0.07627 + assert ticks[0][4] == 0.07657998 + assert ticks[0][5] == 1.85216924 + + assert ticks[9][0] == 1527830400000 + assert ticks[9][1] == 0.07671 + assert ticks[9][2] == 0.07674399 + assert ticks[9][3] == 0.07629216 + assert ticks[9][4] == 0.07655213 + assert ticks[9][5] == 2.31452783 + + def test_cancel_order_dry_run(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.exchange._CONF', default_conf) From 638d98735f237b94ecca0e7c999284c6aee85d3e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 1 Jun 2018 20:58:07 -0700 Subject: [PATCH 41/50] Allow fiat_convert to use same symbol for Crypto and FIAT --- freqtrade/fiat_convert.py | 15 +++++++++++---- freqtrade/tests/test_fiat_convert.py | 7 +++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 17882f51a..7e61a3f2f 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -95,8 +95,11 @@ class CryptoToFiatConverter(object): coinlistings = self._coinmarketcap.listings() self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])), coinlistings["data"])) - except (ValueError, RequestException) as e: - logger.error("Could not load FIAT Cryptocurrency map for the following problem: %s", e) + except (ValueError, RequestException) as exception: + logger.error( + "Could not load FIAT Cryptocurrency map for the following problem: %s", + exception + ) def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float: """ @@ -188,6 +191,10 @@ class CryptoToFiatConverter(object): if not self._is_supported_fiat(fiat=fiat_symbol): raise ValueError('The fiat {} is not supported.'.format(fiat_symbol)) + # No need to convert if both crypto and fiat are the same + if crypto_symbol == fiat_symbol: + return 1.0 + if crypto_symbol not in self._cryptomap: # return 0 for unsupported stake currencies (fiat-convert should not break the bot) logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol) @@ -199,6 +206,6 @@ class CryptoToFiatConverter(object): convert=fiat_symbol )['data']['quotes'][fiat_symbol.upper()]['price'] ) - except BaseException as ex: - logger.error("Error in _find_price: %s", ex) + except BaseException as exception: + logger.error("Error in _find_price: %s", exception) return 0.0 diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index b37ca0f5c..2da427d27 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -126,6 +126,13 @@ def test_fiat_convert_get_price(mocker): assert fiat_convert._pairs[0]._expiration is not expiration +def test_fiat_convert_same_currencies(mocker): + patch_coinmarketcap(mocker) + fiat_convert = CryptoToFiatConverter() + + assert fiat_convert.get_price(crypto_symbol='USD', fiat_symbol='USD') == 1.0 + + def test_loadcryptomap(mocker): patch_coinmarketcap(mocker) From e8a59f4c20f135cb3b977324c574f694d27a1ac0 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 13:26:24 -0700 Subject: [PATCH 42/50] Add a test to check the behavior when converting two FIAT --- freqtrade/tests/test_fiat_convert.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 2da427d27..faf7462c0 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -133,6 +133,13 @@ def test_fiat_convert_same_currencies(mocker): assert fiat_convert.get_price(crypto_symbol='USD', fiat_symbol='USD') == 1.0 +def test_fiat_convert_two_FIAT(mocker): + patch_coinmarketcap(mocker) + fiat_convert = CryptoToFiatConverter() + + assert fiat_convert.get_price(crypto_symbol='USD', fiat_symbol='EUR') == 0.0 + + def test_loadcryptomap(mocker): patch_coinmarketcap(mocker) From cfb06ceb58243618a52a00035cdaacd7c9e857be Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 3 Jun 2018 10:12:07 +0200 Subject: [PATCH 43/50] Update ccxt from 1.14.119 to 1.14.120 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad64e58fd..d96084348 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.119 +ccxt==1.14.120 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 3a158faa30cdfb30d768d6b1e38644f8c46d69fe Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 13:47:36 +0200 Subject: [PATCH 44/50] Refactor fiat-list to constants --- freqtrade/constants.py | 15 +++++++-------- freqtrade/fiat_convert.py | 11 ++--------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 7731ea610..f1cb03d26 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,6 +24,12 @@ TICKER_INTERVAL_MINUTES = { '1w': 10080, } +SUPPORTED_FIAT = [ + "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", + "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", + "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", + "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" + ] # Required json-schema for user specified config CONF_SCHEMA = { @@ -33,14 +39,7 @@ CONF_SCHEMA = { 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, + 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 88eb702c9..a653ad76e 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -9,6 +9,7 @@ from typing import Dict, List from coinmarketcap import Market from requests.exceptions import RequestException +from freqtrade.constants import SUPPORTED_FIAT logger = logging.getLogger(__name__) @@ -67,14 +68,6 @@ class CryptoToFiatConverter(object): __instance = None _coinmarketcap: Market = None - # Constants - SUPPORTED_FIAT = [ - "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", - "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", - "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", - "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" - ] - _cryptomap: Dict = {} def __new__(cls): @@ -175,7 +168,7 @@ class CryptoToFiatConverter(object): fiat = fiat.upper() - return fiat in self.SUPPORTED_FIAT + return fiat in SUPPORTED_FIAT def _find_price(self, crypto_symbol: str, fiat_symbol: str) -> float: """ From 0f352a4b5cb0f41afa948a8883b437af08b2eaf8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 15:14:51 +0200 Subject: [PATCH 45/50] update contributing document to include mypy --- CONTRIBUTING.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 967f57b65..93089495b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,4 +42,16 @@ pip3.6 install flake8 coveralls flake8 freqtrade ``` +## 3. Test if all type-hints are correct +**Install packages** (If not already installed) + +``` bash +pip3.6 install mypy +``` + +**Run mypy** + +``` bash +mypy freqtrade +``` From d3d62e90d321ab02673c79309f458cb49b497728 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 08:36:01 -0700 Subject: [PATCH 46/50] Update Backtesting/Hyperopt usage documentation --- docs/bot-usage.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b42df3ba3..e2c18473c 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -118,21 +118,25 @@ python3 ./freqtrade/main.py -c config.json --dry-run-db Backtesting also uses the config specified via `-c/--config`. ``` -usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation] - [-r] +usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] + [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] optional arguments: -h, --help show this help message and exit - -l, --live using live data - -i INT, --ticker-interval INT - specify ticker interval (default: '5m') + -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + specify ticker interval (1m, 5m, 30m, 1h, 1d) --realistic-simulation uses max_open_trades from config to simulate real world limitations + --timerange TIMERANGE + specify what timerange of data to use. + -l, --live using live data -r, --refresh-pairs-cached - refresh the pairs files in tests/testdata with - the latest data from the exchange. Use it if you want - to run your backtesting with up-to-date data. + refresh the pairs files in tests/testdata with the + latest data from the exchange. Use it if you want to + run your backtesting with up-to-date data. + --export EXPORT export backtest results, argument are: trades Example + --export=trades ``` ### How to use --refresh-pairs-cached parameter? @@ -155,14 +159,25 @@ Hyperopt uses an internal json config return by `hyperopt_optimize_conf()` located in `freqtrade/optimize/hyperopt_conf.py`. ``` -usage: freqtrade hyperopt [-h] [-e INT] [--use-mongodb] +usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] + [--timerange TIMERANGE] [-e INT] [--use-mongodb] + [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: -h, --help show this help message and exit + -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + specify ticker interval (1m, 5m, 30m, 1h, 1d) + --realistic-simulation + uses max_open_trades from config to simulate real + world limitations + --timerange TIMERANGE + specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) --use-mongodb parallelize evaluations with mongodb (requires mongod in PATH) - + -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] + Specify which parameters to hyperopt. Space separate + list. Default: all ``` ## A parameter missing in the configuration? From 43696eff5cfc950074ac221ffca7c3e12d0548f5 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 08:57:13 -0700 Subject: [PATCH 47/50] Add __main__.py to improve how to launch the bot --- freqtrade/__main__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 freqtrade/__main__.py diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py new file mode 100644 index 000000000..fe1318a35 --- /dev/null +++ b/freqtrade/__main__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +""" +__main__.py for Freqtrade +To launch Freqtrade as a module + +> python -m freqtrade (with Python >= 3.6) +""" + +import sys +from freqtrade import main + + +if __name__ == '__main__': + main.set_loggers() + main.main(sys.argv[1:]) From 4eb82959554fba43d3c1ce0362e8efe895304f46 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 3 Jun 2018 19:27:08 +0200 Subject: [PATCH 48/50] Update ccxt from 1.14.120 to 1.14.121 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d96084348..6e7550515 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.120 +ccxt==1.14.121 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From e3227a741c650cd60cf46273e40f55645e2b6164 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 14:52:03 +0200 Subject: [PATCH 49/50] add --export-filename for backtesting --- freqtrade/arguments.py | 10 ++++++++++ freqtrade/configuration.py | 5 +++++ freqtrade/optimize/backtesting.py | 8 +++++--- freqtrade/tests/optimize/test_backtesting.py | 8 +++++++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 97c3d8cb2..7e895177a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -137,6 +137,16 @@ class Arguments(object): default=None, dest='export', ) + parser.add_argument( + '--export-filename', + help='Save backtest results to this filename \ + requires --export to be set as well\ + Example --export-filename=backtest_today.json\ + (default: %(default)s', + type=str, + default='backtest-result.json', + dest='exportfilename', + ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index a800bde78..77b5b4447 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -157,6 +157,11 @@ class Configuration(object): config.update({'export': self.args.export}) logger.info('Parameter --export detected: %s ...', self.args.export) + # If --export-filename is used we add it to the configuration + if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: + config.update({'exportfilename': self.args.exportfilename}) + logger.info('Storing backtest results to %s ...', self.args.exportfilename) + return config def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1d560d309..d7ed45955 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -154,6 +154,7 @@ class Backtesting(object): max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) record = args.get('record', None) + recordfilename = args.get('recordfn', 'backtest-result.json') records = [] trades = [] trade_count_lock: Dict = {} @@ -196,8 +197,8 @@ class Backtesting(object): # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: - logger.info('Dumping backtest results') - file_dump_json('backtest-result.json', records) + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) @@ -257,7 +258,8 @@ class Backtesting(object): 'realistic': self.config.get('realistic_simulation', False), 'sell_profit_only': sell_profit_only, 'use_sell_signal': use_sell_signal, - 'record': self.config.get('export') + 'record': self.config.get('export'), + 'recordfn': self.config.get('exportfilename'), } ) logger.info( diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1b1872404..65820ac09 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -218,7 +218,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--realistic-simulation', '--refresh-pairs-cached', '--timerange', ':100', - '--export', '/bar/foo' + '--export', '/bar/foo', + '--export-filename', 'foo_bar.json' ] config = setup_configuration(get_args(args)) @@ -259,6 +260,11 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'Parameter --export detected: {} ...'.format(config['export']), caplog.record_tuples ) + assert 'exportfilename' in config + assert log_has( + 'Storing backtest results to {} ...'.format(config['exportfilename']), + caplog.record_tuples + ) def test_start(mocker, fee, default_conf, caplog) -> None: From 482d0636389f8b40b2bb05d7a30cef0be2c895ef Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 19:41:34 +0200 Subject: [PATCH 50/50] update documentation for --export-filename --- docs/backtesting.md | 6 ++++++ docs/bot-usage.md | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index 8c4c4180d..0b53d45b7 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -62,6 +62,12 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +**Exporting trades to file specifying a custom filename** +```bash +python3 ./freqtrade/main.py backtesting --export trades --export-filename=backtest_teststrategy.json +``` + + **Running backtest with smaller testset** Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index e2c18473c..cfffd04e9 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -120,6 +120,8 @@ Backtesting also uses the config specified via `-c/--config`. ``` usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] + [--export-filename EXPORTFILENAME] + optional arguments: -h, --help show this help message and exit @@ -137,6 +139,11 @@ optional arguments: run your backtesting with up-to-date data. --export EXPORT export backtest results, argument are: trades Example --export=trades + --export-filename EXPORTFILENAME + Save backtest results to this filename requires + --export to be set as well Example --export- + filename=backtest_today.json (default: backtest- + result.json ``` ### How to use --refresh-pairs-cached parameter?