Merge branch 'develop' into backtest-export

.gitignore

@@ -85,3 +85,5 @@ target/
 .venv
 .idea
 .vscode
+
+hyperopt_trials.pickle

.pylintrc

@@ -1,10 +1,10 @@
 [MASTER]
-extension-pkg-whitelist=numpy,talib
+extension-pkg-whitelist=numpy,talib,talib.abstract
 
 [BASIC]
 good-names=logger
 ignore=vendor
 
 [TYPECHECK]
-ignored-modules=numpy,talib
+ignored-modules=numpy,talib,talib.abstract
 

freqtrade/analyze.py

@@ -4,14 +4,14 @@ Functions to analyze ticker data with indicators and produce buy and sell signal
 import logging
 from datetime import timedelta
 from enum import Enum
-from typing import List, Dict
+from typing import Dict, List
 
 import arrow
 import talib.abstract as ta
 from pandas import DataFrame, to_datetime
 
-from freqtrade.exchange import get_ticker_history
 import freqtrade.vendor.qtpylib.indicators as qtpylib
+from freqtrade.exchange import get_ticker_history
 
 logger = logging.getLogger(__name__)
 

freqtrade/exchange/bittrex.py

@@ -1,7 +1,9 @@
 import logging
-from typing import List, Dict, Optional
+import requests
+from typing import Dict, List, Optional
 
-from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1
+from bittrex.bittrex import Bittrex as _Bittrex
+from bittrex.bittrex import API_V1_1, API_V2_0
 from requests.exceptions import ContentDecodingError
 
 from freqtrade import OperationalException
@@ -13,6 +15,20 @@ _API: _Bittrex = None
 _API_V2: _Bittrex = None
 _EXCHANGE_CONF: dict = {}
 
+# API socket timeout
+API_TIMEOUT = 60
+
+
+def custom_requests(request_url, apisign):
+    """
+    Set timeout for requests
+    """
+    return requests.get(
+        request_url,
+        headers={"apisign": apisign},
+        timeout=API_TIMEOUT
+    ).json()
+
 
 class Bittrex(Exchange):
     """
@@ -31,12 +47,14 @@ class Bittrex(Exchange):
             api_secret=_EXCHANGE_CONF['secret'],
             calls_per_second=1,
             api_version=API_V1_1,
+            dispatch=custom_requests
         )
         _API_V2 = _Bittrex(
             api_key=_EXCHANGE_CONF['key'],
             api_secret=_EXCHANGE_CONF['secret'],
             calls_per_second=1,
             api_version=API_V2_0,
+            dispatch=custom_requests
         )
         self.cached_ticker = {}
 

freqtrade/exchange/interface.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import List, Dict, Optional
+from typing import Dict, List, Optional
 
 
 class Exchange(ABC):

freqtrade/fiat_convert.py

@@ -1,5 +1,6 @@
 import logging
 import time
+
 from pymarketcap import Pymarketcap
 
 logger = logging.getLogger(__name__)

freqtrade/main.py

@@ -5,43 +5,25 @@ import logging
 import sys
 import time
 import traceback
-import arrow
 from datetime import datetime
-from typing import Dict, Optional, List
+from typing import Dict, List, Optional
 
+import arrow
 import requests
-from requests.adapters import TimeoutSauce
 from cachetools import cached, TTLCache
 
-from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \
+from freqtrade import (DependencyException, OperationalException, __version__,
-    OperationalException
+                       exchange, persistence, rpc)
-from freqtrade.analyze import get_signal, SignalType
+from freqtrade.analyze import SignalType, get_signal
-from freqtrade.misc import State, get_state, update_state, parse_args, throttle, \
-    load_config
-from freqtrade.persistence import Trade
 from freqtrade.fiat_convert import CryptoToFiatConverter
+from freqtrade.misc import (State, get_state, load_config, parse_args,
+                            throttle, update_state)
+from freqtrade.persistence import Trade
 
 logger = logging.getLogger('freqtrade')
 
 _CONF = {}
 
-DEFAULT_TIMEOUT = 120
-
-
-# Set requests default timeout (fix for #127)
-class DefaultTimeout(TimeoutSauce):
-    def __init__(self, *args, **kwargs):
-        connect = kwargs.get('connect', DEFAULT_TIMEOUT)
-        read = kwargs.get('read', connect)
-        if connect is None:
-            connect = DEFAULT_TIMEOUT
-        if read is None:
-            read = connect
-        super(DefaultTimeout, self).__init__(connect=connect, read=read)
-
-
-requests.adapters.TimeoutSauce = DefaultTimeout
-
 
 def refresh_whitelist(whitelist: List[str]) -> List[str]:
     """
@@ -197,11 +179,11 @@ def execute_sell(trade: Trade, limit: float) -> None:
     profit_trade = trade.calc_profit(rate=limit)
 
     message = '*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`'.format(
-                    exchange=trade.exchange,
+        exchange=trade.exchange,
-                    pair=trade.pair.replace('_', '/'),
+        pair=trade.pair.replace('_', '/'),
-                    pair_url=exchange.get_pair_detail_url(trade.pair),
+        pair_url=exchange.get_pair_detail_url(trade.pair),
-                    limit=limit
+        limit=limit
-                )
+    )
 
     # For regular case, when the configuration exists
     if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF:
@@ -213,12 +195,12 @@ def execute_sell(trade: Trade, limit: float) -> None:
         )
         message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \
                    '` / {profit_fiat:.3f} {fiat})`'.format(
-                        gain="profit" if fmt_exp_profit > 0 else "loss",
+                       gain="profit" if fmt_exp_profit > 0 else "loss",
-                        profit_percent=fmt_exp_profit,
+                       profit_percent=fmt_exp_profit,
-                        profit_coin=profit_trade,
+                       profit_coin=profit_trade,
-                        coin=_CONF['stake_currency'],
+                       coin=_CONF['stake_currency'],
-                        profit_fiat=profit_fiat,
+                       profit_fiat=profit_fiat,
-                        fiat=_CONF['fiat_display_currency'],
+                       fiat=_CONF['fiat_display_currency'],
                    )
     # Because telegram._forcesell does not have the configuration
     # Ignore the FIAT value and does not show the stake_currency as well

freqtrade/misc.py

@@ -3,10 +3,11 @@ import enum
 import json
 import logging
 import time
-from typing import Any, Callable, List, Dict
+import os
+from typing import Any, Callable, Dict, List
 
-from jsonschema import validate, Draft4Validator
+from jsonschema import Draft4Validator, validate
-from jsonschema.exceptions import best_match, ValidationError
+from jsonschema.exceptions import ValidationError, best_match
 from wrapt import synchronized
 
 from freqtrade import __version__
@@ -62,8 +63,8 @@ def load_config(path: str) -> Dict:
     try:
         validate(conf, CONF_SCHEMA)
         return conf
-    except ValidationError as ex:
+    except ValidationError as exception:
-        logger.fatal('Invalid configuration. See config.json.example. Reason: %s', ex)
+        logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception)
         raise ValidationError(
             best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message
         )
@@ -86,7 +87,7 @@ def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
     return result
 
 
-def parse_args_common(args: List[str], description: str):
+def common_args_parser(description: str):
     """
     Parses given common arguments and returns them as a parsed object.
     """
@@ -122,11 +123,11 @@ def parse_args(args: List[str], description: str):
     Parses given arguments and returns an argparse Namespace instance.
     Returns None if a sub command has been selected and executed.
     """
-    parser = parse_args_common(args, description)
+    parser = common_args_parser(description)
     parser.add_argument(
         '--dry-run-db',
-        help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is \
+        help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \
-             enabled.',  # noqa
+             instead of memory DB. Work only if dry_run is enabled.',
         action='store_true',
         dest='dry_run_db',
     )
@@ -134,13 +135,14 @@ def parse_args(args: List[str], description: str):
         '-dd', '--datadir',
         help='path to backtest data (default freqdata/tests/testdata',
         dest='datadir',
-        default='freqtrade/tests/testdata',
+        default=os.path.join('freqtrade', 'tests', 'testdata'),
         type=str,
         metavar='PATH',
     )
     parser.add_argument(
         '--dynamic-whitelist',
-        help='dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies)',  # noqa
+        help='dynamically generate and update whitelist \
+             based on 24h BaseVolume (Default 20 currencies)',  # noqa
         dest='dynamic_whitelist',
         const=20,
         type=int,

freqtrade/optimize/backtesting.py

@@ -1,19 +1,19 @@
 # pragma pylint: disable=missing-docstring,W0212
 
 import logging
-from typing import Tuple, Dict
+from typing import Dict, Tuple
 
 import arrow
 from pandas import DataFrame, Series
 from tabulate import tabulate
 
+import freqtrade.misc as misc
+import freqtrade.optimize as optimize
 from freqtrade import exchange
 from freqtrade.analyze import populate_buy_trend, populate_sell_trend
 from freqtrade.exchange import Bittrex
 from freqtrade.main import min_roi_reached
-import freqtrade.misc as misc
 from freqtrade.optimize import preprocess
-import freqtrade.optimize as optimize
 from freqtrade.persistence import Trade
 
 logger = logging.getLogger(__name__)
@@ -93,15 +93,14 @@ def get_trade_entry(pair, row, ticker, trade_count_lock, args):
         if min_roi_reached(trade, row2.close, row2.date) or \
             (row2.sell == 1 and use_sell_signal) or \
                 current_profit_percent <= stoploss:
-                current_profit_btc = trade.calc_profit(rate=row2.close)
+            current_profit_btc = trade.calc_profit(rate=row2.close)
-
+            return row2.Index, (pair,
-                return row2.Index, (pair,
+                                current_profit_percent,
-                                    current_profit_percent,
+                                current_profit_btc,
-                                    current_profit_btc,
+                                row2.Index - row.Index,
-                                    row2.Index - row.Index,
+                                current_profit_btc > 0,
-                                    current_profit_btc > 0,
+                                current_profit_btc < 0
-                                    current_profit_btc < 0
+                                )
-                                    )
 
 
 def backtest(args) -> DataFrame:
@@ -217,6 +216,6 @@ def start(args):
                         'record': args.export
                         })
     logger.info(
-        '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa
+        '\n==================================== BACKTESTING REPORT ====================================\n%s',  # noqa
         generate_text_table(data, results, config['stake_currency'], args.ticker_interval)
     )

freqtrade/optimize/hyperopt.py

@@ -4,14 +4,18 @@
 import json
 import logging
 import sys
+import pickle
+import signal
+import os
 from functools import reduce
 from math import exp
 from operator import itemgetter
 
-from hyperopt import fmin, tpe, hp, Trials, STATUS_OK, STATUS_FAIL, space_eval
+from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe
 from hyperopt.mongoexp import MongoTrials
 from pandas import DataFrame
 
+from freqtrade import main  # noqa
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
 from freqtrade.misc import load_config
@@ -27,7 +31,7 @@ logger = logging.getLogger(__name__)
 
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
-TOTAL_TRIES = None
+TOTAL_TRIES = 0
 _CURRENT_TRIES = 0
 CURRENT_BEST_LOSS = 100
 
@@ -43,6 +47,10 @@ EXPECTED_MAX_PROFIT = 3.85
 PROCESSED = None  # optimize.preprocess(optimize.load_data())
 OPTIMIZE_CONFIG = hyperopt_optimize_conf()
 
+# Hyperopt Trials
+TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle')
+TRIALS = Trials()
+
 # Monkey patch config
 from freqtrade import main  # noqa
 main._CONF = OPTIMIZE_CONFIG
@@ -99,6 +107,26 @@ SPACE = {
 }
 
 
+def save_trials(trials, trials_path=TRIALS_FILE):
+    """Save hyperopt trials to file"""
+    logger.info('Saving Trials to \'{}\''.format(trials_path))
+    pickle.dump(trials, open(trials_path, 'wb'))
+
+
+def read_trials(trials_path=TRIALS_FILE):
+    """Read hyperopt trials file"""
+    logger.info('Reading Trials from \'{}\''.format(trials_path))
+    trials = pickle.load(open(trials_path, 'rb'))
+    os.remove(trials_path)
+    return trials
+
+
+def log_trials_result(trials):
+    vals = json.dumps(trials.best_trial['misc']['vals'], indent=4)
+    results = trials.best_trial['result']['result']
+    logger.info('Best result:\n%s\nwith values:\n%s', results, vals)
+
+
 def log_results(results):
     """ log results if it is better than any previous evaluation """
     global CURRENT_BEST_LOSS
@@ -169,7 +197,7 @@ def format_results(results: DataFrame):
                 results.profit_percent.mean() * 100.0,
                 results.profit_BTC.sum(),
                 results.duration.mean() * 5,
-            )
+    )
 
 
 def buy_strategy_generator(params):
@@ -218,7 +246,8 @@ def buy_strategy_generator(params):
 
 
 def start(args):
-    global TOTAL_TRIES, PROCESSED, SPACE
+    global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES
+
     TOTAL_TRIES = args.epochs
 
     exchange._API = Bittrex({'key': '', 'secret': ''})
@@ -240,9 +269,19 @@ def start(args):
         logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')
 
         db_name = 'freqtrade_hyperopt'
-        trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
+        TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
     else:
-        trials = Trials()
+        logger.info('Preparing Trials..')
+        signal.signal(signal.SIGINT, signal_handler)
+        # read trials file if we have one
+        if os.path.exists(TRIALS_FILE):
+            TRIALS = read_trials()
+
+            _CURRENT_TRIES = len(TRIALS.results)
+            TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES
+            logger.info(
+                'Continuing with trials. Current: {}, Total: {}'
+                .format(_CURRENT_TRIES, TOTAL_TRIES))
 
     try:
         best_parameters = fmin(
@@ -250,10 +289,10 @@ def start(args):
             space=SPACE,
             algo=tpe.suggest,
             max_evals=TOTAL_TRIES,
-            trials=trials
+            trials=TRIALS
         )
 
-        results = sorted(trials.results, key=itemgetter('loss'))
+        results = sorted(TRIALS.results, key=itemgetter('loss'))
         best_result = results[0]['result']
 
     except ValueError:
@@ -267,3 +306,15 @@ def start(args):
 
     logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4))
     logger.info('Best Result:\n%s', best_result)
+
+    # Store trials result to file to resume next time
+    save_trials(TRIALS)
+
+
+def signal_handler(sig, frame):
+    """Hyperopt SIGINT handler"""
+    logger.info('Hyperopt received {}'.format(signal.Signals(sig).name))
+
+    save_trials(TRIALS)
+    log_trials_result(TRIALS)
+    sys.exit(0)

freqtrade/optimize/hyperopt_conf.py

@@ -15,10 +15,10 @@ def hyperopt_optimize_conf() -> dict:
         'stake_currency': 'BTC',
         'stake_amount': 0.01,
         "minimal_roi": {
-            '40':  0.0,
+            '40': 0.0,
-            '30':  0.01,
+            '30': 0.01,
-            '20':  0.02,
+            '20': 0.02,
-            '0':  0.04,
+            '0': 0.04,
         },
         'stoploss': -0.10,
         "bid_strategy": {

freqtrade/persistence.py

@@ -1,10 +1,11 @@
 import logging
 from datetime import datetime
 from decimal import Decimal, getcontext
-from typing import Optional, Dict
+from typing import Dict, Optional
 
 import arrow
-from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String, create_engine
+from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
+                        create_engine)
 from sqlalchemy.engine import Engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm.scoping import scoped_session

freqtrade/rpc/telegram.py

@@ -1,21 +1,21 @@
 import logging
 import re
+from datetime import datetime, timedelta
 from decimal import Decimal
-from datetime import timedelta, datetime
+from typing import Any, Callable
-from typing import Callable, Any
 
 import arrow
 from pandas import DataFrame
 from sqlalchemy import and_, func, text
 from tabulate import tabulate
-from telegram import ParseMode, Bot, Update, ReplyKeyboardMarkup
+from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update
 from telegram.error import NetworkError, TelegramError
 from telegram.ext import CommandHandler, Updater
 
-from freqtrade import exchange, __version__
+from freqtrade import __version__, exchange
-from freqtrade.misc import get_state, State, update_state
-from freqtrade.persistence import Trade
 from freqtrade.fiat_convert import CryptoToFiatConverter
+from freqtrade.misc import State, get_state, update_state
+from freqtrade.persistence import Trade
 
 # Remove noisy log messages
 logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO)
@@ -255,7 +255,7 @@ def _daily(bot: Bot, update: Update) -> None:
                 ),
                 symbol=_CONF['fiat_display_currency']
             )
-         ]
+        ]
         for key, value in profit_days.items()
     ]
     stats = tabulate(stats,

freqtrade/tests/conftest.py

@@ -2,10 +2,10 @@
 from datetime import datetime
 from unittest.mock import MagicMock
 
-import pytest
 import arrow
+import pytest
 from jsonschema import validate
-from telegram import Message, Chat, Update
+from telegram import Chat, Message, Update
 
 from freqtrade.misc import CONF_SCHEMA
 
@@ -20,10 +20,10 @@ def default_conf():
         "fiat_display_currency": "USD",
         "dry_run": True,
         "minimal_roi": {
-            "40":  0.0,
+            "40": 0.0,
-            "30":  0.01,
+            "30": 0.01,
-            "20":  0.02,
+            "20": 0.02,
-            "0":  0.04
+            "0": 0.04
         },
         "stoploss": -0.10,
         "unfilledtimeout": 600,

freqtrade/tests/exchange/test_exchange_bittrex.py

@@ -32,7 +32,7 @@ def _stub_config():
             'secret': ''}
 
 
-class Fake_bittrex():
+class FakeBittrex():
     def __init__(self, success=True):
         self.success = True  # Believe in yourself
         self.result = None
@@ -145,7 +145,7 @@ def test_exchange_bittrex_fee():
 
 def test_exchange_bittrex_buy_good(mocker):
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     uuid = wb.buy('BTC_ETH', 1, 1)
     assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid']
 
@@ -156,7 +156,7 @@ def test_exchange_bittrex_buy_good(mocker):
 
 def test_exchange_bittrex_sell_good(mocker):
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     uuid = wb.sell('BTC_ETH', 1, 1)
     assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid']
 
@@ -167,7 +167,7 @@ def test_exchange_bittrex_sell_good(mocker):
 
 def test_exchange_bittrex_get_balance(mocker):
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     bal = wb.get_balance('BTC_ETH')
     assert bal == fb.fake_get_balance(1)['result']['Balance']
 
@@ -178,7 +178,7 @@ def test_exchange_bittrex_get_balance(mocker):
 
 def test_exchange_bittrex_get_balances():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     bals = wb.get_balances()
     assert bals == fb.fake_get_balances()['result']
 
@@ -189,7 +189,7 @@ def test_exchange_bittrex_get_balances():
 
 def test_exchange_bittrex_get_ticker():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
 
     # Poll ticker, which updates the cache
     tick = wb.get_ticker('BTC_ETH')
@@ -210,7 +210,7 @@ def test_exchange_bittrex_get_ticker():
 
 def test_exchange_bittrex_get_ticker_bad():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     fb.result = {'success': True,
                  'result': {'Bid': 1}}  # incomplete result
     with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'):
@@ -222,15 +222,15 @@ def test_exchange_bittrex_get_ticker_bad():
         wb.get_ticker('BTC_ETH')
 
 
-def test_exchange_bittrex_get_ticker_historyOne():
+def test_exchange_bittrex_get_ticker_history_one():
     wb = make_wrap_bittrex()
-    Fake_bittrex()
+    FakeBittrex()
     assert wb.get_ticker_history('BTC_ETH', 1)
 
 
 def test_exchange_bittrex_get_ticker_history():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     assert wb.get_ticker_history('BTC_ETH', 5)
     with pytest.raises(ValueError, match=r'.*Cannot parse tick_interval.*'):
         wb.get_ticker_history('BTC_ETH', 2)
@@ -253,7 +253,7 @@ def test_exchange_bittrex_get_ticker_history():
 
 def test_exchange_bittrex_get_order():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     order = wb.get_order('someUUID')
     assert order['id'] == 'ABC123'
     fb.success = False
@@ -263,7 +263,7 @@ def test_exchange_bittrex_get_order():
 
 def test_exchange_bittrex_cancel_order():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     wb.cancel_order('someUUID')
     with pytest.raises(btx.OperationalException, match=r'no such order'):
         fb.success = False
@@ -284,7 +284,7 @@ def test_exchange_get_pair_detail_url():
 
 def test_exchange_get_markets():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     x = wb.get_markets()
     assert x == ['__']
     with pytest.raises(btx.OperationalException, match=r'market gone'):
@@ -294,7 +294,7 @@ def test_exchange_get_markets():
 
 def test_exchange_get_market_summaries():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     assert ['sum'] == wb.get_market_summaries()
     with pytest.raises(btx.OperationalException, match=r'no summary'):
         fb.success = False
@@ -303,7 +303,7 @@ def test_exchange_get_market_summaries():
 
 def test_exchange_get_wallet_health():
     wb = make_wrap_bittrex()
-    fb = Fake_bittrex()
+    fb = FakeBittrex()
     x = wb.get_wallet_health()
     assert x[0]['Currency'] == 'BTC_ETH'
     with pytest.raises(btx.OperationalException, match=r'bad health'):

freqtrade/tests/optimize/test_hyperopt.py

@@ -1,7 +1,6 @@
 # pragma pylint: disable=missing-docstring,W0212,C0103
-
 from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \
-    log_results
+    log_results, save_trials, read_trials
 
 
 def test_loss_calculation_prefer_correct_trade_count():
@@ -27,16 +26,37 @@ def test_loss_calculation_has_limited_profit():
 
 
 def create_trials(mocker):
+    """
+    When creating trials, mock the hyperopt Trials so that *by default*
+      - we don't create any pickle'd files in the filesystem
+      - we might have a pickle'd file so make sure that we return
+        false when looking for it
+    """
+    mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE',
+                 return_value='freqtrade/tests/optimize/ut_trials.pickle')
+    mocker.patch('freqtrade.optimize.hyperopt.os.path.exists',
+                 return_value=False)
+    mocker.patch('freqtrade.optimize.hyperopt.save_trials',
+                 return_value=None)
+    mocker.patch('freqtrade.optimize.hyperopt.read_trials',
+                 return_value=None)
+    mocker.patch('freqtrade.optimize.hyperopt.os.remove',
+                 return_value=True)
     return mocker.Mock(
         results=[{
             'loss': 1,
-            'result': 'foo'
+            'result': 'foo',
-        }]
+            'status': 'ok'
+        }],
+        best_trial={'misc': {'vals': {'adx': 999}}}
     )
 
 
 def test_start_calls_fmin(mocker):
-    mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker))
+    trials = create_trials(mocker)
+    mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials)
+    mocker.patch('freqtrade.optimize.hyperopt.sorted',
+                 return_value=trials.results)
     mocker.patch('freqtrade.optimize.preprocess')
     mocker.patch('freqtrade.optimize.load_data')
     mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
@@ -141,3 +161,63 @@ def test_fmin_throw_value_error(mocker, caplog):
 
     for line in exists:
         assert line in caplog.text
+
+
+def test_resuming_previous_hyperopt_results_succeeds(mocker):
+    import freqtrade.optimize.hyperopt as hyperopt
+    trials = create_trials(mocker)
+    mocker.patch('freqtrade.optimize.hyperopt.TRIALS',
+                 return_value=trials)
+    mocker.patch('freqtrade.optimize.hyperopt.os.path.exists',
+                 return_value=True)
+    mocker.patch('freqtrade.optimize.hyperopt.len',
+                 return_value=len(trials.results))
+    mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials',
+                             return_value=trials)
+    mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials',
+                             return_value=None)
+    mocker.patch('freqtrade.optimize.hyperopt.sorted',
+                 return_value=trials.results)
+    mocker.patch('freqtrade.optimize.preprocess')
+    mocker.patch('freqtrade.optimize.load_data')
+    mocker.patch('freqtrade.optimize.hyperopt.fmin',
+                 return_value={})
+    args = mocker.Mock(epochs=1,
+                       config='config.json.example',
+                       mongodb=False)
+
+    start(args)
+
+    mock_read.assert_called_once()
+    mock_save.assert_called_once()
+
+    current_tries = hyperopt._CURRENT_TRIES
+    total_tries = hyperopt.TOTAL_TRIES
+
+    assert current_tries == len(trials.results)
+    assert total_tries == (current_tries + len(trials.results))
+
+
+def test_save_trials_saves_trials(mocker):
+    trials = create_trials(mocker)
+    mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump',
+                             return_value=None)
+    trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE',
+                               return_value='ut_trials.pickle')
+    mocker.patch('freqtrade.optimize.hyperopt.open',
+                 return_value=trials_path)
+    save_trials(trials, trials_path)
+
+    mock_dump.assert_called_once_with(trials, trials_path)
+
+
+def test_read_trials_returns_trials_file(mocker):
+    trials = create_trials(mocker)
+    mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load',
+                             return_value=trials)
+    mock_open = mocker.patch('freqtrade.optimize.hyperopt.open',
+                             return_value=mock_load)
+
+    assert read_trials() == trials
+    mock_open.assert_called_once()
+    mock_load.assert_called_once()

freqtrade/tests/optimize/test_optimize.py

@@ -6,7 +6,7 @@ from shutil import copyfile
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
 from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\
-     download_backtesting_testdata, load_tickerdata_file
+    download_backtesting_testdata, load_tickerdata_file
 
 # Change this if modifying BTC_UNITEST testdatafile
 _btc_unittest_length = 13681

freqtrade/tests/test_acl_pair.py

@@ -7,17 +7,17 @@ from freqtrade.main import refresh_whitelist, gen_pair_whitelist
 
 def whitelist_conf():
     return {
-        "stake_currency": "BTC",
+        'stake_currency': 'BTC',
-        "exchange": {
+        'exchange': {
-            "pair_whitelist": [
+            'pair_whitelist': [
-                "BTC_ETH",
+                'BTC_ETH',
-                "BTC_TKN",
+                'BTC_TKN',
-                "BTC_TRST",
+                'BTC_TRST',
-                "BTC_SWT",
+                'BTC_SWT',
-                "BTC_BCC"
+                'BTC_BCC'
             ],
-            "pair_blacklist": [
+            'pair_blacklist': [
-                "BTC_BLK"
+                'BTC_BLK'
             ],
         },
     }
@@ -25,52 +25,51 @@ def whitelist_conf():
 
 def get_market_summaries():
     return [{
-         "MarketName": "BTC-TKN",
+        'MarketName': 'BTC-TKN',
-         "High": 0.00000919,
+        'High': 0.00000919,
-         "Low": 0.00000820,
+        'Low': 0.00000820,
-         "Volume": 74339.61396015,
+        'Volume': 74339.61396015,
-         "Last": 0.00000820,
+        'Last': 0.00000820,
-         "BaseVolume": 1664,
+        'BaseVolume': 1664,
-         "TimeStamp": "2014-07-09T07:19:30.15",
+        'TimeStamp': '2014-07-09T07:19:30.15',
-         "Bid": 0.00000820,
+        'Bid': 0.00000820,
-         "Ask": 0.00000831,
+        'Ask': 0.00000831,
-         "OpenBuyOrders": 15,
+        'OpenBuyOrders': 15,
-         "OpenSellOrders": 15,
+        'OpenSellOrders': 15,
-         "PrevDay": 0.00000821,
+        'PrevDay': 0.00000821,
-         "Created": "2014-03-20T06:00:00",
+        'Created': '2014-03-20T06:00:00',
-         "DisplayMarketName": ""
+        'DisplayMarketName': ''
-      }, {
+    }, {
-         "MarketName": "BTC-ETH",
+        'MarketName': 'BTC-ETH',
-         "High": 0.00000072,
+        'High': 0.00000072,
-         "Low": 0.00000001,
+        'Low': 0.00000001,
-         "Volume": 166340678.42280999,
+        'Volume': 166340678.42280999,
-         "Last": 0.00000005,
+        'Last': 0.00000005,
-         "BaseVolume": 42,
+        'BaseVolume': 42,
-         "TimeStamp": "2014-07-09T07:21:40.51",
+        'TimeStamp': '2014-07-09T07:21:40.51',
-         "Bid": 0.00000004,
+        'Bid': 0.00000004,
-         "Ask": 0.00000005,
+        'Ask': 0.00000005,
-         "OpenBuyOrders": 18,
+        'OpenBuyOrders': 18,
-         "OpenSellOrders": 18,
+        'OpenSellOrders': 18,
-         "PrevDay": 0.00000002,
+        'PrevDay': 0.00000002,
-         "Created": "2014-05-30T07:57:49.637",
+        'Created': '2014-05-30T07:57:49.637',
-         "DisplayMarketName": ""
+        'DisplayMarketName': ''
-      }, {
+    }, {
-         "MarketName": "BTC-BLK",
+        'MarketName': 'BTC-BLK',
-         "High": 0.00000072,
+        'High': 0.00000072,
-         "Low": 0.00000001,
+        'Low': 0.00000001,
-         "Volume": 166340678.42280999,
+        'Volume': 166340678.42280999,
-         "Last": 0.00000005,
+        'Last': 0.00000005,
-         "BaseVolume": 3,
+        'BaseVolume': 3,
-         "TimeStamp": "2014-07-09T07:21:40.51",
+        'TimeStamp': '2014-07-09T07:21:40.51',
-         "Bid": 0.00000004,
+        'Bid': 0.00000004,
-         "Ask": 0.00000005,
+        'Ask': 0.00000005,
-         "OpenBuyOrders": 18,
+        'OpenBuyOrders': 18,
-         "OpenSellOrders": 18,
+        'OpenSellOrders': 18,
-         "PrevDay": 0.00000002,
+        'PrevDay': 0.00000002,
-         "Created": "2014-05-30T07:57:49.637",
+        'Created': '2014-05-30T07:57:49.637',
-         "DisplayMarketName": ""
+        'DisplayMarketName': ''
-      }
+    }]
-    ]
 
 
 def get_health():
@@ -95,7 +94,8 @@ def test_refresh_market_pair_not_in_whitelist(mocker):
     mocker.patch.dict('freqtrade.main._CONF', conf)
     mocker.patch.multiple('freqtrade.main.exchange',
                           get_wallet_health=get_health)
-    refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist'] + ['BTC_XXX'])
+    refreshedwhitelist = refresh_whitelist(
+        conf['exchange']['pair_whitelist'] + ['BTC_XXX'])
     # List ordered by BaseVolume
     whitelist = ['BTC_ETH', 'BTC_TKN']
     # Ensure all except those in whitelist are removed
@@ -123,7 +123,8 @@ def test_refresh_whitelist_dynamic(mocker):
                           get_market_summaries=get_market_summaries)
     # argument: use the whitelist dynamically by exchange-volume
     whitelist = ['BTC_TKN', 'BTC_ETH']
-    refreshedwhitelist = refresh_whitelist(gen_pair_whitelist(conf['stake_currency']))
+    refreshedwhitelist = refresh_whitelist(
+        gen_pair_whitelist(conf['stake_currency']))
     assert whitelist == refreshedwhitelist
 
 

freqtrade/tests/test_analyze.py

@@ -6,8 +6,9 @@ import arrow
 import pytest
 from pandas import DataFrame
 
-from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \
+from freqtrade.analyze import (SignalType, get_signal, parse_ticker_dataframe,
-    get_signal, SignalType, populate_sell_trend
+                               populate_buy_trend, populate_indicators,
+                               populate_sell_trend)
 
 
 @pytest.fixture

freqtrade/tests/test_dataframe.py

@@ -1,7 +1,7 @@
 import pandas
 
-from freqtrade import analyze
 import freqtrade.optimize
+from freqtrade import analyze
 
 _pairs = ['BTC_ETH']
 

freqtrade/tests/test_fiat_convert.py

@@ -1,10 +1,11 @@
 # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103
 
 import time
-import pytest
 from unittest.mock import MagicMock
 
-from freqtrade.fiat_convert import CryptoToFiatConverter, CryptoFiat
+import pytest
+
+from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter
 
 
 def test_pair_convertion_object():

freqtrade/tests/test_main.py

@@ -1,28 +1,27 @@
 # pragma pylint: disable=missing-docstring,C0103
 import copy
+import logging
 from unittest.mock import MagicMock
 
+import arrow
 import pytest
 import requests
-import logging
-import arrow
 from sqlalchemy import create_engine
 
+import freqtrade.main as main
 from freqtrade import DependencyException, OperationalException
 from freqtrade.analyze import SignalType
 from freqtrade.exchange import Exchanges
-from freqtrade.main import create_trade, handle_trade, init, \
+from freqtrade.main import (_process, check_handle_timedout, create_trade,
-    get_target_bid, _process, execute_sell, check_handle_timedout
+                            execute_sell, get_target_bid, handle_trade, init)
-from freqtrade.misc import get_state, State
+from freqtrade.misc import State, get_state
 from freqtrade.persistence import Trade
-import freqtrade.main as main
 
 
-# Test that main() can start backtesting or hyperopt.
-# and also ensure we can pass some specific arguments
-# argument parsing is done in test_misc.py
-
 def test_parse_args_backtesting(mocker):
+    """ Test that main() can start backtesting or hyperopt.
+        and also ensure we can pass some specific arguments
+        argument parsing is done in test_misc.py """
     backtesting_mock = mocker.patch(
         'freqtrade.optimize.backtesting.start', MagicMock())
     with pytest.raises(SystemExit, match=r'0'):
@@ -269,7 +268,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker):
     assert trade.close_date is not None
 
 
-def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog):
+def test_handle_trade_roi(default_conf, ticker, mocker, caplog):
     default_conf.update({'experimental': {'use_sell_signal': True}})
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
 
@@ -301,7 +300,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog)
     assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples
 
 
-def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog):
+def test_handle_trade_experimental(default_conf, ticker, mocker, caplog):
     default_conf.update({'experimental': {'use_sell_signal': True}})
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
 
@@ -353,7 +352,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo
         handle_trade(trade)
 
 
-def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order_old, mocker):
+def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     cancel_order_mock = MagicMock()
     mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
@@ -385,7 +384,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order
     assert len(trades) == 0
 
 
-def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_order_old, mocker):
+def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     cancel_order_mock = MagicMock()
     mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
@@ -418,7 +417,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_ord
 
 
 def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial,
-                                       health, mocker):
+                                       mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     cancel_order_mock = MagicMock()
     mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
@@ -624,54 +623,54 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker):
 
 
 def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker):
-        default_conf['experimental'] = {
+    default_conf['experimental'] = {
-            'use_sell_signal': True,
+        'use_sell_signal': True,
-            'sell_profit_only': True,
+        'sell_profit_only': True,
-        }
+    }
 
-        mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
-        mocker.patch('freqtrade.main.min_roi_reached', return_value=False)
+    mocker.patch('freqtrade.main.min_roi_reached', return_value=False)
-        mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True)
+    mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True)
-        mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
+    mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
-        mocker.patch.multiple('freqtrade.main.exchange',
+    mocker.patch.multiple('freqtrade.main.exchange',
-                              validate_pairs=MagicMock(),
+                          validate_pairs=MagicMock(),
-                              get_ticker=MagicMock(return_value={
+                          get_ticker=MagicMock(return_value={
-                                  'bid': 0.00000172,
+                              'bid': 0.00000172,
-                                  'ask': 0.00000173,
+                              'ask': 0.00000173,
-                                  'last': 0.00000172
+                              'last': 0.00000172
-                              }),
+                          }),
-                              buy=MagicMock(return_value='mocked_limit_buy'))
+                          buy=MagicMock(return_value='mocked_limit_buy'))
 
-        init(default_conf, create_engine('sqlite://'))
+    init(default_conf, create_engine('sqlite://'))
-        create_trade(0.001)
+    create_trade(0.001)
 
-        trade = Trade.query.first()
+    trade = Trade.query.first()
-        trade.update(limit_buy_order)
+    trade.update(limit_buy_order)
-        assert handle_trade(trade) is False
+    assert handle_trade(trade) is False
 
 
 def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker):
-        default_conf['experimental'] = {
+    default_conf['experimental'] = {
-            'use_sell_signal': True,
+        'use_sell_signal': True,
-            'sell_profit_only': False,
+        'sell_profit_only': False,
-        }
+    }
 
-        mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
-        mocker.patch('freqtrade.main.min_roi_reached', return_value=False)
+    mocker.patch('freqtrade.main.min_roi_reached', return_value=False)
-        mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True)
+    mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True)
-        mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
+    mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock())
-        mocker.patch.multiple('freqtrade.main.exchange',
+    mocker.patch.multiple('freqtrade.main.exchange',
-                              validate_pairs=MagicMock(),
+                          validate_pairs=MagicMock(),
-                              get_ticker=MagicMock(return_value={
+                          get_ticker=MagicMock(return_value={
-                                  'bid': 0.00000172,
+                              'bid': 0.00000172,
-                                  'ask': 0.00000173,
+                              'ask': 0.00000173,
-                                  'last': 0.00000172
+                              'last': 0.00000172
-                              }),
+                          }),
-                              buy=MagicMock(return_value='mocked_limit_buy'))
+                          buy=MagicMock(return_value='mocked_limit_buy'))
 
-        init(default_conf, create_engine('sqlite://'))
+    init(default_conf, create_engine('sqlite://'))
-        create_trade(0.001)
+    create_trade(0.001)
 
-        trade = Trade.query.first()
+    trade = Trade.query.first()
-        trade.update(limit_buy_order)
+    trade.update(limit_buy_order)
-        assert handle_trade(trade) is True
+    assert handle_trade(trade) is True

freqtrade/tests/test_misc.py

@@ -1,15 +1,15 @@
 # pragma pylint: disable=missing-docstring,C0103
+import argparse
 import json
 import time
-import argparse
 from copy import deepcopy
 
 import pytest
 from unittest.mock import MagicMock
 from jsonschema import ValidationError
 
-from freqtrade.misc import throttle, parse_args, load_config,\
+from freqtrade.misc import (common_args_parser, load_config, parse_args,
-     parse_args_common, file_dump_json
+                            throttle, file_dump_json)
 
 
 def test_throttle():
@@ -40,12 +40,10 @@ def test_throttle_with_assets():
     assert result == -1
 
 
-# Parse common command-line-arguments
+# Parse common command-line-arguments. Used for all tools
-# used for all tools
-
 
 def test_parse_args_none():
-    args = parse_args_common([], '')
+    args = common_args_parser('')
     assert isinstance(args, argparse.ArgumentParser)
 
 
@@ -88,12 +86,12 @@ def test_parse_args_invalid():
 
 def test_parse_args_dynamic_whitelist():
     args = parse_args(['--dynamic-whitelist'], '')
-    assert args.dynamic_whitelist is 20
+    assert args.dynamic_whitelist == 20
 
 
 def test_parse_args_dynamic_whitelist_10():
     args = parse_args(['--dynamic-whitelist', '10'], '')
-    assert args.dynamic_whitelist is 10
+    assert args.dynamic_whitelist == 10
 
 
 def test_parse_args_dynamic_whitelist_invalid_values():

freqtrade/tests/test_persistence.py

@@ -1,9 +1,10 @@
 # pragma pylint: disable=missing-docstring
+import os
+
 import pytest
 
-import os
 from freqtrade.exchange import Exchanges
-from freqtrade.persistence import init, Trade
+from freqtrade.persistence import Trade, init
 
 
 def test_init_create_session(default_conf, mocker):

scripts/plot_dataframe.py

@@ -6,11 +6,11 @@ import matplotlib  # Install PYQT5 manually if you want to test this helper func
 matplotlib.use("Qt5Agg")
 import matplotlib.pyplot as plt
 from freqtrade import exchange, analyze
-from freqtrade.misc import parse_args_common
+from freqtrade.misc import common_args_parser
 
 
 def plot_parse_args(args ):
-    parser = parse_args_common(args, 'Graph utility')
+    parser = common_args_parser(args, 'Graph utility')
     parser.add_argument(
         '-p', '--pair',
         help = 'What currency pair',