Merge branch 'develop' into rpc-refactor

.gitignore

@@ -7,6 +7,7 @@ config.json
 logfile.txt
 hyperopt_trials.pickle
 user_data/
+freqtrade-plot.html
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

docs/backtesting.md

@@ -51,6 +51,12 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --live
 python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101
 ```
 
+**With a (custom) strategy file**
+```bash
+python3.6 ./freqtrade/main.py -s currentstrategy backtesting
+```
+Where `-s currentstrategy` refers to a filename `currentstrategy.py` in `freqtrade/user_data/strategies`
+
 **Exporting trades to file**
 ```bash
 freqtrade backtesting --export trades

docs/installation.md

@@ -297,6 +297,7 @@ cp config.json.example config.json
 ```bash
 python3.6 -m venv .env
 source .env/bin/activate
+pip3.6 install --upgrade pip
 pip3.6 install -r requirements.txt
 pip3.6 install -e .
 ```

docs/telegram-usage.md

@@ -15,7 +15,7 @@ official commands. You can ask at any moment for help with `/help`.
 |  Command | Default | Description |
 |----------|---------|-------------|
 | `/start` | | Starts the trader
-| `/stop` | | Starts the trader
+| `/stop` | | Stops the trader
 | `/status` | | Lists all open trades
 | `/status table` | | List all open trades in a table format
 | `/count` | | Displays number of trades used and available
@@ -126,4 +126,4 @@ Day         Profit BTC      Profit USD
 ```
 
 ## /version
-> **Version:** `0.14.3` 
+> **Version:** `0.14.3` 

freqtrade/analyze.py

@@ -112,7 +112,7 @@ def get_signal(pair: str, interval: int) -> (bool, bool):
 
     # Check if dataframe is out of date
     signal_date = arrow.get(latest['date'])
-    if signal_date < arrow.now() - timedelta(minutes=10):
+    if signal_date < arrow.now() - timedelta(minutes=(interval + 5)):
         logger.warning('Too old dataframe for pair %s', pair)
         return (False, False)  # return False ?
 

freqtrade/misc.py

@@ -60,7 +60,7 @@ def common_datearray(dfs):
     return np.sort(arr, axis=0)
 
 
-def file_dump_json(filename, data):
+def file_dump_json(filename, data) -> None:
     with open(filename, 'w') as fp:
         json.dump(data, fp)
 
@@ -287,27 +287,27 @@ def hyperopt_options(parser: argparse.ArgumentParser) -> None:
 def parse_timerange(text):
     if text is None:
         return None
-    syntax = [('^-(\d{8})$',        (None,    'date')),
-              ('^(\d{8})-$',        ('date',  None)),
-              ('^(\d{8})-(\d{8})$', ('date',  'date')),
-              ('^(-\d+)$',          (None,    'line')),
-              ('^(\d+)-$',          ('line',  None)),
-              ('^(\d+)-(\d+)$',     ('index', 'index'))]
+    syntax = [(r'^-(\d{8})$', (None, 'date')),
+              (r'^(\d{8})-$', ('date', None)),
+              (r'^(\d{8})-(\d{8})$', ('date', 'date')),
+              (r'^(-\d+)$', (None, 'line')),
+              (r'^(\d+)-$', ('line', None)),
+              (r'^(\d+)-(\d+)$', ('index', 'index'))]
     for rex, stype in syntax:
         # Apply the regular expression to text
-        m = re.match(rex, text)
-        if m:  # Regex has matched
-            rvals = m.groups()
-            n = 0
+        match = re.match(rex, text)
+        if match:  # Regex has matched
+            rvals = match.groups()
+            index = 0
             start = None
             stop = None
             if stype[0]:
-                start = rvals[n]
+                start = rvals[index]
                 if stype[0] != 'date':
                     start = int(start)
-                n += 1
+                index += 1
             if stype[1]:
-                stop = rvals[n]
+                stop = rvals[index]
                 if stype[1] != 'date':
                     stop = int(stop)
             return (stype, start, stop)

freqtrade/optimize/__init__.py

@@ -10,6 +10,7 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
 
 from freqtrade import misc
 from user_data.hyperopt_conf import hyperopt_optimize_conf
+import gzip
 
 logger = logging.getLogger(__name__)
 
@@ -26,8 +27,7 @@ def trim_tickerlist(tickerlist, timerange):
     return tickerlist
 
 
-def load_tickerdata_file(datadir, pair, ticker_interval,
-                         timerange=None):
+def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None):
     """
     Load a pair from file,
     :return dict OR empty if unsuccesful
@@ -38,13 +38,19 @@ def load_tickerdata_file(datadir, pair, ticker_interval,
         pair=pair,
         ticker_interval=ticker_interval,
     )
-    # The file does not exist we download it
-    if not os.path.isfile(file):
+    gzipfile = file + '.gz'
+
+    # If the file does not exist we download it when None is returned.
+    # If file exists, read the file, load the json
+    if os.path.isfile(gzipfile):
+        with gzip.open(gzipfile) as tickerdata:
+            pairdata = json.load(tickerdata)
+    elif os.path.isfile(file):
+        with open(file) as tickerdata:
+            pairdata = json.load(tickerdata)
+    else:
         return None
 
-    # Read the file, load the json
-    with open(file) as tickerdata:
-        pairdata = json.load(tickerdata)
     if timerange:
         pairdata = trim_tickerlist(pairdata, timerange)
     return pairdata

freqtrade/rpc/__init__.py

@@ -322,7 +322,10 @@ def rpc_balance(fiat_display_currency):
         if coin == 'BTC':
             currency["Rate"] = 1.0
         else:
-            currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid']
+            if coin == 'USDT':
+                currency["Rate"] = 1.0 / exchange.get_ticker('USDT_BTC', False)['bid']
+            else:
+                currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid']
         currency['BTC'] = currency["Rate"] * currency["Balance"]
         total = total + currency['BTC']
         output.append({'currency': currency['Currency'],

freqtrade/rpc/telegram.py

@@ -244,7 +244,6 @@ def _profit(bot: Bot, update: Update) -> None:
 def _balance(bot: Bot, update: Update) -> None:
     """
     Handler for /balance
-    Returns current account balance per crypto
     """
     (error, result) = rpc_balance(_CONF['fiat_display_currency'])
     if error:
@@ -259,7 +258,6 @@ def _balance(bot: Bot, update: Update) -> None:
 *Balance*: {balance}
 *Pending*: {pending}
 *Est. BTC*: {est_btc: .8f}
-
 """.format(**currency)
 
     output += """*Estimated Value*:

freqtrade/strategy/interface.py

@@ -17,13 +17,6 @@ class IStrategy(ABC):
         stoploss -> float: optimal stoploss designed for the strategy
         ticker_interval -> int: value of the ticker interval to use for the strategy
     """
-    @property
-    def name(self) -> str:
-        """
-        Name of the strategy.
-        :return: str representation of the class name
-        """
-        return self.__class__.__name__
 
     @abstractmethod
     def populate_indicators(self, dataframe: DataFrame) -> DataFrame:

freqtrade/strategy/strategy.py

@@ -1,3 +1,5 @@
+# pragma pylint: disable=attribute-defined-outside-init
+
 """
 This module load custom strategies
 """
@@ -21,7 +23,7 @@ class Strategy(object):
 
     DEFAULT_STRATEGY = 'default_strategy'
 
-    def __new__(cls):
+    def __new__(cls) -> object:
         """
         Used to create the Singleton
         :return: Strategy object
@@ -30,15 +32,7 @@ class Strategy(object):
             Strategy.__instance = object.__new__(cls)
         return Strategy.__instance
 
-    def __init__(self):
-        if Strategy.__instance is None:
-            self.logger = None
-            self.minimal_roi = None
-            self.stoploss = None
-            self.ticker_interval = None
-            self.custom_strategy = None
-
-    def init(self, config):
+    def init(self, config: dict) -> None:
         """
         Load the custom class from config parameter
         :param config:

freqtrade/tests/optimize/test_optimize.py

@@ -1,12 +1,14 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 
 import os
+import json
 import logging
+import uuid
 from shutil import copyfile
 from freqtrade import exchange, optimize
 from freqtrade.exchange import Bittrex
 from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\
-    download_backtesting_testdata, load_tickerdata_file
+    download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, file_dump_json
 
 # Change this if modifying BTC_UNITEST testdatafile
 _BTC_UNITTEST_LENGTH = 13681
@@ -202,9 +204,14 @@ def test_download_backtesting_testdata2(mocker):
 
 
 def test_load_tickerdata_file():
+    # 7 does not exist in either format.
     assert not load_tickerdata_file(None, 'BTC_UNITEST', 7)
+    # 1 exists only as a .json
     tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 1)
     assert _BTC_UNITTEST_LENGTH == len(tickerdata)
+    # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json
+    tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 8)
+    assert _BTC_UNITTEST_LENGTH == len(tickerdata)
 
 
 def test_init(default_conf, mocker):
@@ -220,3 +227,73 @@ def test_tickerdata_to_dataframe():
     tickerlist = {'BTC_UNITEST': tick}
     data = optimize.tickerdata_to_dataframe(tickerlist)
     assert len(data['BTC_UNITEST']) == 100
+
+
+def test_trim_tickerlist():
+    with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file:
+        ticker_list = json.load(data_file)
+    ticker_list_len = len(ticker_list)
+
+    # Test the pattern ^(-\d+)$
+    # This pattern remove X element from the beginning
+    timerange = ((None, 'line'), None, 5)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_list_len == ticker_len + 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[-1] is ticker[-1]  # The last element must be the same
+
+    # Test the pattern ^(\d+)-$
+    # This pattern keep X element from the end
+    timerange = (('line', None), 5, None)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is ticker[0]  # The first element must be the same
+    assert ticker_list[-1] is not ticker[-1]  # The last element should be different
+
+    # Test the pattern ^(\d+)-(\d+)$
+    # This pattern extract a window
+    timerange = (('index', 'index'), 5, 10)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_len == 5
+    assert ticker_list[0] is not ticker[0]  # The first element should be different
+    assert ticker_list[5] is ticker[0]  # The list starts at the index 5
+    assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
+
+    # Test a wrong pattern
+    # This pattern must return the list unchanged
+    timerange = ((None, None), None, 5)
+    ticker = trim_tickerlist(ticker_list, timerange)
+    ticker_len = len(ticker)
+
+    assert ticker_list_len == ticker_len
+
+
+def test_file_dump_json():
+
+    file = 'freqtrade/tests/testdata/test_{id}.json'.format(id=str(uuid.uuid4()))
+    data = {'bar': 'foo'}
+
+    # check the file we will create does not exist
+    assert os.path.isfile(file) is False
+
+    # Create the Json file
+    file_dump_json(file, data)
+
+    # Check the file was create
+    assert os.path.isfile(file) is True
+
+    # Open the Json file created and test the data is in it
+    with open(file) as data_file:
+        json_from_file = json.load(data_file)
+
+    assert 'bar' in json_from_file
+    assert json_from_file['bar'] == 'foo'
+
+    # Remove the file
+    _clean_test_file(file)

freqtrade/tests/rpc/test_rpc_telegram.py

@@ -381,8 +381,7 @@ def test_performance_handle(
     assert '<code>BTC_ETH\t6.20% (1)</code>' in msg_mock.call_args_list[0][0][0]
 
 
-def test_daily_handle(
-        default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker):
+def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False))
     msg_mock = MagicMock()
@@ -445,6 +444,25 @@ def test_daily_handle(
     assert str('  2.798 USD') in msg_mock.call_args_list[0][0][0]
     assert str('  3 trades') in msg_mock.call_args_list[0][0][0]
 
+
+def test_daily_wrong_input(default_conf, update, ticker, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False))
+    msg_mock = MagicMock()
+    mocker.patch('freqtrade.main.rpc.send_msg', MagicMock())
+    mocker.patch.multiple('freqtrade.rpc.telegram',
+                          _CONF=default_conf,
+                          init=MagicMock(),
+                          send_msg=msg_mock)
+    mocker.patch.multiple('freqtrade.main.exchange',
+                          validate_pairs=MagicMock(),
+                          get_ticker=ticker)
+    mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap',
+                          ticker=MagicMock(return_value={'price_usd': 15000.0}),
+                          _cache_symbols=MagicMock(return_value={'BTC': 1}))
+    mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0)
+    init(default_conf, create_engine('sqlite://'))
+
     # Try invalid data
     msg_mock.reset_mock()
     update_state(State.RUNNING)
@@ -453,6 +471,13 @@ def test_daily_handle(
     assert msg_mock.call_count == 1
     assert 'must be an integer greater than 0' in msg_mock.call_args_list[0][0][0]
 
+    # Try invalid data
+    msg_mock.reset_mock()
+    update_state(State.RUNNING)
+    update.message.text = '/daily today'
+    _daily(bot=MagicMock(), update=update)
+    assert str('Daily Profit over the last 7 days') in msg_mock.call_args_list[0][0][0]
+
 
 def test_count_handle(default_conf, update, ticker, mocker):
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
@@ -593,7 +618,34 @@ def test_telegram_balance_handle(default_conf, update, mocker):
         'Available': 0.0,
         'Pending': 0.0,
         'CryptoAddress': 'XXXX',
+    }, {
+        'Currency': 'USDT',
+        'Balance': 10000.0,
+        'Available': 0.0,
+        'Pending': 0.0,
+        'CryptoAddress': 'XXXX',
+    }, {
+        'Currency': 'LTC',
+        'Balance': 10.0,
+        'Available': 10.0,
+        'Pending': 0.0,
+        'CryptoAddress': 'XXXX',
     }]
+
+    def mock_ticker(symbol, refresh):
+        if symbol == 'USDT_BTC':
+            return {
+                'bid': 10000.00,
+                'ask': 10000.00,
+                'last': 10000.00,
+            }
+        else:
+            return {
+                'bid': 0.1,
+                'ask': 0.1,
+                'last': 0.1,
+            }
+
     mocker.patch.dict('freqtrade.main._CONF', default_conf)
     msg_mock = MagicMock()
     mocker.patch.multiple('freqtrade.rpc.telegram',
@@ -605,12 +657,32 @@ def test_telegram_balance_handle(default_conf, update, mocker):
     mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap',
                           ticker=MagicMock(return_value={'price_usd': 15000.0}),
                           _cache_symbols=MagicMock(return_value={'BTC': 1}))
+    mocker.patch('freqtrade.main.exchange.get_ticker', side_effect=mock_ticker)
 
     _balance(bot=MagicMock(), update=update)
+    result = msg_mock.call_args_list[0][0][0]
     assert msg_mock.call_count == 1
-    assert '*Currency*: BTC' in msg_mock.call_args_list[0][0][0]
-    assert 'Balance' in msg_mock.call_args_list[0][0][0]
-    assert 'Est. BTC' in msg_mock.call_args_list[0][0][0]
+    assert '*Currency*: BTC' in result
+    assert '*Currency*: ETH' not in result
+    assert '*Currency*: USDT' in result
+    assert 'Balance' in result
+    assert 'Est. BTC' in result
+    assert '*BTC*:  12.00000000' in result
+
+
+def test_zero_balance_handle(default_conf, update, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    msg_mock = MagicMock()
+    mocker.patch.multiple('freqtrade.rpc.telegram',
+                          _CONF=default_conf,
+                          init=MagicMock(),
+                          send_msg=msg_mock)
+    mocker.patch.multiple('freqtrade.main.exchange',
+                          get_balances=MagicMock(return_value=[]))
+    _balance(bot=MagicMock(), update=update)
+    result = msg_mock.call_args_list[0][0][0]
+    assert msg_mock.call_count == 1
+    assert '`All balances are zero.`' in result
 
 
 def test_help_handle(default_conf, update, mocker):
@@ -666,3 +738,18 @@ def test_send_msg_network_error(default_conf, mocker):
 
     # Bot should've tried to send it twice
     assert len(bot.method_calls) == 2
+
+
+def test_init(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker):
+    mocker.patch.dict('freqtrade.main._CONF', default_conf)
+    mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False))
+    msg_mock = MagicMock()
+    mocker.patch('freqtrade.main.rpc.send_msg', MagicMock())
+    mocker.patch.multiple('freqtrade.rpc.telegram',
+                          _CONF=default_conf,
+                          init=MagicMock(),
+                          send_msg=msg_mock)
+    mocker.patch.multiple('freqtrade.main.exchange',
+                          validate_pairs=MagicMock(),
+                          get_ticker=ticker)
+    init(default_conf, create_engine('sqlite://'))

freqtrade/tests/strategy/test_strategy.py

@@ -16,6 +16,7 @@ def test_sanitize_module_name():
 
 def test_search_strategy():
     assert Strategy._search_strategy('default_strategy') == '.'
+    assert Strategy._search_strategy('test_strategy') == 'user_data.strategies.'
     assert Strategy._search_strategy('super_duper') is None
 
 

freqtrade/tests/test_misc.py

@@ -5,11 +5,12 @@ import time
 from copy import deepcopy
 from unittest.mock import MagicMock
 
+import datetime
 import pytest
 from jsonschema import ValidationError
-
+from freqtrade.analyze import parse_ticker_dataframe
 from freqtrade.misc import (common_args_parser, file_dump_json, load_config,
-                            parse_args, parse_timerange, throttle)
+                            parse_args, parse_timerange, throttle, datesarray_to_datetimearray)
 
 
 def test_throttle():
@@ -178,3 +179,18 @@ def test_load_config_missing_attributes(default_conf, mocker):
             read_data=json.dumps(conf)))
     with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'):
         load_config('somefile')
+
+
+def test_datesarray_to_datetimearray(ticker_history):
+    dataframes = parse_ticker_dataframe(ticker_history)
+    dates = datesarray_to_datetimearray(dataframes['date'])
+
+    assert isinstance(dates[0], datetime.datetime)
+    assert dates[0].year == 2017
+    assert dates[0].month == 11
+    assert dates[0].day == 26
+    assert dates[0].hour == 8
+    assert dates[0].minute == 50
+
+    date_len = len(dates)
+    assert date_len == 3

freqtrade/tests/testdata/BTC_UNITEST-8.json

@@ -0,0 +1,3 @@
+[
+  {"O": 0.00162008, "H": 0.00162008, "L": 0.00162008, "C": 0.00162008, "V": 108.14853839, "T": "2017-11-04T23:02:00", "BV": 0.17520927}
+]