merged

.github/ISSUE_TEMPLATE.md

@@ -6,10 +6,12 @@ If it hasn't been reported, please create a new issue.
 ## Step 2: Describe your environment
 
   * Python Version: _____ (`python -V`)
+  * CCXT version: _____ (`pip freeze | grep ccxt`)
   * Branch: Master | Develop
   * Last Commit ID: _____ (`git log --format="%H" -n 1`)
- 
+  
 ## Step 3: Describe the problem:
+
 *Explain the problem you have encountered*
 
 ### Steps to reproduce:

CONTRIBUTING.md

@@ -42,6 +42,11 @@ pip3.6 install flake8 coveralls
 flake8 freqtrade
 ```
 
+We receive a lot of code that fails the `flake8` checks.
+To help with that, we encourage you to install the git pre-commit 
+hook that will warn you when you try to commit code that fails these checks. 
+Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html).
+
 ## 3. Test if all type-hints are correct
 
 **Install packages** (If not already installed)

docs/telegram-usage.md

@@ -16,6 +16,7 @@ official commands. You can ask at any moment for help with `/help`.
 |----------|---------|-------------|
 | `/start` | | Starts the trader
 | `/stop` | | Stops the trader
+| `/reload_conf` | | Reloads the configuration file
 | `/status` | | Lists all open trades
 | `/status table` | | List all open trades in a table format
 | `/count` | | Displays number of trades used and available

freqtrade/freqtradebot.py

@@ -76,17 +76,14 @@ class FreqtradeBot(object):
         else:
             self.state = State.STOPPED
 
-    def clean(self) -> bool:
+    def cleanup(self) -> None:
         """
-        Cleanup the application state und finish all pending tasks
+        Cleanup pending resources on an already stopped bot
         :return: None
         """
-        self.rpc.send_msg('*Status:* `Stopping trader...`')
-        logger.info('Stopping trader and cleaning up modules...')
-        self.state = State.STOPPED
+        logger.info('Cleaning up modules ...')
         self.rpc.cleanup()
         persistence.cleanup()
-        return True
 
     def worker(self, old_state: State = None) -> State:
         """

freqtrade/main.py

@@ -5,12 +5,14 @@ Read the documentation to know what cli arguments you need.
 """
 import logging
 import sys
+from argparse import Namespace
 from typing import List
 
 from freqtrade import OperationalException
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.freqtradebot import FreqtradeBot
+from freqtrade.state import State
 
 logger = logging.getLogger('freqtrade')
 
@@ -44,6 +46,8 @@ def main(sysargv: List[str]) -> None:
         state = None
         while 1:
             state = freqtrade.worker(old_state=state)
+            if state == State.RELOAD_CONF:
+                freqtrade = reconfigure(freqtrade, args)
 
     except KeyboardInterrupt:
         logger.info('SIGINT received, aborting ...')
@@ -55,10 +59,28 @@ def main(sysargv: List[str]) -> None:
         logger.exception('Fatal exception!')
     finally:
         if freqtrade:
-            freqtrade.clean()
+            freqtrade.rpc.send_msg('*Status:* `Process died ...`')
+            freqtrade.cleanup()
         sys.exit(return_code)
 
 
+def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot:
+    """
+    Cleans up current instance, reloads the configuration and returns the new instance
+    """
+    # Clean up current modules
+    freqtrade.cleanup()
+
+    # Create new instance
+    freqtrade = FreqtradeBot(Configuration(args).get_config())
+    freqtrade.rpc.send_msg(
+        '*Status:* `Config reloaded ...`'.format(
+            freqtrade.state.name.lower()
+        )
+    )
+    return freqtrade
+
+
 def set_loggers() -> None:
     """
     Set the logger level for Third party libs

freqtrade/optimize/backtesting.py

@@ -237,6 +237,9 @@ class Backtesting(object):
                 timerange=timerange
             )
 
+        if not data:
+            logger.critical("No data found. Terminating.")
+            return
         # Ignore max_open_trades in backtesting, except realistic flag was passed
         if self.config.get('realistic_simulation', False):
             max_open_trades = self.config['max_open_trades']

freqtrade/rpc/rpc.py

@@ -302,6 +302,11 @@ class RPC(object):
 
         return True, '*Status:* `already stopped`'
 
+    def rpc_reload_conf(self) -> str:
+        """ Handler for reload_conf. """
+        self.freqtrade.state = State.RELOAD_CONF
+        return '*Status:* `Reloading config ...`'
+
     # FIX: no test for this!!!!
     def rpc_forcesell(self, trade_id) -> Tuple[bool, Any]:
         """

freqtrade/rpc/telegram.py

@@ -93,6 +93,7 @@ class Telegram(RPC):
             CommandHandler('performance', self._performance),
             CommandHandler('daily', self._daily),
             CommandHandler('count', self._count),
+            CommandHandler('reload_conf', self._reload_conf),
             CommandHandler('help', self._help),
             CommandHandler('version', self._version),
         ]
@@ -300,6 +301,18 @@ class Telegram(RPC):
         (error, msg) = self.rpc_stop()
         self.send_msg(msg, bot=bot)
 
+    @authorized_only
+    def _reload_conf(self, bot: Bot, update: Update) -> None:
+        """
+        Handler for /reload_conf.
+        Triggers a config file reload
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+        msg = self.rpc_reload_conf()
+        self.send_msg(msg, bot=bot)
+
     @authorized_only
     def _forcesell(self, bot: Bot, update: Update) -> None:
         """

freqtrade/state.py

@@ -8,7 +8,8 @@ import enum
 
 class State(enum.Enum):
     """
-    Bot running states
+    Bot application states
     """
     RUNNING = 0
     STOPPED = 1
+    RELOAD_CONF = 2

freqtrade/tests/optimize/test_backtesting.py

@@ -418,6 +418,40 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
         assert log_has(line, caplog.record_tuples)
 
 
+def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
+    """
+    Test Backtesting.start() method if no data is found
+    """
+
+    def get_timeframe(input1, input2):
+        return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
+
+    mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock())
+    mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={}))
+    mocker.patch('freqtrade.exchange.get_ticker_history')
+    mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True))
+    mocker.patch.multiple(
+        'freqtrade.optimize.backtesting.Backtesting',
+        backtest=MagicMock(),
+        _generate_text_table=MagicMock(return_value='1'),
+        get_timeframe=get_timeframe,
+    )
+
+    conf = deepcopy(default_conf)
+    conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
+    conf['ticker_interval'] = "1m"
+    conf['live'] = False
+    conf['datadir'] = None
+    conf['export'] = None
+    conf['timerange'] = '20180101-20180102'
+
+    backtesting = Backtesting(conf)
+    backtesting.start()
+    # check the logs, that will contain the backtest result
+
+    assert log_has('No data found. Terminating.', caplog.record_tuples)
+
+
 def test_backtest(default_conf, fee, mocker) -> None:
     """
     Test Backtesting.backtest() method

freqtrade/tests/rpc/test_rpc_telegram.py

@@ -70,12 +70,12 @@ def test_init(default_conf, mocker, caplog) -> None:
     assert start_polling.call_count == 0
 
     # number of handles registered
-    assert start_polling.dispatcher.add_handler.call_count == 11
+    assert start_polling.dispatcher.add_handler.call_count > 0
     assert start_polling.start_polling.call_count == 1
 
     message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \
                   "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \
-                  "['count'], ['help'], ['version']]"
+                  "['count'], ['reload_conf'], ['help'], ['version']]"
 
     assert log_has(message_str, caplog.record_tuples)
 
@@ -745,6 +745,29 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None:
     assert 'already stopped' in msg_mock.call_args_list[0][0][0]
 
 
+def test_reload_conf_handle(default_conf, update, mocker) -> None:
+    """ Test _reload_conf() method """
+    patch_coinmarketcap(mocker)
+    mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock())
+    msg_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.rpc.telegram.Telegram',
+        _init=MagicMock(),
+        send_msg=msg_mock
+    )
+    mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
+
+    freqtradebot = FreqtradeBot(default_conf)
+    telegram = Telegram(freqtradebot)
+
+    freqtradebot.state = State.RUNNING
+    assert freqtradebot.state == State.RUNNING
+    telegram._reload_conf(bot=MagicMock(), update=update)
+    assert freqtradebot.state == State.RELOAD_CONF
+    assert msg_mock.call_count == 1
+    assert 'Reloading config' in msg_mock.call_args_list[0][0][0]
+
+
 def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, mocker) -> None:
     """
     Test _forcesell() method

freqtrade/tests/test_freqtradebot.py

@@ -68,7 +68,7 @@ def test_freqtradebot_object() -> None:
     Test the FreqtradeBot object has the mandatory public methods
     """
     assert hasattr(FreqtradeBot, 'worker')
-    assert hasattr(FreqtradeBot, 'clean')
+    assert hasattr(FreqtradeBot, 'cleanup')
     assert hasattr(FreqtradeBot, 'create_trade')
     assert hasattr(FreqtradeBot, 'get_target_bid')
     assert hasattr(FreqtradeBot, 'process_maybe_execute_buy')
@@ -93,7 +93,7 @@ def test_freqtradebot(mocker, default_conf) -> None:
     assert freqtrade.state is State.STOPPED
 
 
-def test_clean(mocker, default_conf, caplog) -> None:
+def test_cleanup(mocker, default_conf, caplog) -> None:
     """
     Test clean() method
     """
@@ -101,11 +101,8 @@ def test_clean(mocker, default_conf, caplog) -> None:
     mocker.patch('freqtrade.persistence.cleanup', mock_cleanup)
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    assert freqtrade.state == State.RUNNING
-
-    assert freqtrade.clean()
-    assert freqtrade.state == State.STOPPED
-    assert log_has('Stopping trader and cleaning up modules...', caplog.record_tuples)
+    freqtrade.cleanup()
+    assert log_has('Cleaning up modules ...', caplog.record_tuples)
     assert mock_cleanup.call_count == 1
 
 

freqtrade/tests/test_main.py

@@ -3,12 +3,16 @@ Unit test file for main.py
 """
 
 import logging
+from copy import deepcopy
 from unittest.mock import MagicMock
 
 import pytest
 
 from freqtrade import OperationalException
-from freqtrade.main import main, set_loggers
+from freqtrade.arguments import Arguments
+from freqtrade.freqtradebot import FreqtradeBot
+from freqtrade.main import main, set_loggers, reconfigure
+from freqtrade.state import State
 from freqtrade.tests.conftest import log_has
 
 
@@ -70,7 +74,7 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None:
         'freqtrade.freqtradebot.FreqtradeBot',
         _init_modules=MagicMock(),
         worker=MagicMock(side_effect=Exception),
-        clean=MagicMock(),
+        cleanup=MagicMock(),
     )
     mocker.patch(
         'freqtrade.configuration.Configuration._load_config_file',
@@ -97,7 +101,7 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None:
         'freqtrade.freqtradebot.FreqtradeBot',
         _init_modules=MagicMock(),
         worker=MagicMock(side_effect=KeyboardInterrupt),
-        clean=MagicMock(),
+        cleanup=MagicMock(),
     )
     mocker.patch(
         'freqtrade.configuration.Configuration._load_config_file',
@@ -124,7 +128,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
         'freqtrade.freqtradebot.FreqtradeBot',
         _init_modules=MagicMock(),
         worker=MagicMock(side_effect=OperationalException('Oh snap!')),
-        clean=MagicMock(),
+        cleanup=MagicMock(),
     )
     mocker.patch(
         'freqtrade.configuration.Configuration._load_config_file',
@@ -140,3 +144,69 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
         main(args)
     assert log_has('Using config: config.json.example ...', caplog.record_tuples)
     assert log_has('Oh snap!', caplog.record_tuples)
+
+
+def test_main_reload_conf(mocker, default_conf, caplog) -> None:
+    """
+    Test main() function
+    In this test we are skipping the while True loop by throwing an exception.
+    """
+    mocker.patch.multiple(
+        'freqtrade.freqtradebot.FreqtradeBot',
+        _init_modules=MagicMock(),
+        worker=MagicMock(return_value=State.RELOAD_CONF),
+        cleanup=MagicMock(),
+    )
+    mocker.patch(
+        'freqtrade.configuration.Configuration._load_config_file',
+        lambda *args, **kwargs: default_conf
+    )
+    mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
+
+    # Raise exception as side effect to avoid endless loop
+    reconfigure_mock = mocker.patch(
+        'freqtrade.main.reconfigure', MagicMock(side_effect=Exception)
+    )
+
+    with pytest.raises(SystemExit):
+        main(['-c', 'config.json.example'])
+
+    assert reconfigure_mock.call_count == 1
+    assert log_has('Using config: config.json.example ...', caplog.record_tuples)
+
+
+def test_reconfigure(mocker, default_conf) -> None:
+    """ Test recreate() function """
+    mocker.patch.multiple(
+        'freqtrade.freqtradebot.FreqtradeBot',
+        _init_modules=MagicMock(),
+        worker=MagicMock(side_effect=OperationalException('Oh snap!')),
+        cleanup=MagicMock(),
+    )
+    mocker.patch(
+        'freqtrade.configuration.Configuration._load_config_file',
+        lambda *args, **kwargs: default_conf
+    )
+    mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
+
+    freqtrade = FreqtradeBot(default_conf)
+
+    # Renew mock to return modified data
+    conf = deepcopy(default_conf)
+    conf['stake_amount'] += 1
+    mocker.patch(
+        'freqtrade.configuration.Configuration._load_config_file',
+        lambda *args, **kwargs: conf
+    )
+
+    # reconfigure should return a new instance
+    freqtrade2 = reconfigure(
+        freqtrade,
+        Arguments(['-c', 'config.json.example'], '').get_parsed_arg()
+    )
+
+    # Verify we have a new instance with the new config
+    assert freqtrade is not freqtrade2
+    assert freqtrade.config['stake_amount'] + 1 == freqtrade2.config['stake_amount']

requirements.txt

@@ -1,4 +1,4 @@
-ccxt==1.14.169
+ccxt==1.14.177
 SQLAlchemy==1.2.8
 python-telegram-bot==10.1.0
 arrow==0.12.1

scripts/download_backtest_data.py

@@ -30,6 +30,8 @@ if not os.path.isfile(pairs_file):
 with open(pairs_file) as file:
     PAIRS = list(set(json.load(file)))
 
+PAIRS.sort()
+
 since_time = None
 if args.days:
     since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000
@@ -40,8 +42,15 @@ print(f'About to download pairs: {PAIRS} to {dl_path}')
 exchange._API = exchange.init_ccxt({'key': '',
                                     'secret': '',
                                     'name': args.exchange})
+pairs_not_available = []
+# Make sure API markets is initialized
+exchange._API.load_markets()
 
 for pair in PAIRS:
+    if pair not in exchange._API.markets:
+        pairs_not_available.append(pair)
+        print(f"skipping pair {pair}")
+        continue
     for tick_interval in timeframes:
         print(f'downloading pair {pair}, interval {tick_interval}')
 
@@ -58,3 +67,7 @@ for pair in PAIRS:
         pair_print = pair.replace('/', '_')
         filename = f'{pair_print}-{tick_interval}.json'
         misc.file_dump_json(os.path.join(dl_path, filename), data)
+
+
+if pairs_not_available:
+    print(f"Pairs [{','.join(pairs_not_available)}] not availble.")