Merge branch 'develop' into mypy_typecheck

docs/backtesting.md

@@ -83,6 +83,8 @@ The full timerange specification:
 - Use tickframes till 2018/01/31: `--timerange=-20180131`
 - Use tickframes since 2018/01/31: `--timerange=20180131-`
 - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301`
+- Use tickframes between POSIX timestamps 1527595200 1527618600:
+                                                `--timerange=1527595200-1527618600`
 
 
 **Update testdata directory**

freqtrade/arguments.py

@@ -223,6 +223,9 @@ class Arguments(object):
         syntax = [(r'^-(\d{8})$', (None, 'date')),
                   (r'^(\d{8})-$', ('date', None)),
                   (r'^(\d{8})-(\d{8})$', ('date', 'date')),
+                  (r'^-(\d{10})$', (None, 'date')),
+                  (r'^(\d{10})-$', ('date', None)),
+                  (r'^(\d{10})-(\d{10})$', ('date', 'date')),
                   (r'^(-\d+)$', (None, 'line')),
                   (r'^(\d+)-$', ('line', None)),
                   (r'^(\d+)-(\d+)$', ('index', 'index'))]
@@ -237,14 +240,16 @@ class Arguments(object):
                 if stype[0]:
                     starts = rvals[index]
                     if stype[0] == 'date':
-                        start = arrow.get(starts, 'YYYYMMDD').timestamp
+                        start = int(starts) if len(starts) == 10 \
+                            else arrow.get(starts, 'YYYYMMDD').timestamp
                     else:
                         start = int(starts)
                     index += 1
                 if stype[1]:
                     stops = rvals[index]
                     if stype[1] == 'date':
-                        stop = arrow.get(stops, 'YYYYMMDD').timestamp
+                        stop = int(stops) if len(stops) == 10 \
+                            else arrow.get(stops, 'YYYYMMDD').timestamp
                     else:
                         stop = int(stops)
                 return stype, start, stop

freqtrade/configuration.py

@@ -145,7 +145,7 @@ class Configuration(object):
         # If --datadir is used we add it to the configuration
         if 'datadir' in self.args and self.args.datadir:
             config.update({'datadir': self.args.datadir})
-            logger.info('Parameter --datadir detected: %s ...', self.args.datadir)
+            logger.info('Using data folder: %s ...', self.args.datadir)
 
         # If -r/--refresh-pairs-cached is used we add it to the configuration
         if 'refresh_pairs' in self.args and self.args.refresh_pairs:

freqtrade/optimize/__init__.py

@@ -103,7 +103,12 @@ def load_data(datadir: str,
         if pairdata:
             result[pair] = pairdata
         else:
-            logger.warn('No data for pair %s, use --update-pairs-cached to download the data', pair)
+            logger.warning(
+                'No data for pair: "%s", Interval: %s. '
+                'Use --refresh-pairs-cached to download the data',
+                pair,
+                ticker_interval
+            )
 
     return result
 

freqtrade/optimize/backtesting.py

@@ -94,7 +94,7 @@ class Backtesting(object):
             len(results[results.profit_BTC > 0]),
             len(results[results.profit_BTC < 0])
         ])
-        return tabulate(tabular_data, headers=headers, floatfmt=floatfmt)
+        return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
 
     def _get_sell_trade_entry(
             self, pair: str, buy_row: DataFrame,

freqtrade/optimize/hyperopt.py

@@ -455,6 +455,7 @@ class Hyperopt(Backtesting):
 
         if trade_count == 0 or trade_duration > self.max_accepted_trade_duration:
             print('.', end='')
+            sys.stdout.flush()
             return {
                 'status': STATUS_FAIL,
                 'loss': float('inf')
@@ -479,16 +480,16 @@ class Hyperopt(Backtesting):
             'result': result_explanation,
         }
 
-    @staticmethod
+    def format_results(self, results: DataFrame) -> str:
-    def format_results(results: DataFrame) -> str:
         """
         Return the format result in a string
         """
         return ('{:6d} trades. Avg profit {: 5.2f}%. '
-                'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format(
+                'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format(
                     len(results.index),
                     results.profit_percent.mean() * 100.0,
                     results.profit_BTC.sum(),
+                    self.config['stake_currency'],
                     results.profit_percent.sum(),
                     results.duration.mean(),
                 )

freqtrade/rpc/rpc.py

@@ -316,8 +316,10 @@ class RPC(object):
                         and order['side'] == 'buy':
                     exchange.cancel_order(trade.open_order_id, trade.pair)
                     trade.close(order.get('price') or trade.open_rate)
-                    # TODO: sell amount which has been bought already
+                    # Do the best effort, if we don't know 'filled' amount, don't try selling
-                    return
+                    if order['filled'] is None:
+                        return
+                    trade.amount = order['filled']
 
                 # Ignore trades with an attached LIMIT_SELL order
                 if order and order['status'] == 'open' \

freqtrade/tests/optimize/test_backtesting.py

@@ -181,7 +181,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
     assert log_has(
-        'Parameter --datadir detected: {} ...'.format(config['datadir']),
+        'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples
     )
     assert 'ticker_interval' in config
@@ -229,7 +229,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
     assert log_has(
-        'Parameter --datadir detected: {} ...'.format(config['datadir']),
+        'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples
     )
     assert 'ticker_interval' in config
@@ -357,16 +357,15 @@ def test_generate_text_table(default_conf, mocker):
     )
 
     result_str = (
-        'pair       buy count    avg profit %    '
+        '| pair    |   buy count |   avg profit % |   '
-        'total profit BTC    avg duration    profit    loss\n'
+        'total profit BTC |   avg duration |   profit |   loss |\n'
-        '-------  -----------  --------------  '
+        '|:--------|------------:|---------------:|'
-        '------------------  --------------  --------  ------\n'
+        '-------------------:|---------------:|---------:|-------:|\n'
-        'ETH/BTC            2           15.00          '
+        '| ETH/BTC |           2 |          15.00 |         '
-        '0.60000000            20.0         2       0\n'
+        '0.60000000 |           20.0 |        2 |      0 |\n'
-        'TOTAL              2           15.00          '
+        '| TOTAL   |           2 |          15.00 |         '
-        '0.60000000            20.0         2       0'
+        '0.60000000 |           20.0 |        2 |      0 |'
     )
-
     assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str
 
 
@@ -615,7 +614,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
         'Parameter -l/--live detected ...',
         'Using max_open_trades: 1 ...',
         'Parameter --timerange detected: -100 ..',
-        'Parameter --datadir detected: freqtrade/tests/testdata ...',
+        'Using data folder: freqtrade/tests/testdata ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
         'Downloading data for all pairs in whitelist ...',

freqtrade/tests/optimize/test_hyperopt.py

@@ -389,10 +389,12 @@ def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None:
 # test buy_strategy_generator def populate_buy_trend
 # test optimizer if 'ro_t1' in params
 
-def test_format_results():
+def test_format_results(init_hyperopt):
     """
     Test Hyperopt.format_results()
     """
+
+    # Test with BTC as stake_currency
     trades = [
         ('ETH/BTC', 2, 2, 123),
         ('LTC/BTC', 1, 1, 123),
@@ -400,8 +402,21 @@ def test_format_results():
     ]
     labels = ['currency', 'profit_percent', 'profit_BTC', 'duration']
     df = pd.DataFrame.from_records(trades, columns=labels)
-    x = Hyperopt.format_results(df)
+
-    assert x.find(' 66.67%')
+    result = _HYPEROPT.format_results(df)
+    assert result.find(' 66.67%')
+    assert result.find('Total profit 1.00000000 BTC')
+    assert result.find('2.0000Σ %')
+
+    # Test with EUR as stake_currency
+    trades = [
+        ('ETH/EUR', 2, 2, 123),
+        ('LTC/EUR', 1, 1, 123),
+        ('XPR/EUR', -1, -2, -246)
+    ]
+    df = pd.DataFrame.from_records(trades, columns=labels)
+    result = _HYPEROPT.format_results(df)
+    assert result.find('Total profit 1.00000000 EUR')
 
 
 def test_signal_handler(mocker, init_hyperopt):

freqtrade/tests/optimize/test_optimize.py

@@ -105,7 +105,8 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None:
                        refresh_pairs=False,
                        pairs=['MEME/BTC'])
     assert os.path.isfile(file) is False
-    assert log_has('No data for pair MEME/BTC, use --update-pairs-cached to download the data',
+    assert log_has('No data for pair: "MEME/BTC", Interval: 1m. '
+                   'Use --refresh-pairs-cached to download the data',
                    caplog.record_tuples)
 
     # download a new pair if refresh_pairs is set

freqtrade/tests/rpc/test_rpc.py

@@ -449,20 +449,44 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     freqtradebot.state = State.RUNNING
     assert cancel_order_mock.call_count == 0
     # make an limit-buy open trade
+    trade = Trade.query.filter(Trade.id == '1').first()
+    filled_amount = trade.amount / 2
     mocker.patch(
         'freqtrade.freqtradebot.exchange.get_order',
         return_value={
             'status': 'open',
             'type': 'limit',
-            'side': 'buy'
+            'side': 'buy',
+            'filled': filled_amount
         }
     )
-    # check that the trade is called, which is done
+    # check that the trade is called, which is done by ensuring exchange.cancel_order is called
-    # by ensuring exchange.cancel_order is called
+    # and trade amount is updated
     (error, res) = rpc.rpc_forcesell('1')
     assert not error
     assert res == ''
     assert cancel_order_mock.call_count == 1
+    assert trade.amount == filled_amount
+
+    freqtradebot.create_trade()
+    trade = Trade.query.filter(Trade.id == '2').first()
+    amount = trade.amount
+    # make an limit-buy open trade, if there is no 'filled', don't sell it
+    mocker.patch(
+        'freqtrade.freqtradebot.exchange.get_order',
+        return_value={
+            'status': 'open',
+            'type': 'limit',
+            'side': 'buy',
+            'filled': None
+        }
+    )
+    # check that the trade is called, which is done by ensuring exchange.cancel_order is called
+    (error, res) = rpc.rpc_forcesell('2')
+    assert not error
+    assert res == ''
+    assert cancel_order_mock.call_count == 2
+    assert trade.amount == amount
 
     freqtradebot.create_trade()
     # make an limit-sell open trade
@@ -474,11 +498,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
             'side': 'sell'
         }
     )
-    (error, res) = rpc.rpc_forcesell('2')
+    (error, res) = rpc.rpc_forcesell('3')
     assert not error
     assert res == ''
     # status quo, no exchange calls
-    assert cancel_order_mock.call_count == 1
+    assert cancel_order_mock.call_count == 2
 
 
 def test_performance_handle(default_conf, ticker, limit_buy_order, fee,

freqtrade/tests/test_arguments.py

@@ -116,6 +116,12 @@ def test_parse_timerange_incorrect() -> None:
     timerange = Arguments.parse_timerange('20100522-20150730')
     assert timerange == (('date', 'date'), 1274486400, 1438214400)
 
+    # Added test for unix timestamp - BTC genesis date
+    assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-')
+    assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000')
+    timerange = Arguments.parse_timerange('1231006505-1233360000')
+    assert timerange == (('date', 'date'), 1231006505, 1233360000)
+
     with pytest.raises(Exception, match=r'Incorrect syntax.*'):
         Arguments.parse_timerange('-')
 

freqtrade/tests/test_configuration.py

@@ -236,7 +236,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
     assert log_has(
-        'Parameter --datadir detected: {} ...'.format(config['datadir']),
+        'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples
     )
     assert 'ticker_interval' in config
@@ -287,7 +287,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
     assert log_has(
-        'Parameter --datadir detected: {} ...'.format(config['datadir']),
+        'Using data folder: {} ...'.format(config['datadir']),
         caplog.record_tuples
     )
     assert 'ticker_interval' in config

requirements.txt

@@ -1,4 +1,4 @@
-ccxt==1.14.73
+ccxt==1.14.120
 SQLAlchemy==1.2.8
 python-telegram-bot==10.1.0
 arrow==0.12.1

setup.sh

@@ -2,16 +2,17 @@
 #encoding=utf8
 
 function updateenv () {
-    echo "
+    echo "-------------------------"
-    -------------------------
+    echo "Update your virtual env"
-    Update your virtual env
+    echo "-------------------------"
-    -------------------------
-    "
     source .env/bin/activate
-    pip3.6 install --upgrade pip
+    echo "pip3 install in-progress. Please wait..."
-    pip3 install -r requirements.txt --upgrade
+    pip3.6 install --quiet --upgrade pip
-    pip3 install -r requirements.txt
+    pip3 install --quiet -r requirements.txt --upgrade
-    pip3 install -e .
+    pip3 install --quiet -r requirements.txt
+    pip3 install --quiet -e .
+    echo "pip3 install completed"
+    echo
 }
 
 # Install tab lib
@@ -29,10 +30,11 @@ function install_macos () {
         echo "-------------------------"
         echo "Install Brew"
         echo "-------------------------"
-        echo
         /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
     fi
     brew install python3 wget ta-lib
+
+    test_and_fix_python_on_mac
 }
 
 # Install bot Debian_ubuntu
@@ -54,7 +56,6 @@ function reset () {
     echo "----------------------------"
     echo "Reset branch and virtual env"
     echo "----------------------------"
-    echo
     if [ "1" == $(git branch -vv |grep -cE "\* develop|\* master") ]
     then
         if [ -d ".env" ]; then
@@ -77,18 +78,30 @@ function reset () {
         echo "Reset ignored because you are not on 'master' or 'develop'."
     fi
 
+    echo
     python3.6 -m venv .env
     updateenv
 }
 
+function test_and_fix_python_on_mac() {
+
+    if ! [ -x "$(command -v python3.6)" ]
+    then
+        echo "-------------------------"
+        echo "Fixing Python"
+        echo "-------------------------"
+        echo "Python 3.6 is not linked in your system. Fixing it..."
+        brew link --overwrite python
+        echo
+    fi
+}
+
 function config_generator () {
 
     echo "Starting to generate config.json"
-
+    echo
-    echo "-------------------------"
     echo "General configuration"
     echo "-------------------------"
-    echo
     default_max_trades=3
     read -p "Max open trades: (Default: $default_max_trades) " max_trades
     max_trades=${max_trades:-$default_max_trades}
@@ -105,14 +118,13 @@ function config_generator () {
     read -p "Fiat currency: (Default: $default_fiat_currency) " fiat_currency
     fiat_currency=${fiat_currency:-$default_fiat_currency}
 
-    echo "------------------------"
-    echo "Bittrex config generator"
-    echo "------------------------"
     echo
+    echo "Exchange config generator"
+    echo "------------------------"
     read -p "Exchange API key: " api_key
     read -p "Exchange API Secret: " api_secret
 
-    echo "-------------------------"
+    echo
     echo "Telegram config generator"
     echo "-------------------------"
     read -p "Telegram Token: " token
@@ -131,6 +143,10 @@ function config_generator () {
 }
 
 function config () {
+
+    echo "-------------------------"
+    echo "Config file generator"
+    echo "-------------------------"
     if [ -f config.json ]
     then
     read -p "A config file already exist, do you want to override it [Y/N]? "
@@ -144,22 +160,26 @@ function config () {
         config_generator
     fi
 
+    echo
+    echo "-------------------------"
+    echo "Config file generated"
+    echo "-------------------------"
     echo "Edit ./config.json to modify Pair and other configurations."
+    echo
 }
 
 function install () {
     echo "-------------------------"
     echo "Install mandatory dependencies"
     echo "-------------------------"
-    echo
 
     if [ "$(uname -s)" == "Darwin" ]
     then
-        echo "- You are on macOS"
+        echo "macOS detected. Setup for this system in-progress"
         install_macos
     elif [ -x "$(command -v apt-get)" ]
     then
-        echo "- You are on Debian/Ubuntu"
+        echo "Debian/Ubuntu detected. Setup for this system in-progress"
         install_debian
     else
         echo "This script does not support your OS."
@@ -167,12 +187,13 @@ function install () {
         echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell."
         sleep 10
     fi
+    echo
     reset
-    echo "
-    - Install complete.
-    "
     config
-    echo "You can now use the bot by executing 'source .env/bin/activate; python3 freqtrade/main.py'."
+    echo "-------------------------"
+    echo "Run the bot"
+    echo "-------------------------"
+    echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'."
 }
 
 function plot () {