Merge branch 'develop' into test_backtest

config.json.example

@@ -34,8 +34,8 @@
     },
     "telegram": {
         "enabled": true,
-        "token": "your_instagram_token",
+        "token": "your_telegram_token",
-        "chat_id": "your_instagram_chat_id"
+        "chat_id": "your_telegram_chat_id"
     },
     "initial_state": "running",
     "internals": {

config_full.json.example

@@ -42,8 +42,8 @@
     },
     "telegram": {
         "enabled": true,
-        "token": "your_instagram_token",
+        "token": "your_telegram_token",
-        "chat_id": "your_instagram_chat_id"
+        "chat_id": "your_telegram_chat_id"
     },
     "initial_state": "running",
     "internals": {

freqtrade/main.py

@@ -301,12 +301,13 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) -
         return True
 
     # Check if time matches and current rate is above threshold
-    time_diff = (current_time - trade.open_date).total_seconds() / 60
+    time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60
-    for duration, threshold in sorted(strategy.minimal_roi.items()):
+    for duration_string, threshold in strategy.minimal_roi.items():
-        if time_diff > float(duration) and current_profit > threshold:
+        duration = float(duration_string)
+        if time_diff > duration and current_profit > threshold:
             return True
-
+        if time_diff < duration:
-    logger.debug('Threshold not reached. (cur_profit: %1.2f%%)', float(current_profit) * 100.0)
+            return False
     return False
 
 

freqtrade/misc.py

@@ -145,15 +145,15 @@ def common_args_parser(description: str):
     )
     parser.add_argument(
         '-c', '--config',
-        help='specify configuration file (default: config.json)',
+        help='specify configuration file (default: %(default)s)',
         dest='config',
         default='config.json',
         type=str,
         metavar='PATH',
     )
     parser.add_argument(
-        '--datadir',
+        '-d', '--datadir',
-        help='path to backtest data (default freqdata/tests/testdata)',
+        help='path to backtest data (default: %(default)s',
         dest='datadir',
         default=os.path.join('freqtrade', 'tests', 'testdata'),
         type=str,
@@ -161,7 +161,7 @@ def common_args_parser(description: str):
     )
     parser.add_argument(
         '-s', '--strategy',
-        help='specify strategy file (default: freqtrade/strategy/default_strategy.py)',
+        help='specify strategy file (default: %(default)s)',
         dest='strategy',
         default='default_strategy',
         type=str,
@@ -254,7 +254,7 @@ def backtesting_options(parser: argparse.ArgumentParser) -> None:
 def hyperopt_options(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         '-e', '--epochs',
-        help='specify number of epochs (default: 100)',
+        help='specify number of epochs (default: %(default)d)',
         dest='epochs',
         default=100,
         type=int,

freqtrade/optimize/backtesting.py

@@ -33,7 +33,7 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
 
 
 def generate_text_table(
-        data: Dict[str, Dict], results: DataFrame, stake_currency, ticker_interval) -> str:
+        data: Dict[str, Dict], results: DataFrame, stake_currency) -> str:
     """
     Generates and returns a text table for the given backtest data and the results dataframe
     :return: pretty printed table with tabulate as str
@@ -49,7 +49,7 @@ def generate_text_table(
             len(result.index),
             result.profit_percent.mean() * 100.0,
             result.profit_BTC.sum(),
-            result.duration.mean() * ticker_interval,
+            result.duration.mean(),
             len(result[result.profit_BTC > 0]),
             len(result[result.profit_BTC < 0])
         ])
@@ -60,7 +60,7 @@ def generate_text_table(
         len(results.index),
         results.profit_percent.mean() * 100.0,
         results.profit_BTC.sum(),
-        results.duration.mean() * ticker_interval,
+        results.duration.mean(),
         len(results[results.profit_BTC > 0]),
         len(results[results.profit_BTC < 0])
     ])
@@ -71,28 +71,26 @@ def get_sell_trade_entry(pair, row, buy_subset, ticker, trade_count_lock, args):
     stake_amount = args['stake_amount']
     max_open_trades = args.get('max_open_trades', 0)
     trade = Trade(open_rate=row.close,
-                  open_date=row.date,
+                  open_date=row.Index,
                   stake_amount=stake_amount,
                   amount=stake_amount / row.open,
                   fee=exchange.get_fee()
                   )
 
     # calculate win/lose forwards from buy point
-    sell_subset = ticker[ticker.date > row.date][['close', 'date', 'sell']]
+    sell_subset = ticker[ticker.index > row.Index][['close', 'sell', 'buy']]
     for row2 in sell_subset.itertuples(index=True):
         if max_open_trades > 0:
             # Increase trade_count_lock for every iteration
-            trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1
+            trade_count_lock[row2.Index] = trade_count_lock.get(row2.Index, 0) + 1
 
-        # Buy is on is in the buy_subset there is a row that matches the date
+        buy_signal = row2.buy
-        # of the sell event
+        if(should_sell(trade, row2.close, row2.Index, buy_signal, row2.sell)):
-        buy_signal = not buy_subset[buy_subset.date == row2.date].empty
-        if(should_sell(trade, row2.close, row2.date, buy_signal, row2.sell)):
             return row2, (pair,
                           trade.calc_profit_percent(rate=row2.close),
                           trade.calc_profit(rate=row2.close),
-                          row2.Index - row.Index
+                          (row2.Index - row.Index).seconds // 60
-                          ), row2.date
+                          ), row2.Index
     return None
 
 
@@ -120,22 +118,24 @@ def backtest(args) -> DataFrame:
     for pair, pair_data in processed.items():
         pair_data['buy'], pair_data['sell'] = 0, 0
         ticker = populate_sell_trend(populate_buy_trend(pair_data))
+        if 'date' in ticker:
+            ticker.set_index('date', inplace=True)
         # for each buy point
         lock_pair_until = None
-        headers = ['buy', 'open', 'close', 'date', 'sell']
+        headers = ['buy', 'open', 'close', 'sell']
         buy_subset = ticker[(ticker.buy == 1) & (ticker.sell == 0)][headers]
         for row in buy_subset.itertuples(index=True):
             if realistic:
-                if lock_pair_until is not None and row.date <= lock_pair_until:
+                if lock_pair_until is not None and row.Index <= lock_pair_until:
                     continue
             if max_open_trades > 0:
                 # Check if max_open_trades has already been reached for the given date
-                if not trade_count_lock.get(row.date, 0) < max_open_trades:
+                if not trade_count_lock.get(row.Index, 0) < max_open_trades:
                     continue
 
             if max_open_trades > 0:
                 # Increase lock
-                trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
+                trade_count_lock[row.Index] = trade_count_lock.get(row.Index, 0) + 1
 
             ret = get_sell_trade_entry(pair, row, buy_subset, ticker,
                                        trade_count_lock, args)
@@ -148,8 +148,8 @@ def backtest(args) -> DataFrame:
                     # record a tuple of pair, current_profit_percent,
                     # entry-date, duration
                     records.append((pair, trade_entry[1],
-                                    row.date.strftime('%s'),
+                                    row.Index.strftime('%s'),
-                                    row2.date.strftime('%s'),
+                                    row2.Index.strftime('%s'),
                                     row.Index, trade_entry[3]))
     # For now export inside backtest(), maybe change so that backtest()
     # returns a tuple like: (dataframe, records, logs, etc)
@@ -232,5 +232,5 @@ def start(args):
                         })
     logger.info(
         '\n==================================== BACKTESTING REPORT ====================================\n%s',  # noqa
-        generate_text_table(data, results, config['stake_currency'], strategy.ticker_interval)
+        generate_text_table(data, results, config['stake_currency'])
     )

freqtrade/optimize/hyperopt.py

@@ -406,7 +406,7 @@ def optimizer(params):
 
     total_profit = results.profit_percent.sum()
     trade_count = len(results.index)
-    trade_duration = results.duration.mean() * 5
+    trade_duration = results.duration.mean()
 
     if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION:
         print('.', end='')

freqtrade/persistence.py

@@ -187,8 +187,8 @@ class Trade(_DECL_BASE):
         """
         open_trade_price = self.calc_open_trade_price()
         close_trade_price = self.calc_close_trade_price(
-            rate=Decimal(rate or self.close_rate),
+            rate=(rate or self.close_rate),
-            fee=Decimal(fee or self.fee)
+            fee=(fee or self.fee)
         )
         return float("{0:.8f}".format(close_trade_price - open_trade_price))
 
@@ -206,8 +206,8 @@ class Trade(_DECL_BASE):
 
         open_trade_price = self.calc_open_trade_price()
         close_trade_price = self.calc_close_trade_price(
-            rate=Decimal(rate or self.close_rate),
+            rate=(rate or self.close_rate),
-            fee=Decimal(fee or self.fee)
+            fee=(fee or self.fee)
         )
 
         return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1))

freqtrade/strategy/strategy.py

@@ -7,6 +7,7 @@ import os
 import sys
 import logging
 import importlib
+from collections import OrderedDict
 
 from pandas import DataFrame
 from freqtrade.strategy.interface import IStrategy
@@ -69,7 +70,9 @@ class Strategy(object):
             )
 
         # Minimal ROI designed for the strategy
-        self.minimal_roi = self.custom_strategy.minimal_roi
+        self.minimal_roi = OrderedDict(sorted(
+            self.custom_strategy.minimal_roi.items(),
+            key=lambda tuple: float(tuple[0])))  # sort after converting to number
 
         # Optimal stoploss designed for the strategy
         self.stoploss = self.custom_strategy.stoploss

freqtrade/tests/optimize/test_backtesting.py

@@ -94,12 +94,12 @@ def test_generate_text_table():
             'loss': [0, 0]
         }
     )
-    print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5))
+    print(generate_text_table({'BTC_ETH': {}}, results, 'BTC'))
-    assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == (
+    assert generate_text_table({'BTC_ETH': {}}, results, 'BTC') == (
         'pair       buy count    avg profit %    total profit BTC    avg duration    profit    loss\n'  # noqa
         '-------  -----------  --------------  ------------------  --------------  --------  ------\n'  # noqa
-        'BTC_ETH            2           15.00          0.60000000           100.0         2       0\n'  # noqa
+        'BTC_ETH            2           15.00          0.60000000            20.0         2       0\n'  # noqa
-        'TOTAL              2           15.00          0.60000000           100.0         2       0')  # noqa
+        'TOTAL              2           15.00          0.60000000            20.0         2       0')  # noqa
 
 
 def test_get_timeframe(default_strategy):
@@ -262,14 +262,16 @@ def test_backtest_record(default_conf, mocker, default_strategy):
     assert len(records) == 3
     # ('BTC_UNITEST', 0.00331158, '1510684320', '1510691700', 0, 117)
     # Below follows just a typecheck of the schema/type of trade-records
-    oix = -1
+    oix = None
     for (pair, profit, date_buy, date_sell, buy_index, dur) in records:
         assert pair == 'BTC_UNITEST'
         isinstance(profit, float)
         # FIX: buy/sell should be converted to ints
         isinstance(date_buy, str)
         isinstance(date_sell, str)
-        assert buy_index > oix
+        isinstance(buy_index, pd._libs.tslib.Timestamp)
+        if oix:
+            assert buy_index > oix
         oix = buy_index
         assert dur > 0
 

freqtrade/tests/test_persistence.py

@@ -271,10 +271,6 @@ def test_calc_profit(limit_buy_order, limit_sell_order):
     # Lower than open rate
     assert trade.calc_profit(rate=0.00000123, fee=0.003) == -0.00089092
 
-    # Only custom fee without sell order applied
-    with pytest.raises(TypeError):
-        trade.calc_profit(fee=0.003)
-
     # Test when we apply a Sell order. Sell higher than open rate @ 0.00001173
     trade.update(limit_sell_order)
     assert trade.calc_profit() == 0.00006217
@@ -299,10 +295,6 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order):
     # Get percent of profit with a custom rate (Lower than open rate)
     assert trade.calc_profit_percent(rate=0.00000123) == -0.88863827
 
-    # Only custom fee without sell order applied
-    with pytest.raises(TypeError):
-        trade.calc_profit_percent(fee=0.003)
-
     # Test when we apply a Sell order. Sell higher than open rate @ 0.00001173
     trade.update(limit_sell_order)
     assert trade.calc_profit_percent() == 0.06201057

requirements.txt

@@ -19,7 +19,7 @@ hyperopt==0.1
 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325
 networkx==1.11
 tabulate==0.8.2
-pymarketcap==3.3.154
+pymarketcap==3.3.158
 
 # Required for plotting data
 #plotly==2.3.0

scripts/plot_dataframe.py

@@ -2,24 +2,15 @@
 
 import sys
 import logging
-import argparse
-import os
 
-from pandas import DataFrame
-import talib.abstract as ta
-
-import plotly
 from plotly import tools
 from plotly.offline import plot
 import plotly.graph_objs as go
 
-import freqtrade.vendor.qtpylib.indicators as qtpylib
 from freqtrade import exchange, analyze
-from freqtrade.misc import common_args_parser
 from freqtrade.strategy.strategy import Strategy
 import freqtrade.misc as misc
 import freqtrade.optimize as optimize
-import freqtrade.analyze as analyze
 
 
 logger = logging.getLogger(__name__)
@@ -61,18 +52,17 @@ def plot_analyzed_dataframe(args) -> None:
     dataframe = dataframes[pair]
     dataframe = analyze.populate_buy_trend(dataframe)
     dataframe = analyze.populate_sell_trend(dataframe)
-    dates = misc.datesarray_to_datetimearray(dataframe['date'])
 
     if (len(dataframe.index) > 750):
         logger.warn('Ticker contained more than 750 candles, clipping.')
     df = dataframe.tail(750)
 
     candles = go.Candlestick(x=df.date,
-                        open=df.open,
+                             open=df.open,
-                        high=df.high,
+                             high=df.high,
-                        low=df.low,
+                             low=df.low,
-                        close=df.close,
+                             close=df.close,
-                        name='Price')
+                             name='Price')
 
     df_buy = df[df['buy'] == 1]
     buys = go.Scattergl(
@@ -80,7 +70,14 @@ def plot_analyzed_dataframe(args) -> None:
         y=df_buy.close,
         mode='markers',
         name='buy',
-        marker=dict(symbol='x-dot')
+        marker=dict(
+            symbol='triangle-up-dot',
+            size=9,
+            line=dict(
+                width=1,
+            ),
+            color='green',
+        )
     )
     df_sell = df[df['sell'] == 1]
     sells = go.Scattergl(
@@ -88,7 +85,14 @@ def plot_analyzed_dataframe(args) -> None:
         y=df_sell.close,
         mode='markers',
         name='sell',
-        marker=dict(symbol='diamond')
+        marker=dict(
+            symbol='triangle-down-dot',
+            size=9,
+            line=dict(
+                width=1,
+            ),
+            color='red',
+        )
     )
 
     bb_lower = go.Scatter(
@@ -102,28 +106,20 @@ def plot_analyzed_dataframe(args) -> None:
         y=df.bb_upperband,
         name='BB upper',
         fill="tonexty",
-        fillcolor="rgba(0,176,246,0.2)", 
+        fillcolor="rgba(0,176,246,0.2)",
         line={'color': "transparent"},
     )
+    macd = go.Scattergl(x=df['date'], y=df['macd'], name='MACD')
+    macdsignal = go.Scattergl(x=df['date'], y=df['macdsignal'], name='MACD signal')
+    volume = go.Bar(x=df['date'], y=df['volume'], name='Volume')
 
-    macd = go.Scattergl(
+    fig = tools.make_subplots(
-        x=df['date'],
+        rows=3,
-        y=df['macd'],
+        cols=1,
-        name='MACD'
+        shared_xaxes=True,
+        row_width=[1, 1, 4],
+        vertical_spacing=0.0001,
     )
-    macdsignal = go.Scattergl(
-        x=df['date'],
-        y=df['macdsignal'],
-        name='MACD signal'
-    )
-
-    volume = go.Bar(
-        x=df['date'],
-        y=df['volume'],
-        name='Volume'
-    )
-
-    fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 4])
 
     fig.append_trace(candles, 1, 1)
     fig.append_trace(bb_lower, 1, 1)

scripts/plot_profit.py

@@ -4,14 +4,12 @@ import sys
 import json
 import numpy as np
 
-import plotly
 from plotly import tools
 from plotly.offline import plot
 import plotly.graph_objs as go
 
 import freqtrade.optimize as optimize
 import freqtrade.misc as misc
-import freqtrade.exchange as exchange
 from freqtrade.strategy.strategy import Strategy
 
 

setup.sh

@@ -116,8 +116,8 @@ function config_generator () {
         -e "s/\"fiat_display_currency\": \"USD\",/\"fiat_display_currency\": \"$fiat_currency\",/g" \
         -e "s/\"your_exchange_key\"/\"$api_key\"/g" \
         -e "s/\"your_exchange_secret\"/\"$api_secret\"/g" \
-        -e "s/\"your_instagram_token\"/\"$token\"/g" \
+        -e "s/\"your_telegram_token\"/\"$token\"/g" \
-        -e "s/\"your_instagram_chat_id\"/\"$chat_id\"/g"
+        -e "s/\"your_telegram_chat_id\"/\"$chat_id\"/g"
         -e "s/\"dry_run\": false,/\"dry_run\": true,/g" config.json.example > config.json
 
 }