Merge branch 'develop' into feat/objectify-ccxt

freqtrade/optimize/__init__.py

@@ -113,45 +113,38 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: str) -> bool:
 
 
 # FIX: 20180110, suggest rename interval to tick_interval
-def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m') -> bool:
+def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m') -> None:
     """
     Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
-    :param pairs: list of pairs to download
-    :return: bool
     """
 
     path = make_testdata_path(datadir)
     logger.info(
-        'Download the pair: "%s", Interval: %s',
-        pair,
-        interval
+        'Download the pair: "%s", Interval: %s', pair, interval
     )
 
-    filepair = pair.replace("/", "_")
     filename = os.path.join(path, '{pair}-{interval}.json'.format(
-        pair=filepair,
+        pair=pair.replace("/", "_"),
         interval=interval,
     ))
 
     if os.path.isfile(filename):
         with open(filename, "rt") as file:
             data = json.load(file)
-        logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]))
-        logger.debug("Current End: %s", misc.format_ms_time(data[-1:][0][0]))
     else:
         data = []
-        logger.debug("Current Start: None")
-        logger.debug("Current End: None")
 
-    new_data = get_ticker_history(pair=pair, tick_interval=interval)
-    for row in new_data:
-        if row not in data:
-            data.append(row)
-    logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
-    logger.debug("New End: %s", misc.format_ms_time(data[-1:][0][0]))
-    data = sorted(data, key=lambda data: data[0])
+    logger.debug('Current Start: %s', data[0][0] if data else None)
+    logger.debug('Current End: %s', data[-1:][0][0] if data else None)
 
+    # Extend data with new ticker history
+    data.extend([
+        row for row in get_ticker_history(pair=pair, tick_interval=interval)
+        if row not in data
+    ])
+
+    data = sorted(data, key=lambda _data: _data[0])
+    logger.debug('New Start: %s', data[0][0])
+    logger.debug('New End: %s', data[-1:][0][0])
     misc.file_dump_json(filename, data)
-
-    return True

freqtrade/optimize/backtesting.py

@@ -4,11 +4,12 @@
 This module contains the backtesting logic
 """
 import logging
+import operator
 from argparse import Namespace
 from typing import Dict, Tuple, Any, List, Optional
 
 import arrow
-from pandas import DataFrame, Series
+from pandas import DataFrame
 from tabulate import tabulate
 
 import freqtrade.optimize as optimize
@@ -19,7 +20,6 @@ from freqtrade.configuration import Configuration
 from freqtrade.misc import file_dump_json
 from freqtrade.persistence import Trade
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -66,11 +66,12 @@ class Backtesting(object):
         :param data: dictionary with preprocessed backtesting data
         :return: tuple containing min_date, max_date
         """
-        all_dates = Series([])
-        for pair_data in data.values():
-            all_dates = all_dates.append(pair_data['date'])
-        all_dates.sort_values(inplace=True)
-        return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1])
+        timeframe = [
+            (arrow.get(min(frame.date)), arrow.get(max(frame.date)))
+            for frame in data.values()
+        ]
+        return min(timeframe, key=operator.itemgetter(0))[0], \
+            max(timeframe, key=operator.itemgetter(1))[1]
 
     def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str:
         """
@@ -201,9 +202,9 @@ class Backtesting(object):
                         # record a tuple of pair, current_profit_percent,
                         # entry-date, duration
                         records.append((pair, trade_entry[1],
-                                        row.date.timestamp(),
-                                        row2.date.timestamp(),
-                                        row.date, trade_entry[3]))
+                                        row.date.strftime('%s'),
+                                        row2.date.strftime('%s'),
+                                        index, trade_entry[3]))
         # For now export inside backtest(), maybe change so that backtest()
         # returns a tuple like: (dataframe, records, logs, etc)
         if record and record.find('trades') >= 0:
@@ -304,12 +305,9 @@ def start(args: Namespace) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-
-    # Initialize logger
-    logger.info('Starting freqtrade in Backtesting mode')
-
     # Initialize configuration
     config = setup_configuration(args)
+    logger.info('Starting freqtrade in Backtesting mode')
 
     # Initialize backtesting object
     backtesting = Backtesting(config)

freqtrade/optimize/hyperopt.py

@@ -29,7 +29,6 @@ from freqtrade.optimize import load_data
 from freqtrade.optimize.backtesting import Backtesting
 from user_data.hyperopt_conf import hyperopt_optimize_conf
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -591,11 +590,11 @@ def start(args: Namespace) -> None:
     logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING)
     logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
 
-    logger.info('Starting freqtrade in Hyperopt mode')
-
     # Initialize configuration
     # Monkey patch the configuration with hyperopt_conf.py
     configuration = Configuration(args)
+    logger.info('Starting freqtrade in Hyperopt mode')
+
     optimize_config = hyperopt_optimize_conf()
     config = configuration._load_common_config(optimize_config)
     config = configuration._load_backtesting_config(config)