From f398888865c732df55b0ecee3c775aba4d4a9ff4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 21 May 2021 08:26:19 +0200
Subject: [PATCH] Refactor preprocessed trimming to seperate method

---
 freqtrade/data/converter.py       | 22 ++++++++++++++++++++++
 freqtrade/optimize/backtesting.py | 12 ++----------
 freqtrade/optimize/hyperopt.py    | 12 +++---------
 3 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py
index af6c6a2ef..7098834ec 100644
--- a/freqtrade/data/converter.py
+++ b/freqtrade/data/converter.py
@@ -10,6 +10,7 @@ from typing import Any, Dict, List
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
+from freqtrade.configuration.timerange import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
 
 
@@ -145,6 +146,27 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
     return df
 
 
+def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange: TimeRange,
+                    startup_candles: int) -> Dict[str, DataFrame]:
+    """
+    Trim startup period from analyzed dataframes
+    :param preprocessed: Dict of pair: dataframe
+    :param timerange: timerange (use start and end date if available)
+    :param startup_candles: Startup-candles that should be removed
+    :return: Dict of trimmed dataframes
+    """
+    processed: Dict[str, DataFrame] = {}
+
+    for pair, df in preprocessed.items():
+        trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles)
+        if not trimed_df.empty:
+            processed[pair] = trimed_df
+        else:
+            logger.warning(f'{pair} has no data left after adjusting for startup candles, '
+                           f'skipping.')
+    return processed
+
+
 def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     """
     TODO: This should get a dedicated test
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 0144d934b..aff09921c 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -15,7 +15,7 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.btanalysis import trade_list_to_dataframe
-from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
@@ -462,15 +462,7 @@ class Backtesting:
         preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
 
         # Trim startup period from analyzed dataframe
-        for pair in list(preprocessed):
-            df = preprocessed[pair]
-            df = trim_dataframe(df, timerange, startup_candles=self.required_startup)
-            if len(df) > 0:
-                preprocessed[pair] = df
-            else:
-                logger.warning(f'{pair} has no data left after adjusting for startup candles, '
-                               f'skipping.')
-                del preprocessed[pair]
+        preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
 
         if not preprocessed:
             raise OperationalException(
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 3d4588145..cf5559a24 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -20,7 +20,7 @@ from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_
 from pandas import DataFrame
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
-from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.converter import trim_dataframes
 from freqtrade.data.history import get_timerange
 from freqtrade.misc import file_dump_json, plural
 from freqtrade.optimize.backtesting import Backtesting
@@ -350,16 +350,10 @@ class Hyperopt:
         data, timerange = self.backtesting.load_bt_data()
         logger.info("Dataload complete. Calculating indicators")
 
-        processed: Dict[str, DataFrame] = {}
         preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
+
         # Trim startup period from analyzed dataframe
-        for pair, df in preprocessed.items():
-            trimed_df = trim_dataframe(df, timerange,
-                                       startup_candles=self.backtesting.required_startup)
-            if not trimed_df.empty:
-                processed[pair] = trimed_df
-            else:
-                logger.warn(f'Pair {pair} got removed because triming dataframe left nothing')
+        processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
 
         self.min_date, self.max_date = get_timerange(processed)