Move trim_dataframe from history to converter

freqtrade/data/converter.py

@@ -2,10 +2,12 @@
 Functions to convert data from one format to another
 """
 import logging
+from datetime import datetime, timezone
 
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
+from freqtrade.configuration.timerange import TimeRange
 
 logger = logging.getLogger(__name__)
 
@@ -112,6 +114,23 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     return df
 
 
+def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date') -> DataFrame:
+    """
+    Trim dataframe based on given timerange
+    :param df: Dataframe to trim
+    :param timerange: timerange (use start and end date if available)
+    :param: df_date_col: Column in the dataframe to use as Date column
+    :return: trimmed dataframe
+    """
+    if timerange.starttype == 'date':
+        start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+        df = df.loc[df[df_date_col] >= start, :]
+    if timerange.stoptype == 'date':
+        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
+        df = df.loc[df[df_date_col] <= stop, :]
+    return df
+
+
 def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     """
     Gets order book list, returns dataframe with below format per suggested by creslin

freqtrade/data/datahandlers/jsondatahandler.py

@@ -7,8 +7,7 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.data.converter import clean_ohlcv_dataframe
-from freqtrade.data.history import trim_dataframe
+from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
 
 from .idatahandler import IDataHandler
 

freqtrade/data/history.py

@@ -50,23 +50,6 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     return tickerlist[start_index:stop_index]
 
 
-def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date') -> DataFrame:
-    """
-    Trim dataframe based on given timerange
-    :param df: Dataframe to trim
-    :param timerange: timerange (use start and end date if available)
-    :param: df_date_col: Column in the dataframe to use as Date column
-    :return: trimmed dataframe
-    """
-    if timerange.starttype == 'date':
-        start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
-        df = df.loc[df[df_date_col] >= start, :]
-    if timerange.stoptype == 'date':
-        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
-        df = df.loc[df[df_date_col] <= stop, :]
-    return df
-
-
 def load_tickerdata_file(datadir: Path, pair: str, timeframe: str,
                          timerange: Optional[TimeRange] = None) -> List[Dict]:
     """
@@ -115,15 +98,6 @@ def store_trades_file(datadir: Path, pair: str,
     misc.file_dump_json(filename, data, is_zip=is_zip)
 
 
-def _validate_pairdata(pair, pairdata, timerange: TimeRange):
-    if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000:
-        logger.warning('Missing data at start for pair %s, data starts at %s',
-                       pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
-    if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000:
-        logger.warning('Missing data at end for pair %s, data ends at %s',
-                       pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
-
-
 def load_pair_history(pair: str,
                       timeframe: str,
                       datadir: Path, *,

freqtrade/optimize/backtesting.py

@@ -16,6 +16,7 @@ from freqtrade import OperationalException
 from freqtrade.configuration import (TimeRange, remove_credentials,
                                      validate_config_consistency)
 from freqtrade.data import history
+from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.misc import file_dump_json
@@ -482,7 +483,7 @@ class Backtesting:
 
             # Trim startup period from analyzed dataframe
             for pair, df in preprocessed.items():
-                preprocessed[pair] = history.trim_dataframe(df, timerange)
+                preprocessed[pair] = trim_dataframe(df, timerange)
             min_date, max_date = history.get_timerange(preprocessed)
 
             logger.info(

freqtrade/optimize/hyperopt.py

@@ -23,7 +23,8 @@ from joblib import (Parallel, cpu_count, delayed, dump, load,
 from pandas import DataFrame
 
 from freqtrade import OperationalException
-from freqtrade.data.history import get_timerange, trim_dataframe
+from freqtrade.data.history import get_timerange
+from freqtrade.data.converter import trim_dataframe
 from freqtrade.misc import plural, round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules

freqtrade/plot/plotting.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, List
 import pandas as pd
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
+from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.btanalysis import (combine_tickers_with_mean,
                                        create_cum_profit,
                                        extract_trades_of_period, load_trades)
@@ -47,7 +48,7 @@ def init_plotscript(config):
                          db_url=config.get('db_url'),
                          exportfilename=config.get('exportfilename'),
                          )
-    trades = history.trim_dataframe(trades, timerange, 'open_time')
+    trades = trim_dataframe(trades, timerange, 'open_time')
     return {"tickers": tickers,
             "trades": trades,
             "pairs": pairs,