From a3144cb2f0c07c777f566326f0df9246dfbc36d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Dec 2019 10:49:30 +0100 Subject: [PATCH] remove trim_tickerlist --- freqtrade/data/history.py | 59 ++++---------------------------------- tests/data/test_history.py | 54 +--------------------------------- 2 files changed, 6 insertions(+), 107 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 59695da2e..9dad90864 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -15,7 +15,7 @@ from typing import Dict, List, Optional, Tuple import arrow from pandas import DataFrame -from freqtrade import OperationalException, misc +from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv @@ -26,48 +26,6 @@ from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) -def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: - """ - Trim tickerlist based on given timerange - """ - if not tickerlist: - return tickerlist - - start_index = 0 - stop_index = len(tickerlist) - - if timerange.starttype == 'date': - while (start_index < len(tickerlist) and - tickerlist[start_index][0] < timerange.startts * 1000): - start_index += 1 - - if timerange.stoptype == 'date': - while (stop_index > 0 and - tickerlist[stop_index-1][0] > timerange.stopts * 1000): - stop_index -= 1 - - if start_index > stop_index: - raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') - - return tickerlist[start_index:stop_index] - - -def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None) -> List[Dict]: - """ - Load a pair from file, either .json.gz or .json - :return: tickerlist or None if unsuccessful - """ - filename = pair_data_filename(datadir, pair, timeframe) - pairdata = misc.file_load_json(filename) - if not pairdata: - return [] - - if timerange: - pairdata = trim_tickerlist(pairdata, timerange) - return pairdata - - def load_pair_history(pair: str, timeframe: str, datadir: Path, *, @@ -170,12 +128,6 @@ def refresh_data(datadir: Path, exchange=exchange, data_handler=data_handler) -def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = pair.replace("/", "_") - filename = datadir.joinpath(f'{pair_s}-{timeframe}.json') - return filename - - def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: """ @@ -291,11 +243,10 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes continue for timeframe in timeframes: - dl_file = pair_data_filename(datadir, pair, timeframe) - if erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {timeframe}.') - dl_file.unlink() + if erase: + if data_handler.ohlcv_purge(pair, timeframe): + logger.info( + f'Deleting existing data for pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.') _download_pair_history(datadir=datadir, exchange=exchange, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 969e58223..7e030b523 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -22,7 +22,7 @@ from freqtrade.data.history import (_download_pair_history, load_data, load_pair_history, refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, - trim_tickerlist, validate_backtest_data) + validate_backtest_data) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy @@ -358,58 +358,6 @@ def test_init_with_refresh(default_conf, mocker) -> None: ) -def test_trim_tickerlist(testdatadir) -> None: - file = testdatadir / 'UNITTEST_BTC-1m.json' - with open(file) as data_file: - ticker_list = json.load(data_file) - ticker_list_len = len(ticker_list) - - # Test the pattern ^(\d{8})-(\d{8})$ - # This pattern extract a window between the dates - timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[5] is ticker[0] # The list starts at the index 5 - assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) - - # Test the pattern ^-(\d{8})$ - # This pattern extracts elements from the start to the date - timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 10 - assert ticker_list[0] is ticker[0] # The start of the list is included - assert ticker_list[9] is ticker[-1] # The element 10 is not included - - # Test the pattern ^(\d{8})-$ - # This pattern extracts elements from the date to now - timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == ticker_list_len - 10 - assert ticker_list[10] is ticker[0] # The first element is element #10 - assert ticker_list[-1] is ticker[-1] # The last element is the same - - # Test a wrong pattern - # This pattern must return the list unchanged - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_list_len == ticker_len - - # passing empty list - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist([], timerange) - assert 0 == len(ticker) - assert not ticker - - def test_file_dump_json_tofile(testdatadir) -> None: file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'}