From 13662d4e116a6972391a66bad0970109baa38504 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 31 May 2018 22:22:51 +0300 Subject: [PATCH 1/5] Test if timerange in json file before trimming. trim_tickerlist throws an index out of range error when parsing a json file for a timerange outside its current contents. This patch test for that condition and returns the the file as-is with a log warning to refresh-cache on fail Here is an example of the error prior to path and after patching. This is from binannce using the pair "ZEN/BTC" and timerange "20180522-20180523" """ File "/Users/creslin/PycharmProjects/freqtrade/freqtrade/optimize/__init__.py", line 107, in load_data pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) File "/Users/creslin/PycharmProjects/freqtrade/freqtrade/optimize/__init__.py", line 84, in load_tickerdata_file pairdata = trim_tickerlist(pairdata, timerange) File "/Users/creslin/PycharmProjects/freqtrade/freqtrade/optimize/__init__.py", line 36, in trim_tickerlist while tickerlist[start_index][0] < start * 1000: IndexError: list index out of range """" """ 2018-05-31 22:01:07,060 - freqtrade.configuration - INFO - Parameter --timerange detected: 20180522-20180523 ... 2018-05-31 22:01:07,060 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... 2018-05-31 22:01:13,168 - freqtrade.optimize - WARNING - start timerange for ZEN/BTC not in cache, to update cache use 2018-05-31 22:01:13,168 - freqtrade.optimize - INFO - --refresh-pairs-cached. *Nb The coin may be newer to the exchange """ --- freqtrade/optimize/__init__.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f6f1ba47a..0467d6c4b 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -74,6 +74,23 @@ def load_tickerdata_file( pairdata = json.load(tickerdata) else: return None + """ + Check if timerange is fully within the pairdata loaded + Do not call trim_tickerlist if not, as will fail with -index out of range- error. + Log to user to run with --refresh-pairs-cached. + """ + if timerange: + stype, start, stop = timerange + if stype[0] == 'date': + if ((pairdata[0][0]) > (start * 1000)): + logger.warn('Start timerange for %s not in cache, to update cache use', pair) + logger.info('--refresh-pairs-cached. *NB The coin may be newer to the exchange') + return pairdata + if stype[1] == 'date': + if (pairdata[(len(pairdata) - 1)][0]) < (stop * 1000): + logger.warn('End timerange for %s not in cache, to update cache use', pair) + logger.info('--refresh-pairs-cached. *NB The coin may no longer be on the exchange') + return pairdata if timerange: pairdata = trim_tickerlist(pairdata, timerange) From 36e23142667ea90e4121441f2dfc2e4eec9a2039 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 1 Jun 2018 18:21:45 +0300 Subject: [PATCH 2/5] prevent index out of range in trim_tickerlist There is a bug in trim_tickerlist, it will throw an index out of range exception and crash the programe if passed a start / stop date outside the range of those records in the cached json pair file. This change calls a function that sanitizes the dates prior to calling trim_tickerlist. --- freqtrade/optimize/__init__.py | 79 ++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 0467d6c4b..07af5796d 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -15,6 +15,63 @@ from user_data.hyperopt_conf import hyperopt_optimize_conf logger = logging.getLogger(__name__) +def sanitize_start_stop_date(tickerlist: List[Dict], timerange, pair): + """ + This function is called when either timerange start or stop are type date + - + Prevent index out or range error in trim_tickerlist, which occurs when trying + to process a stop or start range outside that in the cached data + returns (start, stop, nothing_to_trim) + nothing_to_trim > 0 is a flag to later not send the pair data to trim_tickerlist + - + Function Logic: + - if cache records begin after timerange start - reset start to first record + - if cache records end before timerange stop - reset stop to last record + - if start or stop are fully after or before cache records - return, nothing_to_trim. + - if only 1 of stop or start are set in timerange use the first or last + record appropriately from the cache as the unset value + """ + + # Do nothing if timerange does not contain a type date. + stype, start, stop = timerange + nothing_to_trim = 0 + if stype[0] != 'date' or stype[1] != 'date': + sanitized_dates = [start, stop, nothing_to_trim] + return sanitized_dates + + # If no arg for stop or start then set to the first or last record in the cache + stop = stop if stype[1] == 'date' else int(((tickerlist[(len(tickerlist) - 1)][0]) / 1000)) + start = start if stype[0] == 'date' else int(((tickerlist[0][0]) / 1000)) + + # If requested range start is after cache records end - no data to be trimmed + # If requested start range is before start of cache, move it to first nearest record + if stype[0] == 'date': + if (tickerlist[0][0]) > (stop * 1000): + logger.warn('No data for %s timerange in cache, update cache ', pair) + nothing_to_trim = nothing_to_trim + 1 + elif (tickerlist[0][0]) > (start * 1000): + start = (tickerlist[0][0] / 1000) + logger.warn('Requested start timerange for %s not in cache, update cache ', pair) + + # If requested range stop is before cache records begin - no data to be trimmed + # If requested stop range is after end of cache, move it to last nearest record + if stype[1] == 'date': + if (tickerlist[(len(tickerlist) - 1)][0]) < (start * 1000): + logger.warn('No data for %s timerange in cache, update cache ', pair) + nothing_to_trim = nothing_to_trim + 2 + elif (tickerlist[(len(tickerlist) - 1)][0]) < (stop * 1000): + stop = (tickerlist[(len(tickerlist) - 1)][0] / 1000) + logger.warn('Requested stop timerange for %s not in cache, update cache ', pair) + + # Impossible range, nothing to trim. + if start > stop: + nothing_to_trim = nothing_to_trim + 4 + logger.warn('Check timerange for %s', pair) + + sanitized_dates = [start, stop, nothing_to_trim] + return sanitized_dates + + def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: if not tickerlist: return tickerlist @@ -23,7 +80,6 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) - start_index = 0 stop_index = len(tickerlist) - if stype[0] == 'line': stop_index = start if stype[0] == 'index': @@ -74,22 +130,19 @@ def load_tickerdata_file( pairdata = json.load(tickerdata) else: return None + """ - Check if timerange is fully within the pairdata loaded - Do not call trim_tickerlist if not, as will fail with -index out of range- error. - Log to user to run with --refresh-pairs-cached. + Call to function to catch if a start or stop date from timerange is outside + range of records in the cached ticker list. + This prevents "index out of range" error. """ if timerange: stype, start, stop = timerange - if stype[0] == 'date': - if ((pairdata[0][0]) > (start * 1000)): - logger.warn('Start timerange for %s not in cache, to update cache use', pair) - logger.info('--refresh-pairs-cached. *NB The coin may be newer to the exchange') - return pairdata - if stype[1] == 'date': - if (pairdata[(len(pairdata) - 1)][0]) < (stop * 1000): - logger.warn('End timerange for %s not in cache, to update cache use', pair) - logger.info('--refresh-pairs-cached. *NB The coin may no longer be on the exchange') + if stype[0] == 'date' or stype[1] == 'date': + sanitized_dates = sanitize_start_stop_date(pairdata, timerange, pair) + timerange = (('date', 'date'), int(sanitized_dates[0]), int(sanitized_dates[1])) + # If no overlap of timerange to cache data return pairdata, do not call trim_tickerlist + if sanitized_dates[2] > 0: return pairdata if timerange: From fed17de4606ddc400bdd86048d185b5c7fc5dfdd Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 2 Jun 2018 15:45:29 +0300 Subject: [PATCH 3/5] Extend timerange to support unxtime Extend timerange argument to support unixtime This allows finer granularity over backtesting. --- freqtrade/arguments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..00869f974 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,6 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'timestamp')), + (r'^(\d{10})-$', ('timestamp', None)), + (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -237,6 +240,8 @@ class Arguments(object): start = rvals[index] if stype[0] == 'date': start = arrow.get(start, 'YYYYMMDD').timestamp + elif stype[0] == 'timestamp': + start = arrow.get(start).timestamp else: start = int(start) index += 1 @@ -244,6 +249,8 @@ class Arguments(object): stop = rvals[index] if stype[1] == 'date': stop = arrow.get(stop, 'YYYYMMDD').timestamp + elif stype[1] == 'timestamp': + stop = arrow.get(stop).timestamp else: stop = int(stop) return stype, start, stop From bc3d4518ff9ddf8a19d5c2031b644049971a1dfb Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 2 Jun 2018 17:31:31 +0300 Subject: [PATCH 4/5] Revert "Extend timerange to support unxtime" This reverts commit fed17de4606ddc400bdd86048d185b5c7fc5dfdd. --- freqtrade/arguments.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 00869f974..afcb05511 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,9 +222,6 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^-(\d{10})$', (None, 'timestamp')), - (r'^(\d{10})-$', ('timestamp', None)), - (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -240,8 +237,6 @@ class Arguments(object): start = rvals[index] if stype[0] == 'date': start = arrow.get(start, 'YYYYMMDD').timestamp - elif stype[0] == 'timestamp': - start = arrow.get(start).timestamp else: start = int(start) index += 1 @@ -249,8 +244,6 @@ class Arguments(object): stop = rvals[index] if stype[1] == 'date': stop = arrow.get(stop, 'YYYYMMDD').timestamp - elif stype[1] == 'timestamp': - stop = arrow.get(stop).timestamp else: stop = int(stop) return stype, start, stop From b54e6c5af0a607394ab8c4d1e947b94beb026092 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 2 Jun 2018 17:39:17 +0300 Subject: [PATCH 5/5] Revert "Revert "Extend timerange to support unxtime"" This reverts commit bc3d4518ff9ddf8a19d5c2031b644049971a1dfb. --- freqtrade/arguments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..00869f974 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,6 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'timestamp')), + (r'^(\d{10})-$', ('timestamp', None)), + (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -237,6 +240,8 @@ class Arguments(object): start = rvals[index] if stype[0] == 'date': start = arrow.get(start, 'YYYYMMDD').timestamp + elif stype[0] == 'timestamp': + start = arrow.get(start).timestamp else: start = int(start) index += 1 @@ -244,6 +249,8 @@ class Arguments(object): stop = rvals[index] if stype[1] == 'date': stop = arrow.get(stop, 'YYYYMMDD').timestamp + elif stype[1] == 'timestamp': + stop = arrow.get(stop).timestamp else: stop = int(stop) return stype, start, stop