adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into fix/pd-mem-leak

Browse Source
This commit is contained in:
Timothy Pogue 2022-11-17 12:04:49 -07:00
commit 49ecc83061
10 changed files with 85 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
freqtrade/configuration/timerange.py
View File

@ -3,11 +3,12 @@ This module contains the argument manager class
"""
import logging
import re
from datetime import datetime
from datetime import datetime, timezone
from typing import Optional
import arrow
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.exceptions import OperationalException
@ -29,6 +30,52 @@ class TimeRange:
self.startts: int = startts
self.stopts: int = stopts
@property
def startdt(self) -> Optional[datetime]:
if self.startts:
return datetime.fromtimestamp(self.startts, tz=timezone.utc)
return None
@property
def stopdt(self) -> Optional[datetime]:
if self.stopts:
return datetime.fromtimestamp(self.stopts, tz=timezone.utc)
return None
@property
def timerange_str(self) -> str:
"""
Returns a string representation of the timerange as used by parse_timerange.
Follows the format yyyymmdd-yyyymmdd - leaving out the parts that are not set.
"""
start = ''
stop = ''
if startdt := self.startdt:
start = startdt.strftime('%Y%m%d')
if stopdt := self.stopdt:
stop = stopdt.strftime('%Y%m%d')
return f"{start}-{stop}"
@property
def start_fmt(self) -> str:
"""
Returns a string representation of the start date
"""
val = 'unbounded'
if (startdt := self.startdt) is not None:
val = startdt.strftime(DATETIME_PRINT_FORMAT)
return val
@property
def stop_fmt(self) -> str:
"""
Returns a string representation of the stop date
"""
val = 'unbounded'
if (stopdt := self.stopdt) is not None:
val = stopdt.strftime(DATETIME_PRINT_FORMAT)
return val
def __eq__(self, other):
"""Override the default Equals behavior"""
return (self.starttype == other.starttype and self.stoptype == other.stoptype

7
freqtrade/data/converter.py
View File

@ -3,7 +3,6 @@ Functions to convert data from one format to another
"""
import itertools
import logging
from datetime import datetime, timezone
from operator import itemgetter
from typing import Dict, List
@ -138,11 +137,9 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
df = df.iloc[startup_candles:, :]
else:
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
df = df.loc[df[df_date_col] >= start, :]
df = df.loc[df[df_date_col] >= timerange.startdt, :]
if timerange.stoptype == 'date':
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
df = df.loc[df[df_date_col] <= stop, :]
df = df.loc[df[df_date_col] <= timerange.stopdt, :]
return df

6
freqtrade/data/history/history_utils.py
View File

@ -1,6 +1,6 @@
import logging
import operator
from datetime import datetime, timezone
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Tuple
@ -160,9 +160,9 @@ def _load_cached_data_for_updating(
end = None
if timerange:
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
start = timerange.startdt
if timerange.stoptype == 'date':
end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
end = timerange.stopdt
# Intentionally don't pass timerange in - since we need to load the full dataset.
data = data_handler.ohlcv_load(pair, timeframe=timeframe,

6
freqtrade/data/history/idatahandler.py
View File

@ -366,13 +366,11 @@ class IDataHandler(ABC):
"""
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
if pairdata.iloc[0]['date'] > start:
if pairdata.iloc[0]['date'] > timerange.startdt:
logger.warning(f"{pair}, {candle_type}, {timeframe}, "
f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}")
if timerange.stoptype == 'date':
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
if pairdata.iloc[-1]['date'] < stop:
if pairdata.iloc[-1]['date'] < timerange.stopdt:
logger.warning(f"{pair}, {candle_type}, {timeframe}, "
f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")

18
freqtrade/freqai/data_kitchen.py
View File

@ -433,9 +433,7 @@ class FreqaiDataKitchen:
timerange_train.stopts = timerange_train.startts + train_period_days
first = False
start = datetime.fromtimestamp(timerange_train.startts, tz=timezone.utc)
stop = datetime.fromtimestamp(timerange_train.stopts, tz=timezone.utc)
tr_training_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d"))
tr_training_list.append(timerange_train.timerange_str)
tr_training_list_timerange.append(copy.deepcopy(timerange_train))
# associated backtest period
@ -447,9 +445,7 @@ class FreqaiDataKitchen:
if timerange_backtest.stopts > config_timerange.stopts:
timerange_backtest.stopts = config_timerange.stopts
start = datetime.fromtimestamp(timerange_backtest.startts, tz=timezone.utc)
stop = datetime.fromtimestamp(timerange_backtest.stopts, tz=timezone.utc)
tr_backtesting_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d"))
tr_backtesting_list.append(timerange_backtest.timerange_str)
tr_backtesting_list_timerange.append(copy.deepcopy(timerange_backtest))
# ensure we are predicting on exactly same amount of data as requested by user defined
@ -491,11 +487,9 @@ class FreqaiDataKitchen:
it is sliced down to just the present training period.
"""
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
df = df.loc[df["date"] >= start, :]
df = df.loc[df["date"] >= timerange.startdt, :]
if not self.live:
df = df.loc[df["date"] < stop, :]
df = df.loc[df["date"] < timerange.stopdt, :]
return df
@ -1058,9 +1052,7 @@ class FreqaiDataKitchen:
backtest_timerange.startts = (
backtest_timerange.startts - backtest_period_days * SECONDS_IN_DAY
)
start = datetime.fromtimestamp(backtest_timerange.startts, tz=timezone.utc)
stop = datetime.fromtimestamp(backtest_timerange.stopts, tz=timezone.utc)
full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")
full_timerange = backtest_timerange.timerange_str
config_path = Path(self.config["config_files"][0])
if not self.full_path.is_dir():

23
freqtrade/freqai/freqai_interface.py
View File

@ -13,7 +13,7 @@ from numpy.typing import NDArray
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
from freqtrade.constants import Config
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
@ -788,14 +788,8 @@ class IFreqaiModel(ABC):
:return: if the data exists or not
"""
if self.config.get("freqai_backtest_live_models", False) and len(dataframe_backtest) == 0:
tr_backtest_startts_str = datetime.fromtimestamp(
tr_backtest.startts,
tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
tr_backtest_stopts_str = datetime.fromtimestamp(
tr_backtest.stopts,
tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
logger.info(f"No data found for pair {pair} from {tr_backtest_startts_str} "
f" from {tr_backtest_startts_str} to {tr_backtest_stopts_str}. "
logger.info(f"No data found for pair {pair} from "
f"from { tr_backtest.start_fmt} to {tr_backtest.stop_fmt}. "
"Probably more than one training within the same candle period.")
return False
return True
@ -810,18 +804,11 @@ class IFreqaiModel(ABC):
:param pair: the current pair
:param total_trains: total trains (total number of slides for the sliding window)
"""
tr_train_startts_str = datetime.fromtimestamp(
tr_train.startts,
tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
tr_train_stopts_str = datetime.fromtimestamp(
tr_train.stopts,
tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
if not self.config.get("freqai_backtest_live_models", False):
logger.info(
f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs"
f" from {tr_train_startts_str} "
f"to {tr_train_stopts_str}, {train_it}/{total_trains} "
f" from {tr_train.start_fmt} "
f"to {tr_train.stop_fmt}, {train_it}/{total_trains} "
"trains"
)
# Following methods which are overridden by user made prediction models.

5
freqtrade/freqai/utils.py
View File

@ -230,7 +230,4 @@ def get_timerange_backtest_live_models(config: Config) -> str:
dk = FreqaiDataKitchen(config)
models_path = dk.get_full_models_path(config)
timerange, _ = dk.get_timerange_and_assets_end_dates_from_ready_models(models_path)
start_date = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
end_date = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
tr = f"{start_date.strftime('%Y%m%d')}-{end_date.strftime('%Y%m%d')}"
return tr
return timerange.timerange_str

3
freqtrade/optimize/backtesting.py
View File

@ -1286,8 +1286,7 @@ class Backtesting:
def _get_min_cached_backtest_date(self):
min_backtest_date = None
backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
if self.timerange.stopts == 0 or datetime.fromtimestamp(
self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
if self.timerange.stopts == 0 or self.timerange.stopdt > datetime.now(tz=timezone.utc):
logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
elif backtest_cache_age == 'day':
min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1)

2
freqtrade/rpc/telegram.py
View File

@ -1062,7 +1062,7 @@ class Telegram(RPCHandler):
self._rpc._rpc_force_entry(pair, price, order_side=order_side)
except RPCException as e:
logger.exception("Forcebuy error!")
self._send_msg(str(e))
self._send_msg(str(e), ParseMode.HTML)
def _force_enter_inline(self, update: Update, _: CallbackContext) -> None:
if update.callback_query:

19
tests/test_timerange.py
View File

@ -1,4 +1,6 @@
# pragma pylint: disable=missing-docstring, C0103
from datetime import datetime, timezone
import arrow
import pytest
@ -8,16 +10,28 @@ from freqtrade.exceptions import OperationalException
def test_parse_timerange_incorrect():
assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-')
assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522')
timerange = TimeRange.parse_timerange('20100522-')
assert TimeRange('date', None, 1274486400, 0) == timerange
assert timerange.timerange_str == '20100522-'
timerange = TimeRange.parse_timerange('-20100522')
assert TimeRange(None, 'date', 0, 1274486400) == timerange
assert timerange.timerange_str == '-20100522'
timerange = TimeRange.parse_timerange('20100522-20150730')
assert timerange == TimeRange('date', 'date', 1274486400, 1438214400)
assert timerange.timerange_str == '20100522-20150730'
assert timerange.start_fmt == '2010-05-22 00:00:00'
assert timerange.stop_fmt == '2015-07-30 00:00:00'
# Added test for unix timestamp - BTC genesis date
assert TimeRange('date', None, 1231006505, 0) == TimeRange.parse_timerange('1231006505-')
assert TimeRange(None, 'date', 0, 1233360000) == TimeRange.parse_timerange('-1233360000')
timerange = TimeRange.parse_timerange('1231006505-1233360000')
assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange
assert isinstance(timerange.startdt, datetime)
assert isinstance(timerange.stopdt, datetime)
assert timerange.startdt == datetime.fromtimestamp(1231006505, tz=timezone.utc)
assert timerange.stopdt == datetime.fromtimestamp(1233360000, tz=timezone.utc)
assert timerange.timerange_str == '20090103-20090131'
timerange = TimeRange.parse_timerange('1231006505000-1233360000000')
assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange
@ -45,6 +59,7 @@ def test_subtract_start():
x = TimeRange(None, 'date', 0, 1438214400)
x.subtract_start(300)
assert not x.startts
assert not x.startdt
x = TimeRange('date', None, 1274486400, 0)
x.subtract_start(300)