From ecddaa663b3c8be6af8f305efd5f48548e52d2b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Oct 2020 19:58:04 +0200 Subject: [PATCH 1/4] Convert timestamp to int_timestamp for all arrow occurances --- freqtrade/configuration/timerange.py | 8 ++++---- freqtrade/edge/edge_positioning.py | 4 ++-- freqtrade/exchange/exchange.py | 12 ++++++------ freqtrade/optimize/optimize_reports.py | 4 ++-- freqtrade/wallets.py | 4 ++-- tests/commands/test_commands.py | 5 +++-- tests/conftest.py | 12 ++++++------ tests/data/test_history.py | 4 ++-- tests/edge/test_edge.py | 12 ++++++------ tests/exchange/test_exchange.py | 23 ++++++++++++----------- 10 files changed, 45 insertions(+), 43 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 151003999..32bbd02a0 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -52,11 +52,11 @@ class TimeRange: :return: None (Modifies the object in place) """ if (not self.starttype or (startup_candles - and min_date.timestamp >= self.startts)): + and min_date.int_timestamp >= self.startts)): # If no startts was defined, or backtest-data starts at the defined backtest-date logger.warning("Moving start-date by %s candles to account for startup time.", startup_candles) - self.startts = (min_date.timestamp + timeframe_secs * startup_candles) + self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles) self.starttype = 'date' @staticmethod @@ -89,7 +89,7 @@ class TimeRange: if stype[0]: starts = rvals[index] if stype[0] == 'date' and len(starts) == 8: - start = arrow.get(starts, 'YYYYMMDD').timestamp + start = arrow.get(starts, 'YYYYMMDD').int_timestamp elif len(starts) == 13: start = int(starts) // 1000 else: @@ -98,7 +98,7 @@ class TimeRange: if stype[1]: stops = rvals[index] if stype[1] == 'date' and len(stops) == 8: - stop = arrow.get(stops, 'YYYYMMDD').timestamp + stop = arrow.get(stops, 'YYYYMMDD').int_timestamp elif len(stops) == 13: stop = int(stops) // 1000 else: diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index a40b63d67..037717c68 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -87,7 +87,7 @@ class Edge: heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().timestamp): + self._last_updated + heartbeat > arrow.utcnow().int_timestamp): return False data: Dict[str, Any] = {} @@ -146,7 +146,7 @@ class Edge: # Fill missing, calculable columns, profit, duration , abs etc. trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().timestamp + self._last_updated = arrow.utcnow().int_timestamp return True diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index bbb94e61f..d6836ee73 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -291,7 +291,7 @@ class Exchange: try: self._api.load_markets() self._load_async_markets() - self._last_markets_refresh = arrow.utcnow().timestamp + self._last_markets_refresh = arrow.utcnow().int_timestamp except ccxt.BaseError as e: logger.warning('Unable to initialize markets. Reason: %s', e) @@ -300,14 +300,14 @@ class Exchange: # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( self._last_markets_refresh + self.markets_refresh_interval - > arrow.utcnow().timestamp): + > arrow.utcnow().int_timestamp): return None logger.debug("Performing scheduled market reload..") try: self._api.load_markets(reload=True) # Also reload async markets to avoid issues with newly listed pairs self._load_async_markets(reload=True) - self._last_markets_refresh = arrow.utcnow().timestamp + self._last_markets_refresh = arrow.utcnow().int_timestamp except ccxt.BaseError: logger.exception("Could not reload markets.") @@ -501,7 +501,7 @@ class Exchange: 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), - 'timestamp': int(arrow.utcnow().timestamp * 1000), + 'timestamp': int(arrow.utcnow().int_timestamp * 1000), 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} @@ -696,7 +696,7 @@ class Exchange: ) input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in - range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] results = await asyncio.gather(*input_coroutines, return_exceptions=True) @@ -759,7 +759,7 @@ class Exchange: interval_in_sec = timeframe_to_seconds(timeframe) return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) - + interval_in_sec) >= arrow.utcnow().timestamp) + + interval_in_sec) >= arrow.utcnow().int_timestamp) @retrier_async async def _async_get_candle_history(self, pair: str, timeframe: str, diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 3db9a312a..c977a991b 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -268,9 +268,9 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'profit_total': results['profit_percent'].sum(), 'profit_total_abs': results['profit_abs'].sum(), 'backtest_start': min_date.datetime, - 'backtest_start_ts': min_date.timestamp * 1000, + 'backtest_start_ts': min_date.int_timestamp * 1000, 'backtest_end': max_date.datetime, - 'backtest_end_ts': max_date.timestamp * 1000, + 'backtest_end_ts': max_date.int_timestamp * 1000, 'backtest_days': backtest_days, 'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0, diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 21a9466e1..3680dd416 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -108,13 +108,13 @@ class Wallets: for trading operations, the latest balance is needed. :param require_update: Allow skipping an update if balances were recently refreshed """ - if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().timestamp)): + if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().int_timestamp)): if self._config['dry_run']: self._update_dry() else: self._update_live() logger.info('Wallets synced.') - self._last_wallet_refresh = arrow.utcnow().timestamp + self._last_wallet_refresh = arrow.utcnow().int_timestamp def get_all_balances(self) -> Dict[str, Any]: return self._wallets diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 5b125697c..bf845a2e1 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -579,7 +579,7 @@ def test_download_data_timerange(mocker, caplog, markets): start_download_data(get_args(args)) assert dl_mock.call_count == 1 # 20days ago - days_ago = arrow.get(arrow.utcnow().shift(days=-20).date()).timestamp + days_ago = arrow.get(arrow.utcnow().shift(days=-20).date()).int_timestamp assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago dl_mock.reset_mock() @@ -592,7 +592,8 @@ def test_download_data_timerange(mocker, caplog, markets): start_download_data(get_args(args)) assert dl_mock.call_count == 1 - assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow(2020, 1, 1).timestamp + assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow( + 2020, 1, 1).int_timestamp def test_download_data_no_markets(mocker, caplog): diff --git a/tests/conftest.py b/tests/conftest.py index 2153fd327..a99404ac2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -792,7 +792,7 @@ def limit_buy_order_open(): 'side': 'buy', 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().timestamp, + 'timestamp': arrow.utcnow().int_timestamp, 'price': 0.00001099, 'amount': 90.99181073, 'filled': 0.0, @@ -911,7 +911,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': None, - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', @@ -932,7 +932,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': 'AZNPFF-4AC4N-7MKTAT', 'clientOrderId': None, - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'status': 'canceled', @@ -953,7 +953,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', @@ -974,7 +974,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', @@ -1000,7 +1000,7 @@ def limit_sell_order_open(): 'side': 'sell', 'pair': 'mocked', 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().timestamp, + 'timestamp': arrow.utcnow().int_timestamp, 'price': 0.00001173, 'amount': 90.99181073, 'filled': 0.0, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index c8324cf0b..bbc6e55b4 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -323,7 +323,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') data = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, - timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) assert log_has( 'Using indicator startup period: 20 ...', caplog ) @@ -339,7 +339,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') data = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], - timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(data['UNITTEST/BTC']) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index a4bfa1085..f25dad35b 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -50,7 +50,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): 'date': tests_start_time.shift( minutes=( ohlc[0] * - timeframe_in_minute)).timestamp * + timeframe_in_minute)).int_timestamp * 1000, 'buy': ohlc[1], 'open': ohlc[2], @@ -71,7 +71,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): def _time_on_candle(number): return np.datetime64(tests_start_time.shift( - minutes=(number * timeframe_in_minute)).timestamp * 1000, 'ms') + minutes=(number * timeframe_in_minute)).int_timestamp * 1000, 'ms') # End helper functions @@ -251,7 +251,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): heartbeat = edge_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + edge._last_updated = arrow.utcnow().int_timestamp - heartbeat + 1 assert edge.calculate() is False @@ -263,7 +263,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', NEOBTC = [ [ - tests_start_time.shift(minutes=(x * timeframe_in_minute)).timestamp * 1000, + tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -275,7 +275,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', base = 0.002 LTCBTC = [ [ - tests_start_time.shift(minutes=(x * timeframe_in_minute)).timestamp * 1000, + tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -299,7 +299,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): assert edge.calculate() assert len(edge._cached_pairs) == 2 - assert edge._last_updated <= arrow.utcnow().timestamp + 2 + assert edge._last_updated <= arrow.utcnow().int_timestamp + 2 def test_edge_process_no_data(mocker, edge_conf, caplog): diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7be9c77ac..7df596098 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -385,7 +385,7 @@ def test_reload_markets(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) exchange._load_async_markets = MagicMock() - exchange._last_markets_refresh = arrow.utcnow().timestamp + exchange._last_markets_refresh = arrow.utcnow().int_timestamp updated_markets = {'ETH/BTC': {}, "LTC/BTC": {}} assert exchange.markets == initial_markets @@ -396,7 +396,7 @@ def test_reload_markets(default_conf, mocker, caplog): assert exchange._load_async_markets.call_count == 0 # more than 10 minutes have passed, reload is executed - exchange._last_markets_refresh = arrow.utcnow().timestamp - 15 * 60 + exchange._last_markets_refresh = arrow.utcnow().int_timestamp - 15 * 60 exchange.reload_markets() assert exchange.markets == updated_markets assert exchange._load_async_markets.call_count == 1 @@ -1264,7 +1264,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -1281,7 +1281,8 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 - ret = exchange.get_historic_ohlcv(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) + ret = exchange.get_historic_ohlcv(pair, "5m", int(( + arrow.utcnow().int_timestamp - since) * 1000)) assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data @@ -1291,7 +1292,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ - (arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms + (arrow.utcnow().int_timestamp - 1) * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -1299,7 +1300,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: 5, # volume (in quote currency) ], [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 3, # open 1, # high 4, # low @@ -1345,7 +1346,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): ohlcv = [ [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -1380,14 +1381,14 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", - (arrow.utcnow().timestamp - 2000) * 1000) + (arrow.utcnow().int_timestamp - 2000) * 1000) with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical candle \(OHLCV\) data\..*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", - (arrow.utcnow().timestamp - 2000) * 1000) + (arrow.utcnow().int_timestamp - 2000) * 1000) @pytest.mark.asyncio @@ -1599,13 +1600,13 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, with pytest.raises(OperationalException, match=r'Could not fetch trade data*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical trade data\..*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) @pytest.mark.asyncio From 685d18940a52eff4534d1a87bdb440cabb413bf1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Oct 2020 08:13:31 +0200 Subject: [PATCH 2/4] specify min-version for arrow int_timestamp was introduced in this version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9b57e8d2c..b47427709 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup(name='freqtrade', 'ccxt>=1.24.96', 'SQLAlchemy', 'python-telegram-bot', - 'arrow', + 'arrow>=0.17.0', 'cachetools', 'requests', 'urllib3', From adffd402ea6aff03719d21bf725448f8da0d8b27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Oct 2020 20:11:38 +0200 Subject: [PATCH 3/4] Replace some pointless occurances of arrow --- freqtrade/commands/data_commands.py | 4 ++-- freqtrade/data/dataprovider.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 7102eee38..c12c20ddd 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -1,9 +1,9 @@ import logging import sys from collections import defaultdict +from datetime import datetime, timedelta from typing import Any, Dict, List -import arrow from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format @@ -29,7 +29,7 @@ def start_download_data(args: Dict[str, Any]) -> None: "You can only specify one or the other.") timerange = TimeRange() if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') if 'timerange' in config: diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 07dd94fc1..ba43044a1 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -8,7 +8,6 @@ import logging from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple -from arrow import Arrow from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe @@ -38,7 +37,7 @@ class DataProvider: :param timeframe: Timeframe to get data for :param dataframe: analyzed dataframe """ - self.__cached_pairs[(pair, timeframe)] = (dataframe, Arrow.utcnow().datetime) + self.__cached_pairs[(pair, timeframe)] = (dataframe, datetime.now(timezone.utc)) def add_pairlisthandler(self, pairlists) -> None: """ From fd6018f67aaa9620c237098f8139bcfb06832d62 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Oct 2020 06:21:13 +0200 Subject: [PATCH 4/4] Fix dependency sorting --- freqtrade/commands/data_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index c12c20ddd..b28e0c357 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -4,7 +4,6 @@ from collections import defaultdict from datetime import datetime, timedelta from typing import Any, Dict, List - from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,