From da57396d071131102c366d9821e13566cf594998 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Nov 2019 06:55:07 +0100 Subject: [PATCH 1/2] Fix UTC handling of timestamp() conversation in fetch_my_trades --- freqtrade/data/btanalysis.py | 6 +++--- freqtrade/data/history.py | 7 +++---- freqtrade/exchange/exchange.py | 19 ++++++++++++++++++- tests/exchange/test_exchange.py | 17 +++++++++++++++-- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 388deb4b3..2f7a234ce 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -7,7 +7,7 @@ from typing import Dict import numpy as np import pandas as pd -import pytz +from datetime import timezone from freqtrade import persistence from freqtrade.misc import json_load @@ -106,8 +106,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: "stop_loss", "initial_stop_loss", "strategy", "ticker_interval"] trades = pd.DataFrame([(t.pair, - t.open_date.replace(tzinfo=pytz.UTC), - t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None, + t.open_date.replace(tzinfo=timezone.utc), + t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None, t.calc_profit(), t.calc_profit_percent(), t.open_rate, t.close_rate, t.amount, (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index b1e4313ca..3dd40d2b4 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -9,12 +9,11 @@ Includes: import logging import operator from copy import deepcopy -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import arrow -import pytz from pandas import DataFrame from freqtrade import OperationalException, misc @@ -56,10 +55,10 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame: Trim dataframe based on given timerange """ if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc) + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) df = df.loc[df['date'] >= start, :] if timerange.stoptype == 'date': - stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc) + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) df = df.loc[df['date'] <= stop, :] return df diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 430a2ff54..a198e8cdb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -875,6 +875,22 @@ class Exchange: @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ if self._config['dry_run']: return [] if not self.exchange_has('fetchMyTrades'): @@ -882,7 +898,8 @@ class Exchange: try: # Allow 5s offset to catch slight time offsets (discovered in #1185) # since needs to be int in milliseconds - my_trades = self._api.fetch_my_trades(pair, int((since.timestamp() - 5) * 1000)) + my_trades = self._api.fetch_my_trades( + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8a4121d80..1bb643e24 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1586,8 +1586,20 @@ def test_name(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_trades_for_order(default_conf, mocker, exchange_name): + """ + Crucial part in this test is the "since" calculation. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + """ order_id = 'ABCD-ABCD' - since = datetime(2018, 5, 5, tzinfo=timezone.utc) + since = datetime(2018, 5, 5, 0, 0, 0) default_conf["dry_run"] = False mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) api_mock = MagicMock() @@ -1623,7 +1635,8 @@ def test_get_trades_for_order(default_conf, mocker, exchange_name): assert api_mock.fetch_my_trades.call_args[0][0] == 'LTC/BTC' # Same test twice, hardcoded number and doing the same calculation assert api_mock.fetch_my_trades.call_args[0][1] == 1525478395000 - assert api_mock.fetch_my_trades.call_args[0][1] == int(since.timestamp() - 5) * 1000 + assert api_mock.fetch_my_trades.call_args[0][1] == int(since.replace( + tzinfo=timezone.utc).timestamp() - 5) * 1000 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, 'get_trades_for_order', 'fetch_my_trades', From dd47bd04cd905c8822f5c27a285db8d37d72847f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Nov 2019 01:32:08 -0500 Subject: [PATCH 2/2] Move description to correct place --- tests/exchange/test_exchange.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1bb643e24..925a53c95 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1586,18 +1586,7 @@ def test_name(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_trades_for_order(default_conf, mocker, exchange_name): - """ - Crucial part in this test is the "since" calculation. - The "since" argument passed in is coming from the database and is in UTC, - as timezone-native datetime object. - From the python documentation: - > Naive datetime instances are assumed to represent local time - Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the - transformation from local timezone to UTC. - This works for timezones UTC+ since then the result will contain trades from a few hours - instead of from the last 5 seconds, however fails for UTC- timezones, - since we're then asking for trades with a "since" argument in the future. - """ + order_id = 'ABCD-ABCD' since = datetime(2018, 5, 5, 0, 0, 0) default_conf["dry_run"] = False