Merge pull request #2494 from freqtrade/fix/timezone_timestamp

Fix UTC handling of timestamp() conversation in fetch_my_trades

freqtrade/data/btanalysis.py

@@ -7,7 +7,7 @@ from typing import Dict
 
 import numpy as np
 import pandas as pd
-import pytz
+from datetime import timezone
 
 from freqtrade import persistence
 from freqtrade.misc import json_load
@@ -106,8 +106,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
                "stop_loss", "initial_stop_loss", "strategy", "ticker_interval"]
 
     trades = pd.DataFrame([(t.pair,
-                            t.open_date.replace(tzinfo=pytz.UTC),
-                            t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None,
+                            t.open_date.replace(tzinfo=timezone.utc),
+                            t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
                             t.calc_profit(), t.calc_profit_percent(),
                             t.open_rate, t.close_rate, t.amount,
                             (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)

freqtrade/data/history.py

@@ -9,12 +9,11 @@ Includes:
 import logging
 import operator
 from copy import deepcopy
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
-import pytz
 from pandas import DataFrame
 
 from freqtrade import OperationalException, misc
@@ -56,10 +55,10 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame:
     Trim dataframe based on given timerange
     """
     if timerange.starttype == 'date':
-        start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc)
+        start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
         df = df.loc[df['date'] >= start, :]
     if timerange.stoptype == 'date':
-        stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc)
+        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
         df = df.loc[df['date'] <= stop, :]
     return df
 

freqtrade/exchange/exchange.py

@@ -875,6 +875,22 @@ class Exchange:
 
     @retrier
     def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List:
+        """
+        Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id.
+        The "since" argument passed in is coming from the database and is in UTC,
+        as timezone-native datetime object.
+        From the python documentation:
+            > Naive datetime instances are assumed to represent local time
+        Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the
+        transformation from local timezone to UTC.
+        This works for timezones UTC+ since then the result will contain trades from a few hours
+        instead of from the last 5 seconds, however fails for UTC- timezones,
+        since we're then asking for trades with a "since" argument in the future.
+
+        :param order_id order_id: Order-id as given when creating the order
+        :param pair: Pair the order is for
+        :param since: datetime object of the order creation time. Assumes object is in UTC.
+        """
         if self._config['dry_run']:
             return []
         if not self.exchange_has('fetchMyTrades'):
@@ -882,7 +898,8 @@ class Exchange:
         try:
             # Allow 5s offset to catch slight time offsets (discovered in #1185)
             # since needs to be int in milliseconds
-            my_trades = self._api.fetch_my_trades(pair, int((since.timestamp() - 5) * 1000))
+            my_trades = self._api.fetch_my_trades(
+                pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000))
             matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
 
             return matched_trades

tests/exchange/test_exchange.py

@@ -1586,8 +1586,9 @@ def test_name(default_conf, mocker, exchange_name):
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_get_trades_for_order(default_conf, mocker, exchange_name):
+
     order_id = 'ABCD-ABCD'
-    since = datetime(2018, 5, 5, tzinfo=timezone.utc)
+    since = datetime(2018, 5, 5, 0, 0, 0)
     default_conf["dry_run"] = False
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
     api_mock = MagicMock()
@@ -1623,7 +1624,8 @@ def test_get_trades_for_order(default_conf, mocker, exchange_name):
     assert api_mock.fetch_my_trades.call_args[0][0] == 'LTC/BTC'
     # Same test twice, hardcoded number and doing the same calculation
     assert api_mock.fetch_my_trades.call_args[0][1] == 1525478395000
-    assert api_mock.fetch_my_trades.call_args[0][1] == int(since.timestamp() - 5) * 1000
+    assert api_mock.fetch_my_trades.call_args[0][1] == int(since.replace(
+        tzinfo=timezone.utc).timestamp() - 5) * 1000
 
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
                            'get_trades_for_order', 'fetch_my_trades',