From 055229a44a8288b4f32ccd2604d08c1d1e2dd045 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sat, 3 Jul 2021 11:39:14 +0200
Subject: [PATCH 01/35] first iteration of volume pairlist with range lookback

---
 freqtrade/plugins/pairlist/VolumePairList.py | 71 +++++++++++++++++++-
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 8eff137b0..af26201a9 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -11,6 +11,9 @@ from cachetools.ttl import TTLCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
+import arrow
+from copy import deepcopy
+from freqtrade.exchange import timeframe_to_minutes
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +39,25 @@ class VolumePairList(IPairList):
         self._min_value = self._pairlistconfig.get('min_value', 0)
         self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
         self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
+        self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
+        self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
+
+        # overwrite lookback timeframe and days when lookback_days is set
+        if self._lookback_days > 0:
+            self._lookback_timeframe = '1d'
+            self._lookback_period = self._lookback_days
+
+        self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
+        self._tf_in_secs = self._tf_in_min * 60
+
+        self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
+
+        if self._use_range & (self._refresh_period < self._tf_in_secs):
+            raise OperationalException(
+                f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
+                f'Please adjust refresh_period to at least {self._tf_in_secs} and restart the bot.'
+            )
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -47,6 +69,14 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
+
+        if self._lookback_period < 0:
+            raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
+        if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
+            raise OperationalException("VolumeFilter requires lookback_period to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
+
     @property
     def needstickers(self) -> bool:
         """
@@ -78,7 +108,6 @@ class VolumePairList(IPairList):
             # Item found - no refresh necessary
             return pairlist
         else:
-
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
             filtered_tickers = [
@@ -100,9 +129,44 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        # Use the incoming pairlist.
+
+        # Use the incoming pairlist.      
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
+        if self._use_range == True:
+            since_ms = int(arrow.utcnow()
+                        .floor('minute')
+                        .shift(minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
+                        .float_timestamp) * 1000
+
+            self.log_once(f"Using volume range of {self._lookback_period} {self._lookback_timeframe} candles from {since_ms}", logger.info)
+            needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
+            # Get all candles
+            candles = {}
+            if needed_pairs:
+                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
+                                                            cache=False)
+
+            for i,p in enumerate(filtered_tickers):
+            # for p in deepcopy([s['symbol'] for s in filtered_tickers]):
+                pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
+                #print(p['symbol'], " 24h quote volume   = ",filtered_tickers[i]['quoteVolume'])
+
+                #if p['symbol'] == 'BCC/USDT':
+                    #print(pair_candles)
+                    #quit()
+
+                print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
+                if not pair_candles.empty:
+                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
+                    pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
+                    # print(p['symbol'], " range quote volume = ", pair_candles['quoteVolume'].sum())
+                    filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
+                else:
+                    filtered_tickers[i]['quoteVolume'] = 0
+
+                print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
+
         if self._min_value > 0:
             filtered_tickers = [
                     v for v in filtered_tickers if v[self._sort_key] > self._min_value]
@@ -112,9 +176,12 @@ class VolumePairList(IPairList):
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
         pairs = self.verify_blacklist(pairs, logger.info)
+        
         # Limit pairlist to the requested number of pairs
+        
         pairs = pairs[:self._number_pairs]
 
         self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
 
+        quit()
         return pairs

From 62da4b452ccbab16044adc03f9c74d2821710d13 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sat, 3 Jul 2021 11:47:17 +0200
Subject: [PATCH 02/35] code cleanup and comments

---
 freqtrade/plugins/pairlist/VolumePairList.py | 36 +++++++-------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index af26201a9..ae6be54ba 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -4,16 +4,14 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
+import arrow
 from typing import Any, Dict, List
 
 from cachetools.ttl import TTLCache
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.plugins.pairlist.IPairList import IPairList
-
-import arrow
-from copy import deepcopy
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.IPairList import IPairList
 
 logger = logging.getLogger(__name__)
 
@@ -48,15 +46,16 @@ class VolumePairList(IPairList):
             self._lookback_timeframe = '1d'
             self._lookback_period = self._lookback_days
 
+        # get timeframe in minutes and seconds 
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
-        self._tf_in_secs = self._tf_in_min * 60
+        self._tf_in_sec = self._tf_in_min * 60
 
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
         if self._use_range & (self._refresh_period < self._tf_in_secs):
             raise OperationalException(
                 f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
-                f'Please adjust refresh_period to at least {self._tf_in_secs} and restart the bot.'
+                f'Please adjust refresh_period to at least {self._tf_in_sec} and restart the bot.'
             )
 
         if not self._exchange.exchange_has('fetchTickers'):
@@ -129,10 +128,10 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-
         # Use the incoming pairlist.      
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
+        # get lookback period in ms, for exchange ohlcv fetch
         if self._use_range == True:
             since_ms = int(arrow.utcnow()
                         .floor('minute')
@@ -141,31 +140,25 @@ class VolumePairList(IPairList):
 
             self.log_once(f"Using volume range of {self._lookback_period} {self._lookback_timeframe} candles from {since_ms}", logger.info)
             needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
+
             # Get all candles
             candles = {}
             if needed_pairs:
-                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
-                                                            cache=False)
+                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
 
             for i,p in enumerate(filtered_tickers):
-            # for p in deepcopy([s['symbol'] for s in filtered_tickers]):
                 pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
-                #print(p['symbol'], " 24h quote volume   = ",filtered_tickers[i]['quoteVolume'])
-
-                #if p['symbol'] == 'BCC/USDT':
-                    #print(pair_candles)
-                    #quit()
-
-                print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
+                # print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
+                # in case of candle data calculate typical price and quoteVolume for candle
                 if not pair_candles.empty:
                     pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
                     pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
-                    # print(p['symbol'], " range quote volume = ", pair_candles['quoteVolume'].sum())
+                    # replace quoteVolume with range sum
                     filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
                 else:
                     filtered_tickers[i]['quoteVolume'] = 0
 
-                print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
+                # print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
 
         if self._min_value > 0:
             filtered_tickers = [
@@ -175,13 +168,10 @@ class VolumePairList(IPairList):
 
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        pairs = self.verify_blacklist(pairs, logger.info)
-        
+        pairs = self.verify_blacklist(pairs, logger.info)       
         # Limit pairlist to the requested number of pairs
-        
         pairs = pairs[:self._number_pairs]
 
         self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
 
-        quit()
         return pairs

From 53f963dd736188c2cc5efe27a52ccbb40c2af12c Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sat, 3 Jul 2021 11:49:05 +0200
Subject: [PATCH 03/35] fixed `self._tf_in_secs` to `self._tf_in_sec`

---
 freqtrade/plugins/pairlist/VolumePairList.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index ae6be54ba..4e9102243 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -52,7 +52,7 @@ class VolumePairList(IPairList):
 
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
-        if self._use_range & (self._refresh_period < self._tf_in_secs):
+        if self._use_range & (self._refresh_period < self._tf_in_sec):
             raise OperationalException(
                 f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
                 f'Please adjust refresh_period to at least {self._tf_in_sec} and restart the bot.'

From 3d9f3eeb07bc67e08d7ce9b3ba1506a422dae774 Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Sat, 3 Jul 2021 23:58:04 +0700
Subject: [PATCH 04/35] feat(agefilter): add max_days_listed

---
 config_full.json.example                |  2 +-
 docs/includes/pairlists.md              |  4 +--
 freqtrade/plugins/pairlist/AgeFilter.py | 17 +++++++++---
 tests/plugins/test_pairlist.py          | 37 +++++++++++++++++++++----
 4 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/config_full.json.example b/config_full.json.example
index d404391a4..6df4a8253 100644
--- a/config_full.json.example
+++ b/config_full.json.example
@@ -67,7 +67,7 @@
             "sort_key": "quoteVolume",
             "refresh_period": 1800
         },
-        {"method": "AgeFilter", "min_days_listed": 10},
+        {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300},
         {"method": "PrecisionFilter"},
         {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010},
         {"method": "SpreadFilter", "max_spread_ratio": 0.005},
diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index f19c5a181..b07bde62f 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -87,7 +87,7 @@ When pairs are first listed on an exchange they can suffer huge price drops and
 in the first few days while the pair goes through its price-discovery period. Bots can often
 be caught out buying before the pair has finished dropping in price.
 
-This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days.
+This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`.
 
 #### PerformanceFilter
 
@@ -212,7 +212,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
         "number_assets": 20,
         "sort_key": "quoteVolume"
     },
-    {"method": "AgeFilter", "min_days_listed": 10},
+    {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300},
     {"method": "PrecisionFilter"},
     {"method": "PriceFilter", "low_price_ratio": 0.01},
     {"method": "SpreadFilter", "max_spread_ratio": 0.005},
diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 8f623b062..56643cca8 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -27,6 +27,7 @@ class AgeFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
+        self._max_days_listed = pairlistconfig.get('max_days_listed', None)
 
         if self._min_days_listed < 1:
             raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
@@ -34,6 +35,8 @@ class AgeFilter(IPairList):
             raise OperationalException("AgeFilter requires min_days_listed to not exceed "
                                        "exchange max request size "
                                        f"({exchange.ohlcv_candle_limit('1d')})")
+        if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
+            raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
 
     @property
     def needstickers(self) -> bool:
@@ -49,7 +52,9 @@ class AgeFilter(IPairList):
         Short whitelist method description - used for startup-messages
         """
         return (f"{self.name} - Filtering pairs with age less than "
-                f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
+                f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
+                " or more than "
+                f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}")
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
@@ -61,9 +66,10 @@ class AgeFilter(IPairList):
         if not needed_pairs:
             return pairlist
 
+        since_days = -(self._max_days_listed if self._max_days_listed else self._min_days_listed) - 1
         since_ms = int(arrow.utcnow()
                        .floor('day')
-                       .shift(days=-self._min_days_listed - 1)
+                       .shift(days=since_days)
                        .float_timestamp) * 1000
         candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
         if self._enabled:
@@ -86,7 +92,8 @@ class AgeFilter(IPairList):
             return True
 
         if daily_candles is not None:
-            if len(daily_candles) >= self._min_days_listed:
+            if len(daily_candles) >= self._min_days_listed and \
+                    len(daily_candles) <= self._max_days_listed:
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
@@ -94,6 +101,8 @@ class AgeFilter(IPairList):
             else:
                 self.log_once(f"Removed {pair} from whitelist, because age "
                               f"{len(daily_candles)} is less than {self._min_days_listed} "
-                              f"{plural(self._min_days_listed, 'day')}", logger.info)
+                              f"{plural(self._min_days_listed, 'day')} or more than "
+                              f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}",
+                              logger.info)
                 return False
         return False
diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index ae8f6e958..03d8fc563 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -79,7 +79,8 @@ def whitelist_conf_agefilter(default_conf):
         },
         {
             "method": "AgeFilter",
-            "min_days_listed": 2
+            "min_days_listed": 2,
+            "max_days_listed": 100
         }
     ]
     return default_conf
@@ -302,7 +303,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     # No pair for ETH, all handlers
     ([{"method": "StaticPairList"},
       {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "AgeFilter", "min_days_listed": 2},
+      {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": None},
       {"method": "PrecisionFilter"},
       {"method": "PriceFilter", "low_price_ratio": 0.03},
       {"method": "SpreadFilter", "max_spread_ratio": 0.005},
@@ -310,11 +311,15 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
      "ETH", []),
     # AgeFilter and VolumePairList (require 2 days only, all should pass age test)
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "AgeFilter", "min_days_listed": 2}],
+      {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": 100}],
      "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
     # AgeFilter and VolumePairList (require 10 days, all should fail age test)
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "AgeFilter", "min_days_listed": 10}],
+      {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": None}],
+     "BTC", []),
+    # AgeFilter and VolumePairList (all pair listed > 2, all should fail age test)
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}],
      "BTC", []),
     # Precisionfilter and quote volume
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
@@ -480,9 +485,13 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
 
         for pairlist in pairlists:
             if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
-                    len(ohlcv_history) <= pairlist['min_days_listed']:
+                    len(ohlcv_history) < pairlist['min_days_listed']:
                 assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
-                                  r'.* day.*', caplog)
+                                  r'.* day.* or more than .* day', caplog)
+            if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \
+                    len(ohlcv_history) > pairlist['max_days_listed']:
+                assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
+                                  r'.* day.* or more than .* day', caplog)
             if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
                 assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
                                   r'would be <= stop limit.*', caplog)
@@ -650,6 +659,22 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick
         get_patched_freqtradebot(mocker, default_conf)
 
 
+def test_agefilter_max_days_lower_than_min_days(mocker, default_conf, markets, tickers):
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'AgeFilter', 'min_days_listed': 3,
+                                 "max_days_listed": 2}]
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    with pytest.raises(OperationalException,
+                       match=r'AgeFilter max_days_listed <= min_days_listed not permitted'):
+        get_patched_freqtradebot(mocker, default_conf)
+
+
 def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'AgeFilter', 'min_days_listed': 99999}]

From b72bbebccbe5bec7ad920122b1cd7bdfa1541246 Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Sun, 4 Jul 2021 01:46:51 +0700
Subject: [PATCH 05/35] fix flake8

---
 freqtrade/plugins/pairlist/AgeFilter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 56643cca8..e792e0343 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -66,7 +66,9 @@ class AgeFilter(IPairList):
         if not needed_pairs:
             return pairlist
 
-        since_days = -(self._max_days_listed if self._max_days_listed else self._min_days_listed) - 1
+        since_days = -(
+            self._max_days_listed if self._max_days_listedelse else self._min_days_listed
+        ) - 1
         since_ms = int(arrow.utcnow()
                        .floor('day')
                        .shift(days=since_days)

From f6511c3e3f75597cdc725e4ba9c4cb078291e680 Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Sun, 4 Jul 2021 02:20:53 +0700
Subject: [PATCH 06/35] fix typo and add blocker

---
 freqtrade/plugins/pairlist/AgeFilter.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index e792e0343..63a9ecfeb 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -37,6 +37,10 @@ class AgeFilter(IPairList):
                                        f"({exchange.ohlcv_candle_limit('1d')})")
         if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
             raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
+        if self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
+            raise OperationalException("AgeFilter requires max_days_listed to not exceed "
+                                       "exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
 
     @property
     def needstickers(self) -> bool:
@@ -67,7 +71,7 @@ class AgeFilter(IPairList):
             return pairlist
 
         since_days = -(
-            self._max_days_listed if self._max_days_listedelse else self._min_days_listed
+            self._max_days_listed if self._max_days_listed else self._min_days_listed
         ) - 1
         since_ms = int(arrow.utcnow()
                        .floor('day')

From 348dbeff3f99e848666e7b5242ea02e205c73a64 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 11:16:33 +0200
Subject: [PATCH 07/35] added meaningful logging of used lookback range

---
 freqtrade/plugins/pairlist/VolumePairList.py | 23 +++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 4e9102243..5820bc667 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -12,6 +12,7 @@ from cachetools.ttl import TTLCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.misc import format_ms_time
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +42,12 @@ class VolumePairList(IPairList):
         self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
         self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
 
+        if (self._lookback_days > 0) & (self._lookback_period > 0):
+            raise OperationalException(
+                f'Ambigous configuration: lookback_days and lookback_period both set in pairlist config. '
+                f'Please set lookback_days only or lookback_period and lookback_timeframe and restart the bot.'
+            )
+
         # overwrite lookback timeframe and days when lookback_days is set
         if self._lookback_days > 0:
             self._lookback_timeframe = '1d'
@@ -49,7 +56,8 @@ class VolumePairList(IPairList):
         # get timeframe in minutes and seconds 
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
         self._tf_in_sec = self._tf_in_min * 60
-
+        
+        # wether to use range lookback or not
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
         if self._use_range & (self._refresh_period < self._tf_in_sec):
@@ -138,7 +146,13 @@ class VolumePairList(IPairList):
                         .shift(minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
                         .float_timestamp) * 1000
 
-            self.log_once(f"Using volume range of {self._lookback_period} {self._lookback_timeframe} candles from {since_ms}", logger.info)
+            to_ms = int(arrow.utcnow()
+                        .floor('minute')
+                        .shift(minutes=-self._tf_in_min)
+                        .float_timestamp) * 1000
+
+            # todo: utc date output for starting date
+            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: {self._lookback_timeframe}, starting from {format_ms_time(since_ms)} till {format_ms_time(to_ms)}", logger.info)
             needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
 
             # Get all candles
@@ -148,18 +162,15 @@ class VolumePairList(IPairList):
 
             for i,p in enumerate(filtered_tickers):
                 pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
-                # print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
                 # in case of candle data calculate typical price and quoteVolume for candle
                 if not pair_candles.empty:
                     pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
                     pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
-                    # replace quoteVolume with range sum
+                    # replace quoteVolume with range quoteVolume sum calculated above
                     filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
                 else:
                     filtered_tickers[i]['quoteVolume'] = 0
 
-                # print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
-
         if self._min_value > 0:
             filtered_tickers = [
                     v for v in filtered_tickers if v[self._sort_key] > self._min_value]

From 9919061c7899e7e7e5d9c4a0ef59d78152a6a9ec Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 11:40:45 +0200
Subject: [PATCH 08/35] PEP8 compliance

---
 freqtrade/plugins/pairlist/VolumePairList.py | 62 +++++++++++++-------
 1 file changed, 40 insertions(+), 22 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 5820bc667..40e6afa07 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -4,15 +4,16 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
-import arrow
 from typing import Any, Dict, List
 
+import arrow
 from cachetools.ttl import TTLCache
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
-from freqtrade.plugins.pairlist.IPairList import IPairList
 from freqtrade.misc import format_ms_time
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +45,9 @@ class VolumePairList(IPairList):
 
         if (self._lookback_days > 0) & (self._lookback_period > 0):
             raise OperationalException(
-                f'Ambigous configuration: lookback_days and lookback_period both set in pairlist config. '
-                f'Please set lookback_days only or lookback_period and lookback_timeframe and restart the bot.'
+                'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
+                'config. Please set lookback_days only or lookback_period and lookback_timeframe '
+                'and restart the bot.'
             )
 
         # overwrite lookback timeframe and days when lookback_days is set
@@ -53,17 +55,18 @@ class VolumePairList(IPairList):
             self._lookback_timeframe = '1d'
             self._lookback_period = self._lookback_days
 
-        # get timeframe in minutes and seconds 
+        # get timeframe in minutes and seconds
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
         self._tf_in_sec = self._tf_in_min * 60
-        
+
         # wether to use range lookback or not
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
         if self._use_range & (self._refresh_period < self._tf_in_sec):
             raise OperationalException(
-                f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
-                f'Please adjust refresh_period to at least {self._tf_in_sec} and restart the bot.'
+                f'Refresh period of {self._refresh_period} seconds is smaller than one '
+                f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
+                f'to at least {self._tf_in_sec} and restart the bot.'
             )
 
         if not self._exchange.exchange_has('fetchTickers'):
@@ -76,7 +79,6 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
-
         if self._lookback_period < 0:
             raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
         if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
@@ -136,15 +138,16 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        # Use the incoming pairlist.      
+        # Use the incoming pairlist.
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
         # get lookback period in ms, for exchange ohlcv fetch
-        if self._use_range == True:
+        if self._use_range:
             since_ms = int(arrow.utcnow()
-                        .floor('minute')
-                        .shift(minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
-                        .float_timestamp) * 1000
+                           .floor('minute')
+                           .shift(minutes=-(self._lookback_period * self._tf_in_min)
+                                  - self._tf_in_min)
+                           .float_timestamp) * 1000
 
             to_ms = int(arrow.utcnow()
                         .floor('minute')
@@ -152,20 +155,35 @@ class VolumePairList(IPairList):
                         .float_timestamp) * 1000
 
             # todo: utc date output for starting date
-            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: {self._lookback_timeframe}, starting from {format_ms_time(since_ms)} till {format_ms_time(to_ms)}", logger.info)
-            needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
+            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
+                          f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
+                          f"till {format_ms_time(to_ms)}", logger.info)
+            needed_pairs = [
+                (p, self._lookback_timeframe) for p in
+                [
+                    s['symbol'] for s in filtered_tickers
+                ] if p not in self._pair_cache
+            ]
 
             # Get all candles
             candles = {}
             if needed_pairs:
-                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
+                candles = self._exchange.refresh_latest_ohlcv(
+                    needed_pairs, since_ms=since_ms, cache=False
+                )
 
-            for i,p in enumerate(filtered_tickers):
-                pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
+            for i, p in enumerate(filtered_tickers):
+                pair_candles = candles[
+                    (p['symbol'], self._lookback_timeframe)
+                ] if (p['symbol'], self._lookback_timeframe) in candles else None
                 # in case of candle data calculate typical price and quoteVolume for candle
                 if not pair_candles.empty:
-                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
-                    pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
+                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+                                                     + pair_candles['close']) / 3
+                    pair_candles['quoteVolume'] = (
+                        pair_candles['volume'] * pair_candles['typical_price']
+                    )
+
                     # replace quoteVolume with range quoteVolume sum calculated above
                     filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
                 else:
@@ -179,7 +197,7 @@ class VolumePairList(IPairList):
 
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        pairs = self.verify_blacklist(pairs, logger.info)       
+        pairs = self.verify_blacklist(pairs, logger.info)
         # Limit pairlist to the requested number of pairs
         pairs = pairs[:self._number_pairs]
 

From 2d5ced780116088aca71216c29683021754b22dd Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Sun, 4 Jul 2021 21:59:59 +0700
Subject: [PATCH 09/35] fix testcase

---
 config_full.json.example                | 2 +-
 docs/includes/pairlists.md              | 4 ++--
 freqtrade/plugins/pairlist/AgeFilter.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/config_full.json.example b/config_full.json.example
index 6df4a8253..d404391a4 100644
--- a/config_full.json.example
+++ b/config_full.json.example
@@ -67,7 +67,7 @@
             "sort_key": "quoteVolume",
             "refresh_period": 1800
         },
-        {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300},
+        {"method": "AgeFilter", "min_days_listed": 10},
         {"method": "PrecisionFilter"},
         {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010},
         {"method": "SpreadFilter", "max_spread_ratio": 0.005},
diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index b07bde62f..908eaa459 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -81,7 +81,7 @@ Filtering instances (not the first position in the list) will not apply any cach
 
 #### AgeFilter
 
-Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`).
+Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
 
 When pairs are first listed on an exchange they can suffer huge price drops and volatility
 in the first few days while the pair goes through its price-discovery period. Bots can often
@@ -212,7 +212,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
         "number_assets": 20,
         "sort_key": "quoteVolume"
     },
-    {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": 7300},
+    {"method": "AgeFilter", "min_days_listed": 10},
     {"method": "PrecisionFilter"},
     {"method": "PriceFilter", "low_price_ratio": 0.01},
     {"method": "SpreadFilter", "max_spread_ratio": 0.005},
diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 63a9ecfeb..ef3953776 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -37,7 +37,7 @@ class AgeFilter(IPairList):
                                        f"({exchange.ohlcv_candle_limit('1d')})")
         if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
             raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
-        if self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
+        if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
             raise OperationalException("AgeFilter requires max_days_listed to not exceed "
                                        "exchange max request size "
                                        f"({exchange.ohlcv_candle_limit('1d')})")

From 85c7b557503d1c25330101266ab898642ca41272 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 20:46:24 +0200
Subject: [PATCH 10/35] improvements: - `float_timestamp` switched to
 `int_timestamp` - added documentation to pairlists.md

---
 docs/includes/pairlists.md                   | 31 +++++++++++++++++++-
 freqtrade/plugins/pairlist/VolumePairList.py |  4 +--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index f19c5a181..3f8baab29 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -63,7 +63,7 @@ The `refresh_period` setting allows to define the period (in seconds), at which
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
 Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
 
-`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
+`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
 * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
 
@@ -76,6 +76,35 @@ Filtering instances (not the first position in the list) will not apply any cach
 }],
 ```
 
+`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3), and multiplies it with every candle data's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
+
+For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
+
+```json
+"pairlists": [{
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 86400,
+        "lookback_days": 7
+}],
+```
+!!! Warning "Range look back and refresh period"
+    When used in conjuction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
+
+More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
+
+```json
+"pairlists": [{
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 3600,
+        "lookback_timeframe": "1h",
+        "lookback_timeframe": 72
+}],
+```
+
 !!! Note
     `VolumePairList` does not support backtesting mode.
 
diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 40e6afa07..352d028ac 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -147,12 +147,12 @@ class VolumePairList(IPairList):
                            .floor('minute')
                            .shift(minutes=-(self._lookback_period * self._tf_in_min)
                                   - self._tf_in_min)
-                           .float_timestamp) * 1000
+                           .int_timestamp) * 1000
 
             to_ms = int(arrow.utcnow()
                         .floor('minute')
                         .shift(minutes=-self._tf_in_min)
-                        .float_timestamp) * 1000
+                        .int_timestamp) * 1000
 
             # todo: utc date output for starting date
             self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "

From 7ac55e5415f94d7db285c20d093a902b039d05de Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 21:08:42 +0200
Subject: [PATCH 11/35] AgeFilter, RangeStabilityFilter, VolatilityFilter
 changed `float_timestamp` to `int_timestamp`

---
 freqtrade/plugins/pairlist/AgeFilter.py            | 4 ++--
 freqtrade/plugins/pairlist/VolatilityFilter.py     | 2 +-
 freqtrade/plugins/pairlist/rangestabilityfilter.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 8f623b062..09d8588c1 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -64,7 +64,7 @@ class AgeFilter(IPairList):
         since_ms = int(arrow.utcnow()
                        .floor('day')
                        .shift(days=-self._min_days_listed - 1)
-                       .float_timestamp) * 1000
+                       .int_timestamp) * 1000
         candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
         if self._enabled:
             for p in deepcopy(pairlist):
@@ -89,7 +89,7 @@ class AgeFilter(IPairList):
             if len(daily_candles) >= self._min_days_listed:
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
-                self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
+                self._symbolsChecked[pair] = int(arrow.utcnow().int_timestamp) * 1000
                 return True
             else:
                 self.log_once(f"Removed {pair} from whitelist, because age "
diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py
index 5ae8e3e9f..a50bf55b9 100644
--- a/freqtrade/plugins/pairlist/VolatilityFilter.py
+++ b/freqtrade/plugins/pairlist/VolatilityFilter.py
@@ -72,7 +72,7 @@ class VolatilityFilter(IPairList):
         since_ms = int(arrow.utcnow()
                        .floor('day')
                        .shift(days=-self._days - 1)
-                       .float_timestamp) * 1000
+                       .int_timestamp) * 1000
         # Get all candles
         candles = {}
         if needed_pairs:
diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py
index 8be61166b..6f013c750 100644
--- a/freqtrade/plugins/pairlist/rangestabilityfilter.py
+++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py
@@ -65,7 +65,7 @@ class RangeStabilityFilter(IPairList):
         since_ms = int(arrow.utcnow()
                        .floor('day')
                        .shift(days=-self._days - 1)
-                       .float_timestamp) * 1000
+                       .int_timestamp) * 1000
         # Get all candles
         candles = {}
         if needed_pairs:

From 0c8afea38271c2519be61179a09160ef6f1806fc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Jul 2021 04:30:54 +0000
Subject: [PATCH 12/35] Bump pandas from 1.2.5 to 1.3.0

Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.2.5 to 1.3.0.
- [Release notes](https://github.com/pandas-dev/pandas/releases)
- [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md)
- [Commits](https://github.com/pandas-dev/pandas/compare/v1.2.5...v1.3.0)

---
updated-dependencies:
- dependency-name: pandas
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index e1c784608..0003ec031 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 numpy==1.21.0
-pandas==1.2.5
+pandas==1.3.0
 
 ccxt==1.52.40
 # Pin cryptography for now due to rust build errors with piwheels

From 5626ca5a06dcbc29cad9a925945351765188ace1 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Mon, 5 Jul 2021 10:39:22 +0200
Subject: [PATCH 13/35] removed unnecessary casting to int()

---
 freqtrade/plugins/pairlist/AgeFilter.py            | 8 ++++----
 freqtrade/plugins/pairlist/VolatilityFilter.py     | 8 ++++----
 freqtrade/plugins/pairlist/rangestabilityfilter.py | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 09d8588c1..744b1268d 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -61,10 +61,10 @@ class AgeFilter(IPairList):
         if not needed_pairs:
             return pairlist
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._min_days_listed - 1)
-                       .int_timestamp) * 1000
+        since_ms = (arrow.utcnow()
+                         .floor('day')
+                         .shift(days=-self._min_days_listed - 1)
+                         .int_timestamp) * 1000
         candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
         if self._enabled:
             for p in deepcopy(pairlist):
diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py
index a50bf55b9..9383e5d06 100644
--- a/freqtrade/plugins/pairlist/VolatilityFilter.py
+++ b/freqtrade/plugins/pairlist/VolatilityFilter.py
@@ -69,10 +69,10 @@ class VolatilityFilter(IPairList):
         """
         needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._days - 1)
-                       .int_timestamp) * 1000
+        since_ms = (arrow.utcnow()
+                         .floor('day')
+                         .shift(days=-self._days - 1)
+                         .int_timestamp) * 1000
         # Get all candles
         candles = {}
         if needed_pairs:
diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py
index 6f013c750..a6d1820de 100644
--- a/freqtrade/plugins/pairlist/rangestabilityfilter.py
+++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py
@@ -62,10 +62,10 @@ class RangeStabilityFilter(IPairList):
         """
         needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
 
-        since_ms = int(arrow.utcnow()
-                       .floor('day')
-                       .shift(days=-self._days - 1)
-                       .int_timestamp) * 1000
+        since_ms = (arrow.utcnow()
+                         .floor('day')
+                         .shift(days=-self._days - 1)
+                         .int_timestamp) * 1000
         # Get all candles
         candles = {}
         if needed_pairs:

From 346d66748b65cf833696bc342d4c08582ad88e1a Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Mon, 5 Jul 2021 12:50:56 +0200
Subject: [PATCH 14/35] first version of OffsetFilter

---
 freqtrade/constants.py                     |  6 +--
 freqtrade/plugins/pairlist/OffsetFilter.py | 54 ++++++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)
 create mode 100644 freqtrade/plugins/pairlist/OffsetFilter.py

diff --git a/freqtrade/constants.py b/freqtrade/constants.py
index f4c32387b..acd143708 100644
--- a/freqtrade/constants.py
+++ b/freqtrade/constants.py
@@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
                          'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
-                       'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
-                       'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
-                       'SpreadFilter', 'VolatilityFilter']
+                       'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
+                       'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
+                       'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000
diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py
new file mode 100644
index 000000000..4579204d9
--- /dev/null
+++ b/freqtrade/plugins/pairlist/OffsetFilter.py
@@ -0,0 +1,54 @@
+"""
+Offset pair list filter
+"""
+import logging
+from typing import Any, Dict, List
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class OffsetFilter(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._offset = pairlistconfig.get('offset', 0)
+
+        if self._offset < 0:
+            raise OperationalException("OffsetFilter requires offset to be >= 0")
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty Dict is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - Offseting pairs by {self._offset}."
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new whitelist
+        """
+        if self._offset > len(pairlist):
+            self.log_once(f"Offset of {self._offset} is larger than " +
+                          f"pair count of {len(pairlist)}", logger.warn)
+        pairs = pairlist[self._offset:]
+        self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
+        return pairs

From 10998eb0faa8da854b777bdf2b4605caa03dfb7c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 5 Jul 2021 19:51:14 +0200
Subject: [PATCH 15/35] Remove further usages of int(int_timestamp)

---
 freqtrade/data/history/history_utils.py | 6 +++---
 freqtrade/exchange/exchange.py          | 2 +-
 freqtrade/plugins/pairlist/AgeFilter.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py
index eecb63d07..1459dfd78 100644
--- a/freqtrade/data/history/history_utils.py
+++ b/freqtrade/data/history/history_utils.py
@@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path,
         new_data = exchange.get_historic_ohlcv(pair=pair,
                                                timeframe=timeframe,
                                                since_ms=since_ms if since_ms else
-                                               int(arrow.utcnow().shift(
-                                                   days=-new_pairs_days).float_timestamp) * 1000
+                                               arrow.utcnow().shift(
+                                                   days=-new_pairs_days).int_timestamp * 1000
                                                )
         # TODO: Maybe move parsing to exchange class (?)
         new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -272,7 +272,7 @@ def _download_trades_history(exchange: Exchange,
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
         else:
-            since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
+            since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
 
         trades = data_handler.trades_load(pair)
 
diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 235f03269..42e86db3e 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -578,7 +578,7 @@ class Exchange:
             'side': side,
             'remaining': _amount,
             'datetime': arrow.utcnow().isoformat(),
-            'timestamp': int(arrow.utcnow().int_timestamp * 1000),
+            'timestamp': arrow.utcnow().int_timestamp * 1000,
             'status': "closed" if ordertype == "market" else "open",
             'fee': None,
             'info': {}
diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 744b1268d..4c364bfce 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -89,7 +89,7 @@ class AgeFilter(IPairList):
             if len(daily_candles) >= self._min_days_listed:
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
-                self._symbolsChecked[pair] = int(arrow.utcnow().int_timestamp) * 1000
+                self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
                 return True
             else:
                 self.log_once(f"Removed {pair} from whitelist, because age "

From 1e87225e916283fc27f18efb8f1d605b84d0cecd Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Mon, 5 Jul 2021 20:59:27 +0200
Subject: [PATCH 16/35] added `test_VolumePairList_range` to test_pairlist.py

---
 freqtrade/plugins/pairlist/VolumePairList.py |  3 +-
 tests/plugins/test_pairlist.py               | 95 ++++++++++++++++++++
 2 files changed, 96 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 352d028ac..14b9d7024 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -171,13 +171,12 @@ class VolumePairList(IPairList):
                 candles = self._exchange.refresh_latest_ohlcv(
                     needed_pairs, since_ms=since_ms, cache=False
                 )
-
             for i, p in enumerate(filtered_tickers):
                 pair_candles = candles[
                     (p['symbol'], self._lookback_timeframe)
                 ] if (p['symbol'], self._lookback_timeframe) in candles else None
                 # in case of candle data calculate typical price and quoteVolume for candle
-                if not pair_candles.empty:
+                if pair_candles is not None and not pair_candles.empty:
                     pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
                                                      + pair_candles['close']) / 3
                     pair_candles['quoteVolume'] = (
diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index ae8f6e958..7812ee733 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -507,6 +507,101 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                 assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog)
 
 
+@pytest.mark.parametrize("pairlists,base_currency,volumefilter_result", [
+    # default refresh of 1800 to small for daily candle lookback
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_days": 1}],
+     "BTC", "default_refresh_too_short"),  # OperationalException expected
+    # ambigous configuration with lookback days and period
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_days": 1, "lookback_period": 1}],
+     "BTC", "lookback_days_and_period"),  # OperationalException expected
+    # negative lookback period
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": -1}],
+     "BTC", "lookback_period_negative"),  # OperationalException expected
+    # lookback range exceedes exchange limit
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}],
+     "BTC", 'lookback_exceeds_exchange_request_size'),  # OperationalException expected
+    # expecing pairs as given
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
+     "BTC", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
+    # expecting pairs from default tickers, because 1h candles are not available
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']),
+])
+def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
+                              pairlists, base_currency, volumefilter_result, caplog) -> None:
+    whitelist_conf['pairlists'] = pairlists
+    whitelist_conf['stake_currency'] = base_currency
+
+    ohlcv_history_high_vola = ohlcv_history.copy()
+    ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
+
+    # create candles for high volume
+    ohlcv_history_high_volume = ohlcv_history.copy()
+    ohlcv_history_high_volume.loc[ohlcv_history_high_volume.index == 1, 'volume'] = 10
+
+    ohlcv_data = {
+        ('ETH/BTC', '1d'): ohlcv_history,
+        ('TKN/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history_high_volume,
+        ('XRP/BTC', '1d'): ohlcv_history_high_vola,
+        ('HOT/BTC', '1d'): ohlcv_history,
+    }
+
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    if volumefilter_result == 'default_refresh_too_short':
+        with pytest.raises(OperationalException,
+                           match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe '
+                                 r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ '
+                                 r'and restart the bot\.'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        return
+    elif volumefilter_result == 'lookback_days_and_period':
+        with pytest.raises(OperationalException,
+                           match=r'Ambigous configuration: lookback_days and lookback_period both '
+                                 r'set in pairlist config\..*'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    elif volumefilter_result == 'lookback_period_negative':
+        with pytest.raises(OperationalException,
+                           match=r'VolumeFilter requires lookback_period to be >= 0'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    elif volumefilter_result == 'lookback_exceeds_exchange_request_size':
+        with pytest.raises(OperationalException,
+                           match=r'VolumeFilter requires lookback_period to not exceed '
+                           r'exchange max request size \([0-9]+\)'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    else:
+        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            get_tickers=tickers,
+            markets=PropertyMock(return_value=shitcoinmarkets)
+        )
+
+        # remove ohlcv when looback_timeframe != 1d
+        # to enforce fallback to ticker data
+        if 'lookback_timeframe' in pairlists[0]:
+            if pairlists[0]['lookback_timeframe'] != '1d':
+                ohlcv_data = []
+
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
+        )
+
+        freqtrade.pairlists.refresh_pairlist()
+        whitelist = freqtrade.pairlists.whitelist
+
+        assert isinstance(whitelist, list)
+        assert whitelist == volumefilter_result
+
+
 def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
     whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}]
     del whitelist_conf['stoploss']

From dec523eef0c384630c325789dd8b6eae83ee2d38 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 6 Jul 2021 07:20:05 +0200
Subject: [PATCH 17/35] Display verison of installed FreqUI

---
 freqtrade/rpc/api_server/web_ui.py | 11 +++++++++++
 tests/rpc/test_rpc_apiserver.py    |  9 +++++++++
 2 files changed, 20 insertions(+)

diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py
index a8c737e04..76c8ed8f2 100644
--- a/freqtrade/rpc/api_server/web_ui.py
+++ b/freqtrade/rpc/api_server/web_ui.py
@@ -18,6 +18,17 @@ async def fallback():
     return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html'))
 
 
+@router_ui.get('/ui_version', include_in_schema=False)
+async def ui_version():
+    from freqtrade.commands.deploy_commands import read_ui_version
+    uibase = Path(__file__).parent / 'ui/installed/'
+    version = read_ui_version(uibase)
+
+    return {
+        "version": version if version else "not_installed",
+    }
+
+
 @router_ui.get('/{rest_of_path:path}', include_in_schema=False)
 async def index_html(rest_of_path: str):
     """
diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py
index b8dd112c9..89da68da7 100644
--- a/tests/rpc/test_rpc_apiserver.py
+++ b/tests/rpc/test_rpc_apiserver.py
@@ -105,6 +105,15 @@ def test_api_ui_fallback(botclient):
     assert rc.status_code == 200
 
 
+def test_api_ui_version(botclient, mocker):
+    ftbot, client = botclient
+
+    mocker.patch('freqtrade.commands.deploy_commands.read_ui_version', return_value='0.1.2')
+    rc = client_get(client, "/ui_version")
+    assert rc.status_code == 200
+    assert rc.json()['version'] == '0.1.2'
+
+
 def test_api_auth():
     with pytest.raises(ValueError):
         create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234', token_type="NotATokenType")

From 502c69dce3b2e9b8d95e87bc293948165b5e34fc Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Tue, 6 Jul 2021 19:36:42 +0700
Subject: [PATCH 18/35] change short desc

---
 freqtrade/plugins/pairlist/AgeFilter.py | 24 +++++++++++++++---------
 tests/plugins/test_pairlist.py          |  2 +-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index ef3953776..c482f6217 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -55,10 +55,13 @@ class AgeFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
-        return (f"{self.name} - Filtering pairs with age less than "
-                f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
-                " or more than "
-                f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}")
+        return (
+            f"{self.name} - Filtering pairs with age less than "
+            f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
+        ) + (
+            " or more than "
+            f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
+        ) if self._max_days_listed else ''
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
@@ -105,10 +108,13 @@ class AgeFilter(IPairList):
                 self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
                 return True
             else:
-                self.log_once(f"Removed {pair} from whitelist, because age "
-                              f"{len(daily_candles)} is less than {self._min_days_listed} "
-                              f"{plural(self._min_days_listed, 'day')} or more than "
-                              f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}",
-                              logger.info)
+                self.log_once((
+                    f"Removed {pair} from whitelist, because age "
+                    f"{len(daily_candles)} is less than {self._min_days_listed} "
+                    f"{plural(self._min_days_listed, 'day')}"
+                ) + (
+                    " or more than "
+                    f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
+                ) if self.max_days_listed else '', logger.info)
                 return False
         return False
diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index 03d8fc563..5724ca39c 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -487,7 +487,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
             if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
                     len(ohlcv_history) < pairlist['min_days_listed']:
                 assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
-                                  r'.* day.* or more than .* day', caplog)
+                                  r'.* day.*', caplog)
             if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \
                     len(ohlcv_history) > pairlist['max_days_listed']:
                 assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '

From d1104bd434e4c60ba3d7f2582c093a0027c65af0 Mon Sep 17 00:00:00 2001
From: octaviusgus <lmlukamilosevic@gmail.com>
Date: Tue, 6 Jul 2021 22:47:39 +0200
Subject: [PATCH 19/35] fix daily profit data and daily profit curve example

---
 freqtrade/optimize/optimize_reports.py              |  2 +-
 freqtrade/templates/strategy_analysis_example.ipynb | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index 89cf70437..5dc8554c4 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -272,7 +272,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
     winning_days = sum(daily_profit > 0)
     draw_days = sum(daily_profit == 0)
     losing_days = sum(daily_profit < 0)
-    daily_profit_list = daily_profit.tolist()
+    daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
 
     return {
         'backtest_best_day': best_rel,
diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb
index f3b0d8d03..99720ae6e 100644
--- a/freqtrade/templates/strategy_analysis_example.ipynb
+++ b/freqtrade/templates/strategy_analysis_example.ipynb
@@ -215,13 +215,18 @@
     "stats = load_backtest_stats(backtest_dir)\n",
     "strategy_stats = stats['strategy'][strategy]\n",
     "\n",
+    "dates = []\n",
+    "profits = []\n",
+    "for date_profit in strategy_stats['daily_profit']:\n",
+    "    dates.append(date_profit[0])\n",
+    "    profits.append(date_profit[1])\n",
+    "\n",
     "equity = 0\n",
     "equity_daily = []\n",
-    "for dp in strategy_stats['daily_profit']:\n",
+    "for daily_profit in profits:\n",
     "    equity_daily.append(equity)\n",
-    "    equity += float(dp)\n",
+    "    equity += float(daily_profit)\n",
     "\n",
-    "dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end'])\n",
     "\n",
     "df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n",
     "\n",

From 3c3772703bdc8897abcf03c5c4ee6bf45c27e802 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 09:46:05 +0200
Subject: [PATCH 20/35] changed quoteVolume to be built over a rolling period
 using lookback_period to avoid pair_candles being larger than requested
 lookback_period

---
 freqtrade/plugins/pairlist/VolumePairList.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 14b9d7024..d6b8aaaa3 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -183,8 +183,15 @@ class VolumePairList(IPairList):
                         pair_candles['volume'] * pair_candles['typical_price']
                     )
 
+                    # ensure that a rolling sum over the lookback_period is built
+                    # if pair_candles contains more candles than lookback_period
+                    quoteVolume = (pair_candles['quoteVolume']
+                                   .rolling(self._lookback_period)
+                                   .sum()
+                                   .iloc[-1])
+
                     # replace quoteVolume with range quoteVolume sum calculated above
-                    filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
+                    filtered_tickers[i]['quoteVolume'] = quoteVolume
                 else:
                     filtered_tickers[i]['quoteVolume'] = 0
 

From f30e300f181046e6e09f3c896d59563d09e0e119 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 11:28:35 +0200
Subject: [PATCH 21/35] adjusted `test_pairlist.py` for fixed rolling sum

---
 tests/plugins/test_pairlist.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index 7812ee733..f6f86d0b5 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -527,7 +527,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     # expecing pairs as given
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
        "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
-     "BTC", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
+     "BTC", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']),
     # expecting pairs from default tickers, because 1h candles are not available
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
        "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
@@ -541,16 +541,20 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers,
     ohlcv_history_high_vola = ohlcv_history.copy()
     ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
 
-    # create candles for high volume
+    # create candles for medium overall volume with last candle high volume
+    ohlcv_history_medium_volume = ohlcv_history.copy()
+    ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5
+
+    # create candles for high volume with all candles high volume
     ohlcv_history_high_volume = ohlcv_history.copy()
-    ohlcv_history_high_volume.loc[ohlcv_history_high_volume.index == 1, 'volume'] = 10
+    ohlcv_history_high_volume.loc[:, 'volume'] = 10
 
     ohlcv_data = {
         ('ETH/BTC', '1d'): ohlcv_history,
         ('TKN/BTC', '1d'): ohlcv_history,
-        ('LTC/BTC', '1d'): ohlcv_history_high_volume,
+        ('LTC/BTC', '1d'): ohlcv_history_medium_volume,
         ('XRP/BTC', '1d'): ohlcv_history_high_vola,
-        ('HOT/BTC', '1d'): ohlcv_history,
+        ('HOT/BTC', '1d'): ohlcv_history_high_volume,
     }
 
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))

From 6cea0ef2d72e1e66bb6e496b820193b58f9d4624 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 11:48:26 +0200
Subject: [PATCH 22/35] documentation for `OffsetFilter`

---
 docs/includes/pairlists.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index f19c5a181..3ae3b89ec 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)
 * [`VolumePairList`](#volume-pair-list)
 * [`AgeFilter`](#agefilter)
+* [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
 * [`PrecisionFilter`](#precisionfilter)
 * [`PriceFilter`](#pricefilter)
@@ -89,6 +90,30 @@ be caught out buying before the pair has finished dropping in price.
 
 This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days.
 
+#### OffsetFilter
+
+Offsets an incoming pairlist by a given `offset` value.
+
+As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split
+a larger pairlist on two bot instances.
+
+Example to remove the first 10 pairs from the pairlist:
+
+```json
+"pairlists": [{
+        "method": "OffsetFilter",
+        "offset": 10
+}],
+```
+
+!!! Warning
+    When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` 
+    it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the
+    `VolumeFilter`
+
+!!! Note
+    An offset larger then the total length of the incoming pairlist will result in an empty pairlist.
+
 #### PerformanceFilter
 
 Sorts pairs by past trade performance, as follows:

From c44e87cd30a7a43a968319df326858280fffbf72 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 12:06:55 +0200
Subject: [PATCH 23/35] added tests for `OffsetFilter to `test_pairlist.py`

---
 tests/plugins/test_pairlist.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index ae8f6e958..975273e60 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -417,7 +417,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "StaticPairList"},
       {"method": "VolatilityFilter", "lookback_days": 3,
        "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}],
-     "BTC", ['ETH/BTC', 'TKN/BTC'])
+     "BTC", ['ETH/BTC', 'TKN/BTC']),
+    # VolumePairList with no offset = unchanged pairlist
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "OffsetFilter", "offset": 0}],
+     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']),
+    # VolumePairList with offset = 2
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "OffsetFilter", "offset": 2}],
+     "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']),
+    # VolumePairList with higher offset, than total pairlist
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "OffsetFilter", "offset": 100}],
+     "USDT", [])
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                       ohlcv_history, pairlists, base_currency,
@@ -695,6 +707,18 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o
     assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
 
 
+def test_OffsetFilter_error(mocker, whitelist_conf) -> None:
+    whitelist_conf['pairlists'] = (
+        [{"method": "StaticPairList"}, {"method": "OffsetFilter", "offset": -1}]
+    )
+
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    with pytest.raises(OperationalException,
+                       match=r'OffsetFilter requires offset to be >= 0'):
+        PairListManager(MagicMock, whitelist_conf)
+
+
 def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
     default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
                                  {'method': 'RangeStabilityFilter', 'lookback_days': 99999}]

From 00a1931f40e1351191fb030cdd649671aee7380c Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Wed, 7 Jul 2021 21:24:44 +0700
Subject: [PATCH 24/35] fix test

---
 freqtrade/plugins/pairlist/AgeFilter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 44c5d2e5e..810a87082 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -102,7 +102,7 @@ class AgeFilter(IPairList):
 
         if daily_candles is not None:
             if len(daily_candles) >= self._min_days_listed and \
-                    len(daily_candles) <= self._max_days_listed:
+                    (True if not self._max_days_listed else len(daily_candles) <= self._max_days_listed):
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
@@ -112,9 +112,9 @@ class AgeFilter(IPairList):
                     f"Removed {pair} from whitelist, because age "
                     f"{len(daily_candles)} is less than {self._min_days_listed} "
                     f"{plural(self._min_days_listed, 'day')}"
-                ) + (
+                ) + ((
                     " or more than "
                     f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
-                ) if self.max_days_listed else '', logger.info)
+                ) if self._max_days_listed else ''), logger.info)
                 return False
         return False

From 8248d1acd19c1b229fecd894d480bbd320912fc3 Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Wed, 7 Jul 2021 22:10:22 +0700
Subject: [PATCH 25/35] run flake8

---
 freqtrade/plugins/pairlist/AgeFilter.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 810a87082..133285884 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -102,7 +102,8 @@ class AgeFilter(IPairList):
 
         if daily_candles is not None:
             if len(daily_candles) >= self._min_days_listed and \
-                    (True if not self._max_days_listed else len(daily_candles) <= self._max_days_listed):
+                    (True if not self._max_days_listed
+                        else len(daily_candles) <= self._max_days_listed):
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000

From 682f880630dbf31bd99cf604d94256c989f90df1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 7 Jul 2021 20:05:56 +0200
Subject: [PATCH 26/35] Slightly simplify if statement, add additional test

---
 freqtrade/plugins/pairlist/AgeFilter.py |  7 ++++---
 tests/plugins/test_pairlist.py          | 10 +++++++++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 133285884..23250e5c1 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -101,9 +101,10 @@ class AgeFilter(IPairList):
             return True
 
         if daily_candles is not None:
-            if len(daily_candles) >= self._min_days_listed and \
-                    (True if not self._max_days_listed
-                        else len(daily_candles) <= self._max_days_listed):
+            if (
+                len(daily_candles) >= self._min_days_listed
+                and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed)
+            ):
                 # We have fetched at least the minimum required number of daily candles
                 # Add to cache, store the time we last checked this symbol
                 self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index 5724ca39c..f8c5acba3 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -321,6 +321,14 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}],
      "BTC", []),
+    # AgeFilter and VolumePairList LTC/BTC has 6 candles - removes all
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 5}],
+     "BTC", []),
+    # AgeFilter and VolumePairList LTC/BTC has 6 candles - passes
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
+      {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 10}],
+     "BTC", ["LTC/BTC"]),
     # Precisionfilter and quote volume
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PrecisionFilter"}],
@@ -436,7 +444,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     ohlcv_data = {
         ('ETH/BTC', '1d'): ohlcv_history,
         ('TKN/BTC', '1d'): ohlcv_history,
-        ('LTC/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history.append(ohlcv_history),
         ('XRP/BTC', '1d'): ohlcv_history,
         ('HOT/BTC', '1d'): ohlcv_history_high_vola,
     }

From 4d4ed82db8d41904d45302df2999bc660f8daed3 Mon Sep 17 00:00:00 2001
From: nightshift2k <82537832+nightshift2k@users.noreply.github.com>
Date: Wed, 7 Jul 2021 20:29:52 +0200
Subject: [PATCH 27/35] Update docs/includes/pairlists.md

Co-authored-by: Matthias <xmatthias@outlook.com>
---
 docs/includes/pairlists.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index 3ae3b89ec..5b2e892e8 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -109,7 +109,7 @@ Example to remove the first 10 pairs from the pairlist:
 !!! Warning
     When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` 
     it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the
-    `VolumeFilter`
+    `VolumeFilter`.
 
 !!! Note
     An offset larger then the total length of the incoming pairlist will result in an empty pairlist.

From 5a2bc192d4728f7c26e2ded30a68a833df0aa6c1 Mon Sep 17 00:00:00 2001
From: nightshift2k <82537832+nightshift2k@users.noreply.github.com>
Date: Wed, 7 Jul 2021 20:29:55 +0200
Subject: [PATCH 28/35] Update docs/includes/pairlists.md

Co-authored-by: Matthias <xmatthias@outlook.com>
---
 docs/includes/pairlists.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index 5b2e892e8..ecb6231ec 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -100,10 +100,12 @@ a larger pairlist on two bot instances.
 Example to remove the first 10 pairs from the pairlist:
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "OffsetFilter",
         "offset": 10
-}],
+    }
+],
 ```
 
 !!! Warning

From 7dc826d6b398f4220d3d96b9a88fa2fe29d6da93 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 20:43:37 +0200
Subject: [PATCH 29/35] warning for range based lookback performance more
 readable formatting of examples

---
 docs/includes/pairlists.md | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index 3f8baab29..9ea9b41a4 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -68,12 +68,14 @@ Filtering instances (not the first position in the list) will not apply any cach
 * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 1800
-}],
+    }
+],
 ```
 
 `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3), and multiplies it with every candle data's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
@@ -81,28 +83,36 @@ Filtering instances (not the first position in the list) will not apply any cach
 For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 86400,
         "lookback_days": 7
-}],
+    }
+],
 ```
 !!! Warning "Range look back and refresh period"
     When used in conjuction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
 
+!!! Warning "Performance implications when using lookback range"
+    If used in first position in combination with lookback, the computation of the range based volume can be time and ressource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
+
+
 More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 3600,
         "lookback_timeframe": "1h",
-        "lookback_timeframe": 72
-}],
+        "lookback_period": 72
+    }
+],
 ```
 
 !!! Note

From e5da7ff6db70ce4894f58e9b36a683631e921327 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 8 Jul 2021 07:02:40 +0200
Subject: [PATCH 30/35] Fix typos and improve wording in docs

---
 docs/includes/pairlists.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index 9ea9b41a4..5a771e055 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -78,7 +78,7 @@ Filtering instances (not the first position in the list) will not apply any cach
 ],
 ```
 
-`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3), and multiplies it with every candle data's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
+`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
 
 For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
 
@@ -93,12 +93,12 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl
     }
 ],
 ```
+
 !!! Warning "Range look back and refresh period"
-    When used in conjuction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
+    When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
 
 !!! Warning "Performance implications when using lookback range"
-    If used in first position in combination with lookback, the computation of the range based volume can be time and ressource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
-
+    If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
 
 More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
 

From 863391122ffa55895fec87212159c653f3c148cf Mon Sep 17 00:00:00 2001
From: kevinjulian <sayakeju@gmail.com>
Date: Thu, 8 Jul 2021 13:42:52 +0700
Subject: [PATCH 31/35] fix short desc not appear

---
 freqtrade/plugins/pairlist/AgeFilter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py
index 133285884..a49842e94 100644
--- a/freqtrade/plugins/pairlist/AgeFilter.py
+++ b/freqtrade/plugins/pairlist/AgeFilter.py
@@ -58,10 +58,10 @@ class AgeFilter(IPairList):
         return (
             f"{self.name} - Filtering pairs with age less than "
             f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
-        ) + (
+        ) + ((
             " or more than "
             f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
-        ) if self._max_days_listed else ''
+        ) if self._max_days_listed else '')
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """

From 03861945a39af7441f8898f00bf7416f11f7d7ca Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 8 Jul 2021 20:04:47 +0200
Subject: [PATCH 32/35] Update documentation page too.

---
 docs/strategy_analysis_example.md | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md
index 27c620c3d..27192aa2f 100644
--- a/docs/strategy_analysis_example.md
+++ b/docs/strategy_analysis_example.md
@@ -148,13 +148,18 @@ import pandas as pd
 stats = load_backtest_stats(backtest_dir)
 strategy_stats = stats['strategy'][strategy]
 
+dates = []
+profits = []
+for date_profit in strategy_stats['daily_profit']:
+    dates.append(date_profit[0])
+    profits.append(date_profit[1])
+
 equity = 0
 equity_daily = []
-for dp in strategy_stats['daily_profit']:
+for daily_profit in profits:
     equity_daily.append(equity)
-    equity += float(dp)
+    equity += float(daily_profit)
 
-dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end'])
 
 df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})
 

From 2f33b97b95c62ab0fec089098a6cf1eeae01d041 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 9 Jul 2021 07:19:44 +0200
Subject: [PATCH 33/35] Validate startup candles for backtesting correctly

closes #5250
---
 freqtrade/optimize/backtesting.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 7c6b7cbc3..8f818047d 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -116,6 +116,7 @@ class Backtesting:
 
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
+        self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
 
     def __del__(self):
         LoggingMixin.show_output = True

From 6129c5ca9e9ccd418f8f9c6659db54f2a87d8579 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 9 Jul 2021 20:46:38 +0200
Subject: [PATCH 34/35] Fix deprecation warnings from pandas 1.3.0

closes #5251
---
 freqtrade/optimize/hyperopt_tools.py       | 2 +-
 freqtrade/optimize/optimize_reports.py     | 5 +++--
 freqtrade/plugins/pairlist/OffsetFilter.py | 2 +-
 freqtrade/rpc/rpc.py                       | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py
index 90976d34e..439016c14 100755
--- a/freqtrade/optimize/hyperopt_tools.py
+++ b/freqtrade/optimize/hyperopt_tools.py
@@ -75,7 +75,7 @@ class HyperoptTools():
             if fn:
                 HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json'))
             else:
-                logger.warn("Strategy not found, not exporting parameter file.")
+                logger.warning("Strategy not found, not exporting parameter file.")
 
     @staticmethod
     def has_space(config: Dict[str, Any], space: str) -> bool:
diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index 5dc8554c4..eefacbbab 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -325,8 +325,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
                     key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
     worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
                      key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
-    results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
-    results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
+    if not results.empty:
+        results['open_timestamp'] = results['open_date'].view(int64) // 1e6
+        results['close_timestamp'] = results['close_date'].view(int64) // 1e6
 
     backtest_days = (max_date - min_date).days
     strat_stats = {
diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py
index 4579204d9..573a573a6 100644
--- a/freqtrade/plugins/pairlist/OffsetFilter.py
+++ b/freqtrade/plugins/pairlist/OffsetFilter.py
@@ -48,7 +48,7 @@ class OffsetFilter(IPairList):
         """
         if self._offset > len(pairlist):
             self.log_once(f"Offset of {self._offset} is larger than " +
-                          f"pair count of {len(pairlist)}", logger.warn)
+                          f"pair count of {len(pairlist)}", logger.warning)
         pairs = pairlist[self._offset:]
         self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
         return pairs
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 538e95f40..e0aaefe50 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -761,7 +761,7 @@ class RPC:
         sell_signals = 0
         if has_content:
 
-            dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
+            dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
             # Move open to seperate column when signal for easy plotting
             if 'buy' in dataframe.columns:
                 buy_mask = (dataframe['buy'] == 1)

From e4e2340f91ffacdfe971be8387cb9d1c36ad074f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 10 Jul 2021 10:02:05 +0200
Subject: [PATCH 35/35] Fix bug where currencies are duplicated

in case there is dust
---
 freqtrade/rpc/telegram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index 171a53ca1..319a6c9c0 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -600,8 +600,8 @@ class Telegram(RPCHandler):
                 )
             total_dust_balance = 0
             total_dust_currencies = 0
-            curr_output = ''
             for curr in result['currencies']:
+                curr_output = ''
                 if curr['est_stake'] > balance_dust_level:
                     curr_output = (
                         f"*{curr['currency']}:*\n"