From 055229a44a8288b4f32ccd2604d08c1d1e2dd045 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sat, 3 Jul 2021 11:39:14 +0200
Subject: [PATCH 01/11] first iteration of volume pairlist with range lookback

---
 freqtrade/plugins/pairlist/VolumePairList.py | 71 +++++++++++++++++++-
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 8eff137b0..af26201a9 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -11,6 +11,9 @@ from cachetools.ttl import TTLCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
+import arrow
+from copy import deepcopy
+from freqtrade.exchange import timeframe_to_minutes
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +39,25 @@ class VolumePairList(IPairList):
         self._min_value = self._pairlistconfig.get('min_value', 0)
         self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
         self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
+        self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
+        self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
+
+        # overwrite lookback timeframe and days when lookback_days is set
+        if self._lookback_days > 0:
+            self._lookback_timeframe = '1d'
+            self._lookback_period = self._lookback_days
+
+        self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
+        self._tf_in_secs = self._tf_in_min * 60
+
+        self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
+
+        if self._use_range & (self._refresh_period < self._tf_in_secs):
+            raise OperationalException(
+                f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
+                f'Please adjust refresh_period to at least {self._tf_in_secs} and restart the bot.'
+            )
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -47,6 +69,14 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
+
+        if self._lookback_period < 0:
+            raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
+        if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
+            raise OperationalException("VolumeFilter requires lookback_period to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
+
     @property
     def needstickers(self) -> bool:
         """
@@ -78,7 +108,6 @@ class VolumePairList(IPairList):
             # Item found - no refresh necessary
             return pairlist
         else:
-
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
             filtered_tickers = [
@@ -100,9 +129,44 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        # Use the incoming pairlist.
+
+        # Use the incoming pairlist.      
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
+        if self._use_range == True:
+            since_ms = int(arrow.utcnow()
+                        .floor('minute')
+                        .shift(minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
+                        .float_timestamp) * 1000
+
+            self.log_once(f"Using volume range of {self._lookback_period} {self._lookback_timeframe} candles from {since_ms}", logger.info)
+            needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
+            # Get all candles
+            candles = {}
+            if needed_pairs:
+                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
+                                                            cache=False)
+
+            for i,p in enumerate(filtered_tickers):
+            # for p in deepcopy([s['symbol'] for s in filtered_tickers]):
+                pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
+                #print(p['symbol'], " 24h quote volume   = ",filtered_tickers[i]['quoteVolume'])
+
+                #if p['symbol'] == 'BCC/USDT':
+                    #print(pair_candles)
+                    #quit()
+
+                print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
+                if not pair_candles.empty:
+                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
+                    pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
+                    # print(p['symbol'], " range quote volume = ", pair_candles['quoteVolume'].sum())
+                    filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
+                else:
+                    filtered_tickers[i]['quoteVolume'] = 0
+
+                print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
+
         if self._min_value > 0:
             filtered_tickers = [
                     v for v in filtered_tickers if v[self._sort_key] > self._min_value]
@@ -112,9 +176,12 @@ class VolumePairList(IPairList):
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
         pairs = self.verify_blacklist(pairs, logger.info)
+        
         # Limit pairlist to the requested number of pairs
+        
         pairs = pairs[:self._number_pairs]
 
         self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
 
+        quit()
         return pairs

From 62da4b452ccbab16044adc03f9c74d2821710d13 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sat, 3 Jul 2021 11:47:17 +0200
Subject: [PATCH 02/11] code cleanup and comments

---
 freqtrade/plugins/pairlist/VolumePairList.py | 36 +++++++-------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index af26201a9..ae6be54ba 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -4,16 +4,14 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
+import arrow
 from typing import Any, Dict, List
 
 from cachetools.ttl import TTLCache
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.plugins.pairlist.IPairList import IPairList
-
-import arrow
-from copy import deepcopy
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.IPairList import IPairList
 
 logger = logging.getLogger(__name__)
 
@@ -48,15 +46,16 @@ class VolumePairList(IPairList):
             self._lookback_timeframe = '1d'
             self._lookback_period = self._lookback_days
 
+        # get timeframe in minutes and seconds 
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
-        self._tf_in_secs = self._tf_in_min * 60
+        self._tf_in_sec = self._tf_in_min * 60
 
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
         if self._use_range & (self._refresh_period < self._tf_in_secs):
             raise OperationalException(
                 f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
-                f'Please adjust refresh_period to at least {self._tf_in_secs} and restart the bot.'
+                f'Please adjust refresh_period to at least {self._tf_in_sec} and restart the bot.'
             )
 
         if not self._exchange.exchange_has('fetchTickers'):
@@ -129,10 +128,10 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-
         # Use the incoming pairlist.      
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
+        # get lookback period in ms, for exchange ohlcv fetch
         if self._use_range == True:
             since_ms = int(arrow.utcnow()
                         .floor('minute')
@@ -141,31 +140,25 @@ class VolumePairList(IPairList):
 
             self.log_once(f"Using volume range of {self._lookback_period} {self._lookback_timeframe} candles from {since_ms}", logger.info)
             needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
+
             # Get all candles
             candles = {}
             if needed_pairs:
-                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
-                                                            cache=False)
+                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
 
             for i,p in enumerate(filtered_tickers):
-            # for p in deepcopy([s['symbol'] for s in filtered_tickers]):
                 pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
-                #print(p['symbol'], " 24h quote volume   = ",filtered_tickers[i]['quoteVolume'])
-
-                #if p['symbol'] == 'BCC/USDT':
-                    #print(pair_candles)
-                    #quit()
-
-                print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
+                # print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
+                # in case of candle data calculate typical price and quoteVolume for candle
                 if not pair_candles.empty:
                     pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
                     pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
-                    # print(p['symbol'], " range quote volume = ", pair_candles['quoteVolume'].sum())
+                    # replace quoteVolume with range sum
                     filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
                 else:
                     filtered_tickers[i]['quoteVolume'] = 0
 
-                print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
+                # print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
 
         if self._min_value > 0:
             filtered_tickers = [
@@ -175,13 +168,10 @@ class VolumePairList(IPairList):
 
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        pairs = self.verify_blacklist(pairs, logger.info)
-        
+        pairs = self.verify_blacklist(pairs, logger.info)       
         # Limit pairlist to the requested number of pairs
-        
         pairs = pairs[:self._number_pairs]
 
         self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
 
-        quit()
         return pairs

From 53f963dd736188c2cc5efe27a52ccbb40c2af12c Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sat, 3 Jul 2021 11:49:05 +0200
Subject: [PATCH 03/11] fixed `self._tf_in_secs` to `self._tf_in_sec`

---
 freqtrade/plugins/pairlist/VolumePairList.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index ae6be54ba..4e9102243 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -52,7 +52,7 @@ class VolumePairList(IPairList):
 
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
-        if self._use_range & (self._refresh_period < self._tf_in_secs):
+        if self._use_range & (self._refresh_period < self._tf_in_sec):
             raise OperationalException(
                 f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
                 f'Please adjust refresh_period to at least {self._tf_in_sec} and restart the bot.'

From 348dbeff3f99e848666e7b5242ea02e205c73a64 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 11:16:33 +0200
Subject: [PATCH 04/11] added meaningful logging of used lookback range

---
 freqtrade/plugins/pairlist/VolumePairList.py | 23 +++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 4e9102243..5820bc667 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -12,6 +12,7 @@ from cachetools.ttl import TTLCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.misc import format_ms_time
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +42,12 @@ class VolumePairList(IPairList):
         self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
         self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
 
+        if (self._lookback_days > 0) & (self._lookback_period > 0):
+            raise OperationalException(
+                f'Ambigous configuration: lookback_days and lookback_period both set in pairlist config. '
+                f'Please set lookback_days only or lookback_period and lookback_timeframe and restart the bot.'
+            )
+
         # overwrite lookback timeframe and days when lookback_days is set
         if self._lookback_days > 0:
             self._lookback_timeframe = '1d'
@@ -49,7 +56,8 @@ class VolumePairList(IPairList):
         # get timeframe in minutes and seconds 
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
         self._tf_in_sec = self._tf_in_min * 60
-
+        
+        # wether to use range lookback or not
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
         if self._use_range & (self._refresh_period < self._tf_in_sec):
@@ -138,7 +146,13 @@ class VolumePairList(IPairList):
                         .shift(minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
                         .float_timestamp) * 1000
 
-            self.log_once(f"Using volume range of {self._lookback_period} {self._lookback_timeframe} candles from {since_ms}", logger.info)
+            to_ms = int(arrow.utcnow()
+                        .floor('minute')
+                        .shift(minutes=-self._tf_in_min)
+                        .float_timestamp) * 1000
+
+            # todo: utc date output for starting date
+            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: {self._lookback_timeframe}, starting from {format_ms_time(since_ms)} till {format_ms_time(to_ms)}", logger.info)
             needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
 
             # Get all candles
@@ -148,18 +162,15 @@ class VolumePairList(IPairList):
 
             for i,p in enumerate(filtered_tickers):
                 pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
-                # print(p['symbol'], " 24h quote volume     = ",filtered_tickers[i]['quoteVolume'])
                 # in case of candle data calculate typical price and quoteVolume for candle
                 if not pair_candles.empty:
                     pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
                     pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
-                    # replace quoteVolume with range sum
+                    # replace quoteVolume with range quoteVolume sum calculated above
                     filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
                 else:
                     filtered_tickers[i]['quoteVolume'] = 0
 
-                # print(p['symbol'], " range quote volume   = ",filtered_tickers[i]['quoteVolume'])
-
         if self._min_value > 0:
             filtered_tickers = [
                     v for v in filtered_tickers if v[self._sort_key] > self._min_value]

From 9919061c7899e7e7e5d9c4a0ef59d78152a6a9ec Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 11:40:45 +0200
Subject: [PATCH 05/11] PEP8 compliance

---
 freqtrade/plugins/pairlist/VolumePairList.py | 62 +++++++++++++-------
 1 file changed, 40 insertions(+), 22 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 5820bc667..40e6afa07 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -4,15 +4,16 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
-import arrow
 from typing import Any, Dict, List
 
+import arrow
 from cachetools.ttl import TTLCache
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
-from freqtrade.plugins.pairlist.IPairList import IPairList
 from freqtrade.misc import format_ms_time
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +45,9 @@ class VolumePairList(IPairList):
 
         if (self._lookback_days > 0) & (self._lookback_period > 0):
             raise OperationalException(
-                f'Ambigous configuration: lookback_days and lookback_period both set in pairlist config. '
-                f'Please set lookback_days only or lookback_period and lookback_timeframe and restart the bot.'
+                'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
+                'config. Please set lookback_days only or lookback_period and lookback_timeframe '
+                'and restart the bot.'
             )
 
         # overwrite lookback timeframe and days when lookback_days is set
@@ -53,17 +55,18 @@ class VolumePairList(IPairList):
             self._lookback_timeframe = '1d'
             self._lookback_period = self._lookback_days
 
-        # get timeframe in minutes and seconds 
+        # get timeframe in minutes and seconds
         self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
         self._tf_in_sec = self._tf_in_min * 60
-        
+
         # wether to use range lookback or not
         self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
 
         if self._use_range & (self._refresh_period < self._tf_in_sec):
             raise OperationalException(
-                f'Refresh period of {self._refresh_period} seconds is smaller than one timeframe of {self._lookback_timeframe}. '
-                f'Please adjust refresh_period to at least {self._tf_in_sec} and restart the bot.'
+                f'Refresh period of {self._refresh_period} seconds is smaller than one '
+                f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
+                f'to at least {self._tf_in_sec} and restart the bot.'
             )
 
         if not self._exchange.exchange_has('fetchTickers'):
@@ -76,7 +79,6 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
-
         if self._lookback_period < 0:
             raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
         if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
@@ -136,15 +138,16 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        # Use the incoming pairlist.      
+        # Use the incoming pairlist.
         filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
         # get lookback period in ms, for exchange ohlcv fetch
-        if self._use_range == True:
+        if self._use_range:
             since_ms = int(arrow.utcnow()
-                        .floor('minute')
-                        .shift(minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
-                        .float_timestamp) * 1000
+                           .floor('minute')
+                           .shift(minutes=-(self._lookback_period * self._tf_in_min)
+                                  - self._tf_in_min)
+                           .float_timestamp) * 1000
 
             to_ms = int(arrow.utcnow()
                         .floor('minute')
@@ -152,20 +155,35 @@ class VolumePairList(IPairList):
                         .float_timestamp) * 1000
 
             # todo: utc date output for starting date
-            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: {self._lookback_timeframe}, starting from {format_ms_time(since_ms)} till {format_ms_time(to_ms)}", logger.info)
-            needed_pairs = [(p, self._lookback_timeframe) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache]
+            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
+                          f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
+                          f"till {format_ms_time(to_ms)}", logger.info)
+            needed_pairs = [
+                (p, self._lookback_timeframe) for p in
+                [
+                    s['symbol'] for s in filtered_tickers
+                ] if p not in self._pair_cache
+            ]
 
             # Get all candles
             candles = {}
             if needed_pairs:
-                candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
+                candles = self._exchange.refresh_latest_ohlcv(
+                    needed_pairs, since_ms=since_ms, cache=False
+                )
 
-            for i,p in enumerate(filtered_tickers):
-                pair_candles = candles[(p['symbol'], self._lookback_timeframe)] if (p['symbol'], self._lookback_timeframe) in candles else None
+            for i, p in enumerate(filtered_tickers):
+                pair_candles = candles[
+                    (p['symbol'], self._lookback_timeframe)
+                ] if (p['symbol'], self._lookback_timeframe) in candles else None
                 # in case of candle data calculate typical price and quoteVolume for candle
                 if not pair_candles.empty:
-                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3
-                    pair_candles['quoteVolume'] = pair_candles['volume'] * pair_candles['typical_price']
+                    pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+                                                     + pair_candles['close']) / 3
+                    pair_candles['quoteVolume'] = (
+                        pair_candles['volume'] * pair_candles['typical_price']
+                    )
+
                     # replace quoteVolume with range quoteVolume sum calculated above
                     filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
                 else:
@@ -179,7 +197,7 @@ class VolumePairList(IPairList):
 
         # Validate whitelist to only have active market pairs
         pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
-        pairs = self.verify_blacklist(pairs, logger.info)       
+        pairs = self.verify_blacklist(pairs, logger.info)
         # Limit pairlist to the requested number of pairs
         pairs = pairs[:self._number_pairs]
 

From 85c7b557503d1c25330101266ab898642ca41272 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Sun, 4 Jul 2021 20:46:24 +0200
Subject: [PATCH 06/11] improvements: - `float_timestamp` switched to
 `int_timestamp` - added documentation to pairlists.md

---
 docs/includes/pairlists.md                   | 31 +++++++++++++++++++-
 freqtrade/plugins/pairlist/VolumePairList.py |  4 +--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index f19c5a181..3f8baab29 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -63,7 +63,7 @@ The `refresh_period` setting allows to define the period (in seconds), at which
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
 Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
 
-`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
+`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
 * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
 
@@ -76,6 +76,35 @@ Filtering instances (not the first position in the list) will not apply any cach
 }],
 ```
 
+`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3), and multiplies it with every candle data's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
+
+For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
+
+```json
+"pairlists": [{
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 86400,
+        "lookback_days": 7
+}],
+```
+!!! Warning "Range look back and refresh period"
+    When used in conjuction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
+
+More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
+
+```json
+"pairlists": [{
+        "method": "VolumePairList",
+        "number_assets": 20,
+        "sort_key": "quoteVolume",
+        "refresh_period": 3600,
+        "lookback_timeframe": "1h",
+        "lookback_timeframe": 72
+}],
+```
+
 !!! Note
     `VolumePairList` does not support backtesting mode.
 
diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 40e6afa07..352d028ac 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -147,12 +147,12 @@ class VolumePairList(IPairList):
                            .floor('minute')
                            .shift(minutes=-(self._lookback_period * self._tf_in_min)
                                   - self._tf_in_min)
-                           .float_timestamp) * 1000
+                           .int_timestamp) * 1000
 
             to_ms = int(arrow.utcnow()
                         .floor('minute')
                         .shift(minutes=-self._tf_in_min)
-                        .float_timestamp) * 1000
+                        .int_timestamp) * 1000
 
             # todo: utc date output for starting date
             self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "

From 1e87225e916283fc27f18efb8f1d605b84d0cecd Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Mon, 5 Jul 2021 20:59:27 +0200
Subject: [PATCH 07/11] added `test_VolumePairList_range` to test_pairlist.py

---
 freqtrade/plugins/pairlist/VolumePairList.py |  3 +-
 tests/plugins/test_pairlist.py               | 95 ++++++++++++++++++++
 2 files changed, 96 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 352d028ac..14b9d7024 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -171,13 +171,12 @@ class VolumePairList(IPairList):
                 candles = self._exchange.refresh_latest_ohlcv(
                     needed_pairs, since_ms=since_ms, cache=False
                 )
-
             for i, p in enumerate(filtered_tickers):
                 pair_candles = candles[
                     (p['symbol'], self._lookback_timeframe)
                 ] if (p['symbol'], self._lookback_timeframe) in candles else None
                 # in case of candle data calculate typical price and quoteVolume for candle
-                if not pair_candles.empty:
+                if pair_candles is not None and not pair_candles.empty:
                     pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
                                                      + pair_candles['close']) / 3
                     pair_candles['quoteVolume'] = (
diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index ae8f6e958..7812ee733 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -507,6 +507,101 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                 assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog)
 
 
+@pytest.mark.parametrize("pairlists,base_currency,volumefilter_result", [
+    # default refresh of 1800 to small for daily candle lookback
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_days": 1}],
+     "BTC", "default_refresh_too_short"),  # OperationalException expected
+    # ambigous configuration with lookback days and period
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_days": 1, "lookback_period": 1}],
+     "BTC", "lookback_days_and_period"),  # OperationalException expected
+    # negative lookback period
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": -1}],
+     "BTC", "lookback_period_negative"),  # OperationalException expected
+    # lookback range exceedes exchange limit
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}],
+     "BTC", 'lookback_exceeds_exchange_request_size'),  # OperationalException expected
+    # expecing pairs as given
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
+     "BTC", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
+    # expecting pairs from default tickers, because 1h candles are not available
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
+       "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']),
+])
+def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history,
+                              pairlists, base_currency, volumefilter_result, caplog) -> None:
+    whitelist_conf['pairlists'] = pairlists
+    whitelist_conf['stake_currency'] = base_currency
+
+    ohlcv_history_high_vola = ohlcv_history.copy()
+    ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
+
+    # create candles for high volume
+    ohlcv_history_high_volume = ohlcv_history.copy()
+    ohlcv_history_high_volume.loc[ohlcv_history_high_volume.index == 1, 'volume'] = 10
+
+    ohlcv_data = {
+        ('ETH/BTC', '1d'): ohlcv_history,
+        ('TKN/BTC', '1d'): ohlcv_history,
+        ('LTC/BTC', '1d'): ohlcv_history_high_volume,
+        ('XRP/BTC', '1d'): ohlcv_history_high_vola,
+        ('HOT/BTC', '1d'): ohlcv_history,
+    }
+
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    if volumefilter_result == 'default_refresh_too_short':
+        with pytest.raises(OperationalException,
+                           match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe '
+                                 r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ '
+                                 r'and restart the bot\.'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        return
+    elif volumefilter_result == 'lookback_days_and_period':
+        with pytest.raises(OperationalException,
+                           match=r'Ambigous configuration: lookback_days and lookback_period both '
+                                 r'set in pairlist config\..*'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    elif volumefilter_result == 'lookback_period_negative':
+        with pytest.raises(OperationalException,
+                           match=r'VolumeFilter requires lookback_period to be >= 0'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    elif volumefilter_result == 'lookback_exceeds_exchange_request_size':
+        with pytest.raises(OperationalException,
+                           match=r'VolumeFilter requires lookback_period to not exceed '
+                           r'exchange max request size \([0-9]+\)'):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+    else:
+        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            get_tickers=tickers,
+            markets=PropertyMock(return_value=shitcoinmarkets)
+        )
+
+        # remove ohlcv when looback_timeframe != 1d
+        # to enforce fallback to ticker data
+        if 'lookback_timeframe' in pairlists[0]:
+            if pairlists[0]['lookback_timeframe'] != '1d':
+                ohlcv_data = []
+
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
+        )
+
+        freqtrade.pairlists.refresh_pairlist()
+        whitelist = freqtrade.pairlists.whitelist
+
+        assert isinstance(whitelist, list)
+        assert whitelist == volumefilter_result
+
+
 def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
     whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}]
     del whitelist_conf['stoploss']

From 3c3772703bdc8897abcf03c5c4ee6bf45c27e802 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 09:46:05 +0200
Subject: [PATCH 08/11] changed quoteVolume to be built over a rolling period
 using lookback_period to avoid pair_candles being larger than requested
 lookback_period

---
 freqtrade/plugins/pairlist/VolumePairList.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py
index 14b9d7024..d6b8aaaa3 100644
--- a/freqtrade/plugins/pairlist/VolumePairList.py
+++ b/freqtrade/plugins/pairlist/VolumePairList.py
@@ -183,8 +183,15 @@ class VolumePairList(IPairList):
                         pair_candles['volume'] * pair_candles['typical_price']
                     )
 
+                    # ensure that a rolling sum over the lookback_period is built
+                    # if pair_candles contains more candles than lookback_period
+                    quoteVolume = (pair_candles['quoteVolume']
+                                   .rolling(self._lookback_period)
+                                   .sum()
+                                   .iloc[-1])
+
                     # replace quoteVolume with range quoteVolume sum calculated above
-                    filtered_tickers[i]['quoteVolume'] = pair_candles['quoteVolume'].sum()
+                    filtered_tickers[i]['quoteVolume'] = quoteVolume
                 else:
                     filtered_tickers[i]['quoteVolume'] = 0
 

From f30e300f181046e6e09f3c896d59563d09e0e119 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 11:28:35 +0200
Subject: [PATCH 09/11] adjusted `test_pairlist.py` for fixed rolling sum

---
 tests/plugins/test_pairlist.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py
index 7812ee733..f6f86d0b5 100644
--- a/tests/plugins/test_pairlist.py
+++ b/tests/plugins/test_pairlist.py
@@ -527,7 +527,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
     # expecing pairs as given
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
        "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}],
-     "BTC", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
+     "BTC", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']),
     # expecting pairs from default tickers, because 1h candles are not available
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume",
        "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}],
@@ -541,16 +541,20 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers,
     ohlcv_history_high_vola = ohlcv_history.copy()
     ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090
 
-    # create candles for high volume
+    # create candles for medium overall volume with last candle high volume
+    ohlcv_history_medium_volume = ohlcv_history.copy()
+    ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5
+
+    # create candles for high volume with all candles high volume
     ohlcv_history_high_volume = ohlcv_history.copy()
-    ohlcv_history_high_volume.loc[ohlcv_history_high_volume.index == 1, 'volume'] = 10
+    ohlcv_history_high_volume.loc[:, 'volume'] = 10
 
     ohlcv_data = {
         ('ETH/BTC', '1d'): ohlcv_history,
         ('TKN/BTC', '1d'): ohlcv_history,
-        ('LTC/BTC', '1d'): ohlcv_history_high_volume,
+        ('LTC/BTC', '1d'): ohlcv_history_medium_volume,
         ('XRP/BTC', '1d'): ohlcv_history_high_vola,
-        ('HOT/BTC', '1d'): ohlcv_history,
+        ('HOT/BTC', '1d'): ohlcv_history_high_volume,
     }
 
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))

From 7dc826d6b398f4220d3d96b9a88fa2fe29d6da93 Mon Sep 17 00:00:00 2001
From: nightshift2k <join.the.nightshift@protonmail.com>
Date: Wed, 7 Jul 2021 20:43:37 +0200
Subject: [PATCH 10/11] warning for range based lookback performance more
 readable formatting of examples

---
 docs/includes/pairlists.md | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index 3f8baab29..9ea9b41a4 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -68,12 +68,14 @@ Filtering instances (not the first position in the list) will not apply any cach
 * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours.
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 1800
-}],
+    }
+],
 ```
 
 `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3), and multiplies it with every candle data's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
@@ -81,28 +83,36 @@ Filtering instances (not the first position in the list) will not apply any cach
 For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 86400,
         "lookback_days": 7
-}],
+    }
+],
 ```
 !!! Warning "Range look back and refresh period"
     When used in conjuction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
 
+!!! Warning "Performance implications when using lookback range"
+    If used in first position in combination with lookback, the computation of the range based volume can be time and ressource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
+
+
 More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
 
 ```json
-"pairlists": [{
+"pairlists": [
+    {
         "method": "VolumePairList",
         "number_assets": 20,
         "sort_key": "quoteVolume",
         "refresh_period": 3600,
         "lookback_timeframe": "1h",
-        "lookback_timeframe": 72
-}],
+        "lookback_period": 72
+    }
+],
 ```
 
 !!! Note

From e5da7ff6db70ce4894f58e9b36a683631e921327 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 8 Jul 2021 07:02:40 +0200
Subject: [PATCH 11/11] Fix typos and improve wording in docs

---
 docs/includes/pairlists.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md
index 9ea9b41a4..5a771e055 100644
--- a/docs/includes/pairlists.md
+++ b/docs/includes/pairlists.md
@@ -78,7 +78,7 @@ Filtering instances (not the first position in the list) will not apply any cach
 ],
 ```
 
-`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3), and multiplies it with every candle data's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
+`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
 
 For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
 
@@ -93,12 +93,12 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl
     }
 ],
 ```
+
 !!! Warning "Range look back and refresh period"
-    When used in conjuction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
+    When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API.
 
 !!! Warning "Performance implications when using lookback range"
-    If used in first position in combination with lookback, the computation of the range based volume can be time and ressource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
-
+    If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
 
 More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles: