Improvements

- add StaticAnnouncementsPairList in AVAILABLE_PAIRLISTS
- handle Database persistance of BinanceAnnouncementsData

freqtrade/constants.py

@@ -26,7 +26,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
                          'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily',
                          'MaxDrawDownHyperOptLoss']
-AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AnnouncementsPairList',
+AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AnnouncementsPairList', 'StaticAnnouncementsPairList',
                        'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
                        'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']

freqtrade/persistence/__init__.py

@@ -3,3 +3,4 @@
 from freqtrade.persistence.models import (LocalTrade, Order, Trade, clean_dry_run_db, cleanup_db,
                                           init_db)
 from freqtrade.persistence.pairlock_middleware import PairLocks
+from freqtrade.persistence import binanceannouncements

freqtrade/persistence/binanceannouncements.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+import pandas as pd
+import pytz
+
+from sqlalchemy import create_engine
+from sqlalchemy.exc import ProgrammingError
+from sqlalchemy.types import DateTime
+
+TABLE = "binanceannouncements"
+
+
+def get_engine(uri: str):
+    return create_engine(uri, pool_recycle=3600)
+
+
+def get_connection(uri: str):
+    return get_engine(uri).connect()
+
+
+def get_df(uri):
+    """Get dataframe and the first time create DB."""
+    connection = get_connection(uri)
+    try:
+        return pd.read_sql_table(
+            table_name=TABLE,
+            con=connection,
+            index_col='index',
+            columns=['Token', 'Text', 'Link', 'Datetime discover', 'Datetime announcement'],
+        )
+    except ValueError as e:
+        return None
+    finally:
+        connection.close()
+
+
+def save_df(df, uri):
+    """Save dataframe on DB."""
+    connection = get_connection(uri)
+    try:
+        df.to_sql(
+            name=TABLE,
+            con=connection,
+            index=True,
+            index_label='index',
+            if_exists='replace',
+            dtype={"Datetime discover": DateTime(timezone=pytz.utc),
+                   "Datetime announcement": DateTime(timezone=pytz.utc)}
+        )
+    finally:
+        connection.close()

freqtrade/plugins/pairlist/AnnouncementsPairList.py

@@ -9,6 +9,7 @@ Supported exchanges:
 """
 from abc import abstractmethod
 from bs4 import BeautifulSoup
+from cachetools import cached
 from cachetools.ttl import TTLCache
 from datetime import datetime, timedelta
 from requests import get
@@ -22,7 +23,7 @@ import pandas as pd
 
 from freqtrade.exceptions import OperationalException, TemporaryError
 from freqtrade.plugins.pairlist.IPairList import IPairList
-
+from freqtrade.persistence import binanceannouncements
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +39,6 @@ class AnnouncementMixin:
     ANNOUNCEMENT_COL: str
 
     def __init__(self, refresh_period: Optional[int] = None, *args, **kwargs):
-        # 0 is not allowed, so `or` clause will work everytime
         self._refresh_period = refresh_period or self.REFRESH_PERIOD
 
     @abstractmethod
@@ -72,13 +72,16 @@ class BinanceAnnouncement(AnnouncementMixin):
 
     # storage
     COLS = ['Token', 'Text', 'Link', 'Datetime discover', 'Datetime announcement']
-    DB = "BinanceAnnouncements_announcements.csv"
 
     TOKEN_COL = 'Token'
     ANNOUNCEMENT_COL = 'Datetime announcement'
 
     _df: Optional[pd.DataFrame] = None
 
+    def __init__(self, refresh_period: Optional[int] = None, *args, **kwargs):
+        self.db_path = kwargs.get('db_path', "user_data/data/BinanceAnnouncements_announcements.csv")
+        self._refresh_period = refresh_period or self.REFRESH_PERIOD
+
     def update_announcements(self, page_number=1, page_size=10, history=False) -> pd.DataFrame:
         response = None
         url = self.get_api_url(page_number, page_size)
@@ -91,7 +94,7 @@ class BinanceAnnouncement(AnnouncementMixin):
 
         try:
             now = datetime.now(tz=pytz.utc)
-            df = self._get_df()
+            df = self.get_df()
 
             try:
                 response = get(url)
@@ -163,7 +166,7 @@ class BinanceAnnouncement(AnnouncementMixin):
                 (item in article_text_lower)  # key matched
                 and (
                     not have_df  # is first time data
-                    or not (token is None or token in df['Token'].values)  # not an existing or null token
+                    or not (token is None or token in df[self.TOKEN_COL].values)  # not an existing or null token
                 )
             )
             if conditions_buy:
@@ -181,21 +184,30 @@ class BinanceAnnouncement(AnnouncementMixin):
     def _get_tokens(self, text: str):
         return self.BINANCE_TOKEN_REGEX.findall(text)
 
-    def _get_df(self):
+    def get_df(self):
+        """Get Dataframe from CSV file or from Database"""
         if self._df is None:
-            try:
+            self.load_csv_db() if self.db_path.endswith('csv') else self.load_db()
-                self._df = pd.read_csv(self.db_path, parse_dates=['Datetime announcement', 'Datetime discover'])
-            except FileNotFoundError:
-                pass
         return self._df
 
+    def load_csv_db(self):
+        try:
+            self._df = pd.read_csv(self.db_path, parse_dates=['Datetime discover', 'Datetime announcement'])
+        except FileNotFoundError:
+            pass
+
+    def load_db(self):
+        try:
+            self._df = binanceannouncements.get_df(self.db_path)
+        except binanceannouncements.ProgrammingError:
+            pass
+
+    def save_db(self):
+        binanceannouncements.save_df(self._df, self.db_path)
+
     def _save_df(self, df: pd.DataFrame):
         self._df = df.sort_values(by='Datetime announcement')
-        self._df.to_csv(self.db_path, index=False)
+        self._df.to_csv(self.db_path, index=False) if self.db_path.endswith('csv') else self.save_db()
-
-    @property
-    def db_path(self) -> str:
-        return "".join(["user_data/data/", self.DB])
 
     def get_datetime_announcement(self, announcement_url: str):
         response = get(announcement_url)
@@ -257,7 +269,7 @@ class AnnouncementsPairList(IPairList):
 
         pair_exchange_kwargs = self._pairlistconfig.get('exchange_kwargs', {})
         try:
-            self.pair_exchange = self._init_pair_exchange(**pair_exchange_kwargs)
+            self.pair_exchange = self._init_pair_exchange(config, **pair_exchange_kwargs)
         except AssertionError as e:
             raise OperationalException(f"Announcement class is improperly configured. Exception: {e}") from e
 
@@ -268,7 +280,14 @@ class AnnouncementsPairList(IPairList):
         If no Pairlist requires tickers, an empty Dict is passed
         as tickers argument to filter_pairlist
         """
-        return True
+        # Set at False.
+        # This PairList needs tickers but not with standard frequency.
+        # So we take it caching for more time. see _get_cached_tickers. ttl=3600
+        return False
+
+    @cached(TTLCache(maxsize=1, ttl=3600))
+    def _get_cached_tickers(self):
+        return self._exchange.get_tickers()
 
     def short_desc(self) -> str:
         """
@@ -287,6 +306,7 @@ class AnnouncementsPairList(IPairList):
             # Item found - no refresh necessary
             return pairlist.copy()
         else:
+            tickers = self._get_cached_tickers()
             filtered_tickers = [
                 v for k, v in tickers.items()
                 if (self._exchange.get_pair_quote_currency(k) == self._stake_currency)
@@ -306,26 +326,23 @@ class AnnouncementsPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        if self.STATIC:
+        df = self.pair_exchange.get_df() if self.STATIC else self.pair_exchange.update_announcements()
-            df = self.pair_exchange._get_df()
+        df = df[df[self.pair_exchange.ANNOUNCEMENT_COL] > (datetime.now(tz=pytz.utc) - timedelta(hours=self._hours))]
-        else:
+        if df.empty:
-            df = self.pair_exchange.update_announcements()
+            return []
 
-        # TODO improve performance
+        pairs = [get_token_from_pair(v, 0) for v in pairlist]
-        pairlist = [
+        df = df[df[self.pair_exchange.TOKEN_COL].isin(pairs)]
-            v for v in pairlist if not df[
+        return [f"{token}/{self._stake_currency}" for token in df[self.pair_exchange.TOKEN_COL]]
-                (df[self.pair_exchange.TOKEN_COL] == get_token_from_pair(v, 0)) &
-                (df[self.pair_exchange.ANNOUNCEMENT_COL] > (
-                    datetime.now().replace(tzinfo=pytz.utc) - timedelta(hours=self._hours)
-                ))
-            ].empty
-        ]
-        return pairlist
 
-    def _init_pair_exchange(self, **pair_exchange_kwargs):
+    def _init_pair_exchange(self, config, **pair_exchange_kwargs):
         exchange = None
         if self._pair_exchange == 'binance':
-            exchange = BinanceAnnouncement(refresh_period=self._refresh_period, **pair_exchange_kwargs)
+            exchange = BinanceAnnouncement(
+                db_path=config.get('ba_database_uri', None),
+                refresh_period=self._refresh_period,
+                **pair_exchange_kwargs
+            )
 
         if exchange:
             assert hasattr(exchange, 'update_announcements'), '`update_announcements` method is required'
@@ -335,6 +352,9 @@ class AnnouncementsPairList(IPairList):
 
         raise OperationalException(f'Exchange `{self._pair_exchange}` is not supported yet')
 
+    def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
+        return NotImplemented
+
 
 class StaticAnnouncementsPairList(AnnouncementsPairList):
     STATIC = True