Improve imports

docs/backtesting.md

@@ -274,20 +274,19 @@ A backtesting result will look like that:
 | XRP/BTC  |      35 |           0.66 |          22.96 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
 | ZEC/BTC  |      22 |          -0.46 |         -10.18 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
 | TOTAL    |     429 |           0.36 |         152.41 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
-====================================================== LEFT OPEN TRADES REPORT ======================================================
-| Pair     |  Entries |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
-|:---------|---------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
-| ADA/BTC  |        1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
-| LTC/BTC  |        1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
-| TOTAL    |        2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
-==================== EXIT REASON STATS ====================
+========================================================= EXIT REASON STATS ==========================================================
 | Exit Reason        |   Exits |  Wins |  Draws |  Losses |
 |:-------------------|--------:|------:|-------:|--------:|
 | trailing_stop_loss |     205 |   150 |      0 |      55 |
 | stop_loss          |     166 |     0 |      0 |     166 |
 | exit_signal        |      56 |    36 |      0 |      20 |
 | force_exit         |       2 |     0 |      0 |       2 |
-
+====================================================== LEFT OPEN TRADES REPORT ======================================================
+| Pair     |  Entries |   Avg Profit % |   Cum Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
+|:---------|---------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
+| ADA/BTC  |        1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
+| LTC/BTC  |        1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
+| TOTAL    |        2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
 ================== SUMMARY METRICS ==================
 | Metric                      | Value               |
 |-----------------------------+---------------------|

docs/telegram-usage.md

@@ -279,6 +279,7 @@ Return a summary of your profit/loss and performance.
 >   ∙ `33.095 EUR`
 >
 > **Total Trade Count:** `138`
+> **Bot started:** `2022-07-11 18:40:44`
 > **First Trade opened:** `3 days ago`
 > **Latest Trade opened:** `2 minutes ago`
 > **Avg. Duration:** `2:33:45`
@@ -292,6 +293,7 @@ The relative profit of `15.2 Σ%` is be based on the starting capital - so in th
 Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits.
 Profit Factor is calculated as gross profits / gross losses - and should serve as an overall metric for the strategy.
 Max drawdown corresponds to the backtesting metric `Absolute Drawdown (Account)` - calculated as `(Absolute Drawdown) / (DrawdownHigh + startingBalance)`.
+Bot started date will refer to the date the bot was first started. For older bots, this will default to the first trade's open date.
 
 ### /forceexit <trade_id>
 

freqtrade/freqai/data_kitchen.py

@@ -1291,7 +1291,7 @@ class FreqaiDataKitchen:
 
         return dataframe
 
-    def use_strategy_to_populate_indicators(  # noqa: C901
+    def use_strategy_to_populate_indicators(
         self,
         strategy: IStrategy,
         corr_dataframes: dict = {},
@@ -1362,12 +1362,12 @@ class FreqaiDataKitchen:
                 dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
                                                    corr_dataframes, base_dataframes, True)
 
-        if self.live:
         dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata)
-            dataframe = self.remove_special_chars_from_feature_names(dataframe)
 
         self.get_unique_classes_from_labels(dataframe)
 
+        dataframe = self.remove_special_chars_from_feature_names(dataframe)
+
         if self.config.get('reduce_df_footprint', False):
             dataframe = reduce_dataframe_footprint(dataframe)
 

freqtrade/freqai/freqai_interface.py

@@ -306,7 +306,7 @@ class IFreqaiModel(ABC):
                 if check_features:
                     self.dd.load_metadata(dk)
                     dataframe_dummy_features = self.dk.use_strategy_to_populate_indicators(
-                        strategy, prediction_dataframe=dataframe.tail(1), pair=pair
+                        strategy, prediction_dataframe=dataframe.tail(1), pair=metadata["pair"]
                     )
                     dk.find_features(dataframe_dummy_features)
                     self.check_if_feature_list_matches_strategy(dk)
@@ -316,7 +316,7 @@ class IFreqaiModel(ABC):
             else:
                 if populate_indicators:
                     dataframe = self.dk.use_strategy_to_populate_indicators(
-                        strategy, prediction_dataframe=dataframe, pair=pair
+                        strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
                     )
                     populate_indicators = False
 
@@ -332,10 +332,6 @@ class IFreqaiModel(ABC):
                 dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train)
                 dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest)
 
-                dataframe_train = dk.remove_special_chars_from_feature_names(dataframe_train)
-                dataframe_backtest = dk.remove_special_chars_from_feature_names(dataframe_backtest)
-                dk.get_unique_classes_from_labels(dataframe_train)
-
                 if not self.model_exists(dk):
                     dk.find_features(dataframe_train)
                     dk.find_labels(dataframe_train)

freqtrade/freqtradebot.py

@@ -26,6 +26,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, timeframe_to_minutes, time
 from freqtrade.misc import safe_value_fallback, safe_value_fallback2
 from freqtrade.mixins import LoggingMixin
 from freqtrade.persistence import Order, PairLocks, Trade, init_db
+from freqtrade.persistence.key_value_store import set_startup_time
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -182,6 +183,7 @@ class FreqtradeBot(LoggingMixin):
         performs startup tasks
         """
         migrate_binance_futures_names(self.config)
+        set_startup_time()
 
         self.rpc.startup_messages(self.config, self.pairlists, self.protections)
         # Update older trades with precision and precision mode

freqtrade/loggers.py

@@ -1,11 +1,24 @@
 import logging
+import sys
 from logging import Formatter
-from logging.handlers import RotatingFileHandler, SysLogHandler
+from logging.handlers import BufferingHandler, RotatingFileHandler, SysLogHandler
 
 from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
-from freqtrade.loggers.buffering_handler import FTBufferingHandler
-from freqtrade.loggers.std_err_stream_handler import FTStdErrStreamHandler
+
+
+class FTBufferingHandler(BufferingHandler):
+    def flush(self):
+        """
+        Override Flush behaviour - we keep half of the configured capacity
+        otherwise, we have moments with "empty" logs.
+        """
+        self.acquire()
+        try:
+            # Keep half of the records in buffer.
+            self.buffer = self.buffer[-int(self.capacity / 2):]
+        finally:
+            self.release()
 
 
 logger = logging.getLogger(__name__)
@@ -56,7 +69,7 @@ def setup_logging_pre() -> None:
     logging.basicConfig(
         level=logging.INFO,
         format=LOGFORMAT,
-        handlers=[FTStdErrStreamHandler(), bufferHandler]
+        handlers=[logging.StreamHandler(sys.stderr), bufferHandler]
     )
 
 

freqtrade/loggers/buffering_handler.py

@@ -1,15 +0,0 @@
-from logging.handlers import BufferingHandler
-
-
-class FTBufferingHandler(BufferingHandler):
-    def flush(self):
-        """
-        Override Flush behaviour - we keep half of the configured capacity
-        otherwise, we have moments with "empty" logs.
-        """
-        self.acquire()
-        try:
-            # Keep half of the records in buffer.
-            self.buffer = self.buffer[-int(self.capacity / 2):]
-        finally:
-            self.release()

freqtrade/loggers/std_err_stream_handler.py

@@ -1,26 +0,0 @@
-import sys
-from logging import Handler
-
-
-class FTStdErrStreamHandler(Handler):
-    def flush(self):
-        """
-        Override Flush behaviour - we keep half of the configured capacity
-        otherwise, we have moments with "empty" logs.
-        """
-        self.acquire()
-        try:
-            sys.stderr.flush()
-        finally:
-            self.release()
-
-    def emit(self, record):
-        try:
-            msg = self.format(record)
-            # Don't keep a reference to stderr - this can be problematic with progressbars.
-            sys.stderr.write(msg + '\n')
-            self.flush()
-        except RecursionError:
-            raise
-        except Exception:
-            self.handleError(record)

freqtrade/optimize/hyperopt.py

@@ -13,13 +13,13 @@ from math import ceil
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
+import progressbar
 import rapidjson
+from colorama import Fore, Style
 from colorama import init as colorama_init
 from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
 from joblib.externals import cloudpickle
 from pandas import DataFrame
-from rich.progress import (BarColumn, MofNCompleteColumn, Progress, TaskProgressColumn, TextColumn,
-                           TimeElapsedColumn, TimeRemainingColumn)
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
 from freqtrade.data.converter import trim_dataframes
@@ -44,6 +44,8 @@ with warnings.catch_warnings():
     from skopt import Optimizer
     from skopt.space import Dimension
 
+progressbar.streams.wrap_stderr()
+progressbar.streams.wrap_stdout()
 logger = logging.getLogger(__name__)
 
 
@@ -518,6 +520,29 @@ class Hyperopt:
         else:
             return self.opt.ask(n_points=n_points), [False for _ in range(n_points)]
 
+    def get_progressbar_widgets(self):
+        if self.print_colorized:
+            widgets = [
+                ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
+                ' (', progressbar.Percentage(), ')] ',
+                progressbar.Bar(marker=progressbar.AnimatedMarker(
+                    fill='\N{FULL BLOCK}',
+                    fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
+                    marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
+                )),
+                ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
+            ]
+        else:
+            widgets = [
+                ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
+                ' (', progressbar.Percentage(), ')] ',
+                progressbar.Bar(marker=progressbar.AnimatedMarker(
+                    fill='\N{FULL BLOCK}',
+                )),
+                ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
+            ]
+        return widgets
+
     def evaluate_result(self, val: Dict[str, Any], current: int, is_random: bool):
         """
         Evaluate results returned from generate_optimizer
@@ -577,19 +602,11 @@ class Hyperopt:
                 logger.info(f'Effective number of parallel workers used: {jobs}')
 
                 # Define progressbar
-                with Progress(
-                    TextColumn("[progress.description]{task.description}"),
-                    BarColumn(bar_width=None),
-                    MofNCompleteColumn(),
-                    TaskProgressColumn(),
-                    "•",
-                    TimeElapsedColumn(),
-                    "•",
-                    TimeRemainingColumn(),
-                    expand=True,
+                widgets = self.get_progressbar_widgets()
+                with progressbar.ProgressBar(
+                    max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
+                    widgets=widgets
                 ) as pbar:
-                    task = pbar.add_task("Epochs", total=self.total_epochs)
-
                     start = 0
 
                     if self.analyze_per_epoch:
@@ -599,7 +616,7 @@ class Hyperopt:
                         f_val0 = self.generate_optimizer(asked[0])
                         self.opt.tell(asked, [f_val0['loss']])
                         self.evaluate_result(f_val0, 1, is_random[0])
-                        pbar.update(task, advance=1)
+                        pbar.update(1)
                         start += 1
 
                     evals = ceil((self.total_epochs - start) / jobs)
@@ -613,12 +630,14 @@ class Hyperopt:
                         f_val = self.run_optimizer_parallel(parallel, asked)
                         self.opt.tell(asked, [v['loss'] for v in f_val])
 
+                        # Calculate progressbar outputs
                         for j, val in enumerate(f_val):
                             # Use human-friendly indexes here (starting from 1)
                             current = i * jobs + j + 1 + start
 
                             self.evaluate_result(val, current, is_random[j])
-                            pbar.update(task, advance=1)
+
+                            pbar.update(current)
 
         except KeyboardInterrupt:
             print('User interrupted..')

freqtrade/optimize/optimize_reports.py

@@ -865,11 +865,6 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
         print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
     print(table)
 
-    table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
-    if isinstance(table, str) and len(table) > 0:
-        print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
-    print(table)
-
     if (results.get('results_per_enter_tag') is not None
             or results.get('results_per_buy_tag') is not None):
         # results_per_buy_tag is deprecated and should be removed 2 versions after short golive.
@@ -889,6 +884,11 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
         print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '='))
     print(table)
 
+    table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
+    if isinstance(table, str) and len(table) > 0:
+        print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
+    print(table)
+
     for period in backtest_breakdown:
         days_breakdown_stats = generate_periodic_breakdown_stats(
             trade_list=results['trades'], period=period)
@@ -917,11 +917,11 @@ def show_backtest_results(config: Config, backtest_stats: Dict):
             strategy, results, stake_currency,
             config.get('backtest_breakdown', []))
 
-    if len(backtest_stats['strategy']) > 0:
+    if len(backtest_stats['strategy']) > 1:
         # Print Strategy summary table
 
         table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency)
-        print(f"Backtested {results['backtest_start']} -> {results['backtest_end']} |"
+        print(f"{results['backtest_start']} -> {results['backtest_end']} |"
               f" Max open trades : {results['max_open_trades']}")
         print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
         print(table)

freqtrade/persistence/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade

freqtrade/persistence/key_value_store.py

@@ -0,0 +1,179 @@
+from datetime import datetime, timezone
+from enum import Enum
+from typing import ClassVar, Optional, Union
+
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from freqtrade.persistence.base import ModelBase, SessionType
+
+
+ValueTypes = Union[str, datetime, float, int]
+
+
+class ValueTypesEnum(str, Enum):
+    STRING = 'str'
+    DATETIME = 'datetime'
+    FLOAT = 'float'
+    INT = 'int'
+
+
+class KeyStoreKeys(str, Enum):
+    BOT_START_TIME = 'bot_start_time'
+    STARTUP_TIME = 'startup_time'
+
+
+class _KeyValueStoreModel(ModelBase):
+    """
+    Pair Locks database model.
+    """
+    __tablename__ = 'KeyValueStore'
+    session: ClassVar[SessionType]
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+
+    key: Mapped[KeyStoreKeys] = mapped_column(String(25), nullable=False, index=True)
+
+    value_type: Mapped[ValueTypesEnum] = mapped_column(String(20), nullable=False)
+
+    string_value: Mapped[Optional[str]]
+    datetime_value: Mapped[Optional[datetime]]
+    float_value: Mapped[Optional[float]]
+    int_value: Mapped[Optional[int]]
+
+
+class KeyValueStore():
+    """
+    Generic bot-wide, persistent key-value store
+    Can be used to store generic values, e.g. very first bot startup time.
+    Supports the types str, datetime, float and int.
+    """
+
+    @staticmethod
+    def store_value(key: KeyStoreKeys, value: ValueTypes) -> None:
+        """
+        Store the given value for the given key.
+        :param key: Key to store the value for - can be used in get-value to retrieve the key
+        :param value: Value to store - can be str, datetime, float or int
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key).first()
+        if kv is None:
+            kv = _KeyValueStoreModel(key=key)
+        if isinstance(value, str):
+            kv.value_type = ValueTypesEnum.STRING
+            kv.string_value = value
+        elif isinstance(value, datetime):
+            kv.value_type = ValueTypesEnum.DATETIME
+            kv.datetime_value = value
+        elif isinstance(value, float):
+            kv.value_type = ValueTypesEnum.FLOAT
+            kv.float_value = value
+        elif isinstance(value, int):
+            kv.value_type = ValueTypesEnum.INT
+            kv.int_value = value
+        else:
+            raise ValueError(f'Unknown value type {kv.value_type}')
+        _KeyValueStoreModel.session.add(kv)
+        _KeyValueStoreModel.session.commit()
+
+    @staticmethod
+    def delete_value(key: KeyStoreKeys) -> None:
+        """
+        Delete the value for the given key.
+        :param key: Key to delete the value for
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key).first()
+        if kv is not None:
+            _KeyValueStoreModel.session.delete(kv)
+            _KeyValueStoreModel.session.commit()
+
+    @staticmethod
+    def get_value(key: KeyStoreKeys) -> Optional[ValueTypes]:
+        """
+        Get the value for the given key.
+        :param key: Key to get the value for
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key).first()
+        if kv is None:
+            return None
+        if kv.value_type == ValueTypesEnum.STRING:
+            return kv.string_value
+        if kv.value_type == ValueTypesEnum.DATETIME and kv.datetime_value is not None:
+            return kv.datetime_value.replace(tzinfo=timezone.utc)
+        if kv.value_type == ValueTypesEnum.FLOAT:
+            return kv.float_value
+        if kv.value_type == ValueTypesEnum.INT:
+            return kv.int_value
+        # This should never happen unless someone messed with the database manually
+        raise ValueError(f'Unknown value type {kv.value_type}')  # pragma: no cover
+
+    @staticmethod
+    def get_string_value(key: KeyStoreKeys) -> Optional[str]:
+        """
+        Get the value for the given key.
+        :param key: Key to get the value for
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key,
+            _KeyValueStoreModel.value_type == ValueTypesEnum.STRING).first()
+        if kv is None:
+            return None
+        return kv.string_value
+
+    @staticmethod
+    def get_datetime_value(key: KeyStoreKeys) -> Optional[datetime]:
+        """
+        Get the value for the given key.
+        :param key: Key to get the value for
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key,
+            _KeyValueStoreModel.value_type == ValueTypesEnum.DATETIME).first()
+        if kv is None or kv.datetime_value is None:
+            return None
+        return kv.datetime_value.replace(tzinfo=timezone.utc)
+
+    @staticmethod
+    def get_float_value(key: KeyStoreKeys) -> Optional[float]:
+        """
+        Get the value for the given key.
+        :param key: Key to get the value for
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key,
+            _KeyValueStoreModel.value_type == ValueTypesEnum.FLOAT).first()
+        if kv is None:
+            return None
+        return kv.float_value
+
+    @staticmethod
+    def get_int_value(key: KeyStoreKeys) -> Optional[int]:
+        """
+        Get the value for the given key.
+        :param key: Key to get the value for
+        """
+        kv = _KeyValueStoreModel.session.query(_KeyValueStoreModel).filter(
+            _KeyValueStoreModel.key == key,
+            _KeyValueStoreModel.value_type == ValueTypesEnum.INT).first()
+        if kv is None:
+            return None
+        return kv.int_value
+
+
+def set_startup_time():
+    """
+    sets bot_start_time to the first trade open date - or "now" on new databases.
+    sets startup_time to "now"
+    """
+    st = KeyValueStore.get_value('bot_start_time')
+    if st is None:
+        from freqtrade.persistence import Trade
+        t = Trade.session.query(Trade).order_by(Trade.open_date.asc()).first()
+        if t is not None:
+            KeyValueStore.store_value('bot_start_time', t.open_date_utc)
+        else:
+            KeyValueStore.store_value('bot_start_time', datetime.now(timezone.utc))
+    KeyValueStore.store_value('startup_time', datetime.now(timezone.utc))

freqtrade/persistence/models.py

@@ -13,6 +13,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import ModelBase
+from freqtrade.persistence.key_value_store import _KeyValueStoreModel
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
@@ -76,6 +77,7 @@ def init_db(db_url: str) -> None:
         bind=engine, autoflush=False), scopefunc=get_request_or_thread_id)
     Order.session = Trade.session
     PairLock.session = Trade.session
+    _KeyValueStoreModel.session = Trade.session
 
     previous_tables = inspect(engine).get_table_names()
     ModelBase.metadata.create_all(engine)

freqtrade/rpc/api_server/api_schemas.py

@@ -108,6 +108,8 @@ class Profit(BaseModel):
     max_drawdown: float
     max_drawdown_abs: float
     trading_volume: Optional[float]
+    bot_start_timestamp: int
+    bot_start_date: str
 
 
 class SellReason(BaseModel):

freqtrade/rpc/rpc.py

@@ -26,7 +26,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
 from freqtrade.misc import decimals_per_coin, shorten_date
-from freqtrade.persistence import Order, PairLocks, Trade
+from freqtrade.persistence import KeyStoreKeys, KeyValueStore, Order, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -543,6 +543,7 @@ class RPC:
         first_date = trades[0].open_date if trades else None
         last_date = trades[-1].open_date if trades else None
         num = float(len(durations) or 1)
+        bot_start = KeyValueStore.get_datetime_value(KeyStoreKeys.BOT_START_TIME)
         return {
             'profit_closed_coin': profit_closed_coin_sum,
             'profit_closed_percent_mean': round(profit_closed_ratio_mean * 100, 2),
@@ -576,6 +577,8 @@ class RPC:
             'max_drawdown': max_drawdown,
             'max_drawdown_abs': max_drawdown_abs,
             'trading_volume': trading_volume,
+            'bot_start_timestamp': int(bot_start.timestamp() * 1000) if bot_start else 0,
+            'bot_start_date': bot_start.strftime(DATETIME_PRINT_FORMAT) if bot_start else '',
         }
 
     def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:

freqtrade/rpc/telegram.py

@@ -819,7 +819,7 @@ class Telegram(RPCHandler):
         best_pair = stats['best_pair']
         best_pair_profit_ratio = stats['best_pair_profit_ratio']
         if stats['trade_count'] == 0:
-            markdown_msg = 'No trades yet.'
+            markdown_msg = f"No trades yet.\n*Bot started:* `{stats['bot_start_date']}`"
         else:
             # Message to display
             if stats['closed_trade_count'] > 0:
@@ -838,6 +838,7 @@ class Telegram(RPCHandler):
                 f"({profit_all_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
                 f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n"
                 f"*Total Trade Count:* `{trade_count}`\n"
+                f"*Bot started:* `{stats['bot_start_date']}`\n"
                 f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* "
                 f"`{first_trade_date}`\n"
                 f"*Latest Trade opened:* `{latest_trade_date}`\n"

requirements-hyperopt.txt

@@ -6,3 +6,4 @@ scipy==1.10.1
 scikit-learn==1.1.3
 scikit-optimize==0.9.0
 filelock==3.10.6
+progressbar2==4.2.0

requirements.txt

@@ -20,7 +20,6 @@ jinja2==3.1.2
 tables==3.8.0
 blosc==1.11.1
 joblib==1.2.0
-rich==13.3.3
 pyarrow==11.0.0; platform_machine != 'armv7l'
 
 # find first, C search in arrays

setup.py

@@ -8,6 +8,7 @@ hyperopt = [
     'scikit-learn',
     'scikit-optimize>=0.7.0',
     'filelock',
+    'progressbar2',
 ]
 
 freqai = [
@@ -81,7 +82,6 @@ setup(
         'numpy',
         'pandas',
         'joblib>=1.2.0',
-        'rich',
         'pyarrow; platform_machine != "armv7l"',
         'fastapi',
         'pydantic>=1.8.0',

tests/freqai/conftest.py

@@ -119,7 +119,6 @@ def make_unfiltered_dataframe(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     freqai.dk.pair = "ADA/BTC"
     data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(data_load_timerange, freqai.dk)
@@ -153,7 +152,6 @@ def make_data_dictionary(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     freqai.dk.pair = "ADA/BTC"
     data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(data_load_timerange, freqai.dk)

tests/freqai/test_freqai_datadrawer.py

@@ -19,7 +19,6 @@ def test_update_historic_data(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180114")
 
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
@@ -42,7 +41,6 @@ def test_load_all_pairs_histories(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180114")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
 
@@ -62,7 +60,6 @@ def test_get_base_and_corr_dataframes(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180114")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
     sub_timerange = TimeRange.parse_timerange("20180111-20180114")
@@ -90,7 +87,6 @@ def test_use_strategy_to_populate_indicators(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180114")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
     sub_timerange = TimeRange.parse_timerange("20180111-20180114")
@@ -107,9 +103,8 @@ def test_get_timerange_from_live_historic_predictions(mocker, freqai_conf):
     exchange = get_patched_exchange(mocker, freqai_conf)
     strategy.dp = DataProvider(freqai_conf, exchange)
     freqai = strategy.freqai
-    freqai.live = False
+    freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = False
     timerange = TimeRange.parse_timerange("20180126-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
     sub_timerange = TimeRange.parse_timerange("20180128-20180130")

tests/freqai/test_freqai_datakitchen.py

@@ -180,7 +180,6 @@ def test_get_full_model_path(mocker, freqai_conf, model):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
 

tests/freqai/test_freqai_interface.py

@@ -87,7 +87,6 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca,
     freqai.live = True
     freqai.can_short = can_short
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     freqai.dk.set_paths('ADA/BTC', 10000)
     timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
@@ -136,7 +135,6 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
 
@@ -180,7 +178,6 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
 
@@ -374,9 +371,6 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog):
     sub_timerange = TimeRange.parse_timerange("20180129-20180130")
     corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
     df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
-    df = strategy.set_freqai_targets(df.copy(), metadata={"pair": "LTC/BTC"})
-    df = freqai.dk.remove_special_chars_from_feature_names(df)
-    freqai.dk.get_unique_classes_from_labels(df)
     freqai.dk.pair = "ADA/BTC"
     freqai.dk.full_df = df.fillna(0)
     freqai.dk.full_df
@@ -400,7 +394,6 @@ def test_principal_component_analysis(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
 
@@ -432,12 +425,10 @@ def test_plot_feature_importance(mocker, freqai_conf):
     freqai = strategy.freqai
     freqai.live = True
     freqai.dk = FreqaiDataKitchen(freqai_conf)
-    freqai.dk.live = True
     timerange = TimeRange.parse_timerange("20180110-20180130")
     freqai.dd.load_all_pair_histories(timerange, freqai.dk)
 
-    freqai.dd.pair_dict = {"ADA/BTC": {"model_filename": "fake_name",
-                                       "trained_timestamp": 1, "data_path": "", "extras": {}}}
+    freqai.dd.pair_dict = MagicMock()
 
     data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
     new_timerange = TimeRange.parse_timerange("20180120-20180130")

tests/persistence/test_key_value_store.py

@@ -0,0 +1,69 @@
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from freqtrade.persistence.key_value_store import KeyValueStore, set_startup_time
+from tests.conftest import create_mock_trades_usdt
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_key_value_store(time_machine):
+    start = datetime(2023, 1, 1, 4, tzinfo=timezone.utc)
+    time_machine.move_to(start, tick=False)
+
+    KeyValueStore.store_value("test", "testStringValue")
+    KeyValueStore.store_value("test_dt", datetime.now(timezone.utc))
+    KeyValueStore.store_value("test_float", 22.51)
+    KeyValueStore.store_value("test_int", 15)
+
+    assert KeyValueStore.get_value("test") == "testStringValue"
+    assert KeyValueStore.get_value("test") == "testStringValue"
+    assert KeyValueStore.get_string_value("test") == "testStringValue"
+    assert KeyValueStore.get_value("test_dt") == datetime.now(timezone.utc)
+    assert KeyValueStore.get_datetime_value("test_dt") == datetime.now(timezone.utc)
+    assert KeyValueStore.get_string_value("test_dt") is None
+    assert KeyValueStore.get_float_value("test_dt") is None
+    assert KeyValueStore.get_int_value("test_dt") is None
+    assert KeyValueStore.get_value("test_float") == 22.51
+    assert KeyValueStore.get_float_value("test_float") == 22.51
+    assert KeyValueStore.get_value("test_int") == 15
+    assert KeyValueStore.get_int_value("test_int") == 15
+    assert KeyValueStore.get_datetime_value("test_int") is None
+
+    time_machine.move_to(start + timedelta(days=20, hours=5), tick=False)
+    assert KeyValueStore.get_value("test_dt") != datetime.now(timezone.utc)
+    assert KeyValueStore.get_value("test_dt") == start
+    # Test update works
+    KeyValueStore.store_value("test_dt", datetime.now(timezone.utc))
+    assert KeyValueStore.get_value("test_dt") == datetime.now(timezone.utc)
+
+    KeyValueStore.store_value("test_float", 23.51)
+    assert KeyValueStore.get_value("test_float") == 23.51
+    # test deleting
+    KeyValueStore.delete_value("test_float")
+    assert KeyValueStore.get_value("test_float") is None
+    # Delete same value again (should not fail)
+    KeyValueStore.delete_value("test_float")
+
+    with pytest.raises(ValueError, match=r"Unknown value type"):
+        KeyValueStore.store_value("test_float", {'some': 'dict'})
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_set_startup_time(fee, time_machine):
+    create_mock_trades_usdt(fee)
+    start = datetime.now(timezone.utc)
+    time_machine.move_to(start, tick=False)
+    set_startup_time()
+
+    assert KeyValueStore.get_value("startup_time") == start
+    initial_time = KeyValueStore.get_value("bot_start_time")
+    assert initial_time <= start
+
+    # Simulate bot restart
+    new_start = start + timedelta(days=5)
+    time_machine.move_to(new_start, tick=False)
+    set_startup_time()
+
+    assert KeyValueStore.get_value("startup_time") == new_start
+    assert KeyValueStore.get_value("bot_start_time") == initial_time

tests/rpc/test_rpc_apiserver.py

@@ -883,6 +883,8 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected)
         'max_drawdown': ANY,
         'max_drawdown_abs': ANY,
         'trading_volume': expected['trading_volume'],
+        'bot_start_timestamp': 0,
+        'bot_start_date': '',
     }
 
 

tests/test_configuration.py

@@ -23,8 +23,7 @@ from freqtrade.configuration.load_config import (load_config_file, load_file, lo
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, _set_loggers,
-                               setup_logging, setup_logging_pre)
+from freqtrade.loggers import FTBufferingHandler, _set_loggers, setup_logging, setup_logging_pre
 from tests.conftest import (CURRENT_TEST_STRATEGY, log_has, log_has_re,
                             patched_configuration_load_config_file)
 
@@ -659,7 +658,7 @@ def test_set_loggers_syslog():
     setup_logging(config)
     assert len(logger.handlers) == 3
     assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler]
-    assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler]
+    assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
     assert [x for x in logger.handlers if type(x) == FTBufferingHandler]
     # setting up logging again should NOT cause the loggers to be added a second time.
     setup_logging(config)
@@ -682,7 +681,7 @@ def test_set_loggers_Filehandler(tmpdir):
     setup_logging(config)
     assert len(logger.handlers) == 3
     assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler]
-    assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler]
+    assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
     assert [x for x in logger.handlers if type(x) == FTBufferingHandler]
     # setting up logging again should NOT cause the loggers to be added a second time.
     setup_logging(config)
@@ -707,7 +706,7 @@ def test_set_loggers_journald(mocker):
     setup_logging(config)
     assert len(logger.handlers) == 3
     assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"]
-    assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler]
+    assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
     # reset handlers to not break pytest
     logger.handlers = orig_handlers