Merge branch 'develop' into backtest_live_models

docs/freqai-parameter-table.md

@@ -18,6 +18,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)). <br> **Datatype:** Positive integer.
 | `follow_mode` | Use a `follower` that will look for models associated with a specific `identifier` and load those for inferencing. A `follower` will **not** train new models. <br> **Datatype:** Boolean. <br> Default: `False`.
 | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)). <br> **Datatype:** Boolean. <br> Default: `False`.
+| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file. <br> **Datatype:** Boolean. <br> Default: `False`
 |  |  **Feature parameters**
 | `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md). <br> **Datatype:** Dictionary.
 | `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base indicators dataset. <br> **Datatype:** List of timeframes (strings).
@@ -37,7 +38,6 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `noise_standard_deviation` | If set, FreqAI adds noise to the training features with the aim of preventing overfitting. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in FreqAI is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation). <br> **Datatype:** Integer. <br> Default: `0`.
 | `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset. <br> **Datatype:** Float. <br> Default: `30`.
 | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it. <br> **Datatype:** Boolean. <br> Default: `False` (no reversal).
-| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file. <br> **Datatype:** Boolean. <br> Default: `False`
 |  |  **Data split parameters**
 | `data_split_parameters` | Include any additional parameters available from Scikit-learn `test_train_split()`, which are shown [here](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) (external website). <br> **Datatype:** Dictionary.
 | `test_size` | The fraction of data that should be used for testing instead of training. <br> **Datatype:** Positive float < 1.

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
-mkdocs==1.4.1
+mkdocs==1.4.2
-mkdocs-material==8.5.7
+mkdocs-material==8.5.8
 mdx_truly_sane_lists==1.3
 pymdown-extensions==9.7
 jinja2==3.1.2

freqtrade/constants.py

@@ -542,7 +542,7 @@ CONF_SCHEMA = {
                 "keras": {"type": "boolean", "default": False},
                 "write_metrics_to_disk": {"type": "boolean", "default": False},
                 "purge_old_models": {"type": "boolean", "default": True},
-                "conv_width": {"type": "integer", "default": 2},
+                "conv_width": {"type": "integer", "default": 1},
                 "train_period_days": {"type": "integer", "default": 0},
                 "backtest_period_days": {"type": "number", "default": 7},
                 "identifier": {"type": "string", "default": "example"},

freqtrade/data/btanalysis.py

@@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
                    'profit_ratio', 'profit_abs', 'exit_reason',
                    'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
                    'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag',
-                   'is_short', 'open_timestamp', 'close_timestamp', 'orders'
+                   'leverage', 'is_short', 'open_timestamp', 'close_timestamp', 'orders'
                    ]
 
 
@@ -280,6 +280,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
             # Compatibility support for pre short Columns
             if 'is_short' not in df.columns:
                 df['is_short'] = 0
+            if 'leverage' not in df.columns:
+                df['leverage'] = 1.0
             if 'enter_tag' not in df.columns:
                 df['enter_tag'] = df['buy_tag']
                 df = df.drop(['buy_tag'], axis=1)

freqtrade/data/history/idatahandler.py

@@ -102,6 +102,11 @@ class IDataHandler(ABC):
         :return: (min, max)
         """
         data = self._ohlcv_load(pair, timeframe, None, candle_type)
+        if data.empty:
+            return (
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                datetime.fromtimestamp(0, tz=timezone.utc)
+            )
         return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime()
 
     @abstractmethod

freqtrade/exchange/exchange.py

@@ -1689,6 +1689,17 @@ class Exchange:
     @retrier
     def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1,
                 price: float = 1, taker_or_maker: MakerTaker = 'maker') -> float:
+        """
+        Retrieve fee from exchange
+        :param symbol: Pair
+        :param type: Type of order (market, limit, ...)
+        :param side: Side of order (buy, sell)
+        :param amount: Amount of order
+        :param price: Price of order
+        :param taker_or_maker: 'maker' or 'taker' (ignored if "type" is provided)
+        """
+        if type and type == 'market':
+            taker_or_maker = 'taker'
         try:
             if self._config['dry_run'] and self._config.get('fee', None) is not None:
                 return self._config['fee']

freqtrade/freqai/data_drawer.py

@@ -636,6 +636,8 @@ class FreqaiDataDrawer:
                         axis=0,
                     )
 
+            self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date']
+
     def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None:
         """
         Load pair histories for all whitelist and corr_pairlist pairs.

freqtrade/freqai/data_kitchen.py

@@ -1177,11 +1177,13 @@ class FreqaiDataKitchen:
         pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
 
         for pair in pairs:
+            pair = pair.replace(':', '')  # lightgbm doesnt like colons
             valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"]
             pair_cols = [col for col in dataframe.columns if
                          any(substr in col for substr in valid_strs)]
-            pair_cols.insert(0, 'date')
+            if pair_cols:
-            corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
+                pair_cols.insert(0, 'date')
+                corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
 
         return corr_dataframes
 
@@ -1199,8 +1201,9 @@ class FreqaiDataKitchen:
                     ready for training
         """
         pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
-
+        current_pair = current_pair.replace(':', '')
         for pair in pairs:
+            pair = pair.replace(':', '')  # lightgbm doesnt work with colons
             if current_pair != pair:
                 dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date')
 
@@ -1270,6 +1273,8 @@ class FreqaiDataKitchen:
 
         self.get_unique_classes_from_labels(dataframe)
 
+        dataframe = self.remove_special_chars_from_feature_names(dataframe)
+
         return dataframe
 
     def fit_labels(self) -> None:
@@ -1471,3 +1476,16 @@ class FreqaiDataKitchen:
                     assets_end_dates[asset].append(model_end_date)
 
         return assets_end_dates
+
+    def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        """
+        Remove all special characters from feature strings (:)
+        :param dataframe: the dataframe that just finished indicator population. (unfiltered)
+        :return: dataframe with cleaned featrue names
+        """
+
+        spec_chars = [':']
+        for c in spec_chars:
+            dataframe.columns = dataframe.columns.str.replace(c, "")
+
+        return dataframe

freqtrade/freqai/freqai_interface.py

@@ -68,6 +68,9 @@ class IFreqaiModel(ABC):
         if self.save_backtest_models:
             logger.info('Backtesting module configured to save all models.')
         self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
+        # set current candle to arbitrary historical date
+        self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
+        self.dd.current_candle = self.current_candle
         self.scanning = False
         self.ft_params = self.freqai_info["feature_parameters"]
         self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", [])
@@ -75,7 +78,7 @@ class IFreqaiModel(ABC):
         if self.keras and self.ft_params.get("DI_threshold", 0):
             self.ft_params["DI_threshold"] = 0
             logger.warning("DI threshold is not configured for Keras models yet. Deactivating.")
-        self.CONV_WIDTH = self.freqai_info.get("conv_width", 2)
+        self.CONV_WIDTH = self.freqai_info.get('conv_width', 1)
         if self.ft_params.get("inlier_metric_window", 0):
             self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
         self.pair_it = 0
@@ -93,7 +96,6 @@ class IFreqaiModel(ABC):
         # get_corr_dataframes is controlling the caching of corr_dataframes
         # for improved performance. Careful with this boolean.
         self.get_corr_dataframes: bool = True
-
         self._threads: List[threading.Thread] = []
         self._stop_event = threading.Event()
 
@@ -338,6 +340,7 @@ class IFreqaiModel(ABC):
         if self.dd.historic_data:
             self.dd.update_historic_data(strategy, dk)
             logger.debug(f'Updating historic data on pair {metadata["pair"]}')
+            self.track_current_candle()
 
         if not self.follow_mode:
 
@@ -682,8 +685,6 @@ class IFreqaiModel(ABC):
                                    " avoid blinding open trades and degrading performance.")
                 self.pair_it = 0
                 self.inference_time = 0
-                if self.corr_pairlist:
-                    self.get_corr_dataframes = True
         return
 
     def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''):
@@ -759,12 +760,24 @@ class IFreqaiModel(ABC):
                                "is included in the column names when you are creating features "
                                "in `populate_any_indicators()`.")
             self.get_corr_dataframes = not bool(self.corr_dataframes)
-        else:
+        elif self.corr_dataframes:
             dataframe = dk.attach_corr_pair_columns(
                 dataframe, self.corr_dataframes, dk.pair)
 
         return dataframe
 
+    def track_current_candle(self):
+        """
+        Checks if the latest candle appended by the datadrawer is
+        equivalent to the latest candle seen by FreqAI. If not, it
+        asks to refresh the cached corr_dfs, and resets the pair
+        counter.
+        """
+        if self.dd.current_candle > self.current_candle:
+            self.get_corr_dataframes = True
+            self.pair_it = 1
+            self.current_candle = self.dd.current_candle
+
     def ensure_data_exists(self, dataframe_backtest: DataFrame,
                            tr_backtest: TimeRange, pair: str) -> bool:
         """

requirements-dev.txt

@@ -10,7 +10,7 @@ flake8==5.0.4
 flake8-tidy-imports==4.8.0
 mypy==0.982
 pre-commit==2.20.0
-pytest==7.1.3
+pytest==7.2.0
 pytest-asyncio==0.20.1
 pytest-cov==4.0.0
 pytest-mock==3.10.0

requirements-hyperopt.txt

@@ -6,4 +6,4 @@ scipy==1.9.3
 scikit-learn==1.1.3
 scikit-optimize==0.9.0
 filelock==3.8.0
-progressbar2==4.1.1
+progressbar2==4.2.0

requirements.txt

@@ -2,17 +2,17 @@ numpy==1.23.4
 pandas==1.5.1
 pandas-ta==0.3.14b
 
-ccxt==2.0.96
+ccxt==2.1.33
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==38.0.1
 aiohttp==3.8.3
-SQLAlchemy==1.4.42
+SQLAlchemy==1.4.43
 python-telegram-bot==13.14
 arrow==1.2.3
 cachetools==4.2.2
 requests==2.28.1
 urllib3==1.26.12
-jsonschema==4.16.0
+jsonschema==4.17.0
 TA-Lib==0.4.25
 technical==1.3.0
 tabulate==0.9.0
@@ -29,7 +29,7 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.9
 # Properly format api responses
-orjson==3.8.0
+orjson==3.8.1
 
 # Notify systemd
 sdnotify==0.3.2
@@ -46,7 +46,7 @@ psutil==5.9.3
 colorama==0.4.6
 # Building config files interactively
 questionary==1.10.0
-prompt-toolkit==3.0.31
+prompt-toolkit==3.0.32
 # Extensions to datetime library
 python-dateutil==2.8.2
 

tests/data/test_datahandler.py

@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 
 import re
+from datetime import datetime, timezone
 from pathlib import Path
 from unittest.mock import MagicMock
 
@@ -154,6 +155,23 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
     assert df.columns.equals(df1.columns)
 
 
+def test_datahandler_ohlcv_data_min_max(testdatadir):
+    dh = JsonDataHandler(testdatadir)
+    min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '5m', 'spot')
+    assert len(min_max) == 2
+
+    # Empty pair
+    min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '8m', 'spot')
+    assert len(min_max) == 2
+    assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc)
+    assert min_max[0] == min_max[1]
+    # Empty pair2
+    min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '4m', 'spot')
+    assert len(min_max) == 2
+    assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc)
+    assert min_max[0] == min_max[1]
+
+
 def test_datahandler__check_empty_df(testdatadir, caplog):
     dh = JsonDataHandler(testdatadir)
     expected_text = r"Price jump in UNITTEST/USDT, 1h, spot between"

tests/freqai/test_freqai_datadrawer.py

@@ -22,6 +22,7 @@ def test_update_historic_data(mocker, freqai_conf):
     historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
     dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m"))
     candle_difference = dp_candles - historic_candles
+    freqai.dk.pair = "ADA/BTC"
     freqai.dd.update_historic_data(strategy, freqai.dk)
 
     updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])

tests/freqai/test_freqai_interface.py

@@ -194,6 +194,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
     corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
 
     df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
+    df = freqai.cache_corr_pairlist_dfs(df, freqai.dk)
     for i in range(5):
         df[f'%-constant_{i}'] = i
         # df.loc[:, f'%-constant_{i}'] = i
@@ -339,6 +340,7 @@ def test_follow_mode(mocker, freqai_conf):
 
     df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m')
 
+    freqai.dk.pair = "ADA/BTC"
     freqai.start_live(df, metadata, strategy, freqai.dk)
 
     assert len(freqai.dk.return_dataframe.index) == 5702

tests/optimize/test_backtesting.py

@@ -764,6 +764,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
          'max_rate': [0.10501, 0.1038888],
          'is_open': [False, False],
          'enter_tag': [None, None],
+         "leverage": [1.0, 1.0],
          "is_short": [False, False],
          'open_timestamp': [1517251200000, 1517283000000],
          'close_timestamp': [1517265300000, 1517285400000],
@@ -788,13 +789,14 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
         assert len(t['orders']) == 2
         ln = data_pair.loc[data_pair["date"] == t["open_date"]]
         # Check open trade rate alignes to open rate
-        assert ln is not None
+        assert not ln.empty
         assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6)
         # check close trade rate alignes to close rate or is between high and low
-        ln = data_pair.loc[data_pair["date"] == t["close_date"]]
+        ln1 = data_pair.loc[data_pair["date"] == t["close_date"]]
-        assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or
+        assert not ln1.empty
-                round(ln.iloc[0]["low"], 6) < round(
+        assert (round(ln1.iloc[0]["open"], 6) == round(t["close_rate"], 6) or
-                t["close_rate"], 6) < round(ln.iloc[0]["high"], 6))
+                round(ln1.iloc[0]["low"], 6) < round(
+                t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6))
 
 
 def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) -> None:

tests/optimize/test_backtesting_adjust_position.py

@@ -72,6 +72,7 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
          'max_rate': [0.10481985, 0.1038888],
          'is_open': [False, False],
          'enter_tag': [None, None],
+         'leverage': [1.0, 1.0],
          'is_short': [False, False],
          'open_timestamp': [1517251200000, 1517283000000],
          'close_timestamp': [1517265300000, 1517285400000],

tests/test_freqtradebot.py

@@ -5305,7 +5305,7 @@ def test_get_valid_price(mocker, default_conf_usdt) -> None:
 ])
 def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, time_machine,
                                       t1, t2):
-    time_machine.move_to(f"{t1} +00:00")
+    time_machine.move_to(f"{t1} +00:00", tick=False)
 
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -5314,7 +5314,7 @@ def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls,
     default_conf['margin_mode'] = 'isolated'
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    time_machine.move_to(f"{t2} +00:00")
+    time_machine.move_to(f"{t2} +00:00", tick=False)
     # Check schedule jobs in debugging with freqtrade._schedule.jobs
     freqtrade._schedule.run_pending()