Merge branch 'develop' into freqai_bt_from_predictions_improvement

.github/workflows/ci.yml

@@ -410,7 +410,7 @@ jobs:
         python setup.py sdist bdist_wheel
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.6.1
       if: (github.event_name == 'release')
       with:
         user: __token__
@@ -418,7 +418,7 @@ jobs:
         repository_url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.6.1
       if: (github.event_name == 'release')
       with:
         user: __token__

docs/exchanges.md

@@ -54,6 +54,9 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t
 
 ## Binance
 
+!!! Warning "Server location and geo-ip restrictions"
+    Please be aware that binance restrict api access regarding the server country. The currents and non exhaustive countries blocked are United States, Malaysia (Singapour), Ontario (Canada). Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list.
+
 Binance supports [time_in_force](configuration.md#understand-order_time_in_force).
 
 !!! Tip "Stoploss on Exchange"

docs/freqai-parameter-table.md

@@ -37,7 +37,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `populate_any_indicators()` for indicator creation. FreqAI uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN. <br> **Datatype:** Positive integer.
 | `indicator_periods_candles` | Time periods to calculate indicators for. The indicators are added to the base indicator dataset. <br> **Datatype:** List of positive integers.
 | `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) <br> **Datatype:** Boolean. <br> Default: `False`.
-| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. <br> **Datatype:** Integer. <br> Default: `0`.
+| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. Plot is stored in `user_data/models/<identifier>/sub-train-<COIN>_<timestamp>.html`. <br> **Datatype:** Integer. <br> Default: `0`.
 | `DI_threshold` | Activates the use of the Dissimilarity Index for outlier detection when set to > 0. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Positive float (typically < 1).
 | `use_SVM_to_remove_outliers` | Train a support vector machine to detect and remove outliers from the training dataset, as well as from incoming data points. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm). <br> **Datatype:** Boolean.
 | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. See details about some select parameters [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm). <br> **Datatype:** Dictionary.

docs/freqai-reinforcement-learning.md

@@ -243,7 +243,7 @@ cd freqtrade
 tensorboard --logdir user_data/models/unique-id
 ```
 
-where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6060 (6060 is the default port used by Tensorboard).
+where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard).
 
 ![tensorboard](assets/tensorboard.jpg)
 

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.4.2
-mkdocs-material==8.5.10
+mkdocs-material==8.5.11
 mdx_truly_sane_lists==1.3
-pymdown-extensions==9.8
+pymdown-extensions==9.9
 jinja2==3.1.2

freqtrade/freqai/RL/BaseEnvironment.py

@@ -194,12 +194,12 @@ class BaseEnvironment(gym.Env):
         if self._position == Positions.Neutral:
             return 0.
         elif self._position == Positions.Short:
-            current_price = self.add_exit_fee(self.prices.iloc[self._current_tick].open)
-            last_trade_price = self.add_entry_fee(self.prices.iloc[self._last_trade_tick].open)
-            return (last_trade_price - current_price) / last_trade_price
-        elif self._position == Positions.Long:
             current_price = self.add_entry_fee(self.prices.iloc[self._current_tick].open)
             last_trade_price = self.add_exit_fee(self.prices.iloc[self._last_trade_tick].open)
+            return (last_trade_price - current_price) / last_trade_price
+        elif self._position == Positions.Long:
+            current_price = self.add_exit_fee(self.prices.iloc[self._current_tick].open)
+            last_trade_price = self.add_entry_fee(self.prices.iloc[self._last_trade_tick].open)
             return (current_price - last_trade_price) / last_trade_price
         else:
             return 0.

freqtrade/freqai/RL/BaseReinforcementLearningModel.py

@@ -64,7 +64,7 @@ class BaseReinforcementLearningModel(IFreqaiModel):
         self.policy_type = self.freqai_info['rl_config']['policy_type']
         self.unset_outlier_removal()
         self.net_arch = self.rl_config.get('net_arch', [128, 128])
-        self.dd.model_type = "stable_baselines"
+        self.dd.model_type = import_str
 
     def unset_outlier_removal(self):
         """

freqtrade/freqai/data_drawer.py

@@ -503,7 +503,7 @@ class FreqaiDataDrawer:
             dump(model, save_path / f"{dk.model_filename}_model.joblib")
         elif self.model_type == 'keras':
             model.save(save_path / f"{dk.model_filename}_model.h5")
-        elif 'stable_baselines' in self.model_type:
+        elif 'stable_baselines' in self.model_type or 'sb3_contrib' == self.model_type:
             model.save(save_path / f"{dk.model_filename}_model.zip")
 
         if dk.svm_model is not None:
@@ -589,9 +589,9 @@ class FreqaiDataDrawer:
         elif self.model_type == 'keras':
             from tensorflow import keras
             model = keras.models.load_model(dk.data_path / f"{dk.model_filename}_model.h5")
-        elif self.model_type == 'stable_baselines':
+        elif 'stable_baselines' in self.model_type or 'sb3_contrib' == self.model_type:
             mod = importlib.import_module(
-                'stable_baselines3', self.freqai_info['rl_config']['model_type'])
+                self.model_type, self.freqai_info['rl_config']['model_type'])
             MODELCLASS = getattr(mod, self.freqai_info['rl_config']['model_type'])
             model = MODELCLASS.load(dk.data_path / f"{dk.model_filename}_model")
 

freqtrade/main.py

@@ -7,6 +7,8 @@ import logging
 import sys
 from typing import Any, List
 
+from freqtrade.util.gc_setup import gc_set_threshold
+
 
 # check min. python version
 if sys.version_info < (3, 8):  # pragma: no cover
@@ -36,6 +38,7 @@ def main(sysargv: List[str] = None) -> None:
         # Call subcommand.
         if 'func' in args:
             logger.info(f'freqtrade {__version__}')
+            gc_set_threshold()
             return_code = args['func'](args)
         else:
             # No subcommand was issued.

freqtrade/util/gc_setup.py

@@ -0,0 +1,18 @@
+import gc
+import logging
+import platform
+
+
+logger = logging.getLogger(__name__)
+
+
+def gc_set_threshold():
+    """
+    Reduce number of GC runs to improve performance (explanation video)
+    https://www.youtube.com/watch?v=p4Sn6UcFTOU
+
+    """
+    if platform.python_implementation() == "CPython":
+        # allocs, g1, g2 = gc.get_threshold()
+        gc.set_threshold(50_000, 500, 1000)
+        logger.debug("Adjusting python allocations to reduce GC runs")

requirements-dev.txt

@@ -15,7 +15,7 @@ pytest==7.2.0
 pytest-asyncio==0.20.2
 pytest-cov==4.0.0
 pytest-mock==3.10.0
-pytest-random-order==1.0.4
+pytest-random-order==1.1.0
 isort==5.10.1
 # For datetime mocking
 time-machine==2.8.2

requirements-freqai-rl.txt

@@ -2,7 +2,7 @@
 -r requirements-freqai.txt
 
 # Required for freqai-rl
-torch==1.12.1
+torch==1.13.0
 stable-baselines3==1.6.2
 sb3-contrib==1.6.2
 # Gym is forced to this version by stable-baselines3.

requirements.txt

@@ -1,8 +1,8 @@
 numpy==1.23.5
-pandas==1.5.1
+pandas==1.5.2
 pandas-ta==0.3.14b
 
-ccxt==2.2.36
+ccxt==2.2.67
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==38.0.1; platform_machine == 'armv7l'
 cryptography==38.0.4; platform_machine != 'armv7l'
@@ -13,7 +13,7 @@ arrow==1.2.3
 cachetools==4.2.2
 requests==2.28.1
 urllib3==1.26.13
-jsonschema==4.17.1
+jsonschema==4.17.3
 TA-Lib==0.4.25
 technical==1.3.0
 tabulate==0.9.0
@@ -30,13 +30,13 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.9
 # Properly format api responses
-orjson==3.8.2
+orjson==3.8.3
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.87.0
+fastapi==0.88.0
 pydantic==1.10.2
 uvicorn==0.20.0
 pyjwt==2.6.0

tests/exchange/test_ccxt_compat.py

@@ -28,15 +28,15 @@ EXCHANGES = {
         'leverage_tiers_public': False,
         'leverage_in_spot_market': False,
     },
-    'binance': {
+    # 'binance': {
-        'pair': 'BTC/USDT',
+    #     'pair': 'BTC/USDT',
-        'stake_currency': 'USDT',
+    #     'stake_currency': 'USDT',
-        'hasQuoteVolume': True,
+    #     'hasQuoteVolume': True,
-        'timeframe': '5m',
+    #     'timeframe': '5m',
-        'futures': True,
+    #     'futures': True,
-        'leverage_tiers_public': False,
+    #     'leverage_tiers_public': False,
-        'leverage_in_spot_market': False,
+    #     'leverage_in_spot_market': False,
-    },
+    # },
     'kraken': {
         'pair': 'BTC/USDT',
         'stake_currency': 'USDT',