diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e730d1489..273fb7ea0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -410,7 +410,7 @@ jobs: python setup.py sdist bdist_wheel - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.6.1 if: (github.event_name == 'release') with: user: __token__ @@ -418,7 +418,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.6.1 if: (github.event_name == 'release') with: user: __token__ diff --git a/docs/exchanges.md b/docs/exchanges.md index b4eb7e023..7070fc690 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -54,6 +54,9 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t ## Binance +!!! Warning "Server location and geo-ip restrictions" + Please be aware that binance restrict api access regarding the server country. The currents and non exhaustive countries blocked are United States, Malaysia (Singapour), Ontario (Canada). Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. + Binance supports [time_in_force](configuration.md#understand-order_time_in_force). !!! Tip "Stoploss on Exchange" diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index f2a52a9b8..d05ce80f3 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -37,7 +37,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `populate_any_indicators()` for indicator creation. FreqAI uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN.
**Datatype:** Positive integer. | `indicator_periods_candles` | Time periods to calculate indicators for. The indicators are added to the base indicator dataset.
**Datatype:** List of positive integers. | `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis)
**Datatype:** Boolean.
Default: `False`. -| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features.
**Datatype:** Integer.
Default: `0`. +| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. Plot is stored in `user_data/models//sub-train-_.html`.
**Datatype:** Integer.
Default: `0`. | `DI_threshold` | Activates the use of the Dissimilarity Index for outlier detection when set to > 0. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di).
**Datatype:** Positive float (typically < 1). | `use_SVM_to_remove_outliers` | Train a support vector machine to detect and remove outliers from the training dataset, as well as from incoming data points. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm).
**Datatype:** Boolean. | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. See details about some select parameters [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm).
**Datatype:** Dictionary. diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md index 353d7a2cc..b1a212a92 100644 --- a/docs/freqai-reinforcement-learning.md +++ b/docs/freqai-reinforcement-learning.md @@ -243,7 +243,7 @@ cd freqtrade tensorboard --logdir user_data/models/unique-id ``` -where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6060 (6060 is the default port used by Tensorboard). +where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard). ![tensorboard](assets/tensorboard.jpg) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 224e9b548..fd4f66d71 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.4.2 -mkdocs-material==8.5.10 +mkdocs-material==8.5.11 mdx_truly_sane_lists==1.3 -pymdown-extensions==9.8 +pymdown-extensions==9.9 jinja2==3.1.2 diff --git a/freqtrade/freqai/RL/BaseEnvironment.py b/freqtrade/freqai/RL/BaseEnvironment.py index 66bdb8435..e7bd26a92 100644 --- a/freqtrade/freqai/RL/BaseEnvironment.py +++ b/freqtrade/freqai/RL/BaseEnvironment.py @@ -194,12 +194,12 @@ class BaseEnvironment(gym.Env): if self._position == Positions.Neutral: return 0. elif self._position == Positions.Short: - current_price = self.add_exit_fee(self.prices.iloc[self._current_tick].open) - last_trade_price = self.add_entry_fee(self.prices.iloc[self._last_trade_tick].open) - return (last_trade_price - current_price) / last_trade_price - elif self._position == Positions.Long: current_price = self.add_entry_fee(self.prices.iloc[self._current_tick].open) last_trade_price = self.add_exit_fee(self.prices.iloc[self._last_trade_tick].open) + return (last_trade_price - current_price) / last_trade_price + elif self._position == Positions.Long: + current_price = self.add_exit_fee(self.prices.iloc[self._current_tick].open) + last_trade_price = self.add_entry_fee(self.prices.iloc[self._last_trade_tick].open) return (current_price - last_trade_price) / last_trade_price else: return 0. diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index 9d2fae583..81f8edfc4 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -64,7 +64,7 @@ class BaseReinforcementLearningModel(IFreqaiModel): self.policy_type = self.freqai_info['rl_config']['policy_type'] self.unset_outlier_removal() self.net_arch = self.rl_config.get('net_arch', [128, 128]) - self.dd.model_type = "stable_baselines" + self.dd.model_type = import_str def unset_outlier_removal(self): """ diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 99e3686b3..848fb20eb 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -503,7 +503,7 @@ class FreqaiDataDrawer: dump(model, save_path / f"{dk.model_filename}_model.joblib") elif self.model_type == 'keras': model.save(save_path / f"{dk.model_filename}_model.h5") - elif 'stable_baselines' in self.model_type: + elif 'stable_baselines' in self.model_type or 'sb3_contrib' == self.model_type: model.save(save_path / f"{dk.model_filename}_model.zip") if dk.svm_model is not None: @@ -589,9 +589,9 @@ class FreqaiDataDrawer: elif self.model_type == 'keras': from tensorflow import keras model = keras.models.load_model(dk.data_path / f"{dk.model_filename}_model.h5") - elif self.model_type == 'stable_baselines': + elif 'stable_baselines' in self.model_type or 'sb3_contrib' == self.model_type: mod = importlib.import_module( - 'stable_baselines3', self.freqai_info['rl_config']['model_type']) + self.model_type, self.freqai_info['rl_config']['model_type']) MODELCLASS = getattr(mod, self.freqai_info['rl_config']['model_type']) model = MODELCLASS.load(dk.data_path / f"{dk.model_filename}_model") diff --git a/freqtrade/main.py b/freqtrade/main.py index 754c536d0..0a46747ea 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -7,6 +7,8 @@ import logging import sys from typing import Any, List +from freqtrade.util.gc_setup import gc_set_threshold + # check min. python version if sys.version_info < (3, 8): # pragma: no cover @@ -36,6 +38,7 @@ def main(sysargv: List[str] = None) -> None: # Call subcommand. if 'func' in args: logger.info(f'freqtrade {__version__}') + gc_set_threshold() return_code = args['func'](args) else: # No subcommand was issued. diff --git a/freqtrade/util/gc_setup.py b/freqtrade/util/gc_setup.py new file mode 100644 index 000000000..a3532cbab --- /dev/null +++ b/freqtrade/util/gc_setup.py @@ -0,0 +1,18 @@ +import gc +import logging +import platform + + +logger = logging.getLogger(__name__) + + +def gc_set_threshold(): + """ + Reduce number of GC runs to improve performance (explanation video) + https://www.youtube.com/watch?v=p4Sn6UcFTOU + + """ + if platform.python_implementation() == "CPython": + # allocs, g1, g2 = gc.get_threshold() + gc.set_threshold(50_000, 500, 1000) + logger.debug("Adjusting python allocations to reduce GC runs") diff --git a/requirements-dev.txt b/requirements-dev.txt index ffce3d696..463d2656a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,7 @@ pytest==7.2.0 pytest-asyncio==0.20.2 pytest-cov==4.0.0 pytest-mock==3.10.0 -pytest-random-order==1.0.4 +pytest-random-order==1.1.0 isort==5.10.1 # For datetime mocking time-machine==2.8.2 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index df541c701..67bd66102 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -2,7 +2,7 @@ -r requirements-freqai.txt # Required for freqai-rl -torch==1.12.1 +torch==1.13.0 stable-baselines3==1.6.2 sb3-contrib==1.6.2 # Gym is forced to this version by stable-baselines3. diff --git a/requirements.txt b/requirements.txt index dab8ae414..313e0ff9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ numpy==1.23.5 -pandas==1.5.1 +pandas==1.5.2 pandas-ta==0.3.14b -ccxt==2.2.36 +ccxt==2.2.67 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.4; platform_machine != 'armv7l' @@ -13,7 +13,7 @@ arrow==1.2.3 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.13 -jsonschema==4.17.1 +jsonschema==4.17.3 TA-Lib==0.4.25 technical==1.3.0 tabulate==0.9.0 @@ -30,13 +30,13 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.2 +orjson==3.8.3 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.87.0 +fastapi==0.88.0 pydantic==1.10.2 uvicorn==0.20.0 pyjwt==2.6.0 diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 55d463c68..280876ae8 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -28,15 +28,15 @@ EXCHANGES = { 'leverage_tiers_public': False, 'leverage_in_spot_market': False, }, - 'binance': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': True, - 'timeframe': '5m', - 'futures': True, - 'leverage_tiers_public': False, - 'leverage_in_spot_market': False, - }, + # 'binance': { + # 'pair': 'BTC/USDT', + # 'stake_currency': 'USDT', + # 'hasQuoteVolume': True, + # 'timeframe': '5m', + # 'futures': True, + # 'leverage_tiers_public': False, + # 'leverage_in_spot_market': False, + # }, 'kraken': { 'pair': 'BTC/USDT', 'stake_currency': 'USDT',