From 257c83383146f430d334839b6958a94dcd56657b Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 4 Nov 2022 18:10:04 +0100 Subject: [PATCH] add doc for single precision, dont allow half precision, add test --- docs/freqai-parameter-table.md | 1 + freqtrade/freqai/data_kitchen.py | 15 ++------------- tests/freqai/test_freqai_interface.py | 13 +++++++------ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 28a15913b..f4fbcbf1c 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -18,6 +18,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)).
**Datatype:** Positive integer. | `follow_mode` | Use a `follower` that will look for models associated with a specific `identifier` and load those for inferencing. A `follower` will **not** train new models.
**Datatype:** Boolean.
Default: `False`. | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)).
**Datatype:** Boolean.
Default: `False`. +| `convert_df_to_float32` | Recast all numeric columns to float32, with the objective of reducing ram/disk usage and decreasing train/inference timing.
**Datatype:** Boolean.
Default: `False`. | | **Feature parameters** | `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md).
**Datatype:** Dictionary. | `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base indicators dataset.
**Datatype:** List of timeframes (strings). diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1e3a518d0..f1c1fa26d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1357,22 +1357,11 @@ class FreqaiDataKitchen: for col in df.columns[1:]: col_type = df[col].dtype - if col_type != object: - c_min = df[col].min() - c_max = df[col].max() if str(col_type)[:3] == "int": - if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max: - df[col] = df[col].astype(np.int8) - elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max: - df[col] = df[col].astype(np.int16) - elif c_min > np.iinfo(np.int32).min: - df[col] = df[col].astype(np.int32) + df[col] = df[col].astype(np.int32) else: - if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max: - df[col] = df[col].astype(np.float16) - elif c_min > np.finfo(np.float32).min: - df[col] = df[col].astype(np.float32) + df[col] = df[col].astype(np.float32) end_mem = df.memory_usage().sum() / 1024**2 print("Memory usage after optimization is: {:.2f} MB".format(end_mem)) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 2bc65d52e..a2eed92e3 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -27,13 +27,13 @@ def is_mac() -> bool: return "Darwin" in machine -@pytest.mark.parametrize('model, pca, dbscan', [ - ('LightGBMRegressor', True, False), - ('XGBoostRegressor', False, True), - ('XGBoostRFRegressor', False, False), - ('CatboostRegressor', False, False), +@pytest.mark.parametrize('model, pca, dbscan, float32', [ + ('LightGBMRegressor', True, False, True), + ('XGBoostRegressor', False, True, False), + ('XGBoostRFRegressor', False, False, False), + ('CatboostRegressor', False, False, False), ]) -def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan): +def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan, float32): if is_arm() and model == 'CatboostRegressor': pytest.skip("CatBoost is not supported on ARM") @@ -43,6 +43,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, freqai_conf.update({"strategy": "freqai_test_strat"}) freqai_conf['freqai']['feature_parameters'].update({"principal_component_analysis": pca}) freqai_conf['freqai']['feature_parameters'].update({"use_DBSCAN_to_remove_outliers": dbscan}) + freqai_conf['freqai'].update({"convert_df_to_float32": float32}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf)