diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 28a15913b..8a240c372 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -18,6 +18,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)).
**Datatype:** Positive integer. | `follow_mode` | Use a `follower` that will look for models associated with a specific `identifier` and load those for inferencing. A `follower` will **not** train new models.
**Datatype:** Boolean.
Default: `False`. | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)).
**Datatype:** Boolean.
Default: `False`. +| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | | **Feature parameters** | `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md).
**Datatype:** Dictionary. | `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base indicators dataset.
**Datatype:** List of timeframes (strings). @@ -37,7 +38,6 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `noise_standard_deviation` | If set, FreqAI adds noise to the training features with the aim of preventing overfitting. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in FreqAI is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation).
**Datatype:** Integer.
Default: `0`. | `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset.
**Datatype:** Float.
Default: `30`. | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it.
**Datatype:** Boolean.
Default: `False` (no reversal). -| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | | **Data split parameters** | `data_split_parameters` | Include any additional parameters available from Scikit-learn `test_train_split()`, which are shown [here](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) (external website).
**Datatype:** Dictionary. | `test_size` | The fraction of data that should be used for testing instead of training.
**Datatype:** Positive float < 1. diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 0e1e80e09..d7d2e27b7 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 -mkdocs==1.4.1 -mkdocs-material==8.5.7 +mkdocs==1.4.2 +mkdocs-material==8.5.8 mdx_truly_sane_lists==1.3 pymdown-extensions==9.7 jinja2==3.1.2 diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 70f60867b..022cbd400 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -542,7 +542,7 @@ CONF_SCHEMA = { "keras": {"type": "boolean", "default": False}, "write_metrics_to_disk": {"type": "boolean", "default": False}, "purge_old_models": {"type": "boolean", "default": True}, - "conv_width": {"type": "integer", "default": 2}, + "conv_width": {"type": "integer", "default": 1}, "train_period_days": {"type": "integer", "default": 0}, "backtest_period_days": {"type": "number", "default": 7}, "identifier": {"type": "string", "default": "example"}, diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index c32db9165..9bc543a9d 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'profit_ratio', 'profit_abs', 'exit_reason', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag', - 'is_short', 'open_timestamp', 'close_timestamp', 'orders' + 'leverage', 'is_short', 'open_timestamp', 'close_timestamp', 'orders' ] @@ -280,6 +280,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non # Compatibility support for pre short Columns if 'is_short' not in df.columns: df['is_short'] = 0 + if 'leverage' not in df.columns: + df['leverage'] = 1.0 if 'enter_tag' not in df.columns: df['enter_tag'] = df['buy_tag'] df = df.drop(['buy_tag'], axis=1) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index cbc3f1a34..b82d2055b 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -102,6 +102,11 @@ class IDataHandler(ABC): :return: (min, max) """ data = self._ohlcv_load(pair, timeframe, None, candle_type) + if data.empty: + return ( + datetime.fromtimestamp(0, tz=timezone.utc), + datetime.fromtimestamp(0, tz=timezone.utc) + ) return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime() @abstractmethod diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7a2b8ce7d..32d4f3435 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1689,6 +1689,17 @@ class Exchange: @retrier def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, price: float = 1, taker_or_maker: MakerTaker = 'maker') -> float: + """ + Retrieve fee from exchange + :param symbol: Pair + :param type: Type of order (market, limit, ...) + :param side: Side of order (buy, sell) + :param amount: Amount of order + :param price: Price of order + :param taker_or_maker: 'maker' or 'taker' (ignored if "type" is provided) + """ + if type and type == 'market': + taker_or_maker = 'taker' try: if self._config['dry_run'] and self._config.get('fee', None) is not None: return self._config['fee'] diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 0e9d2e605..dda8ebdbf 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -636,6 +636,8 @@ class FreqaiDataDrawer: axis=0, ) + self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date'] + def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None: """ Load pair histories for all whitelist and corr_pairlist pairs. diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 35148a6e2..12a3cd519 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1177,11 +1177,13 @@ class FreqaiDataKitchen: pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: + pair = pair.replace(':', '') # lightgbm doesnt like colons valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] pair_cols = [col for col in dataframe.columns if any(substr in col for substr in valid_strs)] - pair_cols.insert(0, 'date') - corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) + if pair_cols: + pair_cols.insert(0, 'date') + corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) return corr_dataframes @@ -1199,8 +1201,9 @@ class FreqaiDataKitchen: ready for training """ pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) - + current_pair = current_pair.replace(':', '') for pair in pairs: + pair = pair.replace(':', '') # lightgbm doesnt work with colons if current_pair != pair: dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date') @@ -1270,6 +1273,8 @@ class FreqaiDataKitchen: self.get_unique_classes_from_labels(dataframe) + dataframe = self.remove_special_chars_from_feature_names(dataframe) + return dataframe def fit_labels(self) -> None: @@ -1471,3 +1476,16 @@ class FreqaiDataKitchen: assets_end_dates[asset].append(model_end_date) return assets_end_dates + + def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame: + """ + Remove all special characters from feature strings (:) + :param dataframe: the dataframe that just finished indicator population. (unfiltered) + :return: dataframe with cleaned featrue names + """ + + spec_chars = [':'] + for c in spec_chars: + dataframe.columns = dataframe.columns.str.replace(c, "") + + return dataframe diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index ee0fbebc3..c0be5a69e 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -68,6 +68,9 @@ class IFreqaiModel(ABC): if self.save_backtest_models: logger.info('Backtesting module configured to save all models.') self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) + # set current candle to arbitrary historical date + self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) + self.dd.current_candle = self.current_candle self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", []) @@ -75,7 +78,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info.get("conv_width", 2) + self.CONV_WIDTH = self.freqai_info.get('conv_width', 1) if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 @@ -93,7 +96,6 @@ class IFreqaiModel(ABC): # get_corr_dataframes is controlling the caching of corr_dataframes # for improved performance. Careful with this boolean. self.get_corr_dataframes: bool = True - self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() @@ -338,6 +340,7 @@ class IFreqaiModel(ABC): if self.dd.historic_data: self.dd.update_historic_data(strategy, dk) logger.debug(f'Updating historic data on pair {metadata["pair"]}') + self.track_current_candle() if not self.follow_mode: @@ -682,8 +685,6 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 - if self.corr_pairlist: - self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): @@ -759,12 +760,24 @@ class IFreqaiModel(ABC): "is included in the column names when you are creating features " "in `populate_any_indicators()`.") self.get_corr_dataframes = not bool(self.corr_dataframes) - else: + elif self.corr_dataframes: dataframe = dk.attach_corr_pair_columns( dataframe, self.corr_dataframes, dk.pair) return dataframe + def track_current_candle(self): + """ + Checks if the latest candle appended by the datadrawer is + equivalent to the latest candle seen by FreqAI. If not, it + asks to refresh the cached corr_dfs, and resets the pair + counter. + """ + if self.dd.current_candle > self.current_candle: + self.get_corr_dataframes = True + self.pair_it = 1 + self.current_candle = self.dd.current_candle + def ensure_data_exists(self, dataframe_backtest: DataFrame, tr_backtest: TimeRange, pair: str) -> bool: """ diff --git a/requirements-dev.txt b/requirements-dev.txt index 66da72969..a46ec7e48 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8==5.0.4 flake8-tidy-imports==4.8.0 mypy==0.982 pre-commit==2.20.0 -pytest==7.1.3 +pytest==7.2.0 pytest-asyncio==0.20.1 pytest-cov==4.0.0 pytest-mock==3.10.0 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index fd2731256..4f59ad1fa 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,4 +6,4 @@ scipy==1.9.3 scikit-learn==1.1.3 scikit-optimize==0.9.0 filelock==3.8.0 -progressbar2==4.1.1 +progressbar2==4.2.0 diff --git a/requirements.txt b/requirements.txt index cae9cf3b7..0363a4740 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,17 +2,17 @@ numpy==1.23.4 pandas==1.5.1 pandas-ta==0.3.14b -ccxt==2.0.96 +ccxt==2.1.33 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1 aiohttp==3.8.3 -SQLAlchemy==1.4.42 +SQLAlchemy==1.4.43 python-telegram-bot==13.14 arrow==1.2.3 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.12 -jsonschema==4.16.0 +jsonschema==4.17.0 TA-Lib==0.4.25 technical==1.3.0 tabulate==0.9.0 @@ -29,7 +29,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.0 +orjson==3.8.1 # Notify systemd sdnotify==0.3.2 @@ -46,7 +46,7 @@ psutil==5.9.3 colorama==0.4.6 # Building config files interactively questionary==1.10.0 -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.32 # Extensions to datetime library python-dateutil==2.8.2 diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 67eeda7d0..c067d0339 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import re +from datetime import datetime, timezone from pathlib import Path from unittest.mock import MagicMock @@ -154,6 +155,23 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog): assert df.columns.equals(df1.columns) +def test_datahandler_ohlcv_data_min_max(testdatadir): + dh = JsonDataHandler(testdatadir) + min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '5m', 'spot') + assert len(min_max) == 2 + + # Empty pair + min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '8m', 'spot') + assert len(min_max) == 2 + assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) + assert min_max[0] == min_max[1] + # Empty pair2 + min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '4m', 'spot') + assert len(min_max) == 2 + assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) + assert min_max[0] == min_max[1] + + def test_datahandler__check_empty_df(testdatadir, caplog): dh = JsonDataHandler(testdatadir) expected_text = r"Price jump in UNITTEST/USDT, 1h, spot between" diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 1d1c44a1e..7ab963507 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -22,6 +22,7 @@ def test_update_historic_data(mocker, freqai_conf): historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) candle_difference = dp_candles - historic_candles + freqai.dk.pair = "ADA/BTC" freqai.dd.update_historic_data(strategy, freqai.dk) updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 2bc65d52e..e00718486 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -194,6 +194,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + df = freqai.cache_corr_pairlist_dfs(df, freqai.dk) for i in range(5): df[f'%-constant_{i}'] = i # df.loc[:, f'%-constant_{i}'] = i @@ -339,6 +340,7 @@ def test_follow_mode(mocker, freqai_conf): df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m') + freqai.dk.pair = "ADA/BTC" freqai.start_live(df, metadata, strategy, freqai.dk) assert len(freqai.dk.return_dataframe.index) == 5702 diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 290e08455..053ed7e5e 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -764,6 +764,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'max_rate': [0.10501, 0.1038888], 'is_open': [False, False], 'enter_tag': [None, None], + "leverage": [1.0, 1.0], "is_short": [False, False], 'open_timestamp': [1517251200000, 1517283000000], 'close_timestamp': [1517265300000, 1517285400000], @@ -788,13 +789,14 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: assert len(t['orders']) == 2 ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate - assert ln is not None + assert not ln.empty assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) # check close trade rate alignes to close rate or is between high and low - ln = data_pair.loc[data_pair["date"] == t["close_date"]] - assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or - round(ln.iloc[0]["low"], 6) < round( - t["close_rate"], 6) < round(ln.iloc[0]["high"], 6)) + ln1 = data_pair.loc[data_pair["date"] == t["close_date"]] + assert not ln1.empty + assert (round(ln1.iloc[0]["open"], 6) == round(t["close_rate"], 6) or + round(ln1.iloc[0]["low"], 6) < round( + t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6)) def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) -> None: diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 135ec6b15..b97b45e26 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -72,6 +72,7 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'max_rate': [0.10481985, 0.1038888], 'is_open': [False, False], 'enter_tag': [None, None], + 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], 'close_timestamp': [1517265300000, 1517285400000], diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a0d38563e..10e6228f8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -5305,7 +5305,7 @@ def test_get_valid_price(mocker, default_conf_usdt) -> None: ]) def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, time_machine, t1, t2): - time_machine.move_to(f"{t1} +00:00") + time_machine.move_to(f"{t1} +00:00", tick=False) patch_RPCManager(mocker) patch_exchange(mocker) @@ -5314,7 +5314,7 @@ def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, default_conf['margin_mode'] = 'isolated' freqtrade = get_patched_freqtradebot(mocker, default_conf) - time_machine.move_to(f"{t2} +00:00") + time_machine.move_to(f"{t2} +00:00", tick=False) # Check schedule jobs in debugging with freqtrade._schedule.jobs freqtrade._schedule.run_pending()