From 0ec38e0cfd1549d502114172eabdbf7d3fcf3c6d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Aug 2022 08:23:07 +0200 Subject: [PATCH 01/22] Fix typo in docs --- docs/rest-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index 1ec9b6c12..d9840a09c 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -317,7 +317,7 @@ whitelist ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. -This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs/ - but it'll depend on your settings. +This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs - but it'll depend on your settings. ### Advanced API usage using JWT tokens From 085f81ec9e3dee2e885533722ed8429d2ba19566 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Aug 2022 08:24:14 +0200 Subject: [PATCH 02/22] Fix header indent on stake-size-management --- docs/strategy-callbacks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index a9b032818..8d46f42e1 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -75,7 +75,7 @@ class AwesomeStrategy(IStrategy): ``` -### Stake size management +## Stake size management Called before entering a trade, makes it possible to manage your position size when placing a new trade. From 87a3115073562e9abc7efd792569020435a01f6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Aug 2022 10:03:04 +0200 Subject: [PATCH 03/22] Add get_open_trade_count() to simplify getting open trade count. --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence/trade_model.py | 10 ++++++++++ tests/test_freqtradebot.py | 2 -- tests/test_persistence.py | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4e3af64ea..5b71d6ec4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -271,7 +271,7 @@ class FreqtradeBot(LoggingMixin): Return the number of free open trades slots or 0 if max number of open trades reached """ - open_trades = len(Trade.get_open_trades()) + open_trades = Trade.get_open_trade_count() return max(0, self.config['max_open_trades'] - open_trades) def update_funding_fees(self): diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index b954fee20..0c438956a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1044,6 +1044,16 @@ class LocalTrade(): """ return Trade.get_trades_proxy(is_open=True) + @staticmethod + def get_open_trade_count() -> int: + """ + get open trade count + """ + if Trade.use_db: + return Trade.query.filter(Trade.is_open.is_(True)).count() + else: + return len(LocalTrade.trades_open) + @staticmethod def stoploss_reinitialization(desired_stoploss): """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a1a16c039..138527053 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -473,8 +473,6 @@ def test_create_trade_no_signal(default_conf_usdt, fee, mocker) -> None: freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade, enter_long=False, exit_long=False) - Trade.query = MagicMock() - Trade.query.filter = MagicMock() assert not freqtrade.create_trade('ETH/USDT') diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 50d0788ca..d8973e9d0 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1689,6 +1689,7 @@ def test_get_open(fee, is_short, use_db): create_mock_trades(fee, is_short, use_db) assert len(Trade.get_open_trades()) == 4 + assert Trade.get_open_trade_count() == 4 Trade.use_db = True @@ -1701,6 +1702,7 @@ def test_get_open_lev(fee, use_db): create_mock_trades_with_leverage(fee, use_db) assert len(Trade.get_open_trades()) == 5 + assert Trade.get_open_trade_count() == 5 Trade.use_db = True From f6d832c6d918607d6828b5c49ad857433b91d69c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Aug 2022 17:48:13 +0200 Subject: [PATCH 04/22] Add get_option to expose ft_has via method --- freqtrade/commands/data_commands.py | 2 +- freqtrade/data/history/history_utils.py | 4 ++-- freqtrade/exchange/exchange.py | 6 ++++++ freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/plugins/pairlist/VolumePairList.py | 4 ++-- tests/exchange/test_exchange.py | 16 ++++++++-------- 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 36a86bece..ce26d39ab 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -81,7 +81,7 @@ def start_download_data(args: Dict[str, Any]) -> None: data_format_trades=config['dataformat_trades'], ) else: - if not exchange._ft_has.get('ohlcv_has_history', True): + if not exchange.get_option('ohlcv_has_history', True): raise OperationalException( f"Historic klines not available for {exchange.name}. " "Please use `--dl-trades` instead for this exchange " diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index cba1b60db..04d0bf6d3 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -302,8 +302,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. - tf_mark = exchange._ft_has['mark_ohlcv_timeframe'] - fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) + tf_mark = exchange.get_option('mark_ohlcv_timeframe') + fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price')) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7ef6858a5..a2944c784 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -674,6 +674,12 @@ class Exchange: f"Freqtrade does not support {mm_value} {trading_mode.value} on {self.name}" ) + def get_option(self, param: str, default: Any = None) -> Any: + """ + Get parameter value from _ft_has + """ + return self._ft_has.get(param, default) + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 12536c333..e32eae533 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -267,7 +267,7 @@ class Backtesting: funding_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'], + timeframe=self.exchange.get_option('mark_ohlcv_timeframe'), timerange=self.timerange, startup_candles=0, fail_without_data=True, @@ -279,12 +279,12 @@ class Backtesting: mark_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'], + timeframe=self.exchange.get_option('mark_ohlcv_timeframe'), timerange=self.timerange, startup_candles=0, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), - candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"]) + candle_type=CandleType.from_string(self.exchange.get_option("mark_ohlcv_price")) ) # Combine data to avoid combining the data per trade. unavailable_pairs = [] diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index e364e1a69..8138a5fb6 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -73,7 +73,7 @@ class VolumePairList(IPairList): if (not self._use_range and not ( self._exchange.exchange_has('fetchTickers') - and self._exchange._ft_has["tickers_have_quoteVolume"])): + and self._exchange.get_option("tickers_have_quoteVolume"))): raise OperationalException( "Exchange does not support dynamic whitelist in this configuration. " "Please edit your config and either remove Volumepairlist, " @@ -193,7 +193,7 @@ class VolumePairList(IPairList): ) in candles else None # in case of candle data calculate typical price and quoteVolume for candle if pair_candles is not None and not pair_candles.empty: - if self._exchange._ft_has["ohlcv_volume_currency"] == "base": + if self._exchange.get_option("ohlcv_volume_currency") == "base": pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ec259d703..4f9c92efc 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2352,10 +2352,10 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name) order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val) assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC' # Not all exchanges support all limits for orderbook - if not exchange._ft_has['l2_limit_range'] or val in exchange._ft_has['l2_limit_range']: + if not exchange.get_option('l2_limit_range') or val in exchange.get_option('l2_limit_range'): assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val else: - next_limit = exchange.get_next_limit_in_list(val, exchange._ft_has['l2_limit_range']) + next_limit = exchange.get_next_limit_in_list(val, exchange.get_option('l2_limit_range')) assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == next_limit @@ -3311,16 +3311,16 @@ def test_merge_ft_has_dict(default_conf, mocker): ex = Kraken(default_conf) assert ex._ft_has != Exchange._ft_has_default - assert ex._ft_has['trades_pagination'] == 'id' - assert ex._ft_has['trades_pagination_arg'] == 'since' + assert ex.get_option('trades_pagination') == 'id' + assert ex.get_option('trades_pagination_arg') == 'since' # Binance defines different values ex = Binance(default_conf) assert ex._ft_has != Exchange._ft_has_default - assert ex._ft_has['stoploss_on_exchange'] - assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc'] - assert ex._ft_has['trades_pagination'] == 'id' - assert ex._ft_has['trades_pagination_arg'] == 'fromId' + assert ex.get_option('stoploss_on_exchange') + assert ex.get_option('order_time_in_force') == ['gtc', 'fok', 'ioc'] + assert ex.get_option('trades_pagination') == 'id' + assert ex.get_option('trades_pagination_arg') == 'fromId' conf = copy.deepcopy(default_conf) conf['exchange']['_ft_has_params'] = {"DeadBeef": 20, From 2dc34779d5877810d035f31a644f0a204d8868dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Aug 2022 18:07:41 +0200 Subject: [PATCH 05/22] Fix line length --- tests/exchange/test_exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 4f9c92efc..be8d4b2cf 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2352,7 +2352,8 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name) order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val) assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC' # Not all exchanges support all limits for orderbook - if not exchange.get_option('l2_limit_range') or val in exchange.get_option('l2_limit_range'): + if (not exchange.get_option('l2_limit_range') + or val in exchange.get_option('l2_limit_range')): assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val else: next_limit = exchange.get_next_limit_in_list(val, exchange.get_option('l2_limit_range')) From dfa7d1fc2746823ba2d38ab544cfbcaf0ffadf6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 03:01:51 +0000 Subject: [PATCH 06/22] Bump fastapi from 0.79.0 to 0.79.1 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.79.0 to 0.79.1. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.79.0...0.79.1) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77925f98b..a5e344a54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.7.12 sdnotify==0.3.2 # API Server -fastapi==0.79.0 +fastapi==0.79.1 uvicorn==0.18.2 pyjwt==2.4.0 aiofiles==0.8.0 From 3958e53aaa63d3b94125d1f4e9b0ac2103867c82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 03:01:55 +0000 Subject: [PATCH 07/22] Bump mkdocs-material from 8.4.0 to 8.4.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.4.0 to 8.4.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.4.0...8.4.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a53e909e0..bffc04d1c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.3.1 -mkdocs-material==8.4.0 +mkdocs-material==8.4.1 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 From 70848a258dfa115835f311643fb50c6f05518b8f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 03:01:59 +0000 Subject: [PATCH 08/22] Bump types-requests from 2.28.8 to 2.28.9 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.8 to 2.28.9. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0cd4a6a6c..f1c8851b4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -25,6 +25,6 @@ nbconvert==6.5.3 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.8 +types-requests==2.28.9 types-tabulate==0.8.11 types-python-dateutil==2.8.19 From eeb177110eaa1ce7037fbbaaa272fb835a52a550 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 03:02:03 +0000 Subject: [PATCH 09/22] Bump jsonschema from 4.9.1 to 4.14.0 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.9.1 to 4.14.0. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.9.1...v4.14.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77925f98b..50e7d6c8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.11 -jsonschema==4.9.1 +jsonschema==4.14.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 From 354d3c0cda13ead678c4f65078630c0e1db398a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 03:02:10 +0000 Subject: [PATCH 10/22] Bump time-machine from 2.7.1 to 2.8.1 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.7.1 to 2.8.1. - [Release notes](https://github.com/adamchainz/time-machine/releases) - [Changelog](https://github.com/adamchainz/time-machine/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.7.1...2.8.1) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0cd4a6a6c..8316a4d46 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,7 +17,7 @@ pytest-mock==3.8.2 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.7.1 +time-machine==2.8.1 # Convert jupyter notebooks to markdown documents nbconvert==6.5.3 From ff9ed1abad041a38e7552840d0ddcd221b93df9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 03:02:25 +0000 Subject: [PATCH 11/22] Bump ccxt from 1.92.20 to 1.92.52 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.92.20 to 1.92.52. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.92.20...1.92.52) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77925f98b..23bb9ecac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.2 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.92.20 +ccxt==1.92.52 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From 93d2f7fc85184da2f29f9720eb62fcac42a43e76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 06:37:26 +0200 Subject: [PATCH 12/22] types-requests bump pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a205f24ec..86c4ec1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.8 + - types-requests==2.28.9 - types-tabulate==0.8.11 - types-python-dateutil==2.8.19 # stages: [push] From 015be770c33fde9405ae98f76004d0cae68c14f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 06:42:14 +0200 Subject: [PATCH 13/22] ccxt now defaults to base volume for all markets --- freqtrade/exchange/gateio.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index c6ed0c66c..426a4b64d 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -25,7 +25,6 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, - "ohlcv_volume_currency": "quote", "time_in_force_parameter": "timeInForce", "order_time_in_force": ['gtc', 'ioc'], "stoploss_order_types": {"limit": "limit"}, @@ -34,7 +33,6 @@ class Gateio(Exchange): _ft_has_futures: Dict = { "needs_trading_fees": True, - "ohlcv_volume_currency": "base", "fee_cost_in_contracts": False, # Set explicitly to false for clarity "order_props_in_contracts": ['amount', 'filled', 'remaining'], } From f55d5ffd8c65dc0cc76ab2255e21551a406f5a66 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 09:18:51 +0000 Subject: [PATCH 14/22] Don't fail when `--strategy-path` is not a valid directory. closes #7264 --- freqtrade/resolvers/iresolver.py | 5 ++++- tests/strategy/test_strategy_loading.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 74b28dffe..b99e7a94b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -193,7 +193,10 @@ class IResolver: :return: List of dicts containing 'name', 'class' and 'location' entries """ logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") - objects = [] + objects: List[Dict[str, Any]] = [] + if not directory.is_dir(): + logger.info(f"'{directory}' is not a directory, skipping.") + return objects for entry in directory.iterdir(): if ( recursive and entry.is_dir() diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 5b6f15d11..b794cdc99 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -48,6 +48,10 @@ def test_search_all_strategies_with_failed(): assert len([x for x in strategies if x['class'] is not None]) == 9 assert len([x for x in strategies if x['class'] is None]) == 1 + directory = Path(__file__).parent / "strats_nonexistingdir" + strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) + assert len(strategies) == 0 + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', From 96d8882f1e6740f6c0a859c6e5f52a5a30ddb007 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 22 Aug 2022 13:30:30 +0200 Subject: [PATCH 15/22] Plug mem leak, add training timer --- freqtrade/freqai/data_drawer.py | 6 ++--- freqtrade/freqai/data_kitchen.py | 7 ------ freqtrade/freqai/freqai_interface.py | 37 +++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index c8dbdf5e5..b3060deff 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -421,7 +421,7 @@ class FreqaiDataDrawer: ) # if self.live: - self.model_dictionary[dk.model_filename] = model + self.model_dictionary[coin] = model self.pair_dict[coin]["model_filename"] = dk.model_filename self.pair_dict[coin]["data_path"] = str(dk.data_path) self.save_drawer_to_disk() @@ -460,8 +460,8 @@ class FreqaiDataDrawer: ) # try to access model in memory instead of loading object from disk to save time - if dk.live and dk.model_filename in self.model_dictionary: - model = self.model_dictionary[dk.model_filename] + if dk.live and coin in self.model_dictionary: + model = self.model_dictionary[coin] elif not dk.keras: model = load(dk.data_path / f"{dk.model_filename}_model.joblib") else: diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 421b30bf5..9351e92d6 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -873,13 +873,6 @@ class FreqaiDataKitchen: data_load_timerange.stopts = int(time) retrain = True - # logger.info( - # f"downloading data for " - # f"{(data_load_timerange.stopts-data_load_timerange.startts)/SECONDS_IN_DAY:.2f} " - # " days. " - # f"Extension of {additional_seconds/SECONDS_IN_DAY:.2f} days" - # ) - return retrain, trained_timerange, data_load_timerange def set_new_model_names(self, pair: str, trained_timerange: TimeRange): diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 49e4ce5c3..4106f24e0 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -80,12 +80,15 @@ class IFreqaiModel(ABC): logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") self.CONV_WIDTH = self.freqai_info.get("conv_width", 2) self.pair_it = 0 + self.pair_it_train = 0 self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) self.last_trade_database_summary: DataFrame = {} self.current_trade_database_summary: DataFrame = {} self.analysis_lock = Lock() self.inference_time: float = 0 + self.train_time: float = 0 self.begin_time: float = 0 + self.begin_time_train: float = 0 self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) def assert_config(self, config: Dict[str, Any]) -> None: @@ -128,11 +131,20 @@ class IFreqaiModel(ABC): dk = self.start_backtesting(dataframe, metadata, self.dk) dataframe = dk.remove_features_from_df(dk.return_dataframe) - del dk + self.clean_up() if self.live: self.inference_timer('stop') return dataframe + def clean_up(self): + """ + Objects that should be handled by GC already between coins, but + are explicitly shown here to help demonstrate the non-persistence of these + objects. + """ + self.model = None + self.dk = None + @threaded def start_scanning(self, strategy: IStrategy) -> None: """ @@ -159,9 +171,11 @@ class IFreqaiModel(ABC): dk.set_paths(pair, new_trained_timerange.stopts) if retrain: + self.train_timer('start') self.train_model_in_series( new_trained_timerange, pair, strategy, dk, data_load_timerange ) + self.train_timer('stop') self.dd.save_historic_predictions_to_disk() @@ -480,8 +494,7 @@ class IFreqaiModel(ABC): data_load_timerange: TimeRange, ): """ - Retrieve data and train model in single threaded mode (only used if model directory is empty - upon startup for dry/live ) + Retrieve data and train model. :param new_trained_timerange: TimeRange = the timerange to train the model on :param metadata: dict = strategy provided metadata :param strategy: IStrategy = user defined strategy object @@ -612,6 +625,24 @@ class IFreqaiModel(ABC): self.inference_time = 0 return + def train_timer(self, do='start'): + """ + Timer designed to track the cumulative time spent training the full pairlist in + FreqAI. + """ + if do == 'start': + self.pair_it_train += 1 + self.begin_time_train = time.time() + elif do == 'stop': + end = time.time() + self.train_time += (end - self.begin_time_train) + if self.pair_it_train == self.total_pairs: + logger.info( + f'Total time spent training pairlist {self.train_time:.2f} seconds') + self.pair_it_train = 0 + self.train_time = 0 + return + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. From 5ce1c6980356d764b240ac2fdbd7d29572cc31de Mon Sep 17 00:00:00 2001 From: th0rntwig <107926911+th0rntwig@users.noreply.github.com> Date: Mon, 22 Aug 2022 19:57:20 +0200 Subject: [PATCH 16/22] Improve DBSCAN epsilon identification (#7269) * Improve DBSCAN epsilon identification --- freqtrade/freqai/data_kitchen.py | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9351e92d6..e480ab135 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -601,6 +601,8 @@ class FreqaiDataKitchen: is an outlier. """ + from math import cos, sin + if predict: train_ft_df = self.data_dictionary['train_features'] pred_ft_df = self.data_dictionary['prediction_features'] @@ -619,23 +621,47 @@ class FreqaiDataKitchen: else: + def normalise_distances(distances): + normalised_distances = (distances - distances.min()) / \ + (distances.max() - distances.min()) + return normalised_distances + + def rotate_point(origin, point, angle): + # rotate a point counterclockwise by a given angle (in radians) + # around a given origin + x = origin[0] + cos(angle) * (point[0] - origin[0]) - \ + sin(angle) * (point[1] - origin[1]) + y = origin[1] + sin(angle) * (point[0] - origin[0]) + \ + cos(angle) * (point[1] - origin[1]) + return (x, y) + MinPts = len(self.data_dictionary['train_features'].columns) * 2 # measure pairwise distances to train_features.shape[1]*2 nearest neighbours neighbors = NearestNeighbors( n_neighbors=MinPts, n_jobs=self.thread_count) neighbors_fit = neighbors.fit(self.data_dictionary['train_features']) distances, _ = neighbors_fit.kneighbors(self.data_dictionary['train_features']) - distances = np.sort(distances, axis=0) - index_ten_pct = int(len(distances[:, 1]) * 0.1) - distances = distances[index_ten_pct:, 1] - epsilon = distances[-1] + distances = np.sort(distances, axis=0).mean(axis=1) + + normalised_distances = normalise_distances(distances) + x_range = np.linspace(0, 1, len(distances)) + line = np.linspace(normalised_distances[0], + normalised_distances[-1], len(normalised_distances)) + deflection = np.abs(normalised_distances - line) + max_deflection_loc = np.where(deflection == deflection.max())[0][0] + origin = x_range[max_deflection_loc], line[max_deflection_loc] + point = x_range[max_deflection_loc], normalised_distances[max_deflection_loc] + rot_angle = np.pi / 4 + elbow_loc = rotate_point(origin, point, rot_angle) + + epsilon = elbow_loc[1] * (distances[-1] - distances[0]) + distances[0] clustering = DBSCAN(eps=epsilon, min_samples=MinPts, n_jobs=int(self.thread_count)).fit( self.data_dictionary['train_features'] ) - logger.info(f'DBSCAN found eps of {epsilon}.') + logger.info(f'DBSCAN found eps of {epsilon:.2f}.') self.data['DBSCAN_eps'] = epsilon self.data['DBSCAN_min_samples'] = MinPts @@ -698,7 +724,7 @@ class FreqaiDataKitchen: if (len(do_predict) - do_predict.sum()) > 0: logger.info( - f"DI tossed {len(do_predict) - do_predict.sum():.2f} predictions for " + f"DI tossed {len(do_predict) - do_predict.sum()} predictions for " "being too far from training data" ) From 5f38a574ce8159e125822a4f27ee5efec27220cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 20:23:19 +0200 Subject: [PATCH 17/22] Add okx broker id --- freqtrade/exchange/exchange.py | 10 +++++----- freqtrade/exchange/okx.py | 2 ++ tests/exchange/test_exchange.py | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a2944c784..06f83f4df 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -54,8 +54,8 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} - # Additional headers - added to the ccxt object - _headers: Dict = {} + # Additional parameters - added to the ccxt object + _ccxt_params: Dict = {} # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has @@ -242,9 +242,9 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) - if self._headers: - # Inject static headers after the above output to not confuse users. - ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if self._ccxt_params: + # Inject static options after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts(self._ccxt_params, ccxt_kwargs) if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 540e76fca..80373e071 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -39,6 +39,8 @@ class Okx(Exchange): net_only = True + _ccxt_params: Dict = {'options': {'brokerId': 'ffb5405ad327SUDE'}} + def ohlcv_candle_limit( self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index be8d4b2cf..7e61f76cd 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -181,11 +181,11 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) assert log_has(asynclogmsg, caplog) # Test additional headers case - Exchange._headers = {'hello': 'world'} + Exchange._ccxt_params = {'hello': 'world'} ex = Exchange(conf) assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) - assert ex._api.headers == {'hello': 'world'} + assert ex._api.hello == 'world' assert ex._ccxt_config == {} Exchange._headers = {} From 1b0f37a93cc54fbd10926c1d0b3a1bddd701e71d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 20:17:04 +0200 Subject: [PATCH 18/22] Fix documentation typo --- docs/data-download.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-download.md b/docs/data-download.md index 55c2ad738..be36d579d 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -186,7 +186,7 @@ Freqtrade currently supports 3 data-formats for both OHLCV and trades data: By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively. -To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: +To persist this change, you should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: ``` jsonc // ... From 6036018f352ff2e60a8198cf78ad65d38186843e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 20:28:33 +0200 Subject: [PATCH 19/22] Extract contracts_to_amount and amount_to_contracts to standalone functions --- freqtrade/exchange/exchange.py | 47 +++++++++++++++++++++--------- tests/exchange/test_ccxt_compat.py | 4 +-- tests/exchange/test_exchange.py | 6 ++-- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 06f83f4df..44288e696 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -408,7 +408,7 @@ class Exchange: else: return DataFrame() - def _get_contract_size(self, pair: str) -> float: + def get_contract_size(self, pair: str) -> float: if self.trading_mode == TradingMode.FUTURES: market = self.markets[pair] contract_size: float = 1.0 @@ -421,7 +421,7 @@ class Exchange: def _trades_contracts_to_amount(self, trades: List) -> List: if len(trades) > 0 and 'symbol' in trades[0]: - contract_size = self._get_contract_size(trades[0]['symbol']) + contract_size = self.get_contract_size(trades[0]['symbol']) if contract_size != 1: for trade in trades: trade['amount'] = trade['amount'] * contract_size @@ -429,7 +429,7 @@ class Exchange: def _order_contracts_to_amount(self, order: Dict) -> Dict: if 'symbol' in order and order['symbol'] is not None: - contract_size = self._get_contract_size(order['symbol']) + contract_size = self.get_contract_size(order['symbol']) if contract_size != 1: for prop in self._ft_has.get('order_props_in_contracts', []): if prop in order and order[prop] is not None: @@ -438,19 +438,13 @@ class Exchange: def _amount_to_contracts(self, pair: str, amount: float) -> float: - contract_size = self._get_contract_size(pair) - if contract_size and contract_size != 1: - return amount / contract_size - else: - return amount + contract_size = self.get_contract_size(pair) + return amount_to_contracts(amount, contract_size) def _contracts_to_amount(self, pair: str, num_contracts: float) -> float: - contract_size = self._get_contract_size(pair) - if contract_size and contract_size != 1: - return num_contracts * contract_size - else: - return num_contracts + contract_size = self.get_contract_size(pair) + return contracts_to_amount(num_contracts, contract_size) def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: if exchange_config.get('sandbox'): @@ -2898,6 +2892,33 @@ def market_is_active(market: Dict) -> bool: return market.get('active', True) is not False +def amount_to_contracts(amount: float, contract_size: float) -> float: + """ + Convert amount to contracts. + :param amount: amount to convert + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: num-contracts + """ + if contract_size and contract_size != 1: + return amount / contract_size + else: + return amount + + +def contracts_to_amount(num_contracts: float, contract_size: float) -> float: + """ + Takes num-contracts and converts it to contract size + :param num_contracts: number of contracts + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: Amount + """ + + if contract_size and contract_size != 1: + return num_contracts * contract_size + else: + return num_contracts + + def amount_to_precision(amount: float, amount_precision: Optional[float], precisionMode: Optional[int]) -> float: """ diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index c3a5fe17f..29b317c3c 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -409,14 +409,14 @@ class TestCCXTExchange(): assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int)) assert futures_leverage >= 1.0 - def test_ccxt__get_contract_size(self, exchange_futures): + def test_ccxt_get_contract_size(self, exchange_futures): futures, futures_name = exchange_futures if futures: futures_pair = EXCHANGES[futures_name].get( 'futures_pair', EXCHANGES[futures_name]['pair'] ) - contract_size = futures._get_contract_size(futures_pair) + contract_size = futures.get_contract_size(futures_pair) assert (isinstance(contract_size, float) or isinstance(contract_size, int)) assert contract_size >= 0.0 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7e61f76cd..5002a33e1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4288,7 +4288,7 @@ def test__fetch_and_calculate_funding_fees_datetime_called( ('XLTCUSDT', 0.01, 'futures'), ('ETH/USDT:USDT', 10, 'futures') ]) -def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode): +def est__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode): api_mock = MagicMock() default_conf['trading_mode'] = trading_mode default_conf['margin_mode'] = 'isolated' @@ -4307,7 +4307,7 @@ def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_m 'contractSize': '10', } }) - size = exchange._get_contract_size(pair) + size = exchange.get_contract_size(pair) assert expected_size == size @@ -5146,7 +5146,7 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange._get_contract_size = MagicMock(return_value=contract_size) + exchange.get_contract_size = MagicMock(return_value=contract_size) api_mock.create_order.reset_mock() order = exchange.stoploss( From 78b161e14cc262740b386758db70b79e1b88abb8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 20:32:55 +0200 Subject: [PATCH 20/22] add contract_size to database --- freqtrade/freqtradebot.py | 3 ++- freqtrade/persistence/migrations.py | 7 ++++--- freqtrade/persistence/trade_model.py | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5b71d6ec4..bb66303a8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -290,13 +290,14 @@ class FreqtradeBot(LoggingMixin): def startup_backpopulate_precision(self): - trades = Trade.get_trades([Trade.precision_mode.is_(None)]) + trades = Trade.get_trades([Trade.contract_size.is_(None)]) for trade in trades: if trade.exchange != self.exchange.id: continue trade.precision_mode = self.exchange.precisionMode trade.amount_precision = self.exchange.get_precision_amount(trade.pair) trade.price_precision = self.exchange.get_precision_price(trade.pair) + trade.contract_size = self.exchange.get_contract_size(trade.pair) Trade.commit() def startup_update_open_orders(self): diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 311554359..1131c88b4 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -133,6 +133,7 @@ def migrate_trades_and_orders_table( amount_precision = get_column_def(cols, 'amount_precision', 'null') price_precision = get_column_def(cols, 'price_precision', 'null') precision_mode = get_column_def(cols, 'precision_mode', 'null') + contract_size = get_column_def(cols, 'contract_size', 'null') # Schema migration necessary with engine.begin() as connection: @@ -161,7 +162,7 @@ def migrate_trades_and_orders_table( timeframe, open_trade_value, close_profit_abs, trading_mode, leverage, liquidation_price, is_short, interest_rate, funding_fees, realized_profit, - amount_precision, price_precision, precision_mode + amount_precision, price_precision, precision_mode, contract_size ) select id, lower(exchange), pair, {base_currency} base_currency, {stake_currency} stake_currency, @@ -189,7 +190,7 @@ def migrate_trades_and_orders_table( {is_short} is_short, {interest_rate} interest_rate, {funding_fees} funding_fees, {realized_profit} realized_profit, {amount_precision} amount_precision, {price_precision} price_precision, - {precision_mode} precision_mode + {precision_mode} precision_mode, {contract_size} contract_size from {trade_back_name} """)) @@ -308,7 +309,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # if ('orders' not in previous_tables # or not has_column(cols_orders, 'stop_price')): migrating = False - if not has_column(cols_trades, 'precision_mode'): + if not has_column(cols_trades, 'contract_size'): migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 0c438956a..a46a2dd9f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -296,6 +296,7 @@ class LocalTrade(): amount_precision: Optional[float] = None price_precision: Optional[float] = None precision_mode: Optional[int] = None + contract_size: Optional[float] = None # Leverage trading properties liquidation_price: Optional[float] = None @@ -1142,6 +1143,7 @@ class Trade(_DECL_BASE, LocalTrade): amount_precision = Column(Float, nullable=True) price_precision = Column(Float, nullable=True) precision_mode = Column(Integer, nullable=True) + contract_size = Column(Float, nullable=True) # Leverage trading properties leverage = Column(Float, nullable=True, default=1.0) From fe7108ae755733b75d76736e028d2dcfccb8769a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 20:48:02 +0200 Subject: [PATCH 21/22] Convert amount to contracts before comparing for close closes #7279 --- freqtrade/exchange/__init__.py | 3 ++- freqtrade/exchange/exchange.py | 4 ++-- freqtrade/freqtradebot.py | 1 + freqtrade/optimize/backtesting.py | 18 +++++++++++------- freqtrade/persistence/trade_model.py | 7 ++++++- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index cb63c6b9a..57114a342 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -9,7 +9,8 @@ from freqtrade.exchange.bitpanda import Bitpanda from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.coinbasepro import Coinbasepro -from freqtrade.exchange.exchange import (amount_to_precision, available_exchanges, ccxt_exchanges, +from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision, + available_exchanges, ccxt_exchanges, contracts_to_amount, date_minus_candles, is_exchange_known_ccxt, is_exchange_officially_supported, market_is_active, price_to_precision, timeframe_to_minutes, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 44288e696..7d2a2f5c5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2892,7 +2892,7 @@ def market_is_active(market: Dict) -> bool: return market.get('active', True) is not False -def amount_to_contracts(amount: float, contract_size: float) -> float: +def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float: """ Convert amount to contracts. :param amount: amount to convert @@ -2905,7 +2905,7 @@ def amount_to_contracts(amount: float, contract_size: float) -> float: return amount -def contracts_to_amount(num_contracts: float, contract_size: float) -> float: +def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float: """ Takes num-contracts and converts it to contract size :param num_contracts: number of contracts diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bb66303a8..5791a816d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -756,6 +756,7 @@ class FreqtradeBot(LoggingMixin): amount_precision=self.exchange.get_precision_amount(pair), price_precision=self.exchange.get_precision_price(pair), precision_mode=self.exchange.precisionMode, + contract_size=self.exchange.get_contract_size(pair), ) else: # This is additional buy, we reset fee_open_currency so timeout checking can work diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e32eae533..e81698eef 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,7 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.exchange.exchange import amount_to_precision +from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision, + contracts_to_amount) from freqtrade.mixins import LoggingMixin from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress @@ -823,11 +824,13 @@ class Backtesting: self.order_id_counter += 1 base_currency = self.exchange.get_pair_base_currency(pair) amount_p = (stake_amount / propose_rate) * leverage - amount = self.exchange._contracts_to_amount( - pair, amount_to_precision( - self.exchange._amount_to_contracts(pair, amount_p), - self.exchange.get_precision_amount(pair), self.precision_mode) - ) + contract_size = self.exchange.get_contract_size(pair) + precision_amount = self.exchange.get_precision_amount(pair) + amount = contracts_to_amount( + amount_to_precision( + amount_to_contracts(amount_p, contract_size), + precision_amount, self.precision_mode), + contract_size) # Backcalculate actual stake amount. stake_amount = amount * propose_rate / leverage @@ -859,9 +862,10 @@ class Backtesting: trading_mode=self.trading_mode, leverage=leverage, # interest_rate=interest_rate, - amount_precision=self.exchange.get_precision_amount(pair), + amount_precision=precision_amount, price_precision=self.exchange.get_precision_price(pair), precision_mode=self.precision_mode, + contract_size=contract_size, orders=[], ) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index a46a2dd9f..b25487993 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -15,6 +15,7 @@ from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPE from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import amount_to_precision, price_to_precision +from freqtrade.exchange.exchange import amount_to_contracts, contracts_to_amount from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE from freqtrade.util import FtPrecise @@ -624,7 +625,11 @@ class LocalTrade(): else: logger.warning( f'Got different open_order_id {self.open_order_id} != {order.order_id}') - amount_tr = amount_to_precision(self.amount, self.amount_precision, self.precision_mode) + amount_tr = contracts_to_amount( + amount_to_precision( + amount_to_contracts(self.amount, self.contract_size), + self.amount_precision, self.precision_mode), + self.contract_size) if isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC): self.close(order.safe_price) else: From a6d78a8615a91e18d9adc830bd615bca0f1faa72 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 Aug 2022 21:18:02 +0200 Subject: [PATCH 22/22] initialize Since parameter properly closes #7285 --- freqtrade/data/history/history_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 04d0bf6d3..7a3fa4e0c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -330,13 +330,12 @@ def _download_trades_history(exchange: Exchange, try: until = None + since = 0 if timerange: if timerange.starttype == 'date': since = timerange.startts * 1000 if timerange.stoptype == 'date': until = timerange.stopts * 1000 - else: - since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 trades = data_handler.trades_load(pair) @@ -349,6 +348,9 @@ def _download_trades_history(exchange: Exchange, logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") trades = [] + if not since: + since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 + from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: # Reset since to the last available point