From 97e8bb09e8eb093455af9e1bc341a1e1cec193f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Aug 2022 17:16:03 +0200 Subject: [PATCH 01/18] Update exchange documentation with note about leverage --- docs/leverage.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/leverage.md b/docs/leverage.md index 429aff86c..0a265277e 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -92,6 +92,8 @@ One account is used to share collateral between markets (trading pairs). Margin "margin_mode": "cross" ``` +Please read the [exchange specific notes](exchanges.md) for exchanges that support this mode and how they differ. + ## Set leverage to use Different strategies and risk profiles will require different levels of leverage. From 6498e352c183b276ba7b084668474b25b23d347b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 31 Dec 2022 10:23:20 +0100 Subject: [PATCH 02/18] Remove pointless default --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 51341588d..8a40ff3ff 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2808,7 +2808,7 @@ class Exchange: def get_maintenance_ratio_and_amt( self, pair: str, - nominal_value: float = 0.0, + nominal_value: float, ) -> Tuple[float, Optional[float]]: """ Important: Must be fetching data from cached values as this is used by backtesting! From cd7bd9bf9ab26cf9ed3ca56caed09b2bd7bbfb69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 31 Dec 2022 10:08:51 +0100 Subject: [PATCH 03/18] Update gate liquidation price link --- freqtrade/exchange/exchange.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8a40ff3ff..d691738fe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2764,11 +2764,16 @@ class Exchange: """ Important: Must be fetching data from cached values as this is used by backtesting! PERPETUAL: - gateio: https://www.gate.io/help/futures/perpetual/22160/calculation-of-liquidation-price + gateio: https://www.gate.io/help/futures/futures/27724/liquidation-price-bankruptcy-price + > Liquidation Price = (Entry Price ± Margin / Contract Multiplier / Size) / + [ 1 ± (Maintenance Margin Ratio + Taker Rate)] + Wherein, "+" or "-" depends on whether the contract goes long or short: + "-" for long, and "+" for short. + okex: https://www.okex.com/support/hc/en-us/articles/ 360053909592-VI-Introduction-to-the-isolated-mode-of-Single-Multi-currency-Portfolio-margin - :param exchange_name: + :param pair: Pair to calculate liquidation price for :param open_rate: Entry price of position :param is_short: True if the trade is a short, false otherwise :param amount: Absolute value of position size incl. leverage (in base currency) From 74b924471a32b544886bfc9f15b29b2b581faea0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 31 Dec 2022 10:59:26 +0100 Subject: [PATCH 04/18] type ccxt_compat tests --- tests/exchange/test_ccxt_compat.py | 103 +++++++++++++++-------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 7f23c2031..e721ee2c9 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -8,16 +8,19 @@ suitable to run with freqtrade. from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path +from typing import Tuple import pytest from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date -from freqtrade.exchange.exchange import timeframe_to_msecs +from freqtrade.exchange.exchange import Exchange, timeframe_to_msecs from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_default_conf_usdt +EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str] + # Exchanges that should be tested EXCHANGES = { 'bittrex': { @@ -141,19 +144,19 @@ def exchange_futures(request, exchange_conf, class_mocker): @pytest.mark.longrun class TestCCXTExchange(): - def test_load_markets(self, exchange): - exchange, exchangename = exchange + def test_load_markets(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] - markets = exchange.markets + markets = exch.markets assert pair in markets assert isinstance(markets[pair], dict) - assert exchange.market_is_spot(markets[pair]) + assert exch.market_is_spot(markets[pair]) - def test_has_validations(self, exchange): + def test_has_validations(self, exchange: EXCHANGE_FIXTURE_TYPE): - exchange, exchangename = exchange + exch, exchangename = exchange - exchange.validate_ordertypes({ + exch.validate_ordertypes({ 'entry': 'limit', 'exit': 'limit', 'stoploss': 'limit', @@ -162,13 +165,13 @@ class TestCCXTExchange(): if exchangename == 'gateio': # gateio doesn't have market orders on spot return - exchange.validate_ordertypes({ + exch.validate_ordertypes({ 'entry': 'market', 'exit': 'market', 'stoploss': 'market', }) - def test_load_markets_futures(self, exchange_futures): + def test_load_markets_futures(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges @@ -181,11 +184,11 @@ class TestCCXTExchange(): assert exchange.market_is_future(markets[pair]) - def test_ccxt_fetch_tickers(self, exchange): - exchange, exchangename = exchange + def test_ccxt_fetch_tickers(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] - tickers = exchange.get_tickers() + tickers = exch.get_tickers() assert pair in tickers assert 'ask' in tickers[pair] assert tickers[pair]['ask'] is not None @@ -195,11 +198,11 @@ class TestCCXTExchange(): if EXCHANGES[exchangename].get('hasQuoteVolume'): assert tickers[pair]['quoteVolume'] is not None - def test_ccxt_fetch_ticker(self, exchange): - exchange, exchangename = exchange + def test_ccxt_fetch_ticker(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] - ticker = exchange.fetch_ticker(pair) + ticker = exch.fetch_ticker(pair) assert 'ask' in ticker assert ticker['ask'] is not None assert 'bid' in ticker @@ -208,21 +211,21 @@ class TestCCXTExchange(): if EXCHANGES[exchangename].get('hasQuoteVolume'): assert ticker['quoteVolume'] is not None - def test_ccxt_fetch_l2_orderbook(self, exchange): - exchange, exchangename = exchange + def test_ccxt_fetch_l2_orderbook(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] - l2 = exchange.fetch_l2_order_book(pair) + l2 = exch.fetch_l2_order_book(pair) assert 'asks' in l2 assert 'bids' in l2 assert len(l2['asks']) >= 1 assert len(l2['bids']) >= 1 - l2_limit_range = exchange._ft_has['l2_limit_range'] - l2_limit_range_required = exchange._ft_has['l2_limit_range_required'] + l2_limit_range = exch._ft_has['l2_limit_range'] + l2_limit_range_required = exch._ft_has['l2_limit_range_required'] if exchangename == 'gateio': # TODO: Gateio is unstable here at the moment, ignoring the limit partially. return for val in [1, 2, 5, 25, 100]: - l2 = exchange.fetch_l2_order_book(pair, val) + l2 = exch.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: if val > 50: # Orderbooks are not always this deep. @@ -232,7 +235,7 @@ class TestCCXTExchange(): assert len(l2['asks']) == val assert len(l2['bids']) == val else: - next_limit = exchange.get_next_limit_in_list( + next_limit = exch.get_next_limit_in_list( val, l2_limit_range, l2_limit_range_required) if next_limit is None: assert len(l2['asks']) > 100 @@ -245,23 +248,23 @@ class TestCCXTExchange(): assert len(l2['asks']) == next_limit assert len(l2['asks']) == next_limit - def test_ccxt_fetch_ohlcv(self, exchange): - exchange, exchangename = exchange + def test_ccxt_fetch_ohlcv(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] pair_tf = (pair, timeframe, CandleType.SPOT) - ohlcv = exchange.refresh_latest_ohlcv([pair_tf]) + ohlcv = exch.refresh_latest_ohlcv([pair_tf]) assert isinstance(ohlcv, dict) - assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) - # assert len(exchange.klines(pair_tf)) > 200 + assert len(ohlcv[pair_tf]) == len(exch.klines(pair_tf)) + # assert len(exch.klines(pair_tf)) > 200 # Assume 90% uptime ... - assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit( + assert len(exch.klines(pair_tf)) > exch.ohlcv_candle_limit( timeframe, CandleType.SPOT) * 0.90 # Check if last-timeframe is within the last 2 intervals now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) - assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) + assert exch.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) def ccxt__async_get_candle_history(self, exchange, exchangename, pair, timeframe, candle_type): @@ -289,17 +292,17 @@ class TestCCXTExchange(): assert len(candles) >= min(candle_count, candle_count1) assert candles[0][0] == since_ms or (since_ms + timeframe_ms) - def test_ccxt__async_get_candle_history(self, exchange): - exchange, exchangename = exchange + def test_ccxt__async_get_candle_history(self, exchange: EXCHANGE_FIXTURE_TYPE): + exc, exchangename = exchange # For some weired reason, this test returns random lengths for bittrex. - if not exchange._ft_has['ohlcv_has_history'] or exchangename in ('bittrex'): + if not exc._ft_has['ohlcv_has_history'] or exchangename in ('bittrex'): return pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] self.ccxt__async_get_candle_history( - exchange, exchangename, pair, timeframe, CandleType.SPOT) + exc, exchangename, pair, timeframe, CandleType.SPOT) - def test_ccxt__async_get_candle_history_futures(self, exchange_futures): + def test_ccxt__async_get_candle_history_futures(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges @@ -309,7 +312,7 @@ class TestCCXTExchange(): self.ccxt__async_get_candle_history( exchange, exchangename, pair, timeframe, CandleType.FUTURES) - def test_ccxt_fetch_funding_rate_history(self, exchange_futures): + def test_ccxt_fetch_funding_rate_history(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges @@ -347,7 +350,7 @@ class TestCCXTExchange(): (rate['open'].min() != rate['open'].max()) ) - def test_ccxt_fetch_mark_price_history(self, exchange_futures): + def test_ccxt_fetch_mark_price_history(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges @@ -371,7 +374,7 @@ class TestCCXTExchange(): assert mark_candles[mark_candles['date'] == prev_hour].iloc[0]['open'] != 0.0 assert mark_candles[mark_candles['date'] == this_hour].iloc[0]['open'] != 0.0 - def test_ccxt__calculate_funding_fees(self, exchange_futures): + def test_ccxt__calculate_funding_fees(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): exchange, exchangename = exchange_futures if not exchange: # exchange_futures only returns values for supported exchanges @@ -387,16 +390,16 @@ class TestCCXTExchange(): # TODO: tests fetch_trades (?) - def test_ccxt_get_fee(self, exchange): - exchange, exchangename = exchange + def test_ccxt_get_fee(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] threshold = 0.01 - assert 0 < exchange.get_fee(pair, 'limit', 'buy') < threshold - assert 0 < exchange.get_fee(pair, 'limit', 'sell') < threshold - assert 0 < exchange.get_fee(pair, 'market', 'buy') < threshold - assert 0 < exchange.get_fee(pair, 'market', 'sell') < threshold + assert 0 < exch.get_fee(pair, 'limit', 'buy') < threshold + assert 0 < exch.get_fee(pair, 'limit', 'sell') < threshold + assert 0 < exch.get_fee(pair, 'market', 'buy') < threshold + assert 0 < exch.get_fee(pair, 'market', 'sell') < threshold - def test_ccxt_get_max_leverage_spot(self, exchange): + def test_ccxt_get_max_leverage_spot(self, exchange: EXCHANGE_FIXTURE_TYPE): spot, spot_name = exchange if spot: leverage_in_market_spot = EXCHANGES[spot_name].get('leverage_in_spot_market') @@ -406,7 +409,7 @@ class TestCCXTExchange(): assert (isinstance(spot_leverage, float) or isinstance(spot_leverage, int)) assert spot_leverage >= 1.0 - def test_ccxt_get_max_leverage_futures(self, exchange_futures): + def test_ccxt_get_max_leverage_futures(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures if futures: leverage_tiers_public = EXCHANGES[futures_name].get('leverage_tiers_public') @@ -419,7 +422,7 @@ class TestCCXTExchange(): assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int)) assert futures_leverage >= 1.0 - def test_ccxt_get_contract_size(self, exchange_futures): + def test_ccxt_get_contract_size(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures if futures: futures_pair = EXCHANGES[futures_name].get( @@ -430,7 +433,7 @@ class TestCCXTExchange(): assert (isinstance(contract_size, float) or isinstance(contract_size, int)) assert contract_size >= 0.0 - def test_ccxt_load_leverage_tiers(self, exchange_futures): + def test_ccxt_load_leverage_tiers(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures if futures and EXCHANGES[futures_name].get('leverage_tiers_public'): leverage_tiers = futures.load_leverage_tiers() @@ -463,7 +466,7 @@ class TestCCXTExchange(): oldminNotional = tier['minNotional'] oldmaxNotional = tier['maxNotional'] - def test_ccxt_dry_run_liquidation_price(self, exchange_futures): + def test_ccxt_dry_run_liquidation_price(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures if futures and EXCHANGES[futures_name].get('leverage_tiers_public'): @@ -494,7 +497,7 @@ class TestCCXTExchange(): assert (isinstance(liquidation_price, float)) assert liquidation_price >= 0.0 - def test_ccxt_get_max_pair_stake_amount(self, exchange_futures): + def test_ccxt_get_max_pair_stake_amount(self, exchange_futures: EXCHANGE_FIXTURE_TYPE): futures, futures_name = exchange_futures if futures: futures_pair = EXCHANGES[futures_name].get( From d304f95c13591e16acd5062e2ba0779144c21d90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 03:00:55 +0000 Subject: [PATCH 05/18] Bump filelock from 3.8.2 to 3.9.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.8.2 to 3.9.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.8.2...3.9.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index fcae2cbdd..0cfd6cfa1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,5 +5,5 @@ scipy==1.9.3 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.8.2 +filelock==3.9.0 progressbar2==4.2.0 From 488b4512e0f14ea91f3a2ec37fd13de0d58e0bf9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 03:01:00 +0000 Subject: [PATCH 06/18] Bump time-machine from 2.8.2 to 2.9.0 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.8.2 to 2.9.0. - [Release notes](https://github.com/adamchainz/time-machine/releases) - [Changelog](https://github.com/adamchainz/time-machine/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.8.2...2.9.0) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c1fd160ee..cf7a75d98 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ pytest-mock==3.10.0 pytest-random-order==1.1.0 isort==5.11.4 # For datetime mocking -time-machine==2.8.2 +time-machine==2.9.0 # fastapi testing httpx==0.23.1 From 724465c798351f979926223be1a18cad5521f923 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 03:01:07 +0000 Subject: [PATCH 07/18] Bump pydantic from 1.10.2 to 1.10.4 Bumps [pydantic](https://github.com/pydantic/pydantic) from 1.10.2 to 1.10.4. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v1.10.4/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v1.10.2...v1.10.4) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf6b270ab..b132971df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ sdnotify==0.3.2 # API Server fastapi==0.88.0 -pydantic==1.10.2 +pydantic==1.10.4 uvicorn==0.20.0 pyjwt==2.6.0 aiofiles==22.1.0 From 2c430c806c37ef5f32f47422c05281f963a3adbb Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Mon, 2 Jan 2023 15:54:49 +0000 Subject: [PATCH 08/18] Fix ROI table comma and spacing THanks to `@topdollar` in discord for noticing the typos. --- docs/hyperopt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 6b6c2a772..e72b850ca 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -365,7 +365,7 @@ class MyAwesomeStrategy(IStrategy): timeframe = '15m' minimal_roi = { "0": 0.10 - }, + } # Define the parameter spaces buy_ema_short = IntParameter(3, 50, default=5) buy_ema_long = IntParameter(15, 200, default=50) @@ -400,7 +400,7 @@ class MyAwesomeStrategy(IStrategy): return dataframe def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - conditions = [] + conditions = [] conditions.append(qtpylib.crossed_above( dataframe[f'ema_long_{self.buy_ema_long.value}'], dataframe[f'ema_short_{self.buy_ema_short.value}'] )) From 63db1fd89468fd00ecbc6e81f8651d3180620d25 Mon Sep 17 00:00:00 2001 From: zhanglei14 Date: Wed, 4 Jan 2023 01:16:52 +0800 Subject: [PATCH 09/18] Fix Backtesting Analysis Column Wrong --- freqtrade/data/entryexitanalysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 565a279b1..936134976 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -52,7 +52,7 @@ def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_cand return analysed_trades_dict -def _analyze_candles_and_indicators(pair, trades, signal_candles): +def _analyze_candles_and_indicators(pair, trades:pd.DataFrame, signal_candles:pd.DataFrame): buyf = signal_candles if len(buyf) > 0: @@ -120,7 +120,7 @@ def _do_group_table_output(bigdf, glist): else: agg_mask = {'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} + 'profit_ratio': ['median', 'mean', 'sum']} agg_cols = ['num_buys', 'profit_abs_sum', 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', 'total_profit_pct'] From 6f031f005db32f7d366204de836ee2c7319502e7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Jan 2023 20:29:08 +0100 Subject: [PATCH 10/18] Fix flake error --- freqtrade/data/entryexitanalysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 936134976..baa1cca3a 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -52,7 +52,7 @@ def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_cand return analysed_trades_dict -def _analyze_candles_and_indicators(pair, trades:pd.DataFrame, signal_candles:pd.DataFrame): +def _analyze_candles_and_indicators(pair, trades: pd.DataFrame, signal_candles: pd.DataFrame): buyf = signal_candles if len(buyf) > 0: From c384d1357ed38245e5d54e385daa01c12d940c97 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Tue, 3 Jan 2023 21:52:16 +0100 Subject: [PATCH 11/18] Update FreqaiExampleStrategy.py --- freqtrade/templates/FreqaiExampleStrategy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index fc39b0ab4..4690a6ccb 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -28,7 +28,7 @@ class FreqaiExampleStrategy(IStrategy): plot_config = { "main_plot": {}, "subplots": { - "prediction": {"prediction": {"color": "blue"}}, + "&-s_close": {"prediction": {"color": "blue"}}, "do_predict": { "do_predict": {"color": "brown"}, }, @@ -140,7 +140,8 @@ class FreqaiExampleStrategy(IStrategy): # If user wishes to use multiple targets, they can add more by # appending more columns with '&'. User should keep in mind that multi targets # requires a multioutput prediction model such as - # templates/CatboostPredictionMultiModel.py, + # freqai/prediction_models/CatboostRegressorMultiTarget.py, + # freqtrade trade --freqaimodel CatboostRegressorMultiTarget # df["&-s_range"] = ( # df["close"] From 6470635753625a88c6e2873ad1d6d3aef69df462 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Jan 2023 17:55:24 +0100 Subject: [PATCH 12/18] In cases of no losing trade, sortino ratio can't be calculated. closes #7977 --- freqtrade/data/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py index 09dd60208..1d2757f8f 100644 --- a/freqtrade/data/metrics.py +++ b/freqtrade/data/metrics.py @@ -239,7 +239,7 @@ def calculate_sortino(trades: pd.DataFrame, min_date: datetime, max_date: dateti down_stdev = np.std(trades.loc[trades['profit_abs'] < 0, 'profit_abs'] / starting_balance) - if down_stdev != 0: + if down_stdev != 0 and not np.isnan(down_stdev): sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) else: # Define high (negative) sortino ratio to be clear that this is NOT optimal. From 8e5b4750d64c265c138f14f420d79c39627afa75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Jan 2023 18:08:45 +0100 Subject: [PATCH 13/18] Continue in "regular backtest" case (no detail-data available). link to #7967 --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2b8b96cba..81e05ade0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1177,6 +1177,7 @@ class Backtesting: open_trade_count_start = self.backtest_loop( row, pair, current_time, end_date, max_open_trades, open_trade_count_start) + continue detail_data.loc[:, 'enter_long'] = row[LONG_IDX] detail_data.loc[:, 'exit_long'] = row[ELONG_IDX] detail_data.loc[:, 'enter_short'] = row[SHORT_IDX] From 5257e8b3ed33b33c6a074bb12894d13aba74bc92 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jan 2023 09:12:09 +0100 Subject: [PATCH 14/18] Fix random test failures on 3.8 --- tests/commands/test_commands.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index d568f48f6..967dbe296 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -746,9 +746,7 @@ def test_download_data_no_exchange(mocker, caplog): start_download_data(pargs) -def test_download_data_no_pairs(mocker, caplog): - - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) +def test_download_data_no_pairs(mocker): mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) @@ -770,8 +768,6 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_all_pairs(mocker, markets): - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) From 75b0a3e63deab3f038a791d445ff5c8fc566a7f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jan 2023 11:30:15 +0100 Subject: [PATCH 15/18] Use dedicated type for OHLCV response --- freqtrade/exchange/binance.py | 4 ++-- freqtrade/exchange/exchange.py | 6 +++--- freqtrade/exchange/types.py | 7 ++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 7462e4f81..9942a4268 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -11,7 +11,7 @@ from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier -from freqtrade.exchange.types import Tickers +from freqtrade.exchange.types import OHLCVResponse, Tickers from freqtrade.misc import deep_merge_dicts, json_load @@ -112,7 +112,7 @@ class Binance(Exchange): since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, until_ms: Optional[int] = None - ) -> Tuple[str, str, str, List]: + ) -> OHLCVResponse: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d691738fe..bcbdc0be8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -36,7 +36,7 @@ from freqtrade.exchange.exchange_utils import (CcxtModuleType, amount_to_contrac price_to_precision, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) -from freqtrade.exchange.types import Ticker, Tickers +from freqtrade.exchange.types import OHLCVResponse, Ticker, Tickers from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json, safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -1838,7 +1838,7 @@ class Exchange: since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, until_ms: Optional[int] = None - ) -> Tuple[str, str, str, List]: + ) -> OHLCVResponse: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading @@ -2025,7 +2025,7 @@ class Exchange: timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None, - ) -> Tuple[str, str, str, List]: + ) -> OHLCVResponse: """ Asynchronously get candle history data using fetch_ohlcv :param candle_type: '', mark, index, premiumIndex, or funding_rate diff --git a/freqtrade/exchange/types.py b/freqtrade/exchange/types.py index a60b454d4..3b543fa86 100644 --- a/freqtrade/exchange/types.py +++ b/freqtrade/exchange/types.py @@ -1,4 +1,6 @@ -from typing import Dict, Optional, TypedDict +from typing import Dict, List, Optional, Tuple, TypedDict + +from freqtrade.enums import CandleType class Ticker(TypedDict): @@ -14,3 +16,6 @@ class Ticker(TypedDict): Tickers = Dict[str, Ticker] + +# pair, timeframe, candleType, OHLCV +OHLCVResponse = Tuple[str, str, CandleType, List] From 4bac66ff0e4ec17b88d929ec63e30b007e2e0324 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jan 2023 11:33:47 +0100 Subject: [PATCH 16/18] Type ohlcv coroutine --- freqtrade/exchange/exchange.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index bcbdc0be8..ebcf6ff72 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1876,8 +1876,9 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) return pair, timeframe, candle_type, data - def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType, - since_ms: Optional[int], cache: bool) -> Coroutine: + def _build_coroutine( + self, pair: str, timeframe: str, candle_type: CandleType, + since_ms: Optional[int], cache: bool) -> Coroutine[Any, Any, OHLCVResponse]: not_all_data = cache and self.required_candle_call_count > 1 if cache and (pair, timeframe, candle_type) in self._klines: candle_limit = self.ohlcv_candle_limit(timeframe, candle_type) @@ -1914,7 +1915,7 @@ class Exchange: """ Build Coroutines to execute as part of refresh_latest_ohlcv """ - input_coroutines = [] + input_coroutines: List[Coroutine[Any, Any, OHLCVResponse]] = [] cached_pairs = [] for pair, timeframe, candle_type in set(pair_list): if (timeframe not in self.timeframes From bdf6537c60d75d9fbb1c215d2d5c7754e968cb43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jan 2023 11:45:15 +0100 Subject: [PATCH 17/18] Remove unused (and pointless) exchange method --- freqtrade/exchange/exchange.py | 14 --------- tests/exchange/test_exchange.py | 56 --------------------------------- 2 files changed, 70 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ebcf6ff72..6ce62c1f4 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1820,20 +1820,6 @@ class Exchange: logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data - def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, - since_ms: int, candle_type: CandleType) -> DataFrame: - """ - Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe - :param pair: Pair to download - :param timeframe: Timeframe to get data for - :param since_ms: Timestamp in milliseconds to get history from - :param candle_type: Any of the enum CandleType (must match trading mode!) - :return: OHLCV DataFrame - """ - ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, candle_type=candle_type) - return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 280e20ff0..843a5fbc1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1988,62 +1988,6 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ assert log_has_re(r"Async code raised an exception: .*", caplog) -@pytest.mark.parametrize("exchange_name", EXCHANGES) -@pytest.mark.parametrize('candle_type', ['mark', '']) -def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name, candle_type): - exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - ohlcv = [ - [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ], - [ - arrow.utcnow().shift(minutes=5).int_timestamp * 1000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ], - [ - arrow.utcnow().shift(minutes=10).int_timestamp * 1000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - pair = 'ETH/BTC' - - async def mock_candle_hist(pair, timeframe, candle_type, since_ms): - return pair, timeframe, candle_type, ohlcv - - exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) - # one_call calculation * 1.8 should do 2 calls - - since = 5 * 60 * exchange.ohlcv_candle_limit('5m', CandleType.SPOT) * 1.8 - ret = exchange.get_historic_ohlcv_as_df( - pair, - "5m", - int((arrow.utcnow().int_timestamp - since) * 1000), - candle_type=candle_type - ) - - assert exchange._async_get_candle_history.call_count == 2 - # Returns twice the above OHLCV data - assert len(ret) == 2 - assert isinstance(ret, DataFrame) - assert 'date' in ret.columns - assert 'open' in ret.columns - assert 'close' in ret.columns - assert 'high' in ret.columns - - @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) @pytest.mark.parametrize('candle_type', [CandleType.MARK, CandleType.SPOT]) From 787d292ba05a9d7008ea6006a5194051be1b03d5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jan 2023 22:31:32 +0100 Subject: [PATCH 18/18] Move "drop_candle" decision to coroutine --- freqtrade/exchange/exchange.py | 16 ++++++++-------- freqtrade/exchange/types.py | 4 ++-- tests/exchange/test_binance.py | 4 ++-- tests/exchange/test_exchange.py | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6ce62c1f4..b4b5f8342 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1813,7 +1813,7 @@ class Exchange: :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List with candle (OHLCV) data """ - pair, _, _, data = self.loop.run_until_complete( + pair, _, _, data, _ = self.loop.run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, is_new_pair=is_new_pair, candle_type=candle_type)) @@ -1855,12 +1855,12 @@ class Exchange: continue else: # Deconstruct tuple if it's not an exception - p, _, c, new_data = res + p, _, c, new_data, _ = res if p == pair and c == candle_type: data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - return pair, timeframe, candle_type, data + return pair, timeframe, candle_type, data, self._ohlcv_partial_candle def _build_coroutine( self, pair: str, timeframe: str, candle_type: CandleType, @@ -1965,7 +1965,6 @@ class Exchange: :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) - drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete # Gather coroutines to run input_coroutines, cached_pairs = self._build_ohlcv_dl_jobs(pair_list, since_ms, cache) @@ -1983,8 +1982,9 @@ class Exchange: if isinstance(res, Exception): logger.warning(f"Async code raised an exception: {repr(res)}") continue - # Deconstruct tuple (has 4 elements) - pair, timeframe, c_type, ticks = res + # Deconstruct tuple (has 5 elements) + pair, timeframe, c_type, ticks, drop_hint = res + drop_incomplete = drop_hint if drop_incomplete is None else drop_incomplete ohlcv_df = self._process_ohlcv_df( pair, timeframe, c_type, ticks, cache, drop_incomplete) @@ -2052,9 +2052,9 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, timeframe, candle_type, [] + return pair, timeframe, candle_type, [], self._ohlcv_partial_candle logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe) - return pair, timeframe, candle_type, data + return pair, timeframe, candle_type, data, self._ohlcv_partial_candle except ccxt.NotSupported as e: raise OperationalException( diff --git a/freqtrade/exchange/types.py b/freqtrade/exchange/types.py index 3b543fa86..813b09297 100644 --- a/freqtrade/exchange/types.py +++ b/freqtrade/exchange/types.py @@ -17,5 +17,5 @@ class Ticker(TypedDict): Tickers = Dict[str, Ticker] -# pair, timeframe, candleType, OHLCV -OHLCVResponse = Tuple[str, str, CandleType, List] +# pair, timeframe, candleType, OHLCV, drop last?, +OHLCVResponse = Tuple[str, str, CandleType, List, bool] diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 306a30985..189f0488d 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -557,7 +557,7 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, c exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/BTC' - respair, restf, restype, res = await exchange._async_get_historic_ohlcv( + respair, restf, restype, res, _ = await exchange._async_get_historic_ohlcv( pair, "5m", 1500000000000, is_new_pair=False, candle_type=candle_type) assert respair == pair assert restf == '5m' @@ -566,7 +566,7 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, c assert exchange._api_async.fetch_ohlcv.call_count > 400 # assert res == ohlcv exchange._api_async.fetch_ohlcv.reset_mock() - _, _, _, res = await exchange._async_get_historic_ohlcv( + _, _, _, res, _ = await exchange._async_get_historic_ohlcv( pair, "5m", 1500000000000, is_new_pair=True, candle_type=candle_type) # Called twice - one "init" call - and one to get the actual data. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 843a5fbc1..3714291d1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1955,7 +1955,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ pair = 'ETH/BTC' async def mock_candle_hist(pair, timeframe, candle_type, since_ms): - return pair, timeframe, candle_type, ohlcv + return pair, timeframe, candle_type, ohlcv, True exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls @@ -2007,7 +2007,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/USDT' - respair, restf, _, res = await exchange._async_get_historic_ohlcv( + respair, restf, _, res, _ = await exchange._async_get_historic_ohlcv( pair, "5m", 1500000000000, candle_type=candle_type, is_new_pair=False) assert respair == pair assert restf == '5m' @@ -2018,7 +2018,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ exchange._api_async.fetch_ohlcv.reset_mock() end_ts = 1_500_500_000_000 start_ts = 1_500_000_000_000 - respair, restf, _, res = await exchange._async_get_historic_ohlcv( + respair, restf, _, res, _ = await exchange._async_get_historic_ohlcv( pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False, until_ms=end_ts ) @@ -2250,7 +2250,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT) assert type(res) is tuple - assert len(res) == 4 + assert len(res) == 5 assert res[0] == pair assert res[1] == "5m" assert res[2] == CandleType.SPOT @@ -2337,7 +2337,7 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT) assert type(res) is tuple - assert len(res) == 4 + assert len(res) == 5 assert res[0] == pair assert res[1] == "5m" assert res[2] == CandleType.SPOT