diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bafe9cb8..0ff57b270 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,6 +309,9 @@ jobs: webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} cleanup-prior-runs: + permissions: + actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it + contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch runs-on: ubuntu-20.04 steps: - name: Cleanup previous runs on this branch @@ -321,6 +324,10 @@ jobs: notify-complete: needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 + # Discord notification can't handle schedule events + if: (github.event_name != 'schedule') + permissions: + repository-projects: read steps: - name: Check user permission diff --git a/README.md b/README.md index 679dbcab0..cad39f9ac 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,14 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +### Experimentally, freqtrade also supports futures on the following exchanges + +- [X] [Binance](https://www.binance.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) +- [X] [OKX](https://okx.com/). + +Please make sure to read the [exchange specific notes](docs/exchanges.md), as well as the [trading with leverage](docs/leverage.md) documentation before diving in. + ### Community tested Exchanges confirmed working by the community: diff --git a/config_examples/config_binance.example.json b/config_examples/config_binance.example.json index ad8862afa..35b9fcd20 100644 --- a/config_examples/config_binance.example.json +++ b/config_examples/config_binance.example.json @@ -90,7 +90,7 @@ }, "bot_name": "freqtrade", "initial_state": "running", - "force_enter_enable": false, + "force_entry_enable": false, "internals": { "process_throttle_secs": 5 } diff --git a/docs/index.md b/docs/index.md index 2aa80c240..e0a88a381 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,6 +51,14 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +### Experimentally, freqtrade also supports futures on the following exchanges: + +- [X] [Binance](https://www.binance.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) +- [X] [OKX](https://okx.com/). + +Please make sure to read the [exchange specific notes](exchanges.md), as well as the [trading with leverage](leverage.md) documentation before diving in. + ### Community tested Exchanges confirmed working by the community: diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index b01959e10..27777c2ce 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -146,11 +146,11 @@ See [Dataframe access](strategy-advanced.md#dataframe-access) for more informati ## Custom stoploss -Called for open trade every throttling iteration (roughly every 5 seconds) until a trade is closed. +Called for open trade every iteration (roughly every 5 seconds) until a trade is closed. The usage of the custom stoploss method must be enabled by setting `use_custom_stoploss=True` on the strategy object. -The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss (before this method is called for the first time for a trade). +The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss (before this method is called for the first time for a trade), and is still mandatory. The method must return a stoploss value (float / number) as a percentage of the current price. E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoploss price 2% lower, at 196 USD. @@ -389,30 +389,30 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) new_entryprice = dataframe['bollinger_10_lowerband'].iat[-1] - + return new_entryprice def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) new_exitprice = dataframe['bollinger_10_upperband'].iat[-1] - + return new_exitprice ``` !!! Warning - Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. - **Example**: + Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. + **Example**: If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98, which is 2% below the current (proposed) rate. !!! Warning "Backtesting" @@ -454,7 +454,7 @@ class AwesomeStrategy(IStrategy): 'exit': 60 * 25 } - def check_entry_timeout(self, pair: str, trade: 'Trade', order: 'Order', + def check_entry_timeout(self, pair: str, trade: 'Trade', order: 'Order', current_time: datetime, **kwargs) -> bool: if trade.open_rate > 100 and trade.open_date_utc < current_time - timedelta(minutes=5): return True @@ -532,7 +532,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, current_time: datetime, entry_tag: Optional[str], + time_in_force: str, current_time: datetime, entry_tag: Optional[str], side: str, **kwargs) -> bool: """ Called right before placing a entry order. @@ -640,35 +640,35 @@ from freqtrade.persistence import Trade class DigDeeperStrategy(IStrategy): - + position_adjustment_enable = True - + # Attempts to handle large drops with DCA. High stoploss is required. stoploss = -0.30 - + # ... populate_* methods - + # Example specific variables max_entry_position_adjustment = 3 # This number is explained a bit further down max_dca_multiplier = 5.5 - + # This is called when placing the initial order (opening trade) def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, proposed_stake: float, min_stake: float, max_stake: float, entry_tag: Optional[str], side: str, **kwargs) -> float: - + # We need to leave most of the funds for possible further DCA orders # This also applies to fixed stakes return proposed_stake / self.max_dca_multiplier - + def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, min_stake: float, max_stake: float, **kwargs): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased. This means extra buy orders with additional fees. - + :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Current buy rate. @@ -678,7 +678,7 @@ class DigDeeperStrategy(IStrategy): :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: Stake amount to adjust your trade """ - + if current_profit > -0.05: return None diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 206a6f5f3..0c8e721c0 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -12,7 +12,8 @@ import pandas as pd from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.exceptions import OperationalException -from freqtrade.misc import get_backtest_metadata_filename, json_load +from freqtrade.misc import json_load +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename from freqtrade.persistence import LocalTrade, Trade, init_db diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d4741bd64..82dcacb51 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -9,6 +9,7 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone from math import ceil +from threading import Lock from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union import arrow @@ -96,6 +97,9 @@ class Exchange: self._markets: Dict = {} self._trading_fees: Dict[str, Any] = {} self._leverage_tiers: Dict[str, List[Dict]] = {} + # Lock event loop. This is necessary to avoid race-conditions when using force* commands + # Due to funding fee fetching. + self._loop_lock = Lock() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self._config: Dict = {} @@ -167,7 +171,7 @@ class Exchange: self._api_async = self._init_ccxt( exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config) - logger.info('Using Exchange "%s"', self.name) + logger.info(f'Using Exchange "{self.name}"') if validate: # Check if timeframe is available @@ -555,7 +559,7 @@ class Exchange: # Therefore we also show that. raise OperationalException( f"The ccxt library does not provide the list of timeframes " - f"for the exchange \"{self.name}\" and this exchange " + f"for the exchange {self.name} and this exchange " f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}") if timeframe and (timeframe not in self.timeframes): @@ -785,7 +789,9 @@ class Exchange: rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' - _amount = self.amount_to_precision(pair, amount) + # Rounding here must respect to contract sizes + _amount = self._contracts_to_amount( + pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount))) dry_order: Dict[str, Any] = { 'id': order_id, 'symbol': pair, @@ -1775,7 +1781,8 @@ class Exchange: async def gather_stuff(): return await asyncio.gather(*input_coro, return_exceptions=True) - results = self.loop.run_until_complete(gather_stuff()) + with self._loop_lock: + results = self.loop.run_until_complete(gather_stuff()) for res in results: if isinstance(res, Exception): @@ -2032,9 +2039,10 @@ class Exchange: if not self.exchange_has("fetchTrades"): raise OperationalException("This exchange does not support downloading Trades.") - return self.loop.run_until_complete( - self._async_get_trade_history(pair=pair, since=since, - until=until, from_id=from_id)) + with self._loop_lock: + return self.loop.run_until_complete( + self._async_get_trade_history(pair=pair, since=since, + until=until, from_id=from_id)) @retrier def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bee6c4746..b6f0daeed 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -587,7 +587,6 @@ class FreqtradeBot(LoggingMixin): Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY :param stake_amount: amount of stake-currency for the pair - :param leverage: amount of leverage applied to this trade :return: True if a buy order is created, false if it fails. """ time_in_force = self.strategy.order_time_in_force['entry'] @@ -666,16 +665,6 @@ class FreqtradeBot(LoggingMixin): amount = safe_value_fallback(order, 'filled', 'amount') enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') - # TODO: this might be unnecessary, as we're calling it in update_trade_state. - isolated_liq = self.exchange.get_liquidation_price( - leverage=leverage, - pair=pair, - amount=amount, - open_rate=enter_limit_filled_price, - is_short=is_short - ) - interest_rate = self.exchange.get_interest_rate() - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') base_currency = self.exchange.get_pair_base_currency(pair) @@ -704,8 +693,6 @@ class FreqtradeBot(LoggingMixin): timeframe=timeframe_to_minutes(self.config['timeframe']), leverage=leverage, is_short=is_short, - interest_rate=interest_rate, - liquidation_price=isolated_liq, trading_mode=self.trading_mode, funding_fees=funding_fees ) @@ -1375,7 +1362,8 @@ class FreqtradeBot(LoggingMixin): default_retval=proposed_limit_rate)( pair=trade.pair, trade=trade, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_limit_rate, current_profit=current_profit) + proposed_rate=proposed_limit_rate, current_profit=current_profit, + exit_tag=exit_check.exit_reason) limit = self.get_valid_price(custom_exit_price, proposed_limit_rate) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 55a533725..c3968e61c 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,13 +2,11 @@ Various tool function for Freqtrade and scripts """ import gzip -import hashlib import logging import re -from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import Any, Iterator, List, Union +from typing import Any, Iterator, List from typing.io import IO from urllib.parse import urlparse @@ -251,34 +249,3 @@ def parse_db_uri_for_logging(uri: str): return uri pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') - - -def get_strategy_run_id(strategy) -> str: - """ - Generate unique identification hash for a backtest run. Identical config and strategy file will - always return an identical hash. - :param strategy: strategy object. - :return: hex string id. - """ - digest = hashlib.sha1() - config = deepcopy(strategy.config) - - # Options that have no impact on results of individual backtest. - not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') - for k in not_important_keys: - if k in config: - del config[k] - - # Explicitly allow NaN values (e.g. max_open_trades). - # as it does not matter for getting the hash. - digest.update(rapidjson.dumps(config, default=str, - number_mode=rapidjson.NM_NAN).encode('utf-8')) - with open(strategy.__file__, 'rb') as fp: - digest.update(fp.read()) - return digest.hexdigest().lower() - - -def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: - """Return metadata filename for specified backtest results file.""" - filename = Path(filename) - return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtest_caching.py b/freqtrade/optimize/backtest_caching.py new file mode 100644 index 000000000..d9d270072 --- /dev/null +++ b/freqtrade/optimize/backtest_caching.py @@ -0,0 +1,40 @@ +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Union + +import rapidjson + + +def get_strategy_run_id(strategy) -> str: + """ + Generate unique identification hash for a backtest run. Identical config and strategy file will + always return an identical hash. + :param strategy: strategy object. + :return: hex string id. + """ + digest = hashlib.sha1() + config = deepcopy(strategy.config) + + # Options that have no impact on results of individual backtest. + not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') + for k in not_important_keys: + if k in config: + del config[k] + + # Explicitly allow NaN values (e.g. max_open_trades). + # as it does not matter for getting the hash. + digest.update(rapidjson.dumps(config, default=str, + number_mode=rapidjson.NM_NAN).encode('utf-8')) + # Include _ft_params_from_file - so changing parameter files cause cache eviction + digest.update(rapidjson.dumps( + strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8')) + with open(strategy.__file__, 'rb') as fp: + digest.update(fp.read()) + return digest.hexdigest().lower() + + +def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: + """Return metadata filename for specified backtest results file.""" + filename = Path(filename) + return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1789290bc..7456c40cf 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, @@ -54,6 +54,11 @@ ESHORT_IDX = 8 # Exit short ENTER_TAG_IDX = 9 EXIT_TAG_IDX = 10 +# Every change to this headers list must evaluate further usages of the resulting tuple +# and eventually change the constants for indexes at the top +HEADERS = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', + 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] + class Backtesting: """ @@ -265,10 +270,18 @@ class Backtesting: candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"]) ) # Combine data to avoid combining the data per trade. + unavailable_pairs = [] for pair in self.pairlists.whitelist: + if pair not in self.exchange._leverage_tiers: + unavailable_pairs.append(pair) + continue self.futures_data[pair] = funding_rates_dict[pair].merge( mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"]) + if unavailable_pairs: + raise OperationalException( + f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. " + "It is therefore impossible to backtest with this pair at the moment.") else: self.futures_data = {} @@ -306,10 +319,7 @@ class Backtesting: :param processed: a processed dictionary with format {pair, data}, which gets cleared to optimize memory usage! """ - # Every change to this headers list must evaluate further usages of the resulting tuple - # and eventually change the constants for indexes at the top - headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] + data: Dict = {} self.progress.init_step(BacktestState.CONVERT, len(processed)) @@ -321,7 +331,7 @@ class Backtesting: if not pair_data.empty: # Cleanup from prior runs - pair_data.drop(headers[5:] + ['buy', 'sell'], axis=1, errors='ignore') + pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore') df_analyzed = self.strategy.advise_exit( self.strategy.advise_entry(pair_data, {'pair': pair}), @@ -340,7 +350,7 @@ class Backtesting: # To avoid using data from future, we use entry/exit signals shifted # from the previous candle - for col in headers[5:]: + for col in HEADERS[5:]: tag_col = col in ('enter_tag', 'exit_tag') if col in df_analyzed.columns: df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace( @@ -352,7 +362,7 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - data[pair] = df_analyzed[headers].values.tolist() if not df_analyzed.empty else [] + data[pair] = df_analyzed[HEADERS].values.tolist() if not df_analyzed.empty else [] return data def _get_close_rate(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, @@ -516,10 +526,10 @@ class Backtesting: exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX] - exit_ = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] + exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] exit_ = self.strategy.should_exit( trade, row[OPEN_IDX], exit_candle_time, # type: ignore - enter=enter, exit_=exit_, + enter=enter, exit_=exit_sig, low=row[LOW_IDX], high=row[HIGH_IDX] ) @@ -541,7 +551,8 @@ class Backtesting: default_retval=closerate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, - proposed_rate=closerate, current_profit=current_profit) + proposed_rate=closerate, current_profit=current_profit, + exit_tag=exit_.exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: @@ -568,6 +579,7 @@ class Backtesting: len(row) > EXIT_TAG_IDX and row[EXIT_TAG_IDX] is not None and len(row[EXIT_TAG_IDX]) > 0 + and exit_.exit_type in (ExitType.EXIT_SIGNAL,) ): trade.exit_reason = row[EXIT_TAG_IDX] @@ -626,9 +638,7 @@ class Backtesting: detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX] detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX] detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX] - headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] - for det_row in detail_data[headers].values.tolist(): + for det_row in detail_data[HEADERS].values.tolist(): res = self._get_exit_trade_entry_for_candle(trade, det_row) if res: return res diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3ae975ca7..1dafb483c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -468,6 +468,7 @@ class Hyperopt: self.backtesting.exchange._api = None self.backtesting.exchange._api_async = None self.backtesting.exchange.loop = None # type: ignore + self.backtesting.exchange._loop_lock = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index dd058aff4..1d58dc339 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,8 +11,8 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, - get_backtest_metadata_filename, round_coin_value) +from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a9c07f12c..2cacc06e2 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -429,12 +429,10 @@ class LocalTrade(): def __repr__(self): open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' - leverage = self.leverage or 1.0 - is_short = self.is_short or False return ( f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' - f'is_short={is_short}, leverage={leverage}, ' + f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})' ) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 63812f52f..0da129583 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -2,7 +2,7 @@ import logging from ipaddress import IPv4Address from typing import Any, Dict -import rapidjson +import orjson import uvicorn from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -24,7 +24,7 @@ class FTJSONResponse(JSONResponse): Use rapidjson for responses Handles NaN and Inf / -Inf in a javascript way by default. """ - return rapidjson.dumps(content).encode("utf-8") + return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY) class ApiServer(RPCHandler): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c2531fec3..1a9be4503 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -943,7 +943,7 @@ class Telegram(RPCHandler): else: fiat_currency = self._config.get('fiat_display_currency', '') try: - statlist, head, fiat_profit_sum = self._rpc._rpc_status_table( + statlist, _, _ = self._rpc._rpc_status_table( self._config['stake_currency'], fiat_currency) except RPCException: self._send_msg(msg='No open trade found.') diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 40744ed08..a23269770 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -362,7 +362,7 @@ class IStrategy(ABC, HyperStrategyMixin): def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: """ Custom exit price logic, returning the new exit price. @@ -375,6 +375,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param exit_tag: Exit reason. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New exit price value if provided """ diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index d5e2ea8ce..ed40ef509 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -32,7 +32,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: """ Custom exit price logic, returning the new exit price. @@ -45,6 +45,7 @@ def custom_exit_price(self, pair: str, trade: 'Trade', :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param exit_tag: Exit reason. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New exit price value if provided """ diff --git a/pyproject.toml b/pyproject.toml index 50f0242a8..e8d5ed47e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ exclude = ''' line_length = 100 multi_line_output=0 lines_after_imports=2 -skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*"] +skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*", "**/user_data/*"] [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/requirements.txt b/requirements.txt index de14b9f2c..ab8329979 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,8 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 +# Properly format api responses +orjson==3.6.8 # Notify systemd sdnotify==0.3.2 diff --git a/setup.py b/setup.py index 250cafdc9..c5e418d0d 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( 'pycoingecko', 'py_find_1st', 'python-rapidjson', + 'orjson', 'sdnotify', 'colorama', 'jinja2', diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a6918b6d4..689ffa4ce 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -909,7 +909,7 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' - r'for the exchange ".*" and this exchange ' + r'for the exchange .* and this exchange ' r'is therefore not supported. *'): Exchange(default_conf) @@ -930,7 +930,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' - r'for the exchange ".*" and this exchange ' + r'for the exchange .* and this exchange ' r'is therefore not supported. *'): Exchange(default_conf) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 16d1aa2a5..c87a0ef73 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,7 +22,7 @@ from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange.exchange import timeframe_to_next_date -from freqtrade.misc import get_strategy_run_id +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver @@ -1342,6 +1342,39 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat assert 'STRATEGY SUMMARY' in captured.out +@pytest.mark.filterwarnings("ignore:deprecated") +def test_backtest_start_futures_noliq(default_conf_usdt, mocker, + caplog, testdatadir, capsys): + # Tests detail-data loading + default_conf_usdt.update({ + "trading_mode": "futures", + "margin_mode": "isolated", + "use_exit_signal": True, + "exit_profit_only": False, + "exit_profit_offset": 0.0, + "ignore_roi_if_entry_signal": False, + "strategy": CURRENT_TEST_STRATEGY, + }) + patch_exchange(mocker) + + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) + # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf_usdt) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '1h', + ] + args = get_args(args) + with pytest.raises(OperationalException, match=r"Pairs .* got no leverage tiers available\."): + start_backtesting(args) + + @pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_nomock_futures(default_conf_usdt, mocker, caplog, testdatadir, capsys): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 4910213b4..43f783a53 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -13,7 +13,6 @@ import uvicorn from fastapi import FastAPI from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient -from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ @@ -985,7 +984,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, assert_response(rc) resp_values = rc.json() assert len(resp_values) == 4 - assert isnan(resp_values[0]['profit_abs']) + assert resp_values[0]['profit_abs'] is None def test_api_version(botclient): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3737c7c05..89fe88a2c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -717,12 +717,12 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) (True, 'spot', 'gateio', None, 0.0, None), (False, 'spot', 'okx', None, 0.0, None), (True, 'spot', 'okx', None, 0.0, None), - (True, 'futures', 'binance', 'isolated', 0.0, 11.89108910891089), - (False, 'futures', 'binance', 'isolated', 0.0, 8.070707070707071), + (True, 'futures', 'binance', 'isolated', 0.0, 11.88151815181518), + (False, 'futures', 'binance', 'isolated', 0.0, 8.080471380471382), (True, 'futures', 'gateio', 'isolated', 0.0, 11.87413417771621), (False, 'futures', 'gateio', 'isolated', 0.0, 8.085708510208207), - (True, 'futures', 'binance', 'isolated', 0.05, 11.796534653465345), - (False, 'futures', 'binance', 'isolated', 0.05, 8.167171717171717), + (True, 'futures', 'binance', 'isolated', 0.05, 11.7874422442244), + (False, 'futures', 'binance', 'isolated', 0.05, 8.17644781144781), (True, 'futures', 'gateio', 'isolated', 0.05, 11.7804274688304), (False, 'futures', 'gateio', 'isolated', 0.05, 8.181423084697796), (True, 'futures', 'okx', 'isolated', 0.0, 11.87413417771621), @@ -845,6 +845,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade.open_order_id is None assert trade.open_rate == 10 assert trade.stake_amount == round(order['price'] * order['filled'] / leverage, 8) + assert pytest.approx(trade.liquidation_price) == liq_price # In case of rejected or expired order and partially filled order['status'] = 'expired' @@ -932,8 +933,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade.open_rate_requested == 10 # In case of custom entry price not float type - freqtrade.exchange.get_maintenance_ratio_and_amt = MagicMock(return_value=(0.01, 0.01)) - freqtrade.exchange.name = exchange_name order['status'] = 'open' order['id'] = '5568' freqtrade.strategy.custom_entry_price = lambda **kwargs: "string price" @@ -946,7 +945,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade.is_short = is_short assert trade assert trade.open_rate_requested == 10 - assert trade.liquidation_price == liq_price # In case of too high stake amount @@ -3221,7 +3219,7 @@ def test_execute_trade_exit_custom_exit_price( freqtrade.execute_trade_exit( trade=trade, limit=ticker_usdt_sell_up()['ask' if is_short else 'bid'], - exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL) + exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL, exit_reason='foo') ) # Sell price must be different to default bid price @@ -3249,8 +3247,8 @@ def test_execute_trade_exit_custom_exit_price( 'profit_ratio': profit_ratio, 'stake_currency': 'USDT', 'fiat_currency': 'USD', - 'sell_reason': ExitType.EXIT_SIGNAL.value, - 'exit_reason': ExitType.EXIT_SIGNAL.value, + 'sell_reason': 'foo', + 'exit_reason': 'foo', 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY,