diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a205f24ec..86c4ec1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.8 + - types-requests==2.28.9 - types-tabulate==0.8.11 - types-python-dateutil==2.8.19 # stages: [push] diff --git a/docs/data-download.md b/docs/data-download.md index 55c2ad738..be36d579d 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -186,7 +186,7 @@ Freqtrade currently supports 3 data-formats for both OHLCV and trades data: By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively. -To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: +To persist this change, you should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: ``` jsonc // ... diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a53e909e0..bffc04d1c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.3.1 -mkdocs-material==8.4.0 +mkdocs-material==8.4.1 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 diff --git a/docs/rest-api.md b/docs/rest-api.md index 1ec9b6c12..d9840a09c 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -317,7 +317,7 @@ whitelist ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. -This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs/ - but it'll depend on your settings. +This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs - but it'll depend on your settings. ### Advanced API usage using JWT tokens diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index a9b032818..8d46f42e1 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -75,7 +75,7 @@ class AwesomeStrategy(IStrategy): ``` -### Stake size management +## Stake size management Called before entering a trade, makes it possible to manage your position size when placing a new trade. diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 36a86bece..ce26d39ab 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -81,7 +81,7 @@ def start_download_data(args: Dict[str, Any]) -> None: data_format_trades=config['dataformat_trades'], ) else: - if not exchange._ft_has.get('ohlcv_has_history', True): + if not exchange.get_option('ohlcv_has_history', True): raise OperationalException( f"Historic klines not available for {exchange.name}. " "Please use `--dl-trades` instead for this exchange " diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index cba1b60db..7a3fa4e0c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -302,8 +302,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. - tf_mark = exchange._ft_has['mark_ohlcv_timeframe'] - fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) + tf_mark = exchange.get_option('mark_ohlcv_timeframe') + fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price')) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): @@ -330,13 +330,12 @@ def _download_trades_history(exchange: Exchange, try: until = None + since = 0 if timerange: if timerange.starttype == 'date': since = timerange.startts * 1000 if timerange.stoptype == 'date': until = timerange.stopts * 1000 - else: - since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 trades = data_handler.trades_load(pair) @@ -349,6 +348,9 @@ def _download_trades_history(exchange: Exchange, logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") trades = [] + if not since: + since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 + from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: # Reset since to the last available point diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index cb63c6b9a..57114a342 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -9,7 +9,8 @@ from freqtrade.exchange.bitpanda import Bitpanda from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.coinbasepro import Coinbasepro -from freqtrade.exchange.exchange import (amount_to_precision, available_exchanges, ccxt_exchanges, +from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision, + available_exchanges, ccxt_exchanges, contracts_to_amount, date_minus_candles, is_exchange_known_ccxt, is_exchange_officially_supported, market_is_active, price_to_precision, timeframe_to_minutes, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7ef6858a5..7d2a2f5c5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -54,8 +54,8 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} - # Additional headers - added to the ccxt object - _headers: Dict = {} + # Additional parameters - added to the ccxt object + _ccxt_params: Dict = {} # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has @@ -242,9 +242,9 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) - if self._headers: - # Inject static headers after the above output to not confuse users. - ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if self._ccxt_params: + # Inject static options after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts(self._ccxt_params, ccxt_kwargs) if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -408,7 +408,7 @@ class Exchange: else: return DataFrame() - def _get_contract_size(self, pair: str) -> float: + def get_contract_size(self, pair: str) -> float: if self.trading_mode == TradingMode.FUTURES: market = self.markets[pair] contract_size: float = 1.0 @@ -421,7 +421,7 @@ class Exchange: def _trades_contracts_to_amount(self, trades: List) -> List: if len(trades) > 0 and 'symbol' in trades[0]: - contract_size = self._get_contract_size(trades[0]['symbol']) + contract_size = self.get_contract_size(trades[0]['symbol']) if contract_size != 1: for trade in trades: trade['amount'] = trade['amount'] * contract_size @@ -429,7 +429,7 @@ class Exchange: def _order_contracts_to_amount(self, order: Dict) -> Dict: if 'symbol' in order and order['symbol'] is not None: - contract_size = self._get_contract_size(order['symbol']) + contract_size = self.get_contract_size(order['symbol']) if contract_size != 1: for prop in self._ft_has.get('order_props_in_contracts', []): if prop in order and order[prop] is not None: @@ -438,19 +438,13 @@ class Exchange: def _amount_to_contracts(self, pair: str, amount: float) -> float: - contract_size = self._get_contract_size(pair) - if contract_size and contract_size != 1: - return amount / contract_size - else: - return amount + contract_size = self.get_contract_size(pair) + return amount_to_contracts(amount, contract_size) def _contracts_to_amount(self, pair: str, num_contracts: float) -> float: - contract_size = self._get_contract_size(pair) - if contract_size and contract_size != 1: - return num_contracts * contract_size - else: - return num_contracts + contract_size = self.get_contract_size(pair) + return contracts_to_amount(num_contracts, contract_size) def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: if exchange_config.get('sandbox'): @@ -674,6 +668,12 @@ class Exchange: f"Freqtrade does not support {mm_value} {trading_mode.value} on {self.name}" ) + def get_option(self, param: str, default: Any = None) -> Any: + """ + Get parameter value from _ft_has + """ + return self._ft_has.get(param, default) + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. @@ -2892,6 +2892,33 @@ def market_is_active(market: Dict) -> bool: return market.get('active', True) is not False +def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float: + """ + Convert amount to contracts. + :param amount: amount to convert + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: num-contracts + """ + if contract_size and contract_size != 1: + return amount / contract_size + else: + return amount + + +def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float: + """ + Takes num-contracts and converts it to contract size + :param num_contracts: number of contracts + :param contract_size: contract size - taken from exchange.get_contract_size(pair) + :return: Amount + """ + + if contract_size and contract_size != 1: + return num_contracts * contract_size + else: + return num_contracts + + def amount_to_precision(amount: float, amount_precision: Optional[float], precisionMode: Optional[int]) -> float: """ diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index c6ed0c66c..426a4b64d 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -25,7 +25,6 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, - "ohlcv_volume_currency": "quote", "time_in_force_parameter": "timeInForce", "order_time_in_force": ['gtc', 'ioc'], "stoploss_order_types": {"limit": "limit"}, @@ -34,7 +33,6 @@ class Gateio(Exchange): _ft_has_futures: Dict = { "needs_trading_fees": True, - "ohlcv_volume_currency": "base", "fee_cost_in_contracts": False, # Set explicitly to false for clarity "order_props_in_contracts": ['amount', 'filled', 'remaining'], } diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 540e76fca..80373e071 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -39,6 +39,8 @@ class Okx(Exchange): net_only = True + _ccxt_params: Dict = {'options': {'brokerId': 'ffb5405ad327SUDE'}} + def ohlcv_candle_limit( self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index c8dbdf5e5..b3060deff 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -421,7 +421,7 @@ class FreqaiDataDrawer: ) # if self.live: - self.model_dictionary[dk.model_filename] = model + self.model_dictionary[coin] = model self.pair_dict[coin]["model_filename"] = dk.model_filename self.pair_dict[coin]["data_path"] = str(dk.data_path) self.save_drawer_to_disk() @@ -460,8 +460,8 @@ class FreqaiDataDrawer: ) # try to access model in memory instead of loading object from disk to save time - if dk.live and dk.model_filename in self.model_dictionary: - model = self.model_dictionary[dk.model_filename] + if dk.live and coin in self.model_dictionary: + model = self.model_dictionary[coin] elif not dk.keras: model = load(dk.data_path / f"{dk.model_filename}_model.joblib") else: diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1a80a1f86..6f87be31d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -601,6 +601,8 @@ class FreqaiDataKitchen: is an outlier. """ + from math import cos, sin + if predict: train_ft_df = self.data_dictionary['train_features'] pred_ft_df = self.data_dictionary['prediction_features'] @@ -619,23 +621,47 @@ class FreqaiDataKitchen: else: + def normalise_distances(distances): + normalised_distances = (distances - distances.min()) / \ + (distances.max() - distances.min()) + return normalised_distances + + def rotate_point(origin, point, angle): + # rotate a point counterclockwise by a given angle (in radians) + # around a given origin + x = origin[0] + cos(angle) * (point[0] - origin[0]) - \ + sin(angle) * (point[1] - origin[1]) + y = origin[1] + sin(angle) * (point[0] - origin[0]) + \ + cos(angle) * (point[1] - origin[1]) + return (x, y) + MinPts = len(self.data_dictionary['train_features'].columns) * 2 # measure pairwise distances to train_features.shape[1]*2 nearest neighbours neighbors = NearestNeighbors( n_neighbors=MinPts, n_jobs=self.thread_count) neighbors_fit = neighbors.fit(self.data_dictionary['train_features']) distances, _ = neighbors_fit.kneighbors(self.data_dictionary['train_features']) - distances = np.sort(distances, axis=0) - index_ten_pct = int(len(distances[:, 1]) * 0.1) - distances = distances[index_ten_pct:, 1] - epsilon = distances[-1] + distances = np.sort(distances, axis=0).mean(axis=1) + + normalised_distances = normalise_distances(distances) + x_range = np.linspace(0, 1, len(distances)) + line = np.linspace(normalised_distances[0], + normalised_distances[-1], len(normalised_distances)) + deflection = np.abs(normalised_distances - line) + max_deflection_loc = np.where(deflection == deflection.max())[0][0] + origin = x_range[max_deflection_loc], line[max_deflection_loc] + point = x_range[max_deflection_loc], normalised_distances[max_deflection_loc] + rot_angle = np.pi / 4 + elbow_loc = rotate_point(origin, point, rot_angle) + + epsilon = elbow_loc[1] * (distances[-1] - distances[0]) + distances[0] clustering = DBSCAN(eps=epsilon, min_samples=MinPts, n_jobs=int(self.thread_count)).fit( self.data_dictionary['train_features'] ) - logger.info(f'DBSCAN found eps of {epsilon}.') + logger.info(f'DBSCAN found eps of {epsilon:.2f}.') self.data['DBSCAN_eps'] = epsilon self.data['DBSCAN_min_samples'] = MinPts @@ -806,7 +832,7 @@ class FreqaiDataKitchen: if (len(do_predict) - do_predict.sum()) > 0: logger.info( - f"DI tossed {len(do_predict) - do_predict.sum():.2f} predictions for " + f"DI tossed {len(do_predict) - do_predict.sum()} predictions for " "being too far from training data" ) @@ -981,13 +1007,6 @@ class FreqaiDataKitchen: data_load_timerange.stopts = int(time) retrain = True - # logger.info( - # f"downloading data for " - # f"{(data_load_timerange.stopts-data_load_timerange.startts)/SECONDS_IN_DAY:.2f} " - # " days. " - # f"Extension of {additional_seconds/SECONDS_IN_DAY:.2f} days" - # ) - return retrain, trained_timerange, data_load_timerange def set_new_model_names(self, pair: str, trained_timerange: TimeRange): diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 07303b49f..239cb1869 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -82,12 +82,15 @@ class IFreqaiModel(ABC): if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 + self.pair_it_train = 0 self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) self.last_trade_database_summary: DataFrame = {} self.current_trade_database_summary: DataFrame = {} self.analysis_lock = Lock() self.inference_time: float = 0 + self.train_time: float = 0 self.begin_time: float = 0 + self.begin_time_train: float = 0 self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) def assert_config(self, config: Dict[str, Any]) -> None: @@ -130,11 +133,20 @@ class IFreqaiModel(ABC): dk = self.start_backtesting(dataframe, metadata, self.dk) dataframe = dk.remove_features_from_df(dk.return_dataframe) - del dk + self.clean_up() if self.live: self.inference_timer('stop') return dataframe + def clean_up(self): + """ + Objects that should be handled by GC already between coins, but + are explicitly shown here to help demonstrate the non-persistence of these + objects. + """ + self.model = None + self.dk = None + @threaded def start_scanning(self, strategy: IStrategy) -> None: """ @@ -161,9 +173,11 @@ class IFreqaiModel(ABC): dk.set_paths(pair, new_trained_timerange.stopts) if retrain: + self.train_timer('start') self.train_model_in_series( new_trained_timerange, pair, strategy, dk, data_load_timerange ) + self.train_timer('stop') self.dd.save_historic_predictions_to_disk() @@ -490,8 +504,7 @@ class IFreqaiModel(ABC): data_load_timerange: TimeRange, ): """ - Retrieve data and train model in single threaded mode (only used if model directory is empty - upon startup for dry/live ) + Retrieve data and train model. :param new_trained_timerange: TimeRange = the timerange to train the model on :param metadata: dict = strategy provided metadata :param strategy: IStrategy = user defined strategy object @@ -622,6 +635,24 @@ class IFreqaiModel(ABC): self.inference_time = 0 return + def train_timer(self, do='start'): + """ + Timer designed to track the cumulative time spent training the full pairlist in + FreqAI. + """ + if do == 'start': + self.pair_it_train += 1 + self.begin_time_train = time.time() + elif do == 'stop': + end = time.time() + self.train_time += (end - self.begin_time_train) + if self.pair_it_train == self.total_pairs: + logger.info( + f'Total time spent training pairlist {self.train_time:.2f} seconds') + self.pair_it_train = 0 + self.train_time = 0 + return + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4e3af64ea..5791a816d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -271,7 +271,7 @@ class FreqtradeBot(LoggingMixin): Return the number of free open trades slots or 0 if max number of open trades reached """ - open_trades = len(Trade.get_open_trades()) + open_trades = Trade.get_open_trade_count() return max(0, self.config['max_open_trades'] - open_trades) def update_funding_fees(self): @@ -290,13 +290,14 @@ class FreqtradeBot(LoggingMixin): def startup_backpopulate_precision(self): - trades = Trade.get_trades([Trade.precision_mode.is_(None)]) + trades = Trade.get_trades([Trade.contract_size.is_(None)]) for trade in trades: if trade.exchange != self.exchange.id: continue trade.precision_mode = self.exchange.precisionMode trade.amount_precision = self.exchange.get_precision_amount(trade.pair) trade.price_precision = self.exchange.get_precision_price(trade.pair) + trade.contract_size = self.exchange.get_contract_size(trade.pair) Trade.commit() def startup_update_open_orders(self): @@ -755,6 +756,7 @@ class FreqtradeBot(LoggingMixin): amount_precision=self.exchange.get_precision_amount(pair), price_precision=self.exchange.get_precision_price(pair), precision_mode=self.exchange.precisionMode, + contract_size=self.exchange.get_contract_size(pair), ) else: # This is additional buy, we reset fee_open_currency so timeout checking can work diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 12536c333..e81698eef 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,7 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.exchange.exchange import amount_to_precision +from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision, + contracts_to_amount) from freqtrade.mixins import LoggingMixin from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress @@ -267,7 +268,7 @@ class Backtesting: funding_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'], + timeframe=self.exchange.get_option('mark_ohlcv_timeframe'), timerange=self.timerange, startup_candles=0, fail_without_data=True, @@ -279,12 +280,12 @@ class Backtesting: mark_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'], + timeframe=self.exchange.get_option('mark_ohlcv_timeframe'), timerange=self.timerange, startup_candles=0, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), - candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"]) + candle_type=CandleType.from_string(self.exchange.get_option("mark_ohlcv_price")) ) # Combine data to avoid combining the data per trade. unavailable_pairs = [] @@ -823,11 +824,13 @@ class Backtesting: self.order_id_counter += 1 base_currency = self.exchange.get_pair_base_currency(pair) amount_p = (stake_amount / propose_rate) * leverage - amount = self.exchange._contracts_to_amount( - pair, amount_to_precision( - self.exchange._amount_to_contracts(pair, amount_p), - self.exchange.get_precision_amount(pair), self.precision_mode) - ) + contract_size = self.exchange.get_contract_size(pair) + precision_amount = self.exchange.get_precision_amount(pair) + amount = contracts_to_amount( + amount_to_precision( + amount_to_contracts(amount_p, contract_size), + precision_amount, self.precision_mode), + contract_size) # Backcalculate actual stake amount. stake_amount = amount * propose_rate / leverage @@ -859,9 +862,10 @@ class Backtesting: trading_mode=self.trading_mode, leverage=leverage, # interest_rate=interest_rate, - amount_precision=self.exchange.get_precision_amount(pair), + amount_precision=precision_amount, price_precision=self.exchange.get_precision_price(pair), precision_mode=self.precision_mode, + contract_size=contract_size, orders=[], ) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 311554359..1131c88b4 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -133,6 +133,7 @@ def migrate_trades_and_orders_table( amount_precision = get_column_def(cols, 'amount_precision', 'null') price_precision = get_column_def(cols, 'price_precision', 'null') precision_mode = get_column_def(cols, 'precision_mode', 'null') + contract_size = get_column_def(cols, 'contract_size', 'null') # Schema migration necessary with engine.begin() as connection: @@ -161,7 +162,7 @@ def migrate_trades_and_orders_table( timeframe, open_trade_value, close_profit_abs, trading_mode, leverage, liquidation_price, is_short, interest_rate, funding_fees, realized_profit, - amount_precision, price_precision, precision_mode + amount_precision, price_precision, precision_mode, contract_size ) select id, lower(exchange), pair, {base_currency} base_currency, {stake_currency} stake_currency, @@ -189,7 +190,7 @@ def migrate_trades_and_orders_table( {is_short} is_short, {interest_rate} interest_rate, {funding_fees} funding_fees, {realized_profit} realized_profit, {amount_precision} amount_precision, {price_precision} price_precision, - {precision_mode} precision_mode + {precision_mode} precision_mode, {contract_size} contract_size from {trade_back_name} """)) @@ -308,7 +309,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # if ('orders' not in previous_tables # or not has_column(cols_orders, 'stop_price')): migrating = False - if not has_column(cols_trades, 'precision_mode'): + if not has_column(cols_trades, 'contract_size'): migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index b954fee20..b25487993 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -15,6 +15,7 @@ from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPE from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import amount_to_precision, price_to_precision +from freqtrade.exchange.exchange import amount_to_contracts, contracts_to_amount from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE from freqtrade.util import FtPrecise @@ -296,6 +297,7 @@ class LocalTrade(): amount_precision: Optional[float] = None price_precision: Optional[float] = None precision_mode: Optional[int] = None + contract_size: Optional[float] = None # Leverage trading properties liquidation_price: Optional[float] = None @@ -623,7 +625,11 @@ class LocalTrade(): else: logger.warning( f'Got different open_order_id {self.open_order_id} != {order.order_id}') - amount_tr = amount_to_precision(self.amount, self.amount_precision, self.precision_mode) + amount_tr = contracts_to_amount( + amount_to_precision( + amount_to_contracts(self.amount, self.contract_size), + self.amount_precision, self.precision_mode), + self.contract_size) if isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC): self.close(order.safe_price) else: @@ -1044,6 +1050,16 @@ class LocalTrade(): """ return Trade.get_trades_proxy(is_open=True) + @staticmethod + def get_open_trade_count() -> int: + """ + get open trade count + """ + if Trade.use_db: + return Trade.query.filter(Trade.is_open.is_(True)).count() + else: + return len(LocalTrade.trades_open) + @staticmethod def stoploss_reinitialization(desired_stoploss): """ @@ -1132,6 +1148,7 @@ class Trade(_DECL_BASE, LocalTrade): amount_precision = Column(Float, nullable=True) price_precision = Column(Float, nullable=True) precision_mode = Column(Integer, nullable=True) + contract_size = Column(Float, nullable=True) # Leverage trading properties leverage = Column(Float, nullable=True, default=1.0) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index e364e1a69..8138a5fb6 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -73,7 +73,7 @@ class VolumePairList(IPairList): if (not self._use_range and not ( self._exchange.exchange_has('fetchTickers') - and self._exchange._ft_has["tickers_have_quoteVolume"])): + and self._exchange.get_option("tickers_have_quoteVolume"))): raise OperationalException( "Exchange does not support dynamic whitelist in this configuration. " "Please edit your config and either remove Volumepairlist, " @@ -193,7 +193,7 @@ class VolumePairList(IPairList): ) in candles else None # in case of candle data calculate typical price and quoteVolume for candle if pair_candles is not None and not pair_candles.empty: - if self._exchange._ft_has["ohlcv_volume_currency"] == "base": + if self._exchange.get_option("ohlcv_volume_currency") == "base": pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + pair_candles['close']) / 3 diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 74b28dffe..b99e7a94b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -193,7 +193,10 @@ class IResolver: :return: List of dicts containing 'name', 'class' and 'location' entries """ logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") - objects = [] + objects: List[Dict[str, Any]] = [] + if not directory.is_dir(): + logger.info(f"'{directory}' is not a directory, skipping.") + return objects for entry in directory.iterdir(): if ( recursive and entry.is_dir() diff --git a/requirements-dev.txt b/requirements-dev.txt index 0cd4a6a6c..9c45e7277 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,7 +17,7 @@ pytest-mock==3.8.2 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.7.1 +time-machine==2.8.1 # Convert jupyter notebooks to markdown documents nbconvert==6.5.3 @@ -25,6 +25,6 @@ nbconvert==6.5.3 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.8 +types-requests==2.28.9 types-tabulate==0.8.11 types-python-dateutil==2.8.19 diff --git a/requirements.txt b/requirements.txt index 77925f98b..4a0531ea8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.2 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.92.20 +ccxt==1.92.52 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.11 -jsonschema==4.9.1 +jsonschema==4.14.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 @@ -34,7 +34,7 @@ orjson==3.7.12 sdnotify==0.3.2 # API Server -fastapi==0.79.0 +fastapi==0.79.1 uvicorn==0.18.2 pyjwt==2.4.0 aiofiles==0.8.0 diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index c3a5fe17f..29b317c3c 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -409,14 +409,14 @@ class TestCCXTExchange(): assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int)) assert futures_leverage >= 1.0 - def test_ccxt__get_contract_size(self, exchange_futures): + def test_ccxt_get_contract_size(self, exchange_futures): futures, futures_name = exchange_futures if futures: futures_pair = EXCHANGES[futures_name].get( 'futures_pair', EXCHANGES[futures_name]['pair'] ) - contract_size = futures._get_contract_size(futures_pair) + contract_size = futures.get_contract_size(futures_pair) assert (isinstance(contract_size, float) or isinstance(contract_size, int)) assert contract_size >= 0.0 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ec259d703..5002a33e1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -181,11 +181,11 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) assert log_has(asynclogmsg, caplog) # Test additional headers case - Exchange._headers = {'hello': 'world'} + Exchange._ccxt_params = {'hello': 'world'} ex = Exchange(conf) assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) - assert ex._api.headers == {'hello': 'world'} + assert ex._api.hello == 'world' assert ex._ccxt_config == {} Exchange._headers = {} @@ -2352,10 +2352,11 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name) order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val) assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC' # Not all exchanges support all limits for orderbook - if not exchange._ft_has['l2_limit_range'] or val in exchange._ft_has['l2_limit_range']: + if (not exchange.get_option('l2_limit_range') + or val in exchange.get_option('l2_limit_range')): assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val else: - next_limit = exchange.get_next_limit_in_list(val, exchange._ft_has['l2_limit_range']) + next_limit = exchange.get_next_limit_in_list(val, exchange.get_option('l2_limit_range')) assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == next_limit @@ -3311,16 +3312,16 @@ def test_merge_ft_has_dict(default_conf, mocker): ex = Kraken(default_conf) assert ex._ft_has != Exchange._ft_has_default - assert ex._ft_has['trades_pagination'] == 'id' - assert ex._ft_has['trades_pagination_arg'] == 'since' + assert ex.get_option('trades_pagination') == 'id' + assert ex.get_option('trades_pagination_arg') == 'since' # Binance defines different values ex = Binance(default_conf) assert ex._ft_has != Exchange._ft_has_default - assert ex._ft_has['stoploss_on_exchange'] - assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc'] - assert ex._ft_has['trades_pagination'] == 'id' - assert ex._ft_has['trades_pagination_arg'] == 'fromId' + assert ex.get_option('stoploss_on_exchange') + assert ex.get_option('order_time_in_force') == ['gtc', 'fok', 'ioc'] + assert ex.get_option('trades_pagination') == 'id' + assert ex.get_option('trades_pagination_arg') == 'fromId' conf = copy.deepcopy(default_conf) conf['exchange']['_ft_has_params'] = {"DeadBeef": 20, @@ -4287,7 +4288,7 @@ def test__fetch_and_calculate_funding_fees_datetime_called( ('XLTCUSDT', 0.01, 'futures'), ('ETH/USDT:USDT', 10, 'futures') ]) -def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode): +def est__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode): api_mock = MagicMock() default_conf['trading_mode'] = trading_mode default_conf['margin_mode'] = 'isolated' @@ -4306,7 +4307,7 @@ def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_m 'contractSize': '10', } }) - size = exchange._get_contract_size(pair) + size = exchange.get_contract_size(pair) assert expected_size == size @@ -5145,7 +5146,7 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange._get_contract_size = MagicMock(return_value=contract_size) + exchange.get_contract_size = MagicMock(return_value=contract_size) api_mock.create_order.reset_mock() order = exchange.stoploss( diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 5b6f15d11..b794cdc99 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -48,6 +48,10 @@ def test_search_all_strategies_with_failed(): assert len([x for x in strategies if x['class'] is not None]) == 9 assert len([x for x in strategies if x['class'] is None]) == 1 + directory = Path(__file__).parent / "strats_nonexistingdir" + strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) + assert len(strategies) == 0 + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a1a16c039..138527053 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -473,8 +473,6 @@ def test_create_trade_no_signal(default_conf_usdt, fee, mocker) -> None: freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade, enter_long=False, exit_long=False) - Trade.query = MagicMock() - Trade.query.filter = MagicMock() assert not freqtrade.create_trade('ETH/USDT') diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 50d0788ca..d8973e9d0 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1689,6 +1689,7 @@ def test_get_open(fee, is_short, use_db): create_mock_trades(fee, is_short, use_db) assert len(Trade.get_open_trades()) == 4 + assert Trade.get_open_trade_count() == 4 Trade.use_db = True @@ -1701,6 +1702,7 @@ def test_get_open_lev(fee, use_db): create_mock_trades_with_leverage(fee, use_db) assert len(Trade.get_open_trades()) == 5 + assert Trade.get_open_trade_count() == 5 Trade.use_db = True