diff --git a/Dockerfile b/Dockerfile index 8d4f0ebe6..4b399174b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.1-slim-buster as base +FROM python:3.9.2-slim-buster as base # Setup env ENV LANG C.UTF-8 diff --git a/README.md b/README.md index bb136d7f2..7ef0d4ce7 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,21 @@ expect. We strongly recommend you to have coding and Python knowledge. Do not hesitate to read the source code and understand the mechanism of this bot. -## Exchange marketplaces supported +## Supported Exchange marketplaces + +Please read the [exchange specific notes](docs/exchanges.md) to learn about eventual, special configurations needed for each exchange. - [X] [Bittrex](https://bittrex.com/) - [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists)) - [X] [Kraken](https://kraken.com/) -- [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +- [X] [FTX](https://ftx.com) +- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ + +### Community tested + +Exchanges confirmed working by the community: + +- [X] [Bitvavo](https://bitvavo.com/) ## Documentation diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 9bc1aa0a6..d987bcc69 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -51,6 +51,8 @@ fi docker images docker push ${IMAGE_NAME} +docker push ${IMAGE_NAME}:$TAG_PLOT +docker push ${IMAGE_NAME}:$TAG if [ $? -ne 0 ]; then echo "failed pushing repo" return 1 diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index bead18038..d2237b3e8 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -40,6 +40,11 @@ For the sample below, you then need to add the command line parameter `--hyperop A sample of this can be found below, which is identical to the Default Hyperopt loss implementation. A full sample can be found in [userdata/hyperopts](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_loss.py). ``` python +from datetime import datetime +from typing import Dict + +from pandas import DataFrame + from freqtrade.optimize.hyperopt import IHyperOptLoss TARGET_TRADES = 600 @@ -54,6 +59,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, + config: Dict, processed: Dict[str, DataFrame], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results @@ -81,6 +87,8 @@ Currently, the arguments are: * `trade_count`: Amount of trades (identical to `len(results)`) * `min_date`: Start date of the timerange used * `min_date`: End date of the timerange used +* `config`: Config object used (Note: Not all strategy-related parameters will be updated here if they are part of a hyperopt space). +* `processed`: Dict of Dataframes with the pair as keys containing the data used for backtesting. This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. diff --git a/docs/includes/protections.md b/docs/includes/protections.md index de34383ac..6bc57153e 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -40,7 +40,9 @@ All protection end times are rounded up to the next candle to avoid sudden, unex #### Stoploss Guard -`StoplossGuard` selects all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`). +`StoplossGuard` selects all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`). +If `trade_limit` or more trades resulted in stoploss, trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`). + This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. diff --git a/docs/index.md b/docs/index.md index b489861f0..db5088707 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,22 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python - Control/Monitor: Use Telegram or a REST API (start/stop the bot, show profit/loss, daily summary, current open trades results, etc.). - Analyse: Further analysis can be performed on either Backtesting data or Freqtrade trading history (SQL database), including automated standard plots, and methods to load the data into [interactive environments](data-analysis.md). +## Supported exchange marketplaces + +Please read the [exchange specific notes](exchanges.md) to learn about eventual, special configurations needed for each exchange. + +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](exchanges.md#blacklists)) +- [X] [Bittrex](https://bittrex.com/) +- [X] [FTX](https://ftx.com) +- [X] [Kraken](https://kraken.com/) +- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ + +### Community tested + +Exchanges confirmed working by the community: + +- [X] [Bitvavo](https://bitvavo.com/) + ## Requirements ### Hardware requirements diff --git a/docs/plotting.md b/docs/plotting.md index 19ddb4f57..d7ed5ab1f 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -188,7 +188,7 @@ Sample configuration with inline comments explaining the process: 'senkou_a': { 'color': 'green', #optional 'fill_to': 'senkou_b', - 'fill_label': 'Ichimoku Cloud' #optional, + 'fill_label': 'Ichimoku Cloud', #optional 'fill_color': 'rgba(255,76,46,0.2)', #optional }, # plot senkou_b, too. Not only the area to it. diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 85bd72323..94b2fca39 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==6.2.7 +mkdocs-material==6.2.8 mdx_truly_sane_lists==1.2 pymdown-extensions==8.1.1 diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 7e998570f..fdc95a3c1 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -315,11 +315,11 @@ class AwesomeStrategy(IStrategy): def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Check if the entry already exists - if not metadata["pair"] in self._cust_info: + if not metadata["pair"] in self.cust_info: # Create empty entry for this pair - self._cust_info[metadata["pair"]] = {} + self.cust_info[metadata["pair"]] = {} - if "crosstime" in self.cust_info[metadata["pair"]: + if "crosstime" in self.cust_info[metadata["pair"]]: self.cust_info[metadata["pair"]]["crosstime"] += 1 else: self.cust_info[metadata["pair"]]["crosstime"] = 1 @@ -444,14 +444,19 @@ It can also be used in specific callbacks to get the signal that caused the acti ``` python # fetch current dataframe if self.dp: - dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], - timeframe=self.timeframe) + if self.dp.runmode.value in ('live', 'dry_run'): + dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], + timeframe=self.timeframe) ``` !!! Note "No data available" Returns an empty dataframe if the requested pair was not cached. This should not happen when using whitelisted pairs. + +!!! Warning "Warning about backtesting" + This method will return an empty dataframe during backtesting. + ### *orderbook(pair, maximum)* ``` python @@ -462,8 +467,8 @@ if self.dp: dataframe['best_ask'] = ob['asks'][0][0] ``` -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used. +!!! Warning "Warning about backtesting" + The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used, as the method will return uptodate values. ### *ticker(pair)* diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 90e39fd76..5c479aa0b 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -24,7 +24,7 @@ config["strategy"] = "SampleStrategy" # Location of the data data_location = Path(config['user_data_dir'], 'data', 'binance') # Pair to analyze - Only use one pair here -pair = "BTC_USDT" +pair = "BTC/USDT" ``` @@ -34,7 +34,9 @@ from freqtrade.data.history import load_pair_history candles = load_pair_history(datadir=data_location, timeframe=config["timeframe"], - pair=pair) + pair=pair, + data_format = "hdf5", + ) # Confirm success print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {data_location}") diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 69301ca0e..802ddc2b1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -45,6 +45,16 @@ USERPATH_NOTEBOOKS = 'notebooks' TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] + +# Define decimals per coin for outputs +# Only used for outputs. +DECIMAL_PER_COIN_FALLBACK = 3 # Should be low to avoid listing all possible FIAT's +DECIMALS_PER_COIN = { + 'BTC': 8, + 'ETH': 5, +} + + # Soure files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 828fb78f3..8e851a8e8 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -383,3 +383,21 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date' high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] low_date = profit_results.loc[idxmin, date_col] return abs(min(max_drawdown_df['drawdown'])), high_date, low_date + + +def calculate_csum(trades: pd.DataFrame) -> Tuple[float, float]: + """ + Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :return: Tuple (float, float) with cumsum of profit_abs + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + + csum_df = pd.DataFrame() + csum_df['sum'] = trades['profit_abs'].cumsum() + csum_min = csum_df['sum'].min() + csum_max = csum_df['sum'].max() + + return csum_min, csum_max diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index e549a3701..ff86e522e 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -104,6 +104,7 @@ class Edge: exchange=self.exchange, timeframe=self.strategy.timeframe, timerange=self._timerange, + data_format=self.config.get('dataformat_ohlcv', 'json'), ) data = load_data( @@ -159,7 +160,8 @@ class Edge: available_capital = (total_capital + capital_in_trade) * self._capital_ratio allowed_capital_at_risk = available_capital * self._allowed_risk max_position_size = abs(allowed_capital_at_risk / stoploss) - position_size = min(max_position_size, free_capital) + # Position size must be below available capital. + position_size = min(min(max_position_size, free_capital), available_capital) if pair in self._cached_pairs: logger.info( 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4318f9cf0..fd7d47668 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -19,5 +19,11 @@ class Bittrex(Exchange): """ _ft_has: Dict = { + "ohlcv_candle_limit_per_timeframe": { + '1m': 1440, + '5m': 288, + '1h': 744, + '1d': 365, + }, "l2_limit_range": [1, 25, 500], } diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c7625b53c..617cd6c26 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -3,6 +3,7 @@ Cryptocurrency Exchanges support """ import asyncio +import http import inspect import logging from copy import deepcopy @@ -34,6 +35,12 @@ CcxtModuleType = Any logger = logging.getLogger(__name__) +# Workaround for adding samesite support to pre 3.8 python +# Only applies to python3.7, and only on certain exchanges (kraken) +# Replicates the fix from starlette (which is actually causing this problem) +http.cookies.Morsel._reserved["samesite"] = "SameSite" # type: ignore + + class Exchange: _config: Dict = {} @@ -94,7 +101,6 @@ class Exchange: logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) # Assign this directly for easy access - self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit'] self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle'] self._trades_pagination = self._ft_has['trades_pagination'] @@ -130,7 +136,8 @@ class Exchange: self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) - self.validate_required_startup_candles(config.get('startup_candle_count', 0)) + self.validate_required_startup_candles(config.get('startup_candle_count', 0), + config.get('timeframe', '')) # Converts the interval provided in minutes in config to seconds self.markets_refresh_interval: int = exchange_config.get( @@ -191,11 +198,6 @@ class Exchange: def timeframes(self) -> List[str]: return list((self._api.timeframes or {}).keys()) - @property - def ohlcv_candle_limit(self) -> int: - """exchange ohlcv candle limit""" - return int(self._ohlcv_candle_limit) - @property def markets(self) -> Dict: """exchange ccxt markets""" @@ -209,6 +211,17 @@ class Exchange: """exchange ccxt precisionMode""" return self._api.precisionMode + def ohlcv_candle_limit(self, timeframe: str) -> int: + """ + Exchange ohlcv candle limit + Uses ohlcv_candle_limit_per_timeframe if the exchange has different limts + per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + :param timeframe: Timeframe to check + :return: Candle limit as integer + """ + return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( + timeframe, self._ft_has.get('ohlcv_candle_limit'))) + def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None, pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]: """ @@ -421,15 +434,16 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') - def validate_required_startup_candles(self, startup_candles: int) -> None: + def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> None: """ - Checks if required startup_candles is more than ohlcv_candle_limit. + Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ - if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']: + candle_limit = self.ohlcv_candle_limit(timeframe) + if startup_candles + 5 > candle_limit: raise OperationalException( f"This strategy requires {startup_candles} candles to start. " - f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.") + f"{self.name} only provides {candle_limit} for {timeframe}.") def exchange_has(self, endpoint: str) -> bool: """ @@ -714,7 +728,7 @@ class Exchange: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. - Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. + Async over one pair, assuming we get `self.ohlcv_candle_limit()` candles per call. :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from @@ -744,7 +758,7 @@ class Exchange: Download historic ohlcv """ - one_call = timeframe_to_msecs(timeframe) * self._ohlcv_candle_limit + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) logger.debug( "one_call: %s msecs (%s)", one_call, @@ -846,7 +860,7 @@ class Exchange: data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, - limit=self._ohlcv_candle_limit) + limit=self.ohlcv_candle_limit(timeframe)) # Some exchanges sort OHLCV in ASC order and others in DESC. # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) @@ -1019,7 +1033,7 @@ class Exchange: """ Get trade history data using asyncio. Handles all async work and returns the list of candles. - Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. + Async over one pair, assuming we get `self.ohlcv_candle_limit()` candles per call. :param pair: Pair to download :param since: Timestamp in milliseconds to get history from :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a6eb75d5b..d546dd6d2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -179,6 +179,7 @@ class FreqtradeBot(LoggingMixin): # Without this, freqtrade my try to recreate stoploss_on_exchange orders # while selling is in process, since telegram messages arrive in an different thread. with self._sell_lock: + trades = Trade.get_open_trades() # First process current opened trades (positions) self.exit_positions(trades) @@ -1183,6 +1184,7 @@ class FreqtradeBot(LoggingMixin): trade.orders.append(order_obj) trade.open_order_id = order['id'] + trade.sell_order_status = '' trade.close_rate_requested = limit trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 22e14b564..7bbc24056 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -11,10 +11,35 @@ from typing.io import IO import rapidjson +from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN + logger = logging.getLogger(__name__) +def decimals_per_coin(coin: str): + """ + Helper method getting decimal amount for this coin + example usage: f".{decimals_per_coin('USD')}f" + :param coin: Which coin are we printing the price / value for + """ + return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK) + + +def round_coin_value(value: float, coin: str, show_coin_name=True) -> str: + """ + Get price value for this coin + :param value: Value to be printed + :param coin: Which coin are we printing the price / value for + :param show_coin_name: Return string in format: "222.22 USDT" or "222.22" + :return: Formatted / rounded value (with or without coin name) + """ + if show_coin_name: + return f"{value:.{decimals_per_coin(coin)}f} {coin}" + else: + return f"{value:.{decimals_per_coin(coin)}f}" + + def shorten_date(_date: str) -> str: """ Trim the date so it fits on small screens diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d0cdceaeb..eee0f13b3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -546,10 +546,11 @@ class Hyperopt: ) return self._get_results_dict(backtesting_results, min_date, max_date, - params_dict, params_details) + params_dict, params_details, + processed=processed) def _get_results_dict(self, backtesting_results, min_date, max_date, - params_dict, params_details): + params_dict, params_details, processed: Dict[str, DataFrame]): results_metrics = self._calculate_results_metrics(backtesting_results) results_explanation = self._format_results_explanation_string(results_metrics) @@ -563,7 +564,8 @@ class Hyperopt: loss: float = MAX_LOSS if trade_count >= self.config['hyperopt_min_trades']: loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count, - min_date=min_date.datetime, max_date=max_date.datetime) + min_date=min_date.datetime, max_date=max_date.datetime, + config=self.config, processed=processed) return { 'loss': loss, 'params_dict': params_dict, diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index 48407a8a8..b5aa588b2 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -5,6 +5,7 @@ This module defines the interface for the loss-function for hyperopt from abc import ABC, abstractmethod from datetime import datetime +from typing import Dict from pandas import DataFrame @@ -19,7 +20,9 @@ class IHyperOptLoss(ABC): @staticmethod @abstractmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, - min_date: datetime, max_date: datetime, *args, **kwargs) -> float: + min_date: datetime, max_date: datetime, + config: Dict, processed: Dict[str, DataFrame], + *args, **kwargs) -> float: """ Objective function, returns smaller number for better results """ diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 8edfbaf8d..88b2028ba 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -9,8 +9,9 @@ from pandas import DataFrame from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import calculate_market_change, calculate_max_drawdown -from freqtrade.misc import file_dump_json +from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, + calculate_max_drawdown) +from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value logger = logging.getLogger(__name__) @@ -38,11 +39,12 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) -def _get_line_floatfmt() -> List[str]: +def _get_line_floatfmt(stake_currency: str) -> List[str]: """ Generate floatformat (goes in line with _generate_result_line()) """ - return ['s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', 'd', 'd', 'd'] + return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f', + '.2f', 'd', 'd', 'd', 'd'] def _get_line_header(first_column: str, stake_currency: str) -> List[str]: @@ -323,6 +325,13 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'drawdown_end': drawdown_end, 'drawdown_end_ts': drawdown_end.timestamp() * 1000, }) + + csum_min, csum_max = calculate_csum(results) + strat_stats.update({ + 'csum_min': csum_min, + 'csum_max': csum_max + }) + except ValueError: strat_stats.update({ 'max_drawdown': 0.0, @@ -330,6 +339,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'drawdown_start_ts': 0, 'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc), 'drawdown_end_ts': 0, + 'csum_min': 0, + 'csum_max': 0 }) strategy_results = generate_strategy_metrics(all_results=all_results) @@ -352,7 +363,7 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st """ headers = _get_line_header('Pair', stake_currency) - floatfmt = _get_line_floatfmt() + floatfmt = _get_line_floatfmt(stake_currency) output = [[ t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses'] @@ -383,7 +394,9 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren output = [[ t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'], - t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_total_pct'], + t['profit_mean_pct'], t['profit_sum_pct'], + round_coin_value(t['profit_total_abs'], stake_currency, False), + t['profit_total_pct'], ] for t in sell_reason_stats] return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") @@ -396,7 +409,7 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str: :param all_results: Dict of containing results for all strategies :return: pretty printed table with tabulate as string """ - floatfmt = _get_line_floatfmt() + floatfmt = _get_line_floatfmt(stake_currency) headers = _get_line_header('Strategy', stake_currency) output = [[ @@ -436,6 +449,12 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), ('', ''), # Empty line to improve readability + + ('Abs Profit Min', round_coin_value(strat_results['csum_min'], + strat_results['stake_currency'])), + ('Abs Profit Max', round_coin_value(strat_results['csum_max'], + strat_results['stake_currency'])), + ('Max Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"), ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)), ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)), diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 375709423..dff59819c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -171,6 +171,10 @@ class Order(_DECL_BASE): """ Get all non-closed orders - useful when trying to batch-update orders """ + if not isinstance(order, dict): + logger.warning(f"{order} is not a valid response object.") + return + filtered_orders = [o for o in orders if o.order_id == order.get('id')] if filtered_orders: oobj = filtered_orders[0] diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f45ba9b25..4325e537e 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -53,7 +53,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): data_format=config.get('dataformat_ohlcv', 'json'), ) - if startup_candles: + if startup_candles and data: min_date, max_date = get_timerange(data) logger.info(f"Loading data from {min_date} to {max_date}") timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')), @@ -67,14 +67,16 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): if not filename.is_dir() and not filename.is_file(): logger.warning("Backtest file is missing skipping trades.") no_trades = True - - trades = load_trades( - config['trade_source'], - db_url=config.get('db_url'), - exportfilename=filename, - no_trades=no_trades, - strategy=config.get('strategy'), - ) + try: + trades = load_trades( + config['trade_source'], + db_url=config.get('db_url'), + exportfilename=filename, + no_trades=no_trades, + strategy=config.get('strategy'), + ) + except ValueError as e: + raise OperationalException(e) from e trades = trim_dataframe(trades, timerange, 'open_date') return {"ohlcv": data, diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 8c3a5d22f..8a5379ca6 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -30,10 +30,10 @@ class AgeFilter(IPairList): if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") - if self._min_days_listed > exchange.ohlcv_candle_limit: + if self._min_days_listed > exchange.ohlcv_candle_limit('1d'): raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " - f"({exchange.ohlcv_candle_limit})") + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f2e84930b..db51a9c77 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -32,10 +32,10 @@ class RangeStabilityFilter(IPairList): if self._days < 1: raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit: + if self._days > exchange.ohlcv_candle_limit('1d'): raise OperationalException("RangeStabilityFilter requires lookback_days to not " "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit})") + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 92fae54cb..5a9b9ddd0 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -58,13 +58,13 @@ class StoplossGuard(IProtection): SellType.STOPLOSS_ON_EXCHANGE.value) and trade.close_profit < 0)] - if len(trades) > self._trade_limit: - self.log_once(f"Trading stopped due to {self._trade_limit} " - f"stoplosses within {self._lookback_period} minutes.", logger.info) - until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason() + if len(trades) < self._trade_limit: + return False, None, None - return False, None, None + self.log_once(f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + return True, until, self._reason() def global_stop(self, date_now: datetime) -> ProtectionReturn: """ diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 4faefb5fc..050540cc6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -113,7 +113,7 @@ class Daily(BaseModel): class ShowConfig(BaseModel): - dry_run: str + dry_run: bool stake_currency: str stake_amount: Union[float, str] max_open_trades: int diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index a2082103b..3588f2196 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -167,7 +167,7 @@ def reload_config(rpc: RPC = Depends(get_rpc)): @router.get('/pair_candles', response_model=PairHistory, tags=['candle data']) -def pair_candles(pair: str, timeframe: str, limit: Optional[int], rpc=Depends(get_rpc)): +def pair_candles(pair: str, timeframe: str, limit: Optional[int], rpc: RPC = Depends(get_rpc)): return rpc._rpc_analysed_dataframe(pair, timeframe, limit) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index f3eaa1ebc..8a5c958e9 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -2,6 +2,7 @@ import logging from ipaddress import IPv4Address from typing import Any, Dict +import rapidjson import uvicorn from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -14,6 +15,17 @@ from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler logger = logging.getLogger(__name__) +class FTJSONResponse(JSONResponse): + media_type = "application/json" + + def render(self, content: Any) -> bytes: + """ + Use rapidjson for responses + Handles NaN and Inf / -Inf in a javascript way by default. + """ + return rapidjson.dumps(content).encode("utf-8") + + class ApiServer(RPCHandler): _rpc: RPC @@ -32,6 +44,7 @@ class ApiServer(RPCHandler): self.app = FastAPI(title="Freqtrade API", docs_url='/docs' if api_config.get('enable_openapi', False) else None, redoc_url=None, + default_response_class=FTJSONResponse, ) self.configure_app(self.app, self._config) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 27563f73b..7549c38be 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -9,7 +9,7 @@ from math import isnan from typing import Any, Dict, List, Optional, Tuple, Union import arrow -from numpy import NAN, int64, mean +from numpy import NAN, inf, int64, mean from pandas import DataFrame from freqtrade.configuration.timerange import TimeRange @@ -747,6 +747,7 @@ class RPC: sell_mask = (dataframe['sell'] == 1) sell_signals = int(sell_mask.sum()) dataframe.loc[sell_mask, '_sell_signal_open'] = dataframe.loc[sell_mask, 'open'] + dataframe = dataframe.replace([inf, -inf], NAN) dataframe = dataframe.replace({NAN: None}) res = { @@ -775,7 +776,8 @@ class RPC: }) return res - def _rpc_analysed_dataframe(self, pair: str, timeframe: str, limit: int) -> Dict[str, Any]: + def _rpc_analysed_dataframe(self, pair: str, timeframe: str, + limit: Optional[int]) -> Dict[str, Any]: _data, last_analyzed = self._freqtrade.dataprovider.get_analyzed_dataframe( pair, timeframe) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0f7005639..88019601c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -18,6 +18,7 @@ from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ from freqtrade.exceptions import OperationalException +from freqtrade.misc import round_coin_value from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType @@ -189,14 +190,14 @@ class Telegram(RPCHandler): else: msg['stake_amount_fiat'] = 0 - message = ("\N{LARGE BLUE CIRCLE} *{exchange}:* Buying {pair}\n" - "*Amount:* `{amount:.8f}`\n" - "*Open Rate:* `{limit:.8f}`\n" - "*Current Rate:* `{current_rate:.8f}`\n" - "*Total:* `({stake_amount:.6f} {stake_currency}").format(**msg) + message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}\n" + f"*Amount:* `{msg['amount']:.8f}`\n" + f"*Open Rate:* `{msg['limit']:.8f}`\n" + f"*Current Rate:* `{msg['current_rate']:.8f}`\n" + f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}") if msg.get('fiat_currency', None): - message += ", {stake_amount_fiat:.3f} {fiat_currency}".format(**msg) + message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" message += ")`" elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: @@ -365,7 +366,7 @@ class Telegram(RPCHandler): ) stats_tab = tabulate( [[day['date'], - f"{day['abs_profit']:.8f} {stats['stake_currency']}", + f"{round_coin_value(day['abs_profit'], stats['stake_currency'])}", f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}", f"{day['trade_count']} trades"] for day in stats['data']], headers=[ @@ -415,18 +416,18 @@ class Telegram(RPCHandler): # Message to display if stats['closed_trade_count'] > 0: markdown_msg = ("*ROI:* Closed trades\n" - f"∙ `{profit_closed_coin:.8f} {stake_cur} " + f"∙ `{round_coin_value(profit_closed_coin, stake_cur)} " f"({profit_closed_percent_mean:.2f}%) " f"({profit_closed_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{profit_closed_fiat:.3f} {fiat_disp_cur}`\n") + f"∙ `{round_coin_value(profit_closed_fiat, fiat_disp_cur)}`\n") else: markdown_msg = "`No closed trade` \n" markdown_msg += (f"*ROI:* All trades\n" - f"∙ `{profit_all_coin:.8f} {stake_cur} " + f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " f"({profit_all_percent_mean:.2f}%) " f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{profit_all_fiat:.3f} {fiat_disp_cur}`\n" + f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" f"*Total Trade Count:* `{trade_count}`\n" f"*First Trade opened:* `{first_trade_date}`\n" f"*Latest Trade opened:* `{latest_trade_date}\n`" @@ -494,15 +495,17 @@ class Telegram(RPCHandler): "Starting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" ) - for currency in result['currencies']: - if currency['est_stake'] > 0.0001: - curr_output = ("*{currency}:*\n" - "\t`Available: {free: .8f}`\n" - "\t`Balance: {balance: .8f}`\n" - "\t`Pending: {used: .8f}`\n" - "\t`Est. {stake}: {est_stake: .8f}`\n").format(**currency) + for curr in result['currencies']: + if curr['est_stake'] > 0.0001: + curr_output = ( + f"*{curr['currency']}:*\n" + f"\t`Available: {curr['free']:.8f}`\n" + f"\t`Balance: {curr['balance']:.8f}`\n" + f"\t`Pending: {curr['used']:.8f}`\n" + f"\t`Est. {curr['stake']}: " + f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n") else: - curr_output = "*{currency}:* not showing <1$ amount \n".format(**currency) + curr_output = f"*{curr['currency']}:* not showing <1$ amount \n" # Handle overflowing messsage length if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: @@ -512,8 +515,9 @@ class Telegram(RPCHandler): output += curr_output output += ("\n*Estimated Value*:\n" - "\t`{stake}: {total: .8f}`\n" - "\t`{symbol}: {value: .2f}`\n").format(**result) + f"\t`{result['stake']}: {result['total']: .8f}`\n" + f"\t`{result['symbol']}: " + f"{round_coin_value(result['value'], result['symbol'], False)}`\n") self._send_msg(output) except RPCException as e: self._send_msg(str(e)) diff --git a/freqtrade/templates/sample_hyperopt_loss.py b/freqtrade/templates/sample_hyperopt_loss.py index a2b28f948..343349508 100644 --- a/freqtrade/templates/sample_hyperopt_loss.py +++ b/freqtrade/templates/sample_hyperopt_loss.py @@ -1,5 +1,6 @@ from datetime import datetime from math import exp +from typing import Dict from pandas import DataFrame @@ -35,6 +36,7 @@ class SampleHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, + config: Dict, processed: Dict[str, DataFrame], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index c6e64c74e..491afbdd7 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -40,7 +40,7 @@ "# Location of the data\n", "data_location = Path(config['user_data_dir'], 'data', 'binance')\n", "# Pair to analyze - Only use one pair here\n", - "pair = \"BTC_USDT\"" + "pair = \"BTC/USDT\"" ] }, { @@ -54,7 +54,9 @@ "\n", "candles = load_pair_history(datadir=data_location,\n", " timeframe=config[\"timeframe\"],\n", - " pair=pair)\n", + " pair=pair,\n", + " data_format = \"hdf5\",\n", + " )\n", "\n", "# Confirm success\n", "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {data_location}\")\n", diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 104fbf454..8e87a434c 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,9 +2,9 @@ -r requirements.txt # Required for hyperopt -scipy==1.6.0 +scipy==1.6.1 scikit-learn==0.24.1 scikit-optimize==0.8.1 filelock==3.0.12 -joblib==1.0.0 +joblib==1.0.1 progressbar2==3.53.1 diff --git a/requirements.txt b/requirements.txt index 299c07734..51b1ed3d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,12 @@ -numpy==1.20.0 -pandas==1.2.1 +numpy==1.20.1 +pandas==1.2.2 -ccxt==1.41.62 +ccxt==1.42.19 +# Pin cryptography for now due to rust build errors with piwheels +cryptography==3.4.6 aiohttp==3.7.3 -SQLAlchemy==1.3.22 -python-telegram-bot==13.1 +SQLAlchemy==1.3.23 +python-telegram-bot==13.3 arrow==0.17.0 cachetools==4.2.1 requests==2.25.1 @@ -12,14 +14,14 @@ urllib3==1.26.3 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.19 -tabulate==0.8.7 +tabulate==0.8.9 pycoingecko==1.4.0 jinja2==2.11.3 tables==3.6.1 blosc==1.10.2 # find first, C search in arrays -py_find_1st==1.1.4 +py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.0 @@ -29,7 +31,7 @@ sdnotify==0.3.2 # API Server fastapi==0.63.0 -uvicorn==0.13.3 +uvicorn==0.13.4 pyjwt==2.0.1 aiofiles==0.6.0 @@ -37,4 +39,4 @@ aiofiles==0.6.0 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.14 +prompt-toolkit==3.0.16 diff --git a/setup.py b/setup.py index 030980c96..148803cd6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ if readme_file.is_file(): readme_long = (Path(__file__).parent / "README.md").read_text() # Requirements used for submodules -api = ['flask', 'flask-jwt-extended', 'flask-cors'] +api = ['fastapi', 'uvicorn', 'pyjwt', 'aiofiles'] plot = ['plotly>=4.0'] hyperopt = [ 'scipy', diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 96ac6f63c..3c4687745 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -8,11 +8,12 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_MID, BT_DATA_COLUMNS_OLD, - analyze_trade_parallelism, calculate_market_change, - calculate_max_drawdown, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, - get_latest_backtest_filename, get_latest_hyperopt_file, - load_backtest_data, load_trades, load_trades_from_db) + analyze_trade_parallelism, calculate_csum, + calculate_market_change, calculate_max_drawdown, + combine_dataframes_with_mean, create_cum_profit, + extract_trades_of_period, get_latest_backtest_filename, + get_latest_hyperopt_file, load_backtest_data, load_trades, + load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from tests.conftest import create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT @@ -284,6 +285,20 @@ def test_calculate_max_drawdown(testdatadir): drawdown, h, low = calculate_max_drawdown(DataFrame()) +def test_calculate_csum(testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + csum_min, csum_max = calculate_csum(bt_data) + + assert isinstance(csum_min, float) + assert isinstance(csum_max, float) + assert csum_min < 0.01 + assert csum_max > 0.02 + + with pytest.raises(ValueError, match='Trade dataframe empty.'): + csum_min, csum_max = calculate_csum(DataFrame()) + + def test_calculate_max_drawdown2(): values = [0.011580, 0.010048, 0.011340, 0.012161, 0.010416, 0.010009, 0.020024, -0.024662, -0.022350, 0.020496, -0.029859, -0.030511, 0.010041, 0.010872, diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index f25dad35b..c30bce6a4 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -209,7 +209,7 @@ def test_nonexisting_stoploss(mocker, edge_conf): assert edge.stoploss('N/O') == -0.1 -def test_stake_amount(mocker, edge_conf): +def test_edge_stake_amount(mocker, edge_conf): freqtrade = get_patched_freqtradebot(mocker, edge_conf) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( @@ -217,20 +217,33 @@ def test_stake_amount(mocker, edge_conf): 'E/F': PairInfo(-0.02, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - free = 100 - total = 100 - in_trade = 25 - assert edge.stake_amount('E/F', free, total, in_trade) == 31.25 + assert edge._capital_ratio == 0.5 + assert edge.stake_amount('E/F', free_capital=100, total_capital=100, + capital_in_trade=25) == 31.25 - free = 20 - total = 100 - in_trade = 25 - assert edge.stake_amount('E/F', free, total, in_trade) == 20 + assert edge.stake_amount('E/F', free_capital=20, total_capital=100, + capital_in_trade=25) == 20 - free = 0 - total = 100 - in_trade = 25 - assert edge.stake_amount('E/F', free, total, in_trade) == 0 + assert edge.stake_amount('E/F', free_capital=0, total_capital=100, + capital_in_trade=25) == 0 + + # Test with increased allowed_risk + # Result should be no more than allowed capital + edge._allowed_risk = 0.4 + edge._capital_ratio = 0.5 + assert edge.stake_amount('E/F', free_capital=100, total_capital=100, + capital_in_trade=25) == 62.5 + + assert edge.stake_amount('E/F', free_capital=100, total_capital=100, + capital_in_trade=0) == 50 + + edge._capital_ratio = 1 + # Full capital is available + assert edge.stake_amount('E/F', free_capital=100, total_capital=100, + capital_in_trade=0) == 100 + # Full capital is available + assert edge.stake_amount('E/F', free_capital=0, total_capital=100, + capital_in_trade=0) == 0 def test_nonexisting_stake_amount(mocker, edge_conf): diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 8e1d074aa..03cb30d62 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -5,10 +5,12 @@ However, these tests should give a good idea to determine if a new exchange is suitable to run with freqtrade. """ +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_default_conf @@ -122,7 +124,10 @@ class TestCCXTExchange(): assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) # assert len(exchange.klines(pair_tf)) > 200 # Assume 90% uptime ... - assert len(exchange.klines(pair_tf)) > exchange._ohlcv_candle_limit * 0.90 + assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit(timeframe) * 0.90 + # Check if last-timeframe is within the last 2 intervals + now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) + assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) # TODO: tests fetch_trades (?) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index cd24e113e..75db2de26 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1417,7 +1417,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 ret = exchange.get_historic_ohlcv(pair, "5m", int(( arrow.utcnow().int_timestamp - since) * 1000)) @@ -1473,7 +1473,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int(( arrow.utcnow().int_timestamp - since) * 1000)) @@ -2072,9 +2072,9 @@ def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, cap def test_cancel_order(default_conf, mocker, exchange_name): default_conf['dry_run'] = False api_mock = MagicMock() - api_mock.cancel_order = MagicMock(return_value=123) + api_mock.cancel_order = MagicMock(return_value={'id': '123'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 + assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == {'id': '123'} with pytest.raises(InvalidOrderException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) @@ -2091,9 +2091,9 @@ def test_cancel_order(default_conf, mocker, exchange_name): def test_cancel_stoploss_order(default_conf, mocker, exchange_name): default_conf['dry_run'] = False api_mock = MagicMock() - api_mock.cancel_order = MagicMock(return_value=123) + api_mock.cancel_order = MagicMock(return_value={'id': '123'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - assert exchange.cancel_stoploss_order(order_id='_', pair='TKN/BTC') == 123 + assert exchange.cancel_stoploss_order(order_id='_', pair='TKN/BTC') == {'id': '123'} with pytest.raises(InvalidOrderException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) @@ -2418,6 +2418,19 @@ def test_get_markets_error(default_conf, mocker): ex.get_markets('LTC', 'USDT', True, False) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + timeframes = ('1m', '5m', '1h') + expected = exchange._ft_has['ohlcv_candle_limit'] + for timeframe in timeframes: + if 'ohlcv_candle_limit_per_timeframe' in exchange._ft_has: + expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe] + # This should only run for bittrex + assert exchange_name == 'bittrex' + assert exchange.ohlcv_candle_limit(timeframe) == expected + + def test_timeframe_to_minutes(): assert timeframe_to_minutes("5m") == 5 assert timeframe_to_minutes("10m") == 10 @@ -2462,6 +2475,9 @@ def test_timeframe_to_prev_date(): date = datetime.now(tz=timezone.utc) assert timeframe_to_prev_date("5m") < date + # Does not round + time = datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc) + assert timeframe_to_prev_date('5m', time) == time def test_timeframe_to_next_date(): diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5f811e2e5..c8d4338af 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -341,12 +341,14 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') + sbs = mocker.patch('freqtrade.optimize.backtesting.store_backtest_stats') mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) default_conf['timeframe'] = '1m' default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'trades' + default_conf['exportfilename'] = 'export.txt' default_conf['timerange'] = '-1510694220' backtesting = Backtesting(default_conf) @@ -361,6 +363,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: assert log_has(line, caplog) assert backtesting.strategy.dp._pairlists is not None assert backtesting.strategy.bot_loop_start.call_count == 1 + assert sbs.call_count == 1 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index e36900a96..2e42c1be4 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -83,7 +83,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): "method": "StoplossGuard", "lookback_period": 60, "stop_duration": 40, - "trade_limit": 2 + "trade_limit": 3 }] freqtrade = get_patched_freqtradebot(mocker, default_conf) message = r"Trading stopped due to .*" @@ -136,7 +136,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, - "trade_limit": 1, + "trade_limit": 2, "stop_duration": 60, "only_per_pair": only_per_pair }] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 263950d83..d7d69d0ae 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -11,9 +11,11 @@ import uvicorn from fastapi import FastAPI from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient +from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ +from freqtrade.exceptions import ExchangeError from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC @@ -295,7 +297,7 @@ def test_api_run(default_conf, mocker, caplog): "Please make sure that this is intentional!", caplog) assert log_has_re("SECURITY WARNING - `jwt_secret_key` seems to be default.*", caplog) - # Test crashing flask + # Test crashing API server caplog.clear() mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', MagicMock(side_effect=Exception)) @@ -789,6 +791,15 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'exchange': 'bittrex', }] + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) + + rc = client_get(client, f"{BASE_URI}/status") + assert_response(rc) + resp_values = rc.json() + assert len(resp_values) == 1 + assert isnan(resp_values[0]['profit_abs']) + def test_api_version(botclient): ftbot, client = botclient diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 1c34b6b26..f065bb4c5 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -519,7 +519,7 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick assert '*EUR:*' in result assert 'Balance:' in result assert 'Est. BTC:' in result - assert 'BTC: 12.00000000' in result + assert 'BTC: 12.00000000' in result assert '*XRP:* not showing <1$ amount' in result @@ -1205,7 +1205,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None: '*Amount:* `1333.33333333`\n' \ '*Open Rate:* `0.00001099`\n' \ '*Current Rate:* `0.00001099`\n' \ - '*Total:* `(0.001000 BTC, 12.345 USD)`' + '*Total:* `(0.00100000 BTC, 12.345 USD)`' freqtradebot.config['telegram']['notification_settings'] = {'buy': 'off'} caplog.clear() @@ -1389,7 +1389,7 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00001099`\n' '*Current Rate:* `0.00001099`\n' - '*Total:* `(0.001000 BTC)`') + '*Total:* `(0.00100000 BTC)`') def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index bebbc1508..94c3e24f6 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -743,18 +743,18 @@ def test_set_loggers_journald_importerror(mocker, import_fails): logger.handlers = orig_handlers -def test_set_logfile(default_conf, mocker): +def test_set_logfile(default_conf, mocker, tmpdir): patched_configuration_load_config_file(mocker, default_conf) - + f = Path(tmpdir / "test_file.log") + assert not f.is_file() arglist = [ - 'trade', '--logfile', 'test_file.log', + 'trade', '--logfile', str(f), ] args = Arguments(arglist).get_parsed_arg() configuration = Configuration(args) validated_conf = configuration.load_config() - assert validated_conf['logfile'] == "test_file.log" - f = Path("test_file.log") + assert validated_conf['logfile'] == str(f) assert f.is_file() try: f.unlink() diff --git a/tests/test_misc.py b/tests/test_misc.py index 429da135a..e6ba70aee 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -6,9 +6,31 @@ from unittest.mock import MagicMock import pytest -from freqtrade.misc import (file_dump_json, file_load_json, format_ms_time, pair_to_filename, - plural, render_template, render_template_with_fallback, - safe_value_fallback, safe_value_fallback2, shorten_date) +from freqtrade.misc import (decimals_per_coin, file_dump_json, file_load_json, format_ms_time, + pair_to_filename, plural, render_template, + render_template_with_fallback, round_coin_value, safe_value_fallback, + safe_value_fallback2, shorten_date) + + +def test_decimals_per_coin(): + assert decimals_per_coin('USDT') == 3 + assert decimals_per_coin('EUR') == 3 + assert decimals_per_coin('BTC') == 8 + assert decimals_per_coin('ETH') == 5 + + +def test_round_coin_value(): + assert round_coin_value(222.222222, 'USDT') == '222.222 USDT' + assert round_coin_value(222.2, 'USDT') == '222.200 USDT' + assert round_coin_value(222.12745, 'EUR') == '222.127 EUR' + assert round_coin_value(0.1274512123, 'BTC') == '0.12745121 BTC' + assert round_coin_value(0.1274512123, 'ETH') == '0.12745 ETH' + + assert round_coin_value(222.222222, 'USDT', False) == '222.222' + assert round_coin_value(222.2, 'USDT', False) == '222.200' + assert round_coin_value(222.12745, 'EUR', False) == '222.127' + assert round_coin_value(0.1274512123, 'BTC', False) == '0.12745121' + assert round_coin_value(0.1274512123, 'ETH', False) == '0.12745' def test_shorten_date() -> None: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 9921f541b..d0d29f142 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1074,7 +1074,7 @@ def test_get_best_pair(fee): @pytest.mark.usefixtures("init_persistence") -def test_update_order_from_ccxt(): +def test_update_order_from_ccxt(caplog): # Most basic order return (only has orderid) o = Order.parse_from_ccxt_object({'id': '1234'}, 'ETH/BTC', 'buy') assert isinstance(o, Order) @@ -1120,6 +1120,14 @@ def test_update_order_from_ccxt(): with pytest.raises(DependencyException, match=r"Order-id's don't match"): o.update_from_ccxt_object(ccxt_order) + message = "aaaa is not a valid response object." + assert not log_has(message, caplog) + Order.update_orders([o], 'aaaa') + assert log_has(message, caplog) + + # Call regular update - shouldn't fail. + Order.update_orders([o], {'id': '1234'}) + @pytest.mark.usefixtures("init_persistence") def test_select_order(fee):