diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc8906af5..dc3d324a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -118,7 +118,7 @@ jobs: python-version: [3.7] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -175,7 +175,7 @@ jobs: docs_check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Documentation syntax run: | @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-18.04 if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 diff --git a/Dockerfile b/Dockerfile index 923285f39..d986f20ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.1-slim-buster +FROM python:3.8.2-slim-buster RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ diff --git a/README.md b/README.md index 59799da84..88070d45e 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) ([*Note for binance users](#a-note-on-binance)) +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists)) +- [X] [Kraken](https://kraken.com/) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Documentation diff --git a/docs/bot-usage.md b/docs/bot-usage.md index dbc111d44..78e137676 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -275,7 +275,7 @@ Check the corresponding [Data Downloading](data-download.md) section for more de ## Hyperopt commands To optimize your strategy, you can use hyperopt parameter hyperoptimization -to find optimal parameter values for your stategy. +to find optimal parameter values for your strategy. ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] @@ -323,7 +323,7 @@ optional arguments: --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. + --print-json Print best results in JSON format. -j JOBS, --job-workers JOBS The number of concurrently running jobs for hyperoptimization (hyperopt worker processes). If -1 @@ -341,10 +341,11 @@ optional arguments: class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in - Hyperopt-loss-functions are: DefaultHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss, - SharpeHyperOptLossDaily.(default: - `DefaultHyperOptLoss`). + Hyperopt-loss-functions are: + DefaultHyperOptLoss, OnlyProfitHyperOptLoss, + SharpeHyperOptLoss, SharpeHyperOptLossDaily, + SortinoHyperOptLoss, SortinoHyperOptLossDaily. + (default: `DefaultHyperOptLoss`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/configuration.md b/docs/configuration.md index 0b9519688..b3f032bc6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -340,7 +340,7 @@ This is most of the time the default time in force. It means the order will rema on exchange till it is canceled by user. It can be fully or partially fulfilled. If partially fulfilled, the remaining will stay on the exchange till cancelled. -**FOK (Full Or Kill):** +**FOK (Fill Or Kill):** It means if the order is not executed immediately AND fully then it is canceled by the exchange. @@ -370,16 +370,18 @@ The possible values are: `gtc` (default), `fok` or `ioc`. Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the -[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested -with only Bittrex and Binance. - -The bot was tested with the following exchanges: +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). + However, the bot was tested by the development team with only Bittrex, Binance and Kraken, + so the these are the only officially supported exhanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" +- [Kraken](https://kraken.com/): "kraken" Feel free to test other exchanges and submit your PR to improve the bot. +Some exchanges require special configuration, which can be found on the [Exchange-specific Notes](exchanges.md) documentation page. + #### Sample exchange configuration A exchange configuration for "binance" would look as follows: @@ -532,6 +534,12 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis `refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). +`VolumePairList` is based on the ticker data, as reported by the ccxt library: + +* The `bidVolume` is the volume (amount) of current best bid in the orderbook. +* The `askVolume` is the volume (amount) of current best ask in the orderbook. +* The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. + ```json "pairlists": [{ "method": "VolumePairList", @@ -626,6 +634,11 @@ In production mode, the bot will engage your money. Be careful, since a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. +### Setup your exchange account + +You will need to create API Keys (usually you get `key` and `secret`, some exchanges require an additional `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the `freqtrade new-config` command. +API Keys are usually only required for live trading (trading for real money, bot running in "production mode", executing real orders on the exchange) and are not required for the bot running in dry-run (trade simulation) mode. When you setup the bot in dry-run mode, you may fill these fields with empty values. + ### To switch your bot in production mode **Edit your `config.json` file.** @@ -647,9 +660,6 @@ you run it in production mode. } ``` -!!! Note - If you have an exchange API key yet, [see our tutorial](installation.md#setup-your-exchange-account). - You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. ### Using proxy with Freqtrade diff --git a/docs/developer.md b/docs/developer.md index b128ffd2b..ef9232a59 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -234,7 +234,7 @@ git checkout -b new_release Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. -* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7-1` should we need to do a second release that month. +* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part * push that branch to the remote and create a PR against the master branch @@ -268,11 +268,6 @@ Once the PR against master is merged (best right after merging): * Use "master" as reference (this step comes after the above PR is merged). * Use the above changelog as release comment (as codeblock) -### After-release - -* Update version in develop by postfixing that with `-dev` (`2019.6 -> 2019.6-dev`). -* Create a PR against develop to update that branch. - ## Releases ### pypi diff --git a/docs/exchanges.md b/docs/exchanges.md index f615bc61a..70dae0aa5 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -62,6 +62,11 @@ res = [ f"{x['MarketCurrency']}/{x['BaseCurrency']}" for x in ct.publicGetMarket print(res) ``` +## All exchanges + +Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. + + ## Random notes for other exchanges * The Ocean (exchange id: `theocean`) exchange uses Web3 functionality and requires `web3` python package to be installed: diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 401811a1b..9bc5888ce 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -31,9 +31,9 @@ This will create a new hyperopt file from a template, which will be located unde Depending on the space you want to optimize, only some of the below are required: * fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimzation +* fill `indicator_space` - for buy signal optimization * fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimzation +* fill `sell_indicator_space` - for sell signal optimization !!! Note `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. @@ -81,11 +81,11 @@ There are two places you need to change in your hyperopt file to add a new buy h There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. -2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". +2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower Bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like -"*buy exactly when close price touches lower bollinger band, BUT only if +"*buy exactly when close price touches lower Bollinger band, BUT only if ADX > 10*". If you have updated the buy strategy, i.e. changed the contents of @@ -172,7 +172,7 @@ So let's write the buy strategy using these values: Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) with different value combinations. It will then use the given historical data and make buys based on the buy signals generated with the above function and based on the results -it will end with telling you which paramter combination produced the best profits. +it will end with telling you which parameter combination produced the best profits. The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. When you want to test an indicator that isn't used by the bot currently, remember to @@ -191,8 +191,10 @@ Currently, the following loss functions are builtin: * `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on the trade returns) -* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on daily trade returns) +* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) +* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) +* `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) +* `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. @@ -272,7 +274,7 @@ In some situations, you may need to run Hyperopt (and Backtesting) with the By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one open trade is allowed for every traded pair. The total number of trades open for all pairs is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to -some potential trades to be hidden (or masked) by previosly open trades. +some potential trades to be hidden (or masked) by previously open trades. The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` diff --git a/docs/installation.md b/docs/installation.md index 054cafe9b..88e2ef6eb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,6 +2,8 @@ This page explains how to prepare your environment for running the bot. +Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade evaluating how it operates. + ## Prerequisite ### Requirements @@ -14,15 +16,7 @@ Click each one for install guide: * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) * [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) (install instructions below) -### API keys - -Before running your bot in production you will need to setup few -external API. In production mode, the bot will require valid Exchange API -credentials. We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended). - -### Setup your exchange account - -You will need to create API Keys (Usually you get `key` and `secret`) from the Exchange website and insert this into the appropriate fields in the configuration or when asked by the installation script. + We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended. ## Quick start @@ -65,11 +59,11 @@ usage: ** --install ** -With this option, the script will install everything you need to run the bot: +With this option, the script will install the bot and most dependencies: +You will need to have git and python3.6+ installed beforehand for this to work. * Mandatory software as: `ta-lib` -* Setup your virtualenv -* Configure your `config.json` file +* Setup your virtualenv under `.env/` This option is a combination of installation tasks, `--reset` and `--config`. @@ -83,7 +77,7 @@ This option will hard reset your branch (only if you are on either `master` or ` ** --config ** -Use this option to configure the `config.json` configuration file. The script will interactively ask you questions to setup your bot and create your `config.json`. +DEPRECATED - use `freqtrade new-config -c config.json` instead. ------ diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 07833da34..4aacd3af6 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -249,6 +249,23 @@ minimal_roi = { While technically not completely disabled, this would sell once the trade reaches 10000% Profit. +To use times based on candle duration (ticker_interval or timeframe), the following snippet can be handy. +This will allow you to change the ticket_interval for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...) + +``` python +from freqtrade.exchange import timeframe_to_minutes + +class AwesomeStrategy(IStrategy): + + ticker_interval = "1d" + ticker_interval_mins = timeframe_to_minutes(ticker_interval) + minimal_roi = { + "0": 0.05, # 5% for the first 3 candles + str(ticker_interval_mins * 3)): 0.02, # 2% after 3 candles + str(ticker_interval_mins * 6)): 0.01, # 1% After 6 candles + } +``` + ### Stoploss Setting a stoploss is highly recommended to protect your capital from strong moves against you. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 53b35ca09..d26d684ce 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -121,7 +121,6 @@ from freqtrade.data.btanalysis import analyze_trade_parallelism # Analyze the above parallel_trades = analyze_trade_parallelism(trades, '5m') - parallel_trades.plot() ``` @@ -134,11 +133,14 @@ Freqtrade offers interactive plotting capabilities based on plotly. from freqtrade.plot.plotting import generate_candlestick_graph # Limit graph period to keep plotly quick and reactive +# Filter trades to one pair +trades_red = trades.loc[trades['pair'] == pair] + data_red = data['2019-06-01':'2019-06-10'] # Generate candlestick graph graph = generate_candlestick_graph(pair=pair, data=data_red, - trades=trades, + trades=trades_red, indicators1=['sma20', 'ema50', 'ema55'], indicators2=['rsi', 'macd', 'macdsignal', 'macdhist'] ) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index bccd387b9..acb629cf9 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -324,7 +324,8 @@ AVAILABLE_CLI_OPTIONS = { help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' - 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily.' + 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, ' + 'SortinoHyperOptLoss, SortinoHyperOptLossDaily.' '(default: `%(default)s`).', metavar='NAME', default=constants.DEFAULT_HYPEROPT_LOSS, diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 8c1c80d98..e7f89a375 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -46,18 +46,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: trials = _hyperopt_filter_trials(trials, filteroptions) - # TODO: fetch the interval for epochs to print from the cli option - epoch_start, epoch_stop = 0, None - if print_colorized: colorama_init(autoreset=True) try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, - not filteroptions['only_best'], print_colorized) - + Hyperopt.print_result_table(config, trials, total_epochs, + not filteroptions['only_best'], print_colorized) except KeyboardInterrupt: print('User interrupted..') @@ -75,6 +69,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + print_json = config.get('print_json', False) + no_header = config.get('hyperopt_show_no_header', False) + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + n = config.get('hyperopt_show_index', -1) + filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), @@ -87,10 +87,6 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) } - no_header = config.get('hyperopt_show_no_header', False) - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') # Previous evaluations trials = Hyperopt.load_previous_results(trials_file) @@ -99,20 +95,17 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: trials = _hyperopt_filter_trials(trials, filteroptions) trials_epochs = len(trials) - n = config.get('hyperopt_show_index', -1) if n > trials_epochs: raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") + f"The index of the epoch to show should be less than {trials_epochs + 1}.") if n < -trials_epochs: raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") # Translate epoch index from human-readable format to pythonic if n > 0: n -= 1 - print_json = config.get('print_json', False) - if trials: val = trials[n] Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, @@ -129,52 +122,52 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: trials = [x for x in trials if x['results_metrics']['profit'] > 0] if filteroptions['filter_min_trades'] > 0: trials = [ - x for x in trials - if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] - ] + x for x in trials + if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] + ] if filteroptions['filter_max_trades'] > 0: trials = [ - x for x in trials - if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] - ] + x for x in trials + if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] + ] if filteroptions['filter_min_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] - ] + x for x in trials + if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + ] if filteroptions['filter_max_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] - ] + x for x in trials + if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + ] if filteroptions['filter_min_avg_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - > filteroptions['filter_min_avg_profit'] - ] + x for x in trials + if x['results_metrics']['avg_profit'] + > filteroptions['filter_min_avg_profit'] + ] if filteroptions['filter_max_avg_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - < filteroptions['filter_max_avg_profit'] - ] + x for x in trials + if x['results_metrics']['avg_profit'] + < filteroptions['filter_max_avg_profit'] + ] if filteroptions['filter_min_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] - ] + x for x in trials + if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + ] if filteroptions['filter_max_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] - ] + x for x in trials + if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] + ] logger.info(f"{len(trials)} " + ("best " if filteroptions['only_best'] else "") + diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 49674b81a..327901dc0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -58,7 +58,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: else yellow + "DUPLICATE NAME" + reset) } for s in objs] - print(tabulate(objss_to_print, headers='keys', tablefmt='pipe')) + print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right')) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -192,7 +192,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: else: # print data as a table, with the human-readable summary print(f"{summary_str}:") - print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) + print(tabulate(tabular_data, headers='keys', tablefmt='psql', stralign='right')) elif not (args.get('print_one_column', False) or args.get('list_pairs_print_json', False) or args.get('print_csv', False)): diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 5183ad0b4..5ba7ff294 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -150,15 +150,3 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if (pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") - - if pl.get('method') == 'StaticPairList': - stake = conf['stake_currency'] - invalid_pairs = [] - for pair in conf['exchange'].get('pair_whitelist'): - if not pair.endswith(f'/{stake}'): - invalid_pairs.append(pair) - - if invalid_pairs: - raise OperationalException( - f"Stake-currency '{stake}' not compatible with pair-whitelist. " - f"Please remove the following pairs: {invalid_pairs}") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 810840e49..4f2db4065 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -94,6 +94,8 @@ class Configuration: # Keep a copy of the original configuration file config['original_config'] = deepcopy(config) + self._process_logging_options(config) + self._process_runmode(config) self._process_common_options(config) @@ -144,8 +146,6 @@ class Configuration: def _process_common_options(self, config: Dict[str, Any]) -> None: - self._process_logging_options(config) - # Set strategy if not specified in config and or if it's non default if self.args.get("strategy") or not config.get('strategy'): config.update({'strategy': self.args.get("strategy")}) @@ -166,11 +166,6 @@ class Configuration: if 'sd_notify' in self.args and self.args["sd_notify"]: config['internals'].update({'sd_notify': True}) - self._args_to_config(config, - argname='dry_run', - logstring='Parameter --dry-run detected, ' - 'overriding dry_run to: {} ...') - def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ Extract information for sys.argv and load directory configurations @@ -412,10 +407,15 @@ class Configuration: def _process_runmode(self, config: Dict[str, Any]) -> None: + self._args_to_config(config, + argname='dry_run', + logstring='Parameter --dry-run detected, ' + 'overriding dry_run to: {} ...') + if not self.runmode: # Handle real mode, infer dry/live from config self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE - logger.info(f"Runmode set to {self.runmode}.") + logger.info(f"Runmode set to {self.runmode.value}.") config.update({'runmode': self.runmode}) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3672de5bd..7def3a054 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -422,7 +422,6 @@ CONF_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True }, @@ -430,7 +429,6 @@ CONF_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True }, diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 17b9fd7d7..2b738a94a 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -71,6 +71,8 @@ class JsonDataHandler(IDataHandler): return DataFrame(columns=self._columns) pairdata = read_json(filename, orient='values') pairdata.columns = self._columns + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata['date'] = to_datetime(pairdata['date'], unit='ms', utc=True, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b3b347016..522b4e40e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -66,8 +66,6 @@ class Exchange: self._config.update(config) - self._cached_ticker: Dict[str, Any] = {} - # Holds last candle refreshed time of each pair self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} # Timestamp of last markets refresh @@ -228,6 +226,18 @@ class Exchange: markets = self.markets return sorted(set([x['quote'] for _, x in markets.items()])) + def get_pair_quote_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets.get(pair, {}).get('quote', '') + + def get_pair_base_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets.get(pair, {}).get('base', '') + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] @@ -300,7 +310,7 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return - + invalid_pairs = [] for pair in pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format @@ -322,6 +332,13 @@ class Exchange: logger.warning(f"Pair {pair} is restricted for some users on this exchange." f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") + if (self._config['stake_currency'] and + self.get_pair_quote_currency(pair) != self._config['stake_currency']): + invalid_pairs.append(pair) + if invalid_pairs: + raise OperationalException( + f"Stake-currency '{self._config['stake_currency']}' not compatible with " + f"pair-whitelist. Please remove the following pairs: {invalid_pairs}") def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str: """ @@ -591,28 +608,17 @@ class Exchange: raise OperationalException(e) from e @retrier - def fetch_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: - if refresh or pair not in self._cached_ticker.keys(): - try: - if pair not in self._api.markets or not self._api.markets[pair].get('active'): - raise DependencyException(f"Pair {pair} not available") - data = self._api.fetch_ticker(pair) - try: - self._cached_ticker[pair] = { - 'bid': float(data['bid']), - 'ask': float(data['ask']), - } - except KeyError: - logger.debug("Could not cache ticker data for %s", pair) - return data - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - else: - logger.info("returning cached ticker-data for %s", pair) - return self._cached_ticker[pair] + def fetch_ticker(self, pair: str) -> dict: + try: + if pair not in self._api.markets or not self._api.markets[pair].get('active'): + raise DependencyException(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: @@ -1018,7 +1024,7 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non def is_exchange_officially_supported(exchange_name: str) -> bool: - return exchange_name in ['bittrex', 'binance'] + return exchange_name in ['bittrex', 'binance', 'kraken'] def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 127586437..dffec940c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,11 +6,11 @@ import logging import traceback from datetime import datetime from math import isclose -from os import getpid from threading import Lock from typing import Any, Dict, List, Optional, Tuple import arrow +from cachetools import TTLCache from requests.exceptions import RequestException from freqtrade import __version__, constants, persistence @@ -52,9 +52,8 @@ class FreqtradeBot: # Init objects self.config = config - self._heartbeat_msg = 0 - - self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) + self._sell_rate_cache = TTLCache(maxsize=100, ttl=5) + self._buy_rate_cache = TTLCache(maxsize=100, ttl=5) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -159,11 +158,6 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - if (self.heartbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): - logger.info(f"Bot heartbeat. PID={getpid()}") - self._heartbeat_msg = arrow.utcnow().timestamp - def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: """ Refresh whitelist from pairlist or edge and extend it with trades. @@ -234,11 +228,20 @@ class FreqtradeBot: return trades_created - def get_buy_rate(self, pair: str, refresh: bool, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, refresh: bool) -> float: """ Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data :return: float: Price """ + if not refresh: + rate = self._buy_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.info(f"Using cached buy rate for {pair}.") + return rate + config_bid_strategy = self.config.get('bid_strategy', {}) if 'use_order_book' in config_bid_strategy and\ config_bid_strategy.get('use_order_book', False): @@ -251,11 +254,8 @@ class FreqtradeBot: logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) used_rate = order_book_rate else: - if not tick: - logger.info('Using Last Ask / Last Price') - ticker = self.exchange.fetch_ticker(pair, refresh) - else: - ticker = tick + logger.info('Using Last Ask / Last Price') + ticker = self.exchange.fetch_ticker(pair) if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: @@ -263,6 +263,8 @@ class FreqtradeBot: ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) used_rate = ticker_rate + self._buy_rate_cache[pair] = used_rate + return used_rate def get_trade_stake_amount(self, pair: str) -> float: @@ -566,7 +568,7 @@ class FreqtradeBot: """ Sends rpc notification when a buy cancel occured. """ - current_rate = self.get_buy_rate(trade.pair, True) + current_rate = self.get_buy_rate(trade.pair, False) msg = { 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, @@ -621,8 +623,17 @@ class FreqtradeBot: The orderbook portion is only used for rpc messaging, which would otherwise fail for BitMex (has no bid/ask in fetch_ticker) or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data :return: Bid rate """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.info(f"Using cached sell rate for {pair}.") + return rate + config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): logger.debug('Using order book to get sell rate') @@ -631,7 +642,8 @@ class FreqtradeBot: rate = order_book['bids'][0][0] else: - rate = self.exchange.fetch_ticker(pair, refresh)['bid'] + rate = self.exchange.fetch_ticker(pair)['bid'] + self._sell_rate_cache[pair] = rate return rate def handle_trade(self, trade: Trade) -> bool: @@ -948,8 +960,8 @@ class FreqtradeBot: """ # Update wallets to ensure amounts tied up in a stoploss is now free! self.wallets.update() - - wallet_amount = self.wallets.get_free(pair.split('/')[0]) + trade_base_currency = self.exchange.get_pair_base_currency(pair) + wallet_amount = self.wallets.get_free(trade_base_currency) logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") if wallet_amount >= amount: return amount @@ -1058,7 +1070,7 @@ class FreqtradeBot: """ profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, True) + current_rate = self.get_sell_rate(trade.pair, False) profit_percent = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_percent > 0 else "loss" @@ -1135,12 +1147,13 @@ class FreqtradeBot: if trade.fee_open == 0 or order['status'] == 'open': return order_amount + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible if ('fee' in order and order['fee'] is not None and (order['fee'].keys() >= {'currency', 'cost'})): if (order['fee']['currency'] is not None and order['fee']['cost'] is not None and - trade.pair.startswith(order['fee']['currency'])): + trade_base_currency == order['fee']['currency']): new_amount = order_amount - order['fee']['cost'] logger.info("Applying fee on amount for %s (from %s to %s) from Order", trade, order['amount'], new_amount) @@ -1162,7 +1175,7 @@ class FreqtradeBot: # only applies if fee is in quote currency! if (exectrade['fee']['currency'] is not None and exectrade['fee']['cost'] is not None and - trade.pair.startswith(exectrade['fee']['currency'])): + trade_base_currency == exectrade['fee']['currency']): fee_abs += exectrade['fee']['cost'] if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c18aefc76..94441ce24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -423,28 +423,37 @@ class Backtesting: strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") - print(' BACKTESTING REPORT '.center(133, '=')) - print(generate_text_table(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results)) + table = generate_text_table(data, stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) - print(' SELL REASON STATS '.center(133, '=')) - print(generate_text_table_sell_reason(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results)) + table = generate_text_table_sell_reason(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) - print(' LEFT OPEN TRADES REPORT '.center(133, '=')) - print(generate_text_table(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results.loc[results.open_at_end], skip_nan=True)) + table = generate_text_table(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results.loc[results.open_at_end], skip_nan=True) + if isinstance(table, str): + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + if isinstance(table, str): + print('=' * len(table.splitlines()[0])) print() if len(all_results) > 1: # Print Strategy summary table - print(' STRATEGY SUMMARY '.center(133, '=')) - print(generate_text_table_strategy(self.config['stake_currency'], - self.config['max_open_trades'], - all_results=all_results)) + table = generate_text_table_strategy(self.config['stake_currency'], + self.config['max_open_trades'], + all_results=all_results) + print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) + print(table) + print('=' * len(table.splitlines()[0])) print('\nFor more details, please look at the detail tables above') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 354c8ab58..7385cb7ce 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ import sys import warnings from collections import OrderedDict, deque from math import factorial, log, inf +from numpy import iinfo, int32 from operator import itemgetter from pathlib import Path from pprint import pprint @@ -19,6 +20,12 @@ from typing import Any, Dict, List, Optional import rapidjson from colorama import Fore, Style from colorama import init as colorama_init +from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) +from joblib import parallel_backend +from multiprocessing import Manager +from queue import Queue +from pandas import DataFrame, json_normalize, isna +from tabulate import tabulate from freqtrade.data.converter import trim_dataframe from freqtrade.data.history import get_timerange @@ -30,13 +37,6 @@ import freqtrade.optimize.hyperopt_backend as backend from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, HyperOptResolver) -from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) -from joblib import parallel_backend -from multiprocessing import Manager -from queue import Queue - -from pandas import DataFrame -from numpy import iinfo, int32 # Suppress scikit-learn FutureWarnings from skopt with warnings.catch_warnings(): @@ -364,6 +364,61 @@ class Hyperopt: f"{results['current_epoch']:5d}/{total_epochs}: " + f"{results['results_explanation']} " + f"Objective: {results['loss']:.5f}") + @staticmethod + def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + print_colorized: bool) -> None: + """ + Log result table + """ + if not results: + return + + trials = json_normalize(results, max_level=1) + trials['Best'] = '' + trials = trials[[ + 'Best', 'current_epoch', 'results_metrics.trade_count', 'results_metrics.avg_profit', + 'results_metrics.total_profit', 'results_metrics.profit', 'results_metrics.duration', + 'loss', 'is_initial_point', 'is_best' + ]] + trials.columns = [ + 'Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Profit', 'Avg duration', + 'Objective', 'is_initial_point', 'is_best' + ] + trials['is_profit'] = False + trials.loc[trials['is_initial_point'], 'Best'] = '*' + trials.loc[trials['is_best'], 'Best'] = 'Best' + trials['Objective'] = trials['Objective'].astype(str) + trials.loc[trials['Total profit'] > 0, 'is_profit'] = True + trials['Trades'] = trials['Trades'].astype(str) + + trials['Epoch'] = trials['Epoch'].apply(lambda x: "{}/{}".format(x, total_epochs)) + trials['Avg profit'] = trials['Avg profit'].apply(lambda x: '{:,.2f}%'.format(x) + if not isna(x) else x) + trials['Profit'] = trials['Profit'].apply(lambda x: '{:,.2f}%'.format(x) + if not isna(x) else x) + trials['Total profit'] = trials['Total profit'].apply( + lambda x: '{: 11.8f} '.format(x) + config['stake_currency'] if not isna(x) else x) + trials['Avg duration'] = trials['Avg duration'].apply(lambda x: '{:,.1f}m'.format(x) + if not isna(x) else x) + if print_colorized: + for i in range(len(trials)): + if trials.loc[i]['is_profit']: + for z in range(len(trials.loc[i]) - 3): + trials.iat[i, z] = "{}{}{}".format(Fore.GREEN, str(trials.loc[i][z]), + Fore.RESET) + if trials.loc[i]['is_best'] and highlight_best: + for z in range(len(trials.loc[i]) - 3): + trials.iat[i, z] = "{}{}{}".format(Style.BRIGHT, str(trials.loc[i][z]), + Style.RESET_ALL) + + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + + print( + tabulate(trials.to_dict(orient='list'), + headers='keys', + tablefmt='psql', + stralign="right")) + def has_space(self, space: str) -> bool: """ Tell if the space value is contained in the configuration diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py new file mode 100644 index 000000000..83f644a43 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -0,0 +1,49 @@ +""" +SortinoHyperOptLoss + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +from datetime import datetime + +from pandas import DataFrame +import numpy as np + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLoss(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + """ + total_profit = results["profit_percent"] + days_period = (max_date - min_date).days + + # adding slippage of 0.1% per trade + total_profit = total_profit - 0.0005 + expected_returns_mean = total_profit.sum() / days_period + + results['downside_returns'] = 0 + results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent'] + down_stdev = np.std(results['downside_returns']) + + if np.std(total_profit) != 0.0: + sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py new file mode 100644 index 000000000..16dc26142 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -0,0 +1,70 @@ +""" +SortinoHyperOptLossDaily + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +import math +from datetime import datetime + +from pandas import DataFrame, date_range + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLossDaily(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + + Sortino Ratio calculated as described in + http://www.redrockcapital.com/Sortino__A__Sharper__Ratio_Red_Rock_Capital.pdf + """ + resample_freq = '1D' + slippage_per_trade_ratio = 0.0005 + days_in_year = 365 + minimum_acceptable_return = 0.0 + + # apply slippage per trade to profit_percent + results.loc[:, 'profit_percent_after_slippage'] = \ + results['profit_percent'] - slippage_per_trade_ratio + + # create the index within the min_date and end max_date + t_index = date_range(start=min_date, end=max_date, freq=resample_freq, + normalize=True) + + sum_daily = ( + results.resample(resample_freq, on='close_time').agg( + {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) + ) + + total_profit = sum_daily["profit_percent_after_slippage"] - minimum_acceptable_return + expected_returns_mean = total_profit.mean() + + sum_daily['downside_returns'] = 0 + sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit + total_downside = sum_daily['downside_returns'] + # Here total_downside contains min(0, P - MAR) values, + # where P = sum_daily["profit_percent_after_slippage"] + down_stdev = math.sqrt((total_downside**2).sum() / len(total_downside)) + + if (down_stdev != 0.): + sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(t_index, sum_daily, total_profit) + # print(minimum_acceptable_return, expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index b00adbd48..39bde50a8 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -66,7 +66,7 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra ]) # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore def generate_text_table_sell_reason( @@ -112,7 +112,7 @@ def generate_text_table_sell_reason( profit_percent_tot, ] ) - return tabulate(tabular_data, headers=headers, tablefmt="pipe") + return tabulate(tabular_data, headers=headers, tablefmt="orgtbl", stralign="right") def generate_text_table_strategy(stake_currency: str, max_open_trades: str, @@ -146,7 +146,7 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str, ]) # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore def generate_edge_table(results: dict) -> str: @@ -172,4 +172,4 @@ def generate_edge_table(results: dict) -> str: # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 1ad4da523..d45a329dd 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -99,7 +99,8 @@ class IPairList(ABC): logger.warning(f"Pair {pair} is not compatible with exchange " f"{self._exchange.name}. Removing it from whitelist..") continue - if not pair.endswith(self._config['stake_currency']): + + if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']: logger.warning(f"Pair {pair} is not compatible with your stake currency " f"{self._config['stake_currency']}. Removing it from whitelist..") continue diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index e50dafb63..d067d5e8a 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -91,9 +91,9 @@ class VolumePairList(IPairList): if self._pairlist_pos == 0: # If VolumePairList is the first in the list, use fresh pairlist - # check length so that we make sure that '/' is actually in the string + # Check if pair quote currency equals to the stake currency. filtered_tickers = [v for k, v in tickers.items() - if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency + if (self._exchange.get_pair_quote_currency(k) == base_currency and v[key] is not None)] else: # If other pairlist is in front, use the incomming pairlist. diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3411318bb..1e4eaa3e0 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -460,9 +460,9 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - # Check pair is in stake currency + # Check if pair quote currency equals to the stake currency. stake_currency = self._freqtrade.config.get('stake_currency') - if not pair.endswith(stake_currency): + if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: raise RPCException( f'Wrong pair selected. Please pairs with stake {stake_currency} pairs only') # check if valid pair @@ -517,7 +517,7 @@ class RPC: if add: stake_currency = self._freqtrade.config.get('stake_currency') for pair in add: - if (pair.endswith(stake_currency) + if (self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency and pair not in self._freqtrade.pairlists.blacklist): self._freqtrade.pairlists.blacklist.append(pair) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 92f6aefba..17372e1e0 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -124,24 +124,70 @@ class SampleStrategy(IStrategy): # Momentum Indicators # ------------------------------------ - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # ADX dataframe['adx'] = ta.ADX(dataframe) + # # Plus Directional Indicator / Movement + # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + # dataframe['plus_di'] = ta.PLUS_DI(dataframe) + + # # Minus Directional Indicator / Movement + # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + # dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) - # # Awesome oscillator + # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + # # Keltner Channel + # keltner = qtpylib.keltner_channel(dataframe) + # dataframe["kc_upperband"] = keltner["upper"] + # dataframe["kc_lowerband"] = keltner["lower"] + # dataframe["kc_middleband"] = keltner["mid"] + # dataframe["kc_percent"] = ( + # (dataframe["close"] - dataframe["kc_lowerband"]) / + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) + # ) + # dataframe["kc_width"] = ( + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] + # ) + + # # Ultimate Oscillator + # dataframe['uo'] = ta.ULTOSC(dataframe) + + # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) + # rsi = 0.1 * (dataframe['rsi'] - 50) + # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + + # # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) + # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # # Stochastic Slow + # stoch = ta.STOCH(dataframe) + # dataframe['slowd'] = stoch['slowd'] + # dataframe['slowk'] = stoch['slowk'] + + # Stochastic Fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # # Stochastic RSI + # stoch_rsi = ta.STOCHRSI(dataframe) + # dataframe['fastd_rsi'] = stoch_rsi['fastd'] + # dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -151,60 +197,58 @@ class SampleStrategy(IStrategy): # MFI dataframe['mfi'] = ta.MFI(dataframe) - # # Minus Directional Indicator / Movement - # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # # Plus Directional Indicator / Movement - # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - # dataframe['plus_di'] = ta.PLUS_DI(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) - # # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - # rsi = 0.1 * (dataframe['rsi'] - 50) - # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - - # # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # # Stoch - # stoch = ta.STOCH(dataframe) - # dataframe['slowd'] = stoch['slowd'] - # dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # # Stoch RSI - # stoch_rsi = ta.STOCHRSI(dataframe) - # dataframe['fastd_rsi'] = stoch_rsi['fastd'] - # dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ - # Bollinger bands + # Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] + dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) + ) + dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] + ) + + # Bollinger Bands - Weighted (EMA based instead of SMA) + # weighted_bollinger = qtpylib.weighted_bollinger_bands( + # qtpylib.typical_price(dataframe), window=20, stds=2 + # ) + # dataframe["wbb_upperband"] = weighted_bollinger["upper"] + # dataframe["wbb_lowerband"] = weighted_bollinger["lower"] + # dataframe["wbb_middleband"] = weighted_bollinger["mid"] + # dataframe["wbb_percent"] = ( + # (dataframe["close"] - dataframe["wbb_lowerband"]) / + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) + # ) + # dataframe["wbb_width"] = ( + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / + # dataframe["wbb_middleband"] + # ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + # dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average - # dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) + # dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) + # dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) + # dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) + # dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) + # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) - # SAR Parabol + # Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average @@ -264,7 +308,7 @@ class SampleStrategy(IStrategy): # # Chart type # # ------------------------------------ - # # Heikinashi stategy + # # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 399235cfe..dffa308ce 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -190,7 +190,6 @@ "# Analyze the above\n", "parallel_trades = analyze_trade_parallelism(trades, '5m')\n", "\n", - "\n", "parallel_trades.plot()" ] }, @@ -212,11 +211,14 @@ "from freqtrade.plot.plotting import generate_candlestick_graph\n", "# Limit graph period to keep plotly quick and reactive\n", "\n", + "# Filter trades to one pair\n", + "trades_red = trades.loc[trades['pair'] == pair]\n", + "\n", "data_red = data['2019-06-01':'2019-06-10']\n", "# Generate candlestick graph\n", "graph = generate_candlestick_graph(pair=pair,\n", " data=data_red,\n", - " trades=trades,\n", + " trades=trades_red,\n", " indicators1=['sma20', 'ema50', 'ema55'],\n", " indicators2=['rsi', 'macd', 'macdsignal', 'macdhist']\n", " )\n", diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 879a2daa0..60a358bec 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -2,24 +2,70 @@ # Momentum Indicators # ------------------------------------ -# RSI -dataframe['rsi'] = ta.RSI(dataframe) - # ADX dataframe['adx'] = ta.ADX(dataframe) +# # Plus Directional Indicator / Movement +# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) +# dataframe['plus_di'] = ta.PLUS_DI(dataframe) + +# # Minus Directional Indicator / Movement +# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) +# dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) -# # Awesome oscillator +# # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) -# # Commodity Channel Index: values Oversold:<-100, Overbought:>100 +# # Keltner Channel +# keltner = qtpylib.keltner_channel(dataframe) +# dataframe["kc_upperband"] = keltner["upper"] +# dataframe["kc_lowerband"] = keltner["lower"] +# dataframe["kc_middleband"] = keltner["mid"] +# dataframe["kc_percent"] = ( +# (dataframe["close"] - dataframe["kc_lowerband"]) / +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) +# ) +# dataframe["kc_width"] = ( +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] +# ) + +# # Ultimate Oscillator +# dataframe['uo'] = ta.ULTOSC(dataframe) + +# # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) +# RSI +dataframe['rsi'] = ta.RSI(dataframe) + +# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) +# rsi = 0.1 * (dataframe['rsi'] - 50) +# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + +# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) +# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + +# # Stochastic Slow +# stoch = ta.STOCH(dataframe) +# dataframe['slowd'] = stoch['slowd'] +# dataframe['slowk'] = stoch['slowk'] + +# Stochastic Fast +stoch_fast = ta.STOCHF(dataframe) +dataframe['fastd'] = stoch_fast['fastd'] +dataframe['fastk'] = stoch_fast['fastk'] + +# # Stochastic RSI +# stoch_rsi = ta.STOCHRSI(dataframe) +# dataframe['fastd_rsi'] = stoch_rsi['fastd'] +# dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -29,60 +75,57 @@ dataframe['macdhist'] = macd['macdhist'] # MFI dataframe['mfi'] = ta.MFI(dataframe) -# # Minus Directional Indicator / Movement -# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - -# # Plus Directional Indicator / Movement -# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) -# dataframe['plus_di'] = ta.PLUS_DI(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) -# # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) -# rsi = 0.1 * (dataframe['rsi'] - 50) -# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - -# # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) -# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - -# # Stoch -# stoch = ta.STOCH(dataframe) -# dataframe['slowd'] = stoch['slowd'] -# dataframe['slowk'] = stoch['slowk'] - -# Stoch fast -stoch_fast = ta.STOCHF(dataframe) -dataframe['fastd'] = stoch_fast['fastd'] -dataframe['fastk'] = stoch_fast['fastk'] - -# # Stoch RSI -# stoch_rsi = ta.STOCHRSI(dataframe) -# dataframe['fastd_rsi'] = stoch_rsi['fastd'] -# dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ -# Bollinger bands +# Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] +dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) +) +dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] +) + +# Bollinger Bands - Weighted (EMA based instead of SMA) +# weighted_bollinger = qtpylib.weighted_bollinger_bands( +# qtpylib.typical_price(dataframe), window=20, stds=2 +# ) +# dataframe["wbb_upperband"] = weighted_bollinger["upper"] +# dataframe["wbb_lowerband"] = weighted_bollinger["lower"] +# dataframe["wbb_middleband"] = weighted_bollinger["mid"] +# dataframe["wbb_percent"] = ( +# (dataframe["close"] - dataframe["wbb_lowerband"]) / +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) +# ) +# dataframe["wbb_width"] = ( +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"] +# ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) +# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average -# dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) +# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) +# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) +# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) +# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) +# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) +# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) -# SAR Parabol +# Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average @@ -142,7 +185,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] # # Chart type # # ------------------------------------ -# # Heikinashi stategy +# # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index dd5e34fe6..b913155bc 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -74,7 +74,7 @@ class Wallets: ) for trade in open_trades: - curr = trade.pair.split('/')[0] + curr = self._exchange.get_pair_base_currency(trade.pair) _wallets[curr] = Wallet( curr, trade.amount, diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 64cc97026..4c28ecaeb 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -4,6 +4,7 @@ Main Freqtrade worker class. import logging import time import traceback +from os import getpid from typing import Any, Callable, Dict, Optional import sdnotify @@ -26,12 +27,15 @@ class Worker: """ Init all variables and objects the bot needs to work """ - logger.info('Starting worker %s', __version__) + logger.info(f"Starting worker {__version__}") self._args = args self._config = config self._init(False) + self.last_throttle_start_time: float = 0 + self._heartbeat_msg: float = 0 + # Tell systemd that we completed initialization phase if self._sd_notify: logger.debug("sd_notify: READY=1") @@ -48,10 +52,10 @@ class Worker: # Init the instance of the bot self.freqtrade = FreqtradeBot(self._config) - self._throttle_secs = self._config.get('internals', {}).get( - 'process_throttle_secs', - constants.PROCESS_THROTTLE_SECS - ) + internals_config = self._config.get('internals', {}) + self._throttle_secs = internals_config.get('process_throttle_secs', + constants.PROCESS_THROTTLE_SECS) + self._heartbeat_interval = internals_config.get('heartbeat_interval', 60) self._sd_notify = sdnotify.SystemdNotifier() if \ self._config.get('internals', {}).get('sd_notify', False) else None @@ -63,31 +67,33 @@ class Worker: if state == State.RELOAD_CONF: self._reconfigure() - def _worker(self, old_state: Optional[State], throttle_secs: Optional[float] = None) -> State: + def _worker(self, old_state: Optional[State]) -> State: """ - Trading routine that must be run at each loop + The main routine that runs each throttling iteration and handles the states. :param old_state: the previous service state from the previous call :return: current service state """ state = self.freqtrade.state - if throttle_secs is None: - throttle_secs = self._throttle_secs # Log state transition if state != old_state: self.freqtrade.notify_status(f'{state.name.lower()}') - logger.info('Changing state to: %s', state.name) + logger.info(f"Changing state to: {state.name}") if state == State.RUNNING: self.freqtrade.startup() + # Reset heartbeat timestamp to log the heartbeat message at + # first throttling iteration when the state changes + self._heartbeat_msg = 0 + if state == State.STOPPED: # Ping systemd watchdog before sleeping in the stopped state if self._sd_notify: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.") - time.sleep(throttle_secs) + self._throttle(func=self._process_stopped, throttle_secs=self._throttle_secs) elif state == State.RUNNING: # Ping systemd watchdog before throttling @@ -95,28 +101,40 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.") - self._throttle(func=self._process, min_secs=throttle_secs) + self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) + + if self._heartbeat_interval: + now = time.time() + if (now - self._heartbeat_msg) > self._heartbeat_interval: + logger.info(f"Bot heartbeat. PID={getpid()}, " + f"version='{__version__}', state='{state.name}'") + self._heartbeat_msg = now return state - def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it takes at least `min_secs` to finish execution. :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any + :param throttle_secs: throttling interation execution time limit in seconds + :return: Any (result of execution of func) """ - start = time.time() + self.last_throttle_start_time = time.time() + logger.debug("========================================") result = func(*args, **kwargs) - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) - time.sleep(duration) + time_passed = time.time() - self.last_throttle_start_time + sleep_duration = max(throttle_secs - time_passed, 0.0) + logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " + f"last iteration took {time_passed:.2f} s.") + time.sleep(sleep_duration) return result - def _process(self) -> None: - logger.debug("========================================") + def _process_stopped(self) -> None: + # Maybe do here something in the future... + pass + + def _process_running(self) -> None: try: self.freqtrade.process() except TemporaryError as error: diff --git a/mkdocs.yml b/mkdocs.yml index d53687c64..4e7e6ff75 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,8 @@ site_name: Freqtrade nav: - - About: index.md - - Installation: installation.md + - Home: index.md - Installation Docker: docker.md + - Installation: installation.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md diff --git a/requirements-common.txt b/requirements-common.txt index 2be51ba73..bdb1f1127 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,11 +1,11 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.22.61 +ccxt==1.22.95 SQLAlchemy==1.3.13 python-telegram-bot==12.4.2 arrow==0.15.5 cachetools==4.0.0 -requests==2.22.0 +requests==2.23.0 urllib3==1.25.8 wrapt==1.12.0 jsonschema==3.2.0 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index e97e7f6be..2984229c1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,6 +4,6 @@ # Required for hyperopt scipy==1.4.1 scikit-learn==0.22.1 -scikit-optimize==0.7.2 +scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 diff --git a/requirements-plot.txt b/requirements-plot.txt index 26467d90b..5e62a5e95 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.5.0 +plotly==4.5.1 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a9fe0f637..1877aaa43 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -217,8 +217,9 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active markets: " - "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" + assert ("Exchange Bittrex has 10 active markets: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, " + "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) patch_exchange(mocker, api_mock=api_mock, id="binance") @@ -231,7 +232,7 @@ def test_list_markets(mocker, markets, capsys): pargs['config'] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 9 active markets:\n", + assert re.match("\nExchange Binance has 10 active markets:\n", captured.out) patch_exchange(mocker, api_mock=api_mock, id="bittrex") @@ -243,8 +244,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 11 markets: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, " + assert ("Exchange Bittrex has 12 markets: " + "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) @@ -256,8 +257,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active pairs: " - "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" + assert ("Exchange Bittrex has 9 active pairs: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" in captured.out) # Test list-pairs subcommand with --all: all pairs @@ -268,8 +269,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 10 pairs: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, " + assert ("Exchange Bittrex has 11 pairs: " + "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XRP/BTC.\n" in captured.out) @@ -282,8 +283,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 5 active markets with ETH, LTC as base currencies: " - "ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, XLTCUSDT.\n" + assert ("Exchange Bittrex has 6 active markets with ETH, LTC as base currencies: " + "ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, base=LTC @@ -295,8 +296,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 3 active markets with LTC as base currency: " - "LTC/BTC, LTC/USD, XLTCUSDT.\n" + assert ("Exchange Bittrex has 4 active markets with LTC as base currency: " + "LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT, USD @@ -384,7 +385,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active markets:\n" + assert ("Exchange Bittrex has 10 active markets:\n" in captured.out) # Test tabular output, no markets found @@ -407,7 +408,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC",' + assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/ETH","LTC/USD","NEO/BTC",' '"TKN/BTC","XLTCUSDT","XRP/BTC"]' in captured.out) @@ -446,11 +447,9 @@ def test_create_datadir_failed(caplog): def test_create_datadir(caplog, mocker): - # Ensure that caplog is empty before starting ... - # Should prevent random failures. - caplog.clear() - # Added assert here to analyze random test-failures ... - assert len(caplog.record_tuples) == 0 + + # Capture caplog length here trying to avoid random test failure + len_caplog_before = len(caplog.record_tuples) cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) @@ -463,7 +462,7 @@ def test_create_datadir(caplog, mocker): assert cud.call_count == 1 assert csf.call_count == 1 - assert len(caplog.record_tuples) == 0 + assert len(caplog.record_tuples) == len_caplog_before def test_start_new_strategy(mocker, caplog): @@ -778,6 +777,20 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", + "Sell hyperspace params", "ROI table", "Stoploss"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) args = [ "hyperopt-list", "--no-details", diff --git a/tests/conftest.py b/tests/conftest.py index acb730330..000f62868 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -575,7 +575,34 @@ def get_markets(): } }, 'info': {}, - } + }, + 'LTC/ETH': { + 'id': 'LTCETH', + 'symbol': 'LTC/ETH', + 'base': 'LTC', + 'quote': 'ETH', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 3, + 'price': 5 + }, + 'limits': { + 'amount': { + 'min': 0.001, + 'max': 10000000.0 + }, + 'price': { + 'min': 1e-05, + 'max': 1000.0 + }, + 'cost': { + 'min': 0.01, + 'max': None + } + }, + }, } diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8b2e439c3..6bec53d49 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -400,13 +400,40 @@ def test_validate_stake_currency_error(default_conf, mocker, caplog): def test_get_quote_currencies(default_conf, mocker): ex = get_patched_exchange(mocker, default_conf) - assert set(ex.get_quote_currencies()) == set(['USD', 'BTC', 'USDT']) + assert set(ex.get_quote_currencies()) == set(['USD', 'ETH', 'BTC', 'USDT']) + + +@pytest.mark.parametrize('pair,expected', [ + ('XRP/BTC', 'BTC'), + ('LTC/USD', 'USD'), + ('ETH/USDT', 'USDT'), + ('XLTCUSDT', 'USDT'), + ('XRP/NOCURRENCY', ''), +]) +def test_get_pair_quote_currency(default_conf, mocker, pair, expected): + ex = get_patched_exchange(mocker, default_conf) + assert ex.get_pair_quote_currency(pair) == expected + + +@pytest.mark.parametrize('pair,expected', [ + ('XRP/BTC', 'XRP'), + ('LTC/USD', 'LTC'), + ('ETH/USDT', 'ETH'), + ('XLTCUSDT', 'LTC'), + ('XRP/NOCURRENCY', ''), +]) +def test_get_pair_base_currency(default_conf, mocker, pair, expected): + ex = get_patched_exchange(mocker, default_conf) + assert ex.get_pair_base_currency(pair) == expected def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs directly api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, 'XRP/BTC': {}, 'NEO/BTC': {} + 'ETH/BTC': {'quote': 'BTC'}, + 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, + 'NEO/BTC': {'quote': 'BTC'}, }) id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -454,9 +481,9 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): def test_validate_pairs_restricted(default_conf, mocker, caplog): api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, - 'XRP/BTC': {'info': {'IsRestricted': True}}, - 'NEO/BTC': {'info': 'TestString'}, # info can also be a string ... + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC', 'info': {'IsRestricted': True}}, + 'NEO/BTC': {'quote': 'BTC', 'info': 'TestString'}, # info can also be a string ... }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') @@ -469,6 +496,54 @@ def test_validate_pairs_restricted(default_conf, mocker, caplog): f"on the exchange and eventually remove XRP/BTC from your whitelist.", caplog) +def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + +def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker, caplog): + api_mock = MagicMock() + default_conf['stake_currency'] = '' + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + +def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): + default_conf['exchange']['pair_whitelist'].append('HELLO-WORLD') + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'USDT'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"): + Exchange(default_conf) + + @pytest.mark.parametrize("timeframe", [ ('5m'), ("1m"), ("15m"), ("1h") ]) @@ -1121,25 +1196,16 @@ def test_fetch_ticker(default_conf, mocker, exchange_name): assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 - assert 'ETH/BTC' in exchange._cached_ticker - assert exchange._cached_ticker['ETH/BTC']['bid'] == 0.5 - assert exchange._cached_ticker['ETH/BTC']['ask'] == 1 - - # Test caching - api_mock.fetch_ticker = MagicMock() - exchange.fetch_ticker(pair='ETH/BTC', refresh=False) - assert api_mock.fetch_ticker.call_count == 0 - ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, "fetch_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + pair='ETH/BTC') api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.fetch_ticker(pair='ETH/BTC', refresh=True) + exchange.fetch_ticker(pair='ETH/BTC') with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'): - exchange.fetch_ticker(pair='XRP/ETH', refresh=True) + exchange.fetch_ticker(pair='XRP/ETH') @pytest.mark.parametrize("exchange_name", EXCHANGES) @@ -1828,6 +1894,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): # 'ETH/BTC': 'active': True # 'ETH/USDT': 'active': True # 'LTC/BTC': 'active': False + # 'LTC/ETH': 'active': True # 'LTC/USD': 'active': True # 'LTC/USDT': 'active': True # 'NEO/BTC': 'active': False @@ -1836,26 +1903,26 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): # 'XRP/BTC': 'active': False # all markets ([], [], False, False, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', + ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # active markets ([], [], False, True, - ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # all pairs ([], [], True, False, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', + ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # active pairs ([], [], True, True, - ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # all markets, base=ETH, LTC (['ETH', 'LTC'], [], False, False, - ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), + ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), # all markets, base=LTC (['LTC'], [], False, False, - ['LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), + ['LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), # all markets, quote=USDT ([], ['USDT'], False, False, ['ETH/USDT', 'LTC/USDT', 'XLTCUSDT']), diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index d0f09f74c..1d43878a6 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -330,6 +330,42 @@ def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results assert under > correct +def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 57e928cca..285ecaa02 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -22,14 +22,14 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' - ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------|-------:|---------------:|---------------:|-----------------:|' - '---------------:|:---------------|-------:|--------:|---------:|\n' + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' + ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |' ) assert generate_text_table(data={'ETH/BTC': {}}, stake_currency='BTC', max_open_trades=2, @@ -52,13 +52,13 @@ def test_generate_text_table_sell_reason(default_conf, mocker): ) result_str = ( - '| Sell Reason | Sells | Wins | Draws | Losses |' + '| Sell Reason | Sells | Wins | Draws | Losses |' ' Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' - '|:--------------|--------:|-------:|--------:|---------:|' - '---------------:|---------------:|-----------------:|---------------:|\n' - '| roi | 2 | 2 | 0 | 0 |' + '|---------------+---------+--------+---------+----------+' + '----------------+----------------+------------------+----------------|\n' + '| roi | 2 | 2 | 0 | 0 |' ' 15 | 30 | 0.6 | 15 |\n' - '| stop_loss | 1 | 0 | 0 | 1 |' + '| stop_loss | 1 | 0 | 0 | 1 |' ' -10 | -10 | -0.2 | -5 |' ) assert generate_text_table_sell_reason( @@ -95,14 +95,14 @@ def test_generate_text_table_strategy(default_conf, mocker): ) result_str = ( - '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' - ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------------|-------:|---------------:|---------------:|------' - '-----------:|---------------:|:---------------|-------:|--------:|---------:|\n' - '| TestStrategy1 | 3 | 20.00 | 60.00 | ' - ' 1.10000000 | 30.00 | 0:17:00 | 3 | 0 | 0 |\n' - '| TestStrategy2 | 3 | 30.00 | 90.00 | ' - ' 1.30000000 | 45.00 | 0:20:00 | 3 | 0 | 0 |' + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' + ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' + '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 |' + ' 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 |' + ' 45.00 | 0:20:00 | 3 | 0 | 0 |' ) assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str @@ -111,8 +111,7 @@ def test_generate_edge_table(edge_conf, mocker): results = {} results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - - assert generate_edge_table(results).count(':|') == 7 + assert generate_edge_table(results).count('+') == 7 assert generate_edge_table(results).count('| ETH/BTC |') == 1 assert generate_edge_table(results).count( '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index a35bfa0d6..93b6f6058 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -65,10 +65,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)' } == results[0] - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) assert isnan(results[0]['current_rate']) @@ -134,10 +132,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert 'ETH/BTC' in result[0][1] assert '-0.59% (-0.09)' == result[0][3] - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] @@ -260,10 +256,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' @@ -687,7 +681,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None # Test buy pair not with stakes with pytest.raises(RPCException, match=r'Wrong pair selected. Please pairs with stake.*'): - rpc._rpc_forcebuy('XRP/ETH', 0.0001) + rpc._rpc_forcebuy('LTC/ETH', 0.0001) pair = 'XRP/BTC' # Test not buying diff --git a/tests/test_configuration.py b/tests/test_configuration.py index d810305db..1e9d6440d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -34,13 +34,6 @@ def all_conf(): return conf -def test_load_config_invalid_pair(default_conf) -> None: - default_conf['exchange']['pair_whitelist'].append('ETH-BTC') - - with pytest.raises(ValidationError, match=r'.*does not match.*'): - validate_config_schema(default_conf) - - def test_load_config_missing_attributes(default_conf) -> None: conf = deepcopy(default_conf) conf.pop('exchange') @@ -326,6 +319,7 @@ def test_load_dry_run(default_conf, mocker, config_value, expected, arglist) -> validated_conf = configuration.load_config() assert validated_conf['dry_run'] is expected + assert validated_conf['runmode'] == (RunMode.DRY_RUN if expected else RunMode.LIVE) def test_load_custom_strategy(default_conf, mocker) -> None: @@ -810,12 +804,6 @@ def test_validate_whitelist(default_conf): validate_config_consistency(conf) - conf = deepcopy(default_conf) - conf['stake_currency'] = 'USDT' - with pytest.raises(OperationalException, - match=r"Stake-currency 'USDT' not compatible with pair-whitelist.*"): - validate_config_consistency(conf) - def test_load_config_test_comments() -> None: """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5ed4d296c..852b6b990 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -782,7 +782,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker) -> None: worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - worker._process() + worker._process_running() assert sleep_mock.has_calls() @@ -799,7 +799,7 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert worker.freqtrade.state == State.RUNNING - worker._process() + worker._process_running() assert worker.freqtrade.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] @@ -915,13 +915,21 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: (5, 10, 1.0, 5), # last bigger than ask (5, 10, 0.5, 5), # last bigger than ask ]) -def test_get_buy_rate(mocker, default_conf, ask, last, last_ab, expected) -> None: +def test_get_buy_rate(mocker, default_conf, caplog, ask, last, last_ab, expected) -> None: default_conf['bid_strategy']['ask_last_balance'] = last_ab freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': ask, 'last': last})) assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert freqtrade.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: @@ -2192,6 +2200,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> Non Trade.session = MagicMock() trade = MagicMock() + trade.pair = 'LTC/ETH' limit_buy_order['remaining'] = limit_buy_order['amount'] assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) assert cancel_order_mock.call_count == 1 @@ -2215,6 +2224,7 @@ def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_ Trade.session = MagicMock() trade = MagicMock() + trade.pair = 'LTC/ETH' limit_buy_order['remaining'] = limit_buy_order['amount'] assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) assert cancel_order_mock.call_count == 1 @@ -3614,7 +3624,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order assert freqtrade.handle_trade(trade) is True -def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: +def test_get_sell_rate(default_conf, mocker, caplog, ticker, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -3626,8 +3636,15 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: # Test regular mode ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == 0.00001098 + # Use caching + rate = ft.get_sell_rate(pair, False) + assert rate == 0.00001098 + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + caplog.clear() # Test orderbook mode default_conf['ask_strategy']['use_order_book'] = True @@ -3635,8 +3652,12 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: default_conf['ask_strategy']['order_book_max'] = 2 ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == 0.043936 + rate = ft.get_sell_rate(pair, False) + assert rate == 0.043936 + assert log_has("Using cached sell rate for ETH/BTC.", caplog) def test_startup_state(default_conf, mocker): @@ -3665,30 +3686,6 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): assert reinit_mock.call_count == 0 -def test_process_i_am_alive(default_conf, mocker, caplog): - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - ftbot = get_patched_freqtradebot(mocker, default_conf) - message = r"Bot heartbeat\. PID=.*" - ftbot.process() - assert log_has_re(message, caplog) - assert ftbot._heartbeat_msg != 0 - - caplog.clear() - # Message is not shown before interval is up - ftbot.process() - assert not log_has_re(message, caplog) - - caplog.clear() - # Set clock - 70 seconds - ftbot._heartbeat_msg -= 70 - - ftbot.process() - assert log_has_re(message, caplog) - - @pytest.mark.usefixtures("init_persistence") def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog): default_conf['dry_run'] = True diff --git a/tests/test_worker.py b/tests/test_worker.py index 2fb42d47e..7b446ac6a 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade.data.dataprovider import DataProvider from freqtrade.state import State from freqtrade.worker import Worker -from tests.conftest import get_patched_worker, log_has +from tests.conftest import get_patched_worker, log_has, log_has_re def test_worker_state(mocker, default_conf, markets) -> None: @@ -38,15 +38,13 @@ def test_worker_running(mocker, default_conf, caplog) -> None: def test_worker_stopped(mocker, default_conf, caplog) -> None: mock_throttle = MagicMock() mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mock_sleep = mocker.patch('time.sleep', return_value=None) worker = get_patched_worker(mocker, default_conf) worker.freqtrade.state = State.STOPPED state = worker._worker(old_state=State.RUNNING) assert state is State.STOPPED assert log_has('Changing state to: STOPPED', caplog) - assert mock_throttle.call_count == 0 - assert mock_sleep.call_count == 1 + assert mock_throttle.call_count == 1 def test_throttle(mocker, default_conf, caplog) -> None: @@ -57,14 +55,14 @@ def test_throttle(mocker, default_conf, caplog) -> None: worker = get_patched_worker(mocker, default_conf) start = time.time() - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) end = time.time() assert result == 42 assert end - start > 0.1 - assert log_has('Throttling throttled_func for 0.10 seconds', caplog) + assert log_has_re(r"Throttling with 'throttled_func\(\)': sleep for 0\.10 s.*", caplog) - result = worker._throttle(throttled_func, min_secs=-1) + result = worker._throttle(throttled_func, throttle_secs=-1) assert result == 42 @@ -74,8 +72,54 @@ def test_throttle_with_assets(mocker, default_conf) -> None: worker = get_patched_worker(mocker, default_conf) - result = worker._throttle(throttled_func, min_secs=0.1, nb_assets=666) + result = worker._throttle(throttled_func, throttle_secs=0.1, nb_assets=666) assert result == 666 - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) assert result == -1 + + +def test_worker_heartbeat_running(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='RUNNING'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.RUNNING + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.RUNNING) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + +def test_worker_heartbeat_stopped(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='STOPPED'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.STOPPED + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.STOPPED) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog)