diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc3d324a1..42668e46f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: [3.7] + python-version: [3.7, 3.8] steps: - uses: actions/checkout@v2 @@ -130,8 +130,7 @@ jobs: if: startsWith(runner.os, 'Windows') with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip - restore-keys: ${{ runner.os }}-pip + key: ${{ matrix.os }}-${{ matrix.python-version }}-pip - name: Installation run: | diff --git a/README.md b/README.md index 59799da84..88070d45e 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) ([*Note for binance users](#a-note-on-binance)) +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists)) +- [X] [Kraken](https://kraken.com/) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Documentation diff --git a/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..90626b183 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 138fba208..7dbdd77dd 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -3,7 +3,15 @@ # Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" python -m pip install --upgrade pip -pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl + +$pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" + +if ($pyv -eq '3.7') { + pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl +} +if ($pyv -eq '3.8') { + pip install build_helpers\TA_Lib-0.4.17-cp38-cp38-win_amd64.whl +} pip install -r requirements-dev.txt pip install -e . diff --git a/config_full.json.example b/config_full.json.example index cdb7e841e..f0414bd0d 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -25,6 +25,7 @@ "sell": 30 }, "bid_strategy": { + "price_side": "bid", "use_order_book": false, "ask_last_balance": 0.0, "order_book_top": 1, @@ -34,6 +35,7 @@ } }, "ask_strategy":{ + "price_side": "ask", "use_order_book": false, "order_book_min": 1, "order_book_max": 9, diff --git a/docs/bot-usage.md b/docs/bot-usage.md index dbc111d44..78e137676 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -275,7 +275,7 @@ Check the corresponding [Data Downloading](data-download.md) section for more de ## Hyperopt commands To optimize your strategy, you can use hyperopt parameter hyperoptimization -to find optimal parameter values for your stategy. +to find optimal parameter values for your strategy. ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] @@ -323,7 +323,7 @@ optional arguments: --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. + --print-json Print best results in JSON format. -j JOBS, --job-workers JOBS The number of concurrently running jobs for hyperoptimization (hyperopt worker processes). If -1 @@ -341,10 +341,11 @@ optional arguments: class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in - Hyperopt-loss-functions are: DefaultHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss, - SharpeHyperOptLossDaily.(default: - `DefaultHyperOptLoss`). + Hyperopt-loss-functions are: + DefaultHyperOptLoss, OnlyProfitHyperOptLoss, + SharpeHyperOptLoss, SharpeHyperOptLossDaily, + SortinoHyperOptLoss, SortinoHyperOptLossDaily. + (default: `DefaultHyperOptLoss`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/configuration.md b/docs/configuration.md index b05dab7c9..5580b9c68 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -60,11 +60,13 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer -| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook). +| `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).
*Defaults to `bid`.*
**Datatype:** String (either `ask` or `bid`). +| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook-enabled). | `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
**Datatype:** Boolean | `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled).
*Defaults to `1`.*
**Datatype:** Positive Integer | `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market).
*Defaults to `false`.*
**Datatype:** Boolean | `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market)
*Defaults to `0`.*
**Datatype:** Float (as ratio) +| `ask_strategy.price_side` | Select the side of the spread the bot should look at to get the sell rate. [More information below](#sell-price-side).
*Defaults to `ask`.*
**Datatype:** String (either `ask` or `bid`). | `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled).
**Datatype:** Boolean | `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer | `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer @@ -370,16 +372,18 @@ The possible values are: `gtc` (default), `fok` or `ioc`. Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the -[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested -with only Bittrex and Binance. - -The bot was tested with the following exchanges: +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). + However, the bot was tested by the development team with only Bittrex, Binance and Kraken, + so the these are the only officially supported exhanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" +- [Kraken](https://kraken.com/): "kraken" Feel free to test other exchanges and submit your PR to improve the bot. +Some exchanges require special configuration, which can be found on the [Exchange-specific Notes](exchanges.md) documentation page. + #### Sample exchange configuration A exchange configuration for "binance" would look as follows: @@ -461,23 +465,72 @@ Orderbook `bid` (buy) side depth is then divided by the orderbook `ask` (sell) s !!! Note A delta value below 1 means that `ask` (sell) orderbook side depth is greater than the depth of the `bid` (buy) orderbook side, while a value greater than 1 means opposite (depth of the buy side is higher than the depth of the sell side). +#### Buy price side + +The configuration setting `bid_strategy.price_side` defines the side of the spread the bot looks for when buying. + +The following displays an orderbook. + +``` explanation +... +103 +102 +101 # ask +-------------Current spread +99 # bid +98 +97 +... +``` + +If `bid_strategy.price_side` is set to `"bid"`, then the bot will use 99 as buying price. +In line with that, if `bid_strategy.price_side` is set to `"ask"`, then the bot will use 101 as buying price. + +Using `ask` price often guarantees quicker filled orders, but the bot can also end up paying more than what would have been necessary. +Taker fees instead of maker fees will most likely apply even when using limit buy orders. +Also, prices at the "ask" side of the spread are higher than prices at the "bid" side in the orderbook, so the order behaves similar to a market order (however with a maximum price). + #### Buy price with Orderbook enabled -When buying with the orderbook enabled (`bid_strategy.use_order_book=True`), Freqtrade fetches the `bid_strategy.order_book_top` entries from the orderbook and then uses the entry specified as `bid_strategy.order_book_top` on the `bid` (buy) side of the orderbook. 1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on. +When buying with the orderbook enabled (`bid_strategy.use_order_book=True`), Freqtrade fetches the `bid_strategy.order_book_top` entries from the orderbook and then uses the entry specified as `bid_strategy.order_book_top` on the configured side (`bid_strategy.price_side`) of the orderbook. 1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on. #### Buy price without Orderbook enabled -When not using orderbook (`bid_strategy.use_order_book=False`), Freqtrade uses the best `ask` (sell) price from the ticker if it's below the `last` traded price from the ticker. Otherwise (when the `ask` price is not below the `last` price), it calculates a rate between `ask` and `last` price. +The following section uses `side` as the configured `bid_strategy.price_side`. -The `bid_strategy.ask_last_balance` configuration parameter controls this. A value of `0.0` will use `ask` price, while `1.0` will use the `last` price and values between those interpolate between ask and last price. +When not using orderbook (`bid_strategy.use_order_book=False`), Freqtrade uses the best `side` price from the ticker if it's below the `last` traded price from the ticker. Otherwise (when the `side` price is above the `last` price), it calculates a rate between `side` and `last` price. -Using `ask` price often guarantees quicker success in the bid, but the bot can also end up paying more than what would have been necessary. +The `bid_strategy.ask_last_balance` configuration parameter controls this. A value of `0.0` will use `side` price, while `1.0` will use the `last` price and values between those interpolate between ask and last price. ### Sell price +#### Sell price side + +The configuration setting `ask_strategy.price_side` defines the side of the spread the bot looks for when selling. + +The following displays an orderbook: + +``` explanation +... +103 +102 +101 # ask +-------------Current spread +99 # bid +98 +97 +... +``` + +If `ask_strategy.price_side` is set to `"ask"`, then the bot will use 101 as selling price. +In line with that, if `ask_strategy.price_side` is set to `"bid"`, then the bot will use 99 as selling price. + #### Sell price with Orderbook enabled -When selling with the orderbook enabled (`ask_strategy.use_order_book=True`), Freqtrade fetches the `ask_strategy.order_book_max` entries in the orderbook. Then each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the `ask` orderbook side are validated for a profitable sell-possibility based on the strategy configuration and the sell order is placed at the first profitable spot. +When selling with the orderbook enabled (`ask_strategy.use_order_book=True`), Freqtrade fetches the `ask_strategy.order_book_max` entries in the orderbook. Then each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the configured orderbook side are validated for a profitable sell-possibility based on the strategy configuration (`minimal_roi` conditions) and the sell order is placed at the first profitable spot. + +!!! Note + Using `order_book_max` higher than `order_book_min` only makes sense when ask_strategy.price_side is set to `"ask"`. The idea here is to place the sell order early, to be ahead in the queue. @@ -488,7 +541,7 @@ A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting #### Sell price without Orderbook enabled -When not using orderbook (`ask_strategy.use_order_book=False`), the `bid` price from the ticker will be used as the sell price. +When not using orderbook (`ask_strategy.use_order_book=False`), the price at the `ask_strategy.price_side` side (defaults to `"ask"`) from the ticker will be used as the sell price. ## Pairlists diff --git a/docs/developer.md b/docs/developer.md index b128ffd2b..ef9232a59 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -234,7 +234,7 @@ git checkout -b new_release Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. -* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7-1` should we need to do a second release that month. +* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part * push that branch to the remote and create a PR against the master branch @@ -268,11 +268,6 @@ Once the PR against master is merged (best right after merging): * Use "master" as reference (this step comes after the above PR is merged). * Use the above changelog as release comment (as codeblock) -### After-release - -* Update version in develop by postfixing that with `-dev` (`2019.6 -> 2019.6-dev`). -* Create a PR against develop to update that branch. - ## Releases ### pypi diff --git a/docs/exchanges.md b/docs/exchanges.md index f615bc61a..70dae0aa5 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -62,6 +62,11 @@ res = [ f"{x['MarketCurrency']}/{x['BaseCurrency']}" for x in ct.publicGetMarket print(res) ``` +## All exchanges + +Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. + + ## Random notes for other exchanges * The Ocean (exchange id: `theocean`) exchange uses Web3 functionality and requires `web3` python package to be installed: diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 401811a1b..9bc5888ce 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -31,9 +31,9 @@ This will create a new hyperopt file from a template, which will be located unde Depending on the space you want to optimize, only some of the below are required: * fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimzation +* fill `indicator_space` - for buy signal optimization * fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimzation +* fill `sell_indicator_space` - for sell signal optimization !!! Note `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. @@ -81,11 +81,11 @@ There are two places you need to change in your hyperopt file to add a new buy h There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. -2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". +2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower Bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like -"*buy exactly when close price touches lower bollinger band, BUT only if +"*buy exactly when close price touches lower Bollinger band, BUT only if ADX > 10*". If you have updated the buy strategy, i.e. changed the contents of @@ -172,7 +172,7 @@ So let's write the buy strategy using these values: Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) with different value combinations. It will then use the given historical data and make buys based on the buy signals generated with the above function and based on the results -it will end with telling you which paramter combination produced the best profits. +it will end with telling you which parameter combination produced the best profits. The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. When you want to test an indicator that isn't used by the bot currently, remember to @@ -191,8 +191,10 @@ Currently, the following loss functions are builtin: * `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on the trade returns) -* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on daily trade returns) +* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) +* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) +* `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) +* `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. @@ -272,7 +274,7 @@ In some situations, you may need to run Hyperopt (and Backtesting) with the By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one open trade is allowed for every traded pair. The total number of trades open for all pairs is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to -some potential trades to be hidden (or masked) by previosly open trades. +some potential trades to be hidden (or masked) by previously open trades. The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` diff --git a/docs/plotting.md b/docs/plotting.md index ecd5e1603..3eef8f8e7 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -196,6 +196,7 @@ The first graph is good to get a grip of how the overall market progresses. The second graph will show if your algorithm works or doesn't. Perhaps you want an algorithm that steadily makes small profits, or one that acts less often, but makes big swings. +This graph will also highlight the start (and end) of the Max drawdown period. The third graph can be useful to spot outliers, events in pairs that cause profit spikes. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 53b35ca09..d26d684ce 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -121,7 +121,6 @@ from freqtrade.data.btanalysis import analyze_trade_parallelism # Analyze the above parallel_trades = analyze_trade_parallelism(trades, '5m') - parallel_trades.plot() ``` @@ -134,11 +133,14 @@ Freqtrade offers interactive plotting capabilities based on plotly. from freqtrade.plot.plotting import generate_candlestick_graph # Limit graph period to keep plotly quick and reactive +# Filter trades to one pair +trades_red = trades.loc[trades['pair'] == pair] + data_red = data['2019-06-01':'2019-06-10'] # Generate candlestick graph graph = generate_candlestick_graph(pair=pair, data=data_red, - trades=trades, + trades=trades_red, indicators1=['sma20', 'ema50', 'ema55'], indicators2=['rsi', 'macd', 'macdsignal', 'macdhist'] ) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index a8d4bc198..ef674c5c2 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -257,7 +257,8 @@ AVAILABLE_CLI_OPTIONS = { help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' - 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily.' + 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, ' + 'SortinoHyperOptLoss, SortinoHyperOptLossDaily.' '(default: `%(default)s`).', metavar='NAME', default=constants.DEFAULT_HYPEROPT_LOSS, diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index ccaa59e54..4803f6885 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -51,7 +51,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: try: Hyperopt.print_result_table(config, trials, total_epochs, - not filteroptions['only_best'], print_colorized) + not filteroptions['only_best'], print_colorized, 0) except KeyboardInterrupt: print('User interrupted..') @@ -97,10 +97,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: if n > trials_epochs: raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") + f"The index of the epoch to show should be less than {trials_epochs + 1}.") if n < -trials_epochs: raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") # Translate epoch index from human-readable format to pythonic if n > 0: @@ -122,52 +122,52 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: trials = [x for x in trials if x['results_metrics']['profit'] > 0] if filteroptions['filter_min_trades'] > 0: trials = [ - x for x in trials - if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] - ] + x for x in trials + if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] + ] if filteroptions['filter_max_trades'] > 0: trials = [ - x for x in trials - if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] - ] + x for x in trials + if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] + ] if filteroptions['filter_min_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] - ] + x for x in trials + if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + ] if filteroptions['filter_max_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] - ] + x for x in trials + if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + ] if filteroptions['filter_min_avg_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - > filteroptions['filter_min_avg_profit'] - ] + x for x in trials + if x['results_metrics']['avg_profit'] + > filteroptions['filter_min_avg_profit'] + ] if filteroptions['filter_max_avg_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - < filteroptions['filter_max_avg_profit'] - ] + x for x in trials + if x['results_metrics']['avg_profit'] + < filteroptions['filter_max_avg_profit'] + ] if filteroptions['filter_min_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] - ] + x for x in trials + if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + ] if filteroptions['filter_max_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] - ] + x for x in trials + if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] + ] logger.info(f"{len(trials)} " + ("best " if filteroptions['only_best'] else "") + diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1504d1f1c..ac1a8a6a9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -15,6 +15,7 @@ UNLIMITED_STAKE_AMOUNT = 'unlimited' DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05 REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] +ORDERBOOK_SIDES = ['ask', 'bid'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', @@ -113,15 +114,16 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 1, 'exclusiveMaximum': False, - 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1}, - 'check_depth_of_market': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, - } - }, + }, + 'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'}, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } }, }, 'required': ['ask_last_balance'] @@ -129,6 +131,7 @@ CONF_SCHEMA = { 'ask_strategy': { 'type': 'object', 'properties': { + 'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'ask'}, 'use_order_book': {'type': 'boolean'}, 'order_book_min': {'type': 'integer', 'minimum': 1}, 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50}, @@ -299,6 +302,7 @@ SCHEMA_TRADE_REQUIRED = [ 'last_stake_amount_min_ratio', 'dry_run', 'dry_run_wallet', + 'ask_strategy', 'bid_strategy', 'unfilledtimeout', 'stoploss', diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index c28e462ba..7972c6333 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,7 +3,7 @@ Helpers when analyzing backtest data """ import logging from pathlib import Path -from typing import Dict, Union +from typing import Dict, Union, Tuple import numpy as np import pandas as pd @@ -188,3 +188,28 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, # FFill to get continuous df[col_name] = df[col_name].ffill() return df + + +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time', + value_col: str = 'profitperc' + ) -> Tuple[float, pd.Timestamp, pd.Timestamp]: + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_time and profitperc) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_time') + :param value_col: Column in DataFrame to use for values (defaults to 'profitperc') + :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col) + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + + high_date = profit_results.loc[max_drawdown_df['high_value'].idxmax(), date_col] + low_date = profit_results.loc[max_drawdown_df['drawdown'].idxmin(), date_col] + + return abs(min(max_drawdown_df['drawdown'])), high_date, low_date diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 799ee2c37..522b4e40e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -332,7 +332,8 @@ class Exchange: logger.warning(f"Pair {pair} is restricted for some users on this exchange." f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") - if not self.get_pair_quote_currency(pair) == self._config['stake_currency']: + if (self._config['stake_currency'] and + self.get_pair_quote_currency(pair) != self._config['stake_currency']): invalid_pairs.append(pair) if invalid_pairs: raise OperationalException( @@ -1023,7 +1024,7 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non def is_exchange_officially_supported(exchange_name: str) -> bool: - return exchange_name in ['bittrex', 'binance'] + return exchange_name in ['bittrex', 'binance', 'kraken'] def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f50244dac..4cbacdb1e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -242,25 +242,25 @@ class FreqtradeBot: logger.info(f"Using cached buy rate for {pair}.") return rate - config_bid_strategy = self.config.get('bid_strategy', {}) - if 'use_order_book' in config_bid_strategy and\ - config_bid_strategy.get('use_order_book', False): - logger.info('Getting price from order book') - order_book_top = config_bid_strategy.get('order_book_top', 1) + bid_strategy = self.config.get('bid_strategy', {}) + if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + logger.info( + f"Getting price from order book {bid_strategy['price_side'].capitalize()} side." + ) + order_book_top = bid_strategy.get('order_book_top', 1) order_book = self.exchange.get_order_book(pair, order_book_top) logger.debug('order_book %s', order_book) # top 1 = index 0 - order_book_rate = order_book['bids'][order_book_top - 1][0] - logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) + order_book_rate = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + logger.info(f'...top {order_book_top} order book buy rate {order_book_rate:.8f}') used_rate = order_book_rate else: - logger.info('Using Last Ask / Last Price') + logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") ticker = self.exchange.fetch_ticker(pair) - if ticker['ask'] < ticker['last']: - ticker_rate = ticker['ask'] - else: + ticker_rate = ticker[bid_strategy['price_side']] + if ticker['last'] and ticker_rate > ticker['last']: balance = self.config['bid_strategy']['ask_last_balance'] - ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) used_rate = ticker_rate self._buy_rate_cache[pair] = used_rate @@ -617,6 +617,15 @@ class FreqtradeBot: return trades_closed + def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, + order_book_min: int = 1): + """ + Helper generator to query orderbook in loop (used for early sell-order placing) + """ + order_book = self.exchange.get_order_book(pair, order_book_max) + for i in range(order_book_min, order_book_max + 1): + yield order_book[side][i - 1][0] + def get_sell_rate(self, pair: str, refresh: bool) -> float: """ Get sell rate - either using get-ticker bid or first bid based on orderbook @@ -636,13 +645,12 @@ class FreqtradeBot: config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): + # This code is only used for notifications, selling uses the generator directly logger.debug('Using order book to get sell rate') - - order_book = self.exchange.get_order_book(pair, 1) - rate = order_book['bids'][0][0] + rate = next(self._order_book_gen(pair, f"{config_ask_strategy['price_side']}s")) else: - rate = self.exchange.fetch_ticker(pair)['bid'] + rate = self.exchange.fetch_ticker(pair)[config_ask_strategy['price_side']] self._sell_rate_cache[pair] = rate return rate @@ -672,12 +680,13 @@ class FreqtradeBot: order_book_min = config_ask_strategy.get('order_book_min', 1) order_book_max = config_ask_strategy.get('order_book_max', 1) - order_book = self.exchange.get_order_book(trade.pair, order_book_max) - + order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", + order_book_min=order_book_min, + order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): - order_book_rate = order_book['asks'][i - 1][0] - logger.debug(' order book asks top %s: %0.8f', i, order_book_rate) - sell_rate = order_book_rate + sell_rate = next(order_book) + logger.debug(f" order book {config_ask_strategy['price_side']} top {i}: " + f"{sell_rate:0.8f}") if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c18aefc76..94441ce24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -423,28 +423,37 @@ class Backtesting: strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") - print(' BACKTESTING REPORT '.center(133, '=')) - print(generate_text_table(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results)) + table = generate_text_table(data, stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) - print(' SELL REASON STATS '.center(133, '=')) - print(generate_text_table_sell_reason(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results)) + table = generate_text_table_sell_reason(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) - print(' LEFT OPEN TRADES REPORT '.center(133, '=')) - print(generate_text_table(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results.loc[results.open_at_end], skip_nan=True)) + table = generate_text_table(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results.loc[results.open_at_end], skip_nan=True) + if isinstance(table, str): + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + if isinstance(table, str): + print('=' * len(table.splitlines()[0])) print() if len(all_results) > 1: # Print Strategy summary table - print(' STRATEGY SUMMARY '.center(133, '=')) - print(generate_text_table_strategy(self.config['stake_currency'], - self.config['max_open_trades'], - all_results=all_results)) + table = generate_text_table_strategy(self.config['stake_currency'], + self.config['max_open_trades'], + all_results=all_results) + print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) + print(table) + print('=' * len(table.splitlines()[0])) print('\nFor more details, please look at the detail tables above') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 66ea18bd1..e9ab469f4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,6 +9,7 @@ import logging import random import sys import warnings +from math import ceil from collections import OrderedDict from operator import itemgetter from pathlib import Path @@ -21,7 +22,7 @@ from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) from pandas import DataFrame, json_normalize, isna -from tabulate import tabulate +import tabulate from freqtrade.data.converter import trim_dataframe from freqtrade.data.history import get_timerange @@ -116,6 +117,7 @@ class Hyperopt: self.config['ask_strategy']['use_sell_signal'] = True self.print_all = self.config.get('print_all', False) + self.hyperopt_table_header = 0 self.print_colorized = self.config.get('print_colorized', False) self.print_json = self.config.get('print_json', False) @@ -153,7 +155,7 @@ class Hyperopt: """ num_trials = len(self.trials) if num_trials > self.num_trials_saved: - logger.info(f"Saving {num_trials} {plural(num_trials, 'epoch')}.") + logger.debug(f"Saving {num_trials} {plural(num_trials, 'epoch')}.") dump(self.trials, self.trials_file) self.num_trials_saved = num_trials if final: @@ -272,8 +274,10 @@ class Hyperopt: if not self.print_all: # Separate the results explanation string from dots print("\n") - self.print_results_explanation(results, self.total_epochs, self.print_all, - self.print_colorized) + self.print_result_table(self.config, results, self.total_epochs, + self.print_all, self.print_colorized, + self.hyperopt_table_header) + self.hyperopt_table_header = 2 @staticmethod def print_results_explanation(results, total_epochs, highlight_best: bool, @@ -299,13 +303,15 @@ class Hyperopt: @staticmethod def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, - print_colorized: bool) -> None: + print_colorized: bool, remove_header: int) -> None: """ Log result table """ if not results: return + tabulate.PRESERVE_WHITESPACE = True + trials = json_normalize(results, max_level=1) trials['Best'] = '' trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', @@ -317,35 +323,63 @@ class Hyperopt: trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '*' trials.loc[trials['is_best'], 'Best'] = 'Best' - trials['Objective'] = trials['Objective'].astype(str) trials.loc[trials['Total profit'] > 0, 'is_profit'] = True trials['Trades'] = trials['Trades'].astype(str) trials['Epoch'] = trials['Epoch'].apply( - lambda x: "{}/{}".format(x, total_epochs)) + lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs) + ) trials['Avg profit'] = trials['Avg profit'].apply( - lambda x: '{:,.2f}%'.format(x) if not isna(x) else x) - trials['Profit'] = trials['Profit'].apply( - lambda x: '{:,.2f}%'.format(x) if not isna(x) else x) - trials['Total profit'] = trials['Total profit'].apply( - lambda x: '{: 11.8f} '.format(x) + config['stake_currency'] if not isna(x) else x) + lambda x: ('{:,.2f}%'.format(x)).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') + ) trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: '{:,.1f}m'.format(x) if not isna(x) else x) + lambda x: ('{:,.1f} m'.format(x)).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') + ) + trials['Objective'] = trials['Objective'].apply( + lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') + ) + + trials['Profit'] = trials.apply( + lambda x: '{:,.8f} {} {}'.format( + x['Total profit'], config['stake_currency'], + '({:,.2f}%)'.format(x['Profit']).rjust(10, ' ') + ).rjust(25+len(config['stake_currency'])) + if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])), + axis=1 + ) + trials = trials.drop(columns=['Total profit']) + if print_colorized: for i in range(len(trials)): if trials.loc[i]['is_profit']: - for z in range(len(trials.loc[i])-3): - trials.iat[i, z] = "{}{}{}".format(Fore.GREEN, - str(trials.loc[i][z]), Fore.RESET) + for j in range(len(trials.loc[i])-3): + trials.iat[i, j] = "{}{}{}".format(Fore.GREEN, + str(trials.loc[i][j]), Fore.RESET) if trials.loc[i]['is_best'] and highlight_best: - for z in range(len(trials.loc[i])-3): - trials.iat[i, z] = "{}{}{}".format(Style.BRIGHT, - str(trials.loc[i][z]), Style.RESET_ALL) + for j in range(len(trials.loc[i])-3): + trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT, + str(trials.loc[i][j]), Style.RESET_ALL) trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + if remove_header > 0: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='orgtbl', + headers='keys', stralign="right" + ) - print(tabulate(trials.to_dict(orient='list'), headers='keys', tablefmt='psql', - stralign="right")) + table = table.split("\n", remove_header)[remove_header] + elif remove_header < 0: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='psql', + headers='keys', stralign="right" + ) + table = "\n".join(table.split("\n")[0:remove_header]) + else: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='psql', + headers='keys', stralign="right" + ) + print(table) def has_space(self, space: str) -> bool: """ @@ -533,7 +567,7 @@ class Hyperopt: def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") - + self.hyperopt_table_header = -1 data, timerange = self.backtesting.load_bt_data() preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) @@ -569,16 +603,21 @@ class Hyperopt: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') - EVALS = max(self.total_epochs // jobs, 1) + EVALS = ceil(self.total_epochs / jobs) for i in range(EVALS): - asked = self.opt.ask(n_points=jobs) + # Correct the number of epochs to be processed for the last + # iteration (should not exceed self.total_epochs in total) + n_rest = (i + 1) * jobs - self.total_epochs + current_jobs = jobs - n_rest if n_rest > 0 else jobs + + asked = self.opt.ask(n_points=current_jobs) f_val = self.run_optimizer_parallel(parallel, asked, i) self.opt.tell(asked, [v['loss'] for v in f_val]) self.fix_optimizer_models_list() - for j in range(jobs): + + for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 - val = f_val[j] val['current_epoch'] = current val['is_initial_point'] = current <= INITIAL_POINTS logger.debug(f"Optimizer epoch evaluated: {val}") diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py new file mode 100644 index 000000000..83f644a43 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -0,0 +1,49 @@ +""" +SortinoHyperOptLoss + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +from datetime import datetime + +from pandas import DataFrame +import numpy as np + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLoss(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + """ + total_profit = results["profit_percent"] + days_period = (max_date - min_date).days + + # adding slippage of 0.1% per trade + total_profit = total_profit - 0.0005 + expected_returns_mean = total_profit.sum() / days_period + + results['downside_returns'] = 0 + results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent'] + down_stdev = np.std(results['downside_returns']) + + if np.std(total_profit) != 0.0: + sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py new file mode 100644 index 000000000..16dc26142 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -0,0 +1,70 @@ +""" +SortinoHyperOptLossDaily + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +import math +from datetime import datetime + +from pandas import DataFrame, date_range + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLossDaily(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + + Sortino Ratio calculated as described in + http://www.redrockcapital.com/Sortino__A__Sharper__Ratio_Red_Rock_Capital.pdf + """ + resample_freq = '1D' + slippage_per_trade_ratio = 0.0005 + days_in_year = 365 + minimum_acceptable_return = 0.0 + + # apply slippage per trade to profit_percent + results.loc[:, 'profit_percent_after_slippage'] = \ + results['profit_percent'] - slippage_per_trade_ratio + + # create the index within the min_date and end max_date + t_index = date_range(start=min_date, end=max_date, freq=resample_freq, + normalize=True) + + sum_daily = ( + results.resample(resample_freq, on='close_time').agg( + {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) + ) + + total_profit = sum_daily["profit_percent_after_slippage"] - minimum_acceptable_return + expected_returns_mean = total_profit.mean() + + sum_daily['downside_returns'] = 0 + sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit + total_downside = sum_daily['downside_returns'] + # Here total_downside contains min(0, P - MAR) values, + # where P = sum_daily["profit_percent_after_slippage"] + down_stdev = math.sqrt((total_downside**2).sum() / len(total_downside)) + + if (down_stdev != 0.): + sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(t_index, sum_daily, total_profit) + # print(minimum_acceptable_return, expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index b00adbd48..39bde50a8 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -66,7 +66,7 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra ]) # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore def generate_text_table_sell_reason( @@ -112,7 +112,7 @@ def generate_text_table_sell_reason( profit_percent_tot, ] ) - return tabulate(tabular_data, headers=headers, tablefmt="pipe") + return tabulate(tabular_data, headers=headers, tablefmt="orgtbl", stralign="right") def generate_text_table_strategy(stake_currency: str, max_open_trades: str, @@ -146,7 +146,7 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str, ]) # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore def generate_edge_table(results: dict) -> str: @@ -172,4 +172,4 @@ def generate_edge_table(results: dict) -> str: # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 4a892792a..d979a40e0 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,7 +5,8 @@ from typing import Any, Dict, List import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (combine_tickers_with_mean, +from freqtrade.data.btanalysis import (calculate_max_drawdown, + combine_tickers_with_mean, create_cum_profit, extract_trades_of_period, load_trades) from freqtrade.data.converter import trim_dataframe @@ -111,6 +112,36 @@ def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> make_sub return fig +def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> make_subplots: + """ + Add scatter points indicating max drawdown + """ + try: + max_drawdown, highdate, lowdate = calculate_max_drawdown(trades) + + drawdown = go.Scatter( + x=[highdate, lowdate], + y=[ + df_comb.loc[highdate, 'cum_profit'], + df_comb.loc[lowdate, 'cum_profit'], + ], + mode='markers', + name=f"Max drawdown {max_drawdown:.2f}%", + text=f"Max drawdown {max_drawdown:.2f}%", + marker=dict( + symbol='square-open', + size=9, + line=dict(width=2), + color='green' + + ) + ) + fig.add_trace(drawdown, row, 1) + except ValueError: + logger.warning("No trades found - not plotting max drawdown.") + return fig + + def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: """ Add trades to "fig" @@ -364,6 +395,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], fig.add_trace(avgclose, 1, 1) fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') + fig = add_max_drawdown(fig, 2, trades, df_comb) for pair in pairs: profit_col = f'cum_profit_{pair}' diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 88edeb1e8..0049d59a0 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -11,6 +11,7 @@ "sell": 30 }, "bid_strategy": { + "price_side": "bid", "ask_last_balance": 0.0, "use_order_book": false, "order_book_top": 1, @@ -20,6 +21,7 @@ } }, "ask_strategy": { + "price_side": "ask", "use_order_book": false, "order_book_min": 1, "order_book_max": 9, diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 399235cfe..dffa308ce 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -190,7 +190,6 @@ "# Analyze the above\n", "parallel_trades = analyze_trade_parallelism(trades, '5m')\n", "\n", - "\n", "parallel_trades.plot()" ] }, @@ -212,11 +211,14 @@ "from freqtrade.plot.plotting import generate_candlestick_graph\n", "# Limit graph period to keep plotly quick and reactive\n", "\n", + "# Filter trades to one pair\n", + "trades_red = trades.loc[trades['pair'] == pair]\n", + "\n", "data_red = data['2019-06-01':'2019-06-10']\n", "# Generate candlestick graph\n", "graph = generate_candlestick_graph(pair=pair,\n", " data=data_red,\n", - " trades=trades,\n", + " trades=trades_red,\n", " indicators1=['sma20', 'ema50', 'ema55'],\n", " indicators2=['rsi', 'macd', 'macdsignal', 'macdhist']\n", " )\n", diff --git a/requirements-common.txt b/requirements-common.txt index bdb1f1127..10d567a96 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.22.95 +ccxt==1.23.30 SQLAlchemy==1.3.13 python-telegram-bot==12.4.2 arrow==0.15.5 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 2984229c1..c713317ec 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.4.1 -scikit-learn==0.22.1 +scikit-learn==0.22.2 scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 diff --git a/requirements-plot.txt b/requirements-plot.txt index 5e62a5e95..a70c3e0cf 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.5.1 +plotly==4.5.2 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 5f9bc0aa2..3e1c0a581 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -447,11 +447,6 @@ def test_create_datadir_failed(caplog): def test_create_datadir(caplog, mocker): - # Ensure that caplog is empty before starting ... - # Should prevent random failures. - caplog.clear() - # Added assert here to analyze random test-failures ... - assert len(caplog.record_tuples) == 0 cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) @@ -464,7 +459,6 @@ def test_create_datadir(caplog, mocker): assert cud.call_count == 1 assert csf.call_count == 1 - assert len(caplog.record_tuples) == 0 def test_start_new_strategy(mocker, caplog): diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 60d9c3ea5..7e3c1f077 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -2,15 +2,17 @@ from unittest.mock import MagicMock import pytest from arrow import Arrow -from pandas import DataFrame, DateOffset, to_datetime +from pandas import DataFrame, DateOffset, to_datetime, Timestamp from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, + analyze_trade_parallelism, + calculate_max_drawdown, combine_tickers_with_mean, create_cum_profit, extract_trades_of_period, load_backtest_data, load_trades, - load_trades_from_db, analyze_trade_parallelism) + load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from tests.test_persistence import create_mock_trades @@ -163,3 +165,17 @@ def test_create_cum_profit1(testdatadir): assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 + + +def test_calculate_max_drawdown(testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + drawdown, h, low = calculate_max_drawdown(bt_data) + assert isinstance(drawdown, float) + assert pytest.approx(drawdown) == 0.21142322 + assert isinstance(h, Timestamp) + assert isinstance(low, Timestamp) + assert h == Timestamp('2018-01-24 14:25:00', tz='UTC') + assert low == Timestamp('2018-01-30 04:45:00', tz='UTC') + with pytest.raises(ValueError, match='Trade dataframe empty.'): + drawdown, h, low = calculate_max_drawdown(DataFrame()) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 179566bb0..6bec53d49 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -511,6 +511,22 @@ def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): Exchange(default_conf) +def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker, caplog): + api_mock = MagicMock() + default_conf['stake_currency'] = '' + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'].append('HELLO-WORLD') api_mock = MagicMock() diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index e3212e0cd..0406157f6 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -369,6 +369,42 @@ def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results assert under > correct +def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 @@ -390,17 +426,27 @@ def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) def test_log_results_if_loss_improves(hyperopt, capsys) -> None: hyperopt.current_best_loss = 2 hyperopt.total_epochs = 2 + hyperopt.print_results( { - 'is_best': True, 'loss': 1, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + }, + 'total_profit': 0, 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) - 'results_explanation': 'foo.', - 'is_initial_point': False + 'is_initial_point': False, + 'is_best': True } ) out, err = capsys.readouterr() - assert ' 2/2: foo. Objective: 1.00000' in out + assert all(x in out + for x in ["Best", "2/2", " 1", "0.10%", "0.00100000 BTC (1.00%)", "20.0 m"]) def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: @@ -422,13 +468,11 @@ def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None hyperopt.trials = trials hyperopt.save_trials(final=True) - assert log_has("Saving 1 epoch.", caplog) assert log_has(f"1 epoch saved to '{trials_file}'.", caplog) mock_dump.assert_called_once() hyperopt.trials = trials + trials hyperopt.save_trials(final=True) - assert log_has("Saving 2 epochs.", caplog) assert log_has(f"2 epochs saved to '{trials_file}'.", caplog) @@ -466,8 +510,18 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', - 'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', + 'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + }, + }]) ) patch_exchange(mocker) # Co-test loading ticker-interval from strategy @@ -761,11 +815,23 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, - 'params_details': {'buy': {'mfi-value': None}, - 'sell': {'sell-mfi-value': None}, - 'roi': {}, 'stoploss': {'stoploss': None}, - 'trailing': {'trailing_stop': None}}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': { + 'buy': {'mfi-value': None}, + 'sell': {'sell-mfi-value': None}, + 'roi': {}, 'stoploss': {'stoploss': None}, + 'trailing': {'trailing_stop': None} + }, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -787,7 +853,11 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: parallel.assert_called_once() out, err = capsys.readouterr() - assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null}' in out # noqa: E501 + result_str = ( + '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi"' + ':{},"stoploss":null,"trailing_stop":null}' + ) + assert result_str in out # noqa: E501 assert dumper.called # Should be called twice, once for tickerdata, once to save evaluations assert dumper.call_count == 2 @@ -804,10 +874,22 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, - 'params_details': {'buy': {'mfi-value': None}, - 'sell': {'sell-mfi-value': None}, - 'roi': {}, 'stoploss': {'stoploss': None}}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': { + 'buy': {'mfi-value': None}, + 'sell': {'sell-mfi-value': None}, + 'roi': {}, 'stoploss': {'stoploss': None} + }, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -846,8 +928,18 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, - 'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -887,7 +979,16 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{ - 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}}]) + 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -965,7 +1066,17 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -1012,7 +1123,17 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 57e928cca..285ecaa02 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -22,14 +22,14 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' - ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------|-------:|---------------:|---------------:|-----------------:|' - '---------------:|:---------------|-------:|--------:|---------:|\n' + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' + ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |' ) assert generate_text_table(data={'ETH/BTC': {}}, stake_currency='BTC', max_open_trades=2, @@ -52,13 +52,13 @@ def test_generate_text_table_sell_reason(default_conf, mocker): ) result_str = ( - '| Sell Reason | Sells | Wins | Draws | Losses |' + '| Sell Reason | Sells | Wins | Draws | Losses |' ' Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' - '|:--------------|--------:|-------:|--------:|---------:|' - '---------------:|---------------:|-----------------:|---------------:|\n' - '| roi | 2 | 2 | 0 | 0 |' + '|---------------+---------+--------+---------+----------+' + '----------------+----------------+------------------+----------------|\n' + '| roi | 2 | 2 | 0 | 0 |' ' 15 | 30 | 0.6 | 15 |\n' - '| stop_loss | 1 | 0 | 0 | 1 |' + '| stop_loss | 1 | 0 | 0 | 1 |' ' -10 | -10 | -0.2 | -5 |' ) assert generate_text_table_sell_reason( @@ -95,14 +95,14 @@ def test_generate_text_table_strategy(default_conf, mocker): ) result_str = ( - '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' - ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------------|-------:|---------------:|---------------:|------' - '-----------:|---------------:|:---------------|-------:|--------:|---------:|\n' - '| TestStrategy1 | 3 | 20.00 | 60.00 | ' - ' 1.10000000 | 30.00 | 0:17:00 | 3 | 0 | 0 |\n' - '| TestStrategy2 | 3 | 30.00 | 90.00 | ' - ' 1.30000000 | 45.00 | 0:20:00 | 3 | 0 | 0 |' + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' + ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' + '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 |' + ' 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 |' + ' 45.00 | 0:20:00 | 3 | 0 | 0 |' ) assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str @@ -111,8 +111,7 @@ def test_generate_edge_table(edge_conf, mocker): results = {} results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - - assert generate_edge_table(results).count(':|') == 7 + assert generate_edge_table(results).count('+') == 7 assert generate_edge_table(results).count('| ETH/BTC |') == 1 assert generate_edge_table(results).count( '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 93b6f6058..6319ab9e6 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -51,13 +51,13 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_date_hum': ANY, 'close_date': None, 'close_date_hum': None, - 'open_rate': 1.099e-05, + 'open_rate': 1.098e-05, 'close_rate': None, - 'current_rate': 1.098e-05, - 'amount': 90.99181074, + 'current_rate': 1.099e-05, + 'amount': 91.07468124, 'stake_amount': 0.001, 'close_profit': None, - 'current_profit': -0.59, + 'current_profit': -0.41, 'stop_loss': 0.0, 'initial_stop_loss': 0.0, 'initial_stop_loss_pct': None, @@ -78,10 +78,10 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_date_hum': ANY, 'close_date': None, 'close_date_hum': None, - 'open_rate': 1.099e-05, + 'open_rate': 1.098e-05, 'close_rate': None, 'current_rate': ANY, - 'amount': 90.99181074, + 'amount': 91.07468124, 'stake_amount': 0.001, 'close_profit': None, 'current_profit': ANY, @@ -121,7 +121,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert "Pair" in headers assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] - assert '-0.59%' == result[0][3] + assert '-0.41%' == result[0][3] # Test with fiatconvert rpc._fiat_converter = CryptoToFiatConverter() @@ -130,7 +130,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert "Pair" in headers assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] - assert '-0.59% (-0.09)' == result[0][3] + assert '-0.41% (-0.06)' == result[0][3] mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) @@ -245,9 +245,9 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) - assert prec_satoshi(stats['profit_all_coin'], 5.632e-05) - assert prec_satoshi(stats['profit_all_percent'], 2.81) - assert prec_satoshi(stats['profit_all_fiat'], 0.8448) + assert prec_satoshi(stats['profit_all_coin'], 5.802e-05) + assert prec_satoshi(stats['profit_all_percent'], 2.89) + assert prec_satoshi(stats['profit_all_fiat'], 0.8703) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' @@ -668,7 +668,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None trade = rpc._rpc_forcebuy(pair, None) assert isinstance(trade, Trade) assert trade.pair == pair - assert trade.open_rate == ticker()['ask'] + assert trade.open_rate == ticker()['bid'] # Test buy duplicate with pytest.raises(RPCException, match=r'position for ETH/BTC already open - id: 1'): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 25c971bf7..e0abd886d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -426,20 +426,20 @@ def test_api_status(botclient, mocker, ticker, fee, markets): rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) assert len(rc.json) == 1 - assert rc.json == [{'amount': 90.99181074, + assert rc.json == [{'amount': 91.07468124, 'base_currency': 'BTC', 'close_date': None, 'close_date_hum': None, 'close_profit': None, 'close_rate': None, - 'current_profit': -0.59, - 'current_rate': 1.098e-05, + 'current_profit': -0.41, + 'current_rate': 1.099e-05, 'initial_stop_loss': 0.0, 'initial_stop_loss_pct': None, 'open_date': ANY, 'open_date_hum': 'just now', 'open_order': '(limit buy rem=0.00000000)', - 'open_rate': 1.099e-05, + 'open_rate': 1.098e-05, 'pair': 'ETH/BTC', 'stake_amount': 0.001, 'stop_loss': 0.0, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 1ac03f812..cea863ac8 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -720,13 +720,13 @@ def test_forcesell_handle(default_conf, update, ticker, fee, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'profit', - 'limit': 1.172e-05, - 'amount': 90.99181073703367, + 'limit': 1.173e-05, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.172e-05, - 'profit_amount': 6.126e-05, - 'profit_ratio': 0.0611052, + 'open_rate': 1.098e-05, + 'current_rate': 1.173e-05, + 'profit_amount': 6.314e-05, + 'profit_ratio': 0.0629778, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.FORCE_SELL.value, @@ -779,13 +779,13 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'loss', - 'limit': 1.044e-05, - 'amount': 90.99181073703367, + 'limit': 1.043e-05, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -5.492e-05, - 'profit_ratio': -0.05478342, + 'open_rate': 1.098e-05, + 'current_rate': 1.043e-05, + 'profit_amount': -5.497e-05, + 'profit_ratio': -0.05482878, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.FORCE_SELL.value, @@ -827,13 +827,13 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'loss', - 'limit': 1.098e-05, - 'amount': 90.99181073703367, + 'limit': 1.099e-05, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.098e-05, - 'profit_amount': -5.91e-06, - 'profit_ratio': -0.00589291, + 'open_rate': 1.098e-05, + 'current_rate': 1.099e-05, + 'profit_amount': -4.09e-06, + 'profit_ratio': -0.00408133, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.FORCE_SELL.value, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 61f69bd85..9f6e5ef0c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -761,8 +761,8 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'bittrex' - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073703367 + assert trade.open_rate == 0.00001098 + assert trade.amount == 91.07468123861567 assert log_has( 'Buy signal found: about create a new trade with stake_amount: 0.001 ...', caplog @@ -906,20 +906,37 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0] -@pytest.mark.parametrize("ask,last,last_ab,expected", [ - (20, 10, 0.0, 20), # Full ask side - (20, 10, 1.0, 10), # Full last side - (20, 10, 0.5, 15), # Between ask and last - (20, 10, 0.7, 13), # Between ask and last - (20, 10, 0.3, 17), # Between ask and last - (5, 10, 1.0, 5), # last bigger than ask - (5, 10, 0.5, 5), # last bigger than ask +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('bid', 10, 20, 10, 0.0, 20), # Full bid side + ('bid', 10, 20, 10, 1.0, 10), # Full last side + ('bid', 10, 20, 10, 0.5, 15), # Between bid and last + ('bid', 10, 20, 10, 0.7, 13), # Between bid and last + ('bid', 10, 20, 10, 0.3, 17), # Between bid and last + ('bid', 4, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 4, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 10, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 4, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 4, 5, None, 1, 5), # last not available - uses bid + ('bid', 4, 5, None, 0, 5), # last not available - uses bid ]) -def test_get_buy_rate(mocker, default_conf, caplog, ask, last, last_ab, expected) -> None: +def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected) -> None: default_conf['bid_strategy']['ask_last_balance'] = last_ab + default_conf['bid_strategy']['price_side'] = side freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={'ask': ask, 'last': last})) + MagicMock(return_value={'ask': ask, 'last': last, 'bid': bid})) assert freqtrade.get_buy_rate('ETH/BTC', True) == expected assert not log_has("Using cached buy rate for ETH/BTC.", caplog) @@ -1317,7 +1334,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, stoploss_order_mock.assert_not_called() assert freqtrade.handle_trade(trade) is False - assert trade.stop_loss == 0.00002344 * 0.95 + assert trade.stop_loss == 0.00002346 * 0.95 # setting stoploss_on_exchange_interval to 0 seconds freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 0 @@ -1325,10 +1342,10 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') - stoploss_order_mock.assert_called_once_with(amount=85.25149190110828, + stoploss_order_mock.assert_called_once_with(amount=85.32423208191126, pair='ETH/BTC', order_types=freqtrade.strategy.order_types, - stop_price=0.00002344 * 0.95) + stop_price=0.00002346 * 0.95) # price fell below stoploss, so dry-run sells trade. mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -1510,12 +1527,12 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False # stoploss should be set to 1% as trailing is on - assert trade.stop_loss == 0.00002344 * 0.99 + assert trade.stop_loss == 0.00002346 * 0.99 cancel_order_mock.assert_called_once_with(100, 'NEO/BTC') - stoploss_order_mock.assert_called_once_with(amount=2131074.168797954, + stoploss_order_mock.assert_called_once_with(amount=2132892.491467577, pair='NEO/BTC', order_types=freqtrade.strategy.order_types, - stop_price=0.00002344 * 0.99) + stop_price=0.00002346 * 0.99) def test_enter_positions(mocker, default_conf, caplog) -> None: @@ -2292,12 +2309,12 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N 'pair': 'ETH/BTC', 'gain': 'profit', 'limit': 1.172e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.172e-05, - 'profit_amount': 6.126e-05, - 'profit_ratio': 0.0611052, + 'open_rate': 1.098e-05, + 'current_rate': 1.173e-05, + 'profit_amount': 6.223e-05, + 'profit_ratio': 0.0620716, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value, @@ -2341,12 +2358,12 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.044e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -5.492e-05, - 'profit_ratio': -0.05478342, + 'open_rate': 1.098e-05, + 'current_rate': 1.043e-05, + 'profit_amount': -5.406e-05, + 'profit_ratio': -0.05392257, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, @@ -2397,12 +2414,12 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.08801e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -1.498e-05, - 'profit_ratio': -0.01493766, + 'open_rate': 1.098e-05, + 'current_rate': 1.043e-05, + 'profit_amount': -1.408e-05, + 'profit_ratio': -0.01404051, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, @@ -2587,7 +2604,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) assert not trade.is_open - assert trade.close_profit == 0.0611052 + assert trade.close_profit == 0.0620716 assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -2597,12 +2614,12 @@ def test_execute_sell_market_order(default_conf, ticker, fee, 'pair': 'ETH/BTC', 'gain': 'profit', 'limit': 1.172e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'market', - 'open_rate': 1.099e-05, - 'current_rate': 1.172e-05, - 'profit_amount': 6.126e-05, - 'profit_ratio': 0.0611052, + 'open_rate': 1.098e-05, + 'current_rate': 1.173e-05, + 'profit_amount': 6.223e-05, + 'profit_ratio': 0.0620716, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value, @@ -3624,13 +3641,20 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order assert freqtrade.handle_trade(trade) is True -def test_get_sell_rate(default_conf, mocker, caplog, ticker, order_book_l2) -> None: - - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_order_book=order_book_l2, - fetch_ticker=ticker, - ) +@pytest.mark.parametrize('side,ask,bid,expected', [ + ('bid', 10.0, 11.0, 11.0), + ('bid', 10.0, 11.2, 11.2), + ('bid', 10.0, 11.0, 11.0), + ('bid', 9.8, 11.0, 11.0), + ('bid', 0.0001, 0.002, 0.002), + ('ask', 10.0, 11.0, 10.0), + ('ask', 10.11, 11.2, 10.11), + ('ask', 0.001, 0.002, 0.001), + ('ask', 0.006, 1.0, 0.006), +]) +def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, expected) -> None: + default_conf['ask_strategy']['price_side'] = side + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'bid': bid}) pair = "ETH/BTC" # Test regular mode @@ -3638,25 +3662,33 @@ def test_get_sell_rate(default_conf, mocker, caplog, ticker, order_book_l2) -> N rate = ft.get_sell_rate(pair, True) assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) - assert rate == 0.00001098 + assert rate == expected # Use caching rate = ft.get_sell_rate(pair, False) - assert rate == 0.00001098 + assert rate == expected assert log_has("Using cached sell rate for ETH/BTC.", caplog) - caplog.clear() +@pytest.mark.parametrize('side,expected', [ + ('bid', 0.043936), # Value from order_book_l2 fiture - bids side + ('ask', 0.043949), # Value from order_book_l2 fiture - asks side +]) +def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): # Test orderbook mode + default_conf['ask_strategy']['price_side'] = side default_conf['ask_strategy']['use_order_book'] = True default_conf['ask_strategy']['order_book_min'] = 1 default_conf['ask_strategy']['order_book_max'] = 2 + # TODO: min/max is irrelevant for this test until refactoring + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) - assert rate == 0.043936 + assert rate == expected rate = ft.get_sell_rate(pair, False) - assert rate == 0.043936 + assert rate == expected assert log_has("Using cached sell rate for ETH/BTC.", caplog) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 34d1f2b0c..dd04035b7 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -3,15 +3,16 @@ from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock +import pandas as pd import plotly.graph_objects as go import pytest from plotly.subplots import make_subplots +from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.exceptions import OperationalException -from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.plot.plotting import (add_indicators, add_profit, create_plotconfig, generate_candlestick_graph, @@ -266,6 +267,7 @@ def test_generate_profit_graph(testdatadir): trades = load_backtest_data(filename) timerange = TimeRange.parse_timerange("20180110-20180112") pairs = ["TRX/BTC", "ADA/BTC"] + trades = trades[trades['close_time'] < pd.Timestamp('2018-01-12', tz='UTC')] tickers = history.load_data(datadir=testdatadir, pairs=pairs, @@ -283,13 +285,15 @@ def test_generate_profit_graph(testdatadir): assert fig.layout.yaxis3.title.text == "Profit" figure = fig.layout.figure - assert len(figure.data) == 4 + assert len(figure.data) == 5 avgclose = find_trace_in_fig_data(figure.data, "Avg close price") assert isinstance(avgclose, go.Scatter) profit = find_trace_in_fig_data(figure.data, "Profit") assert isinstance(profit, go.Scatter) + profit = find_trace_in_fig_data(figure.data, "Max drawdown 0.00%") + assert isinstance(profit, go.Scatter) for pair in pairs: profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}") diff --git a/tests/test_worker.py b/tests/test_worker.py index 7b446ac6a..839f7cdac 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -60,7 +60,7 @@ def test_throttle(mocker, default_conf, caplog) -> None: assert result == 42 assert end - start > 0.1 - assert log_has_re(r"Throttling with 'throttled_func\(\)': sleep for 0\.10 s.*", caplog) + assert log_has_re(r"Throttling with 'throttled_func\(\)': sleep for \d\.\d{2} s.*", caplog) result = worker._throttle(throttled_func, throttle_secs=-1) assert result == 42