diff --git a/config_binance.json.example b/config_binance.json.example index 7d616fe91..8dc6f5c3a 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -37,18 +37,21 @@ "rateLimit": 200 }, "pair_whitelist": [ - "AST/BTC", - "ETC/BTC", - "ETH/BTC", + "ALGO/BTC", + "ATOM/BTC", + "BAT/BTC", + "BCH/BTC", + "BRD/BTC", "EOS/BTC", + "ETH/BTC", "IOTA/BTC", + "LINK/BTC", "LTC/BTC", - "MTH/BTC", - "NCASH/BTC", - "TNT/BTC", + "NEO/BTC", + "NXS/BTC", "XMR/BTC", - "XLM/BTC", - "XRP/BTC" + "XRP/BTC", + "XTZ/BTC" ], "pair_blacklist": [ "BNB/BTC" diff --git a/config_kraken.json.example b/config_kraken.json.example index 854aeef71..401ee7c9f 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -38,9 +38,26 @@ "rateLimit": 1000 }, "pair_whitelist": [ - "ETH/EUR", + "ADA/EUR", + "ATOM/EUR", + "BAT/EUR", + "BCH/EUR", "BTC/EUR", - "BCH/EUR" + "DAI/EUR", + "DASH/EUR", + "EOS/EUR", + "ETC/EUR", + "ETH/EUR", + "LINK/EUR", + "LTC/EUR", + "QTUM/EUR", + "REP/EUR", + "WAVES/EUR", + "XLM/EUR", + "XMR/EUR", + "XRP/EUR", + "XTZ/EUR", + "ZEC/EUR" ], "pair_blacklist": [ diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md new file mode 100644 index 000000000..20af0aaab --- /dev/null +++ b/docs/advanced-hyperopt.md @@ -0,0 +1,63 @@ +# Advanced Hyperopt + +This page explains some advanced Hyperopt topics that may require higher +coding skills and Python knowledge than creation of an ordinal hyperoptimization +class. + +## Creating and using a custom loss function + +To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class. +For the sample below, you then need to add the command line parameter `--hyperopt-loss SuperDuperHyperOptLoss` to your hyperopt call so this function is being used. + +A sample of this can be found below, which is identical to the Default Hyperopt loss implementation. A full sample can be found in [userdata/hyperopts](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_loss.py). + +``` python +from freqtrade.optimize.hyperopt import IHyperOptLoss + +TARGET_TRADES = 600 +EXPECTED_MAX_PROFIT = 3.0 +MAX_ACCEPTED_TRADE_DURATION = 300 + +class SuperDuperHyperOptLoss(IHyperOptLoss): + """ + Defines the default loss function for hyperopt + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for better results + This is the legacy algorithm (used until now in freqtrade). + Weights are distributed as follows: + * 0.4 to trade duration + * 0.25: Avoiding trade loss + * 1.0 to total profit, compared to the expected value (`EXPECTED_MAX_PROFIT`) defined above + """ + total_profit = results.profit_percent.sum() + trade_duration = results.trade_duration.mean() + + trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8) + profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) + duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) + result = trade_loss + profit_loss + duration_loss + return result +``` + +Currently, the arguments are: + +* `results`: DataFrame containing the result + The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`): + `pair, profit_percent, profit_abs, open_time, close_time, open_index, close_index, trade_duration, open_at_end, open_rate, close_rate, sell_reason` +* `trade_count`: Amount of trades (identical to `len(results)`) +* `min_date`: Start date of the hyperopting TimeFrame +* `min_date`: End date of the hyperopting TimeFrame + +This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. + +!!! Note + This function is called once per iteration - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. + +!!! Note + Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later. diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 97d52850c..2d3fe36f5 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -34,3 +34,59 @@ as the watchdog. !!! Note The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. + +## Advanced Logging + +On many Linux systems the bot can be configured to send its log messages to `syslog` or `journald` system services. Logging to a remote `syslog` server is also available on Windows. The special values for the `--logfilename` command line option can be used for this. + +### Logging to syslog + +To send Freqtrade log messages to a local or remote `syslog` service use the `--logfilename` command line option with the value in the following format: + +* `--logfilename syslog:` -- send log messages to `syslog` service using the `` as the syslog address. + +The syslog address can be either a Unix domain socket (socket filename) or a UDP socket specification, consisting of IP address and UDP port, separated by the `:` character. + +So, the following are the examples of possible usages: + +* `--logfilename syslog:/dev/log` -- log to syslog (rsyslog) using the `/dev/log` socket, suitable for most systems. +* `--logfilename syslog` -- same as above, the shortcut for `/dev/log`. +* `--logfilename syslog:/var/run/syslog` -- log to syslog (rsyslog) using the `/var/run/syslog` socket. Use this on MacOS. +* `--logfilename syslog:localhost:514` -- log to local syslog using UDP socket, if it listens on port 514. +* `--logfilename syslog::514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to an external syslog server. + +Log messages are send to `syslog` with the `user` facility. So you can see them with the following commands: + +* `tail -f /var/log/user`, or +* install a comprehensive graphical viewer (for instance, 'Log File Viewer' for Ubuntu). + +On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. + +For `rsyslog` the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add +``` +if $programname startswith "freqtrade" then -/var/log/freqtrade.log +``` +to one of the rsyslog configuration files, for example at the end of the `/etc/rsyslog.d/50-default.conf`. + +For `syslog` (`rsyslog`), the reduction mode can be switched on. This will reduce the number of repeating messages. For instance, multiple bot Heartbeat messages will be reduced to a single message when nothing else happens with the bot. To achieve this, set in `/etc/rsyslog.conf`: +``` +# Filter duplicated messages +$RepeatedMsgReduction on +``` + +### Logging to journald + +This needs the `systemd` python package installed as the dependency, which is not available on Windows. Hence, the whole journald logging functionality is not available for a bot running on Windows. + +To send Freqtrade log messages to `journald` system service use the `--logfilename` command line option with the value in the following format: + +* `--logfilename journald` -- send log messages to `journald`. + +Log messages are send to `journald` with the `user` facility. So you can see them with the following commands: + +* `journalctl -f` -- shows Freqtrade log messages sent to `journald` along with other log messages fetched by `journald`. +* `journalctl -f -u freqtrade.service` -- this command can be used when the bot is run as a `systemd` service. + +There are many other options in the `journalctl` utility to filter the messages, see manual pages for this utility. + +On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. diff --git a/docs/backtesting.md b/docs/backtesting.md index 19814303b..68782bb9c 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -11,14 +11,15 @@ Now you have good Buy and Sell strategies and some historic data, you want to te real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pairs) from your config file -and load ticker data from `user_data/data/` by default. -If no data is available for the exchange / pair / ticker interval combination, backtesting will -ask you to download them first using `freqtrade download-data`. +Backtesting will use the crypto-currencies (pairs) from your config file and load ticker data from `user_data/data/` by default. +If no data is available for the exchange / pair / ticker interval combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. The result of backtesting will confirm if your bot has better odds of making a profit than a loss. +!!! Tip "Using dynamic pairlists for backtesting" + While using dynamic pairlists during backtesting is not possible, a dynamic pairlist using current data can be generated via the [`test-pairlist`](utils.md#test-pairlist) command, and needs to be specified as `"pair_whitelist"` attribute in the configuration. + ### Run a backtesting against the currencies listed in your config file #### With 5 min tickers (Per default) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4665878d4..25818aea6 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -89,9 +89,9 @@ The bot allows you to use multiple configuration files by specifying multiple defined in the latter configuration files override parameters with the same name defined in the previous configuration files specified in the command line earlier. -For example, you can make a separate configuration file with your key and secrete +For example, you can make a separate configuration file with your key and secret for the Exchange you use for trading, specify default configuration file with -empty key and secrete values while running in the Dry Mode (which does not actually +empty key and secret values while running in the Dry Mode (which does not actually require them): ```bash @@ -104,7 +104,7 @@ and specify both configuration files when running in the normal Live Trade Mode: freqtrade trade -c ./config.json -c path/to/secrets/keys.config.json ``` -This could help you hide your private Exchange key and Exchange secrete on you local machine +This could help you hide your private Exchange key and Exchange secret on you local machine by setting appropriate file permissions for the file which contains actual secrets and, additionally, prevent unintended disclosure of sensitive private data when you publish examples of your configuration in the project issues or in the Internet. diff --git a/docs/configuration.md b/docs/configuration.md index f913d7dbb..5ad1a886e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,84 +38,92 @@ The prevelance for all Options is as follows: Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. -| Command | Default | Description | -|----------|---------|-------------| -| `max_open_trades` | 3 | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades) -| `stake_currency` | BTC | **Required.** Crypto-currency used for trading. -| `stake_amount` | 0.05 | **Required.** Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to `"unlimited"` to allow the bot to use all available balance. -| `amount_reserve_percent` | 0.05 | Reserve some amount in min pair stake amount. Default is 5%. The bot will reserve `amount_reserve_percent` + stop-loss value when calculating min pair stake amount in order to avoid possible trade refusals. -| `ticker_interval` | [1m, 5m, 15m, 30m, 1h, 1d, ...] | The ticker interval to use (1min, 5 min, 15 min, 30 min, 1 hour or 1 day). Default is 5 minutes. [Strategy Override](#parameters-in-the-strategy). -| `fiat_display_currency` | USD | **Required.** Fiat currency used to show your profits. More information below. -| `dry_run` | true | **Required.** Define if the bot must be in Dry-run or production mode. -| `dry_run_wallet` | 999.9 | Overrides the default amount of 999.9 stake currency units in the wallet used by the bot running in the Dry Run mode if you need it for any reason. -| `process_only_new_candles` | false | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy). -| `minimal_roi` | See below | Set the threshold in percent the bot will use to sell a trade. More information below. [Strategy Override](#parameters-in-the-strategy). -| `stoploss` | -0.10 | Value of the stoploss in percent used by the bot. More information below. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). -| `trailing_stop` | false | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). -| `trailing_stop_positive` | 0 | Changes stop-loss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). -| `trailing_stop_positive_offset` | 0 | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). -| `trailing_only_offset_is_reached` | false | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). -| `unfilledtimeout.buy` | 10 | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. -| `unfilledtimeout.sell` | 10 | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. -| `bid_strategy.ask_last_balance` | 0.0 | **Required.** Set the bidding price. More information [below](#understand-ask_last_balance). -| `bid_strategy.use_order_book` | false | Allows buying of pair using the rates in Order Book Bids. -| `bid_strategy.order_book_top` | 0 | Bot will use the top N rate in Order Book Bids. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. -| `bid_strategy. check_depth_of_market.enabled` | false | Does not buy if the % difference of buy orders and sell orders is met in Order Book. -| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | 0 | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. -| `ask_strategy.use_order_book` | false | Allows selling of open traded pair using the rates in Order Book Asks. -| `ask_strategy.order_book_min` | 0 | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. -| `ask_strategy.order_book_max` | 0 | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. -| `ask_strategy.use_sell_signal` | true | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy). -| `ask_strategy.sell_profit_only` | false | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy). -| `ask_strategy.ignore_roi_if_buy_signal` | false | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy). -| `order_types` | None | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy). -| `order_time_in_force` | None | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). -| `exchange.name` | | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). -| `exchange.sandbox` | false | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details. -| `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** -| `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** -| `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. ***Keep it in secrete, do not disclose publicly.*** -| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)). -| `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)). -| `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) -| `exchange.ccxt_async_config` | None | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) -| `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded. -| `edge` | false | Please refer to [edge configuration document](edge.md) for detailed explanation. -| `experimental.block_bad_exchanges` | true | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now. -| `pairlists` | StaticPairList | Define one or more pairlists to be used. [More information below](#dynamic-pairlists). -| `telegram.enabled` | true | **Required.** Enable or not the usage of Telegram. -| `telegram.token` | token | Your Telegram bot token. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** -| `telegram.chat_id` | chat_id | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. ***Keep it in secrete, do not disclose publicly.*** -| `webhook.enabled` | false | Enable usage of Webhook notifications -| `webhook.url` | false | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. -| `webhook.webhookbuy` | false | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. -| `webhook.webhooksell` | false | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. -| `webhook.webhookstatus` | false | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. -| `db_url` | `sqlite:///tradesv3.sqlite`| Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`. -| `initial_state` | running | Defines the initial application state. More information below. -| `forcebuy_enable` | false | Enables the RPC Commands to force a buy. More information below. -| `strategy` | None | **Required** Defines Strategy class to use. Recommended to set via `--strategy NAME`. -| `strategy_path` | null | Adds an additional strategy lookup path (must be a directory). -| `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. -| `internals.heartbeat_interval` | 60 | Print heartbeat message every X seconds. Set to 0 to disable heartbeat messages. -| `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. -| `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. -| `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`. +| Command | Description | +|----------|-------------| +| `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades).
***Datatype:*** *Positive integer or -1.* +| `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* +| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#understand-stake_amount). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Positive float or `"unlimited"`.* +| `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
***Datatype:*** *Positive Float as ratio.* +| `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* +| `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
***Datatype:*** *String* +| `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
***Datatype:*** *Boolean* +| `dry_run_wallet` | Overrides the default amount of 999.9 stake currency units in the wallet used by the bot running in the Dry Run mode if you need it for any reason.
***Datatype:*** *Float* +| `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* +| `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float (as ratio)* +| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Boolean* +| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float* +| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
***Datatype:*** *Float* +| `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* +| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* +| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#understand-ask_last_balance). +| `bid_strategy.use_order_book` | Enable buying using the rates in Order Book Bids.
***Datatype:*** *Boolean* +| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. *Defaults to `1`.*
***Datatype:*** *Positive Integer* +| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book.
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. *Defaults to `0`.*
***Datatype:*** *Float (as ratio)* +| `ask_strategy.use_order_book` | Enable selling of open trades using Order Book Asks.
***Datatype:*** *Boolean* +| `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* +| `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* +| `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
***Datatype:*** *Boolean* +| `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* +| `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* +| `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* +| `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
***Datatype:*** *String* +| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
***Datatype:*** *Boolean* +| `exchange.key` | API key to use for the exchange. Only required when you are in production mode. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)).
***Datatype:*** *List* +| `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)).
***Datatype:*** *List* +| `exchange.ccxt_config` | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* +| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* +| `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
***Datatype:*** *Positive Integer* +| `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. +| `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
***Datatype:*** *Boolean* +| `pairlists` | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
*Defaults to `StaticPairList`.*
***Datatype:*** *List of Dicts* +| `telegram.enabled` | Enable the usage of Telegram.
***Datatype:*** *Boolean* +| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `webhook.enabled` | Enable usage of Webhook notifications
***Datatype:*** *Boolean* +| `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* +| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* +| `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Boolean* +| `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *IPv4* +| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* +| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* +| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
***Datatype:*** *String, SQLAlchemy connect string* +| `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
***Datatype:*** *Enum, either `stopped` or `running`* +| `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
***Datatype:*** *Boolean* +| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
***Datatype:*** *ClassName* +| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
***Datatype:*** *String* +| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
***Datatype:*** *Positive Integer* +| `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
***Datatype:*** *Positive Integer or 0* +| `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
***Datatype:*** *Boolean* +| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
***Datatype:*** *String* +| `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
***Datatype:*** *String* ### Parameters in the strategy The following parameters can be set in either configuration file or strategy. Values set in the configuration file always overwrite values set in the strategy. -* `ticker_interval` * `minimal_roi` +* `ticker_interval` * `stoploss` * `trailing_stop` * `trailing_stop_positive` * `trailing_stop_positive_offset` +* `trailing_only_offset_is_reached` * `process_only_new_candles` * `order_types` * `order_time_in_force` +* `stake_currency` +* `stake_amount` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) @@ -123,15 +131,19 @@ Values set in the configuration file always overwrite values set in the strategy ### Understand stake_amount The `stake_amount` configuration parameter is an amount of crypto-currency your bot will use for each trade. -The minimal value is 0.0005. If there is not enough crypto-currency in -the account an exception is generated. + +The minimal configuration value is 0.0001. Please check your exchange's trading minimums to avoid problems. + +This setting works in combination with `max_open_trades`. The maximum capital engaged in trades is `stake_amount * max_open_trades`. +For example, the bot will at most use (0.05 BTC x 3) = 0.15 BTC, assuming a configuration of `max_open_trades=3` and `stake_amount=0.05`. + To allow the bot to trade all the available `stake_currency` in your account set ```json "stake_amount" : "unlimited", ``` -In this case a trade amount is calclulated as: +In this case a trade amount is calculated as: ```python currency_balance / (max_open_trades - current_open_trades) @@ -393,6 +405,9 @@ Inactive markets and blacklisted pairs are always removed from the resulting `pa * [`PrecisionFilter`](#precision-filter) * [`PriceFilter`](#price-pair-filter) +!!! Tip "Testing pairlists" + Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) subcommand to test your configuration quickly. + #### Static Pair List By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. @@ -472,7 +487,7 @@ creating trades on the exchange. "db_url": "sqlite:///tradesv3.dryrun.sqlite", ``` -3. Remove your Exchange API key and secrete (change them by empty values or fake credentials): +3. Remove your Exchange API key and secret (change them by empty values or fake credentials): ```json "exchange": { diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 5a3ae7e3a..0661896a0 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -46,8 +46,9 @@ Optional - can also be loaded from a strategy: Rarely you may also need to override: * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) -* `generate_roi_table` - for custom ROI optimization (if you need more than 4 entries in the ROI table) +* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) * `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) +* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) ### 1. Install a Custom Hyperopt File @@ -182,63 +183,7 @@ Currently, the following loss functions are builtin: * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) * `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on the trade returns) -### Creating and using a custom loss function - -To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class. -For the sample below, you then need to add the command line parameter `--hyperopt-loss SuperDuperHyperOptLoss` to your hyperopt call so this fuction is being used. - -A sample of this can be found below, which is identical to the Default Hyperopt loss implementation. A full sample can be found [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_loss.py) - -``` python -from freqtrade.optimize.hyperopt import IHyperOptLoss - -TARGET_TRADES = 600 -EXPECTED_MAX_PROFIT = 3.0 -MAX_ACCEPTED_TRADE_DURATION = 300 - -class SuperDuperHyperOptLoss(IHyperOptLoss): - """ - Defines the default loss function for hyperopt - """ - - @staticmethod - def hyperopt_loss_function(results: DataFrame, trade_count: int, - min_date: datetime, max_date: datetime, - *args, **kwargs) -> float: - """ - Objective function, returns smaller number for better results - This is the legacy algorithm (used until now in freqtrade). - Weights are distributed as follows: - * 0.4 to trade duration - * 0.25: Avoiding trade loss - * 1.0 to total profit, compared to the expected value (`EXPECTED_MAX_PROFIT`) defined above - """ - total_profit = results.profit_percent.sum() - trade_duration = results.trade_duration.mean() - - trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8) - profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) - result = trade_loss + profit_loss + duration_loss - return result -``` - -Currently, the arguments are: - -* `results`: DataFrame containing the result - The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`): - `pair, profit_percent, profit_abs, open_time, close_time, open_index, close_index, trade_duration, open_at_end, open_rate, close_rate, sell_reason` -* `trade_count`: Amount of trades (identical to `len(results)`) -* `min_date`: Start date of the hyperopting TimeFrame -* `min_date`: End date of the hyperopting TimeFrame - -This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. - -!!! Note - This function is called once per iteration - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. - -!!! Note - Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later. +Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. ## Execute Hyperopt @@ -253,10 +198,10 @@ freqtrade hyperopt --config config.json --hyperopt -e 5000 --spac Use `` as the name of the custom hyperopt used. -The `-e` flag will set how many evaluations hyperopt will do. We recommend +The `-e` option will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. -The `--spaces all` flag determines that all possible parameters should be optimized. Possibilities are listed below. +The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. !!! Note By default, hyperopt will erase previous results and start from scratch. Continuation can be archived by using `--continue`. @@ -289,7 +234,7 @@ freqtrade hyperopt --strategy SampleStrategy --customhyperopt SampleHyperopt ### Running Hyperopt with Smaller Search Space -Use the `--spaces` argument to limit the search space used by hyperopt. +Use the `--spaces` option to limit the search space used by hyperopt. Letting Hyperopt optimize everything is a huuuuge search space. Often it might make more sense to start by just searching for initial buy algorithm. Or maybe you just want to optimize your stoploss or roi table for that awesome @@ -302,8 +247,12 @@ Legal values are: * `sell`: just search for a new sell strategy * `roi`: just optimize the minimal profit table for your strategy * `stoploss`: search for the best stoploss value +* `trailing`: search for the best trailing stop values +* `default`: `all` except `trailing` * space-separated list of any of the above values for example `--spaces roi stoploss` +The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy. + ### Position stacking and disabling max market positions In some situations, you may need to run Hyperopt (and Backtesting) with the @@ -377,19 +326,13 @@ You can use the `--print-all` command line option if you would like to see all r ### Understand Hyperopt ROI results -If you are optimizing ROI (i.e. if optimization search-space contains 'all' or 'roi'), your result will look as follows and include a ROI table: +If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table: ``` Best result: 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 -Buy hyperspace params: -{ 'adx-value': 44, - 'rsi-value': 29, - 'adx-enabled': False, - 'rsi-enabled': True, - 'trigger': 'bb_lower'} ROI table: { 0: 0.10674, 21: 0.09158, @@ -413,7 +356,7 @@ As stated in the comment, you can also use it as the value of the `minimal_roi` #### Default ROI Search Space -If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values can vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point): +If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point): | # step | 1m | | 5m | | 1h | | 1d | | |---|---|---|---|---|---|---|---|---| @@ -430,7 +373,7 @@ Override the `roi_space()` method if you need components of the ROI tables to va ### Understand Hyperopt Stoploss results -If you are optimizing stoploss values (i.e. if optimization search-space contains 'all' or 'stoploss'), your result will look as follows and include stoploss: +If you are optimizing stoploss values (i.e. if optimization search-space contains 'all', 'default' or 'stoploss'), your result will look as follows and include stoploss: ``` Best result: @@ -457,12 +400,46 @@ As stated in the comment, you can also use it as the value of the `stoploss` set #### Default Stoploss Search Space -If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimization hyperspace for you. By default, the stoploss values in that hyperspace can vary in the range -0.35...-0.02, which is sufficient in most cases. +If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimization hyperspace for you. By default, the stoploss values in that hyperspace vary in the range -0.35...-0.02, which is sufficient in most cases. If you have the `stoploss_space()` method in your custom hyperopt file, remove it in order to utilize Stoploss hyperoptimization space generated by Freqtrade by default. Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +### Understand Hyperopt Trailing Stop results + +If you are optimizing trailing stop values (i.e. if optimization search-space contains 'all' or 'trailing'), your result will look as follows and include trailing stop parameters: + +``` +Best result: + + 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161 + +Trailing stop: +{ 'trailing_only_offset_is_reached': True, + 'trailing_stop': True, + 'trailing_stop_positive': 0.02001, + 'trailing_stop_positive_offset': 0.06038} +``` + +In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy: + +``` + # Trailing stop + # These attributes will be overridden if the config file contains corresponding values. + trailing_stop = True + trailing_stop_positive = 0.02001 + trailing_stop_positive_offset = 0.06038 + trailing_only_offset_is_reached = True +``` +As stated in the comment, you can also use it as the values of the corresponding settings in the configuration file. + +#### Default Trailing Stop Search Space + +If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges 0.02...0.35 and 0.01...0.1 correspondingly, which is sufficient in most cases. + +Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py). + ### Validate backtesting results Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected. diff --git a/docs/installation.md b/docs/installation.md index 411441aa2..27b7a94c5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -201,7 +201,7 @@ freqtrade trade -c config.json #### 7. (Optional) Post-installation Tasks -On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details. +On Linux, as an optional post-installation task, you may wish to setup the bot to run as a `systemd` service or configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Logging](advanced-setup.md#advanced-logging) for details. ------ diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index c43d8e3f6..4efca7d2f 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -1,4 +1,4 @@ -# Optimization +# Strategy Customization This page explains where to customize your strategies, and add new indicators. diff --git a/docs/utils.md b/docs/utils.md index ca4b645a5..4ad32055a 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -43,20 +43,6 @@ The file will be named inline with your class name, and will not overwrite exist Results will be located in `user_data/strategies/.py`. -### Sample usage of new-strategy - -```bash -freqtrade new-strategy --strategy AwesomeStrategy -``` - -With custom user directory - -```bash -freqtrade new-strategy --userdir ~/.freqtrade/ --strategy AwesomeStrategy -``` - -### new-strategy complete options - ``` output usage: freqtrade new-strategy [-h] [--userdir PATH] [-s NAME] [--template {full,minimal}] @@ -75,6 +61,18 @@ optional arguments: ``` +### Sample usage of new-strategy + +```bash +freqtrade new-strategy --strategy AwesomeStrategy +``` + +With custom user directory + +```bash +freqtrade new-strategy --userdir ~/.freqtrade/ --strategy AwesomeStrategy +``` + ## Create new hyperopt Creates a new hyperopt from a template similar to SampleHyperopt. @@ -82,20 +80,6 @@ The file will be named inline with your class name, and will not overwrite exist Results will be located in `user_data/hyperopts/.py`. -### Sample usage of new-hyperopt - -```bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -With custom user directory - -```bash -freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt -``` - -### new-hyperopt complete options - ``` output usage: freqtrade new-hyperopt [-h] [--userdir PATH] [--hyperopt NAME] [--template {full,minimal}] @@ -112,6 +96,18 @@ optional arguments: `full`. ``` +### Sample usage of new-hyperopt + +```bash +freqtrade new-hyperopt --hyperopt AwesomeHyperopt +``` + +With custom user directory + +```bash +freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt +``` + ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. @@ -234,3 +230,35 @@ $ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT ``` $ freqtrade list-markets --exchange kraken --all ``` + +## Test pairlist + +Use the `test-pairlist` subcommand to test the configuration of [dynamic pairlists](configuration.md#pairlists). + +Requires a configuration with specified `pairlists` attribute. +Can be used to generate static pairlists to be used during backtesting / hyperopt. + +``` +usage: freqtrade test-pairlist [-h] [-c PATH] + [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] + [-1] [--print-json] + +optional arguments: + -h, --help show this help message and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + --quote QUOTE_CURRENCY [QUOTE_CURRENCY ...] + Specify quote currency(-ies). Space-separated list. + -1, --one-column Print output in one column. + --print-json Print list of pairs or market symbols in JSON format. +``` + +### Examples + +Show whitelist when using a [dynamic pairlist](configuration.md#pairlists). + +``` +freqtrade test-pairlist --config config.json --quote USDT BTC +``` diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index e4e051b52..799cd7d20 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -37,6 +37,8 @@ ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column", "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"] +ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pairs_print_json"] + ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] @@ -69,6 +71,7 @@ class Arguments: """ Arguments Class. Manage the arguments received by the cli """ + def __init__(self, args: Optional[List[str]]) -> None: self.args = args self._parsed_arg: Optional[argparse.Namespace] = None @@ -129,7 +132,7 @@ class Arguments: start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_new_hyperopt, start_new_strategy, - start_list_timeframes, start_trading) + start_list_timeframes, start_test_pairlist, start_trading) from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', @@ -218,6 +221,14 @@ class Arguments: list_pairs_cmd.set_defaults(func=partial(start_list_markets, pairs_only=True)) self._build_args(optionlist=ARGS_LIST_PAIRS, parser=list_pairs_cmd) + # Add test-pairlist subcommand + test_pairlist_cmd = subparsers.add_parser( + 'test-pairlist', + help='Test your pairlist configuration.', + ) + test_pairlist_cmd.set_defaults(func=start_test_pairlist) + self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd) + # Add download-data subcommand download_data_cmd = subparsers.add_parser( 'download-data', diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 1dcbd1f96..30902dfe9 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -48,7 +48,8 @@ AVAILABLE_CLI_OPTIONS = { ), "logfile": Arg( '--logfile', - help='Log to the file specified.', + help="Log to the file specified. Special values are: 'syslog', 'journald'. " + "See the documentation for more details.", metavar='FILE', ), "version": Arg( @@ -195,11 +196,10 @@ AVAILABLE_CLI_OPTIONS = { ), "spaces": Arg( '--spaces', - help='Specify which parameters to hyperopt. Space-separated list. ' - 'Default: `%(default)s`.', - choices=['all', 'buy', 'sell', 'roi', 'stoploss'], + help='Specify which parameters to hyperopt. Space-separated list.', + choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'], nargs='+', - default='all', + default='default', ), "print_all": Arg( '--print-all', diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 8a7641a08..4bfd24677 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -61,11 +61,16 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: :param conf: Config in JSON format :return: Returns None if everything is ok, otherwise throw an OperationalException """ + # validating trailing stoploss _validate_trailing_stoploss(conf) _validate_edge(conf) _validate_whitelist(conf) + # validate configuration before returning + logger.info('Validating configuration ...') + validate_config_schema(conf) + def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 6552078c8..e517a0558 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -9,8 +9,6 @@ from typing import Any, Callable, Dict, List, Optional from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.config_validation import (validate_config_consistency, - validate_config_schema) from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings from freqtrade.configuration.directory_operations import (create_datadir, create_userdata_dir) @@ -84,10 +82,6 @@ class Configuration: if 'pairlists' not in config: config['pairlists'] = [] - # validate configuration before returning - logger.info('Validating configuration ...') - validate_config_schema(config) - return config def load_config(self) -> Dict[str, Any]: @@ -118,8 +112,6 @@ class Configuration: process_temporary_deprecated_settings(config) - validate_config_consistency(config) - return config def _process_logging_options(self, config: Dict[str, Any]) -> None: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index bf5d822c6..f5e5969eb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -6,7 +6,6 @@ bot constants DEFAULT_CONFIG = 'config.json' DEFAULT_EXCHANGE = 'bittrex' PROCESS_THROTTLE_SECS = 5 # sec -DEFAULT_TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss' @@ -66,13 +65,13 @@ MINIMAL_CONFIG = { CONF_SCHEMA = { 'type': 'object', 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': -1}, + 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, 'ticker_interval': {'type': 'string', 'enum': TIMEFRAMES}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'XBT', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': { - "type": ["number", "string"], - "minimum": 0.0005, - "pattern": UNLIMITED_STAKE_AMOUNT + 'type': ['number', 'string'], + 'minimum': 0.0001, + 'pattern': UNLIMITED_STAKE_AMOUNT }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, @@ -94,8 +93,8 @@ CONF_SCHEMA = { 'unfilledtimeout': { 'type': 'object', 'properties': { - 'buy': {'type': 'number', 'minimum': 3}, - 'sell': {'type': 'number', 'minimum': 10} + 'buy': {'type': 'number', 'minimum': 1}, + 'sell': {'type': 'number', 'minimum': 1} } }, 'bid_strategy': { @@ -107,7 +106,7 @@ CONF_SCHEMA = { 'maximum': 1, 'exclusiveMaximum': False, 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1}, + 'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1}, 'check_depth_of_market': { 'type': 'object', 'properties': { @@ -123,8 +122,8 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'use_order_book': {'type': 'boolean'}, - 'order_book_min': {'type': 'number', 'minimum': 1}, - 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50}, + 'order_book_min': {'type': 'integer', 'minimum': 1}, + 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50}, 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, 'ignore_roi_if_buy_signal': {'type': 'boolean'} @@ -197,8 +196,8 @@ CONF_SCHEMA = { 'listen_ip_address': {'format': 'ipv4'}, 'listen_port': { 'type': 'integer', - "minimum": 1024, - "maximum": 65535 + 'minimum': 1024, + 'maximum': 65535 }, 'username': {'type': 'string'}, 'password': {'type': 'string'}, @@ -211,7 +210,7 @@ CONF_SCHEMA = { 'internals': { 'type': 'object', 'properties': { - 'process_throttle_secs': {'type': 'number'}, + 'process_throttle_secs': {'type': 'integer'}, 'interval': {'type': 'integer'}, 'sd_notify': {'type': 'boolean'}, } @@ -253,32 +252,32 @@ CONF_SCHEMA = { 'edge': { 'type': 'object', 'properties': { - "enabled": {'type': 'boolean'}, - "process_throttle_secs": {'type': 'integer', 'minimum': 600}, - "calculate_since_number_of_days": {'type': 'integer'}, - "allowed_risk": {'type': 'number'}, - "capital_available_percentage": {'type': 'number'}, - "stoploss_range_min": {'type': 'number'}, - "stoploss_range_max": {'type': 'number'}, - "stoploss_range_step": {'type': 'number'}, - "minimum_winrate": {'type': 'number'}, - "minimum_expectancy": {'type': 'number'}, - "min_trade_number": {'type': 'number'}, - "max_trade_duration_minute": {'type': 'integer'}, - "remove_pumps": {'type': 'boolean'} + 'enabled': {'type': 'boolean'}, + 'process_throttle_secs': {'type': 'integer', 'minimum': 600}, + 'calculate_since_number_of_days': {'type': 'integer'}, + 'allowed_risk': {'type': 'number'}, + 'capital_available_percentage': {'type': 'number'}, + 'stoploss_range_min': {'type': 'number'}, + 'stoploss_range_max': {'type': 'number'}, + 'stoploss_range_step': {'type': 'number'}, + 'minimum_winrate': {'type': 'number'}, + 'minimum_expectancy': {'type': 'number'}, + 'min_trade_number': {'type': 'number'}, + 'max_trade_duration_minute': {'type': 'integer'}, + 'remove_pumps': {'type': 'boolean'} }, 'required': ['process_throttle_secs', 'allowed_risk', 'capital_available_percentage'] } }, - 'anyOf': [ - {'required': ['exchange']} - ], 'required': [ + 'exchange', 'max_open_trades', 'stake_currency', 'stake_amount', 'dry_run', 'bid_strategy', 'unfilledtimeout', + 'stoploss', + 'minimal_roi', ] } diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ec95be874..865893a66 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -146,7 +146,7 @@ def load_pair_history(pair: str, :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period - :return: DataFrame with ohlcv data + :return: DataFrame with ohlcv data, or empty DataFrame """ timerange_startup = deepcopy(timerange) @@ -174,7 +174,7 @@ def load_pair_history(pair: str, f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data' ) - return None + return DataFrame() def load_data(datadir: Path, @@ -216,7 +216,7 @@ def load_data(datadir: Path, exchange=exchange, fill_up_missing=fill_up_missing, startup_candles=startup_candles) - if hist is not None: + if not hist.empty: result[pair] = hist if fail_without_data and not result: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 358c63f90..ec341ff0a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -266,7 +266,11 @@ class FreqtradeBot: amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) - return min(min_stake_amounts) / amount_reserve_percent + + # The value returned should satisfy both limits: for amount (base currency) and + # for cost (quote, stake currency), so max() is used here. + # See also #2575 at github. + return max(min_stake_amounts) / amount_reserve_percent def create_trades(self) -> bool: """ diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index e00f4fc11..27f16ecc3 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -1,9 +1,12 @@ import logging import sys -from logging.handlers import RotatingFileHandler +from logging import Formatter +from logging.handlers import RotatingFileHandler, SysLogHandler from typing import Any, Dict, List +from freqtrade import OperationalException + logger = logging.getLogger(__name__) @@ -36,10 +39,38 @@ def setup_logging(config: Dict[str, Any]) -> None: # Log to stderr log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)] - if config.get('logfile'): - log_handlers.append(RotatingFileHandler(config['logfile'], - maxBytes=1024 * 1024, # 1Mb - backupCount=10)) + logfile = config.get('logfile') + if logfile: + s = logfile.split(':') + if s[0] == 'syslog': + # Address can be either a string (socket filename) for Unix domain socket or + # a tuple (hostname, port) for UDP socket. + # Address can be omitted (i.e. simple 'syslog' used as the value of + # config['logfilename']), which defaults to '/dev/log', applicable for most + # of the systems. + address = (s[1], int(s[2])) if len(s) > 2 else s[1] if len(s) > 1 else '/dev/log' + handler = SysLogHandler(address=address) + # No datetime field for logging into syslog, to allow syslog + # to perform reduction of repeating messages if this is set in the + # syslog config. The messages should be equal for this. + handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) + log_handlers.append(handler) + elif s[0] == 'journald': + try: + from systemd.journal import JournaldLogHandler + except ImportError: + raise OperationalException("You need the systemd python package be installed in " + "order to use logging to journald.") + handler = JournaldLogHandler() + # No datetime field for logging into journald, to allow syslog + # to perform reduction of repeating messages if this is set in the + # syslog config. The messages should be equal for this. + handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) + log_handlers.append(handler) + else: + log_handlers.append(RotatingFileHandler(logfile, + maxBytes=1024 * 1024, # 1Mb + backupCount=10)) logging.basicConfig( level=logging.INFO if verbosity < 1 else logging.DEBUG, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2c2d116a4..d9fb1f2d1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -13,7 +13,8 @@ from pandas import DataFrame from tabulate import tabulate from freqtrade import OperationalException -from freqtrade.configuration import TimeRange, remove_credentials +from freqtrade.configuration import (TimeRange, remove_credentials, + validate_config_consistency) from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -75,10 +76,12 @@ class Backtesting: stratconf = deepcopy(self.config) stratconf['strategy'] = strat self.strategylist.append(StrategyResolver(stratconf).strategy) + validate_config_consistency(stratconf) else: # No strategy list specified, only one strategy self.strategylist.append(StrategyResolver(self.config).strategy) + validate_config_consistency(self.config) if "ticker_interval" not in self.config: raise OperationalException("Ticker-interval needs to be set in either configuration " diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 5a4543884..a667ebb92 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -9,7 +9,8 @@ from typing import Any, Dict from tabulate import tabulate from freqtrade import constants -from freqtrade.configuration import TimeRange, remove_credentials +from freqtrade.configuration import (TimeRange, remove_credentials, + validate_config_consistency) from freqtrade.edge import Edge from freqtrade.exchange import Exchange from freqtrade.resolvers import StrategyResolver @@ -35,6 +36,8 @@ class EdgeCli: self.exchange = Exchange(self.config) self.strategy = StrategyResolver(self.config).strategy + validate_config_consistency(self.config) + self.edge = Edge(config, self.exchange, self.strategy) # Set refresh_pairs to false for edge-cli (it must be true for edge) self.edge._refresh_pairs = False diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f932393aa..97810d75a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -175,6 +175,9 @@ class Hyperopt: if self.has_space('stoploss'): result['stoploss'] = {p.name: params.get(p.name) for p in self.hyperopt_space('stoploss')} + if self.has_space('trailing'): + result['trailing'] = {p.name: params.get(p.name) + for p in self.hyperopt_space('trailing')} return result @@ -196,7 +199,7 @@ class Hyperopt: if print_json: result_dict: Dict = {} - for s in ['buy', 'sell', 'roi', 'stoploss']: + for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: Hyperopt._params_update_for_json(result_dict, params, s) print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) @@ -205,6 +208,7 @@ class Hyperopt: Hyperopt._params_pretty_print(params, 'sell', "Sell hyperspace params:") Hyperopt._params_pretty_print(params, 'roi', "ROI table:") Hyperopt._params_pretty_print(params, 'stoploss', "Stoploss:") + Hyperopt._params_pretty_print(params, 'trailing', "Trailing stop:") @staticmethod def _params_update_for_json(result_dict, params, space: str): @@ -220,7 +224,7 @@ class Hyperopt: result_dict['minimal_roi'] = OrderedDict( (str(k), v) for k, v in space_params.items() ) - else: # 'stoploss' + else: # 'stoploss', 'trailing' result_dict.update(space_params) @staticmethod @@ -285,9 +289,13 @@ class Hyperopt: def has_space(self, space: str) -> bool: """ - Tell if a space value is contained in the configuration + Tell if the space value is contained in the configuration """ - return any(s in self.config['spaces'] for s in [space, 'all']) + # The 'trailing' space is not included in the 'default' set of spaces + if space == 'trailing': + return any(s in self.config['spaces'] for s in [space, 'all']) + else: + return any(s in self.config['spaces'] for s in [space, 'all', 'default']) def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]: """ @@ -297,18 +305,27 @@ class Hyperopt: for all hyperspaces used. """ spaces: List[Dimension] = [] + if space == 'buy' or (space is None and self.has_space('buy')): logger.debug("Hyperopt has 'buy' space") spaces += self.custom_hyperopt.indicator_space() + if space == 'sell' or (space is None and self.has_space('sell')): logger.debug("Hyperopt has 'sell' space") spaces += self.custom_hyperopt.sell_indicator_space() + if space == 'roi' or (space is None and self.has_space('roi')): logger.debug("Hyperopt has 'roi' space") spaces += self.custom_hyperopt.roi_space() + if space == 'stoploss' or (space is None and self.has_space('stoploss')): logger.debug("Hyperopt has 'stoploss' space") spaces += self.custom_hyperopt.stoploss_space() + + if space == 'trailing' or (space is None and self.has_space('trailing')): + logger.debug("Hyperopt has 'trailing' space") + spaces += self.custom_hyperopt.trailing_space() + return spaces def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: @@ -334,6 +351,15 @@ class Hyperopt: if self.has_space('stoploss'): self.backtesting.strategy.stoploss = params_dict['stoploss'] + if self.has_space('trailing'): + self.backtesting.strategy.trailing_stop = params_dict['trailing_stop'] + self.backtesting.strategy.trailing_stop_positive = \ + params_dict['trailing_stop_positive'] + self.backtesting.strategy.trailing_stop_positive_offset = \ + params_dict['trailing_stop_positive_offset'] + self.backtesting.strategy.trailing_only_offset_is_reached = \ + params_dict['trailing_only_offset_is_reached'] + processed = load(self.tickerdata_pickle) min_date, max_date = get_timeframe(processed) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 2b2a81f00..958fb8d66 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -8,7 +8,7 @@ import math from abc import ABC from typing import Dict, Any, Callable, List -from skopt.space import Dimension, Integer, Real +from skopt.space import Categorical, Dimension, Integer, Real from freqtrade import OperationalException from freqtrade.exchange import timeframe_to_minutes @@ -174,6 +174,27 @@ class IHyperOpt(ABC): Real(-0.35, -0.02, name='stoploss'), ] + @staticmethod + def trailing_space() -> List[Dimension]: + """ + Create a trailing stoploss space. + + You may override it in your custom Hyperopt class. + """ + return [ + # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace + # is used. Otherwise hyperopt will vary other parameters that won't have effect if + # trailing_stop is set False. + # This parameter is included into the hyperspace dimensions rather than assigning + # it explicitly in the code in order to have it printed in the results along with + # other 'trailing' hyperspace parameters. + Categorical([True], name='trailing_stop'), + + Real(0.02, 0.35, name='trailing_stop_positive'), + Real(0.01, 0.1, name='trailing_stop_positive_offset'), + Categorical([True, False], name='trailing_only_offset_is_reached'), + ] + # This is needed for proper unpickling the class attribute ticker_interval # which is set to the actual value by the resolver. # Why do I still need such shamanic mantras in modern python? diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index d7b2c96ae..aedcc5a88 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -48,6 +48,7 @@ class PrecisionFilter(IPairList): """ Filters and sorts pairlists and assigns and returns them again. """ + stoploss = None if self._config.get('stoploss') is not None: # Precalculate sanitized stoploss value to avoid recalculation for every pair stoploss = 1 - abs(self._config.get('stoploss')) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index f87165253..8f4cc4787 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -312,7 +312,7 @@ class ApiServer(RPC): logger.info("LocalRPC - Profit Command Called") stats = self._rpc_trade_statistics(self._config['stake_currency'], - self._config['fiat_display_currency'] + self._config.get('fiat_display_currency') ) return self.rest_dump(stats) @@ -354,7 +354,8 @@ class ApiServer(RPC): Returns the current status of the trades in json format """ - results = self._rpc_balance(self._config.get('fiat_display_currency', '')) + results = self._rpc_balance(self._config['stake_currency'], + self._config.get('fiat_display_currency', '')) return self.rest_dump(results) @require_login diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8898f3068..4cebe646e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -297,34 +297,42 @@ class RPC: 'best_rate': round(bp_rate * 100, 2), } - def _rpc_balance(self, fiat_display_currency: str) -> Dict: + def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: """ Returns current account balance per crypto """ output = [] total = 0.0 - for coin, balance in self._freqtrade.exchange.get_balances().items(): - if not balance['total']: + try: + tickers = self._freqtrade.exchange.get_tickers() + except (TemporaryError, DependencyException): + raise RPCException('Error getting current tickers.') + + for coin, balance in self._freqtrade.wallets.get_all_balances().items(): + if not balance.total: continue - if coin == 'BTC': + est_stake: float = 0 + if coin == stake_currency: rate = 1.0 + est_stake = balance.total else: try: - pair = self._freqtrade.exchange.get_valid_pair_combination(coin, "BTC") - if pair.startswith("BTC"): - rate = 1.0 / self._freqtrade.get_sell_rate(pair, False) - else: - rate = self._freqtrade.get_sell_rate(pair, False) + pair = self._freqtrade.exchange.get_valid_pair_combination(coin, stake_currency) + rate = tickers.get(pair, {}).get('bid', None) + if rate: + if pair.startswith(stake_currency): + rate = 1.0 / rate + est_stake = rate * balance.total except (TemporaryError, DependencyException): logger.warning(f" Could not get rate for pair {coin}.") continue - est_btc: float = rate * balance['total'] - total = total + est_btc + total = total + (est_stake or 0) output.append({ 'currency': coin, - 'free': balance['free'] if balance['free'] is not None else 0, - 'balance': balance['total'] if balance['total'] is not None else 0, - 'used': balance['used'] if balance['used'] is not None else 0, - 'est_btc': est_btc, + 'free': balance.free if balance.free is not None else 0, + 'balance': balance.total if balance.total is not None else 0, + 'used': balance.used if balance.used is not None else 0, + 'est_stake': est_stake or 0, + 'stake': stake_currency, }) if total == 0.0: if self._freqtrade.config.get('dry_run', False): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0547af7b0..2ae22f472 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -325,15 +325,16 @@ class Telegram(RPC): def _balance(self, update: Update, context: CallbackContext) -> None: """ Handler for /balance """ try: - result = self._rpc_balance(self._config.get('fiat_display_currency', '')) + result = self._rpc_balance(self._config['stake_currency'], + self._config.get('fiat_display_currency', '')) output = '' for currency in result['currencies']: - if currency['est_btc'] > 0.0001: + if currency['est_stake'] > 0.0001: curr_output = "*{currency}:*\n" \ "\t`Available: {free: .8f}`\n" \ "\t`Balance: {balance: .8f}`\n" \ "\t`Pending: {used: .8f}`\n" \ - "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) + "\t`Est. {stake}: {est_stake: .8f}`\n".format(**currency) else: curr_output = "*{currency}:* not showing <1$ amount \n".format(**currency) diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py index 5634c21ea..b4bbee3fb 100644 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ b/freqtrade/templates/sample_hyperopt_advanced.py @@ -233,6 +233,27 @@ class AdvancedSampleHyperOpt(IHyperOpt): Real(-0.5, -0.02, name='stoploss'), ] + @staticmethod + def trailing_space() -> List[Dimension]: + """ + Create a trailing stoploss space. + + You may override it in your custom Hyperopt class. + """ + return [ + # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace + # is used. Otherwise hyperopt will vary other parameters that won't have effect if + # trailing_stop is set False. + # This parameter is included into the hyperspace dimensions rather than assigning + # it explicitly in the code in order to have it printed in the results along with + # other 'trailing' hyperspace parameters. + Categorical([True], name='trailing_stop'), + + Real(0.02, 0.35, name='trailing_stop_positive'), + Real(0.01, 0.1, name='trailing_stop_positive_offset'), + Categorical([True, False], name='trailing_only_offset_is_reached'), + ] + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators. diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 3fc0c245c..4d170985c 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -326,6 +326,38 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: print(f"{summary_str}.") +def start_test_pairlist(args: Dict[str, Any]) -> None: + """ + Test Pairlist configuration + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange + + quote_currencies = args.get('quote_currencies') + if not quote_currencies: + quote_currencies = [config.get('stake_currency')] + results = {} + for curr in quote_currencies: + config['stake_currency'] = curr + # Do not use ticker_interval set in the config + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist + + for curr, pairlist in results.items(): + if not args.get('print_one_column', False): + print(f"Pairs for {curr}: ") + + if args.get('print_one_column', False): + print('\n'.join(pairlist)) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairlist), default=str)) + else: + print(pairlist) + + def start_hyperopt_list(args: Dict[str, Any]) -> None: """ """ diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 90b68c49d..c674b5286 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -2,7 +2,7 @@ """ Wallet """ import logging -from typing import Dict, NamedTuple +from typing import Dict, NamedTuple, Any from freqtrade.exchange import Exchange from freqtrade import constants @@ -72,3 +72,6 @@ class Wallets: ) logger.info('Wallets synced.') + + def get_all_balances(self) -> Dict[str, Any]: + return self._wallets diff --git a/mkdocs.yml b/mkdocs.yml index 43d6acc1d..d53687c64 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,6 +24,7 @@ nav: - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md - Advanced Post-installation Tasks: advanced-setup.md + - Advanced Hyperopt: advanced-hyperopt.md - Sandbox Testing: sandbox-testing.md - Deprecated Features: deprecated.md - Contributors Guide: developer.md diff --git a/requirements-common.txt b/requirements-common.txt index e5c66590a..8c38cf546 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.19.86 +ccxt==1.20.22 SQLAlchemy==1.3.11 python-telegram-bot==12.2.0 arrow==0.15.4 diff --git a/requirements-dev.txt b/requirements-dev.txt index a60bbf0eb..a2ffa1a06 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,8 +7,8 @@ coveralls==1.8.2 flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==3.1.0 -mypy==0.740 -pytest==5.3.0 +mypy==0.750 +pytest==5.3.1 pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest-mock==1.12.1 diff --git a/tests/conftest.py b/tests/conftest.py index fbd23a0dc..6c567dda8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -325,7 +325,7 @@ def get_markets(): }, 'price': 500000, 'cost': { - 'min': 1, + 'min': 0.0001, 'max': 500000, }, }, @@ -351,7 +351,7 @@ def get_markets(): }, 'price': 500000, 'cost': { - 'min': 1, + 'min': 0.0001, 'max': 500000, }, }, @@ -376,7 +376,7 @@ def get_markets(): }, 'price': 500000, 'cost': { - 'min': 1, + 'min': 0.0001, 'max': 500000, }, }, @@ -401,7 +401,7 @@ def get_markets(): }, 'price': 500000, 'cost': { - 'min': 1, + 'min': 0.0001, 'max': 500000, }, }, @@ -426,7 +426,7 @@ def get_markets(): }, 'price': 500000, 'cost': { - 'min': 1, + 'min': 0.0001, 'max': 500000, }, }, @@ -451,7 +451,7 @@ def get_markets(): }, 'price': 500000, 'cost': { - 'min': 1, + 'min': 0.0001, 'max': 500000, }, }, @@ -479,7 +479,7 @@ def get_markets(): 'max': None }, 'cost': { - 'min': 0.001, + 'min': 0.0001, 'max': None } }, @@ -980,6 +980,28 @@ def tickers(): 'quoteVolume': 62.68220262, 'info': {} }, + 'BTC/USDT': { + 'symbol': 'BTC/USDT', + 'timestamp': 1573758371399, + 'datetime': '2019-11-14T19:06:11.399Z', + 'high': 8800.0, + 'low': 8582.6, + 'bid': 8648.16, + 'bidVolume': 0.238771, + 'ask': 8648.72, + 'askVolume': 0.016253, + 'vwap': 8683.13647806, + 'open': 8759.7, + 'close': 8648.72, + 'last': 8648.72, + 'previousClose': 8759.67, + 'change': -110.98, + 'percentage': -1.267, + 'average': None, + 'baseVolume': 35025.943355, + 'quoteVolume': 304135046.4242901, + 'info': {} + }, 'ETH/USDT': { 'symbol': 'ETH/USDT', 'timestamp': 1522014804118, @@ -1067,7 +1089,29 @@ def tickers(): 'baseVolume': 59698.79897, 'quoteVolume': 29132399.743954, 'info': {} - } + }, + 'XRP/BTC': { + 'symbol': 'XRP/BTC', + 'timestamp': 1573758257534, + 'datetime': '2019-11-14T19:04:17.534Z', + 'high': 3.126e-05, + 'low': 3.061e-05, + 'bid': 3.093e-05, + 'bidVolume': 27901.0, + 'ask': 3.095e-05, + 'askVolume': 10551.0, + 'vwap': 3.091e-05, + 'open': 3.119e-05, + 'close': 3.094e-05, + 'last': 3.094e-05, + 'previousClose': 3.117e-05, + 'change': -2.5e-07, + 'percentage': -0.802, + 'average': None, + 'baseVolume': 37334921.0, + 'quoteVolume': 1154.19266394, + 'info': {} + }, }) @@ -1317,8 +1361,8 @@ def rpc_balance(): 'used': 0.0 }, 'XRP': { - 'total': 1.0, - 'free': 1.0, + 'total': 0.1, + 'free': 0.01, 'used': 0.0 }, 'EUR': { @@ -1343,7 +1387,7 @@ def import_fails() -> None: realimport = builtins.__import__ def mockedimport(name, *args, **kwargs): - if name in ["filelock"]: + if name in ["filelock", 'systemd.journal']: raise ImportError(f"No module named '{name}'") return realimport(name, *args, **kwargs) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 65feaf03e..6d89ab7c5 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -74,8 +74,8 @@ def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> No def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> None: ld = history.load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir) - assert not isinstance(ld, DataFrame) - assert ld is None + assert isinstance(ld, DataFrame) + assert ld.empty assert log_has( 'No history data for pair: "UNITTEST/BTC", timeframe: 7m. ' 'Use `freqtrade download-data` to download the data', caplog diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index e74ead33d..5086891a6 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -494,7 +494,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None)) + mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame())) mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 5617ee4db..032aece4a 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -27,7 +27,7 @@ from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, @pytest.fixture(scope='function') def hyperopt(default_conf, mocker): default_conf.update({ - 'spaces': ['all'], + 'spaces': ['default'], 'hyperopt': 'DefaultHyperOpt', }) patch_exchange(mocker) @@ -113,7 +113,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--enable-position-stacking', '--disable-max-market-positions', '--epochs', '1000', - '--spaces', 'all', + '--spaces', 'default', '--print-all' ] @@ -249,7 +249,7 @@ def test_start(mocker, default_conf, caplog) -> None: def test_start_no_data(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) - mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None)) + mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame)) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -442,7 +442,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: 'hyperopt': 'DefaultHyperOpt', 'epochs': 1, 'timerange': None, - 'spaces': 'all', + 'spaces': 'default', 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) @@ -507,14 +507,38 @@ def test_format_results(hyperopt): assert result.find('Total profit 1.00000000 EUR') -def test_has_space(hyperopt): - hyperopt.config.update({'spaces': ['buy', 'roi']}) - assert hyperopt.has_space('roi') - assert hyperopt.has_space('buy') - assert not hyperopt.has_space('stoploss') - - hyperopt.config.update({'spaces': ['all']}) - assert hyperopt.has_space('buy') +@pytest.mark.parametrize("spaces, expected_results", [ + (['buy'], + {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False}), + (['sell'], + {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False}), + (['roi'], + {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), + (['stoploss'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False}), + (['trailing'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True}), + (['buy', 'sell', 'roi', 'stoploss'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), + (['buy', 'sell', 'roi', 'stoploss', 'trailing'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['buy', 'roi'], + {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), + (['all'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['default'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), + (['default', 'trailing'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['all', 'buy'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['default', 'buy'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), +]) +def test_has_space(hyperopt, spaces, expected_results): + for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: + hyperopt.config.update({'spaces': spaces}) + assert hyperopt.has_space(s) == expected_results[s] def test_populate_indicators(hyperopt, testdatadir) -> None: @@ -609,6 +633,10 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'roi_p2': 0.01, 'roi_p3': 0.1, 'stoploss': -0.4, + 'trailing_stop': True, + 'trailing_stop_positive': 0.02, + 'trailing_stop_positive_offset': 0.1, + 'trailing_only_offset_is_reached': False, } response_expected = { 'loss': 1.9840569076926293, @@ -637,7 +665,11 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'sell-rsi-enabled': False, 'sell-rsi-value': 0, 'sell-trigger': 'macd_cross_signal'}, - 'stoploss': {'stoploss': -0.4}}, + 'stoploss': {'stoploss': -0.4}, + 'trailing': {'trailing_only_offset_is_reached': False, + 'trailing_stop': True, + 'trailing_stop_positive': 0.02, + 'trailing_stop_positive_offset': 0.1}}, 'params_dict': optimizer_param, 'results_metrics': {'avg_profit': 2.3117, 'duration': 100.0, @@ -659,7 +691,7 @@ def test_clean_hyperopt(mocker, default_conf, caplog): 'hyperopt': 'DefaultHyperOpt', 'epochs': 1, 'timerange': None, - 'spaces': 'all', + 'spaces': 'default', 'hyperopt_jobs': 1, }) mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) @@ -676,7 +708,7 @@ def test_continue_hyperopt(mocker, default_conf, caplog): 'hyperopt': 'DefaultHyperOpt', 'epochs': 1, 'timerange': None, - 'spaces': 'all', + 'spaces': 'default', 'hyperopt_jobs': 1, 'hyperopt_continue': True }) @@ -702,7 +734,8 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, 'params_details': {'buy': {'mfi-value': None}, 'sell': {'sell-mfi-value': None}, - 'roi': {}, 'stoploss': {'stoploss': None}}}]) + 'roi': {}, 'stoploss': {'stoploss': None}, + 'trailing': {'trailing_stop': None}}}]) ) patch_exchange(mocker) @@ -723,6 +756,48 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: parallel.assert_called_once() + out, err = capsys.readouterr() + assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null}' in out # noqa: E501 + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + + +def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': {'buy': {'mfi-value': None}, + 'sell': {'sell-mfi-value': None}, + 'roi': {}, 'stoploss': {'stoploss': None}}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'hyperopt': 'DefaultHyperOpt', + 'epochs': 1, + 'timerange': None, + 'spaces': 'default', + 'hyperopt_jobs': 1, + 'print_json': True, + }) + + hyperopt = Hyperopt(default_conf) + hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + hyperopt.start() + + parallel.assert_called_once() + out, err = capsys.readouterr() assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501 assert dumper.called diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 76537880c..43285cdb1 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -100,7 +100,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co markets=PropertyMock(return_value=shitcoinmarkets), ) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC'] + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'] bot.pairlists.refresh_pairlist() assert whitelist == bot.pairlists.whitelist @@ -135,10 +135,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), # Different sorting depending on quote or bid volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "USDT", ['ETH/USDT']), # No pair for ETH ... @@ -146,19 +146,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "ETH", []), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), + {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # Precisionfilter bid ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, - {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'FUEL/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.02} - ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # StaticPairlist Only ([{"method": "StaticPairList"}, ], "BTC", ['ETH/BTC', 'TKN/BTC']), @@ -285,12 +285,7 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): - del whitelist_conf['pairlists'][0]['method'] mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - with pytest.raises(OperationalException, - match=r"No Pairlist defined!"): - get_patched_freqtradebot(mocker, whitelist_conf) - assert log_has_re("No method in .*", caplog) whitelist_conf['pairlists'] = [] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index fb7a5276a..699f2d962 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -355,29 +355,18 @@ def test_rpc_balance_handle_error(default_conf, mocker): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=mock_balance), - get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) + get_tickers=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() - - result = rpc._rpc_balance(default_conf['fiat_display_currency']) - assert prec_satoshi(result['total'], 12) - assert prec_satoshi(result['value'], 180000) - assert 'USD' == result['symbol'] - assert result['currencies'] == [{ - 'currency': 'BTC', - 'free': 10.0, - 'balance': 12.0, - 'used': 2.0, - 'est_btc': 12.0, - }] - assert result['total'] == 12.0 + with pytest.raises(RPCException, match="Error getting current tickers."): + rpc._rpc_balance(default_conf['stake_currency'], default_conf['fiat_display_currency']) -def test_rpc_balance_handle(default_conf, mocker): +def test_rpc_balance_handle(default_conf, mocker, tickers): mock_balance = { 'BTC': { 'free': 10.0, @@ -389,7 +378,7 @@ def test_rpc_balance_handle(default_conf, mocker): 'total': 5.0, 'used': 4.0, }, - 'PAX': { + 'USDT': { 'free': 5.0, 'total': 10.0, 'used': 5.0, @@ -405,10 +394,9 @@ def test_rpc_balance_handle(default_conf, mocker): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=mock_balance), - get_ticker=MagicMock( - side_effect=lambda p, r: {'bid': 100} if p == "BTC/PAX" else {'bid': 0.01}), + get_tickers=tickers, get_valid_pair_combination=MagicMock( - side_effect=lambda a, b: f"{b}/{a}" if a == "PAX" else f"{a}/{b}") + side_effect=lambda a, b: f"{b}/{a}" if a == "USDT" else f"{a}/{b}") ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -416,30 +404,35 @@ def test_rpc_balance_handle(default_conf, mocker): rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() - result = rpc._rpc_balance(default_conf['fiat_display_currency']) - assert prec_satoshi(result['total'], 12.15) - assert prec_satoshi(result['value'], 182250) + result = rpc._rpc_balance(default_conf['stake_currency'], default_conf['fiat_display_currency']) + assert prec_satoshi(result['total'], 12.309096315) + assert prec_satoshi(result['value'], 184636.44472997) assert 'USD' == result['symbol'] assert result['currencies'] == [ {'currency': 'BTC', - 'free': 10.0, - 'balance': 12.0, - 'used': 2.0, - 'est_btc': 12.0, + 'free': 10.0, + 'balance': 12.0, + 'used': 2.0, + 'est_stake': 12.0, + 'stake': 'BTC', }, {'free': 1.0, 'balance': 5.0, 'currency': 'ETH', - 'est_btc': 0.05, - 'used': 4.0 + 'est_stake': 0.30794, + 'used': 4.0, + 'stake': 'BTC', + }, {'free': 5.0, 'balance': 10.0, - 'currency': 'PAX', - 'est_btc': 0.1, - 'used': 5.0} + 'currency': 'USDT', + 'est_stake': 0.0011563153318162476, + 'used': 5.0, + 'stake': 'BTC', + } ] - assert result['total'] == 12.15 + assert result['total'] == 12.309096315331816 def test_rpc_start(mocker, default_conf) -> None: @@ -697,8 +690,8 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None pair = 'XRP/BTC' # Test not buying - default_conf['stake_amount'] = 0.0000001 freqtradebot = get_patched_freqtradebot(mocker, default_conf) + freqtradebot.config['stake_amount'] = 0.0000001 patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'TKN/BTC' diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8eff37023..555fcdc81 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -23,7 +23,7 @@ _TEST_PASS = "SuperSecurePassword1!" def botclient(default_conf, mocker): default_conf.update({"api_server": {"enabled": True, "listen_ip_address": "127.0.0.1", - "listen_port": "8080", + "listen_port": 8080, "username": _TEST_USER, "password": _TEST_PASS, }}) @@ -133,7 +133,10 @@ def test_api__init__(default_conf, mocker): def test_api_run(default_conf, mocker, caplog): default_conf.update({"api_server": {"enabled": True, "listen_ip_address": "127.0.0.1", - "listen_port": "8080"}}) + "listen_port": 8080, + "username": "TestUser", + "password": "testPass", + }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.api_server.threading.Thread', MagicMock()) @@ -146,7 +149,7 @@ def test_api_run(default_conf, mocker, caplog): apiserver.run() assert server_mock.call_count == 1 assert server_mock.call_args_list[0][0][0] == "127.0.0.1" - assert server_mock.call_args_list[0][0][1] == "8080" + assert server_mock.call_args_list[0][0][1] == 8080 assert isinstance(server_mock.call_args_list[0][0][2], Flask) assert hasattr(apiserver, "srv") @@ -158,14 +161,14 @@ def test_api_run(default_conf, mocker, caplog): server_mock.reset_mock() apiserver._config.update({"api_server": {"enabled": True, "listen_ip_address": "0.0.0.0", - "listen_port": "8089", + "listen_port": 8089, "password": "", }}) apiserver.run() assert server_mock.call_count == 1 assert server_mock.call_args_list[0][0][0] == "0.0.0.0" - assert server_mock.call_args_list[0][0][1] == "8089" + assert server_mock.call_args_list[0][0][1] == 8089 assert isinstance(server_mock.call_args_list[0][0][2], Flask) assert log_has("Starting HTTP Server at 0.0.0.0:8089", caplog) assert log_has("Starting Local Rest Server.", caplog) @@ -186,7 +189,10 @@ def test_api_run(default_conf, mocker, caplog): def test_api_cleanup(default_conf, mocker, caplog): default_conf.update({"api_server": {"enabled": True, "listen_ip_address": "127.0.0.1", - "listen_port": "8080"}}) + "listen_port": 8080, + "username": "TestUser", + "password": "testPass", + }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.api_server.threading.Thread', MagicMock()) mocker.patch('freqtrade.rpc.api_server.make_server', MagicMock()) @@ -224,28 +230,10 @@ def test_api_stopbuy(botclient): def test_api_balance(botclient, mocker, rpc_balance): ftbot, client = botclient - def mock_ticker(symbol, refresh): - if symbol == 'BTC/USDT': - return { - 'bid': 10000.00, - 'ask': 10000.00, - 'last': 10000.00, - } - elif symbol == 'XRP/BTC': - return { - 'bid': 0.00001, - 'ask': 0.00001, - 'last': 0.00001, - } - return { - 'bid': 0.1, - 'ask': 0.1, - 'last': 0.1, - } mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', side_effect=mock_ticker) mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") + ftbot.wallets.update() rc = client_get(client, f"{BASE_URI}/balance") assert_response(rc) @@ -256,7 +244,8 @@ def test_api_balance(botclient, mocker, rpc_balance): 'free': 12.0, 'balance': 12.0, 'used': 0.0, - 'est_btc': 12.0, + 'est_stake': 12.0, + 'stake': 'BTC', } diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index c9fbf8c3b..edf6bae4d 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -173,7 +173,10 @@ def test_init_apiserver_enabled(mocker, default_conf, caplog) -> None: default_conf["telegram"]["enabled"] = False default_conf["api_server"] = {"enabled": True, "listen_ip_address": "127.0.0.1", - "listen_port": "8080"} + "listen_port": 8080, + "username": "TestUser", + "password": "TestPass", + } rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) # Sleep to allow the thread to start diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index a33ab8675..367eb8366 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -144,9 +144,9 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: def test_status(default_conf, update, mocker, fee, ticker,) -> None: - update.message.chat.id = 123 + update.message.chat.id = "123" default_conf['telegram']['enabled'] = False - default_conf['telegram']['chat_id'] = 123 + default_conf['telegram']['chat_id'] = "123" mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -461,29 +461,10 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, assert '*Best Performing:* `ETH/BTC: 6.20%`' in msg_mock.call_args_list[-1][0][0] -def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance) -> None: - - def mock_ticker(symbol, refresh): - if symbol == 'BTC/USDT': - return { - 'bid': 10000.00, - 'ask': 10000.00, - 'last': 10000.00, - } - elif symbol == 'XRP/BTC': - return { - 'bid': 0.00001, - 'ask': 0.00001, - 'last': 0.00001, - } - return { - 'bid': 0.1, - 'ask': 0.1, - 'last': 0.1, - } +def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tickers) -> None: mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', side_effect=mock_ticker) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") @@ -564,7 +545,8 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None 'free': 1.0, 'used': 0.5, 'balance': i, - 'est_btc': 1 + 'est_stake': 1, + 'stake': 'BTC', }) mocker.patch('freqtrade.rpc.rpc.RPC._rpc_balance', return_value={ 'currencies': balances, diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index dbbc4cefb..c066aa8e7 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -113,7 +113,7 @@ def test_send_msg(default_conf, mocker): def test_exception_send_msg(default_conf, mocker, caplog): default_conf["webhook"] = get_webhook_dict() - default_conf["webhook"]["webhookbuy"] = None + del default_conf["webhook"]["webhookbuy"] webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION}) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index e971d15ab..ae85c7493 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name import json import logging +import sys import warnings from copy import deepcopy from pathlib import Path @@ -19,7 +20,7 @@ from freqtrade.configuration.deprecated_settings import ( process_temporary_deprecated_settings) from freqtrade.configuration.load_config import load_config_file from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL -from freqtrade.loggers import _set_loggers +from freqtrade.loggers import _set_loggers, setup_logging from freqtrade.state import RunMode from tests.conftest import (log_has, log_has_re, patched_configuration_load_config_file) @@ -40,10 +41,16 @@ def test_load_config_invalid_pair(default_conf) -> None: def test_load_config_missing_attributes(default_conf) -> None: - default_conf.pop('exchange') + conf = deepcopy(default_conf) + conf.pop('exchange') with pytest.raises(ValidationError, match=r".*'exchange' is a required property.*"): - validate_config_schema(default_conf) + validate_config_schema(conf) + + conf = deepcopy(default_conf) + conf.pop('stake_currency') + with pytest.raises(ValidationError, match=r".*'stake_currency' is a required property.*"): + validate_config_schema(conf) def test_load_config_incorrect_stake_amount(default_conf) -> None: @@ -100,7 +107,6 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: assert validated_conf['max_open_trades'] == 0 assert 'internals' in validated_conf - assert log_has('Validating configuration ...', caplog) def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None: @@ -132,7 +138,6 @@ def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None: assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist'] assert 'internals' in validated_conf - assert log_has('Validating configuration ...', caplog) def test_from_config(default_conf, mocker, caplog) -> None: @@ -159,7 +164,6 @@ def test_from_config(default_conf, mocker, caplog) -> None: assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist'] assert validated_conf['fiat_display_currency'] == "EUR" assert 'internals' in validated_conf - assert log_has('Validating configuration ...', caplog) assert isinstance(validated_conf['user_data_dir'], Path) @@ -191,7 +195,6 @@ def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> assert validated_conf['max_open_trades'] > 999999999 assert validated_conf['max_open_trades'] == float('inf') - assert log_has('Validating configuration ...', caplog) assert "runmode" in validated_conf assert validated_conf['runmode'] == RunMode.DRY_RUN @@ -636,6 +639,56 @@ def test_set_loggers() -> None: assert logging.getLogger('telegram').level is logging.INFO +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +def test_set_loggers_syslog(mocker): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'syslog:/dev/log', + } + + setup_logging(config) + assert len(logger.handlers) == 2 + assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] + assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.") +def test_set_loggers_journald(mocker): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + + setup_logging(config) + assert len(logger.handlers) == 2 + assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] + assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +def test_set_loggers_journald_importerror(mocker, import_fails): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + with pytest.raises(OperationalException, + match=r'You need the systemd python package.*'): + setup_logging(config) + logger.handlers = orig_handlers + + def test_set_logfile(default_conf, mocker): patched_configuration_load_config_file(mocker, default_conf) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index b01c8e247..fe58ccf37 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -299,7 +299,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, limit_buy_order, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - default_conf['stake_amount'] = 0.0000098751 + default_conf['stake_amount'] = 0.00098751 default_conf['max_open_trades'] = 2 mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -313,7 +313,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, trade = Trade.query.first() assert trade is not None - assert trade.stake_amount == 0.0000098751 + assert trade.stake_amount == 0.00098751 assert trade.is_open assert trade.open_date is not None @@ -321,11 +321,11 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, trade = Trade.query.order_by(Trade.id.desc()).first() assert trade is not None - assert trade.stake_amount == 0.0000098751 + assert trade.stake_amount == 0.00098751 assert trade.is_open assert trade.open_date is not None - assert Trade.total_open_trades_stakes() == 1.97502e-05 + assert Trade.total_open_trades_stakes() == 1.97502e-03 def test_get_min_pair_stake_amount(mocker, default_conf) -> None: @@ -334,6 +334,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.stoploss = -0.05 markets = {'ETH/BTC': {'symbol': 'ETH/BTC'}} + # no pair found mocker.patch( 'freqtrade.exchange.Exchange.markets', @@ -425,7 +426,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 2) - assert result == min(2, 2 * 2) / 0.9 + assert result == max(2, 2 * 2) / 0.9 # min amount and cost are set (amount is minial) markets["ETH/BTC"]["limits"] = { @@ -437,7 +438,27 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 2) - assert result == min(8, 2 * 2) / 0.9 + assert result == max(8, 2 * 2) / 0.9 + + +def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + freqtrade = FreqtradeBot(default_conf) + freqtrade.strategy.stoploss = -0.05 + markets = {'ETH/BTC': {'symbol': 'ETH/BTC'}} + + # Real Binance data + markets["ETH/BTC"]["limits"] = { + 'cost': {'min': 0.0001}, + 'amount': {'min': 0.001} + } + mocker.patch( + 'freqtrade.exchange.Exchange.markets', + PropertyMock(return_value=markets) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 0.020405) + assert round(result, 8) == round(max(0.0001, 0.001 * 0.020405) / 0.9, 8) def test_create_trades(default_conf, ticker, limit_buy_order, fee, mocker) -> None: @@ -522,8 +543,9 @@ def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_or get_fee=fee, ) - default_conf['stake_amount'] = 0.000000005 freqtrade = FreqtradeBot(default_conf) + freqtrade.config['stake_amount'] = 0.000000005 + patch_get_signal(freqtrade) assert not freqtrade.create_trades() diff --git a/tests/test_utils.py b/tests/test_utils.py index 1258c939c..67ec8409b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,8 +10,9 @@ from freqtrade.utils import (setup_utils_configuration, start_create_userdir, start_download_data, start_list_exchanges, start_list_markets, start_list_timeframes, start_new_hyperopt, start_new_strategy, - start_trading) -from tests.conftest import get_args, log_has, log_has_re, patch_exchange + start_test_pairlist, start_trading) +from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, + patched_configuration_load_config_file) def test_setup_utils_configuration(): @@ -573,7 +574,7 @@ def test_download_data_no_exchange(mocker, caplog): ) args = [ "download-data", - ] + ] pargs = get_args(args) pargs['config'] = None with pytest.raises(OperationalException, @@ -623,3 +624,36 @@ def test_download_data_trades(mocker, caplog): assert dl_mock.call_args[1]['timerange'].starttype == "date" assert dl_mock.call_count == 1 assert convert_mock.call_count == 1 + + +def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys): + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + default_conf['pairlists'] = [ + { + "method": "VolumePairList", + "number_assets": 5, + "sort_key": "quoteVolume", + }, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.02}, + ] + + patched_configuration_load_config_file(mocker, default_conf) + args = [ + 'test-pairlist', + '-c', 'config.json.example' + ] + + start_test_pairlist(get_args(args)) + + assert log_has_re(r"^Using resolved pairlist VolumePairList.*", caplog) + assert log_has_re(r"^Using resolved pairlist PrecisionFilter.*", caplog) + assert log_has_re(r"^Using resolved pairlist PriceFilter.*", caplog) + captured = capsys.readouterr() + assert re.match(r"Pairs for .*", captured.out) + assert re.match("['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', 'XRP/BTC']", captured.out)