diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index c32fb33c2..a18915462 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -24,4 +24,3 @@ Have you search for this feature before requesting it? It's highly likely that a ## Describe the enhancement *Explain the enhancement you would like* - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8df7ab10..0ff57b270 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: - name: Mypy run: | - mypy freqtrade scripts + mypy freqtrade scripts tests - name: Discord notification uses: rjstone/discord-webhook-notify@v1 @@ -157,6 +157,12 @@ jobs: pip install -e . - name: Tests + if: (runner.os != 'Linux' || matrix.python-version != '3.8') + run: | + pytest --random-order + + - name: Tests (with cov) + if: (runner.os == 'Linux' && matrix.python-version == '3.8') run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc @@ -229,7 +235,7 @@ jobs: - name: Tests run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc + pytest --random-order - name: Backtesting run: | @@ -249,7 +255,7 @@ jobs: - name: Mypy run: | - mypy freqtrade scripts + mypy freqtrade scripts tests - name: Discord notification uses: rjstone/discord-webhook-notify@v1 @@ -259,6 +265,21 @@ jobs: details: Test Failed webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} + mypy_version_check: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: pre-commit dependencies + run: | + pip install pyaml + python build_helpers/pre_commit_update.py + docs_check: runs-on: ubuntu-20.04 steps: @@ -271,7 +292,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.9 - name: Documentation build run: | @@ -288,6 +309,9 @@ jobs: webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} cleanup-prior-runs: + permissions: + actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it + contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch runs-on: ubuntu-20.04 steps: - name: Cleanup previous runs on this branch @@ -298,8 +322,12 @@ jobs: # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: - needs: [ build_linux, build_macos, build_windows, docs_check ] + needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 + # Discord notification can't handle schedule events + if: (github.event_name != 'schedule') + permissions: + repository-projects: read steps: - name: Check user permission @@ -319,7 +347,7 @@ jobs: webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} deploy: - needs: [ build_linux, build_macos, build_windows, docs_check ] + needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] runs-on: ubuntu-20.04 if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 822533ee2..4587626f6 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -15,4 +15,3 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKERHUB_REPOSITORY: freqtradeorg/freqtrade - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31af5b7c7..2170b704a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,46 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pycqa/flake8 - rev: '4.0.1' + - repo: https://github.com/pycqa/flake8 + rev: "4.0.1" hooks: - - id: flake8 + - id: flake8 # stages: [push] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.942' + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v0.942" hooks: - - id: mypy + - id: mypy + exclude: build_helpers + additional_dependencies: + - types-cachetools==5.0.1 + - types-filelock==3.2.5 + - types-requests==2.27.20 + - types-tabulate==0.8.7 + - types-python-dateutil==2.8.12 # stages: [push] -- repo: https://github.com/pycqa/isort - rev: '5.10.1' + - repo: https://github.com/pycqa/isort + rev: "5.10.1" hooks: - id: isort name: isort (python) # stages: [push] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: end-of-file-fixer + exclude: | + (?x)^( + tests/.*| + .*\.svg + )$ + - id: mixed-line-ending + - id: debug-statements + - id: check-ast + - id: trailing-whitespace + exclude: | + (?x)^( + .*\.md + )$ diff --git a/.pylintrc b/.pylintrc index dce99c067..0932ecba4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,4 +7,3 @@ ignore=vendor [TYPECHECK] ignored-modules=numpy,talib,talib.abstract - diff --git a/README.md b/README.md index 679dbcab0..cad39f9ac 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,14 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +### Experimentally, freqtrade also supports futures on the following exchanges + +- [X] [Binance](https://www.binance.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) +- [X] [OKX](https://okx.com/). + +Please make sure to read the [exchange specific notes](docs/exchanges.md), as well as the [trading with leverage](docs/leverage.md) documentation before diving in. + ### Community tested Exchanges confirmed working by the community: diff --git a/build_helpers/pre_commit_update.py b/build_helpers/pre_commit_update.py new file mode 100644 index 000000000..8724d8ade --- /dev/null +++ b/build_helpers/pre_commit_update.py @@ -0,0 +1,42 @@ +# File used in CI to ensure pre-commit dependencies are kept uptodate. + +import sys +from pathlib import Path + +import yaml + + +pre_commit_file = Path('.pre-commit-config.yaml') +require_dev = Path('requirements-dev.txt') + +with require_dev.open('r') as rfile: + requirements = rfile.readlines() + +# Extract types only +type_reqs = [r.strip('\n') for r in requirements if r.startswith('types-')] + +with pre_commit_file.open('r') as file: + f = yaml.load(file, Loader=yaml.FullLoader) + + +mypy_repo = [repo for repo in f['repos'] if repo['repo'] + == 'https://github.com/pre-commit/mirrors-mypy'] + +hooks = mypy_repo[0]['hooks'][0]['additional_dependencies'] + +errors = [] +for hook in hooks: + if hook not in type_reqs: + errors.append(f"{hook} is missing in requirements-dev.txt.") + +for req in type_reqs: + if req not in hooks: + errors.append(f"{req} is missing in pre-config file.") + + +if errors: + for e in errors: + print(e) + sys.exit(1) + +sys.exit(0) diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 193cc30bc..6382e1baf 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -182,6 +182,7 @@ "disable_dataframe_checks": false, "strategy": "SampleStrategy", "strategy_path": "user_data/strategies/", + "recursive_strategy_search": false, "add_config_files": [], "dataformat_ohlcv": "json", "dataformat_trades": "jsongz" diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md new file mode 100644 index 000000000..2a484da69 --- /dev/null +++ b/docs/advanced-backtesting.md @@ -0,0 +1,73 @@ +# Advanced Backtesting Analysis + +## Analyze the buy/entry and sell/exit tags + +It can be helpful to understand how a strategy behaves according to the buy/entry tags used to +mark up different buy conditions. You might want to see more complex statistics about each buy and +sell condition above those provided by the default backtesting output. You may also want to +determine indicator values on the signal candle that resulted in a trade opening. + +!!! Note + The following buy reason analysis is only available for backtesting, *not hyperopt*. + +We need to run backtesting with the `--export` option set to `signals` to enable the exporting of +signals **and** trades: + +``` bash +freqtrade backtesting -c --timeframe --strategy --timerange= --export=signals +``` + +This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding +DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy +makes, this file may get quite large, so periodically check your `user_data/backtest_results` +folder to delete old exports. + +To analyze the buy tags, we need to use the `buy_reasons.py` script from +[froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions +in their README to copy the script into your `freqtrade/scripts/` folder. + +Before running your next backtest, make sure you either delete your old backtest results or run +backtesting with the `--cache none` option to make sure no cached results are used. + +If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the +`user_data/backtest_results` folder. + +Now run the `buy_reasons.py` script, supplying a few options: + +``` bash +python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 +``` + +The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) +to the most detailed per pair, per buy and per sell tag (4). More options are available by +running with the `-h` option. + +### Tuning the buy tags and sell tags to display + +To show only certain buy and sell tags in the displayed output, use the following two options: + +``` +--enter_reason_list : Comma separated list of enter signals to analyse. Default: "all" +--exit_reason_list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" +``` + +For example: + +```bash +python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" +``` + +### Outputting signal candle indicators + +The real power of the buy_reasons.py script comes from the ability to print out the indicator +values present on signal candles to allow fine-grained investigation and tuning of buy signal +indicators. To print out a column for a given set of indicators, use the `--indicator-list` +option: + +```bash +python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +``` + +The indicators have to be present in your strategy's main DataFrame (either for your main +timeframe or for informative timeframes) otherwise they will simply be ignored in the script +output. diff --git a/docs/backtesting.md b/docs/backtesting.md index 5d836d01b..a0a304400 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -20,7 +20,8 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--dry-run-wallet DRY_RUN_WALLET] [--timeframe-detail TIMEFRAME_DETAIL] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export {none,trades}] [--export-filename PATH] + [--export {none,trades,signals}] + [--export-filename PATH] [--breakdown {day,week,month} [{day,week,month} ...]] [--cache {none,day,week,month}] @@ -63,18 +64,17 @@ optional arguments: `30m`, `1h`, `1d`). --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to - backtest. Please note that timeframe needs to be - set either in config or via command line. When using - this together with `--export trades`, the strategy- - name is injected into the filename (so `backtest- - data.json` becomes `backtest-data-SampleStrategy.json` - --export {none,trades} + backtest. Please note that timeframe needs to be set + either in config or via command line. When using this + together with `--export trades`, the strategy-name is + injected into the filename (so `backtest-data.json` + becomes `backtest-data-SampleStrategy.json` + --export {none,trades,signals} Export backtest results (default: trades). - --export-filename PATH - Save backtest results to the file with this filename. - Requires `--export` to be set as well. Example: - `--export-filename=user_data/backtest_results/backtest - _today.json` + --export-filename PATH, --backtest-filename PATH + Use this filename for backtest results.Requires + `--export` to be set as well. Example: `--export-filen + ame=user_data/backtest_results/backtest_today.json` --breakdown {day,week,month} [{day,week,month} ...] Show backtesting breakdown per [day, week, month]. --cache {none,day,week,month} @@ -299,6 +299,7 @@ A backtesting result will look like that: | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | +| CAGR % | 460.87% | | Trades per day | 3.575 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | @@ -388,6 +389,7 @@ It contains some useful key metrics about performance of your strategy on backte | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | +| CAGR % | 460.87% | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | diff --git a/docs/configuration.md b/docs/configuration.md index 369c4e2dd..80cd52c5b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ Per default, the bot loads the configuration from the `config.json` file, locate You can specify a different configuration file used by the bot with the `-c/--config` command-line option. -If you used the [Quick start](installation.md/#quick-start) method for installing +If you used the [Quick start](installation.md/#quick-start) method for installing the bot, the installation script should have already created the default configuration file (`config.json`) for you. If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file. @@ -64,7 +64,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as "config-private.json" ] ``` - + ``` bash freqtrade trade --config user_data/config.json <...> ``` @@ -100,7 +100,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as "stake_amount": "unlimited", } ``` - + Resulting combined configuration: ``` json title="Result" @@ -173,6 +173,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `order_types` | Configure order-types depending on the action (`"entry"`, `"exit"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `order_time_in_force` | Configure time in force for entry and exit orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float +| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy.
**Datatype:** Boolean | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String | `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
**Datatype:** Boolean | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 9a79ee5ed..926ed3eae 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -122,5 +122,6 @@ Best avoid relative paths, since this starts at the storage location of the jupy * [Strategy debugging](strategy_analysis_example.md) - also available as Jupyter notebook (`user_data/notebooks/strategy_analysis_example.ipynb`) * [Plotting](plotting.md) +* [Tag Analysis](advanced-backtesting.md) Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. diff --git a/docs/index.md b/docs/index.md index 2aa80c240..e0a88a381 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,6 +51,14 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [OKX](https://okx.com/) (Former OKEX) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +### Experimentally, freqtrade also supports futures on the following exchanges: + +- [X] [Binance](https://www.binance.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) +- [X] [OKX](https://okx.com/). + +Please make sure to read the [exchange specific notes](exchanges.md), as well as the [trading with leverage](leverage.md) documentation before diving in. + ### Community tested Exchanges confirmed working by the community: diff --git a/docs/javascripts/config.js b/docs/javascripts/config.js index 95d619efc..80e81ba59 100644 --- a/docs/javascripts/config.js +++ b/docs/javascripts/config.js @@ -9,4 +9,4 @@ window.MathJax = { ignoreHtmlClass: ".*|", processHtmlClass: "arithmatex" } -}; \ No newline at end of file +}; diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 8d03a38c3..97be17243 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.9 +mkdocs-material==8.2.10 mdx_truly_sane_lists==1.2 -pymdown-extensions==9.3 +pymdown-extensions==9.4 jinja2==3.1.1 diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index c78a2c7e5..8da8bab0f 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -377,7 +377,7 @@ class AwesomeStrategy(IStrategy): def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 2fa84a6df..ae0c6a6a3 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -93,7 +93,7 @@ from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats # if backtest_dir points to a directory, it'll automatically load the last backtest file. backtest_dir = config["user_data_dir"] / "backtest_results" -# backtest_dir can also point to a specific file +# backtest_dir can also point to a specific file # backtest_dir = config["user_data_dir"] / "backtest_results/backtest-result-2020-07-01_20-04-22.json" ``` diff --git a/environment.yml b/environment.yml index f2f961894..19f3c7f5a 100644 --- a/environment.yml +++ b/environment.yml @@ -32,6 +32,7 @@ dependencies: - prompt-toolkit - schedule - python-dateutil + - joblib # ============================ @@ -54,7 +55,6 @@ dependencies: - scikit-learn - filelock - scikit-optimize - - joblib - progressbar2 # ============================ # 4/4 req plot diff --git a/freqtrade.service b/freqtrade.service index df220ed39..6f0c73ee4 100644 --- a/freqtrade.service +++ b/freqtrade.service @@ -11,4 +11,3 @@ Restart=on-failure [Install] WantedBy=default.target - diff --git a/freqtrade.service.watchdog b/freqtrade.service.watchdog index 66ea00d76..dcd32ae18 100644 --- a/freqtrade.service.watchdog +++ b/freqtrade.service.watchdog @@ -27,4 +27,3 @@ WatchdogSec=20 [Install] WantedBy=default.target - diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 7d4624bd1..62b79da2e 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -12,7 +12,7 @@ from freqtrade.constants import DEFAULT_CONFIG ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] -ARGS_STRATEGY = ["strategy", "strategy_path"] +ARGS_STRATEGY = ["strategy", "strategy_path", "recursive_strategy_search"] ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"] @@ -37,7 +37,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] -ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized"] +ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized", + "recursive_strategy_search"] ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 61890d650..df8966e85 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -83,6 +83,11 @@ AVAILABLE_CLI_OPTIONS = { help='Reset sample files to their original state.', action='store_true', ), + "recursive_strategy_search": Arg( + '--recursive-strategy-search', + help='Recursively search for a strategy in the strategies folder.', + action='store_true', + ), # Main options "strategy": Arg( '-s', '--strategy', diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index c4bd0bf4d..2a5223917 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -41,7 +41,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason'])) -def _print_objs_tabular(objs: List, print_colorized: bool) -> None: +def _print_objs_tabular(objs: List, print_colorized: bool, base_dir: Path) -> None: if print_colorized: colorama_init(autoreset=True) red = Fore.RED @@ -55,7 +55,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: names = [s['name'] for s in objs] objs_to_print = [{ 'name': s['name'] if s['name'] else "--", - 'location': s['location'].name, + 'location': s['location'].relative_to(base_dir), 'status': (red + "LOAD FAILED" + reset if s['class'] is None else "OK" if names.count(s['name']) == 1 else yellow + "DUPLICATE NAME" + reset) @@ -77,7 +77,8 @@ def start_list_strategies(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) - strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column']) + strategy_objs = StrategyResolver.search_all_objects( + directory, not args['print_one_column'], config.get('recursive_strategy_search', False)) # Sort alphabetically strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) for obj in strategy_objs: @@ -89,7 +90,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None: if args['print_one_column']: print('\n'.join([s['name'] for s in strategy_objs])) else: - _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) + _print_objs_tabular(strategy_objs, config.get('print_colorized', False), directory) def start_list_timeframes(args: Dict[str, Any]) -> None: diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 331901920..dde56c220 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -248,6 +248,12 @@ class Configuration: self._args_to_config(config, argname='strategy_list', logstring='Using strategy list of {} strategies', logfun=len) + self._args_to_config( + config, + argname='recursive_strategy_search', + logstring='Recursively searching for a strategy in the strategies folder.', + ) + self._args_to_config(config, argname='timeframe', logstring='Overriding timeframe with Command line argument') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index fba6c968d..af5ad00dd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -14,7 +14,7 @@ PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec TIMEOUT_UNITS = ['minutes', 'seconds'] -EXPORT_OPTIONS = ['none', 'trades'] +EXPORT_OPTIONS = ['none', 'trades', 'signals'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 8abcc6747..206a6f5f3 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -553,3 +553,14 @@ def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[f csum_max = csum_df['sum'].max() + starting_balance return csum_min, csum_max + + +def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: + """ + Calculate CAGR + :param days_passed: Days passed between start and ending balance + :param starting_balance: Starting balance + :param final_balance: Final balance to calculate CAGR against + :return: CAGR + """ + return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ab063762b..82dcacb51 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -9,6 +9,7 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone from math import ceil +from threading import Lock from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union import arrow @@ -64,6 +65,7 @@ class Exchange: "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, + "ohlcv_require_since": False, # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency "ohlcv_volume_currency": "base", # "base" or "quote" "tickers_have_quoteVolume": True, @@ -95,6 +97,9 @@ class Exchange: self._markets: Dict = {} self._trading_fees: Dict[str, Any] = {} self._leverage_tiers: Dict[str, List[Dict]] = {} + # Lock event loop. This is necessary to avoid race-conditions when using force* commands + # Due to funding fee fetching. + self._loop_lock = Lock() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self._config: Dict = {} @@ -166,7 +171,7 @@ class Exchange: self._api_async = self._init_ccxt( exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config) - logger.info('Using Exchange "%s"', self.name) + logger.info(f'Using Exchange "{self.name}"') if validate: # Check if timeframe is available @@ -368,6 +373,9 @@ class Exchange: return ( market.get('quote', None) is not None and market.get('base', None) is not None + and (self.precisionMode != TICK_SIZE + # Too low precision will falsify calculations + or market.get('precision', {}).get('price', None) > 1e-11) and ((self.trading_mode == TradingMode.SPOT and self.market_is_spot(market)) or (self.trading_mode == TradingMode.MARGIN and self.market_is_margin(market)) or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market))) @@ -551,7 +559,7 @@ class Exchange: # Therefore we also show that. raise OperationalException( f"The ccxt library does not provide the list of timeframes " - f"for the exchange \"{self.name}\" and this exchange " + f"for the exchange {self.name} and this exchange " f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}") if timeframe and (timeframe not in self.timeframes): @@ -781,7 +789,9 @@ class Exchange: rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' - _amount = self.amount_to_precision(pair, amount) + # Rounding here must respect to contract sizes + _amount = self._contracts_to_amount( + pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount))) dry_order: Dict[str, Any] = { 'id': order_id, 'symbol': pair, @@ -1710,7 +1720,8 @@ class Exchange: def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> Coroutine: - if not since_ms and self.required_candle_call_count > 1: + if (not since_ms + and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)): # Multiple calls for one pair - to get more history one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) move_to = one_call * self.required_candle_call_count @@ -1770,7 +1781,8 @@ class Exchange: async def gather_stuff(): return await asyncio.gather(*input_coro, return_exceptions=True) - results = self.loop.run_until_complete(gather_stuff()) + with self._loop_lock: + results = self.loop.run_until_complete(gather_stuff()) for res in results: if isinstance(res, Exception): @@ -1829,17 +1841,18 @@ class Exchange: pair, timeframe, since_ms, s ) params = deepcopy(self._ft_has.get('ohlcv_params', {})) + candle_limit = self.ohlcv_candle_limit(timeframe) if candle_type != CandleType.SPOT: params.update({'price': candle_type}) if candle_type != CandleType.FUNDING_RATE: data = await self._api_async.fetch_ohlcv( pair, timeframe=timeframe, since=since_ms, - limit=self.ohlcv_candle_limit(timeframe), params=params) + limit=candle_limit, params=params) else: # Funding rate data = await self._api_async.fetch_funding_rate_history( pair, since=since_ms, - limit=self.ohlcv_candle_limit(timeframe)) + limit=candle_limit) # Convert funding rate to candle pattern data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data] # Some exchanges sort OHLCV in ASC order and others in DESC. @@ -2026,9 +2039,10 @@ class Exchange: if not self.exchange_has("fetchTrades"): raise OperationalException("This exchange does not support downloading Trades.") - return self.loop.run_until_complete( - self._async_get_trade_history(pair=pair, since=since, - until=until, from_id=from_id)) + with self._loop_lock: + return self.loop.run_until_complete( + self._async_get_trade_history(pair=pair, since=since, + until=until, from_id=from_id)) @retrier def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float: diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index f20aab138..d2dcf84a6 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -20,6 +20,7 @@ class Ftx(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "ohlcv_candle_limit": 1500, + "ohlcv_require_since": True, "ohlcv_volume_currency": "quote", "mark_ohlcv_price": "index", "mark_ohlcv_timeframe": "1h", diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9febe64fe..b55fee35f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -586,7 +586,6 @@ class FreqtradeBot(LoggingMixin): Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY :param stake_amount: amount of stake-currency for the pair - :param leverage: amount of leverage applied to this trade :return: True if a buy order is created, false if it fails. """ time_in_force = self.strategy.order_time_in_force['entry'] @@ -665,16 +664,6 @@ class FreqtradeBot(LoggingMixin): amount = safe_value_fallback(order, 'filled', 'amount') enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') - # TODO: this might be unnecessary, as we're calling it in update_trade_state. - isolated_liq = self.exchange.get_liquidation_price( - leverage=leverage, - pair=pair, - amount=amount, - open_rate=enter_limit_filled_price, - is_short=is_short - ) - interest_rate = self.exchange.get_interest_rate() - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') base_currency = self.exchange.get_pair_base_currency(pair) @@ -703,8 +692,6 @@ class FreqtradeBot(LoggingMixin): timeframe=timeframe_to_minutes(self.config['timeframe']), leverage=leverage, is_short=is_short, - interest_rate=interest_rate, - liquidation_price=isolated_liq, trading_mode=self.trading_mode, funding_fees=funding_fees ) @@ -1430,7 +1417,8 @@ class FreqtradeBot(LoggingMixin): default_retval=proposed_limit_rate)( pair=trade.pair, trade=trade, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_limit_rate, current_profit=current_profit) + proposed_rate=proposed_limit_rate, current_profit=current_profit, + exit_tag=exit_check.exit_reason) limit = self.get_valid_price(custom_exit_price, proposed_limit_rate) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d5572ea0b..55a533725 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -86,6 +86,22 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = logger.debug(f'done json to "{filename}"') +def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: + """ + Dump object data into a file + :param filename: file to create + :param data: Object data to save + :return: + """ + import joblib + + if log: + logger.info(f'dumping joblib to "{filename}"') + with open(filename, 'wb') as fp: + joblib.dump(data, fp) + logger.debug(f'done joblib dump to "{filename}"') + + def json_load(datafile: IO) -> Any: """ load data with rapidjson diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py old mode 100644 new mode 100755 index bf666abf2..02867d157 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,6 +9,7 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple +import pandas as pd from numpy import nan from pandas import DataFrame @@ -19,13 +20,15 @@ from freqtrade.data import history from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode +from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, RunMode, + TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, + store_backtest_signal_candles, store_backtest_stats) from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager @@ -51,6 +54,11 @@ ESHORT_IDX = 8 # Exit short ENTER_TAG_IDX = 9 EXIT_TAG_IDX = 10 +# Every change to this headers list must evaluate further usages of the resulting tuple +# and eventually change the constants for indexes at the top +HEADERS = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', + 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] + class Backtesting: """ @@ -73,6 +81,8 @@ class Backtesting: self.run_ids: Dict[str, str] = {} self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} + self.processed_dfs: Dict[str, Dict] = {} + self._exchange_name = self.config['exchange']['name'] self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config) self.dataprovider = DataProvider(self.config, self.exchange) @@ -174,7 +184,7 @@ class Backtesting: # Attach Wallets to Strategy baseclass strategy.wallets = self.wallets # Set stoploss_on_exchange to false for backtesting, - # since a "perfect" stoploss-sell is assumed anyway + # since a "perfect" stoploss-exit is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False @@ -259,10 +269,18 @@ class Backtesting: candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"]) ) # Combine data to avoid combining the data per trade. + unavailable_pairs = [] for pair in self.pairlists.whitelist: + if pair not in self.exchange._leverage_tiers: + unavailable_pairs.append(pair) + continue self.futures_data[pair] = funding_rates_dict[pair].merge( mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"]) + if unavailable_pairs: + raise OperationalException( + f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. " + "It is therefore impossible to backtest with this pair at the moment.") else: self.futures_data = {} @@ -300,10 +318,7 @@ class Backtesting: :param processed: a processed dictionary with format {pair, data}, which gets cleared to optimize memory usage! """ - # Every change to this headers list must evaluate further usages of the resulting tuple - # and eventually change the constants for indexes at the top - headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] + data: Dict = {} self.progress.init_step(BacktestState.CONVERT, len(processed)) @@ -315,7 +330,7 @@ class Backtesting: if not pair_data.empty: # Cleanup from prior runs - pair_data.drop(headers[5:] + ['buy', 'sell'], axis=1, errors='ignore') + pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore') df_analyzed = self.strategy.advise_exit( self.strategy.advise_entry(pair_data, {'pair': pair}), @@ -328,13 +343,13 @@ class Backtesting: self.dataprovider._set_cached_df( pair, self.timeframe, df_analyzed, self.config['candle_type_def']) - # Create a copy of the dataframe before shifting, that way the buy signal/tag + # Create a copy of the dataframe before shifting, that way the entry signal/tag # remains on the correct candle for callbacks. df_analyzed = df_analyzed.copy() - # To avoid using data from future, we use buy/sell signals shifted + # To avoid using data from future, we use entry/exit signals shifted # from the previous candle - for col in headers[5:]: + for col in HEADERS[5:]: tag_col = col in ('enter_tag', 'exit_tag') if col in df_analyzed.columns: df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace( @@ -346,27 +361,27 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - data[pair] = df_analyzed[headers].values.tolist() if not df_analyzed.empty else [] + data[pair] = df_analyzed[HEADERS].values.tolist() if not df_analyzed.empty else [] return data - def _get_close_rate(self, row: Tuple, trade: LocalTrade, sell: ExitCheckTuple, + def _get_close_rate(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int) -> float: """ Get close rate for backtesting result """ # Special handling if high or low hit STOP_LOSS or ROI - if sell.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS): - return self._get_close_rate_for_stoploss(row, trade, sell, trade_dur) - elif sell.exit_type == (ExitType.ROI): - return self._get_close_rate_for_roi(row, trade, sell, trade_dur) + if exit.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS): + return self._get_close_rate_for_stoploss(row, trade, exit, trade_dur) + elif exit.exit_type == (ExitType.ROI): + return self._get_close_rate_for_roi(row, trade, exit, trade_dur) else: return row[OPEN_IDX] - def _get_close_rate_for_stoploss(self, row: Tuple, trade: LocalTrade, sell: ExitCheckTuple, + def _get_close_rate_for_stoploss(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int) -> float: # our stoploss was already lower than candle high, # possibly due to a cancelled trade exit. - # sell at open price. + # exit at open price. is_short = trade.is_short or False leverage = trade.leverage or 1.0 side_1 = -1 if is_short else 1 @@ -380,7 +395,7 @@ class Backtesting: # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and # immediately going down to stop price. - if sell.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0: + if exit.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0: if ( not self.strategy.use_custom_stoploss and self.strategy.trailing_stop and self.strategy.trailing_only_offset_is_reached @@ -399,7 +414,7 @@ class Backtesting: else: assert stop_rate < row[HIGH_IDX] - # Limit lower-end to candle low to avoid sells below the low. + # Limit lower-end to candle low to avoid exits below the low. # This still remains "worst case" - but "worst realistic case". if is_short: return min(row[HIGH_IDX], stop_rate) @@ -409,7 +424,7 @@ class Backtesting: # Set close_rate to stoploss return trade.stop_loss - def _get_close_rate_for_roi(self, row: Tuple, trade: LocalTrade, sell: ExitCheckTuple, + def _get_close_rate_for_roi(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple, trade_dur: int) -> float: is_short = trade.is_short or False leverage = trade.leverage or 1.0 @@ -434,7 +449,7 @@ class Backtesting: and roi_entry % self.timeframe_min == 0 and is_new_roi): # new ROI entry came into effect. - # use Open rate if open_rate > calculated sell rate + # use Open rate if open_rate > calculated exit rate return row[OPEN_IDX] if (trade_dur == 0 and ( @@ -457,11 +472,11 @@ class Backtesting: # ROI on opening candles with custom pricing can only # trigger if the entry was at Open or lower wick. # details: https: // github.com/freqtrade/freqtrade/issues/6261 - # If open_rate is < open, only allow sells below the close on red candles. + # If open_rate is < open, only allow exits below the close on red candles. raise ValueError("Opening candle ROI on red candles.") # Use the maximum between close_rate and low as we - # cannot sell outside of a candle. + # cannot exit outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. return min(max(close_rate, row[LOW_IDX]), row[HIGH_IDX]) @@ -496,7 +511,7 @@ class Backtesting: """ Rate is within candle, therefore filled""" return row[LOW_IDX] <= rate <= row[HIGH_IDX] - def _get_sell_trade_entry_for_candle(self, trade: LocalTrade, + def _get_exit_trade_entry_for_candle(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]: # Check if we need to adjust our current positions @@ -508,34 +523,35 @@ class Backtesting: if check_adjust_entry: trade = self._get_adjust_trade_entry_for_candle(trade, row) - sell_candle_time: datetime = row[DATE_IDX].to_pydatetime() + exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX] - exit_ = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] - sell = self.strategy.should_exit( - trade, row[OPEN_IDX], sell_candle_time, # type: ignore - enter=enter, exit_=exit_, + exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX] + exit_ = self.strategy.should_exit( + trade, row[OPEN_IDX], exit_candle_time, # type: ignore + enter=enter, exit_=exit_sig, low=row[LOW_IDX], high=row[HIGH_IDX] ) - if sell.exit_flag: - trade.close_date = sell_candle_time + if exit_.exit_flag: + trade.close_date = exit_candle_time trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: - closerate = self._get_close_rate(row, trade, sell, trade_dur) + closerate = self._get_close_rate(row, trade, exit_, trade_dur) except ValueError: return None # call the custom exit price,with default value as previous closerate current_profit = trade.calc_profit_ratio(closerate) order_type = self.strategy.order_types['exit'] - if sell.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): - # Custom exit pricing only for sell-signals + if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): + # Custom exit pricing only for exit-signals if order_type == 'limit': closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, default_retval=closerate)( pair=trade.pair, trade=trade, - current_time=sell_candle_time, - proposed_rate=closerate, current_profit=current_profit) + current_time=exit_candle_time, + proposed_rate=closerate, current_profit=current_profit, + exit_tag=exit_.exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: @@ -549,12 +565,12 @@ class Backtesting: pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, rate=closerate, time_in_force=time_in_force, - sell_reason=sell.exit_reason, # deprecated - exit_reason=sell.exit_reason, - current_time=sell_candle_time): + sell_reason=exit_.exit_reason, # deprecated + exit_reason=exit_.exit_reason, + current_time=exit_candle_time): return None - trade.exit_reason = sell.exit_reason + trade.exit_reason = exit_.exit_reason # Checks and adds an exit tag, after checking that the length of the # row has the length for an exit tag column @@ -562,6 +578,7 @@ class Backtesting: len(row) > EXIT_TAG_IDX and row[EXIT_TAG_IDX] is not None and len(row[EXIT_TAG_IDX]) > 0 + and exit_.exit_type in (ExitType.EXIT_SIGNAL,) ): trade.exit_reason = row[EXIT_TAG_IDX] @@ -569,8 +586,8 @@ class Backtesting: order = Order( id=self.order_id_counter, ft_trade_id=trade.id, - order_date=sell_candle_time, - order_update_date=sell_candle_time, + order_date=exit_candle_time, + order_update_date=exit_candle_time, ft_is_open=True, ft_pair=trade.pair, order_id=str(self.order_id_counter), @@ -591,8 +608,8 @@ class Backtesting: return None - def _get_sell_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]: - sell_candle_time: datetime = row[DATE_IDX].to_pydatetime() + def _get_exit_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]: + exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() if self.trading_mode == TradingMode.FUTURES: trade.funding_fees = self.exchange.calculate_funding_fees( @@ -600,37 +617,35 @@ class Backtesting: amount=trade.amount, is_short=trade.is_short, open_date=trade.open_date_utc, - close_date=sell_candle_time, + close_date=exit_candle_time, ) if self.timeframe_detail and trade.pair in self.detail_data: - sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min) + exit_candle_end = exit_candle_time + timedelta(minutes=self.timeframe_min) detail_data = self.detail_data[trade.pair] detail_data = detail_data.loc[ - (detail_data['date'] >= sell_candle_time) & - (detail_data['date'] < sell_candle_end) + (detail_data['date'] >= exit_candle_time) & + (detail_data['date'] < exit_candle_end) ].copy() if len(detail_data) == 0: # Fall back to "regular" data if no detail data was found for this candle - return self._get_sell_trade_entry_for_candle(trade, row) + return self._get_exit_trade_entry_for_candle(trade, row) detail_data.loc[:, 'enter_long'] = row[LONG_IDX] detail_data.loc[:, 'exit_long'] = row[ELONG_IDX] detail_data.loc[:, 'enter_short'] = row[SHORT_IDX] detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX] detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX] detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX] - headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'enter_tag', 'exit_tag'] - for det_row in detail_data[headers].values.tolist(): - res = self._get_sell_trade_entry_for_candle(trade, det_row) + for det_row in detail_data[HEADERS].values.tolist(): + res = self._get_exit_trade_entry_for_candle(trade, det_row) if res: return res return None else: - return self._get_sell_trade_entry_for_candle(trade, row) + return self._get_exit_trade_entry_for_candle(trade, row) def get_valid_price_and_stake( self, pair: str, row: Tuple, propose_rate: float, stake_amount: Optional[float], @@ -818,13 +833,13 @@ class Backtesting: if len(open_trades[pair]) > 0: for trade in open_trades[pair]: if trade.open_order_id and trade.nr_of_successful_entries == 0: - # Ignore trade if buy-order did not fill yet + # Ignore trade if entry-order did not fill yet continue - sell_row = data[pair][-1] + exit_row = data[pair][-1] - trade.close_date = sell_row[DATE_IDX].to_pydatetime() + trade.close_date = exit_row[DATE_IDX].to_pydatetime() trade.exit_reason = ExitType.FORCE_EXIT.value - trade.close(sell_row[OPEN_IDX], show_msg=False) + trade.close(exit_row[OPEN_IDX], show_msg=False) LocalTrade.close_bt_trade(trade) # Deepcopy object to have wallets update correctly trade1 = deepcopy(trade) @@ -889,7 +904,7 @@ class Backtesting: # Remove trade due to entry timeout expiration. return True else: - # Close additional buy order + # Close additional entry order del trade.orders[trade.orders.index(order)] if order.side == trade.exit_side: self.timedout_exit_orders += 1 @@ -902,7 +917,7 @@ class Backtesting: self, data: Dict, pair: str, row_index: int, current_time: datetime) -> Optional[Tuple]: try: # Row is treated as "current incomplete candle". - # Buy / sell signals are shifted by 1 to compensate for this. + # entry / exit signals are shifted by 1 to compensate for this. row = data[pair][row_index] except IndexError: # missing Data for one pair at the end. @@ -967,16 +982,16 @@ class Backtesting: self.dataprovider._set_dataframe_max_index(row_index) for t in list(open_trades[pair]): - # 1. Cancel expired buy/sell orders. + # 1. Cancel expired entry/exit orders. if self.check_order_cancel(t, current_time): - # Close trade due to buy timeout expiration. + # Close trade due to entry timeout expiration. open_trade_count -= 1 open_trades[pair].remove(t) self.wallets.update() else: self.check_order_replace(t, current_time, row) - # 2. Process buys. + # 2. Process entries. # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected # don't open on the last row @@ -992,7 +1007,7 @@ class Backtesting: if trade: # TODO: hacky workaround to avoid opening > max_open_trades # This emulates previous behavior - not sure if this is correct - # Prevents buying if the trade-slot was freed in this candle + # Prevents entering if the trade-slot was freed in this candle open_trade_count_start += 1 open_trade_count += 1 # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") @@ -1007,18 +1022,18 @@ class Backtesting: LocalTrade.add_bt_trade(trade) self.wallets.update() - # 4. Create sell orders (if any) + # 4. Create exit orders (if any) if not trade.open_order_id: - self._get_sell_trade_entry(trade, row) # Place sell order if necessary + self._get_exit_trade_entry(trade, row) # Place exit order if necessary - # 5. Process sell orders. + # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): trade.open_order_id = None trade.close_date = current_time trade.close(order.price, show_msg=False) - # logger.debug(f"{pair} - Backtesting sell {trade}") + # logger.debug(f"{pair} - Backtesting exit {trade}") open_trade_count -= 1 open_trades[pair].remove(trade) LocalTrade.close_bt_trade(trade) @@ -1074,7 +1089,7 @@ class Backtesting: "No data left after adjusting for startup candles.") # Use preprocessed_tmp for date generation (the trimmed dataframe). - # Backtesting will re-trim the dataframes after buy/sell signal generation. + # Backtesting will re-trim the dataframes after entry/exit signal generation. min_date, max_date = history.get_timerange(preprocessed_tmp) logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' @@ -1096,8 +1111,31 @@ class Backtesting: }) self.all_results[self.strategy.get_strategy_name()] = results + if (self.config.get('export', 'none') == 'signals' and + self.dataprovider.runmode == RunMode.BACKTEST): + self._generate_trade_signal_candles(preprocessed_tmp, results) + return min_date, max_date + def _generate_trade_signal_candles(self, preprocessed_df, bt_results): + signal_candles_only = {} + for pair in preprocessed_df.keys(): + signal_candles_only_df = DataFrame() + + pairdf = preprocessed_df[pair] + resdf = bt_results['results'] + pairresults = resdf.loc[(resdf["pair"] == pair)] + + if pairdf.shape[0] > 0: + for t, v in pairresults.open_date.items(): + allinds = pairdf.loc[(pairdf['date'] < v)] + signal_inds = allinds.iloc[[-1]] + signal_candles_only_df = pd.concat([signal_candles_only_df, signal_inds]) + + signal_candles_only[pair] = signal_candles_only_df + + self.processed_dfs[self.strategy.get_strategy_name()] = signal_candles_only + def _get_min_cached_backtest_date(self): min_backtest_date = None backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT) @@ -1156,9 +1194,13 @@ class Backtesting: else: self.results = results - if self.config.get('export', 'none') == 'trades': + if self.config.get('export', 'none') in ('trades', 'signals'): store_backtest_stats(self.config['exportfilename'], self.results) + if (self.config.get('export', 'none') == 'signals' and + self.dataprovider.runmode == RunMode.BACKTEST): + store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs) + # Results may be mixed up now. Sort them so they follow --strategy-list order. if 'strategy_list' in self.config and len(self.results) > 0: self.results['strategy_comparison'] = sorted( diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index eb8f7ec94..1dafb483c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,7 +10,7 @@ import warnings from datetime import datetime, timezone from math import ceil from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import progressbar import rapidjson @@ -290,7 +290,7 @@ class Hyperopt: self.assign_params(params_dict, 'protection') if HyperoptTools.has_space(self.config, 'roi'): - self.backtesting.strategy.minimal_roi = ( # type: ignore + self.backtesting.strategy.minimal_roi = ( self.custom_hyperopt.generate_roi_table(params_dict)) if HyperoptTools.has_space(self.config, 'stoploss'): @@ -409,6 +409,51 @@ class Hyperopt: # Store non-trimmed data - will be trimmed after signal generation. dump(preprocessed, self.data_pickle_file) + def get_asked_points(self, n_points: int) -> Tuple[List[List[Any]], List[bool]]: + """ + Enforce points returned from `self.opt.ask` have not been already evaluated + + Steps: + 1. Try to get points using `self.opt.ask` first + 2. Discard the points that have already been evaluated + 3. Retry using `self.opt.ask` up to 3 times + 4. If still some points are missing in respect to `n_points`, random sample some points + 5. Repeat until at least `n_points` points in the `asked_non_tried` list + 6. Return a list with length truncated at `n_points` + """ + def unique_list(a_list): + new_list = [] + for item in a_list: + if item not in new_list: + new_list.append(item) + return new_list + i = 0 + asked_non_tried: List[List[Any]] = [] + is_random: List[bool] = [] + while i < 5 and len(asked_non_tried) < n_points: + if i < 3: + self.opt.cache_ = {} + asked = unique_list(self.opt.ask(n_points=n_points * 5)) + is_random = [False for _ in range(len(asked))] + else: + asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5)) + is_random = [True for _ in range(len(asked))] + is_random += [rand for x, rand in zip(asked, is_random) + if x not in self.opt.Xi + and x not in asked_non_tried] + asked_non_tried += [x for x in asked + if x not in self.opt.Xi + and x not in asked_non_tried] + i += 1 + + if asked_non_tried: + return ( + asked_non_tried[:min(len(asked_non_tried), n_points)], + is_random[:min(len(asked_non_tried), n_points)] + ) + else: + return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") @@ -420,9 +465,10 @@ class Hyperopt: # We don't need exchange instance anymore while running hyperopt self.backtesting.exchange.close() - self.backtesting.exchange._api = None # type: ignore - self.backtesting.exchange._api_async = None # type: ignore + self.backtesting.exchange._api = None + self.backtesting.exchange._api_async = None self.backtesting.exchange.loop = None # type: ignore + self.backtesting.exchange._loop_lock = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore @@ -473,7 +519,7 @@ class Hyperopt: n_rest = (i + 1) * jobs - self.total_epochs current_jobs = jobs - n_rest if n_rest > 0 else jobs - asked = self.opt.ask(n_points=current_jobs) + asked, is_random = self.get_asked_points(n_points=current_jobs) f_val = self.run_optimizer_parallel(parallel, asked, i) self.opt.tell(asked, [v['loss'] for v in f_val]) @@ -492,6 +538,7 @@ class Hyperopt: # evaluations can take different time. Here they are aligned in the # order they will be shown to the user. val['is_best'] = is_best + val['is_random'] = is_random[j] self.print_results(val) if is_best: diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 8d3c3a266..0421e6e38 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -41,7 +41,8 @@ class HyperoptTools(): """ from freqtrade.resolvers.strategy_resolver import StrategyResolver directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) - strategy_objs = StrategyResolver.search_all_objects(directory, False) + strategy_objs = StrategyResolver.search_all_objects( + directory, False, config.get('recursive_strategy_search', False)) strategies = [s for s in strategy_objs if s['name'] == strategy_name] if strategies: strategy = strategies[0] @@ -310,6 +311,8 @@ class HyperoptTools(): if not has_drawdown: # Ensure compatibility with older versions of hyperopt results trials['results_metrics.max_drawdown_account'] = None + if 'is_random' not in trials.columns: + trials['is_random'] = False # New mode, using backtest result for metrics trials['results_metrics.winsdrawslosses'] = trials.apply( @@ -322,12 +325,12 @@ class HyperoptTools(): 'results_metrics.profit_total', 'results_metrics.holding_avg', 'results_metrics.max_drawdown', 'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs', - 'loss', 'is_initial_point', 'is_best']] + 'loss', 'is_initial_point', 'is_random', 'is_best']] trials.columns = [ 'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', 'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account', - 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best' + 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_random', 'is_best' ] return trials @@ -349,9 +352,11 @@ class HyperoptTools(): trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown) trials['is_profit'] = False - trials.loc[trials['is_initial_point'], 'Best'] = '* ' + trials.loc[trials['is_initial_point'] | trials['is_random'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' - trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' + trials.loc[ + (trials['is_initial_point'] | trials['is_random']) & trials['is_best'], + 'Best'] = '* Best' trials.loc[trials['Total profit'] > 0, 'is_profit'] = True trials['Trades'] = trials['Trades'].astype(str) # perc_multi = 1 if legacy_mode else 100 @@ -407,7 +412,7 @@ class HyperoptTools(): trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT, str(trials.loc[i][j]), Style.RESET_ALL) - trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit', 'is_random']) if remove_header > 0: table = tabulate.tabulate( trials.to_dict(orient='list'), tablefmt='orgtbl', diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index c08fa07a1..dd058aff4 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -9,10 +9,10 @@ from pandas import DataFrame, to_datetime from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT -from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, +from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename, - round_coin_value) +from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, + get_backtest_metadata_filename, round_coin_value) logger = logging.getLogger(__name__) @@ -45,6 +45,29 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) +def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path: + """ + Stores backtest trade signal candles + :param recordfilename: Path object, which can either be a filename or a directory. + Filenames will be appended with a timestamp right before the suffix + while for directories, /backtest-result-_signals.pkl will be used + as filename + :param stats: Dict containing the backtesting signal candles + """ + if recordfilename.is_dir(): + filename = (recordfilename / + f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl') + else: + filename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl' + ) + + file_dump_joblib(filename, candles) + + return filename + + def _get_line_floatfmt(stake_currency: str) -> List[str]: """ Generate floatformat (goes in line with _generate_result_line()) @@ -241,7 +264,7 @@ def generate_edge_table(results: dict) -> str: # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") def _get_resample_from_period(period: str) -> str: @@ -423,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str], 'profit_total_abs': results['profit_abs'].sum(), 'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(), 'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(), + 'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']), 'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT), 'backtest_start_ts': int(min_date.timestamp() * 1000), 'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT), @@ -723,6 +747,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{strat_results['profit_total']:.2%}"), + ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), ('Trades per day', strat_results['trades_per_day']), ('Avg. daily profit %', f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a9c07f12c..2cacc06e2 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -429,12 +429,10 @@ class LocalTrade(): def __repr__(self): open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' - leverage = self.leverage or 1.0 - is_short = self.is_short or False return ( f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' - f'is_short={is_short}, leverage={leverage}, ' + f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})' ) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 3ab461041..74b28dffe 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -44,7 +44,7 @@ class IResolver: @classmethod def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, - extra_dir: Optional[str] = None) -> List[Path]: + extra_dirs: List[str] = []) -> List[Path]: abs_paths: List[Path] = [] if cls.initial_search_path: @@ -53,9 +53,9 @@ class IResolver: if user_subdir: abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) - if extra_dir: - # Add extra directory to the top of the search paths - abs_paths.insert(0, Path(extra_dir).resolve()) + # Add extra directory to the top of the search paths + for dir in extra_dirs: + abs_paths.insert(0, Path(dir).resolve()) return abs_paths @@ -164,9 +164,13 @@ class IResolver: :return: Object instance or None """ + extra_dirs: List[str] = [] + if extra_dir: + extra_dirs.append(extra_dir) + abs_paths = cls.build_search_paths(config, user_subdir=cls.user_subdir, - extra_dir=extra_dir) + extra_dirs=extra_dirs) found_object = cls._load_object(paths=abs_paths, object_name=object_name, kwargs=kwargs) @@ -178,18 +182,25 @@ class IResolver: ) @classmethod - def search_all_objects(cls, directory: Path, - enum_failed: bool) -> List[Dict[str, Any]]: + def search_all_objects(cls, directory: Path, enum_failed: bool, + recursive: bool = False) -> List[Dict[str, Any]]: """ Searches a directory for valid objects :param directory: Path to search :param enum_failed: If True, will return None for modules which fail. Otherwise, failing modules are skipped. + :param recursive: Recursively walk directory tree searching for strategies :return: List of dicts containing 'name', 'class' and 'location' entries """ logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") objects = [] for entry in directory.iterdir(): + if ( + recursive and entry.is_dir() + and not entry.name.startswith('__') + and not entry.name.startswith('.') + ): + objects.extend(cls.search_all_objects(entry, enum_failed, recursive=recursive)) # Only consider python files if entry.suffix != '.py': logger.debug('Ignoring %s', entry) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 76515026c..44d590b67 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -7,8 +7,9 @@ import logging import tempfile from base64 import urlsafe_b64decode from inspect import getfullargspec +from os import walk from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from freqtrade.configuration.config_validation import validate_migrated_strategy_settings from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES @@ -216,15 +217,19 @@ class StrategyResolver(IResolver): raise OperationalException( "`populate_exit_trend` or `populate_sell_trend` must be implemented.") - strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) - strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) - strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) + _populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) + _buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) + _sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) if any(x == 2 for x in [ - strategy._populate_fun_len, - strategy._buy_fun_len, - strategy._sell_fun_len + _populate_fun_len, + _buy_fun_len, + _sell_fun_len ]): - strategy.INTERFACE_VERSION = 1 + raise OperationalException( + "Strategy Interface v1 is no longer supported. " + "Please update your strategy to implement " + "`populate_indicators`, `populate_entry_trend` and `populate_exit_trend` " + "with the metadata argument. ") return strategy @staticmethod @@ -237,10 +242,19 @@ class StrategyResolver(IResolver): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ + if config.get('recursive_strategy_search', False): + extra_dirs: List[str] = [ + path[0] for path in walk(f"{config['user_data_dir']}/{USERPATH_STRATEGIES}") + ] # sub-directories + else: + extra_dirs = [] + + if extra_dir: + extra_dirs.append(extra_dir) abs_paths = StrategyResolver.build_search_paths(config, user_subdir=USERPATH_STRATEGIES, - extra_dir=extra_dir) + extra_dirs=extra_dirs) if ":" in strategy_name: logger.info("loading base64 encoded strategy") diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 5021c99f9..a8b9873d7 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -253,7 +253,8 @@ def list_strategies(config=Depends(get_config)): directory = Path(config.get( 'strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) from freqtrade.resolvers.strategy_resolver import StrategyResolver - strategies = StrategyResolver.search_all_objects(directory, False) + strategies = StrategyResolver.search_all_objects( + directory, False, config.get('recursive_strategy_search', False)) strategies = sorted(strategies, key=lambda x: x['name']) return {'strategies': [x['name'] for x in strategies]} diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c2531fec3..1a9be4503 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -943,7 +943,7 @@ class Telegram(RPCHandler): else: fiat_currency = self._config.get('fiat_display_currency', '') try: - statlist, head, fiat_profit_sum = self._rpc._rpc_status_table( + statlist, _, _ = self._rpc._rpc_status_table( self._config['stake_currency'], fiat_currency) except RPCException: self._send_msg(msg='No open trade found.') diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 0dd5320cd..7dfdf5a8c 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -23,7 +23,7 @@ class InformativeData: def informative(timeframe: str, asset: str = '', fmt: Optional[Union[str, Callable[[Any], str]]] = None, *, - candle_type: Optional[CandleType] = None, + candle_type: Optional[Union[CandleType, str]] = None, ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: """ A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 01473391a..0a7580b6f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -3,7 +3,6 @@ IStrategy interface This module defines the interface to apply for strategies """ import logging -import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple, Union @@ -44,14 +43,11 @@ class IStrategy(ABC, HyperStrategyMixin): """ # Strategy interface version # Default to version 2 - # Version 1 is the initial interface without metadata dict + # Version 1 is the initial interface without metadata dict - deprecated and no longer supported. # Version 2 populate_* include metadata dict # Version 3 - First version with short and leverage support INTERFACE_VERSION: int = 3 - _populate_fun_len: int = 0 - _buy_fun_len: int = 0 - _sell_fun_len: int = 0 _ft_params_from_file: Dict # associated minimal roi minimal_roi: Dict = {} @@ -114,7 +110,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. - dp: Optional[DataProvider] + dp: DataProvider wallets: Optional[Wallets] = None # Filled from configuration stake_currency: str @@ -359,7 +355,7 @@ class IStrategy(ABC, HyperStrategyMixin): def custom_exit_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: """ Custom exit price logic, returning the new exit price. @@ -372,6 +368,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param exit_tag: Exit reason. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New exit price value if provided """ @@ -1116,12 +1113,7 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe = _create_and_merge_informative_pair( self, dataframe, metadata, inf_data, populate_fn) - if self._populate_fun_len == 2: - warnings.warn("deprecated - check out the Sample strategy to see " - "the current function headers!", DeprecationWarning) - return self.populate_indicators(dataframe) # type: ignore - else: - return self.populate_indicators(dataframe, metadata) + return self.populate_indicators(dataframe, metadata) def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ @@ -1135,12 +1127,7 @@ class IStrategy(ABC, HyperStrategyMixin): logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.") - if self._buy_fun_len == 2: - warnings.warn("deprecated - check out the Sample strategy to see " - "the current function headers!", DeprecationWarning) - df = self.populate_buy_trend(dataframe) # type: ignore - else: - df = self.populate_entry_trend(dataframe, metadata) + df = self.populate_entry_trend(dataframe, metadata) if 'enter_long' not in df.columns: df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns') @@ -1155,14 +1142,8 @@ class IStrategy(ABC, HyperStrategyMixin): currently traded pair :return: DataFrame with exit column """ - logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.") - if self._sell_fun_len == 2: - warnings.warn("deprecated - check out the Sample strategy to see " - "the current function headers!", DeprecationWarning) - df = self.populate_sell_trend(dataframe) # type: ignore - else: - df = self.populate_exit_trend(dataframe, metadata) + df = self.populate_exit_trend(dataframe, metadata) if 'exit_long' not in df.columns: df = df.rename({'sell': 'exit_long'}, axis='columns') return df diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index a36cb3dbb..43728dc1f 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -56,12 +56,18 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, # Combine the 2 dataframes # all indicators on the informative sample MUST be calculated before this point - dataframe = pd.merge(dataframe, informative, left_on='date', - right_on=date_merge, how='left') + if ffill: + # https://pandas.pydata.org/docs/user_guide/merging.html#timeseries-friendly-merging + # merge_ordered - ffill method is 2.5x faster than seperate ffill() + dataframe = pd.merge_ordered(dataframe, informative, fill_method="ffill", left_on='date', + right_on=date_merge, how='left') + else: + dataframe = pd.merge(dataframe, informative, left_on='date', + right_on=date_merge, how='left') dataframe = dataframe.drop(date_merge, axis=1) - if ffill: - dataframe = dataframe.ffill() + # if ffill: + # dataframe = dataframe.ffill() return dataframe diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index db64b3e07..176f567c7 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -58,7 +58,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, exit_tag: Optional[str], **kwargs) -> float: """ Custom exit price logic, returning the new exit price. @@ -71,6 +71,7 @@ def custom_exit_price(self, pair: str, trade: 'Trade', :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param exit_tag: Exit reason. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New exit price value if provided """ diff --git a/mkdocs.yml b/mkdocs.yml index 8b36ba699..a43322f78 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -29,6 +29,7 @@ nav: - Data Analysis: - Jupyter Notebooks: data-analysis.md - Strategy analysis: strategy_analysis_example.md + - Backtest analysis: advanced-backtesting.md - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - Edge Positioning: edge.md diff --git a/requirements-dev.txt b/requirements-dev.txt index 4fb4456f0..c4fe366a5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,7 +9,7 @@ flake8==4.0.1 flake8-tidy-imports==4.6.0 mypy==0.942 pre-commit==2.18.1 -pytest==7.1.1 +pytest==7.1.2 pytest-asyncio==0.18.3 pytest-cov==3.0.0 pytest-mock==3.7.0 @@ -24,8 +24,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.0.1 types-filelock==3.2.5 -types-requests==2.27.19 +types-requests==2.27.20 types-tabulate==0.8.7 - -# Extensions to datetime library -types-python-dateutil==2.8.11 +types-python-dateutil==2.8.12 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index aeb7be035..32fc3f4b9 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,5 +6,4 @@ scipy==1.8.0 scikit-learn==1.0.2 scikit-optimize==0.9.0 filelock==3.6.0 -joblib==1.1.0 progressbar2==4.0.0 diff --git a/requirements-plot.txt b/requirements-plot.txt index 9eb6a10a3..d9faed301 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -2,4 +2,3 @@ -r requirements.txt plotly==5.7.0 - diff --git a/requirements.txt b/requirements.txt index 24f267b67..de14b9f2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.3 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.79.81 +ccxt==1.80.61 # Pin cryptography for now due to rust build errors with piwheels cryptography==36.0.2 aiohttp==3.8.1 @@ -20,6 +20,7 @@ pycoingecko==2.2.0 jinja2==3.1.1 tables==3.7.0 blosc==1.10.6 +joblib==1.1.0 # find first, C search in arrays py_find_1st==1.1.5 diff --git a/setup.cfg b/setup.cfg index f4a90bda7..edbd320c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,11 @@ exclude = [mypy] ignore_missing_imports = True +warn_unused_ignores = True +exclude = (?x)( + ^build_helpers\.py$ + ) + [mypy-tests.*] ignore_errors = True diff --git a/setup.sh b/setup.sh index 5cde1a589..e0b010387 100755 --- a/setup.sh +++ b/setup.sh @@ -90,7 +90,7 @@ function updateenv() { echo "pip install completed" echo if [[ $dev =~ ^[Yy]$ ]]; then - ${PYTHON} -m pre-commit install + ${PYTHON} -m pre_commit install if [ $? -ne 0 ]; then echo "Failed installing pre-commit" exit 1 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1431bd22a..37eeda86a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -847,7 +847,7 @@ def test_start_convert_trades(mocker, caplog): assert convert_mock.call_count == 1 -def test_start_list_strategies(mocker, caplog, capsys): +def test_start_list_strategies(capsys): args = [ "list-strategies", @@ -859,8 +859,8 @@ def test_start_list_strategies(mocker, caplog, capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" not in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" not in captured.out assert CURRENT_TEST_STRATEGY in captured.out # Test regular output @@ -874,8 +874,8 @@ def test_start_list_strategies(mocker, caplog, capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" in captured.out assert CURRENT_TEST_STRATEGY in captured.out # Test color output @@ -888,10 +888,30 @@ def test_start_list_strategies(mocker, caplog, capsys): # pargs['config'] = None start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestStrategyLegacyV1" in captured.out - assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" in captured.out assert CURRENT_TEST_STRATEGY in captured.out assert "LOAD FAILED" in captured.out + # Recursive + assert "TestStrategyNoImplements" not in captured.out + + # Test recursive + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), + '--no-color', + '--recursive-strategy-search' + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "StrategyTestV2" in captured.out + assert "strategy_test_v2.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "TestStrategyNoImplements" in captured.out + assert str(Path("broken_strats/broken_futures_strategies.py")) in captured.out def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): diff --git a/tests/conftest.py b/tests/conftest.py index 9cbb95531..cc07de1de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1632,40 +1632,6 @@ def limit_buy_order(limit_buy_order_open): return order -@pytest.fixture(scope='function') -def market_buy_order(): - return { - 'id': 'mocked_market_buy', - 'type': 'market', - 'side': 'buy', - 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), - 'price': 0.00004099, - 'amount': 91.99181073, - 'filled': 91.99181073, - 'remaining': 0.0, - 'status': 'closed' - } - - -@pytest.fixture -def market_sell_order(): - return { - 'id': 'mocked_limit_sell', - 'type': 'market', - 'side': 'sell', - 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), - 'price': 0.00004173, - 'amount': 91.99181073, - 'filled': 91.99181073, - 'remaining': 0.0, - 'status': 'closed' - } - - @pytest.fixture def limit_buy_order_old(): return { @@ -2672,6 +2638,7 @@ def saved_hyperopt_results(): 'total_profit': -0.00125625, 'current_epoch': 1, 'is_initial_point': True, + 'is_random': False, 'is_best': True, }, { @@ -2688,6 +2655,7 @@ def saved_hyperopt_results(): 'total_profit': 6.185e-05, 'current_epoch': 2, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': 14.241196856510731, @@ -2698,6 +2666,7 @@ def saved_hyperopt_results(): 'total_profit': -0.13639474, 'current_epoch': 3, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': 100000, @@ -2705,7 +2674,7 @@ def saved_hyperopt_results(): 'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501 'results_metrics': {'total_trades': 0, 'wins': 0, 'draws': 0, 'losses': 0, 'profit_mean': None, 'profit_median': None, 'profit_total': 0, 'profit': 0.0, 'holding_avg': timedelta()}, # noqa: E501 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 - 'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False + 'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_random': False, 'is_best': False # noqa: E501 }, { 'loss': 0.22195522184191518, 'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501 @@ -2715,6 +2684,7 @@ def saved_hyperopt_results(): 'total_profit': -0.002480140000000001, 'current_epoch': 5, 'is_initial_point': True, + 'is_random': False, 'is_best': True }, { 'loss': 0.545315889154162, @@ -2725,6 +2695,7 @@ def saved_hyperopt_results(): 'total_profit': -0.0041773, 'current_epoch': 6, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': 4.713497421432944, @@ -2737,6 +2708,7 @@ def saved_hyperopt_results(): 'total_profit': -0.06339929, 'current_epoch': 7, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': 20.0, # noqa: E501 @@ -2747,6 +2719,7 @@ def saved_hyperopt_results(): 'total_profit': 0.0, 'current_epoch': 8, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': 2.4731817780991223, @@ -2757,6 +2730,7 @@ def saved_hyperopt_results(): 'total_profit': -0.044050070000000004, # noqa: E501 'current_epoch': 9, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': -0.2604606005845212, # noqa: E501 @@ -2767,6 +2741,7 @@ def saved_hyperopt_results(): 'total_profit': 0.00021629, 'current_epoch': 10, 'is_initial_point': True, + 'is_random': False, 'is_best': True }, { 'loss': 4.876465945994304, # noqa: E501 @@ -2778,6 +2753,7 @@ def saved_hyperopt_results(): 'total_profit': -0.07436117, 'current_epoch': 11, 'is_initial_point': True, + 'is_random': False, 'is_best': False }, { 'loss': 100000, @@ -2788,6 +2764,7 @@ def saved_hyperopt_results(): 'total_profit': 0, 'current_epoch': 12, 'is_initial_point': True, + 'is_random': False, 'is_best': False } ] @@ -2935,14 +2912,6 @@ def limit_order(limit_buy_order_usdt, limit_sell_order_usdt): } -@pytest.fixture(scope='function') -def market_order(market_buy_order_usdt, market_sell_order_usdt): - return { - 'buy': market_buy_order_usdt, - 'sell': market_sell_order_usdt - } - - @pytest.fixture(scope='function') def limit_order_open(limit_buy_order_usdt_open, limit_sell_order_usdt_open): return { diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 2b53e4900..eaf703b2d 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -8,13 +8,13 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_csum, - calculate_market_change, calculate_max_drawdown, - calculate_underwater, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, - get_latest_backtest_filename, get_latest_hyperopt_file, - load_backtest_data, load_backtest_metadata, load_trades, - load_trades_from_db) +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr, + calculate_csum, calculate_market_change, + calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit, + extract_trades_of_period, get_latest_backtest_filename, + get_latest_hyperopt_file, load_backtest_data, + load_backtest_metadata, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from freqtrade.exceptions import OperationalException from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades @@ -336,6 +336,19 @@ def test_calculate_csum(testdatadir): csum_min, csum_max = calculate_csum(DataFrame()) +@pytest.mark.parametrize('start,end,days, expected', [ + (64900, 176000, 3 * 365, 0.3945), + (64900, 176000, 365, 1.7119), + (1000, 1000, 365, 0.0), + (1000, 1500, 365, 0.5), + (1000, 1500, 100, 3.3927), # sub year + (0.01000000, 0.01762792, 120, 4.6087), # sub year BTC values +]) +def test_calculate_cagr(start, end, days, expected): + + assert round(calculate_cagr(days, start, end), 4) == expected + + def test_calculate_max_drawdown2(): values = [0.011580, 0.010048, 0.011340, 0.012161, 0.010416, 0.010009, 0.020024, -0.024662, -0.022350, 0.020496, -0.029859, -0.030511, 0.010041, 0.010872, diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index a43e82b22..b30d6f998 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock import arrow import numpy as np import pytest -from pandas import DataFrame, to_datetime +from pandas import DataFrame from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo @@ -30,49 +30,6 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, tests_start_time = arrow.get(2018, 10, 3) timeframe_in_minute = 60 -# Helpers for this test file - - -def _validate_ohlc(buy_ohlc_sell_matrice): - for index, ohlc in enumerate(buy_ohlc_sell_matrice): - # if not high < open < low or not high < close < low - if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: - raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') - return True - - -def _build_dataframe(buy_ohlc_sell_matrice): - _validate_ohlc(buy_ohlc_sell_matrice) - data = [] - for ohlc in buy_ohlc_sell_matrice: - d = { - 'date': tests_start_time.shift( - minutes=( - ohlc[0] * - timeframe_in_minute)).int_timestamp * - 1000, - 'buy': ohlc[1], - 'open': ohlc[2], - 'high': ohlc[3], - 'low': ohlc[4], - 'close': ohlc[5], - 'sell': ohlc[6]} - data.append(d) - - frame = DataFrame(data) - frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) - - return frame - - -def _time_on_candle(number): - return np.datetime64(tests_start_time.shift( - minutes=(number * timeframe_in_minute)).int_timestamp * 1000, 'ms') - - # End helper functions # Open trade should be removed from the end tc0 = BTContainer(data=[ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a6918b6d4..689ffa4ce 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -909,7 +909,7 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' - r'for the exchange ".*" and this exchange ' + r'for the exchange .* and this exchange ' r'is therefore not supported. *'): Exchange(default_conf) @@ -930,7 +930,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' - r'for the exchange ".*" and this exchange ' + r'for the exchange .* and this exchange ' r'is therefore not supported. *'): Exchange(default_conf) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 08957acf9..a51e1b654 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -384,14 +384,16 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') sbs = mocker.patch('freqtrade.optimize.backtesting.store_backtest_stats') + sbc = mocker.patch('freqtrade.optimize.backtesting.store_backtest_signal_candles') mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) default_conf['timeframe'] = '1m' default_conf['datadir'] = testdatadir - default_conf['export'] = 'trades' + default_conf['export'] = 'signals' default_conf['exportfilename'] = 'export.txt' default_conf['timerange'] = '-1510694220' + default_conf['runmode'] = RunMode.BACKTEST backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) @@ -407,6 +409,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: assert backtesting.strategy.dp._pairlists is not None assert backtesting.strategy.bot_loop_start.call_count == 1 assert sbs.call_count == 1 + assert sbc.call_count == 1 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: @@ -497,7 +500,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti Backtesting(default_conf) # Multiple strategies - default_conf['strategy_list'] = [CURRENT_TEST_STRATEGY, 'TestStrategyLegacyV1'] + default_conf['strategy_list'] = [CURRENT_TEST_STRATEGY, 'StrategyTestV2'] with pytest.raises(OperationalException, match='PrecisionFilter not allowed for backtesting multiple strategies.'): Backtesting(default_conf) @@ -711,7 +714,7 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: ) # No data available. - res = backtesting._get_sell_trade_entry(trade, row_sell) + res = backtesting._get_exit_trade_entry(trade, row_sell) assert res is not None assert res.exit_reason == ExitType.ROI.value assert res.close_date_utc == datetime(2020, 1, 1, 5, 0, tzinfo=timezone.utc) @@ -724,13 +727,13 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: [], columns=['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', 'enter_short', 'exit_short', 'long_tag', 'short_tag', 'exit_tag']) - res = backtesting._get_sell_trade_entry(trade, row) + res = backtesting._get_exit_trade_entry(trade, row) assert res is None # Assign backtest-detail data backtesting.detail_data[pair] = row_detail - res = backtesting._get_sell_trade_entry(trade, row_sell) + res = backtesting._get_exit_trade_entry(trade, row_sell) assert res is not None assert res.exit_reason == ExitType.ROI.value # Sell at minute 3 (not available above!) @@ -1195,7 +1198,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): '--disable-max-market-positions', '--strategy-list', CURRENT_TEST_STRATEGY, - 'TestStrategyLegacyV1', + 'StrategyTestV2', ] args = get_args(args) start_backtesting(args) @@ -1218,14 +1221,13 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV2', ] for line in exists: assert log_has(line, caplog) -@pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdatadir, capsys): default_conf.update({ "use_exit_signal": True, @@ -1307,7 +1309,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat '--breakdown', 'day', '--strategy-list', CURRENT_TEST_STRATEGY, - 'TestStrategyLegacyV1', + 'StrategyTestV2', ] args = get_args(args) start_backtesting(args) @@ -1324,7 +1326,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV2', ] for line in exists: @@ -1339,6 +1341,39 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat assert 'STRATEGY SUMMARY' in captured.out +@pytest.mark.filterwarnings("ignore:deprecated") +def test_backtest_start_futures_noliq(default_conf_usdt, mocker, + caplog, testdatadir, capsys): + # Tests detail-data loading + default_conf_usdt.update({ + "trading_mode": "futures", + "margin_mode": "isolated", + "use_exit_signal": True, + "exit_profit_only": False, + "exit_profit_offset": 0.0, + "ignore_roi_if_entry_signal": False, + "strategy": CURRENT_TEST_STRATEGY, + }) + patch_exchange(mocker) + + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) + # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf_usdt) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '1h', + ] + args = get_args(args) + with pytest.raises(OperationalException, match=r"Pairs .* got no leverage tiers available\."): + start_backtesting(args) + + @pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_nomock_futures(default_conf_usdt, mocker, caplog, testdatadir, capsys): @@ -1589,7 +1624,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda min_backtest_date = now - timedelta(weeks=4) load_backtest_metadata = MagicMock(return_value={ 'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()}, - 'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()} + 'StrategyTestV3': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()} }) load_backtest_stats = MagicMock(side_effect=[ { @@ -1598,9 +1633,9 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'strategy_comparison': [{'key': 'StrategyTestV2'}] }, { - 'metadata': {'TestStrategyLegacyV1': {'run_id': '2'}}, - 'strategy': {'TestStrategyLegacyV1': {}}, - 'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}] + 'metadata': {'StrategyTestV3': {'run_id': '2'}}, + 'strategy': {'StrategyTestV3': {}}, + 'strategy_comparison': [{'key': 'StrategyTestV3'}] } ]) mocker.patch('pathlib.Path.glob', return_value=[ @@ -1624,7 +1659,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda '--cache', cache, '--strategy-list', 'StrategyTestV2', - 'TestStrategyLegacyV1', + 'StrategyTestV3', ] args = get_args(args) start_backtesting(args) @@ -1646,7 +1681,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda assert backtestmock.call_count == 2 exists = [ 'Running backtesting for Strategy StrategyTestV2', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV3', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', ] @@ -1654,12 +1689,12 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda assert backtestmock.call_count == 0 exists = [ 'Reusing result of previous backtest for StrategyTestV2', - 'Reusing result of previous backtest for TestStrategyLegacyV1', + 'Reusing result of previous backtest for StrategyTestV3', ] else: exists = [ 'Reusing result of previous backtest for StrategyTestV2', - 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Running backtesting for Strategy StrategyTestV3', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', ] diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index ce6ae1880..75944390e 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -41,6 +41,7 @@ def generate_result_metrics(): 'max_drawdown_abs': 0.001, 'loss': 0.001, 'is_initial_point': 0.001, + 'is_random': False, 'is_best': 1, } @@ -247,6 +248,7 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None: 'total_profit': 0, 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) 'is_initial_point': False, + 'is_random': False, 'is_best': True } ) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 05c0bf575..ff8d420b3 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -2,6 +2,7 @@ import re from datetime import timedelta from pathlib import Path +import joblib import pandas as pd import pytest from arrow import Arrow @@ -19,6 +20,7 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene generate_periodic_breakdown_stats, generate_strategy_comparison, generate_trading_stats, show_sorted_pairlist, + store_backtest_signal_candles, store_backtest_stats, text_table_bt_results, text_table_exit_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver @@ -201,6 +203,62 @@ def test_store_backtest_stats(testdatadir, mocker): assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult')) +def test_store_backtest_candles(testdatadir, mocker): + + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_joblib') + + candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} + + # mock directory exporting + store_backtest_signal_candles(testdatadir, candle_dict) + + assert dump_mock.call_count == 1 + assert isinstance(dump_mock.call_args_list[0][0][0], Path) + assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) + + dump_mock.reset_mock() + # mock file exporting + filename = Path(testdatadir / 'testresult') + store_backtest_signal_candles(filename, candle_dict) + assert dump_mock.call_count == 1 + assert isinstance(dump_mock.call_args_list[0][0][0], Path) + # result will be testdatadir / testresult-_signals.pkl + assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl')) + dump_mock.reset_mock() + + +def test_write_read_backtest_candles(tmpdir): + + candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} + + # test directory exporting + stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict) + scp = open(stored_file, "rb") + pickled_signal_candles = joblib.load(scp) + scp.close() + + assert pickled_signal_candles.keys() == candle_dict.keys() + assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() + assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ + .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) + + _clean_test_file(stored_file) + + # test file exporting + filename = Path(tmpdir / 'testresult') + stored_file = store_backtest_signal_candles(filename, candle_dict) + scp = open(stored_file, "rb") + pickled_signal_candles = joblib.load(scp) + scp.close() + + assert pickled_signal_candles.keys() == candle_dict.keys() + assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys() + assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \ + .equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC']) + + _clean_test_file(stored_file) + + def test_generate_pair_metrics(): results = pd.DataFrame( diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index af8361571..4910213b4 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1389,7 +1389,6 @@ def test_api_strategies(botclient): 'StrategyTestV2', 'StrategyTestV3', 'StrategyTestV3Futures', - 'TestStrategyLegacyV1', ]} diff --git a/tests/strategy/strats/broken_strats/legacy_strategy_v1.py b/tests/strategy/strats/broken_strats/legacy_strategy_v1.py new file mode 100644 index 000000000..f3b8c2696 --- /dev/null +++ b/tests/strategy/strats/broken_strats/legacy_strategy_v1.py @@ -0,0 +1,30 @@ +# type: ignore +from pandas import DataFrame + +from freqtrade.strategy import IStrategy + + +# Dummy strategy - no longer loads but raises an exception. +class TestStrategyLegacyV1(IStrategy): + + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + stoploss = -0.10 + + timeframe = '5m' + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + + return dataframe diff --git a/tests/strategy/strats/legacy_strategy_v1.py b/tests/strategy/strats/legacy_strategy_v1.py deleted file mode 100644 index bad2aa40d..000000000 --- a/tests/strategy/strats/legacy_strategy_v1.py +++ /dev/null @@ -1,85 +0,0 @@ - -# --- Do not remove these libs --- -# Add your lib to import here -import talib.abstract as ta -from pandas import DataFrame - -from freqtrade.strategy import IStrategy - - -# -------------------------------- - -# This class is a sample. Feel free to customize it. -class TestStrategyLegacyV1(IStrategy): - """ - This is a test strategy using the legacy function headers, which will be - removed in a future update. - Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py - for a uptodate version of this template. - """ - - # Minimal ROI designed for the strategy. - # This attribute will be overridden if the config file contains "minimal_roi" - minimal_roi = { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 - } - - # Optimal stoploss designed for the strategy - # This attribute will be overridden if the config file contains "stoploss" - stoploss = -0.10 - - timeframe = '5m' - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - - return dataframe - - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 30) & - (dataframe['tema'] > dataframe['tema'].shift(1)) & - (dataframe['volume'] > 0) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 70) & - (dataframe['tema'] < dataframe['tema'].shift(1)) & - (dataframe['volume'] > 0) - ), - 'sell'] = 1 - return dataframe diff --git a/tests/strategy/strats/strategy_test_v2.py b/tests/strategy/strats/strategy_test_v2.py index 8996b227a..85ff856e1 100644 --- a/tests/strategy/strats/strategy_test_v2.py +++ b/tests/strategy/strats/strategy_test_v2.py @@ -56,19 +56,6 @@ class StrategyTestV2(IStrategy): # By default this strategy does not use Position Adjustments position_adjustment_enable = False - def informative_pairs(self): - """ - Define additional, informative pair/interval combinations to be cached from the exchange. - These pair/interval combinations are non-tradeable, unless they are part - of the whitelist as well. - For more information, please consult the documentation - :return: List of tuples in the format (pair, interval) - Sample: return [("ETH/USDT", "5m"), - ("BTC/USDT", "15m"), - ] - """ - return [] - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a86d69135..ea81fe968 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -686,7 +686,7 @@ def test_is_pair_locked(default_conf): def test_is_informative_pairs_callback(default_conf): - default_conf.update({'strategy': 'TestStrategyLegacyV1'}) + default_conf.update({'strategy': 'StrategyTestV2'}) strategy = StrategyResolver.load_strategy(default_conf) # Should return empty # Uses fallback to base implementation diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 65fb9f6dc..244fd3919 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -68,6 +68,21 @@ def test_merge_informative_pair(): assert result.iloc[7]['date_1h'] == result.iloc[4]['date'] assert result.iloc[8]['date_1h'] == result.iloc[4]['date'] + informative = generate_test_data('1h', 40) + result = merge_informative_pair(data, informative, '15m', '1h', ffill=False) + # First 3 rows are empty + assert result.iloc[0]['date_1h'] is pd.NaT + assert result.iloc[1]['date_1h'] is pd.NaT + assert result.iloc[2]['date_1h'] is pd.NaT + # Next 4 rows contain the starting date (0:00) + assert result.iloc[3]['date_1h'] == result.iloc[0]['date'] + assert result.iloc[4]['date_1h'] is pd.NaT + assert result.iloc[5]['date_1h'] is pd.NaT + assert result.iloc[6]['date_1h'] is pd.NaT + # Next 4 rows contain the next Hourly date original date row 4 + assert result.iloc[7]['date_1h'] == result.iloc[4]['date'] + assert result.iloc[8]['date_1h'] is pd.NaT + def test_merge_informative_pair_same(): data = generate_test_data('15m', 40) diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index e74a2a022..3ed1eb0ce 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging -import warnings from base64 import urlsafe_b64encode from pathlib import Path @@ -35,7 +34,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 6 + assert len(strategies) == 5 assert isinstance(strategies[0], dict) @@ -43,10 +42,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 7 + assert len(strategies) == 6 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 6 + assert len([x for x in strategies if x['class'] is not None]) == 5 assert len([x for x in strategies if x['class'] is None]) == 1 @@ -100,7 +99,7 @@ def test_load_strategy_noname(default_conf): @pytest.mark.filterwarnings("ignore:deprecated") -@pytest.mark.parametrize('strategy_name', ['StrategyTestV2', 'TestStrategyLegacyV1']) +@pytest.mark.parametrize('strategy_name', ['StrategyTestV2']) def test_strategy_pre_v3(result, default_conf, strategy_name): default_conf.update({'strategy': strategy_name}) @@ -346,40 +345,6 @@ def test_strategy_override_use_exit_profit_only(caplog, default_conf): assert log_has("Override strategy 'exit_profit_only' with value in config file: True.", caplog) -@pytest.mark.filterwarnings("ignore:deprecated") -def test_deprecate_populate_indicators(result, default_conf): - default_location = Path(__file__).parent / "strats" - default_conf.update({'strategy': 'TestStrategyLegacyV1', - 'strategy_path': default_location}) - strategy = StrategyResolver.load_strategy(default_conf) - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - indicators = strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - check out the Sample strategy to see the current function headers!" \ - in str(w[-1].message) - - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - strategy.advise_entry(indicators, {'pair': 'ETH/BTC'}) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - check out the Sample strategy to see the current function headers!" \ - in str(w[-1].message) - - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - strategy.advise_exit(indicators, {'pair': 'ETH_BTC'}) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - check out the Sample strategy to see the current function headers!" \ - in str(w[-1].message) - - @pytest.mark.filterwarnings("ignore:deprecated") def test_missing_implements(default_conf, caplog): @@ -438,33 +403,14 @@ def test_missing_implements(default_conf, caplog): StrategyResolver.load_strategy(default_conf) -@pytest.mark.filterwarnings("ignore:deprecated") -def test_call_deprecated_function(result, default_conf, caplog): - default_location = Path(__file__).parent / "strats" +def test_call_deprecated_function(default_conf): + default_location = Path(__file__).parent / "strats/broken_strats/" del default_conf['timeframe'] default_conf.update({'strategy': 'TestStrategyLegacyV1', 'strategy_path': default_location}) - strategy = StrategyResolver.load_strategy(default_conf) - metadata = {'pair': 'ETH/BTC'} - - # Make sure we are using a legacy function - assert strategy._populate_fun_len == 2 - assert strategy._buy_fun_len == 2 - assert strategy._sell_fun_len == 2 - assert strategy.INTERFACE_VERSION == 1 - assert strategy.timeframe == '5m' - - indicator_df = strategy.advise_indicators(result, metadata=metadata) - assert isinstance(indicator_df, DataFrame) - assert 'adx' in indicator_df.columns - - enterdf = strategy.advise_entry(result, metadata=metadata) - assert isinstance(enterdf, DataFrame) - assert 'enter_long' in enterdf.columns - - exitdf = strategy.advise_exit(result, metadata=metadata) - assert isinstance(exitdf, DataFrame) - assert 'exit_long' in exitdf + with pytest.raises(OperationalException, + match=r"Strategy Interface v1 is no longer supported.*"): + StrategyResolver.load_strategy(default_conf) def test_strategy_interface_versioning(result, default_conf): @@ -472,10 +418,6 @@ def test_strategy_interface_versioning(result, default_conf): strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} - # Make sure we are using a legacy function - assert strategy._populate_fun_len == 3 - assert strategy._buy_fun_len == 3 - assert strategy._sell_fun_len == 3 assert strategy.INTERFACE_VERSION == 2 indicator_df = strategy.advise_indicators(result, metadata=metadata) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2026872de..2bef11de2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -717,12 +717,12 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) (True, 'spot', 'gateio', None, 0.0, None), (False, 'spot', 'okx', None, 0.0, None), (True, 'spot', 'okx', None, 0.0, None), - (True, 'futures', 'binance', 'isolated', 0.0, 11.89108910891089), - (False, 'futures', 'binance', 'isolated', 0.0, 8.070707070707071), + (True, 'futures', 'binance', 'isolated', 0.0, 11.88151815181518), + (False, 'futures', 'binance', 'isolated', 0.0, 8.080471380471382), (True, 'futures', 'gateio', 'isolated', 0.0, 11.87413417771621), (False, 'futures', 'gateio', 'isolated', 0.0, 8.085708510208207), - (True, 'futures', 'binance', 'isolated', 0.05, 11.796534653465345), - (False, 'futures', 'binance', 'isolated', 0.05, 8.167171717171717), + (True, 'futures', 'binance', 'isolated', 0.05, 11.7874422442244), + (False, 'futures', 'binance', 'isolated', 0.05, 8.17644781144781), (True, 'futures', 'gateio', 'isolated', 0.05, 11.7804274688304), (False, 'futures', 'gateio', 'isolated', 0.05, 8.181423084697796), (True, 'futures', 'okx', 'isolated', 0.0, 11.87413417771621), @@ -845,6 +845,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade.open_order_id is None assert trade.open_rate == 10 assert trade.stake_amount == round(order['price'] * order['filled'] / leverage, 8) + assert pytest.approx(trade.liquidation_price) == liq_price # In case of rejected or expired order and partially filled order['status'] = 'expired' @@ -932,8 +933,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, assert trade.open_rate_requested == 10 # In case of custom entry price not float type - freqtrade.exchange.get_maintenance_ratio_and_amt = MagicMock(return_value=(0.01, 0.01)) - freqtrade.exchange.name = exchange_name order['status'] = 'open' order['id'] = '5568' freqtrade.strategy.custom_entry_price = lambda **kwargs: "string price" @@ -946,7 +945,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade.is_short = is_short assert trade assert trade.open_rate_requested == 10 - assert trade.liquidation_price == liq_price # In case of too high stake amount @@ -3221,7 +3219,7 @@ def test_execute_trade_exit_custom_exit_price( freqtrade.execute_trade_exit( trade=trade, limit=ticker_usdt_sell_up()['ask' if is_short else 'bid'], - exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL) + exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL, exit_reason='foo') ) # Sell price must be different to default bid price @@ -3249,8 +3247,8 @@ def test_execute_trade_exit_custom_exit_price( 'profit_ratio': profit_ratio, 'stake_currency': 'USDT', 'fiat_currency': 'USD', - 'sell_reason': ExitType.EXIT_SIGNAL.value, - 'exit_reason': ExitType.EXIT_SIGNAL.value, + 'sell_reason': 'foo', + 'exit_reason': 'foo', 'open_date': ANY, 'close_date': ANY, 'close_rate': ANY,