adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feat_readjust_entry

Browse Source
This commit is contained in:
eSeR1805 2022-04-28 23:06:52 +03:00
commit eb23170c43
No known key found for this signature in database
GPG Key ID: BA53686259B46936
66 changed files with 818 additions and 517 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/ISSUE_TEMPLATE/feature_request.md vendored
View File

@ -24,4 +24,3 @@ Have you search for this feature before requesting it? It's highly likely that a
## Describe the enhancement
*Explain the enhancement you would like*

40
.github/workflows/ci.yml vendored
View File

@ -100,7 +100,7 @@ jobs:
- name: Mypy
run: |
mypy freqtrade scripts
mypy freqtrade scripts tests
- name: Discord notification
uses: rjstone/discord-webhook-notify@v1
@ -157,6 +157,12 @@ jobs:
pip install -e .
- name: Tests
if: (runner.os != 'Linux' || matrix.python-version != '3.8')
run: |
pytest --random-order
- name: Tests (with cov)
if: (runner.os == 'Linux' && matrix.python-version == '3.8')
run: |
pytest --random-order --cov=freqtrade --cov-config=.coveragerc
@ -229,7 +235,7 @@ jobs:
- name: Tests
run: |
pytest --random-order --cov=freqtrade --cov-config=.coveragerc
pytest --random-order
- name: Backtesting
run: |
@ -249,7 +255,7 @@ jobs:
- name: Mypy
run: |
mypy freqtrade scripts
mypy freqtrade scripts tests
- name: Discord notification
uses: rjstone/discord-webhook-notify@v1
@ -259,6 +265,21 @@ jobs:
details: Test Failed
webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
mypy_version_check:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: 3.9
- name: pre-commit dependencies
run: |
pip install pyaml
python build_helpers/pre_commit_update.py
docs_check:
runs-on: ubuntu-20.04
steps:
@ -271,7 +292,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: 3.9
- name: Documentation build
run: |
@ -288,6 +309,9 @@ jobs:
webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
cleanup-prior-runs:
permissions:
actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it
contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch
runs-on: ubuntu-20.04
steps:
- name: Cleanup previous runs on this branch
@ -298,8 +322,12 @@ jobs:
# Notify only once - when CI completes (and after deploy) in case it's successfull
notify-complete:
needs: [ build_linux, build_macos, build_windows, docs_check ]
needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ]
runs-on: ubuntu-20.04
# Discord notification can't handle schedule events
if: (github.event_name != 'schedule')
permissions:
repository-projects: read
steps:
- name: Check user permission
@ -319,7 +347,7 @@ jobs:
webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
deploy:
needs: [ build_linux, build_macos, build_windows, docs_check ]
needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ]
runs-on: ubuntu-20.04
if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'

1
.github/workflows/docker_update_readme.yml vendored
View File

@ -15,4 +15,3 @@ jobs:
DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKERHUB_REPOSITORY: freqtradeorg/freqtrade

41
.pre-commit-config.yaml
View File

@ -1,21 +1,46 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pycqa/flake8
rev: '4.0.1'
- repo: https://github.com/pycqa/flake8
rev: "4.0.1"
hooks:
- id: flake8
- id: flake8
# stages: [push]
- repo: https://github.com/pre-commit/mirrors-mypy
rev: 'v0.942'
- repo: https://github.com/pre-commit/mirrors-mypy
rev: "v0.942"
hooks:
- id: mypy
- id: mypy
exclude: build_helpers
additional_dependencies:
- types-cachetools==5.0.1
- types-filelock==3.2.5
- types-requests==2.27.20
- types-tabulate==0.8.7
- types-python-dateutil==2.8.12
# stages: [push]
- repo: https://github.com/pycqa/isort
rev: '5.10.1'
- repo: https://github.com/pycqa/isort
rev: "5.10.1"
hooks:
- id: isort
name: isort (python)
# stages: [push]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.4.0
hooks:
- id: end-of-file-fixer
exclude: |
(?x)^(
tests/.*|
.*\.svg
)$
- id: mixed-line-ending
- id: debug-statements
- id: check-ast
- id: trailing-whitespace
exclude: |
(?x)^(
.*\.md
)$

1
.pylintrc
View File

@ -7,4 +7,3 @@ ignore=vendor
[TYPECHECK]
ignored-modules=numpy,talib,talib.abstract

8
README.md
View File

@ -39,6 +39,14 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
- [X] [OKX](https://okx.com/) (Former OKEX)
- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
### Experimentally, freqtrade also supports futures on the following exchanges
- [X] [Binance](https://www.binance.com/)
- [X] [Gate.io](https://www.gate.io/ref/6266643)
- [X] [OKX](https://okx.com/).
Please make sure to read the [exchange specific notes](docs/exchanges.md), as well as the [trading with leverage](docs/leverage.md) documentation before diving in.
### Community tested
Exchanges confirmed working by the community:

42
build_helpers/pre_commit_update.py Normal file
View File

@ -0,0 +1,42 @@
# File used in CI to ensure pre-commit dependencies are kept uptodate.
import sys
from pathlib import Path
import yaml
pre_commit_file = Path('.pre-commit-config.yaml')
require_dev = Path('requirements-dev.txt')
with require_dev.open('r') as rfile:
requirements = rfile.readlines()
# Extract types only
type_reqs = [r.strip('\n') for r in requirements if r.startswith('types-')]
with pre_commit_file.open('r') as file:
f = yaml.load(file, Loader=yaml.FullLoader)
mypy_repo = [repo for repo in f['repos'] if repo['repo']
== 'https://github.com/pre-commit/mirrors-mypy']
hooks = mypy_repo[0]['hooks'][0]['additional_dependencies']
errors = []
for hook in hooks:
if hook not in type_reqs:
errors.append(f"{hook} is missing in requirements-dev.txt.")
for req in type_reqs:
if req not in hooks:
errors.append(f"{req} is missing in pre-config file.")
if errors:
for e in errors:
print(e)
sys.exit(1)
sys.exit(0)

1
config_examples/config_full.example.json
View File

@ -182,6 +182,7 @@
"disable_dataframe_checks": false,
"strategy": "SampleStrategy",
"strategy_path": "user_data/strategies/",
"recursive_strategy_search": false,
"add_config_files": [],
"dataformat_ohlcv": "json",
"dataformat_trades": "jsongz"

73
docs/advanced-backtesting.md Normal file
View File

@ -0,0 +1,73 @@
# Advanced Backtesting Analysis
## Analyze the buy/entry and sell/exit tags
It can be helpful to understand how a strategy behaves according to the buy/entry tags used to
mark up different buy conditions. You might want to see more complex statistics about each buy and
sell condition above those provided by the default backtesting output. You may also want to
determine indicator values on the signal candle that resulted in a trade opening.
!!! Note
The following buy reason analysis is only available for backtesting, *not hyperopt*.
We need to run backtesting with the `--export` option set to `signals` to enable the exporting of
signals **and** trades:
``` bash
freqtrade backtesting -c <config.json> --timeframe <tf> --strategy <strategy_name> --timerange=<timerange> --export=signals
```
This will tell freqtrade to output a pickled dictionary of strategy, pairs and corresponding
DataFrame of the candles that resulted in buy signals. Depending on how many buys your strategy
makes, this file may get quite large, so periodically check your `user_data/backtest_results`
folder to delete old exports.
To analyze the buy tags, we need to use the `buy_reasons.py` script from
[froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions
in their README to copy the script into your `freqtrade/scripts/` folder.
Before running your next backtest, make sure you either delete your old backtest results or run
backtesting with the `--cache none` option to make sure no cached results are used.
If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the
`user_data/backtest_results` folder.
Now run the `buy_reasons.py` script, supplying a few options:
``` bash
python3 scripts/buy_reasons.py -c <config.json> -s <strategy_name> -t <timerange> -g0,1,2,3,4
```
The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0)
to the most detailed per pair, per buy and per sell tag (4). More options are available by
running with the `-h` option.
### Tuning the buy tags and sell tags to display
To show only certain buy and sell tags in the displayed output, use the following two options:
```
--enter_reason_list : Comma separated list of enter signals to analyse. Default: "all"
--exit_reason_list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss"
```
For example:
```bash
python3 scripts/buy_reasons.py -c <config.json> -s <strategy_name> -t <timerange> -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss"
```
### Outputting signal candle indicators
The real power of the buy_reasons.py script comes from the ability to print out the indicator
values present on signal candles to allow fine-grained investigation and tuning of buy signal
indicators. To print out a column for a given set of indicators, use the `--indicator-list`
option:
```bash
python3 scripts/buy_reasons.py -c <config.json> -s <strategy_name> -t <timerange> -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal"
```
The indicators have to be present in your strategy's main DataFrame (either for your main
timeframe or for informative timeframes) otherwise they will simply be ignored in the script
output.

26
docs/backtesting.md
View File

@ -20,7 +20,8 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[--dry-run-wallet DRY_RUN_WALLET]
[--timeframe-detail TIMEFRAME_DETAIL]
[--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
[--export {none,trades}] [--export-filename PATH]
[--export {none,trades,signals}]
[--export-filename PATH]
[--breakdown {day,week,month} [{day,week,month} ...]]
[--cache {none,day,week,month}]
@ -63,18 +64,17 @@ optional arguments:
`30m`, `1h`, `1d`).
--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
Provide a space-separated list of strategies to
backtest. Please note that timeframe needs to be
set either in config or via command line. When using
this together with `--export trades`, the strategy-
name is injected into the filename (so `backtest-
data.json` becomes `backtest-data-SampleStrategy.json`
--export {none,trades}
backtest. Please note that timeframe needs to be set
either in config or via command line. When using this
together with `--export trades`, the strategy-name is
injected into the filename (so `backtest-data.json`
becomes `backtest-data-SampleStrategy.json`
--export {none,trades,signals}
Export backtest results (default: trades).
--export-filename PATH
Save backtest results to the file with this filename.
Requires `--export` to be set as well. Example:
`--export-filename=user_data/backtest_results/backtest
_today.json`
--export-filename PATH, --backtest-filename PATH
Use this filename for backtest results.Requires
`--export` to be set as well. Example: `--export-filen
ame=user_data/backtest_results/backtest_today.json`
--breakdown {day,week,month} [{day,week,month} ...]
Show backtesting breakdown per [day, week, month].
--cache {none,day,week,month}
@ -299,6 +299,7 @@ A backtesting result will look like that:
| Final balance | 0.01762792 BTC |
| Absolute profit | 0.00762792 BTC |
| Total profit % | 76.2% |
| CAGR % | 460.87% |
| Trades per day | 3.575 |
| Avg. stake amount | 0.001 BTC |
| Total trade volume | 0.429 BTC |
@ -388,6 +389,7 @@ It contains some useful key metrics about performance of your strategy on backte
| Final balance | 0.01762792 BTC |
| Absolute profit | 0.00762792 BTC |
| Total profit % | 76.2% |
| CAGR % | 460.87% |
| Avg. stake amount | 0.001 BTC |
| Total trade volume | 0.429 BTC |
| | |

7
docs/configuration.md
View File

@ -11,7 +11,7 @@ Per default, the bot loads the configuration from the `config.json` file, locate
You can specify a different configuration file used by the bot with the `-c/--config` command-line option.
If you used the [Quick start](installation.md/#quick-start) method for installing
If you used the [Quick start](installation.md/#quick-start) method for installing
the bot, the installation script should have already created the default configuration file (`config.json`) for you.
If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
@ -64,7 +64,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as
"config-private.json"
]
```
``` bash
freqtrade trade --config user_data/config.json <...>
```
@ -100,7 +100,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as
"stake_amount": "unlimited",
}
```
Resulting combined configuration:
``` json title="Result"
@ -173,6 +173,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
| `order_types` | Configure order-types depending on the action (`"entry"`, `"exit"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Dict
| `order_time_in_force` | Configure time in force for entry and exit orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Dict
| `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price. <br>*Defaults to `0.02` 2%).*<br> **Datatype:** Positive float
| `recursive_strategy_search` | Set to `true` to recursively search sub-directories inside `user_data/strategies` for a strategy. <br> **Datatype:** Boolean
| `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). <br> **Datatype:** String
| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.<br> **Datatype:** Boolean
| `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String

1
docs/data-analysis.md
View File

@ -122,5 +122,6 @@ Best avoid relative paths, since this starts at the storage location of the jupy
* [Strategy debugging](strategy_analysis_example.md) - also available as Jupyter notebook (`user_data/notebooks/strategy_analysis_example.ipynb`)
* [Plotting](plotting.md)
* [Tag Analysis](advanced-backtesting.md)
Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data.

8
docs/index.md
View File

@ -51,6 +51,14 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
- [X] [OKX](https://okx.com/) (Former OKEX)
- [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
### Experimentally, freqtrade also supports futures on the following exchanges:
- [X] [Binance](https://www.binance.com/)
- [X] [Gate.io](https://www.gate.io/ref/6266643)
- [X] [OKX](https://okx.com/).
Please make sure to read the [exchange specific notes](exchanges.md), as well as the [trading with leverage](leverage.md) documentation before diving in.
### Community tested
Exchanges confirmed working by the community:

2
docs/javascripts/config.js
View File

@ -9,4 +9,4 @@ window.MathJax = {
ignoreHtmlClass: ".*|",
processHtmlClass: "arithmatex"
}
};
};

4
docs/requirements-docs.txt
View File

@ -1,5 +1,5 @@
mkdocs==1.3.0
mkdocs-material==8.2.9
mkdocs-material==8.2.10
mdx_truly_sane_lists==1.2
pymdown-extensions==9.3
pymdown-extensions==9.4
jinja2==3.1.1

2
docs/strategy-callbacks.md
View File

@ -377,7 +377,7 @@ class AwesomeStrategy(IStrategy):
def custom_exit_price(self, pair: str, trade: Trade,
current_time: datetime, proposed_rate: float,
current_profit: float, **kwargs) -> float:
current_profit: float, exit_tag: Optional[str], **kwargs) -> float:
dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
timeframe=self.timeframe)

2
docs/strategy_analysis_example.md
View File

@ -93,7 +93,7 @@ from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats
# if backtest_dir points to a directory, it'll automatically load the last backtest file.
backtest_dir = config["user_data_dir"] / "backtest_results"
# backtest_dir can also point to a specific file
# backtest_dir can also point to a specific file
# backtest_dir = config["user_data_dir"] / "backtest_results/backtest-result-2020-07-01_20-04-22.json"
```

2
environment.yml
View File

@ -32,6 +32,7 @@ dependencies:
- prompt-toolkit
- schedule
- python-dateutil
- joblib
# ============================
@ -54,7 +55,6 @@ dependencies:
- scikit-learn
- filelock
- scikit-optimize
- joblib
- progressbar2
# ============================
# 4/4 req plot

1
freqtrade.service
View File

@ -11,4 +11,3 @@ Restart=on-failure
[Install]
WantedBy=default.target

1
freqtrade.service.watchdog
View File

@ -27,4 +27,3 @@ WatchdogSec=20
[Install]
WantedBy=default.target

5
freqtrade/commands/arguments.py
View File

@ -12,7 +12,7 @@ from freqtrade.constants import DEFAULT_CONFIG
ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
ARGS_STRATEGY = ["strategy", "strategy_path"]
ARGS_STRATEGY = ["strategy", "strategy_path", "recursive_strategy_search"]
ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
@ -37,7 +37,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized"]
ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized",
"recursive_strategy_search"]
ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"]

5
freqtrade/commands/cli_options.py
View File

@ -83,6 +83,11 @@ AVAILABLE_CLI_OPTIONS = {
help='Reset sample files to their original state.',
action='store_true',
),
"recursive_strategy_search": Arg(
'--recursive-strategy-search',
help='Recursively search for a strategy in the strategies folder.',
action='store_true',
),
# Main options
"strategy": Arg(
'-s', '--strategy',

9
freqtrade/commands/list_commands.py
View File

@ -41,7 +41,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
def _print_objs_tabular(objs: List, print_colorized: bool, base_dir: Path) -> None:
if print_colorized:
colorama_init(autoreset=True)
red = Fore.RED
@ -55,7 +55,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
names = [s['name'] for s in objs]
objs_to_print = [{
'name': s['name'] if s['name'] else "--",
'location': s['location'].name,
'location': s['location'].relative_to(base_dir),
'status': (red + "LOAD FAILED" + reset if s['class'] is None
else "OK" if names.count(s['name']) == 1
else yellow + "DUPLICATE NAME" + reset)
@ -77,7 +77,8 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column'])
strategy_objs = StrategyResolver.search_all_objects(
directory, not args['print_one_column'], config.get('recursive_strategy_search', False))
# Sort alphabetically
strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
for obj in strategy_objs:
@ -89,7 +90,7 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
if args['print_one_column']:
print('\n'.join([s['name'] for s in strategy_objs]))
else:
_print_objs_tabular(strategy_objs, config.get('print_colorized', False))
_print_objs_tabular(strategy_objs, config.get('print_colorized', False), directory)
def start_list_timeframes(args: Dict[str, Any]) -> None:

6
freqtrade/configuration/configuration.py
View File

@ -248,6 +248,12 @@ class Configuration:
self._args_to_config(config, argname='strategy_list',
logstring='Using strategy list of {} strategies', logfun=len)
self._args_to_config(
config,
argname='recursive_strategy_search',
logstring='Recursively searching for a strategy in the strategies folder.',
)
self._args_to_config(config, argname='timeframe',
logstring='Overriding timeframe with Command line argument')

2
freqtrade/constants.py
View File

@ -14,7 +14,7 @@ PROCESS_THROTTLE_SECS = 5 # sec
HYPEROPT_EPOCH = 100 # epochs
RETRY_TIMEOUT = 30 # sec
TIMEOUT_UNITS = ['minutes', 'seconds']
EXPORT_OPTIONS = ['none', 'trades']
EXPORT_OPTIONS = ['none', 'trades', 'signals']
DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
UNLIMITED_STAKE_AMOUNT = 'unlimited'

11
freqtrade/data/btanalysis.py
View File

@ -553,3 +553,14 @@ def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[f
csum_max = csum_df['sum'].max() + starting_balance
return csum_min, csum_max
def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float:
"""
Calculate CAGR
:param days_passed: Days passed between start and ending balance
:param starting_balance: Starting balance
:param final_balance: Final balance to calculate CAGR against
:return: CAGR
"""
return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1

34
freqtrade/exchange/exchange.py
View File

@ -9,6 +9,7 @@ import logging
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from math import ceil
from threading import Lock
from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
import arrow
@ -64,6 +65,7 @@ class Exchange:
"ohlcv_params": {},
"ohlcv_candle_limit": 500,
"ohlcv_partial_candle": True,
"ohlcv_require_since": False,
# Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency
"ohlcv_volume_currency": "base", # "base" or "quote"
"tickers_have_quoteVolume": True,
@ -95,6 +97,9 @@ class Exchange:
self._markets: Dict = {}
self._trading_fees: Dict[str, Any] = {}
self._leverage_tiers: Dict[str, List[Dict]] = {}
# Lock event loop. This is necessary to avoid race-conditions when using force* commands
# Due to funding fee fetching.
self._loop_lock = Lock()
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self._config: Dict = {}
@ -166,7 +171,7 @@ class Exchange:
self._api_async = self._init_ccxt(
exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config)
logger.info('Using Exchange "%s"', self.name)
logger.info(f'Using Exchange "{self.name}"')
if validate:
# Check if timeframe is available
@ -368,6 +373,9 @@ class Exchange:
return (
market.get('quote', None) is not None
and market.get('base', None) is not None
and (self.precisionMode != TICK_SIZE
# Too low precision will falsify calculations
or market.get('precision', {}).get('price', None) > 1e-11)
and ((self.trading_mode == TradingMode.SPOT and self.market_is_spot(market))
or (self.trading_mode == TradingMode.MARGIN and self.market_is_margin(market))
or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market)))
@ -551,7 +559,7 @@ class Exchange:
# Therefore we also show that.
raise OperationalException(
f"The ccxt library does not provide the list of timeframes "
f"for the exchange \"{self.name}\" and this exchange "
f"for the exchange {self.name} and this exchange "
f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}")
if timeframe and (timeframe not in self.timeframes):
@ -781,7 +789,9 @@ class Exchange:
rate: float, leverage: float, params: Dict = {},
stop_loss: bool = False) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount)
# Rounding here must respect to contract sizes
_amount = self._contracts_to_amount(
pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount)))
dry_order: Dict[str, Any] = {
'id': order_id,
'symbol': pair,
@ -1710,7 +1720,8 @@ class Exchange:
def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
since_ms: Optional[int]) -> Coroutine:
if not since_ms and self.required_candle_call_count > 1:
if (not since_ms
and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)):
# Multiple calls for one pair - to get more history
one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
move_to = one_call * self.required_candle_call_count
@ -1770,7 +1781,8 @@ class Exchange:
async def gather_stuff():
return await asyncio.gather(*input_coro, return_exceptions=True)
results = self.loop.run_until_complete(gather_stuff())
with self._loop_lock:
results = self.loop.run_until_complete(gather_stuff())
for res in results:
if isinstance(res, Exception):
@ -1829,17 +1841,18 @@ class Exchange:
pair, timeframe, since_ms, s
)
params = deepcopy(self._ft_has.get('ohlcv_params', {}))
candle_limit = self.ohlcv_candle_limit(timeframe)
if candle_type != CandleType.SPOT:
params.update({'price': candle_type})
if candle_type != CandleType.FUNDING_RATE:
data = await self._api_async.fetch_ohlcv(
pair, timeframe=timeframe, since=since_ms,
limit=self.ohlcv_candle_limit(timeframe), params=params)
limit=candle_limit, params=params)
else:
# Funding rate
data = await self._api_async.fetch_funding_rate_history(
pair, since=since_ms,
limit=self.ohlcv_candle_limit(timeframe))
limit=candle_limit)
# Convert funding rate to candle pattern
data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data]
# Some exchanges sort OHLCV in ASC order and others in DESC.
@ -2026,9 +2039,10 @@ class Exchange:
if not self.exchange_has("fetchTrades"):
raise OperationalException("This exchange does not support downloading Trades.")
return self.loop.run_until_complete(
self._async_get_trade_history(pair=pair, since=since,
until=until, from_id=from_id))
with self._loop_lock:
return self.loop.run_until_complete(
self._async_get_trade_history(pair=pair, since=since,
until=until, from_id=from_id))
@retrier
def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float:

1
freqtrade/exchange/ftx.py
View File

@ -20,6 +20,7 @@ class Ftx(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"ohlcv_candle_limit": 1500,
"ohlcv_require_since": True,
"ohlcv_volume_currency": "quote",
"mark_ohlcv_price": "index",
"mark_ohlcv_timeframe": "1h",

16
freqtrade/freqtradebot.py
View File

@ -586,7 +586,6 @@ class FreqtradeBot(LoggingMixin):
Executes a limit buy for the given pair
:param pair: pair for which we want to create a LIMIT_BUY
:param stake_amount: amount of stake-currency for the pair
:param leverage: amount of leverage applied to this trade
:return: True if a buy order is created, false if it fails.
"""
time_in_force = self.strategy.order_time_in_force['entry']
@ -665,16 +664,6 @@ class FreqtradeBot(LoggingMixin):
amount = safe_value_fallback(order, 'filled', 'amount')
enter_limit_filled_price = safe_value_fallback(order, 'average', 'price')
# TODO: this might be unnecessary, as we're calling it in update_trade_state.
isolated_liq = self.exchange.get_liquidation_price(
leverage=leverage,
pair=pair,
amount=amount,
open_rate=enter_limit_filled_price,
is_short=is_short
)
interest_rate = self.exchange.get_interest_rate()
# Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
base_currency = self.exchange.get_pair_base_currency(pair)
@ -703,8 +692,6 @@ class FreqtradeBot(LoggingMixin):
timeframe=timeframe_to_minutes(self.config['timeframe']),
leverage=leverage,
is_short=is_short,
interest_rate=interest_rate,
liquidation_price=isolated_liq,
trading_mode=self.trading_mode,
funding_fees=funding_fees
)
@ -1430,7 +1417,8 @@ class FreqtradeBot(LoggingMixin):
default_retval=proposed_limit_rate)(
pair=trade.pair, trade=trade,
current_time=datetime.now(timezone.utc),
proposed_rate=proposed_limit_rate, current_profit=current_profit)
proposed_rate=proposed_limit_rate, current_profit=current_profit,
exit_tag=exit_check.exit_reason)
limit = self.get_valid_price(custom_exit_price, proposed_limit_rate)

16
freqtrade/misc.py
View File

@ -86,6 +86,22 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
logger.debug(f'done json to "{filename}"')
def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
"""
Dump object data into a file
:param filename: file to create
:param data: Object data to save
:return:
"""
import joblib
if log:
logger.info(f'dumping joblib to "{filename}"')
with open(filename, 'wb') as fp:
joblib.dump(data, fp)
logger.debug(f'done joblib dump to "{filename}"')
def json_load(datafile: IO) -> Any:
"""
load data with rapidjson

184
freqtrade/optimize/backtesting.py Normal file → Executable file
View File

@ -9,6 +9,7 @@ from copy import deepcopy
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
from numpy import nan
from pandas import DataFrame
@ -19,13 +20,15 @@ from freqtrade.data import history
from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe
from freqtrade.data.converter import trim_dataframe, trim_dataframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import BacktestState, CandleType, ExitCheckTuple, ExitType, TradingMode
from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType, RunMode,
TradingMode)
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.misc import get_strategy_run_id
from freqtrade.mixins import LoggingMixin
from freqtrade.optimize.bt_progress import BTProgress
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_signal_candles,
store_backtest_stats)
from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
from freqtrade.plugins.pairlistmanager import PairListManager
@ -51,6 +54,11 @@ ESHORT_IDX = 8 # Exit short
ENTER_TAG_IDX = 9
EXIT_TAG_IDX = 10
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
HEADERS = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
class Backtesting:
"""
@ -73,6 +81,8 @@ class Backtesting:
self.run_ids: Dict[str, str] = {}
self.strategylist: List[IStrategy] = []
self.all_results: Dict[str, Dict] = {}
self.processed_dfs: Dict[str, Dict] = {}
self._exchange_name = self.config['exchange']['name']
self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config)
self.dataprovider = DataProvider(self.config, self.exchange)
@ -174,7 +184,7 @@ class Backtesting:
# Attach Wallets to Strategy baseclass
strategy.wallets = self.wallets
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# since a "perfect" stoploss-exit is assumed anyway
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
@ -259,10 +269,18 @@ class Backtesting:
candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"])
)
# Combine data to avoid combining the data per trade.
unavailable_pairs = []
for pair in self.pairlists.whitelist:
if pair not in self.exchange._leverage_tiers:
unavailable_pairs.append(pair)
continue
self.futures_data[pair] = funding_rates_dict[pair].merge(
mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"])
if unavailable_pairs:
raise OperationalException(
f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. "
"It is therefore impossible to backtest with this pair at the moment.")
else:
self.futures_data = {}
@ -300,10 +318,7 @@ class Backtesting:
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
data: Dict = {}
self.progress.init_step(BacktestState.CONVERT, len(processed))
@ -315,7 +330,7 @@ class Backtesting:
if not pair_data.empty:
# Cleanup from prior runs
pair_data.drop(headers[5:] + ['buy', 'sell'], axis=1, errors='ignore')
pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore')
df_analyzed = self.strategy.advise_exit(
self.strategy.advise_entry(pair_data, {'pair': pair}),
@ -328,13 +343,13 @@ class Backtesting:
self.dataprovider._set_cached_df(
pair, self.timeframe, df_analyzed, self.config['candle_type_def'])
# Create a copy of the dataframe before shifting, that way the buy signal/tag
# Create a copy of the dataframe before shifting, that way the entry signal/tag
# remains on the correct candle for callbacks.
df_analyzed = df_analyzed.copy()
# To avoid using data from future, we use buy/sell signals shifted
# To avoid using data from future, we use entry/exit signals shifted
# from the previous candle
for col in headers[5:]:
for col in HEADERS[5:]:
tag_col = col in ('enter_tag', 'exit_tag')
if col in df_analyzed.columns:
df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace(
@ -346,27 +361,27 @@ class Backtesting:
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = df_analyzed[headers].values.tolist() if not df_analyzed.empty else []
data[pair] = df_analyzed[HEADERS].values.tolist() if not df_analyzed.empty else []
return data
def _get_close_rate(self, row: Tuple, trade: LocalTrade, sell: ExitCheckTuple,
def _get_close_rate(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple,
trade_dur: int) -> float:
"""
Get close rate for backtesting result
"""
# Special handling if high or low hit STOP_LOSS or ROI
if sell.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS):
return self._get_close_rate_for_stoploss(row, trade, sell, trade_dur)
elif sell.exit_type == (ExitType.ROI):
return self._get_close_rate_for_roi(row, trade, sell, trade_dur)
if exit.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS):
return self._get_close_rate_for_stoploss(row, trade, exit, trade_dur)
elif exit.exit_type == (ExitType.ROI):
return self._get_close_rate_for_roi(row, trade, exit, trade_dur)
else:
return row[OPEN_IDX]
def _get_close_rate_for_stoploss(self, row: Tuple, trade: LocalTrade, sell: ExitCheckTuple,
def _get_close_rate_for_stoploss(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple,
trade_dur: int) -> float:
# our stoploss was already lower than candle high,
# possibly due to a cancelled trade exit.
# sell at open price.
# exit at open price.
is_short = trade.is_short or False
leverage = trade.leverage or 1.0
side_1 = -1 if is_short else 1
@ -380,7 +395,7 @@ class Backtesting:
# Special case: trailing triggers within same candle as trade opened. Assume most
# pessimistic price movement, which is moving just enough to arm stoploss and
# immediately going down to stop price.
if sell.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0:
if exit.exit_type == ExitType.TRAILING_STOP_LOSS and trade_dur == 0:
if (
not self.strategy.use_custom_stoploss and self.strategy.trailing_stop
and self.strategy.trailing_only_offset_is_reached
@ -399,7 +414,7 @@ class Backtesting:
else:
assert stop_rate < row[HIGH_IDX]
# Limit lower-end to candle low to avoid sells below the low.
# Limit lower-end to candle low to avoid exits below the low.
# This still remains "worst case" - but "worst realistic case".
if is_short:
return min(row[HIGH_IDX], stop_rate)
@ -409,7 +424,7 @@ class Backtesting:
# Set close_rate to stoploss
return trade.stop_loss
def _get_close_rate_for_roi(self, row: Tuple, trade: LocalTrade, sell: ExitCheckTuple,
def _get_close_rate_for_roi(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple,
trade_dur: int) -> float:
is_short = trade.is_short or False
leverage = trade.leverage or 1.0
@ -434,7 +449,7 @@ class Backtesting:
and roi_entry % self.timeframe_min == 0
and is_new_roi):
# new ROI entry came into effect.
# use Open rate if open_rate > calculated sell rate
# use Open rate if open_rate > calculated exit rate
return row[OPEN_IDX]
if (trade_dur == 0 and (
@ -457,11 +472,11 @@ class Backtesting:
# ROI on opening candles with custom pricing can only
# trigger if the entry was at Open or lower wick.
# details: https: // github.com/freqtrade/freqtrade/issues/6261
# If open_rate is < open, only allow sells below the close on red candles.
# If open_rate is < open, only allow exits below the close on red candles.
raise ValueError("Opening candle ROI on red candles.")
# Use the maximum between close_rate and low as we
# cannot sell outside of a candle.
# cannot exit outside of a candle.
# Applies when a new ROI setting comes in place and the whole candle is above that.
return min(max(close_rate, row[LOW_IDX]), row[HIGH_IDX])
@ -496,7 +511,7 @@ class Backtesting:
""" Rate is within candle, therefore filled"""
return row[LOW_IDX] <= rate <= row[HIGH_IDX]
def _get_sell_trade_entry_for_candle(self, trade: LocalTrade,
def _get_exit_trade_entry_for_candle(self, trade: LocalTrade,
row: Tuple) -> Optional[LocalTrade]:
# Check if we need to adjust our current positions
@ -508,34 +523,35 @@ class Backtesting:
if check_adjust_entry:
trade = self._get_adjust_trade_entry_for_candle(trade, row)
sell_candle_time: datetime = row[DATE_IDX].to_pydatetime()
exit_candle_time: datetime = row[DATE_IDX].to_pydatetime()
enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX]
exit_ = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX]
sell = self.strategy.should_exit(
trade, row[OPEN_IDX], sell_candle_time, # type: ignore
enter=enter, exit_=exit_,
exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX]
exit_ = self.strategy.should_exit(
trade, row[OPEN_IDX], exit_candle_time, # type: ignore
enter=enter, exit_=exit_sig,
low=row[LOW_IDX], high=row[HIGH_IDX]
)
if sell.exit_flag:
trade.close_date = sell_candle_time
if exit_.exit_flag:
trade.close_date = exit_candle_time
trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
try:
closerate = self._get_close_rate(row, trade, sell, trade_dur)
closerate = self._get_close_rate(row, trade, exit_, trade_dur)
except ValueError:
return None
# call the custom exit price,with default value as previous closerate
current_profit = trade.calc_profit_ratio(closerate)
order_type = self.strategy.order_types['exit']
if sell.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT):
# Custom exit pricing only for sell-signals
if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT):
# Custom exit pricing only for exit-signals
if order_type == 'limit':
closerate = strategy_safe_wrapper(self.strategy.custom_exit_price,
default_retval=closerate)(
pair=trade.pair, trade=trade,
current_time=sell_candle_time,
proposed_rate=closerate, current_profit=current_profit)
current_time=exit_candle_time,
proposed_rate=closerate, current_profit=current_profit,
exit_tag=exit_.exit_reason)
# We can't place orders lower than current low.
# freqtrade does not support this in live, and the order would fill immediately
if trade.is_short:
@ -549,12 +565,12 @@ class Backtesting:
pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
rate=closerate,
time_in_force=time_in_force,
sell_reason=sell.exit_reason, # deprecated
exit_reason=sell.exit_reason,
current_time=sell_candle_time):
sell_reason=exit_.exit_reason, # deprecated
exit_reason=exit_.exit_reason,
current_time=exit_candle_time):
return None
trade.exit_reason = sell.exit_reason
trade.exit_reason = exit_.exit_reason
# Checks and adds an exit tag, after checking that the length of the
# row has the length for an exit tag column
@ -562,6 +578,7 @@ class Backtesting:
len(row) > EXIT_TAG_IDX
and row[EXIT_TAG_IDX] is not None
and len(row[EXIT_TAG_IDX]) > 0
and exit_.exit_type in (ExitType.EXIT_SIGNAL,)
):
trade.exit_reason = row[EXIT_TAG_IDX]
@ -569,8 +586,8 @@ class Backtesting:
order = Order(
id=self.order_id_counter,
ft_trade_id=trade.id,
order_date=sell_candle_time,
order_update_date=sell_candle_time,
order_date=exit_candle_time,
order_update_date=exit_candle_time,
ft_is_open=True,
ft_pair=trade.pair,
order_id=str(self.order_id_counter),
@ -591,8 +608,8 @@ class Backtesting:
return None
def _get_sell_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]:
sell_candle_time: datetime = row[DATE_IDX].to_pydatetime()
def _get_exit_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]:
exit_candle_time: datetime = row[DATE_IDX].to_pydatetime()
if self.trading_mode == TradingMode.FUTURES:
trade.funding_fees = self.exchange.calculate_funding_fees(
@ -600,37 +617,35 @@ class Backtesting:
amount=trade.amount,
is_short=trade.is_short,
open_date=trade.open_date_utc,
close_date=sell_candle_time,
close_date=exit_candle_time,
)
if self.timeframe_detail and trade.pair in self.detail_data:
sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min)
exit_candle_end = exit_candle_time + timedelta(minutes=self.timeframe_min)
detail_data = self.detail_data[trade.pair]
detail_data = detail_data.loc[
(detail_data['date'] >= sell_candle_time) &
(detail_data['date'] < sell_candle_end)
(detail_data['date'] >= exit_candle_time) &
(detail_data['date'] < exit_candle_end)
].copy()
if len(detail_data) == 0:
# Fall back to "regular" data if no detail data was found for this candle
return self._get_sell_trade_entry_for_candle(trade, row)
return self._get_exit_trade_entry_for_candle(trade, row)
detail_data.loc[:, 'enter_long'] = row[LONG_IDX]
detail_data.loc[:, 'exit_long'] = row[ELONG_IDX]
detail_data.loc[:, 'enter_short'] = row[SHORT_IDX]
detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX]
detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX]
detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX]
headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
for det_row in detail_data[headers].values.tolist():
res = self._get_sell_trade_entry_for_candle(trade, det_row)
for det_row in detail_data[HEADERS].values.tolist():
res = self._get_exit_trade_entry_for_candle(trade, det_row)
if res:
return res
return None
else:
return self._get_sell_trade_entry_for_candle(trade, row)
return self._get_exit_trade_entry_for_candle(trade, row)
def get_valid_price_and_stake(
self, pair: str, row: Tuple, propose_rate: float, stake_amount: Optional[float],
@ -818,13 +833,13 @@ class Backtesting:
if len(open_trades[pair]) > 0:
for trade in open_trades[pair]:
if trade.open_order_id and trade.nr_of_successful_entries == 0:
# Ignore trade if buy-order did not fill yet
# Ignore trade if entry-order did not fill yet
continue
sell_row = data[pair][-1]
exit_row = data[pair][-1]
trade.close_date = sell_row[DATE_IDX].to_pydatetime()
trade.close_date = exit_row[DATE_IDX].to_pydatetime()
trade.exit_reason = ExitType.FORCE_EXIT.value
trade.close(sell_row[OPEN_IDX], show_msg=False)
trade.close(exit_row[OPEN_IDX], show_msg=False)
LocalTrade.close_bt_trade(trade)
# Deepcopy object to have wallets update correctly
trade1 = deepcopy(trade)
@ -889,7 +904,7 @@ class Backtesting:
# Remove trade due to entry timeout expiration.
return True
else:
# Close additional buy order
# Close additional entry order
del trade.orders[trade.orders.index(order)]
if order.side == trade.exit_side:
self.timedout_exit_orders += 1
@ -902,7 +917,7 @@ class Backtesting:
self, data: Dict, pair: str, row_index: int, current_time: datetime) -> Optional[Tuple]:
try:
# Row is treated as "current incomplete candle".
# Buy / sell signals are shifted by 1 to compensate for this.
# entry / exit signals are shifted by 1 to compensate for this.
row = data[pair][row_index]
except IndexError:
# missing Data for one pair at the end.
@ -967,16 +982,16 @@ class Backtesting:
self.dataprovider._set_dataframe_max_index(row_index)
for t in list(open_trades[pair]):
# 1. Cancel expired buy/sell orders.
# 1. Cancel expired entry/exit orders.
if self.check_order_cancel(t, current_time):
# Close trade due to buy timeout expiration.
# Close trade due to entry timeout expiration.
open_trade_count -= 1
open_trades[pair].remove(t)
self.wallets.update()
else:
self.check_order_replace(t, current_time, row)
# 2. Process buys.
# 2. Process entries.
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
# don't open on the last row
@ -992,7 +1007,7 @@ class Backtesting:
if trade:
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behavior - not sure if this is correct
# Prevents buying if the trade-slot was freed in this candle
# Prevents entering if the trade-slot was freed in this candle
open_trade_count_start += 1
open_trade_count += 1
# logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
@ -1007,18 +1022,18 @@ class Backtesting:
LocalTrade.add_bt_trade(trade)
self.wallets.update()
# 4. Create sell orders (if any)
# 4. Create exit orders (if any)
if not trade.open_order_id:
self._get_sell_trade_entry(trade, row) # Place sell order if necessary
self._get_exit_trade_entry(trade, row) # Place exit order if necessary
# 5. Process sell orders.
# 5. Process exit orders.
order = trade.select_order(trade.exit_side, is_open=True)
if order and self._get_order_filled(order.price, row):
trade.open_order_id = None
trade.close_date = current_time
trade.close(order.price, show_msg=False)
# logger.debug(f"{pair} - Backtesting sell {trade}")
# logger.debug(f"{pair} - Backtesting exit {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
LocalTrade.close_bt_trade(trade)
@ -1074,7 +1089,7 @@ class Backtesting:
"No data left after adjusting for startup candles.")
# Use preprocessed_tmp for date generation (the trimmed dataframe).
# Backtesting will re-trim the dataframes after buy/sell signal generation.
# Backtesting will re-trim the dataframes after entry/exit signal generation.
min_date, max_date = history.get_timerange(preprocessed_tmp)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
@ -1096,8 +1111,31 @@ class Backtesting:
})
self.all_results[self.strategy.get_strategy_name()] = results
if (self.config.get('export', 'none') == 'signals' and
self.dataprovider.runmode == RunMode.BACKTEST):
self._generate_trade_signal_candles(preprocessed_tmp, results)
return min_date, max_date
def _generate_trade_signal_candles(self, preprocessed_df, bt_results):
signal_candles_only = {}
for pair in preprocessed_df.keys():
signal_candles_only_df = DataFrame()
pairdf = preprocessed_df[pair]
resdf = bt_results['results']
pairresults = resdf.loc[(resdf["pair"] == pair)]
if pairdf.shape[0] > 0:
for t, v in pairresults.open_date.items():
allinds = pairdf.loc[(pairdf['date'] < v)]
signal_inds = allinds.iloc[[-1]]
signal_candles_only_df = pd.concat([signal_candles_only_df, signal_inds])
signal_candles_only[pair] = signal_candles_only_df
self.processed_dfs[self.strategy.get_strategy_name()] = signal_candles_only
def _get_min_cached_backtest_date(self):
min_backtest_date = None
backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
@ -1156,9 +1194,13 @@ class Backtesting:
else:
self.results = results
if self.config.get('export', 'none') == 'trades':
if self.config.get('export', 'none') in ('trades', 'signals'):
store_backtest_stats(self.config['exportfilename'], self.results)
if (self.config.get('export', 'none') == 'signals' and
self.dataprovider.runmode == RunMode.BACKTEST):
store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs)
# Results may be mixed up now. Sort them so they follow --strategy-list order.
if 'strategy_list' in self.config and len(self.results) > 0:
self.results['strategy_comparison'] = sorted(

57
freqtrade/optimize/hyperopt.py
View File

@ -10,7 +10,7 @@ import warnings
from datetime import datetime, timezone
from math import ceil
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
import progressbar
import rapidjson
@ -290,7 +290,7 @@ class Hyperopt:
self.assign_params(params_dict, 'protection')
if HyperoptTools.has_space(self.config, 'roi'):
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.backtesting.strategy.minimal_roi = (
self.custom_hyperopt.generate_roi_table(params_dict))
if HyperoptTools.has_space(self.config, 'stoploss'):
@ -409,6 +409,51 @@ class Hyperopt:
# Store non-trimmed data - will be trimmed after signal generation.
dump(preprocessed, self.data_pickle_file)
def get_asked_points(self, n_points: int) -> Tuple[List[List[Any]], List[bool]]:
"""
Enforce points returned from `self.opt.ask` have not been already evaluated
Steps:
1. Try to get points using `self.opt.ask` first
2. Discard the points that have already been evaluated
3. Retry using `self.opt.ask` up to 3 times
4. If still some points are missing in respect to `n_points`, random sample some points
5. Repeat until at least `n_points` points in the `asked_non_tried` list
6. Return a list with length truncated at `n_points`
"""
def unique_list(a_list):
new_list = []
for item in a_list:
if item not in new_list:
new_list.append(item)
return new_list
i = 0
asked_non_tried: List[List[Any]] = []
is_random: List[bool] = []
while i < 5 and len(asked_non_tried) < n_points:
if i < 3:
self.opt.cache_ = {}
asked = unique_list(self.opt.ask(n_points=n_points * 5))
is_random = [False for _ in range(len(asked))]
else:
asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5))
is_random = [True for _ in range(len(asked))]
is_random += [rand for x, rand in zip(asked, is_random)
if x not in self.opt.Xi
and x not in asked_non_tried]
asked_non_tried += [x for x in asked
if x not in self.opt.Xi
and x not in asked_non_tried]
i += 1
if asked_non_tried:
return (
asked_non_tried[:min(len(asked_non_tried), n_points)],
is_random[:min(len(asked_non_tried), n_points)]
)
else:
return self.opt.ask(n_points=n_points), [False for _ in range(n_points)]
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
@ -420,9 +465,10 @@ class Hyperopt:
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange.close()
self.backtesting.exchange._api = None # type: ignore
self.backtesting.exchange._api_async = None # type: ignore
self.backtesting.exchange._api = None
self.backtesting.exchange._api_async = None
self.backtesting.exchange.loop = None # type: ignore
self.backtesting.exchange._loop_lock = None # type: ignore
# self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore
@ -473,7 +519,7 @@ class Hyperopt:
n_rest = (i + 1) * jobs - self.total_epochs
current_jobs = jobs - n_rest if n_rest > 0 else jobs
asked = self.opt.ask(n_points=current_jobs)
asked, is_random = self.get_asked_points(n_points=current_jobs)
f_val = self.run_optimizer_parallel(parallel, asked, i)
self.opt.tell(asked, [v['loss'] for v in f_val])
@ -492,6 +538,7 @@ class Hyperopt:
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
val['is_random'] = is_random[j]
self.print_results(val)
if is_best:

17
freqtrade/optimize/hyperopt_tools.py
View File

@ -41,7 +41,8 @@ class HyperoptTools():
"""
from freqtrade.resolvers.strategy_resolver import StrategyResolver
directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
strategy_objs = StrategyResolver.search_all_objects(directory, False)
strategy_objs = StrategyResolver.search_all_objects(
directory, False, config.get('recursive_strategy_search', False))
strategies = [s for s in strategy_objs if s['name'] == strategy_name]
if strategies:
strategy = strategies[0]
@ -310,6 +311,8 @@ class HyperoptTools():
if not has_drawdown:
# Ensure compatibility with older versions of hyperopt results
trials['results_metrics.max_drawdown_account'] = None
if 'is_random' not in trials.columns:
trials['is_random'] = False
# New mode, using backtest result for metrics
trials['results_metrics.winsdrawslosses'] = trials.apply(
@ -322,12 +325,12 @@ class HyperoptTools():
'results_metrics.profit_total', 'results_metrics.holding_avg',
'results_metrics.max_drawdown',
'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs',
'loss', 'is_initial_point', 'is_best']]
'loss', 'is_initial_point', 'is_random', 'is_best']]
trials.columns = [
'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account',
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_random', 'is_best'
]
return trials
@ -349,9 +352,11 @@ class HyperoptTools():
trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown)
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '* '
trials.loc[trials['is_initial_point'] | trials['is_random'], 'Best'] = '* '
trials.loc[trials['is_best'], 'Best'] = 'Best'
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
trials.loc[
(trials['is_initial_point'] | trials['is_random']) & trials['is_best'],
'Best'] = '* Best'
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
trials['Trades'] = trials['Trades'].astype(str)
# perc_multi = 1 if legacy_mode else 100
@ -407,7 +412,7 @@ class HyperoptTools():
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
str(trials.loc[i][j]), Style.RESET_ALL)
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit', 'is_random'])
if remove_header > 0:
table = tabulate.tabulate(
trials.to_dict(orient='list'), tablefmt='orgtbl',

33
freqtrade/optimize/optimize_reports.py
View File

@ -9,10 +9,10 @@ from pandas import DataFrame, to_datetime
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change,
from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename,
round_coin_value)
from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json,
get_backtest_metadata_filename, round_coin_value)
logger = logging.getLogger(__name__)
@ -45,6 +45,29 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
file_dump_json(latest_filename, {'latest_backtest': str(filename.name)})
def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path:
"""
Stores backtest trade signal candles
:param recordfilename: Path object, which can either be a filename or a directory.
Filenames will be appended with a timestamp right before the suffix
while for directories, <directory>/backtest-result-<datetime>_signals.pkl will be used
as filename
:param stats: Dict containing the backtesting signal candles
"""
if recordfilename.is_dir():
filename = (recordfilename /
f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl')
else:
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl'
)
file_dump_joblib(filename, candles)
return filename
def _get_line_floatfmt(stake_currency: str) -> List[str]:
"""
Generate floatformat (goes in line with _generate_result_line())
@ -241,7 +264,7 @@ def generate_edge_table(results: dict) -> str:
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
def _get_resample_from_period(period: str) -> str:
@ -423,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str],
'profit_total_abs': results['profit_abs'].sum(),
'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
@ -723,6 +747,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
strat_results['stake_currency'])),
('Total profit %', f"{strat_results['profit_total']:.2%}"),
('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'),
('Trades per day', strat_results['trades_per_day']),
('Avg. daily profit %',
f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),

4
freqtrade/persistence/models.py
View File

@ -429,12 +429,10 @@ class LocalTrade():
def __repr__(self):
open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
leverage = self.leverage or 1.0
is_short = self.is_short or False
return (
f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
f'is_short={is_short}, leverage={leverage}, '
f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, '
f'open_rate={self.open_rate:.8f}, open_since={open_since})'
)

25
freqtrade/resolvers/iresolver.py
View File

@ -44,7 +44,7 @@ class IResolver:
@classmethod
def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None,
extra_dir: Optional[str] = None) -> List[Path]:
extra_dirs: List[str] = []) -> List[Path]:
abs_paths: List[Path] = []
if cls.initial_search_path:
@ -53,9 +53,9 @@ class IResolver:
if user_subdir:
abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir))
if extra_dir:
# Add extra directory to the top of the search paths
abs_paths.insert(0, Path(extra_dir).resolve())
# Add extra directory to the top of the search paths
for dir in extra_dirs:
abs_paths.insert(0, Path(dir).resolve())
return abs_paths
@ -164,9 +164,13 @@ class IResolver:
:return: Object instance or None
"""
extra_dirs: List[str] = []
if extra_dir:
extra_dirs.append(extra_dir)
abs_paths = cls.build_search_paths(config,
user_subdir=cls.user_subdir,
extra_dir=extra_dir)
extra_dirs=extra_dirs)
found_object = cls._load_object(paths=abs_paths, object_name=object_name,
kwargs=kwargs)
@ -178,18 +182,25 @@ class IResolver:
)
@classmethod
def search_all_objects(cls, directory: Path,
enum_failed: bool) -> List[Dict[str, Any]]:
def search_all_objects(cls, directory: Path, enum_failed: bool,
recursive: bool = False) -> List[Dict[str, Any]]:
"""
Searches a directory for valid objects
:param directory: Path to search
:param enum_failed: If True, will return None for modules which fail.
Otherwise, failing modules are skipped.
:param recursive: Recursively walk directory tree searching for strategies
:return: List of dicts containing 'name', 'class' and 'location' entries
"""
logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'")
objects = []
for entry in directory.iterdir():
if (
recursive and entry.is_dir()
and not entry.name.startswith('__')
and not entry.name.startswith('.')
):
objects.extend(cls.search_all_objects(entry, enum_failed, recursive=recursive))
# Only consider python files
if entry.suffix != '.py':
logger.debug('Ignoring %s', entry)

32
freqtrade/resolvers/strategy_resolver.py
View File

@ -7,8 +7,9 @@ import logging
import tempfile
from base64 import urlsafe_b64decode
from inspect import getfullargspec
from os import walk
from pathlib import Path
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
from freqtrade.configuration.config_validation import validate_migrated_strategy_settings
from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES
@ -216,15 +217,19 @@ class StrategyResolver(IResolver):
raise OperationalException(
"`populate_exit_trend` or `populate_sell_trend` must be implemented.")
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
_populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
_buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
_sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
if any(x == 2 for x in [
strategy._populate_fun_len,
strategy._buy_fun_len,
strategy._sell_fun_len
_populate_fun_len,
_buy_fun_len,
_sell_fun_len
]):
strategy.INTERFACE_VERSION = 1
raise OperationalException(
"Strategy Interface v1 is no longer supported. "
"Please update your strategy to implement "
"`populate_indicators`, `populate_entry_trend` and `populate_exit_trend` "
"with the metadata argument. ")
return strategy
@staticmethod
@ -237,10 +242,19 @@ class StrategyResolver(IResolver):
:param extra_dir: additional directory to search for the given strategy
:return: Strategy instance or None
"""
if config.get('recursive_strategy_search', False):
extra_dirs: List[str] = [
path[0] for path in walk(f"{config['user_data_dir']}/{USERPATH_STRATEGIES}")
] # sub-directories
else:
extra_dirs = []
if extra_dir:
extra_dirs.append(extra_dir)
abs_paths = StrategyResolver.build_search_paths(config,
user_subdir=USERPATH_STRATEGIES,
extra_dir=extra_dir)
extra_dirs=extra_dirs)
if ":" in strategy_name:
logger.info("loading base64 encoded strategy")

3
freqtrade/rpc/api_server/api_v1.py
View File

@ -253,7 +253,8 @@ def list_strategies(config=Depends(get_config)):
directory = Path(config.get(
'strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
from freqtrade.resolvers.strategy_resolver import StrategyResolver
strategies = StrategyResolver.search_all_objects(directory, False)
strategies = StrategyResolver.search_all_objects(
directory, False, config.get('recursive_strategy_search', False))
strategies = sorted(strategies, key=lambda x: x['name'])
return {'strategies': [x['name'] for x in strategies]}

2
freqtrade/rpc/telegram.py
View File

@ -943,7 +943,7 @@ class Telegram(RPCHandler):
else:
fiat_currency = self._config.get('fiat_display_currency', '')
try:
statlist, head, fiat_profit_sum = self._rpc._rpc_status_table(
statlist, _, _ = self._rpc._rpc_status_table(
self._config['stake_currency'], fiat_currency)
except RPCException:
self._send_msg(msg='No open trade found.')

2
freqtrade/strategy/informative_decorator.py
View File

@ -23,7 +23,7 @@ class InformativeData:
def informative(timeframe: str, asset: str = '',
fmt: Optional[Union[str, Callable[[Any], str]]] = None,
*,
candle_type: Optional[CandleType] = None,
candle_type: Optional[Union[CandleType, str]] = None,
ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]:
"""
A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to

33
freqtrade/strategy/interface.py
View File

@ -3,7 +3,6 @@ IStrategy interface
This module defines the interface to apply for strategies
"""
import logging
import warnings
from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple, Union
@ -44,14 +43,11 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
# Strategy interface version
# Default to version 2
# Version 1 is the initial interface without metadata dict
# Version 1 is the initial interface without metadata dict - deprecated and no longer supported.
# Version 2 populate_* include metadata dict
# Version 3 - First version with short and leverage support
INTERFACE_VERSION: int = 3
_populate_fun_len: int = 0
_buy_fun_len: int = 0
_sell_fun_len: int = 0
_ft_params_from_file: Dict
# associated minimal roi
minimal_roi: Dict = {}
@ -114,7 +110,7 @@ class IStrategy(ABC, HyperStrategyMixin):
# Class level variables (intentional) containing
# the dataprovider (dp) (access to other candles, historic data, ...)
# and wallets - access to the current balance.
dp: Optional[DataProvider]
dp: DataProvider
wallets: Optional[Wallets] = None
# Filled from configuration
stake_currency: str
@ -359,7 +355,7 @@ class IStrategy(ABC, HyperStrategyMixin):
def custom_exit_price(self, pair: str, trade: Trade,
current_time: datetime, proposed_rate: float,
current_profit: float, **kwargs) -> float:
current_profit: float, exit_tag: Optional[str], **kwargs) -> float:
"""
Custom exit price logic, returning the new exit price.
@ -372,6 +368,7 @@ class IStrategy(ABC, HyperStrategyMixin):
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in exit_pricing.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param exit_tag: Exit reason.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New exit price value if provided
"""
@ -1116,12 +1113,7 @@ class IStrategy(ABC, HyperStrategyMixin):
dataframe = _create_and_merge_informative_pair(
self, dataframe, metadata, inf_data, populate_fn)
if self._populate_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
return self.populate_indicators(dataframe) # type: ignore
else:
return self.populate_indicators(dataframe, metadata)
return self.populate_indicators(dataframe, metadata)
def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
@ -1135,12 +1127,7 @@ class IStrategy(ABC, HyperStrategyMixin):
logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.")
if self._buy_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
df = self.populate_buy_trend(dataframe) # type: ignore
else:
df = self.populate_entry_trend(dataframe, metadata)
df = self.populate_entry_trend(dataframe, metadata)
if 'enter_long' not in df.columns:
df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns')
@ -1155,14 +1142,8 @@ class IStrategy(ABC, HyperStrategyMixin):
currently traded pair
:return: DataFrame with exit column
"""
logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.")
if self._sell_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
df = self.populate_sell_trend(dataframe) # type: ignore
else:
df = self.populate_exit_trend(dataframe, metadata)
df = self.populate_exit_trend(dataframe, metadata)
if 'exit_long' not in df.columns:
df = df.rename({'sell': 'exit_long'}, axis='columns')
return df

14
freqtrade/strategy/strategy_helper.py
View File

@ -56,12 +56,18 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
# Combine the 2 dataframes
# all indicators on the informative sample MUST be calculated before this point
dataframe = pd.merge(dataframe, informative, left_on='date',
right_on=date_merge, how='left')
if ffill:
# https://pandas.pydata.org/docs/user_guide/merging.html#timeseries-friendly-merging
# merge_ordered - ffill method is 2.5x faster than seperate ffill()
dataframe = pd.merge_ordered(dataframe, informative, fill_method="ffill", left_on='date',
right_on=date_merge, how='left')
else:
dataframe = pd.merge(dataframe, informative, left_on='date',
right_on=date_merge, how='left')
dataframe = dataframe.drop(date_merge, axis=1)
if ffill:
dataframe = dataframe.ffill()
# if ffill:
# dataframe = dataframe.ffill()
return dataframe

3
freqtrade/templates/subtemplates/strategy_methods_advanced.j2
View File

@ -58,7 +58,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate:
def custom_exit_price(self, pair: str, trade: 'Trade',
current_time: 'datetime', proposed_rate: float,
current_profit: float, **kwargs) -> float:
current_profit: float, exit_tag: Optional[str], **kwargs) -> float:
"""
Custom exit price logic, returning the new exit price.
@ -71,6 +71,7 @@ def custom_exit_price(self, pair: str, trade: 'Trade',
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in exit_pricing.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param exit_tag: Exit reason.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New exit price value if provided
"""

1
mkdocs.yml
View File

@ -29,6 +29,7 @@ nav:
- Data Analysis:
- Jupyter Notebooks: data-analysis.md
- Strategy analysis: strategy_analysis_example.md
- Backtest analysis: advanced-backtesting.md
- Advanced Topics:
- Advanced Post-installation Tasks: advanced-setup.md
- Edge Positioning: edge.md

8
requirements-dev.txt
View File

@ -9,7 +9,7 @@ flake8==4.0.1
flake8-tidy-imports==4.6.0
mypy==0.942
pre-commit==2.18.1
pytest==7.1.1
pytest==7.1.2
pytest-asyncio==0.18.3
pytest-cov==3.0.0
pytest-mock==3.7.0
@ -24,8 +24,6 @@ nbconvert==6.5.0
# mypy types
types-cachetools==5.0.1
types-filelock==3.2.5
types-requests==2.27.19
types-requests==2.27.20
types-tabulate==0.8.7
# Extensions to datetime library
types-python-dateutil==2.8.11
types-python-dateutil==2.8.12

1
requirements-hyperopt.txt
View File

@ -6,5 +6,4 @@ scipy==1.8.0
scikit-learn==1.0.2
scikit-optimize==0.9.0
filelock==3.6.0
joblib==1.1.0
progressbar2==4.0.0

1
requirements-plot.txt
View File

@ -2,4 +2,3 @@
-r requirements.txt
plotly==5.7.0

3
requirements.txt
View File

@ -2,7 +2,7 @@ numpy==1.22.3
pandas==1.4.2
pandas-ta==0.3.14b
ccxt==1.79.81
ccxt==1.80.61
# Pin cryptography for now due to rust build errors with piwheels
cryptography==36.0.2
aiohttp==3.8.1
@ -20,6 +20,7 @@ pycoingecko==2.2.0
jinja2==3.1.1
tables==3.7.0
blosc==1.10.6
joblib==1.1.0
# find first, C search in arrays
py_find_1st==1.1.5

5
setup.cfg
View File

@ -52,6 +52,11 @@ exclude =
[mypy]
ignore_missing_imports = True
warn_unused_ignores = True
exclude = (?x)(
^build_helpers\.py$
)
[mypy-tests.*]
ignore_errors = True

2
setup.sh
View File

@ -90,7 +90,7 @@ function updateenv() {
echo "pip install completed"
echo
if [[ $dev =~ ^[Yy]$ ]]; then
${PYTHON} -m pre-commit install
${PYTHON} -m pre_commit install
if [ $? -ne 0 ]; then
echo "Failed installing pre-commit"
exit 1

34
tests/commands/test_commands.py
View File

@ -847,7 +847,7 @@ def test_start_convert_trades(mocker, caplog):
assert convert_mock.call_count == 1
def test_start_list_strategies(mocker, caplog, capsys):
def test_start_list_strategies(capsys):
args = [
"list-strategies",
@ -859,8 +859,8 @@ def test_start_list_strategies(mocker, caplog, capsys):
# pargs['config'] = None
start_list_strategies(pargs)
captured = capsys.readouterr()
assert "TestStrategyLegacyV1" in captured.out
assert "legacy_strategy_v1.py" not in captured.out
assert "StrategyTestV2" in captured.out
assert "strategy_test_v2.py" not in captured.out
assert CURRENT_TEST_STRATEGY in captured.out
# Test regular output
@ -874,8 +874,8 @@ def test_start_list_strategies(mocker, caplog, capsys):
# pargs['config'] = None
start_list_strategies(pargs)
captured = capsys.readouterr()
assert "TestStrategyLegacyV1" in captured.out
assert "legacy_strategy_v1.py" in captured.out
assert "StrategyTestV2" in captured.out
assert "strategy_test_v2.py" in captured.out
assert CURRENT_TEST_STRATEGY in captured.out
# Test color output
@ -888,10 +888,30 @@ def test_start_list_strategies(mocker, caplog, capsys):
# pargs['config'] = None
start_list_strategies(pargs)
captured = capsys.readouterr()
assert "TestStrategyLegacyV1" in captured.out
assert "legacy_strategy_v1.py" in captured.out
assert "StrategyTestV2" in captured.out
assert "strategy_test_v2.py" in captured.out
assert CURRENT_TEST_STRATEGY in captured.out
assert "LOAD FAILED" in captured.out
# Recursive
assert "TestStrategyNoImplements" not in captured.out
# Test recursive
args = [
"list-strategies",
"--strategy-path",
str(Path(__file__).parent.parent / "strategy" / "strats"),
'--no-color',
'--recursive-strategy-search'
]
pargs = get_args(args)
# pargs['config'] = None
start_list_strategies(pargs)
captured = capsys.readouterr()
assert "StrategyTestV2" in captured.out
assert "strategy_test_v2.py" in captured.out
assert "StrategyTestV2" in captured.out
assert "TestStrategyNoImplements" in captured.out
assert str(Path("broken_strats/broken_futures_strategies.py")) in captured.out
def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):

55
tests/conftest.py
View File

@ -1632,40 +1632,6 @@ def limit_buy_order(limit_buy_order_open):
return order
@pytest.fixture(scope='function')
def market_buy_order():
return {
'id': 'mocked_market_buy',
'type': 'market',
'side': 'buy',
'symbol': 'mocked',
'timestamp': arrow.utcnow().int_timestamp * 1000,
'datetime': arrow.utcnow().isoformat(),
'price': 0.00004099,
'amount': 91.99181073,
'filled': 91.99181073,
'remaining': 0.0,
'status': 'closed'
}
@pytest.fixture
def market_sell_order():
return {
'id': 'mocked_limit_sell',
'type': 'market',
'side': 'sell',
'symbol': 'mocked',
'timestamp': arrow.utcnow().int_timestamp * 1000,
'datetime': arrow.utcnow().isoformat(),
'price': 0.00004173,
'amount': 91.99181073,
'filled': 91.99181073,
'remaining': 0.0,
'status': 'closed'
}
@pytest.fixture
def limit_buy_order_old():
return {
@ -2672,6 +2638,7 @@ def saved_hyperopt_results():
'total_profit': -0.00125625,
'current_epoch': 1,
'is_initial_point': True,
'is_random': False,
'is_best': True,
}, {
@ -2688,6 +2655,7 @@ def saved_hyperopt_results():
'total_profit': 6.185e-05,
'current_epoch': 2,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': 14.241196856510731,
@ -2698,6 +2666,7 @@ def saved_hyperopt_results():
'total_profit': -0.13639474,
'current_epoch': 3,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': 100000,
@ -2705,7 +2674,7 @@ def saved_hyperopt_results():
'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501
'results_metrics': {'total_trades': 0, 'wins': 0, 'draws': 0, 'losses': 0, 'profit_mean': None, 'profit_median': None, 'profit_total': 0, 'profit': 0.0, 'holding_avg': timedelta()}, # noqa: E501
'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501
'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False
'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_random': False, 'is_best': False # noqa: E501
}, {
'loss': 0.22195522184191518,
'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501
@ -2715,6 +2684,7 @@ def saved_hyperopt_results():
'total_profit': -0.002480140000000001,
'current_epoch': 5,
'is_initial_point': True,
'is_random': False,
'is_best': True
}, {
'loss': 0.545315889154162,
@ -2725,6 +2695,7 @@ def saved_hyperopt_results():
'total_profit': -0.0041773,
'current_epoch': 6,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': 4.713497421432944,
@ -2737,6 +2708,7 @@ def saved_hyperopt_results():
'total_profit': -0.06339929,
'current_epoch': 7,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': 20.0, # noqa: E501
@ -2747,6 +2719,7 @@ def saved_hyperopt_results():
'total_profit': 0.0,
'current_epoch': 8,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': 2.4731817780991223,
@ -2757,6 +2730,7 @@ def saved_hyperopt_results():
'total_profit': -0.044050070000000004, # noqa: E501
'current_epoch': 9,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': -0.2604606005845212, # noqa: E501
@ -2767,6 +2741,7 @@ def saved_hyperopt_results():
'total_profit': 0.00021629,
'current_epoch': 10,
'is_initial_point': True,
'is_random': False,
'is_best': True
}, {
'loss': 4.876465945994304, # noqa: E501
@ -2778,6 +2753,7 @@ def saved_hyperopt_results():
'total_profit': -0.07436117,
'current_epoch': 11,
'is_initial_point': True,
'is_random': False,
'is_best': False
}, {
'loss': 100000,
@ -2788,6 +2764,7 @@ def saved_hyperopt_results():
'total_profit': 0,
'current_epoch': 12,
'is_initial_point': True,
'is_random': False,
'is_best': False
}
]
@ -2935,14 +2912,6 @@ def limit_order(limit_buy_order_usdt, limit_sell_order_usdt):
}
@pytest.fixture(scope='function')
def market_order(market_buy_order_usdt, market_sell_order_usdt):
return {
'buy': market_buy_order_usdt,
'sell': market_sell_order_usdt
}
@pytest.fixture(scope='function')
def limit_order_open(limit_buy_order_usdt_open, limit_sell_order_usdt_open):
return {

27
tests/data/test_btanalysis.py
View File

@ -8,13 +8,13 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_csum,
calculate_market_change, calculate_max_drawdown,
calculate_underwater, combine_dataframes_with_mean,
create_cum_profit, extract_trades_of_period,
get_latest_backtest_filename, get_latest_hyperopt_file,
load_backtest_data, load_backtest_metadata, load_trades,
load_trades_from_db)
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr,
calculate_csum, calculate_market_change,
calculate_max_drawdown, calculate_underwater,
combine_dataframes_with_mean, create_cum_profit,
extract_trades_of_period, get_latest_backtest_filename,
get_latest_hyperopt_file, load_backtest_data,
load_backtest_metadata, load_trades, load_trades_from_db)
from freqtrade.data.history import load_data, load_pair_history
from freqtrade.exceptions import OperationalException
from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades
@ -336,6 +336,19 @@ def test_calculate_csum(testdatadir):
csum_min, csum_max = calculate_csum(DataFrame())
@pytest.mark.parametrize('start,end,days, expected', [
(64900, 176000, 3 * 365, 0.3945),
(64900, 176000, 365, 1.7119),
(1000, 1000, 365, 0.0),
(1000, 1500, 365, 0.5),
(1000, 1500, 100, 3.3927), # sub year
(0.01000000, 0.01762792, 120, 4.6087), # sub year BTC values
])
def test_calculate_cagr(start, end, days, expected):
assert round(calculate_cagr(days, start, end), 4) == expected
def test_calculate_max_drawdown2():
values = [0.011580, 0.010048, 0.011340, 0.012161, 0.010416, 0.010009, 0.020024,
-0.024662, -0.022350, 0.020496, -0.029859, -0.030511, 0.010041, 0.010872,

45
tests/edge/test_edge.py
View File

@ -8,7 +8,7 @@ from unittest.mock import MagicMock
import arrow
import numpy as np
import pytest
from pandas import DataFrame, to_datetime
from pandas import DataFrame
from freqtrade.data.converter import ohlcv_to_dataframe
from freqtrade.edge import Edge, PairInfo
@ -30,49 +30,6 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe,
tests_start_time = arrow.get(2018, 10, 3)
timeframe_in_minute = 60
# Helpers for this test file
def _validate_ohlc(buy_ohlc_sell_matrice):
for index, ohlc in enumerate(buy_ohlc_sell_matrice):
# if not high < open < low or not high < close < low
if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]:
raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!')
return True
def _build_dataframe(buy_ohlc_sell_matrice):
_validate_ohlc(buy_ohlc_sell_matrice)
data = []
for ohlc in buy_ohlc_sell_matrice:
d = {
'date': tests_start_time.shift(
minutes=(
ohlc[0] *
timeframe_in_minute)).int_timestamp *
1000,
'buy': ohlc[1],
'open': ohlc[2],
'high': ohlc[3],
'low': ohlc[4],
'close': ohlc[5],
'sell': ohlc[6]}
data.append(d)
frame = DataFrame(data)
frame['date'] = to_datetime(frame['date'],
unit='ms',
utc=True,
infer_datetime_format=True)
return frame
def _time_on_candle(number):
return np.datetime64(tests_start_time.shift(
minutes=(number * timeframe_in_minute)).int_timestamp * 1000, 'ms')
# End helper functions
# Open trade should be removed from the end
tc0 = BTContainer(data=[

4
tests/exchange/test_exchange.py
View File

@ -909,7 +909,7 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker):
mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
with pytest.raises(OperationalException,
match=r'The ccxt library does not provide the list of timeframes '
r'for the exchange ".*" and this exchange '
r'for the exchange .* and this exchange '
r'is therefore not supported. *'):
Exchange(default_conf)
@ -930,7 +930,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker):
mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
with pytest.raises(OperationalException,
match=r'The ccxt library does not provide the list of timeframes '
r'for the exchange ".*" and this exchange '
r'for the exchange .* and this exchange '
r'is therefore not supported. *'):
Exchange(default_conf)

71
tests/optimize/test_backtesting.py
View File

@ -384,14 +384,16 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats')
mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
sbs = mocker.patch('freqtrade.optimize.backtesting.store_backtest_stats')
sbc = mocker.patch('freqtrade.optimize.backtesting.store_backtest_signal_candles')
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['UNITTEST/BTC']))
default_conf['timeframe'] = '1m'
default_conf['datadir'] = testdatadir
default_conf['export'] = 'trades'
default_conf['export'] = 'signals'
default_conf['exportfilename'] = 'export.txt'
default_conf['timerange'] = '-1510694220'
default_conf['runmode'] = RunMode.BACKTEST
backtesting = Backtesting(default_conf)
backtesting._set_strategy(backtesting.strategylist[0])
@ -407,6 +409,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
assert backtesting.strategy.dp._pairlists is not None
assert backtesting.strategy.bot_loop_start.call_count == 1
assert sbs.call_count == 1
assert sbc.call_count == 1
def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None:
@ -497,7 +500,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti
Backtesting(default_conf)
# Multiple strategies
default_conf['strategy_list'] = [CURRENT_TEST_STRATEGY, 'TestStrategyLegacyV1']
default_conf['strategy_list'] = [CURRENT_TEST_STRATEGY, 'StrategyTestV2']
with pytest.raises(OperationalException,
match='PrecisionFilter not allowed for backtesting multiple strategies.'):
Backtesting(default_conf)
@ -711,7 +714,7 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None:
)
# No data available.
res = backtesting._get_sell_trade_entry(trade, row_sell)
res = backtesting._get_exit_trade_entry(trade, row_sell)
assert res is not None
assert res.exit_reason == ExitType.ROI.value
assert res.close_date_utc == datetime(2020, 1, 1, 5, 0, tzinfo=timezone.utc)
@ -724,13 +727,13 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None:
[], columns=['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'long_tag', 'short_tag', 'exit_tag'])
res = backtesting._get_sell_trade_entry(trade, row)
res = backtesting._get_exit_trade_entry(trade, row)
assert res is None
# Assign backtest-detail data
backtesting.detail_data[pair] = row_detail
res = backtesting._get_sell_trade_entry(trade, row_sell)
res = backtesting._get_exit_trade_entry(trade, row_sell)
assert res is not None
assert res.exit_reason == ExitType.ROI.value
# Sell at minute 3 (not available above!)
@ -1195,7 +1198,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
'--disable-max-market-positions',
'--strategy-list',
CURRENT_TEST_STRATEGY,
'TestStrategyLegacyV1',
'StrategyTestV2',
]
args = get_args(args)
start_backtesting(args)
@ -1218,14 +1221,13 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
'up to 2017-11-14 22:58:00 (0 days).',
'Parameter --enable-position-stacking detected ...',
f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
'Running backtesting for Strategy TestStrategyLegacyV1',
'Running backtesting for Strategy StrategyTestV2',
]
for line in exists:
assert log_has(line, caplog)
@pytest.mark.filterwarnings("ignore:deprecated")
def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdatadir, capsys):
default_conf.update({
"use_exit_signal": True,
@ -1307,7 +1309,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
'--breakdown', 'day',
'--strategy-list',
CURRENT_TEST_STRATEGY,
'TestStrategyLegacyV1',
'StrategyTestV2',
]
args = get_args(args)
start_backtesting(args)
@ -1324,7 +1326,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
'up to 2017-11-14 22:58:00 (0 days).',
'Parameter --enable-position-stacking detected ...',
f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
'Running backtesting for Strategy TestStrategyLegacyV1',
'Running backtesting for Strategy StrategyTestV2',
]
for line in exists:
@ -1339,6 +1341,39 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
assert 'STRATEGY SUMMARY' in captured.out
@pytest.mark.filterwarnings("ignore:deprecated")
def test_backtest_start_futures_noliq(default_conf_usdt, mocker,
caplog, testdatadir, capsys):
# Tests detail-data loading
default_conf_usdt.update({
"trading_mode": "futures",
"margin_mode": "isolated",
"use_exit_signal": True,
"exit_profit_only": False,
"exit_profit_offset": 0.0,
"ignore_roi_if_entry_signal": False,
"strategy": CURRENT_TEST_STRATEGY,
})
patch_exchange(mocker)
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
# mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
patched_configuration_load_config_file(mocker, default_conf_usdt)
args = [
'backtesting',
'--config', 'config.json',
'--datadir', str(testdatadir),
'--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'),
'--timeframe', '1h',
]
args = get_args(args)
with pytest.raises(OperationalException, match=r"Pairs .* got no leverage tiers available\."):
start_backtesting(args)
@pytest.mark.filterwarnings("ignore:deprecated")
def test_backtest_start_nomock_futures(default_conf_usdt, mocker,
caplog, testdatadir, capsys):
@ -1589,7 +1624,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
min_backtest_date = now - timedelta(weeks=4)
load_backtest_metadata = MagicMock(return_value={
'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()},
'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()}
'StrategyTestV3': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()}
})
load_backtest_stats = MagicMock(side_effect=[
{
@ -1598,9 +1633,9 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
'strategy_comparison': [{'key': 'StrategyTestV2'}]
},
{
'metadata': {'TestStrategyLegacyV1': {'run_id': '2'}},
'strategy': {'TestStrategyLegacyV1': {}},
'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}]
'metadata': {'StrategyTestV3': {'run_id': '2'}},
'strategy': {'StrategyTestV3': {}},
'strategy_comparison': [{'key': 'StrategyTestV3'}]
}
])
mocker.patch('pathlib.Path.glob', return_value=[
@ -1624,7 +1659,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
'--cache', cache,
'--strategy-list',
'StrategyTestV2',
'TestStrategyLegacyV1',
'StrategyTestV3',
]
args = get_args(args)
start_backtesting(args)
@ -1646,7 +1681,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
assert backtestmock.call_count == 2
exists = [
'Running backtesting for Strategy StrategyTestV2',
'Running backtesting for Strategy TestStrategyLegacyV1',
'Running backtesting for Strategy StrategyTestV3',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
]
@ -1654,12 +1689,12 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
assert backtestmock.call_count == 0
exists = [
'Reusing result of previous backtest for StrategyTestV2',
'Reusing result of previous backtest for TestStrategyLegacyV1',
'Reusing result of previous backtest for StrategyTestV3',
]
else:
exists = [
'Reusing result of previous backtest for StrategyTestV2',
'Running backtesting for Strategy TestStrategyLegacyV1',
'Running backtesting for Strategy StrategyTestV3',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
]

2
tests/optimize/test_hyperopt.py
View File

@ -41,6 +41,7 @@ def generate_result_metrics():
'max_drawdown_abs': 0.001,
'loss': 0.001,
'is_initial_point': 0.001,
'is_random': False,
'is_best': 1,
}
@ -247,6 +248,7 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
'total_profit': 0,
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
'is_initial_point': False,
'is_random': False,
'is_best': True
}
)

58
tests/optimize/test_optimize_reports.py
View File

@ -2,6 +2,7 @@ import re
from datetime import timedelta
from pathlib import Path
import joblib
import pandas as pd
import pytest
from arrow import Arrow
@ -19,6 +20,7 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene
generate_periodic_breakdown_stats,
generate_strategy_comparison,
generate_trading_stats, show_sorted_pairlist,
store_backtest_signal_candles,
store_backtest_stats, text_table_bt_results,
text_table_exit_reason, text_table_strategy)
from freqtrade.resolvers.strategy_resolver import StrategyResolver
@ -201,6 +203,62 @@ def test_store_backtest_stats(testdatadir, mocker):
assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult'))
def test_store_backtest_candles(testdatadir, mocker):
dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_joblib')
candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}
# mock directory exporting
store_backtest_signal_candles(testdatadir, candle_dict)
assert dump_mock.call_count == 1
assert isinstance(dump_mock.call_args_list[0][0][0], Path)
assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl'))
dump_mock.reset_mock()
# mock file exporting
filename = Path(testdatadir / 'testresult')
store_backtest_signal_candles(filename, candle_dict)
assert dump_mock.call_count == 1
assert isinstance(dump_mock.call_args_list[0][0][0], Path)
# result will be testdatadir / testresult-<timestamp>_signals.pkl
assert str(dump_mock.call_args_list[0][0][0]).endswith(str('_signals.pkl'))
dump_mock.reset_mock()
def test_write_read_backtest_candles(tmpdir):
candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}}
# test directory exporting
stored_file = store_backtest_signal_candles(Path(tmpdir), candle_dict)
scp = open(stored_file, "rb")
pickled_signal_candles = joblib.load(scp)
scp.close()
assert pickled_signal_candles.keys() == candle_dict.keys()
assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys()
assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \
.equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC'])
_clean_test_file(stored_file)
# test file exporting
filename = Path(tmpdir / 'testresult')
stored_file = store_backtest_signal_candles(filename, candle_dict)
scp = open(stored_file, "rb")
pickled_signal_candles = joblib.load(scp)
scp.close()
assert pickled_signal_candles.keys() == candle_dict.keys()
assert pickled_signal_candles['DefStrat'].keys() == pickled_signal_candles['DefStrat'].keys()
assert pickled_signal_candles['DefStrat']['UNITTEST/BTC'] \
.equals(pickled_signal_candles['DefStrat']['UNITTEST/BTC'])
_clean_test_file(stored_file)
def test_generate_pair_metrics():
results = pd.DataFrame(

1
tests/rpc/test_rpc_apiserver.py
View File

@ -1389,7 +1389,6 @@ def test_api_strategies(botclient):
'StrategyTestV2',
'StrategyTestV3',
'StrategyTestV3Futures',
'TestStrategyLegacyV1',
]}

30
tests/strategy/strats/broken_strats/legacy_strategy_v1.py Normal file
View File

@ -0,0 +1,30 @@
# type: ignore
from pandas import DataFrame
from freqtrade.strategy import IStrategy
# Dummy strategy - no longer loads but raises an exception.
class TestStrategyLegacyV1(IStrategy):
minimal_roi = {
"40": 0.0,
"30": 0.01,
"20": 0.02,
"0": 0.04
}
stoploss = -0.10
timeframe = '5m'
def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
return dataframe
def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
return dataframe
def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame:
return dataframe

85
tests/strategy/strats/legacy_strategy_v1.py
View File

@ -1,85 +0,0 @@
# --- Do not remove these libs ---
# Add your lib to import here
import talib.abstract as ta
from pandas import DataFrame
from freqtrade.strategy import IStrategy
# --------------------------------
# This class is a sample. Feel free to customize it.
class TestStrategyLegacyV1(IStrategy):
"""
This is a test strategy using the legacy function headers, which will be
removed in a future update.
Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py
for a uptodate version of this template.
"""
# Minimal ROI designed for the strategy.
# This attribute will be overridden if the config file contains "minimal_roi"
minimal_roi = {
"40": 0.0,
"30": 0.01,
"20": 0.02,
"0": 0.04
}
# Optimal stoploss designed for the strategy
# This attribute will be overridden if the config file contains "stoploss"
stoploss = -0.10
timeframe = '5m'
def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
"""
Adds several different TA indicators to the given DataFrame
Performance Note: For the best performance be frugal on the number of indicators
you are using. Let uncomment only the indicator you are using in your strategies
or your hyperopt configuration, otherwise you will waste your memory and CPU usage.
"""
# Momentum Indicator
# ------------------------------------
# ADX
dataframe['adx'] = ta.ADX(dataframe)
# TEMA - Triple Exponential Moving Average
dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)
return dataframe
def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
"""
Based on TA indicators, populates the buy signal for the given dataframe
:param dataframe: DataFrame
:return: DataFrame with buy column
"""
dataframe.loc[
(
(dataframe['adx'] > 30) &
(dataframe['tema'] > dataframe['tema'].shift(1)) &
(dataframe['volume'] > 0)
),
'buy'] = 1
return dataframe
def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame:
"""
Based on TA indicators, populates the sell signal for the given dataframe
:param dataframe: DataFrame
:return: DataFrame with buy column
"""
dataframe.loc[
(
(dataframe['adx'] > 70) &
(dataframe['tema'] < dataframe['tema'].shift(1)) &
(dataframe['volume'] > 0)
),
'sell'] = 1
return dataframe

13
tests/strategy/strats/strategy_test_v2.py
View File

@ -56,19 +56,6 @@ class StrategyTestV2(IStrategy):
# By default this strategy does not use Position Adjustments
position_adjustment_enable = False
def informative_pairs(self):
"""
Define additional, informative pair/interval combinations to be cached from the exchange.
These pair/interval combinations are non-tradeable, unless they are part
of the whitelist as well.
For more information, please consult the documentation
:return: List of tuples in the format (pair, interval)
Sample: return [("ETH/USDT", "5m"),
("BTC/USDT", "15m"),
]
"""
return []
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Adds several different TA indicators to the given DataFrame

2
tests/strategy/test_interface.py
View File

@ -686,7 +686,7 @@ def test_is_pair_locked(default_conf):
def test_is_informative_pairs_callback(default_conf):
default_conf.update({'strategy': 'TestStrategyLegacyV1'})
default_conf.update({'strategy': 'StrategyTestV2'})
strategy = StrategyResolver.load_strategy(default_conf)
# Should return empty
# Uses fallback to base implementation

15
tests/strategy/test_strategy_helpers.py
View File

@ -68,6 +68,21 @@ def test_merge_informative_pair():
assert result.iloc[7]['date_1h'] == result.iloc[4]['date']
assert result.iloc[8]['date_1h'] == result.iloc[4]['date']
informative = generate_test_data('1h', 40)
result = merge_informative_pair(data, informative, '15m', '1h', ffill=False)
# First 3 rows are empty
assert result.iloc[0]['date_1h'] is pd.NaT
assert result.iloc[1]['date_1h'] is pd.NaT
assert result.iloc[2]['date_1h'] is pd.NaT
# Next 4 rows contain the starting date (0:00)
assert result.iloc[3]['date_1h'] == result.iloc[0]['date']
assert result.iloc[4]['date_1h'] is pd.NaT
assert result.iloc[5]['date_1h'] is pd.NaT
assert result.iloc[6]['date_1h'] is pd.NaT
# Next 4 rows contain the next Hourly date original date row 4
assert result.iloc[7]['date_1h'] == result.iloc[4]['date']
assert result.iloc[8]['date_1h'] is pd.NaT
def test_merge_informative_pair_same():
data = generate_test_data('15m', 40)

76
tests/strategy/test_strategy_loading.py
View File

@ -1,6 +1,5 @@
# pragma pylint: disable=missing-docstring, protected-access, C0103
import logging
import warnings
from base64 import urlsafe_b64encode
from pathlib import Path
@ -35,7 +34,7 @@ def test_search_all_strategies_no_failed():
directory = Path(__file__).parent / "strats"
strategies = StrategyResolver.search_all_objects(directory, enum_failed=False)
assert isinstance(strategies, list)
assert len(strategies) == 6
assert len(strategies) == 5
assert isinstance(strategies[0], dict)
@ -43,10 +42,10 @@ def test_search_all_strategies_with_failed():
directory = Path(__file__).parent / "strats"
strategies = StrategyResolver.search_all_objects(directory, enum_failed=True)
assert isinstance(strategies, list)
assert len(strategies) == 7
assert len(strategies) == 6
# with enum_failed=True search_all_objects() shall find 2 good strategies
# and 1 which fails to load
assert len([x for x in strategies if x['class'] is not None]) == 6
assert len([x for x in strategies if x['class'] is not None]) == 5
assert len([x for x in strategies if x['class'] is None]) == 1
@ -100,7 +99,7 @@ def test_load_strategy_noname(default_conf):
@pytest.mark.filterwarnings("ignore:deprecated")
@pytest.mark.parametrize('strategy_name', ['StrategyTestV2', 'TestStrategyLegacyV1'])
@pytest.mark.parametrize('strategy_name', ['StrategyTestV2'])
def test_strategy_pre_v3(result, default_conf, strategy_name):
default_conf.update({'strategy': strategy_name})
@ -346,40 +345,6 @@ def test_strategy_override_use_exit_profit_only(caplog, default_conf):
assert log_has("Override strategy 'exit_profit_only' with value in config file: True.", caplog)
@pytest.mark.filterwarnings("ignore:deprecated")
def test_deprecate_populate_indicators(result, default_conf):
default_location = Path(__file__).parent / "strats"
default_conf.update({'strategy': 'TestStrategyLegacyV1',
'strategy_path': default_location})
strategy = StrategyResolver.load_strategy(default_conf)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
indicators = strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
assert len(w) == 1
assert issubclass(w[-1].category, DeprecationWarning)
assert "deprecated - check out the Sample strategy to see the current function headers!" \
in str(w[-1].message)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
strategy.advise_entry(indicators, {'pair': 'ETH/BTC'})
assert len(w) == 1
assert issubclass(w[-1].category, DeprecationWarning)
assert "deprecated - check out the Sample strategy to see the current function headers!" \
in str(w[-1].message)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
strategy.advise_exit(indicators, {'pair': 'ETH_BTC'})
assert len(w) == 1
assert issubclass(w[-1].category, DeprecationWarning)
assert "deprecated - check out the Sample strategy to see the current function headers!" \
in str(w[-1].message)
@pytest.mark.filterwarnings("ignore:deprecated")
def test_missing_implements(default_conf, caplog):
@ -438,33 +403,14 @@ def test_missing_implements(default_conf, caplog):
StrategyResolver.load_strategy(default_conf)
@pytest.mark.filterwarnings("ignore:deprecated")
def test_call_deprecated_function(result, default_conf, caplog):
default_location = Path(__file__).parent / "strats"
def test_call_deprecated_function(default_conf):
default_location = Path(__file__).parent / "strats/broken_strats/"
del default_conf['timeframe']
default_conf.update({'strategy': 'TestStrategyLegacyV1',
'strategy_path': default_location})
strategy = StrategyResolver.load_strategy(default_conf)
metadata = {'pair': 'ETH/BTC'}
# Make sure we are using a legacy function
assert strategy._populate_fun_len == 2
assert strategy._buy_fun_len == 2
assert strategy._sell_fun_len == 2
assert strategy.INTERFACE_VERSION == 1
assert strategy.timeframe == '5m'
indicator_df = strategy.advise_indicators(result, metadata=metadata)
assert isinstance(indicator_df, DataFrame)
assert 'adx' in indicator_df.columns
enterdf = strategy.advise_entry(result, metadata=metadata)
assert isinstance(enterdf, DataFrame)
assert 'enter_long' in enterdf.columns
exitdf = strategy.advise_exit(result, metadata=metadata)
assert isinstance(exitdf, DataFrame)
assert 'exit_long' in exitdf
with pytest.raises(OperationalException,
match=r"Strategy Interface v1 is no longer supported.*"):
StrategyResolver.load_strategy(default_conf)
def test_strategy_interface_versioning(result, default_conf):
@ -472,10 +418,6 @@ def test_strategy_interface_versioning(result, default_conf):
strategy = StrategyResolver.load_strategy(default_conf)
metadata = {'pair': 'ETH/BTC'}
# Make sure we are using a legacy function
assert strategy._populate_fun_len == 3
assert strategy._buy_fun_len == 3
assert strategy._sell_fun_len == 3
assert strategy.INTERFACE_VERSION == 2
indicator_df = strategy.advise_indicators(result, metadata=metadata)

18
tests/test_freqtradebot.py
View File

@ -717,12 +717,12 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker)
(True, 'spot', 'gateio', None, 0.0, None),
(False, 'spot', 'okx', None, 0.0, None),
(True, 'spot', 'okx', None, 0.0, None),
(True, 'futures', 'binance', 'isolated', 0.0, 11.89108910891089),
(False, 'futures', 'binance', 'isolated', 0.0, 8.070707070707071),
(True, 'futures', 'binance', 'isolated', 0.0, 11.88151815181518),
(False, 'futures', 'binance', 'isolated', 0.0, 8.080471380471382),
(True, 'futures', 'gateio', 'isolated', 0.0, 11.87413417771621),
(False, 'futures', 'gateio', 'isolated', 0.0, 8.085708510208207),
(True, 'futures', 'binance', 'isolated', 0.05, 11.796534653465345),
(False, 'futures', 'binance', 'isolated', 0.05, 8.167171717171717),
(True, 'futures', 'binance', 'isolated', 0.05, 11.7874422442244),
(False, 'futures', 'binance', 'isolated', 0.05, 8.17644781144781),
(True, 'futures', 'gateio', 'isolated', 0.05, 11.7804274688304),
(False, 'futures', 'gateio', 'isolated', 0.05, 8.181423084697796),
(True, 'futures', 'okx', 'isolated', 0.0, 11.87413417771621),
@ -845,6 +845,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order,
assert trade.open_order_id is None
assert trade.open_rate == 10
assert trade.stake_amount == round(order['price'] * order['filled'] / leverage, 8)
assert pytest.approx(trade.liquidation_price) == liq_price
# In case of rejected or expired order and partially filled
order['status'] = 'expired'
@ -932,8 +933,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order,
assert trade.open_rate_requested == 10
# In case of custom entry price not float type
freqtrade.exchange.get_maintenance_ratio_and_amt = MagicMock(return_value=(0.01, 0.01))
freqtrade.exchange.name = exchange_name
order['status'] = 'open'
order['id'] = '5568'
freqtrade.strategy.custom_entry_price = lambda **kwargs: "string price"
@ -946,7 +945,6 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order,
trade.is_short = is_short
assert trade
assert trade.open_rate_requested == 10
assert trade.liquidation_price == liq_price
# In case of too high stake amount
@ -3221,7 +3219,7 @@ def test_execute_trade_exit_custom_exit_price(
freqtrade.execute_trade_exit(
trade=trade,
limit=ticker_usdt_sell_up()['ask' if is_short else 'bid'],
exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL)
exit_check=ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL, exit_reason='foo')
)
# Sell price must be different to default bid price
@ -3249,8 +3247,8 @@ def test_execute_trade_exit_custom_exit_price(
'profit_ratio': profit_ratio,
'stake_currency': 'USDT',
'fiat_currency': 'USD',
'sell_reason': ExitType.EXIT_SIGNAL.value,
'exit_reason': ExitType.EXIT_SIGNAL.value,
'sell_reason': 'foo',
'exit_reason': 'foo',
'open_date': ANY,
'close_date': ANY,
'close_rate': ANY,