From e2a42d30272f872f28dd17c7991becf8fc6e932b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Sun, 3 Apr 2022 18:57:58 +0530 Subject: [PATCH 01/20] added pre-commit --- .pre-commit-config.yaml | 15 +++++++++++++++ requirements-dev.txt | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..93583de50 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pycqa/flake8 + rev: '4.0.1' + hooks: + - id: flake8 + args: + - max-line-length = 100, + - max-complexity = 12 +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.942' + hooks: + - id: mypy + args: [--ignore-missing-imports] diff --git a/requirements-dev.txt b/requirements-dev.txt index 063cfaa45..5266ad003 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,10 +3,12 @@ -r requirements-plot.txt -r requirements-hyperopt.txt + coveralls==3.3.1 flake8==4.0.1 flake8-tidy-imports==4.6.0 mypy==0.942 +pre-commit==2.18.1 pytest==7.1.1 pytest-asyncio==0.18.3 pytest-cov==3.0.0 From 57af08fde727231a878d32149bdf4a628ab48cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Tue, 5 Apr 2022 10:21:07 +0530 Subject: [PATCH 02/20] updated requested changes in PR #6636 --- .pre-commit-config.yaml | 27 +++++++++++++++++++++++---- CONTRIBUTING.md | 2 ++ setup.sh | 7 +++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93583de50..28eb0ae38 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,11 +5,30 @@ repos: rev: '4.0.1' hooks: - id: flake8 - args: - - max-line-length = 100, - - max-complexity = 12 + stages: [push] + - repo: https://github.com/pre-commit/mirrors-mypy rev: 'v0.942' hooks: - id: mypy - args: [--ignore-missing-imports] + stages: [push] + +- repo: https://github.com/pycqa/isort + rev: '5.10.1' + hooks: + - id: isort + name: isort (python) + stages: [push] + +# https://github.com/pre-commit/pre-commit/issues/761#issuecomment-394167542 +- repo: local + hooks: + - id: pytest + name: pytest + entry: venv/bin/pytest + language: script + pass_filenames: false + # alternatively you could `types: [python]` so it only runs when python files change + # though tests might be invalidated if you were to say change a data file + always_run: true + stages: [push] \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4e0bc024..ae9c5d81e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,8 @@ Best start by reading the [documentation](https://www.freqtrade.io/) to get a fe ## Before sending the PR +Do the following if you disabled pre-commit hook when commiting. + ### 1. Run unit tests All unit tests must pass. If a unit test is broken, change your code to diff --git a/setup.sh b/setup.sh index ebfabaca5..2c3a6710b 100755 --- a/setup.sh +++ b/setup.sh @@ -51,6 +51,7 @@ function updateenv() { echo "pip install in-progress. Please wait..." ${PYTHON} -m pip install --upgrade pip read -p "Do you want to install dependencies for dev [y/N]? " + dev=$REPLY if [[ $REPLY =~ ^[Yy]$ ]] then REQUIREMENTS=requirements-dev.txt @@ -88,6 +89,12 @@ function updateenv() { fi echo "pip install completed" echo + if [[ $dev =~ ^[Yy]$ ]] then + ${PYTHON} -m pre-commit install + if [ $? -ne 0 ]; then + echo "Failed installing pre-commit" + exit 1 + fi } # Install tab lib From 0d93916f79bc57528066fe96a245bb26d72a337a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Tue, 5 Apr 2022 11:28:22 +0530 Subject: [PATCH 03/20] Update developer.md --- docs/developer.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index ee4bac5c2..18465238f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -26,6 +26,8 @@ Alternatively (e.g. if your system is not supported by the setup.sh script), fol This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`. +Then install the git hook scripts by running `pre-commit install` + Before opening a pull request, please familiarize yourself with our [Contributing Guidelines](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md). ### Devcontainer setup From 1ea49ce864be5b40d30b5ff9a5e39a92d824d2b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Apr 2022 20:29:03 +0200 Subject: [PATCH 04/20] Support nested configurations --- freqtrade/configuration/load_config.py | 30 ++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 254ce3126..8718e9fd6 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -75,18 +75,36 @@ def load_config_file(path: str) -> Dict[str, Any]: return config -def load_from_files(files: List[str]) -> Dict[str, Any]: - +def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> Dict[str, Any]: + """ + Recursively load configuration files if specified. + Sub-files are assumed to be relative to the initial config. + """ config: Dict[str, Any] = {} + if level > 5: + raise OperationalException("Config loop detected.") if not files: return deepcopy(MINIMAL_CONFIG) # We expect here a list of config filenames - for path in files: - logger.info(f'Using config: {path} ...') - # Merge config options, overwriting old values - config = deep_merge_dicts(load_config_file(path), config) + for filename in files: + logger.info(f'Using config: {filename} ...') + if filename == '-': + # Immediately load stdin and return + return load_config_file(filename) + file = Path(filename) + if base_path: + # Prepend basepath to allow for relative assignments + file = base_path / file + + config_tmp = load_config_file(str(file)) + if 'files' in config_tmp: + config_sub = load_from_files(config_tmp['files'], file.resolve().parent, level + 1) + deep_merge_dicts(config_sub, config_tmp) + + # Merge config options, overwriting prior values + config = deep_merge_dicts(config_tmp, config) config['config_files'] = files From 3427df06539362f6f5838c537ee63d497a5c3778 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Apr 2022 16:04:54 +0200 Subject: [PATCH 05/20] Add simple test for recursive loading --- config_examples/config_full.example.json | 1 + freqtrade/constants.py | 17 +++++++-------- tests/test_configuration.py | 23 ++++++++++++++++++++- tests/testdata/testconfigs/base_config.json | 12 +++++++++++ tests/testdata/testconfigs/pricing.json | 21 +++++++++++++++++++ tests/testdata/testconfigs/pricing2.json | 18 ++++++++++++++++ tests/testdata/testconfigs/recursive.json | 6 ++++++ tests/testdata/testconfigs/testconfig.json | 6 ++++++ 8 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 tests/testdata/testconfigs/base_config.json create mode 100644 tests/testdata/testconfigs/pricing.json create mode 100644 tests/testdata/testconfigs/pricing2.json create mode 100644 tests/testdata/testconfigs/recursive.json create mode 100644 tests/testdata/testconfigs/testconfig.json diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 915db6c44..b41acb726 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -182,6 +182,7 @@ "disable_dataframe_checks": false, "strategy": "SampleStrategy", "strategy_path": "user_data/strategies/", + "files": [], "dataformat_ohlcv": "json", "dataformat_trades": "jsongz" } diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 8067c1f6a..c6a2ab5d3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -91,15 +91,14 @@ SUPPORTED_FIAT = [ ] MINIMAL_CONFIG = { - 'stake_currency': '', - 'dry_run': True, - 'exchange': { - 'name': '', - 'key': '', - 'secret': '', - 'pair_whitelist': [], - 'ccxt_async_config': { - 'enableRateLimit': True, + "stake_currency": "", + "dry_run": True, + "exchange": { + "name": "", + "key": "", + "secret": "", + "pair_whitelist": [], + "ccxt_async_config": { } } } diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 19355b9eb..39e56f075 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -18,7 +18,8 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti process_removed_setting, process_temporary_deprecated_settings) from freqtrade.configuration.environment_vars import flat_vars_to_nested_dict -from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range +from freqtrade.configuration.load_config import (load_config_file, load_file, load_from_files, + log_config_error_range) from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException @@ -206,6 +207,26 @@ def test_from_config(default_conf, mocker, caplog) -> None: assert isinstance(validated_conf['user_data_dir'], Path) +def test_from_recursive_files(testdatadir) -> None: + files = testdatadir / "testconfigs/testconfig.json" + + conf = Configuration.from_files([files]) + + assert conf + # Exchange comes from "the first config" + assert conf['exchange'] + # Pricing comes from the 2nd config + assert conf['entry_pricing'] + assert conf['entry_pricing']['price_side'] == "same" + assert conf['exit_pricing'] + # The other key comes from pricing2, which is imported by pricing.json + assert conf['exit_pricing']['price_side'] == "other" + + files = testdatadir / "testconfigs/recursive.json" + with pytest.raises(OperationalException, match="Config loop detected."): + load_from_files([files]) + + def test_print_config(default_conf, mocker, caplog) -> None: conf1 = deepcopy(default_conf) # Delete non-json elements from default_conf diff --git a/tests/testdata/testconfigs/base_config.json b/tests/testdata/testconfigs/base_config.json new file mode 100644 index 000000000..d15c5890b --- /dev/null +++ b/tests/testdata/testconfigs/base_config.json @@ -0,0 +1,12 @@ +{ + "stake_currency": "", + "dry_run": true, + "exchange": { + "name": "", + "key": "", + "secret": "", + "pair_whitelist": [], + "ccxt_async_config": { + } + } +} diff --git a/tests/testdata/testconfigs/pricing.json b/tests/testdata/testconfigs/pricing.json new file mode 100644 index 000000000..d8868443f --- /dev/null +++ b/tests/testdata/testconfigs/pricing.json @@ -0,0 +1,21 @@ +{ + "entry_pricing": { + "price_side": "same", + "use_order_book": true, + "order_book_top": 1, + "price_last_balance": 0.0, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "exit_pricing":{ + "price_side": "same", + "use_order_book": true, + "order_book_top": 1, + "price_last_balance": 0.0 + }, + "files": [ + "pricing2.json" + ] +} diff --git a/tests/testdata/testconfigs/pricing2.json b/tests/testdata/testconfigs/pricing2.json new file mode 100644 index 000000000..094783a60 --- /dev/null +++ b/tests/testdata/testconfigs/pricing2.json @@ -0,0 +1,18 @@ +{ + "entry_pricing": { + "price_side": "same", + "use_order_book": true, + "order_book_top": 1, + "price_last_balance": 0.0, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "exit_pricing":{ + "price_side": "other", + "use_order_book": true, + "order_book_top": 1, + "price_last_balance": 0.0 + } +} diff --git a/tests/testdata/testconfigs/recursive.json b/tests/testdata/testconfigs/recursive.json new file mode 100644 index 000000000..28d8ce05a --- /dev/null +++ b/tests/testdata/testconfigs/recursive.json @@ -0,0 +1,6 @@ +{ + // This file fails as it's loading itself over and over + "files": [ + "./recursive.json" + ] +} diff --git a/tests/testdata/testconfigs/testconfig.json b/tests/testdata/testconfigs/testconfig.json new file mode 100644 index 000000000..557926097 --- /dev/null +++ b/tests/testdata/testconfigs/testconfig.json @@ -0,0 +1,6 @@ +{ + "files": [ + "base_config.json", + "pricing.json" + ] +} From 1435d269962f74180df2fbd22e96de751094ddde Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Apr 2022 17:26:51 +0200 Subject: [PATCH 06/20] store config-file loading paths --- freqtrade/configuration/load_config.py | 7 +++++-- tests/test_configuration.py | 6 ++++++ .../{base_config.json => test_base_config.json} | 0 .../testconfigs/{pricing2.json => test_pricing2_conf.json} | 0 .../testconfigs/{pricing.json => test_pricing_conf.json} | 2 +- tests/testdata/testconfigs/testconfig.json | 4 ++-- 6 files changed, 14 insertions(+), 5 deletions(-) rename tests/testdata/testconfigs/{base_config.json => test_base_config.json} (100%) rename tests/testdata/testconfigs/{pricing2.json => test_pricing2_conf.json} (100%) rename tests/testdata/testconfigs/{pricing.json => test_pricing_conf.json} (92%) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 8718e9fd6..5a86ab24a 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -86,7 +86,7 @@ def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> if not files: return deepcopy(MINIMAL_CONFIG) - + files_loaded = [] # We expect here a list of config filenames for filename in files: logger.info(f'Using config: {filename} ...') @@ -101,11 +101,14 @@ def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> config_tmp = load_config_file(str(file)) if 'files' in config_tmp: config_sub = load_from_files(config_tmp['files'], file.resolve().parent, level + 1) + files_loaded.extend(config_sub.get('config_files', [])) deep_merge_dicts(config_sub, config_tmp) + files_loaded.insert(0, str(file)) + # Merge config options, overwriting prior values config = deep_merge_dicts(config_tmp, config) - config['config_files'] = files + config['config_files'] = files_loaded return config diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 39e56f075..957468b86 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -222,6 +222,12 @@ def test_from_recursive_files(testdatadir) -> None: # The other key comes from pricing2, which is imported by pricing.json assert conf['exit_pricing']['price_side'] == "other" + assert len(conf['config_files']) == 4 + assert 'testconfig.json' in conf['config_files'][0] + assert 'test_pricing_conf.json' in conf['config_files'][1] + assert 'test_base_config.json' in conf['config_files'][2] + assert 'test_pricing2_conf.json' in conf['config_files'][3] + files = testdatadir / "testconfigs/recursive.json" with pytest.raises(OperationalException, match="Config loop detected."): load_from_files([files]) diff --git a/tests/testdata/testconfigs/base_config.json b/tests/testdata/testconfigs/test_base_config.json similarity index 100% rename from tests/testdata/testconfigs/base_config.json rename to tests/testdata/testconfigs/test_base_config.json diff --git a/tests/testdata/testconfigs/pricing2.json b/tests/testdata/testconfigs/test_pricing2_conf.json similarity index 100% rename from tests/testdata/testconfigs/pricing2.json rename to tests/testdata/testconfigs/test_pricing2_conf.json diff --git a/tests/testdata/testconfigs/pricing.json b/tests/testdata/testconfigs/test_pricing_conf.json similarity index 92% rename from tests/testdata/testconfigs/pricing.json rename to tests/testdata/testconfigs/test_pricing_conf.json index d8868443f..fbdaede74 100644 --- a/tests/testdata/testconfigs/pricing.json +++ b/tests/testdata/testconfigs/test_pricing_conf.json @@ -16,6 +16,6 @@ "price_last_balance": 0.0 }, "files": [ - "pricing2.json" + "./test_pricing2_conf.json" ] } diff --git a/tests/testdata/testconfigs/testconfig.json b/tests/testdata/testconfigs/testconfig.json index 557926097..96b3b6db8 100644 --- a/tests/testdata/testconfigs/testconfig.json +++ b/tests/testdata/testconfigs/testconfig.json @@ -1,6 +1,6 @@ { "files": [ - "base_config.json", - "pricing.json" + "test_base_config.json", + "test_pricing_conf.json" ] } From 238ff6c9fe7fa5681730f42201810ed57bae9c2d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Apr 2022 17:30:23 +0200 Subject: [PATCH 07/20] Use better naming --- config_examples/config_full.example.json | 2 +- freqtrade/configuration/load_config.py | 4 ++-- tests/testdata/testconfigs/recursive.json | 2 +- tests/testdata/testconfigs/test_pricing_conf.json | 2 +- tests/testdata/testconfigs/testconfig.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index b41acb726..193cc30bc 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -182,7 +182,7 @@ "disable_dataframe_checks": false, "strategy": "SampleStrategy", "strategy_path": "user_data/strategies/", - "files": [], + "add_config_files": [], "dataformat_ohlcv": "json", "dataformat_trades": "jsongz" } diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 5a86ab24a..c6a81d384 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -99,8 +99,8 @@ def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> file = base_path / file config_tmp = load_config_file(str(file)) - if 'files' in config_tmp: - config_sub = load_from_files(config_tmp['files'], file.resolve().parent, level + 1) + if 'add_config_files' in config_tmp: + config_sub = load_from_files(config_tmp['add_config_files'], file.resolve().parent, level + 1) files_loaded.extend(config_sub.get('config_files', [])) deep_merge_dicts(config_sub, config_tmp) diff --git a/tests/testdata/testconfigs/recursive.json b/tests/testdata/testconfigs/recursive.json index 28d8ce05a..33ab12008 100644 --- a/tests/testdata/testconfigs/recursive.json +++ b/tests/testdata/testconfigs/recursive.json @@ -1,6 +1,6 @@ { // This file fails as it's loading itself over and over - "files": [ + "add_config_files": [ "./recursive.json" ] } diff --git a/tests/testdata/testconfigs/test_pricing_conf.json b/tests/testdata/testconfigs/test_pricing_conf.json index fbdaede74..59516d65e 100644 --- a/tests/testdata/testconfigs/test_pricing_conf.json +++ b/tests/testdata/testconfigs/test_pricing_conf.json @@ -15,7 +15,7 @@ "order_book_top": 1, "price_last_balance": 0.0 }, - "files": [ + "add_config_files": [ "./test_pricing2_conf.json" ] } diff --git a/tests/testdata/testconfigs/testconfig.json b/tests/testdata/testconfigs/testconfig.json index 96b3b6db8..87ed6daef 100644 --- a/tests/testdata/testconfigs/testconfig.json +++ b/tests/testdata/testconfigs/testconfig.json @@ -1,5 +1,5 @@ { - "files": [ + "add_config_files": [ "test_base_config.json", "test_pricing_conf.json" ] From b8556498efb22529438594cf8fb68cbcf899127c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Apr 2022 17:46:53 +0200 Subject: [PATCH 08/20] Fix pre-commit to actually work --- .pre-commit-config.yaml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28eb0ae38..31af5b7c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,30 +5,17 @@ repos: rev: '4.0.1' hooks: - id: flake8 - stages: [push] + # stages: [push] - repo: https://github.com/pre-commit/mirrors-mypy rev: 'v0.942' hooks: - id: mypy - stages: [push] + # stages: [push] - repo: https://github.com/pycqa/isort rev: '5.10.1' hooks: - id: isort name: isort (python) - stages: [push] - -# https://github.com/pre-commit/pre-commit/issues/761#issuecomment-394167542 -- repo: local - hooks: - - id: pytest - name: pytest - entry: venv/bin/pytest - language: script - pass_filenames: false - # alternatively you could `types: [python]` so it only runs when python files change - # though tests might be invalidated if you were to say change a data file - always_run: true - stages: [push] \ No newline at end of file + # stages: [push] From ecb0e43c2a29d4dc9912eaf49b97dbb7214796a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Apr 2022 17:50:32 +0200 Subject: [PATCH 09/20] Improve pre-commit docs --- CONTRIBUTING.md | 2 -- docs/developer.md | 3 ++- requirements-dev.txt | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae9c5d81e..b4e0bc024 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,8 +20,6 @@ Best start by reading the [documentation](https://www.freqtrade.io/) to get a fe ## Before sending the PR -Do the following if you disabled pre-commit hook when commiting. - ### 1. Run unit tests All unit tests must pass. If a unit test is broken, change your code to diff --git a/docs/developer.md b/docs/developer.md index 18465238f..0434ecb3e 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -26,7 +26,8 @@ Alternatively (e.g. if your system is not supported by the setup.sh script), fol This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`. -Then install the git hook scripts by running `pre-commit install` +Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing. +This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine. Before opening a pull request, please familiarize yourself with our [Contributing Guidelines](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md). diff --git a/requirements-dev.txt b/requirements-dev.txt index 5266ad003..c510b107d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,6 @@ -r requirements-plot.txt -r requirements-hyperopt.txt - coveralls==3.3.1 flake8==4.0.1 flake8-tidy-imports==4.6.0 @@ -28,4 +27,4 @@ types-requests==2.27.15 types-tabulate==0.8.6 # Extensions to datetime library -types-python-dateutil==2.8.10 \ No newline at end of file +types-python-dateutil==2.8.10 From 16e64ddf97e82ef1613c61b3607074e2380dcbd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Apr 2022 17:36:50 +0200 Subject: [PATCH 10/20] Update docs for multi-config loading --- docs/configuration.md | 24 ++++++++++++++++++++++-- freqtrade/configuration/load_config.py | 3 ++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 49a59c070..0c89bbbdd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -53,14 +53,33 @@ FREQTRADE__EXCHANGE__SECRET= Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. +You can specify additional configuration files in `add_config_files`. Files specified in this parameter will be loaded and merged with the initial config file. The files are resolved relative to the initial configuration file. +This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands. + !!! Tip "Use multiple configuration files to keep secrets secret" You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself. + ``` json title="user_data/config.json" + "add_config_files": [ + "config-private.json" + ] + ``` + + ``` bash + freqtrade trade --config user_data/config.json <...> + ``` + + The 2nd file should only specify what you intend to override. + If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`). + + For one-off commands, you can also use the below syntax by specifying multiple "--config" parameters. + ``` bash freqtrade trade --config user_data/config.json --config user_data/config-private.json <...> ``` - The 2nd file should only specify what you intend to override. - If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`). + + This is equivalent to the example above - but `config-private.json` is specified as cli argument. + ## Configuration parameters @@ -175,6 +194,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String | `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String +| `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.
*Defaults to `[]`*.
**Datatype:** List of strings | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data.
*Defaults to `json`*.
**Datatype:** String | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position).
[Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index c6a81d384..32c2ae0d9 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -100,7 +100,8 @@ def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> config_tmp = load_config_file(str(file)) if 'add_config_files' in config_tmp: - config_sub = load_from_files(config_tmp['add_config_files'], file.resolve().parent, level + 1) + config_sub = load_from_files( + config_tmp['add_config_files'], file.resolve().parent, level + 1) files_loaded.extend(config_sub.get('config_files', [])) deep_merge_dicts(config_sub, config_tmp) From ebcb530d4f0303ca0cb164296b92cd8ea269d032 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 09:56:12 +0200 Subject: [PATCH 11/20] Log if no stake-amount is left for trade --- freqtrade/freqtradebot.py | 1 + freqtrade/rpc/rpc.py | 2 +- tests/rpc/test_rpc.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6e1a0b208..dc2e21ed6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -598,6 +598,7 @@ class FreqtradeBot(LoggingMixin): pair, price, stake_amount, trade_side, enter_tag, trade) if not stake_amount: + logger.info(f"No stake amount to enter a trade for {pair}.") return False if pos_adjust: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 258754b90..8f3d57cf6 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -791,7 +791,7 @@ class RPC: trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade else: - return None + raise RPCException(f'Failed to enter position for {pair}.') def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]: """ diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 8bdb81072..e421b6fe5 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1230,8 +1230,8 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open patch_get_signal(freqtradebot) rpc = RPC(freqtradebot) pair = 'TKN/BTC' - trade = rpc._rpc_force_entry(pair, None) - assert trade is None + with pytest.raises(RPCException, match=r"Failed to enter position for TKN/BTC."): + trade = rpc._rpc_force_entry(pair, None) def test_rpc_force_entry_stopped(mocker, default_conf) -> None: From e6060511028c97036e49a4bb6829efceec262332 Mon Sep 17 00:00:00 2001 From: RafaelDorigo Date: Sat, 9 Apr 2022 11:53:47 +0200 Subject: [PATCH 12/20] Fixed setup.sh --- setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 2c3a6710b..5cde1a589 100755 --- a/setup.sh +++ b/setup.sh @@ -89,12 +89,13 @@ function updateenv() { fi echo "pip install completed" echo - if [[ $dev =~ ^[Yy]$ ]] then + if [[ $dev =~ ^[Yy]$ ]]; then ${PYTHON} -m pre-commit install if [ $? -ne 0 ]; then echo "Failed installing pre-commit" exit 1 fi + fi } # Install tab lib From 8e98a2ff9f4fabf81bf5a4f4e1f772f5c4a091ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 16:42:18 +0200 Subject: [PATCH 13/20] api - provide assset_currency via API --- freqtrade/exchange/exchange.py | 8 ++------ freqtrade/freqtradebot.py | 3 +++ freqtrade/optimize/backtesting.py | 3 +++ freqtrade/persistence/migrations.py | 14 ++++++++----- freqtrade/persistence/models.py | 26 +++++++++++++++++++++++++ freqtrade/rpc/api_server/api_schemas.py | 2 ++ freqtrade/rpc/rpc.py | 1 - freqtrade/rpc/telegram.py | 2 +- tests/rpc/test_rpc.py | 6 ++++-- tests/rpc/test_rpc_apiserver.py | 8 ++++++-- tests/rpc/test_rpc_telegram.py | 3 ++- tests/test_persistence.py | 4 ++++ 12 files changed, 62 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 609dbb83e..82505759a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -341,15 +341,11 @@ class Exchange: return sorted(set([x['quote'] for _, x in markets.items()])) def get_pair_quote_currency(self, pair: str) -> str: - """ - Return a pair's quote currency - """ + """ Return a pair's quote currency (base/quote:settlement) """ return self.markets.get(pair, {}).get('quote', '') def get_pair_base_currency(self, pair: str) -> str: - """ - Return a pair's base currency - """ + """ Return a pair's base currency (base/quote:settlement) """ return self.markets.get(pair, {}).get('base', '') def market_is_future(self, market: Dict[str, Any]) -> bool: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index dc2e21ed6..57d7cac3c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -676,6 +676,7 @@ class FreqtradeBot(LoggingMixin): # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') + base_currency = self.exchange.get_pair_base_currency(pair) open_date = datetime.now(timezone.utc) funding_fees = self.exchange.get_funding_fees( pair=pair, amount=amount, is_short=is_short, open_date=open_date) @@ -683,6 +684,8 @@ class FreqtradeBot(LoggingMixin): if trade is None: trade = Trade( pair=pair, + base_currency=base_currency, + stake_currency=self.config['stake_currency'], stake_amount=stake_amount, amount=amount, is_open=True, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4bb10d39c..438337669 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -726,6 +726,7 @@ class Backtesting: if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): self.order_id_counter += 1 + base_currency = self.exchange.get_pair_base_currency(pair) amount = round((stake_amount / propose_rate) * leverage, 8) is_short = (direction == 'short') # Necessary for Margin trading. Disabled until support is enabled. @@ -738,6 +739,8 @@ class Backtesting: id=self.trade_id_counter, open_order_id=self.order_id_counter, pair=pair, + base_currency=base_currency, + stake_currency=self.config['stake_currency'], open_rate=propose_rate, open_rate_requested=propose_rate, open_date=current_time, diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 9521eae69..996af7341 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -58,6 +58,8 @@ def migrate_trades_and_orders_table( decl_base, inspector, engine, trade_back_name: str, cols: List, order_back_name: str, cols_order: List): + base_currency = get_column_def(cols, 'base_currency', 'null') + stake_currency = get_column_def(cols, 'stake_currency', 'null') fee_open = get_column_def(cols, 'fee_open', 'fee') fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') @@ -130,7 +132,7 @@ def migrate_trades_and_orders_table( # Copy data back - following the correct schema with engine.begin() as connection: connection.execute(text(f"""insert into trades - (id, exchange, pair, is_open, + (id, exchange, pair, base_currency, stake_currency, is_open, fee_open, fee_open_cost, fee_open_currency, fee_close, fee_close_cost, fee_close_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, @@ -142,7 +144,8 @@ def migrate_trades_and_orders_table( trading_mode, leverage, liquidation_price, is_short, interest_rate, funding_fees ) - select id, lower(exchange), pair, + select id, lower(exchange), pair, {base_currency} base_currency, + {stake_currency} stake_currency, is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, {fee_open_currency} fee_open_currency, {fee_close} fee_close, {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, @@ -230,7 +233,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: """ inspector = inspect(engine) - cols = inspector.get_columns('trades') + cols_trades = inspector.get_columns('trades') cols_orders = inspector.get_columns('orders') tabs = get_table_names_for_table(inspector, 'trades') table_back_name = get_backup_name(tabs, 'trades_bak') @@ -241,11 +244,12 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # Migrates both trades and orders table! # if ('orders' not in previous_tables # or not has_column(cols_orders, 'leverage')): - if not has_column(cols, 'exit_order_status'): + if not has_column(cols_trades, 'base_currency'): logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") migrate_trades_and_orders_table( - decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_orders) + decl_base, inspector, engine, table_back_name, cols_trades, + order_table_bak_name, cols_orders) if 'orders' not in previous_tables and 'trades' in previous_tables: logger.info('Moving open orders to Orders table.') diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 3cd9cbd67..05de39caf 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -279,6 +279,8 @@ class LocalTrade(): exchange: str = '' pair: str = '' + base_currency: str = '' + stake_currency: str = '' is_open: bool = True fee_open: float = 0.0 fee_open_cost: Optional[float] = None @@ -397,6 +399,26 @@ class LocalTrade(): else: return "long" + @property + def safe_base_currency(self) -> str: + """ + Compatibility layer for asset - which can be empty for old trades. + """ + try: + return self.base_currency or self.pair.split('/')[0] + except IndexError: + return '' + + @property + def safe_quote_currency(self) -> str: + """ + Compatibility layer for asset - which can be empty for old trades. + """ + try: + return self.stake_currency or self.pair.split('/')[1].split(':')[0] + except IndexError: + return '' + def __init__(self, **kwargs): for key in kwargs: setattr(self, key, kwargs[key]) @@ -423,6 +445,8 @@ class LocalTrade(): return { 'trade_id': self.id, 'pair': self.pair, + 'base_currency': self.safe_base_currency, + 'quote_currency': self.safe_quote_currency, 'is_open': self.is_open, 'exchange': self.exchange, 'amount': round(self.amount, 8), @@ -1051,6 +1075,8 @@ class Trade(_DECL_BASE, LocalTrade): exchange = Column(String(25), nullable=False) pair = Column(String(25), nullable=False, index=True) + base_currency = Column(String(25), nullable=True) + stake_currency = Column(String(25), nullable=True) is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) fee_open_cost = Column(Float, nullable=True) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 03049e0f4..ae797edad 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -203,6 +203,8 @@ class OrderSchema(BaseModel): class TradeSchema(BaseModel): trade_id: int pair: str + base_currency: str + quote_currency: str is_open: bool is_short: bool exchange: str diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8f3d57cf6..be0e8e797 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -197,7 +197,6 @@ class RPC: trade_dict = trade.to_json() trade_dict.update(dict( - base_currency=self._freqtrade.config['stake_currency'], close_profit=trade.close_profit if trade.close_profit is not None else None, current_rate=current_rate, current_profit=current_profit, # Deprecated diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e13e46395..5f6a8b147 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -508,7 +508,7 @@ class Telegram(RPCHandler): lines.append("*Open Order:* `{open_order}`") lines_detail = self._prepare_entry_details( - r['orders'], r['base_currency'], r['is_open']) + r['orders'], r['quote_currency'], r['is_open']) lines.extend(lines_detail if lines_detail else "") # Filter empty lines using list-comprehension diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index e421b6fe5..f4a2f6099 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -52,7 +52,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: assert results[0] == { 'trade_id': 1, 'pair': 'ETH/BTC', - 'base_currency': 'BTC', + 'base_currency': 'ETH', + 'quote_currency': 'BTC', 'open_date': ANY, 'open_timestamp': ANY, 'is_open': ANY, @@ -135,7 +136,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: assert results[0] == { 'trade_id': 1, 'pair': 'ETH/BTC', - 'base_currency': 'BTC', + 'base_currency': 'ETH', + 'quote_currency': 'BTC', 'open_date': ANY, 'open_timestamp': ANY, 'is_open': ANY, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 3e1710c8e..76cef0df0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -931,6 +931,8 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'open_order': None, 'open_rate': 0.123, 'pair': 'ETH/BTC', + 'base_currency': 'ETH', + 'quote_currency': 'BTC', 'stake_amount': 0.001, 'stop_loss_abs': ANY, 'stop_loss_pct': ANY, @@ -1097,7 +1099,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): # Test creating trade fbuy_mock = MagicMock(return_value=Trade( - pair='ETH/ETH', + pair='ETH/BTC', amount=1, amount_requested=1, exchange='binance', @@ -1130,7 +1132,9 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): 'open_date': ANY, 'open_timestamp': ANY, 'open_rate': 0.245441, - 'pair': 'ETH/ETH', + 'pair': 'ETH/BTC', + 'base_currency': 'ETH', + 'quote_currency': 'BTC', 'stake_amount': 1, 'stop_loss_abs': None, 'stop_loss_pct': None, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f104e7153..da853799b 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -184,7 +184,8 @@ def test_telegram_status(default_conf, update, mocker) -> None: _rpc_trade_status=MagicMock(return_value=[{ 'trade_id': 1, 'pair': 'ETH/BTC', - 'base_currency': 'BTC', + 'base_currency': 'ETH', + 'quote_currency': 'BTC', 'open_date': arrow.utcnow(), 'close_date': None, 'open_rate': 1.099e-05, diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8ba8764e0..d30d33d3b 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1561,6 +1561,8 @@ def test_to_json(fee): assert result == {'trade_id': None, 'pair': 'ADA/USDT', + 'base_currency': 'ADA', + 'quote_currency': 'USDT', 'is_open': None, 'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"), 'open_timestamp': int(trade.open_date.timestamp() * 1000), @@ -1637,6 +1639,8 @@ def test_to_json(fee): assert result == {'trade_id': None, 'pair': 'XRP/BTC', + 'base_currency': 'XRP', + 'quote_currency': 'BTC', 'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"), 'open_timestamp': int(trade.open_date.timestamp() * 1000), 'close_date': trade.close_date.strftime("%Y-%m-%d %H:%M:%S"), From ef18d0916123bdfe584ccda8c76792645865b692 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 16:50:38 +0200 Subject: [PATCH 14/20] Call custom_exit also when the trade is not in profit and exit_profit_only is set. --- docs/strategy-callbacks.md | 3 ++- docs/strategy_migration.md | 3 +++ freqtrade/strategy/interface.py | 11 ++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 678899cc6..302ffd5fd 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -91,7 +91,8 @@ For example you could implement a 1:2 risk-reward ROI with `custom_exit()`. Using custom_exit() signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. !!! Note - Returning a (none-empty) `string` or `True` from this method is equal to setting exit signal on a candle at specified time. This method is not called when exit signal is set already, or if exit signals are disabled (`use_exit_signal=False` or `exit_profit_only=True` while profit is below `exit_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters. + Returning a (none-empty) `string` or `True` from this method is equal to setting exit signal on a candle at specified time. This method is not called when exit signal is set already, or if exit signals are disabled (`use_exit_signal=False`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters. + `custom_exit()` will ignore `exit_profit_only`, and will always be called unless `use_exit_signal=False` or if there is an enter signal. An example of how we can use different indicators depending on the current profit and also exit trades that were open longer than one day: diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index eb1729ba7..1fe1f0953 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -145,6 +145,9 @@ Please refer to the [Strategy documentation](strategy-customization.md#exit-sign ### `custom_sell` +`custom_sell` has been renamed to `custom_exit`. +It's now also being called for every iteration, independent of current profit and `exit_profit_only` settings. + ``` python hl_lines="2" class AwesomeStrategy(IStrategy): def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index b0ed6e72d..1c53b2e3e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -881,10 +881,7 @@ class IStrategy(ABC, HyperStrategyMixin): current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) - if (self.exit_profit_only and current_profit <= self.exit_profit_offset): - # exit_profit_only and profit doesn't reach the offset - ignore sell signal - pass - elif self.use_exit_signal and not enter: + if self.use_exit_signal and not enter: if exit_: exit_signal = ExitType.EXIT_SIGNAL else: @@ -902,7 +899,11 @@ class IStrategy(ABC, HyperStrategyMixin): custom_reason = custom_reason[:CUSTOM_EXIT_MAX_LENGTH] else: custom_reason = None - if exit_signal in (ExitType.CUSTOM_EXIT, ExitType.EXIT_SIGNAL): + if ( + exit_signal == ExitType.CUSTOM_EXIT + or (exit_signal == ExitType.EXIT_SIGNAL + and (not self.exit_profit_only or current_profit > self.exit_profit_offset)) + ): logger.debug(f"{trade.pair} - Sell signal received. " f"exit_type=ExitType.{exit_signal.name}" + (f", custom_reason={custom_reason}" if custom_reason else "")) From 139b65835c54c41e3ca01b3a6ff61f235c9e0748 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 17:09:04 +0200 Subject: [PATCH 15/20] Only show long/short signals on telegram for non-spot markets --- freqtrade/rpc/rpc.py | 9 +++++++-- tests/rpc/test_rpc_telegram.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8f3d57cf6..e151a1e07 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -223,6 +223,7 @@ class RPC: def _rpc_status_table(self, stake_currency: str, fiat_display_currency: str) -> Tuple[List, List, float]: trades: List[Trade] = Trade.get_open_trades() + nonspot = self._config.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT if not trades: raise RPCException('no active trade') else: @@ -237,7 +238,7 @@ class RPC: current_rate = NAN trade_profit = trade.calc_profit(current_rate) profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' - direction_str = 'S' if trade.is_short else 'L' + direction_str = ('S' if trade.is_short else 'L') if nonspot else '' if self._fiat_converter: fiat_profit = self._fiat_converter.convert_amount( trade_profit, @@ -267,7 +268,11 @@ class RPC: if self._fiat_converter: profitcol += " (" + fiat_display_currency + ")" - columns = ['ID L/S', 'Pair', 'Since', profitcol] + columns = [ + 'ID L/S' if nonspot else 'ID', + 'Pair', + 'Since', + profitcol] if self._config.get('position_adjustment_enable', False): columns.append('# Entries') return trades_list, columns, fiat_profit_sum diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f104e7153..6d422ce11 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -398,8 +398,8 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: fields = re.sub('[ ]+', ' ', line[2].strip()).split(' ') assert int(fields[0]) == 1 - assert 'L' in fields[1] - assert 'ETH/BTC' in fields[2] + # assert 'L' in fields[1] + assert 'ETH/BTC' in fields[1] assert msg_mock.call_count == 1 From 114591048c9daa227b31b7990fab6be97e0093af Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 17:17:49 +0200 Subject: [PATCH 16/20] Always call custom_sell - also when there's a new enter signal --- docs/strategy-callbacks.md | 4 ++-- freqtrade/strategy/interface.py | 4 ++-- tests/test_freqtradebot.py | 8 +++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 302ffd5fd..bd32f41c3 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -88,11 +88,11 @@ Allows to define custom exit signals, indicating that specified position should For example you could implement a 1:2 risk-reward ROI with `custom_exit()`. -Using custom_exit() signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. +Using `custom_exit()` signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. !!! Note Returning a (none-empty) `string` or `True` from this method is equal to setting exit signal on a candle at specified time. This method is not called when exit signal is set already, or if exit signals are disabled (`use_exit_signal=False`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters. - `custom_exit()` will ignore `exit_profit_only`, and will always be called unless `use_exit_signal=False` or if there is an enter signal. + `custom_exit()` will ignore `exit_profit_only`, and will always be called unless `use_exit_signal=False`, even if there is a new enter signal. An example of how we can use different indicators depending on the current profit and also exit trades that were open longer than one day: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1c53b2e3e..ebaa6568f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -881,8 +881,8 @@ class IStrategy(ABC, HyperStrategyMixin): current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) - if self.use_exit_signal and not enter: - if exit_: + if self.use_exit_signal: + if exit_ and not enter: exit_signal = ExitType.EXIT_SIGNAL else: trade_type = "exit_short" if trade.is_short else "sell" diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e4066413e..3737c7c05 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3663,6 +3663,7 @@ def test_exit_profit_only( }) freqtrade = FreqtradeBot(default_conf_usdt) patch_get_signal(freqtrade, enter_short=is_short, enter_long=not is_short) + freqtrade.strategy.custom_exit = MagicMock(return_value=None) if exit_type == ExitType.EXIT_SIGNAL.value: freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) else: @@ -3671,10 +3672,15 @@ def test_exit_profit_only( freqtrade.enter_positions() trade = Trade.query.first() - trade.is_short = is_short + assert trade.is_short == is_short oobj = Order.parse_from_ccxt_object(limit_order[eside], limit_order[eside]['symbol'], eside) trade.update_trade(oobj) freqtrade.wallets.update() + if profit_only: + assert freqtrade.handle_trade(trade) is False + # Custom-exit is called + freqtrade.strategy.custom_exit.call_count == 1 + patch_get_signal(freqtrade, enter_long=False, exit_short=is_short, exit_long=not is_short) assert freqtrade.handle_trade(trade) is handle_first From 9f9219675ff06a02fde10d7c18cad08957cca46b Mon Sep 17 00:00:00 2001 From: RafaelDorigo Date: Sat, 9 Apr 2022 19:58:58 +0200 Subject: [PATCH 17/20] Update strategy_migration.md --- docs/strategy_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index eb1729ba7..c53e10e6e 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -9,7 +9,7 @@ You can use the quick summary as checklist. Please refer to the detailed section ## Quick summary / migration checklist -Note : `force_exit`, `force_enter`, `emergency_exit` are changed to `force_exit`, `force_enter`, `emergency_exit` respectively. +Note : `forcesell`, `forcebuy`, `emergencysell` are changed to `force_exit`, `force_enter`, `emergency_exit` respectively. * Strategy methods: * [`populate_buy_trend()` -> `populate_entry_trend()`](#populate_buy_trend) From 09b41a6f8de114de3cb7378f487d6df2cd771da4 Mon Sep 17 00:00:00 2001 From: zolbayars Date: Sun, 10 Apr 2022 10:39:48 +0800 Subject: [PATCH 18/20] Docs: update plotting doc to show strategy option is mandatory --- docs/plotting.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index df988c578..6ae0c3f11 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -96,7 +96,7 @@ Strategy arguments: Example: ``` bash -freqtrade plot-dataframe -p BTC/ETH +freqtrade plot-dataframe -p BTC/ETH --strategy AwesomeStrategy ``` The `-p/--pairs` argument can be used to specify pairs you would like to plot. @@ -107,9 +107,6 @@ The `-p/--pairs` argument can be used to specify pairs you would like to plot. Specify custom indicators. Use `--indicators1` for the main plot and `--indicators2` for the subplot below (if values are in a different range than prices). -!!! Tip - You will almost certainly want to specify a custom strategy! This can be done by adding `-s Classname` / `--strategy ClassName` to the command. - ``` bash freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --indicators1 sma ema --indicators2 macd ``` From 850760bc00b68848cf929520a752d2767de20cfc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 17:37:04 +0200 Subject: [PATCH 19/20] Remove migration from very old database (database without Orders table) --- freqtrade/persistence/migrations.py | 28 +++----- tests/test_persistence.py | 100 +++++++++++++++++++++------- 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 996af7341..a28683e04 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -3,6 +3,8 @@ from typing import List from sqlalchemy import inspect, text +from freqtrade.exceptions import OperationalException + logger = logging.getLogger(__name__) @@ -176,23 +178,6 @@ def migrate_trades_and_orders_table( set_sequence_ids(engine, order_id, trade_id) -def migrate_open_orders_to_trades(engine): - with engine.begin() as connection: - connection.execute(text(""" - insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) - select id ft_trade_id, pair ft_pair, open_order_id, - case when close_rate_requested is null then 'buy' - else 'sell' end ft_order_side, 1 ft_is_open - from trades - where open_order_id is not null - union all - select id ft_trade_id, pair ft_pair, stoploss_order_id order_id, - 'stoploss' ft_order_side, 1 ft_is_open - from trades - where stoploss_order_id is not null - """)) - - def drop_orders_table(engine, table_back_name: str): # Drop and recreate orders table as backup # This drops foreign keys, too. @@ -210,7 +195,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): # sqlite does not support literals for booleans with engine.begin() as connection: connection.execute(text(f""" - insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date, order_filled_date, order_update_date, ft_fee_base) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, @@ -252,6 +237,9 @@ def check_migrate(engine, decl_base, previous_tables) -> None: order_table_bak_name, cols_orders) if 'orders' not in previous_tables and 'trades' in previous_tables: - logger.info('Moving open orders to Orders table.') - migrate_open_orders_to_trades(engine) + raise OperationalException( + "Your database seems to be very old. " + "Please update to freqtrade 2022.3 to migrate this database or " + "start with a fresh database.") + set_sqlite_to_wal(engine) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index d30d33d3b..ecac561f8 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1209,6 +1209,27 @@ def test_migrate_new(mocker, default_conf, fee, caplog): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" + create_table_order = """CREATE TABLE orders ( + id INTEGER NOT NULL, + ft_trade_id INTEGER, + ft_order_side VARCHAR(25) NOT NULL, + ft_pair VARCHAR(25) NOT NULL, + ft_is_open BOOLEAN NOT NULL, + order_id VARCHAR(255) NOT NULL, + status VARCHAR(255), + symbol VARCHAR(25), + order_type VARCHAR(50), + side VARCHAR(25), + price FLOAT, + amount FLOAT, + filled FLOAT, + remaining FLOAT, + cost FLOAT, + order_date DATETIME, + order_filled_date DATETIME, + order_update_date DATETIME, + PRIMARY KEY (id) + );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date, stop_loss, initial_stop_loss, max_rate, ticker_interval, @@ -1222,15 +1243,66 @@ def test_migrate_new(mocker, default_conf, fee, caplog): stake=default_conf.get("stake_amount"), amount=amount ) + insert_orders = f""" + insert into orders ( + ft_trade_id, + ft_order_side, + ft_pair, + ft_is_open, + order_id, + status, + symbol, + order_type, + side, + price, + amount, + filled, + remaining, + cost) + values ( + 1, + 'buy', + 'ETC/BTC', + 0, + 'buy_order', + 'closed', + 'ETC/BTC', + 'limit', + 'buy', + 0.00258580, + {amount}, + {amount}, + 0, + {amount * 0.00258580} + ), + ( + 1, + 'stoploss', + 'ETC/BTC', + 0, + 'stop_order_id222', + 'closed', + 'ETC/BTC', + 'limit', + 'sell', + 0.00258580, + {amount}, + {amount}, + 0, + {amount * 0.00258580} + ) + """ engine = create_engine('sqlite://') mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format with engine.begin() as connection: connection.execute(text(create_table_old)) + connection.execute(text(create_table_order)) connection.execute(text("create index ix_trades_is_open on trades(is_open)")) connection.execute(text("create index ix_trades_pair on trades(pair)")) connection.execute(text(insert_table_old)) + connection.execute(text(insert_orders)) # fake previous backup connection.execute(text("create table trades_bak as select * from trades")) @@ -1267,8 +1339,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.open_trade_value == trade._calc_open_trade_value() assert trade.close_profit_abs is None - assert log_has("Moving open orders to Orders table.", caplog) - orders = Order.query.all() + orders = trade.orders assert len(orders) == 2 assert orders[0].order_id == 'buy_order' assert orders[0].ft_order_side == 'buy' @@ -1277,7 +1348,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert orders[1].ft_order_side == 'stoploss' -def test_migrate_mid_state(mocker, default_conf, fee, caplog): +def test_migrate_too_old(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ @@ -1301,6 +1372,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) VALUES ('binance', 'ETC/BTC', 1, {fee}, {fee}, @@ -1319,26 +1391,8 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): connection.execute(text(insert_table_old)) # Run init to test migration - init_db(default_conf['db_url'], default_conf['dry_run']) - - assert len(Trade.query.filter(Trade.id == 1).all()) == 1 - trade = Trade.query.filter(Trade.id == 1).first() - assert trade.fee_open == fee.return_value - assert trade.fee_close == fee.return_value - assert trade.open_rate_requested is None - assert trade.close_rate_requested is None - assert trade.is_open == 1 - assert trade.amount == amount - assert trade.stake_amount == default_conf.get("stake_amount") - assert trade.pair == "ETC/BTC" - assert trade.exchange == "binance" - assert trade.max_rate == 0.0 - assert trade.stop_loss == 0.0 - assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_value == trade._calc_open_trade_value() - assert log_has("trying trades_bak0", caplog) - assert log_has("Running database migration for trades - backup: trades_bak0, orders_bak0", - caplog) + with pytest.raises(OperationalException, match=r'Your database seems to be very old'): + init_db(default_conf['db_url'], default_conf['dry_run']) def test_migrate_get_last_sequence_ids(): From ffff45e76bb1e647f17f5f8711664ba4985f2de2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 10 Apr 2022 08:37:35 +0200 Subject: [PATCH 20/20] simplify exit message --- freqtrade/strategy/interface.py | 3 +-- tests/strategy/test_interface.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index ebaa6568f..ba2eb9636 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -885,7 +885,6 @@ class IStrategy(ABC, HyperStrategyMixin): if exit_ and not enter: exit_signal = ExitType.EXIT_SIGNAL else: - trade_type = "exit_short" if trade.is_short else "sell" custom_reason = strategy_safe_wrapper(self.custom_exit, default_retval=False)( pair=trade.pair, trade=trade, current_time=current_time, current_rate=current_rate, current_profit=current_profit) @@ -893,7 +892,7 @@ class IStrategy(ABC, HyperStrategyMixin): exit_signal = ExitType.CUSTOM_EXIT if isinstance(custom_reason, str): if len(custom_reason) > CUSTOM_EXIT_MAX_LENGTH: - logger.warning(f'Custom {trade_type} reason returned from ' + logger.warning(f'Custom exit reason returned from ' f'custom_exit is too long and was trimmed' f'to {CUSTOM_EXIT_MAX_LENGTH} characters.') custom_reason = custom_reason[:CUSTOM_EXIT_MAX_LENGTH] diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 44a17ac02..a86d69135 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -523,7 +523,7 @@ def test_custom_exit(default_conf, fee, caplog) -> None: assert res.exit_type == ExitType.CUSTOM_EXIT assert res.exit_flag is True assert res.exit_reason == 'h' * 64 - assert log_has_re('Custom sell reason returned from custom_exit is too long.*', caplog) + assert log_has_re('Custom exit reason returned from custom_exit is too long.*', caplog) @pytest.mark.parametrize('side', TRADE_SIDES)