From df033d92ef4743093fec3b85f5d4b8f9e09d7606 Mon Sep 17 00:00:00 2001 From: Simon Ebner Date: Sat, 23 Oct 2021 09:20:00 +0200 Subject: [PATCH 01/11] Improve performance of decimalspace.py decimalspace.py is heavily used in the hyperoptimization. The following benchmark code runs an optimization which is taken from optimizing a real strategy (wtc). The optimized version takes on my machine approx. 11/12s compared to the original 32s. Results are equivalent in both cases. ``` import freqtrade.optimize.space import numpy as np import skopt import timeit def init(): Decimal = freqtrade.optimize.space.decimalspace.SKDecimal Integer = skopt.space.space.Integer dimensions = [Decimal(low=-1.0, high=1.0, decimals=4, prior='uniform', transform='identity')] * 20 return skopt.Optimizer( dimensions, base_estimator="ET", acq_optimizer="auto", n_initial_points=5, acq_optimizer_kwargs={'n_jobs': 96}, random_state=0, model_queue_size=10, ) def test(): opt = init() actual = opt.ask(n_points=2) expected = [[ 0.7515, -0.4723, -0.6941, -0.7988, 0.0448, 0.8605, -0.108, 0.5399, 0.763, -0.2948, 0.8345, -0.7683, 0.7077, -0.2478, -0.333, 0.8575, 0.6108, 0.4514, 0.5982, 0.3506 ], [ 0.5563, 0.7386, -0.6407, 0.9073, -0.5211, -0.8167, -0.3771, -0.0318, 0.2861, 0.1176, 0.0943, -0.6077, -0.9317, -0.5372, -0.4934, -0.3637, -0.8035, -0.8627, -0.5399, 0.6036 ]] absdiff = np.max(np.abs(np.asarray(expected) - np.asarray(actual))) assert absdiff < 1e-5 def time(): opt = init() print('dt', timeit.timeit("opt.ask(n_points=20)", globals=locals())) if __name__ == "__main__": test() time() ``` --- freqtrade/optimize/space/decimalspace.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/space/decimalspace.py b/freqtrade/optimize/space/decimalspace.py index 643999cc1..220502e69 100644 --- a/freqtrade/optimize/space/decimalspace.py +++ b/freqtrade/optimize/space/decimalspace.py @@ -7,11 +7,15 @@ class SKDecimal(Integer): def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None, name=None, dtype=np.int64): self.decimals = decimals - _low = int(low * pow(10, self.decimals)) - _high = int(high * pow(10, self.decimals)) + + self.pow_dot_one = pow(0.1, self.decimals) + self.pow_ten = pow(10, self.decimals) + + _low = int(low * self.pow_ten) + _high = int(high * self.pow_ten) # trunc to precision to avoid points out of space - self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals) - self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals) + self.low_orig = round(_low * self.pow_dot_one, self.decimals) + self.high_orig = round(_high * self.pow_dot_one, self.decimals) super().__init__(_low, _high, prior, base, transform, name, dtype) @@ -25,9 +29,9 @@ class SKDecimal(Integer): return self.low_orig <= point <= self.high_orig def transform(self, Xt): - aa = [int(x * pow(10, self.decimals)) for x in Xt] - return super().transform(aa) + return super().transform([int(v * self.pow_ten) for v in Xt]) def inverse_transform(self, Xt): res = super().inverse_transform(Xt) - return [round(x * pow(0.1, self.decimals), self.decimals) for x in res] + # equivalent to [round(x * pow(0.1, self.decimals), self.decimals) for x in res] + return [int(v) / self.pow_ten for v in res] From f7926083ca9a1eaa16c401bd18bfce4a209a2d74 Mon Sep 17 00:00:00 2001 From: Simon Ebner Date: Sun, 24 Oct 2021 22:59:28 +0200 Subject: [PATCH 02/11] Clean up unclosed file handles Close all file handles that are left dangling to avoid warnings such as ``` ResourceWarning: unclosed file <_io.TextIOWrapper name='...' mode='r' encoding='UTF-8'> params = json_load(filename.open('r')) ``` --- freqtrade/optimize/hyperopt_tools.py | 10 +++++----- freqtrade/strategy/hyper.py | 3 ++- tests/optimize/test_hyperopt_tools.py | 3 ++- tests/strategy/test_strategy_loading.py | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index cfbc2757e..0b2efa5c2 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -1,4 +1,3 @@ - import io import logging from copy import deepcopy @@ -64,10 +63,11 @@ class HyperoptTools(): 'export_time': datetime.now(timezone.utc), } logger.info(f"Dumping parameters to {filename}") - rapidjson.dump(final_params, filename.open('w'), indent=2, - default=hyperopt_serializer, - number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN - ) + with filename.open('w') as f: + rapidjson.dump(final_params, f, indent=2, + default=hyperopt_serializer, + number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN + ) @staticmethod def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict): diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index dad282d7e..eaf41263a 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -381,7 +381,8 @@ class HyperStrategyMixin(object): if filename.is_file(): logger.info(f"Loading parameters from file {filename}") try: - params = json_load(filename.open('r')) + with filename.open('r') as f: + params = json_load(f) if params.get('strategy_name') != self.__class__.__name__: raise OperationalException('Invalid parameter file provided.') return params diff --git a/tests/optimize/test_hyperopt_tools.py b/tests/optimize/test_hyperopt_tools.py index 9c2b2e8fc..d9a52db39 100644 --- a/tests/optimize/test_hyperopt_tools.py +++ b/tests/optimize/test_hyperopt_tools.py @@ -209,7 +209,8 @@ def test_export_params(tmpdir): assert filename.is_file() - content = rapidjson.load(filename.open('r')) + with filename.open('r') as f: + content = rapidjson.load(f) assert content['strategy_name'] == 'StrategyTestV2' assert 'params' in content assert "buy" in content["params"] diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 3a30a824a..3590c3e01 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -62,8 +62,8 @@ def test_load_strategy(default_conf, result): def test_load_strategy_base64(result, caplog, default_conf): - with (Path(__file__).parents[2] / 'freqtrade/templates/sample_strategy.py').open("rb") as file: - encoded_string = urlsafe_b64encode(file.read()).decode("utf-8") + filepath = Path(__file__).parents[2] / 'freqtrade/templates/sample_strategy.py' + encoded_string = urlsafe_b64encode(filepath.read_bytes()).decode("utf-8") default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)}) strategy = StrategyResolver.load_strategy(default_conf) From 520c5687aa56ea60d0d021d2d944e98913913669 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 03:01:05 +0000 Subject: [PATCH 03/11] Bump prompt-toolkit from 3.0.20 to 3.0.21 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.20 to 3.0.21. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b10bbabf6..b12697040 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,4 +41,4 @@ psutil==5.8.0 colorama==0.4.4 # Building config files interactively questionary==1.10.0 -prompt-toolkit==3.0.20 +prompt-toolkit==3.0.21 From b50b38f049374634903c2911636dd0150dd34aa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 03:01:10 +0000 Subject: [PATCH 04/11] Bump sqlalchemy from 1.4.25 to 1.4.26 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.25 to 1.4.26. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b10bbabf6..27649eb7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.58.47 # Pin cryptography for now due to rust build errors with piwheels cryptography==35.0.0 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.25 +SQLAlchemy==1.4.26 python-telegram-bot==13.7 arrow==1.2.0 cachetools==4.2.2 From 3d90305f8e4a130dfda821c7008ab9fdd69d8511 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 03:01:29 +0000 Subject: [PATCH 05/11] Bump ccxt from 1.58.47 to 1.59.2 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.58.47 to 1.59.2. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.58.47...1.59.2) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b10bbabf6..b3e5b1cc7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.4 pandas-ta==0.3.14b -ccxt==1.58.47 +ccxt==1.59.2 # Pin cryptography for now due to rust build errors with piwheels cryptography==35.0.0 aiohttp==3.7.4.post0 From 826d4eb2f42bea1656c6a3bf8f8e5519fbfc5479 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 03:01:33 +0000 Subject: [PATCH 06/11] Bump jsonschema from 4.1.0 to 4.1.2 Bumps [jsonschema](https://github.com/Julian/jsonschema) from 4.1.0 to 4.1.2. - [Release notes](https://github.com/Julian/jsonschema/releases) - [Changelog](https://github.com/Julian/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/Julian/jsonschema/compare/v4.1.0...v4.1.2) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b10bbabf6..504123583 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.0 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 -jsonschema==4.1.0 +jsonschema==4.1.2 TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 From 538d9e8b375d863105855901d4cccde8d98770bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 04:40:24 +0000 Subject: [PATCH 07/11] Bump arrow from 1.2.0 to 1.2.1 Bumps [arrow](https://github.com/arrow-py/arrow) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/compare/1.2.0...1.2.1) --- updated-dependencies: - dependency-name: arrow dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 84fd2b771..10abecd0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ cryptography==35.0.0 aiohttp==3.7.4.post0 SQLAlchemy==1.4.26 python-telegram-bot==13.7 -arrow==1.2.0 +arrow==1.2.1 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 From 4e88bd07fa0073584f1bead80a2ded02f14606ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 04:40:30 +0000 Subject: [PATCH 08/11] Bump numpy from 1.21.2 to 1.21.3 Bumps [numpy](https://github.com/numpy/numpy) from 1.21.2 to 1.21.3. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.21.2...v1.21.3) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 84fd2b771..04683b97a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.21.2 +numpy==1.21.3 pandas==1.3.4 pandas-ta==0.3.14b From cea251c83c771ae2d3a220d4103569bb46cd6040 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Oct 2021 06:46:02 +0200 Subject: [PATCH 09/11] Clarify documentation for /forcebuy closes #5783 --- docs/telegram-usage.md | 2 +- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index b9d01a236..0c45fbbf1 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -171,7 +171,7 @@ official commands. You can ask at any moment for help with `/help`. | `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) | `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). -| `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) +| `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional and only applies to limit orders. (`forcebuy_enable` must be set to True) | `/performance` | Show performance of each finished trade grouped by pair | `/balance` | Show account balance per currency | `/daily ` | Shows profit or loss per day, over the last n days (n defaults to 7) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 846747f40..771d8bfa4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1033,7 +1033,7 @@ class Telegram(RPCHandler): :return: None """ forcebuy_text = ("*/forcebuy []:* `Instantly buys the given pair. " - "Optionally takes a rate at which to buy.` \n") + "Optionally takes a rate at which to buy (only applies to limit orders).` \n") message = ("*/start:* `Starts the trader`\n" "*/stop:* `Stops the trader`\n" "*/status |[table]:* `Lists all open trades`\n" From 262f186a37cf58ec5720f872fe94ad25c3992052 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Oct 2021 07:19:47 +0200 Subject: [PATCH 10/11] . --- freqtrade/rpc/telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 771d8bfa4..073583940 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1033,7 +1033,8 @@ class Telegram(RPCHandler): :return: None """ forcebuy_text = ("*/forcebuy []:* `Instantly buys the given pair. " - "Optionally takes a rate at which to buy (only applies to limit orders).` \n") + "Optionally takes a rate at which to buy " + "(only applies to limit orders).` \n") message = ("*/start:* `Starts the trader`\n" "*/stop:* `Stops the trader`\n" "*/status |[table]:* `Lists all open trades`\n" From f80d3d48e46fee629b3e748c9363e5a89a488393 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Oct 2021 06:29:35 +0200 Subject: [PATCH 11/11] Add default to minimal_roi to avoid failures closes #5796 --- freqtrade/resolvers/strategy_resolver.py | 18 +++++++++++------- freqtrade/strategy/interface.py | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index e7c077e84..a7b95a3c5 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -56,17 +56,21 @@ class StrategyResolver(IResolver): if strategy._ft_params_from_file: # Set parameters from Hyperopt results file params = strategy._ft_params_from_file - strategy.minimal_roi = params.get('roi', strategy.minimal_roi) + strategy.minimal_roi = params.get('roi', getattr(strategy, 'minimal_roi', {})) - strategy.stoploss = params.get('stoploss', {}).get('stoploss', strategy.stoploss) + strategy.stoploss = params.get('stoploss', {}).get( + 'stoploss', getattr(strategy, 'stoploss', -0.1)) trailing = params.get('trailing', {}) - strategy.trailing_stop = trailing.get('trailing_stop', strategy.trailing_stop) - strategy.trailing_stop_positive = trailing.get('trailing_stop_positive', - strategy.trailing_stop_positive) + strategy.trailing_stop = trailing.get( + 'trailing_stop', getattr(strategy, 'trailing_stop', False)) + strategy.trailing_stop_positive = trailing.get( + 'trailing_stop_positive', getattr(strategy, 'trailing_stop_positive', None)) strategy.trailing_stop_positive_offset = trailing.get( - 'trailing_stop_positive_offset', strategy.trailing_stop_positive_offset) + 'trailing_stop_positive_offset', + getattr(strategy, 'trailing_stop_positive_offset', 0)) strategy.trailing_only_offset_is_reached = trailing.get( - 'trailing_only_offset_is_reached', strategy.trailing_only_offset_is_reached) + 'trailing_only_offset_is_reached', + getattr(strategy, 'trailing_only_offset_is_reached', 0.0)) # Set attributes # Check if we need to override configuration diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7420bd9fd..834ba5975 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -65,9 +65,9 @@ class IStrategy(ABC, HyperStrategyMixin): _populate_fun_len: int = 0 _buy_fun_len: int = 0 _sell_fun_len: int = 0 - _ft_params_from_file: Dict = {} + _ft_params_from_file: Dict # associated minimal roi - minimal_roi: Dict + minimal_roi: Dict = {} # associated stoploss stoploss: float