diff --git a/config_examples/config_binance.example.json b/config_examples/config_binance.example.json index ad8862afa..35b9fcd20 100644 --- a/config_examples/config_binance.example.json +++ b/config_examples/config_binance.example.json @@ -90,7 +90,7 @@ }, "bot_name": "freqtrade", "initial_state": "running", - "force_enter_enable": false, + "force_entry_enable": false, "internals": { "process_throttle_secs": 5 } diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 9e22a25b3..462cf604f 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -7,6 +7,7 @@ Depending on the callback used, they may be called when entering / exiting a tra Currently available callbacks: +* [`bot_start()`](#bot-start) * [`bot_loop_start()`](#bot-loop-start) * [`custom_stake_amount()`](#stake-size-management) * [`custom_exit()`](#custom-exit-signal) @@ -22,6 +23,29 @@ Currently available callbacks: !!! Tip "Callback calling sequence" You can find the callback calling sequence in [bot-basics](bot-basics.md#bot-execution-logic) +## Bot start + +A simple callback which is called once when the strategy is loaded. +This can be used to perform actions that must only be performed once and runs after dataprovider and wallet are set + +``` python +import requests + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + def bot_start(self, **kwargs) -> None: + """ + Called only once after bot instantiation. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + if self.config['runmode'].value in ('live', 'dry_run'): + # Assign this to the class by using self.* + # can then be used by populate_* methods + self.cust_remote_data = requests.get('https://some_remote_source.example.com') + +``` ## Bot loop start A simple callback which is called once at the start of every bot throttling iteration (roughly every 5 seconds, unless configured differently). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 333fe7d11..af5ad00dd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -459,8 +459,6 @@ SCHEMA_BACKTEST_REQUIRED = [ 'stake_currency', 'stake_amount', 'dry_run_wallet', - 'stoploss', - 'minimal_roi', 'dataformat_ohlcv', 'dataformat_trades', ] diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 206a6f5f3..0c8e721c0 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -12,7 +12,8 @@ import pandas as pd from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.exceptions import OperationalException -from freqtrade.misc import get_backtest_metadata_filename, json_load +from freqtrade.misc import json_load +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename from freqtrade.persistence import LocalTrade, Trade, init_db diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 49c7050c9..5a16ef562 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -123,6 +123,8 @@ class FreqtradeBot(LoggingMixin): self._schedule.every().day.at(t).do(update) self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc) + self.strategy.bot_start() + def notify_status(self, msg: str) -> None: """ Public method for users of this class (worker, etc.) to send notifications diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 55a533725..c3968e61c 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,13 +2,11 @@ Various tool function for Freqtrade and scripts """ import gzip -import hashlib import logging import re -from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import Any, Iterator, List, Union +from typing import Any, Iterator, List from typing.io import IO from urllib.parse import urlparse @@ -251,34 +249,3 @@ def parse_db_uri_for_logging(uri: str): return uri pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') - - -def get_strategy_run_id(strategy) -> str: - """ - Generate unique identification hash for a backtest run. Identical config and strategy file will - always return an identical hash. - :param strategy: strategy object. - :return: hex string id. - """ - digest = hashlib.sha1() - config = deepcopy(strategy.config) - - # Options that have no impact on results of individual backtest. - not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') - for k in not_important_keys: - if k in config: - del config[k] - - # Explicitly allow NaN values (e.g. max_open_trades). - # as it does not matter for getting the hash. - digest.update(rapidjson.dumps(config, default=str, - number_mode=rapidjson.NM_NAN).encode('utf-8')) - with open(strategy.__file__, 'rb') as fp: - digest.update(fp.read()) - return digest.hexdigest().lower() - - -def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: - """Return metadata filename for specified backtest results file.""" - filename = Path(filename) - return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtest_caching.py b/freqtrade/optimize/backtest_caching.py new file mode 100644 index 000000000..d9d270072 --- /dev/null +++ b/freqtrade/optimize/backtest_caching.py @@ -0,0 +1,40 @@ +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Union + +import rapidjson + + +def get_strategy_run_id(strategy) -> str: + """ + Generate unique identification hash for a backtest run. Identical config and strategy file will + always return an identical hash. + :param strategy: strategy object. + :return: hex string id. + """ + digest = hashlib.sha1() + config = deepcopy(strategy.config) + + # Options that have no impact on results of individual backtest. + not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') + for k in not_important_keys: + if k in config: + del config[k] + + # Explicitly allow NaN values (e.g. max_open_trades). + # as it does not matter for getting the hash. + digest.update(rapidjson.dumps(config, default=str, + number_mode=rapidjson.NM_NAN).encode('utf-8')) + # Include _ft_params_from_file - so changing parameter files cause cache eviction + digest.update(rapidjson.dumps( + strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8')) + with open(strategy.__file__, 'rb') as fp: + digest.update(fp.read()) + return digest.hexdigest().lower() + + +def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: + """Return metadata filename for specified backtest results file.""" + filename = Path(filename) + return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d567e1159..4485d3da3 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, @@ -187,6 +187,7 @@ class Backtesting: # since a "perfect" stoploss-exit is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + self.strategy.bot_start() def _load_protections(self, strategy: IStrategy): if self.config.get('enable_protections', False): diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index cc9bafb0b..30eabecd0 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -44,6 +44,7 @@ class EdgeCli: self.edge._timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + self.strategy.bot_start() def start(self) -> None: result = self.edge.calculate(self.config['exchange']['pair_whitelist']) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index dd058aff4..1d58dc339 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,8 +11,8 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, - get_backtest_metadata_filename, round_coin_value) +from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 747248be7..5337016f3 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -610,6 +610,7 @@ def load_and_plot_trades(config: Dict[str, Any]): exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) IStrategy.dp = DataProvider(config, exchange) + strategy.bot_start() plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count) timerange = plot_elements['timerange'] trades = plot_elements['trades'] diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 63812f52f..0da129583 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -2,7 +2,7 @@ import logging from ipaddress import IPv4Address from typing import Any, Dict -import rapidjson +import orjson import uvicorn from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -24,7 +24,7 @@ class FTJSONResponse(JSONResponse): Use rapidjson for responses Handles NaN and Inf / -Inf in a javascript way by default. """ - return rapidjson.dumps(content).encode("utf-8") + return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY) class ApiServer(RPCHandler): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a472a6943..1ea5f2d5d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -193,6 +193,13 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.populate_sell_trend(dataframe, metadata) + def bot_start(self, **kwargs) -> None: + """ + Called only once after bot instantiation. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + pass + def bot_loop_start(self, **kwargs) -> None: """ Called at the start of the bot iteration (one loop). diff --git a/requirements.txt b/requirements.txt index de14b9f2c..ab8329979 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,8 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 +# Properly format api responses +orjson==3.6.8 # Notify systemd sdnotify==0.3.2 diff --git a/setup.py b/setup.py index 250cafdc9..c5e418d0d 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( 'pycoingecko', 'py_find_1st', 'python-rapidjson', + 'orjson', 'sdnotify', 'colorama', 'jinja2', diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a51e1b654..c87a0ef73 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,7 +22,7 @@ from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange.exchange import timeframe_to_next_date -from freqtrade.misc import get_strategy_run_id +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver @@ -312,6 +312,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: get_fee.assert_called() assert backtesting.fee == 0.5 assert not backtesting.strategy.order_types["stoploss_on_exchange"] + assert backtesting.strategy.bot_started is True def test_backtesting_init_no_timeframe(mocker, default_conf, caplog) -> None: diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index f0f436a43..8241a5362 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -94,6 +94,7 @@ def test_edge_init(mocker, edge_conf) -> None: assert edge_cli.config == edge_conf assert edge_cli.config['stake_amount'] == 'unlimited' assert callable(edge_cli.edge.calculate) + assert edge_cli.strategy.bot_started is True def test_edge_init_fee(mocker, edge_conf) -> None: diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 4910213b4..43f783a53 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -13,7 +13,6 @@ import uvicorn from fastapi import FastAPI from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient -from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ @@ -985,7 +984,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, assert_response(rc) resp_values = rc.json() assert len(resp_values) == 4 - assert isnan(resp_values[0]['profit_abs']) + assert resp_values[0]['profit_abs'] is None def test_api_version(botclient): diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 372e29412..df83d3663 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -82,6 +82,11 @@ class StrategyTestV3(IStrategy): # }) # return prot + bot_started = False + + def bot_start(self): + self.bot_started = True + def informative_pairs(self): return []