adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5243 from freqtrade/feat/webservermode_progress

Browse Source 
Introduce webserver mode subcommand
This commit is contained in:
Matthias
2021-07-18 10:48:55 +02:00
committed by GitHub
parent 0282d13221 52ae95b2a5
commit 7b7d9c02d7
20 changed files with 595 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/commands/__init__.py
View File

@@ -20,3 +20,4 @@ from freqtrade.commands.optimize_commands import start_backtesting, start_edge,
from freqtrade.commands.pairlist_commands import start_test_pairlist
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
from freqtrade.commands.trade_commands import start_trading
from freqtrade.commands.webserver_commands import start_webserver

11
freqtrade/commands/arguments.py
View File

@@ -16,6 +16,8 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
ARGS_WEBSERVER: List[str] = []
ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
"max_open_trades", "stake_amount", "fee", "pairs"]
@@ -176,7 +178,8 @@ class Arguments:
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config, start_new_hyperopt,
start_new_strategy, start_plot_dataframe, start_plot_profit,
start_show_trades, start_test_pairlist, start_trading)
start_show_trades, start_test_pairlist, start_trading,
start_webserver)
subparsers = self.parser.add_subparsers(dest='command',
# Use custom message when no subhandler is added
@@ -384,3 +387,9 @@ class Arguments:
)
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
# Add webserver subcommand
webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
parents=[_common_parser])
webserver_cmd.set_defaults(func=start_webserver)
self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)

15
freqtrade/commands/webserver_commands.py Normal file
View File

@@ -0,0 +1,15 @@
from typing import Any, Dict
from freqtrade.enums import RunMode
def start_webserver(args: Dict[str, Any]) -> None:
"""
Main entry point for webserver mode
"""
from freqtrade.configuration import Configuration
from freqtrade.rpc.api_server import ApiServer
# Initialize configuration
config = Configuration(args, RunMode.WEBSERVER).get_config()
ApiServer(config, standalone=True)

2
freqtrade/configuration/configuration.py
View File

@@ -71,7 +71,7 @@ class Configuration:
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:
config['internals'] = {}

1
freqtrade/enums/__init__.py
View File

@@ -1,4 +1,5 @@
# flake8: noqa: F401
from freqtrade.enums.backteststate import BacktestState
from freqtrade.enums.rpcmessagetype import RPCMessageType
from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
from freqtrade.enums.selltype import SellType

15
freqtrade/enums/backteststate.py Normal file
View File

@@ -0,0 +1,15 @@
from enum import Enum
class BacktestState(Enum):
"""
Bot application states
"""
STARTUP = 1
DATALOAD = 2
ANALYZE = 3
CONVERT = 4
BACKTEST = 5
def __str__(self):
return f"{self.name.lower()}"

1
freqtrade/enums/runmode.py
View File

@@ -14,6 +14,7 @@ class RunMode(Enum):
UTIL_EXCHANGE = "util_exchange"
UTIL_NO_EXCHANGE = "util_no_exchange"
PLOT = "plot"
WEBSERVER = "webserver"
OTHER = "other"

41
freqtrade/optimize/backtesting.py
View File

@@ -17,10 +17,11 @@ from freqtrade.data import history
from freqtrade.data.btanalysis import trade_list_to_dataframe
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import SellType
from freqtrade.enums import BacktestState, SellType
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.mixins import LoggingMixin
from freqtrade.optimize.bt_progress import BTProgress
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_stats)
from freqtrade.persistence import LocalTrade, PairLocks, Trade
@@ -57,6 +58,7 @@ class Backtesting:
LoggingMixin.show_output = False
self.config = config
self.results: Optional[Dict[str, Any]] = None
# Reset keys for backtesting
remove_credentials(self.config)
@@ -118,6 +120,9 @@ class Backtesting:
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
self.progress = BTProgress()
self.abort = False
def __del__(self):
LoggingMixin.show_output = True
PairLocks.use_db = True
@@ -147,6 +152,8 @@ class Backtesting:
Loads backtest data and returns the data combined with the timerange
as tuple.
"""
self.progress.init_step(BacktestState.DATALOAD, 1)
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
@@ -170,6 +177,7 @@ class Backtesting:
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
self.progress.set_new_value(1)
return data, timerange
def prepare_backtest(self, enable_protections):
@@ -184,6 +192,15 @@ class Backtesting:
self.rejected_trades = 0
self.dataprovider.clear_cache()
def check_abort(self):
"""
Check if abort was requested, raise DependencyException if that's the case
Only applies to Interactive backtest mode (webserver mode)
"""
if self.abort:
self.abort = False
raise DependencyException("Stop requested")
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
"""
Helper function to convert a processed dataframes into lists for performance reasons.
@@ -194,8 +211,12 @@ class Backtesting:
# and eventually change the constants for indexes at the top
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
data: Dict = {}
self.progress.init_step(BacktestState.CONVERT, len(processed))
# Create dict with data
for pair, pair_data in processed.items():
self.check_abort()
self.progress.increment()
if not pair_data.empty:
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
@@ -417,10 +438,13 @@ class Backtesting:
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
open_trade_count = 0
self.progress.init_step(BacktestState.BACKTEST, int(
(end_date - start_date) / timedelta(minutes=self.timeframe_min)))
# Loop timerange and get candle for each pair at that point in time
while tmp <= end_date:
open_trade_count_start = open_trade_count
self.check_abort()
for i, pair in enumerate(data):
row_index = indexes[pair]
try:
@@ -476,6 +500,7 @@ class Backtesting:
self.protections.global_stop(tmp)
# Move time one configured time_interval ahead.
self.progress.increment()
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
@@ -491,6 +516,8 @@ class Backtesting:
}
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
self.progress.init_step(BacktestState.ANALYZE, 0)
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
backtest_start_time = datetime.now(timezone.utc)
self._set_strategy(strat)
@@ -517,6 +544,7 @@ class Backtesting:
"No data left after adjusting for startup candles.")
min_date, max_date = history.get_timerange(preprocessed)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days).')
@@ -551,11 +579,12 @@ class Backtesting:
for strat in self.strategylist:
min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
if len(self.strategylist) > 0:
stats = generate_backtest_stats(data, self.all_results,
min_date=min_date, max_date=max_date)
self.results = generate_backtest_stats(data, self.all_results,
min_date=min_date, max_date=max_date)
if self.config.get('export', 'none') == 'trades':
store_backtest_stats(self.config['exportfilename'], stats)
store_backtest_stats(self.config['exportfilename'], self.results)
# Show backtest results
show_backtest_results(self.config, stats)
show_backtest_results(self.config, self.results)

33
freqtrade/optimize/bt_progress.py Normal file
View File

@@ -0,0 +1,33 @@
from freqtrade.enums import BacktestState
class BTProgress:
_action: BacktestState = BacktestState.STARTUP
_progress: float = 0
_max_steps: float = 0
def __init__(self):
pass
def init_step(self, action: BacktestState, max_steps: float):
self._action = action
self._max_steps = max_steps
self._proress = 0
def set_new_value(self, new_value: float):
self._progress = new_value
def increment(self):
self._progress += 1
@property
def progress(self):
"""
Get progress as ratio, capped to be between 0 and 1 (to avoid small calculation errors).
"""
return max(min(round(self._progress / self._max_steps, 5)
if self._max_steps > 0 else 0, 1), 0)
@property
def action(self):
return str(self._action)

176
freqtrade/rpc/api_server/api_backtest.py Normal file
View File

@@ -0,0 +1,176 @@
import asyncio
import logging
from copy import deepcopy
from fastapi import APIRouter, BackgroundTasks, Depends
from freqtrade.enums import BacktestState
from freqtrade.exceptions import DependencyException
from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse
from freqtrade.rpc.api_server.deps import get_config
from freqtrade.rpc.api_server.webserver import ApiServer
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# Private API, protected by authentication
router = APIRouter()
@router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
config=Depends(get_config)):
"""Start backtesting if not done so already"""
if ApiServer._bgtask_running:
raise RPCException('Bot Background task already running')
btconfig = deepcopy(config)
settings = dict(bt_settings)
# Pydantic models will contain all keys, but non-provided ones are None
for setting in settings.keys():
if settings[setting] is not None:
btconfig[setting] = settings[setting]
# Start backtesting
# Initialize backtesting object
def run_backtest():
from freqtrade.optimize.optimize_reports import generate_backtest_stats
from freqtrade.resolvers import StrategyResolver
asyncio.set_event_loop(asyncio.new_event_loop())
try:
# Reload strategy
lastconfig = ApiServer._bt_last_config
strat = StrategyResolver.load_strategy(btconfig)
if (
not ApiServer._bt
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
):
from freqtrade.optimize.backtesting import Backtesting
ApiServer._bt = Backtesting(btconfig)
# Only reload data if timeframe or timerange changed.
if (
not ApiServer._bt_data
or not ApiServer._bt_timerange
or lastconfig.get('timerange') != btconfig['timerange']
or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
or lastconfig.get('protections') != btconfig.get('protections', [])
or lastconfig.get('timeframe') != strat.timeframe
):
lastconfig['timerange'] = btconfig['timerange']
lastconfig['protections'] = btconfig.get('protections', [])
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
lastconfig['timeframe'] = strat.timeframe
ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
ApiServer._bt.abort = False
min_date, max_date = ApiServer._bt.backtest_one_strategy(
strat, ApiServer._bt_data, ApiServer._bt_timerange)
ApiServer._bt.results = generate_backtest_stats(
ApiServer._bt_data, ApiServer._bt.all_results,
min_date=min_date, max_date=max_date)
logger.info("Backtest finished.")
except DependencyException as e:
logger.info(f"Backtesting caused an error: {e}")
pass
finally:
ApiServer._bgtask_running = False
background_tasks.add_task(run_backtest)
ApiServer._bgtask_running = True
return {
"status": "running",
"running": True,
"progress": 0,
"step": str(BacktestState.STARTUP),
"status_msg": "Backtest started",
}
@router.get('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_get_backtest():
"""
Get backtesting result.
Returns Result after backtesting has been ran.
"""
from freqtrade.persistence import LocalTrade
if ApiServer._bgtask_running:
return {
"status": "running",
"running": True,
"step": ApiServer._bt.progress.action if ApiServer._bt else str(BacktestState.STARTUP),
"progress": ApiServer._bt.progress.progress if ApiServer._bt else 0,
"trade_count": len(LocalTrade.trades),
"status_msg": "Backtest running",
}
if not ApiServer._bt:
return {
"status": "not_started",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest not yet executed"
}
return {
"status": "ended",
"running": False,
"status_msg": "Backtest ended",
"step": "finished",
"progress": 1,
"backtest_result": ApiServer._bt.results,
}
@router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_delete_backtest():
"""Reset backtesting"""
if ApiServer._bgtask_running:
return {
"status": "running",
"running": True,
"step": "",
"progress": 0,
"status_msg": "Backtest running",
}
if ApiServer._bt:
del ApiServer._bt
ApiServer._bt = None
del ApiServer._bt_data
ApiServer._bt_data = None
logger.info("Backtesting reset")
return {
"status": "reset",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest reset",
}
@router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_backtest_abort():
if not ApiServer._bgtask_running:
return {
"status": "not_running",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest ended",
}
ApiServer._bt.abort = True
return {
"status": "stopping",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest ended",
}

29
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -123,17 +123,17 @@ class ShowConfig(BaseModel):
stake_currency_decimals: int
max_open_trades: int
minimal_roi: Dict[str, Any]
stoploss: float
trailing_stop: bool
stoploss: Optional[float]
trailing_stop: Optional[bool]
trailing_stop_positive: Optional[float]
trailing_stop_positive_offset: Optional[float]
trailing_only_offset_is_reached: Optional[bool]
use_custom_stoploss: Optional[bool]
timeframe: str
timeframe: Optional[str]
timeframe_ms: int
timeframe_min: int
exchange: str
strategy: str
strategy: Optional[str]
forcebuy_enabled: bool
ask_strategy: Dict[str, Any]
bid_strategy: Dict[str, Any]
@@ -318,3 +318,24 @@ class PairHistory(BaseModel):
json_encoders = {
datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT),
}
class BacktestRequest(BaseModel):
strategy: str
timeframe: Optional[str]
timerange: Optional[str]
max_open_trades: Optional[int]
stake_amount: Optional[Union[float, str]]
enable_protections: bool
dry_run_wallet: Optional[float]
class BacktestResponse(BaseModel):
status: str
running: bool
status_msg: str
step: str
progress: float
trade_count: Optional[float]
# TODO: Properly type backtestresult...
backtest_result: Optional[Dict[str, Any]]

5
freqtrade/rpc/api_server/api_v1.py
View File

@@ -1,3 +1,4 @@
import logging
from copy import deepcopy
from pathlib import Path
from typing import List, Optional
@@ -22,6 +23,8 @@ from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# Public API, requires no auth.
router_public = APIRouter()
# Private API, protected by authentication
@@ -249,7 +252,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option
pair_interval = sorted(pair_interval, key=lambda x: x[0])
pairs = list({x[0] for x in pair_interval})
pairs.sort()
result = {
'length': len(pairs),
'pairs': pairs,

63
freqtrade/rpc/api_server/webserver.py
View File

@@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
from starlette.responses import JSONResponse
from freqtrade.exceptions import OperationalException
from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer
from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler
@@ -28,17 +29,37 @@ class FTJSONResponse(JSONResponse):
class ApiServer(RPCHandler):
__instance = None
__initialized = False
_rpc: RPC
# Backtesting type: Backtesting
_bt = None
_bt_data = None
_bt_timerange = None
_bt_last_config: Dict[str, Any] = {}
_has_rpc: bool = False
_bgtask_running: bool = False
_config: Dict[str, Any] = {}
def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None:
super().__init__(rpc, config)
self._server = None
def __new__(cls, *args, **kwargs):
"""
This class is a singleton.
We'll only have one instance of it around.
"""
if ApiServer.__instance is None:
ApiServer.__instance = object.__new__(cls)
ApiServer.__initialized = False
return ApiServer.__instance
ApiServer._rpc = rpc
ApiServer._has_rpc = True
def __init__(self, config: Dict[str, Any], standalone: bool = False) -> None:
ApiServer._config = config
if self.__initialized and (standalone or self._standalone):
return
self._standalone: bool = standalone
self._server = None
ApiServer.__initialized = True
api_config = self._config['api_server']
self.app = FastAPI(title="Freqtrade API",
@@ -50,12 +71,33 @@ class ApiServer(RPCHandler):
self.start_api()
def add_rpc_handler(self, rpc: RPC):
"""
Attach rpc handler
"""
if not self._has_rpc:
ApiServer._rpc = rpc
ApiServer._has_rpc = True
else:
# This should not happen assuming we didn't mess up.
raise OperationalException('RPC Handler already attached.')
def cleanup(self) -> None:
""" Cleanup pending module resources """
if self._server:
ApiServer._has_rpc = False
del ApiServer._rpc
if self._server and not self._standalone:
logger.info("Stopping API Server")
self._server.cleanup()
@classmethod
def shutdown(cls):
cls.__initialized = False
del cls.__instance
cls.__instance = None
cls._has_rpc = False
cls._rpc = None
def send_msg(self, msg: Dict[str, str]) -> None:
pass
@@ -68,6 +110,7 @@ class ApiServer(RPCHandler):
def configure_app(self, app: FastAPI, config):
from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login
from freqtrade.rpc.api_server.api_backtest import router as api_backtest
from freqtrade.rpc.api_server.api_v1 import router as api_v1
from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public
from freqtrade.rpc.api_server.web_ui import router_ui
@@ -77,6 +120,9 @@ class ApiServer(RPCHandler):
app.include_router(api_v1, prefix="/api/v1",
dependencies=[Depends(http_basic_or_jwt_token)],
)
app.include_router(api_backtest, prefix="/api/v1",
dependencies=[Depends(http_basic_or_jwt_token)],
)
app.include_router(router_login, prefix="/api/v1", tags=["auth"])
# UI Router MUST be last!
app.include_router(router_ui, prefix='')
@@ -125,6 +171,9 @@ class ApiServer(RPCHandler):
)
try:
self._server = UvicornServer(uvconfig)
self._server.run_in_thread()
if self._standalone:
self._server.run()
else:
self._server.run_in_thread()
except Exception:
logger.exception("Api server failed to start.")

4
freqtrade/rpc/rpc.py
View File

@@ -119,9 +119,9 @@ class RPC:
'bot_name': config.get('bot_name', 'freqtrade'),
'timeframe': config.get('timeframe'),
'timeframe_ms': timeframe_to_msecs(config['timeframe']
) if 'timeframe' in config else '',
) if 'timeframe' in config else 0,
'timeframe_min': timeframe_to_minutes(config['timeframe']
) if 'timeframe' in config else '',
) if 'timeframe' in config else 0,
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),

7
freqtrade/rpc/rpc_manager.py
View File

@@ -36,15 +36,16 @@ class RPCManager:
if config.get('api_server', {}).get('enabled', False):
logger.info('Enabling rpc.api_server')
from freqtrade.rpc.api_server import ApiServer
self.registered_modules.append(ApiServer(self._rpc, config))
apiserver = ApiServer(config)
apiserver.add_rpc_handler(self._rpc)
self.registered_modules.append(apiserver)
def cleanup(self) -> None:
""" Stops all enabled rpc modules """
logger.info('Cleaning up rpc modules ...')
while self.registered_modules:
mod = self.registered_modules.pop()
logger.debug('Cleaning up rpc.%s ...', mod.name)
logger.info('Cleaning up rpc.%s ...', mod.name)
mod.cleanup()
del mod