adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into pr/imxuwang/3799

Browse Source
This commit is contained in:
Matthias
2020-10-22 07:55:48 +02:00
parent 1c27aaab72 71410a5a1e
commit 2f91f87ad3
174 changed files with 4148 additions and 1648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/__main__.py
View File

@@ -8,5 +8,6 @@ To launch Freqtrade as a module
from freqtrade import main
if __name__ == '__main__':
main.main()

24
freqtrade/commands/__init__.py
View File

@@ -8,23 +8,15 @@ Note: Be careful with file-scoped imports in these subfiles.
"""
from freqtrade.commands.arguments import Arguments
from freqtrade.commands.build_config_commands import start_new_config
from freqtrade.commands.data_commands import (start_convert_data,
start_download_data,
from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
start_list_data)
from freqtrade.commands.deploy_commands import (start_create_userdir,
start_new_hyperopt,
from freqtrade.commands.deploy_commands import (start_create_userdir, start_new_hyperopt,
start_new_strategy)
from freqtrade.commands.hyperopt_commands import (start_hyperopt_list,
start_hyperopt_show)
from freqtrade.commands.list_commands import (start_list_exchanges,
start_list_hyperopts,
start_list_markets,
start_list_strategies,
start_list_timeframes,
start_show_trades)
from freqtrade.commands.optimize_commands import (start_backtesting,
start_edge, start_hyperopt)
from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts,
start_list_markets, start_list_strategies,
start_list_timeframes, start_show_trades)
from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt
from freqtrade.commands.pairlist_commands import start_test_pairlist
from freqtrade.commands.plot_commands import (start_plot_dataframe,
start_plot_profit)
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
from freqtrade.commands.trade_commands import start_trading

23
freqtrade/commands/arguments.py
View File

@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional
from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
from freqtrade.constants import DEFAULT_CONFIG
ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
ARGS_STRATEGY = ["strategy", "strategy_path"]
@@ -26,7 +27,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"use_max_market_positions", "print_all",
"print_colorized", "print_json", "hyperopt_jobs",
"hyperopt_random_state", "hyperopt_min_trades",
"hyperopt_continue", "hyperopt_loss"]
"hyperopt_loss"]
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
@@ -56,7 +57,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
"timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
@@ -75,10 +76,10 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
"hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit",
"hyperopt_list_min_objective", "hyperopt_list_max_objective",
"print_colorized", "print_json", "hyperopt_list_no_details",
"export_csv"]
"hyperoptexportfilename", "export_csv"]
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperopt_show_no_header"]
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header"]
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies", "list-data",
@@ -161,16 +162,14 @@ class Arguments:
self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
self._build_args(optionlist=['version'], parser=self.parser)
from freqtrade.commands import (start_create_userdir, start_convert_data,
start_download_data, start_list_data,
start_hyperopt_list, start_hyperopt_show,
from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir,
start_download_data, start_edge, start_hyperopt,
start_hyperopt_list, start_hyperopt_show, start_list_data,
start_list_exchanges, start_list_hyperopts,
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config,
start_new_hyperopt, start_new_strategy,
start_plot_dataframe, start_plot_profit, start_show_trades,
start_backtesting, start_hyperopt, start_edge,
start_test_pairlist, start_trading)
start_list_timeframes, start_new_config, start_new_hyperopt,
start_new_strategy, start_plot_dataframe, start_plot_profit,
start_show_trades, start_test_pairlist, start_trading)
subparsers = self.parser.add_subparsers(dest='command',
# Use custom message when no subhandler is added

10
freqtrade/commands/build_config_commands.py
View File

@@ -1,13 +1,15 @@
import logging
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, List
from questionary import Separator, prompt
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
from freqtrade.exchange import available_exchanges, MAP_EXCHANGE_CHILDCLASS
from freqtrade.misc import render_template
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
from freqtrade.misc import render_template
logger = logging.getLogger(__name__)
@@ -46,7 +48,7 @@ def ask_user_config() -> Dict[str, Any]:
Interactive questions built using https://github.com/tmbo/questionary
:returns: Dict with keys to put into template
"""
questions = [
questions: List[Dict[str, Any]] = [
{
"type": "confirm",
"name": "dry_run",

21
freqtrade/commands/cli_options.py
View File

@@ -252,23 +252,20 @@ AVAILABLE_CLI_OPTIONS = {
metavar='INT',
default=1,
),
"hyperopt_continue": Arg(
"--continue",
help="Continue hyperopt from previous runs. "
"By default, temporary files will be removed and hyperopt will start from scratch.",
default=False,
action='store_true',
),
"hyperopt_loss": Arg(
'--hyperopt-loss',
help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). '
'Different functions can generate completely different results, '
'since the target for optimization is different. Built-in Hyperopt-loss-functions are: '
'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, '
'SortinoHyperOptLoss, SortinoHyperOptLossDaily.'
'(default: `%(default)s`).',
'ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, '
'SharpeHyperOptLossDaily, SortinoHyperOptLoss, SortinoHyperOptLossDaily.',
metavar='NAME',
default=constants.DEFAULT_HYPEROPT_LOSS,
),
"hyperoptexportfilename": Arg(
'--hyperopt-filename',
help='Hyperopt result filename.'
'Example: `--hyperopt-filename=hyperopt_results_2020-09-27_16-20-48.pickle`',
metavar='FILENAME',
),
# List exchanges
"print_one_column": Arg(
@@ -375,7 +372,7 @@ AVAILABLE_CLI_OPTIONS = {
help='Specify which tickers to download. Space-separated list. '
'Default: `1m 5m`.',
choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
'6h', '8h', '12h', '1d', '3d', '1w'],
'6h', '8h', '12h', '1d', '3d', '1w', '2w', '1M', '1y'],
default=['1m', '5m'],
nargs='+',
),

16
freqtrade/commands/data_commands.py
View File

@@ -6,16 +6,15 @@ from typing import Any, Dict, List
import arrow
from freqtrade.configuration import TimeRange, setup_utils_configuration
from freqtrade.data.converter import (convert_ohlcv_format,
convert_trades_format)
from freqtrade.data.history import (convert_trades_to_ohlcv,
refresh_backtest_ohlcv_data,
from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
refresh_backtest_trades_data)
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@@ -25,11 +24,17 @@ def start_download_data(args: Dict[str, Any]) -> None:
"""
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
if 'days' in config and 'timerange' in config:
raise OperationalException("--days and --timerange are mutually exclusive. "
"You can only specify one or the other.")
timerange = TimeRange()
if 'days' in config:
time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
timerange = TimeRange.parse_timerange(f'{time_since}-')
if 'timerange' in config:
timerange = timerange.parse_timerange(config['timerange'])
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
@@ -99,8 +104,9 @@ def start_list_data(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
from freqtrade.data.history.idatahandler import get_datahandler
from tabulate import tabulate
from freqtrade.data.history.idatahandler import get_datahandler
dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
paircombs = dhc.ohlcv_get_available_data(config['datadir'])

4
freqtrade/commands/deploy_commands.py
View File

@@ -4,13 +4,13 @@ from pathlib import Path
from typing import Any, Dict
from freqtrade.configuration import setup_utils_configuration
from freqtrade.configuration.directory_operations import (copy_sample_files,
create_userdata_dir)
from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.misc import render_template, render_template_with_fallback
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)

13
freqtrade/commands/hyperopt_commands.py
View File

@@ -5,9 +5,11 @@ from typing import Any, Dict, List
from colorama import init as colorama_init
from freqtrade.configuration import setup_utils_configuration
from freqtrade.data.btanalysis import get_latest_hyperopt_file
from freqtrade.exceptions import OperationalException
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@@ -40,8 +42,9 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
}
results_file = (config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
results_file = get_latest_hyperopt_file(
config['user_data_dir'] / 'hyperopt_results',
config.get('hyperoptexportfilename'))
# Previous evaluations
epochs = Hyperopt.load_previous_results(results_file)
@@ -80,8 +83,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
print_json = config.get('print_json', False)
no_header = config.get('hyperopt_show_no_header', False)
results_file = (config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
results_file = get_latest_hyperopt_file(
config['user_data_dir'] / 'hyperopt_results',
config.get('hyperoptexportfilename'))
n = config.get('hyperopt_show_index', -1)
filteroptions = {

13
freqtrade/commands/list_commands.py
View File

@@ -5,20 +5,20 @@ from collections import OrderedDict
from pathlib import Path
from typing import Any, Dict, List
from colorama import init as colorama_init
from colorama import Fore, Style
import rapidjson
from colorama import Fore, Style
from colorama import init as colorama_init
from tabulate import tabulate
from freqtrade.configuration import setup_utils_configuration
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
market_is_active)
from freqtrade.exchange import available_exchanges, ccxt_exchanges, market_is_active
from freqtrade.misc import plural
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@@ -203,15 +203,16 @@ def start_show_trades(args: Dict[str, Any]) -> None:
"""
Show trades
"""
from freqtrade.persistence import init, Trade
import json
from freqtrade.persistence import Trade, init_db
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if 'db_url' not in config:
raise OperationalException("--db-url is required for this command.")
logger.info(f'Using DB: "{config["db_url"]}"')
init(config['db_url'], clean_open_orders=False)
init_db(config['db_url'], clean_open_orders=False)
tfilter = []
if config.get('trade_ids'):

3
freqtrade/commands/optimize_commands.py
View File

@@ -6,6 +6,7 @@ from freqtrade.configuration import setup_utils_configuration
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@@ -58,6 +59,7 @@ def start_hyperopt(args: Dict[str, Any]) -> None:
# Import here to avoid loading hyperopt module when it's not used
try:
from filelock import FileLock, Timeout
from freqtrade.optimize.hyperopt import Hyperopt
except ImportError as e:
raise OperationalException(
@@ -98,6 +100,7 @@ def start_edge(args: Dict[str, Any]) -> None:
:return: None
"""
from freqtrade.optimize.edge_cli import EdgeCli
# Initialize configuration
config = setup_optimize_configuration(args, RunMode.EDGE)
logger.info('Starting freqtrade in Edge mode')

1
freqtrade/commands/pairlist_commands.py
View File

@@ -7,6 +7,7 @@ from freqtrade.configuration import setup_utils_configuration
from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)

1
freqtrade/commands/trade_commands.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Any, Dict

6
freqtrade/configuration/__init__.py
View File

@@ -1,7 +1,7 @@
# flake8: noqa: F401
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
from freqtrade.configuration.timerange import TimeRange
from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.timerange import TimeRange

6
freqtrade/configuration/check_exchange.py
View File

@@ -2,11 +2,11 @@ import logging
from typing import Any, Dict
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason,
is_exchange_bad, is_exchange_known_ccxt,
is_exchange_officially_supported)
from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad,
is_exchange_known_ccxt, is_exchange_officially_supported)
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)

6
freqtrade/configuration/config_setup.py
View File

@@ -1,10 +1,12 @@
import logging
from typing import Any, Dict
from freqtrade.state import RunMode
from .check_exchange import remove_credentials
from .config_validation import validate_config_consistency
from .configuration import Configuration
from .check_exchange import remove_credentials
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)

1
freqtrade/configuration/config_validation.py
View File

@@ -9,6 +9,7 @@ from freqtrade import constants
from freqtrade.exceptions import OperationalException
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)

10
freqtrade/configuration/configuration.py
View File

@@ -10,14 +10,14 @@ from typing import Any, Callable, Dict, List, Optional
from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import (create_datadir,
create_userdata_dir)
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.load_config import load_config_file
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts, json_load
from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode
logger = logging.getLogger(__name__)
@@ -263,6 +263,9 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_path',
logstring='Using additional Hyperopt lookup path: {}')
self._args_to_config(config, argname='hyperoptexportfilename',
logstring='Using hyperopt file: {}')
self._args_to_config(config, argname='epochs',
logstring='Parameter --epochs detected ... '
'Will run Hyperopt with for {} epochs ...'
@@ -295,9 +298,6 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_min_trades',
logstring='Parameter --min-trades detected: {}')
self._args_to_config(config, argname='hyperopt_continue',
logstring='Hyperopt continue: {}')
self._args_to_config(config, argname='hyperopt_loss',
logstring='Using Hyperopt loss class name: {}')

3
freqtrade/configuration/directory_operations.py
View File

@@ -3,8 +3,9 @@ import shutil
from pathlib import Path
from typing import Any, Dict, Optional
from freqtrade.exceptions import OperationalException
from freqtrade.constants import USER_DATA_FILES
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)

1
freqtrade/configuration/load_config.py
View File

@@ -11,6 +11,7 @@ import rapidjson
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)

15
freqtrade/constants.py
View File

@@ -11,7 +11,6 @@ DEFAULT_EXCHANGE = 'bittrex'
PROCESS_THROTTLE_SECS = 5 # sec
HYPEROPT_EPOCH = 100 # epochs
RETRY_TIMEOUT = 30 # sec
DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss'
DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
UNLIMITED_STAKE_AMOUNT = 'unlimited'
@@ -39,6 +38,8 @@ USERPATH_HYPEROPTS = 'hyperopts'
USERPATH_STRATEGIES = 'strategies'
USERPATH_NOTEBOOKS = 'notebooks'
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
# Soure files with destination directories within user-directory
USER_DATA_FILES = {
'sample_strategy.py': USERPATH_STRATEGIES,
@@ -201,6 +202,18 @@ CONF_SCHEMA = {
'enabled': {'type': 'boolean'},
'token': {'type': 'string'},
'chat_id': {'type': 'string'},
'notification_settings': {
'type': 'object',
'properties': {
'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}
}
}
},
'required': ['enabled', 'token', 'chat_id']
},

64
freqtrade/data/btanalysis.py
View File

@@ -2,17 +2,17 @@
Helpers when analyzing backtest data
"""
import logging
from datetime import timezone
from pathlib import Path
from typing import Dict, Union, Tuple, Any, Optional
from typing import Any, Dict, Optional, Tuple, Union
import numpy as np
import pandas as pd
from datetime import timezone
from freqtrade import persistence
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.misc import json_load
from freqtrade.persistence import Trade
from freqtrade.persistence import Trade, init_db
logger = logging.getLogger(__name__)
@@ -21,10 +21,11 @@ BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index",
"open_rate", "close_rate", "open_at_end", "sell_reason"]
def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
"""
Get latest backtest export based on '.last_result.json'.
:param directory: Directory to search for last result
:param variant: 'backtest' or 'hyperopt' - the method to return
:return: string containing the filename of the latest backtest result
:raises: ValueError in the following cases:
* Directory does not exist
@@ -44,10 +45,57 @@ def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
with filename.open() as file:
data = json_load(file)
if 'latest_backtest' not in data:
if f'latest_{variant}' not in data:
raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.")
return data['latest_backtest']
return data[f'latest_{variant}']
def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
"""
Get latest backtest export based on '.last_result.json'.
:param directory: Directory to search for last result
:return: string containing the filename of the latest backtest result
:raises: ValueError in the following cases:
* Directory does not exist
* `directory/.last_result.json` does not exist
* `directory/.last_result.json` has the wrong content
"""
return get_latest_optimize_filename(directory, 'backtest')
def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str:
"""
Get latest hyperopt export based on '.last_result.json'.
:param directory: Directory to search for last result
:return: string containing the filename of the latest hyperopt result
:raises: ValueError in the following cases:
* Directory does not exist
* `directory/.last_result.json` does not exist
* `directory/.last_result.json` has the wrong content
"""
try:
return get_latest_optimize_filename(directory, 'hyperopt')
except ValueError:
# Return default (legacy) pickle filename
return 'hyperopt_results.pickle'
def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = None) -> Path:
"""
Get latest hyperopt export based on '.last_result.json'.
:param directory: Directory to search for last result
:return: string containing the filename of the latest hyperopt result
:raises: ValueError in the following cases:
* Directory does not exist
* `directory/.last_result.json` does not exist
* `directory/.last_result.json` has the wrong content
"""
if isinstance(directory, str):
directory = Path(directory)
if predef_filename:
return directory / predef_filename
return directory / get_latest_hyperopt_filename(directory)
def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
@@ -169,7 +217,7 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
Can also serve as protection to load the correct result.
:return: Dataframe containing Trades
"""
persistence.init(db_url, clean_open_orders=False)
init_db(db_url, clean_open_orders=False)
columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
"open_rate", "close_rate", "amount", "trade_duration", "sell_reason",

4
freqtrade/data/converter.py
View File

@@ -10,8 +10,8 @@ from typing import Any, Dict, List
import pandas as pd
from pandas import DataFrame, to_datetime
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
DEFAULT_TRADES_COLUMNS)
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
logger = logging.getLogger(__name__)

1
freqtrade/data/dataprovider.py
View File

@@ -17,6 +17,7 @@ from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)

10
freqtrade/data/history/__init__.py
View File

@@ -5,10 +5,8 @@ Includes:
* load data for a pair (or a list of pairs) from disk
* download data from exchange and store to disk
"""
from .history_utils import (convert_trades_to_ohlcv, # noqa: F401
get_timerange, load_data, load_pair_history,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data, refresh_data,
# flake8: noqa: F401
from .history_utils import (convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history,
refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data,
validate_backtest_data)
from .idatahandler import get_datahandler # noqa: F401
from .idatahandler import get_datahandler

4
freqtrade/data/history/hdf5datahandler.py
View File

@@ -7,12 +7,12 @@ import pandas as pd
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
DEFAULT_TRADES_COLUMNS,
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
ListPairsWithTimeframes)
from .idatahandler import IDataHandler, TradeList
logger = logging.getLogger(__name__)

8
freqtrade/data/history/history_utils.py
View File

@@ -9,15 +9,14 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import (clean_ohlcv_dataframe,
ohlcv_to_dataframe,
trades_remove_duplicates,
trades_to_ohlcv)
from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe,
trades_remove_duplicates, trades_to_ohlcv)
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.misc import format_ms_time
logger = logging.getLogger(__name__)
@@ -136,7 +135,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
start = None
if timerange:
if timerange.starttype == 'date':
# TODO: convert to date for conversion
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
# Intentionally don't pass timerange in - since we need to load the full dataset.

4
freqtrade/data/history/idatahandler.py
View File

@@ -14,10 +14,10 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.converter import (clean_ohlcv_dataframe,
trades_remove_duplicates, trim_dataframe)
from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe
from freqtrade.exchange import timeframe_to_seconds
logger = logging.getLogger(__name__)
# Type for trades list

4
freqtrade/data/history/jsondatahandler.py
View File

@@ -8,12 +8,12 @@ from pandas import DataFrame, read_json, to_datetime
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
ListPairsWithTimeframes)
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes
from freqtrade.data.converter import trades_dict_to_list
from .idatahandler import IDataHandler, TradeList
logger = logging.getLogger(__name__)

11
freqtrade/edge/edge_positioning.py
View File

@@ -9,11 +9,12 @@ import utils_find_1st as utf1st
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, DATETIME_PRINT_FORMAT
from freqtrade.exceptions import OperationalException
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.exceptions import OperationalException
from freqtrade.strategy.interface import SellType
logger = logging.getLogger(__name__)
@@ -309,8 +310,10 @@ class Edge:
# Calculating number of losing trades, average win and average loss
df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']
df['average_win'] = df['profit_sum'] / df['nb_win_trades']
df['average_loss'] = df['loss_sum'] / df['nb_loss_trades']
df['average_win'] = np.where(df['nb_win_trades'] == 0, 0.0,
df['profit_sum'] / df['nb_win_trades'])
df['average_loss'] = np.where(df['nb_loss_trades'] == 0, 0.0,
df['loss_sum'] / df['nb_loss_trades'])
# Win rate = number of profitable trades / number of trades
df['winrate'] = df['nb_win_trades'] / df['nb_trades']

7
freqtrade/exceptions.py
View File

@@ -51,6 +51,13 @@ class RetryableOrderError(InvalidOrderException):
"""
class InsufficientFundsError(InvalidOrderException):
"""
This error is used when there are not enough funds available on the exchange
to create an order.
"""
class TemporaryError(ExchangeError):
"""
Temporary network or exchange related error.

25
freqtrade/exchange/__init__.py
View File

@@ -1,19 +1,16 @@
# flake8: noqa: F401
# isort: off
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS
from freqtrade.exchange.exchange import Exchange
from freqtrade.exchange.exchange import (get_exchange_bad_reason,
is_exchange_bad,
is_exchange_known_ccxt,
is_exchange_officially_supported,
ccxt_exchanges,
available_exchanges)
from freqtrade.exchange.exchange import (timeframe_to_seconds,
timeframe_to_minutes,
timeframe_to_msecs,
timeframe_to_next_date,
timeframe_to_prev_date)
from freqtrade.exchange.exchange import (market_is_active)
from freqtrade.exchange.kraken import Kraken
from freqtrade.exchange.binance import Binance
# isort: on
from freqtrade.exchange.bibox import Bibox
from freqtrade.exchange.binance import Binance
from freqtrade.exchange.bittrex import Bittrex
from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
get_exchange_bad_reason, is_exchange_bad,
is_exchange_known_ccxt, is_exchange_officially_supported,
market_is_active, timeframe_to_minutes, timeframe_to_msecs,
timeframe_to_next_date, timeframe_to_prev_date,
timeframe_to_seconds)
from freqtrade.exchange.ftx import Ftx
from freqtrade.exchange.kraken import Kraken

1
freqtrade/exchange/bibox.py
View File

@@ -4,6 +4,7 @@ from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)

21
freqtrade/exchange/binance.py
View File

@@ -4,12 +4,12 @@ from typing import Dict
import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError,
InvalidOrderException, OperationalException,
TemporaryError)
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
logger = logging.getLogger(__name__)
@@ -20,20 +20,9 @@ class Binance(Exchange):
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"trades_pagination": "id",
"trades_pagination_arg": "fromId",
"l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
}
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""
get order book level 2 from exchange
20180619: binance support limits but only on specific range
"""
limit_range = [5, 10, 20, 50, 100, 500, 1000]
# get next-higher step in the limit_range list
limit = min(list(filter(lambda x: limit <= x, limit_range)))
return super().fetch_l2_order_book(pair, limit)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
@@ -80,7 +69,7 @@ class Binance(Exchange):
'stop price: %s. limit: %s', pair, stop_price, rate)
return order
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to sell amount {amount} at rate {rate}. '
f'Message: {e}') from e

23
freqtrade/exchange/bittrex.py Normal file
View File

@@ -0,0 +1,23 @@
""" Bittrex exchange subclass """
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Bittrex(Exchange):
"""
Bittrex exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
_ft_has: Dict = {
"l2_limit_range": [1, 25, 500],
}

8
freqtrade/exchange/common.py
View File

@@ -3,13 +3,17 @@ import logging
import time
from functools import wraps
from freqtrade.exceptions import (DDosProtection, RetryableOrderError,
TemporaryError)
from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError
logger = logging.getLogger(__name__)
# Maximum default retry count.
# Functions are always called RETRY_COUNT + 1 times (for the original call)
API_RETRY_COUNT = 4
API_FETCH_ORDER_RETRY_COUNT = 5
BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "

52
freqtrade/exchange/exchange.py
View File

@@ -8,24 +8,25 @@ import logging
from copy import deepcopy
from datetime import datetime, timezone
from math import ceil
from random import randint
from typing import Any, Dict, List, Optional, Tuple
import arrow
import ccxt
import ccxt.async_support as ccxt_async
from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE,
TRUNCATE, decimal_to_precision)
from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE,
decimal_to_precision)
from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.exceptions import (DDosProtection, ExchangeError,
InvalidOrderException, OperationalException,
RetryableOrderError, TemporaryError)
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
InvalidOrderException, OperationalException, RetryableOrderError,
TemporaryError)
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
CcxtModuleType = Any
@@ -52,7 +53,7 @@ class Exchange:
"ohlcv_partial_candle": True,
"trades_pagination": "time", # Possible are "time" or "id"
"trades_pagination_arg": "since",
"l2_limit_range": None,
}
_ft_has: Dict = {}
@@ -487,11 +488,11 @@ class Exchange:
def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{randint(0, 10**6)}'
order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount)
dry_order = {
"id": order_id,
'pair': pair,
'id': order_id,
'symbol': pair,
'price': rate,
'average': rate,
'amount': _amount,
@@ -500,6 +501,7 @@ class Exchange:
'side': side,
'remaining': _amount,
'datetime': arrow.utcnow().isoformat(),
'timestamp': int(arrow.utcnow().timestamp * 1000),
'status': "closed" if ordertype == "market" else "open",
'fee': None,
'info': {}
@@ -538,7 +540,7 @@ class Exchange:
amount, rate_for_order, params)
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
f'Tried to {side} amount {amount} at rate {rate}.'
f'Message: {e}') from e
@@ -1027,7 +1029,7 @@ class Exchange:
return order
@retrier(retries=5)
@retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:
try:
@@ -1056,6 +1058,27 @@ class Exchange:
# Assign method to fetch_stoploss_order to allow easy overriding in other classes
fetch_stoploss_order = fetch_order
def fetch_order_or_stoploss_order(self, order_id: str, pair: str,
stoploss_order: bool = False) -> Dict:
"""
Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
the stoploss_order parameter
:param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
"""
if stoploss_order:
return self.fetch_stoploss_order(order_id, pair)
return self.fetch_order(order_id, pair)
@staticmethod
def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]):
"""
Get next greater value in the list.
Used by fetch_l2_order_book if the api only supports a limited range
"""
if not limit_range:
return limit
return min([x for x in limit_range if limit <= x] + [max(limit_range)])
@retrier
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""
@@ -1064,9 +1087,10 @@ class Exchange:
Returns a dict in the format
{'asks': [price, volume], 'bids': [price, volume]}
"""
limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'])
try:
return self._api.fetch_l2_order_book(pair, limit)
return self._api.fetch_l2_order_book(pair, limit1)
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching order book.'

12
freqtrade/exchange/ftx.py
View File

@@ -4,11 +4,11 @@ from typing import Any, Dict
import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError,
InvalidOrderException, OperationalException,
TemporaryError)
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
logger = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ class Ftx(Exchange):
'stop price: %s.', pair, stop_price)
return order
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e
@@ -88,7 +88,7 @@ class Ftx(Exchange):
except ccxt.BaseError as e:
raise OperationalException(e) from e
@retrier(retries=5)
@retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:
try:

8
freqtrade/exchange/kraken.py
View File

@@ -4,12 +4,12 @@ from typing import Any, Dict
import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError,
InvalidOrderException, OperationalException,
TemporaryError)
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
logger = logging.getLogger(__name__)
@@ -98,7 +98,7 @@ class Kraken(Exchange):
'stop price: %s.', pair, stop_price)
return order
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e

214
freqtrade/freqtradebot.py
View File

@@ -12,17 +12,17 @@ from typing import Any, Dict, List, Optional
import arrow
from cachetools import TTLCache
from freqtrade import __version__, constants, persistence
from freqtrade import __version__, constants
from freqtrade.configuration import validate_config_consistency
from freqtrade.data.converter import order_book_to_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.exceptions import (DependencyException, ExchangeError,
from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
InvalidOrderException, PricingError)
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.misc import safe_value_fallback, safe_value_fallback2
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Trade
from freqtrade.persistence import Order, Trade, cleanup_db, init_db
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State
@@ -30,6 +30,7 @@ from freqtrade.strategy.interface import IStrategy, SellType
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
logger = logging.getLogger(__name__)
@@ -57,8 +58,8 @@ class FreqtradeBot:
# Cache values for 1800 to avoid frequent polling of the exchange for prices
# Caching only applies to RPC methods, so prices for open trades are still
# refreshed once every iteration.
self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800)
self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800)
self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)
@@ -67,7 +68,7 @@ class FreqtradeBot:
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
persistence.init(self.config.get('db_url', None), clean_open_orders=self.config['dry_run'])
init_db(self.config.get('db_url', None), clean_open_orders=self.config['dry_run'])
self.wallets = Wallets(self.config, self.exchange)
@@ -122,7 +123,7 @@ class FreqtradeBot:
self.check_for_open_trades()
self.rpc.cleanup()
persistence.cleanup()
cleanup_db()
def startup(self) -> None:
"""
@@ -134,6 +135,10 @@ class FreqtradeBot:
# Adjust stoploss if it was changed
Trade.stoploss_reinitialization(self.strategy.stoploss)
# Only update open orders on startup
# This will update the database after the initial migration
self.update_open_orders()
def process(self) -> None:
"""
Queries the persistence layer for open trades and handles them,
@@ -144,6 +149,8 @@ class FreqtradeBot:
# Check whether markets have to be reloaded and reload them when it's needed
self.exchange.reload_markets()
self.update_closed_trades_without_assigned_fees()
# Query trades from persistence layer
trades = Trade.get_open_trades()
@@ -227,6 +234,104 @@ class FreqtradeBot:
open_trades = len(Trade.get_open_trades())
return max(0, self.config['max_open_trades'] - open_trades)
def update_open_orders(self):
"""
Updates open orders based on order list kept in the database.
Mainly updates the state of orders - but may also close trades
"""
orders = Order.get_open_orders()
logger.info(f"Updating {len(orders)} open orders.")
for order in orders:
try:
fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
order.ft_order_side == 'stoploss')
self.update_trade_state(order.trade, order.order_id, fo)
except ExchangeError as e:
logger.warning(f"Error updating Order {order.order_id} due to {e}")
def update_closed_trades_without_assigned_fees(self):
"""
Update closed trades without close fees assigned.
Only acts when Orders are in the database, otherwise the last orderid is unknown.
"""
trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees()
for trade in trades:
if not trade.is_open and not trade.fee_updated('sell'):
# Get sell fee
order = trade.select_order('sell', False)
if order:
logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id,
stoploss_order=order.ft_order_side == 'stoploss')
trades: List[Trade] = Trade.get_open_trades_without_assigned_fees()
for trade in trades:
if trade.is_open and not trade.fee_updated('buy'):
order = trade.select_order('buy', False)
if order:
logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id)
def handle_insufficient_funds(self, trade: Trade):
"""
Determine if we ever opened a sell order for this trade.
If not, try update buy fees - otherwise "refind" the open order we obviously lost.
"""
sell_order = trade.select_order('sell', None)
if sell_order:
self.refind_lost_order(trade)
else:
self.reupdate_buy_order_fees(trade)
def reupdate_buy_order_fees(self, trade: Trade):
"""
Get buy order from database, and try to reupdate.
Handles trades where the initial fee-update did not work.
"""
logger.info(f"Trying to reupdate buy fees for {trade}")
order = trade.select_order('buy', False)
if order:
logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id)
def refind_lost_order(self, trade):
"""
Try refinding a lost trade.
Only used when InsufficientFunds appears on sell orders (stoploss or sell).
Tries to walk the stored orders and sell them off eventually.
"""
logger.info(f"Trying to refind lost order for {trade}")
for order in trade.orders:
logger.info(f"Trying to refind {order}")
fo = None
if not order.ft_is_open:
logger.debug(f"Order {order} is no longer open.")
continue
if order.ft_order_side == 'buy':
# Skip buy side - this is handled by reupdate_buy_order_fees
continue
try:
fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
order.ft_order_side == 'stoploss')
if order.ft_order_side == 'stoploss':
if fo and fo['status'] == 'open':
# Assume this as the open stoploss order
trade.stoploss_order_id = order.order_id
elif order.ft_order_side == 'sell':
if fo and fo['status'] == 'open':
# Assume this as the open order
trade.open_order_id = order.order_id
if fo:
logger.info(f"Found {order} for trade {trade}.jj")
self.update_trade_state(trade, order.order_id, fo,
stoploss_order=order.ft_order_side == 'stoploss')
except ExchangeError:
logger.warning(f"Error updating {order.order_id}.")
#
# BUY / enter positions / open trades logic and methods
#
@@ -528,6 +633,7 @@ class FreqtradeBot:
order = self.exchange.buy(pair=pair, ordertype=order_type,
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
order_id = order['id']
order_status = order.get('status', None)
@@ -556,7 +662,6 @@ class FreqtradeBot:
stake_amount = order['cost']
amount = safe_value_fallback(order, 'filled', 'amount')
buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
order_id = None
# in case of FOK the order may be filled immediately and fully
elif order_status == 'closed':
@@ -581,10 +686,11 @@ class FreqtradeBot:
strategy=self.strategy.get_strategy_name(),
timeframe=timeframe_to_minutes(self.config['timeframe'])
)
trade.orders.append(order_obj)
# Update fees if order is closed
if order_status == 'closed':
self.update_trade_state(trade, order)
self.update_trade_state(trade, order_id, order)
Trade.session.add(trade)
Trade.session.flush()
@@ -783,8 +889,16 @@ class FreqtradeBot:
stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount,
stop_price=stop_price,
order_types=self.strategy.order_types)
order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss')
trade.orders.append(order_obj)
trade.stoploss_order_id = str(stoploss_order['id'])
return True
except InsufficientFundsError as e:
logger.warning(f"Unable to place stoploss order {e}.")
# Try to figure out what went wrong
self.handle_insufficient_funds(trade)
except InvalidOrderException as e:
trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}')
@@ -814,13 +928,17 @@ class FreqtradeBot:
except InvalidOrderException as exception:
logger.warning('Unable to fetch stoploss order: %s', exception)
if stoploss_order:
trade.update_order(stoploss_order)
# We check if stoploss order is fulfilled
if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
self.update_trade_state(trade, stoploss_order, sl_order=True)
self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
stoploss_order=True)
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair,
timeframe_to_next_date(self.config['timeframe']))
self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe']),
reason='Auto lock')
self._notify_sell(trade, "stoploss")
return True
@@ -869,10 +987,11 @@ class FreqtradeBot:
update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
# cancelling the current stoploss on exchange first
logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) '
'in order to add another one ...', order['id'])
logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
f"(orderid:{order['id']}) in order to add another one ...")
try:
self.exchange.cancel_stoploss_order(order['id'], trade.pair)
co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
trade.update_order(co)
except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {order['id']} "
f"for pair {trade.pair}")
@@ -927,7 +1046,7 @@ class FreqtradeBot:
logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
continue
fully_cancelled = self.update_trade_state(trade, order)
fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order)
if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and (
fully_cancelled
@@ -995,8 +1114,7 @@ class FreqtradeBot:
if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
logger.info('Buy order fully cancelled. Removing %s from database.', trade)
# if trade is not partially completed, just delete the trade
Trade.session.delete(trade)
Trade.session.flush()
trade.delete()
was_trade_fully_canceled = True
reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
else:
@@ -1007,7 +1125,7 @@ class FreqtradeBot:
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = filled_amount
trade.stake_amount = trade.amount * trade.open_rate
self.update_trade_state(trade, corder, trade.amount)
self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None
logger.info('Partial buy order timeout for %s.', trade)
@@ -1121,23 +1239,33 @@ class FreqtradeBot:
logger.info(f"User requested abortion of selling {trade.pair}")
return False
# Execute sell and update trade record
order = self.exchange.sell(pair=str(trade.pair),
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=time_in_force
)
try:
# Execute sell and update trade record
order = self.exchange.sell(pair=trade.pair,
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=time_in_force
)
except InsufficientFundsError as e:
logger.warning(f"Unable to place order {e}.")
# Try to figure out what went wrong
self.handle_insufficient_funds(trade)
return False
order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell')
trade.orders.append(order_obj)
trade.open_order_id = order['id']
trade.close_rate_requested = limit
trade.sell_reason = sell_reason.value
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') == 'closed':
self.update_trade_state(trade, order)
self.update_trade_state(trade, trade.open_order_id, order)
Trade.session.flush()
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe']))
self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe']),
reason='Auto lock')
self._notify_sell(trade, order_type)
@@ -1230,30 +1358,35 @@ class FreqtradeBot:
# Common update trade state methods
#
def update_trade_state(self, trade: Trade, action_order: dict = None,
order_amount: float = None, sl_order: bool = False) -> bool:
def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None,
stoploss_order: bool = False) -> bool:
"""
Checks trades with open orders and updates the amount if necessary
Handles closing both buy and sell orders.
:param trade: Trade object of the trade we're analyzing
:param order_id: Order-id of the order we're analyzing
:param action_order: Already aquired order object
:return: True if order has been cancelled without being filled partially, False otherwise
"""
# Get order details for actual price per unit
if trade.open_order_id:
order_id = trade.open_order_id
elif trade.stoploss_order_id and sl_order:
order_id = trade.stoploss_order_id
else:
if not order_id:
logger.warning(f'Orderid for trade {trade} is empty.')
return False
# Update trade with order values
logger.info('Found open order for %s', trade)
try:
order = action_order or self.exchange.fetch_order(order_id, trade.pair)
order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id,
trade.pair,
stoploss_order)
except InvalidOrderException as exception:
logger.warning('Unable to fetch order %s: %s', order_id, exception)
return False
trade.update_order(order)
# Try update amount (binance-fix)
try:
new_amount = self.get_real_amount(trade, order, order_amount)
new_amount = self.get_real_amount(trade, order)
if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount,
abs_tol=constants.MATH_CLOSE_PREC):
order['amount'] = new_amount
@@ -1291,7 +1424,7 @@ class FreqtradeBot:
return real_amount
return amount
def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
def get_real_amount(self, trade: Trade, order: Dict) -> float:
"""
Detect and update trade fee.
Calls trade.update_fee() uppon correct detection.
@@ -1300,8 +1433,7 @@ class FreqtradeBot:
:return: identical (or new) amount for the trade
"""
# Init variables
if order_amount is None:
order_amount = safe_value_fallback(order, 'filled', 'amount')
order_amount = safe_value_fallback(order, 'filled', 'amount')
# Only run for closed orders
if trade.fee_updated(order.get('side', '')) or order['status'] == 'open':
return order_amount
@@ -1325,7 +1457,7 @@ class FreqtradeBot:
"""
fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
"""
trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair,
trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
trade.open_date)
if len(trades) == 0:

4
freqtrade/loggers.py
View File

@@ -1,12 +1,12 @@
import logging
import sys
from logging import Formatter
from logging.handlers import (BufferingHandler, RotatingFileHandler,
SysLogHandler)
from logging.handlers import BufferingHandler, RotatingFileHandler, SysLogHandler
from typing import Any, Dict
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

1
freqtrade/main.py
View File

@@ -7,6 +7,7 @@ import logging
import sys
from typing import Any, List
# check min. python version
if sys.version_info < (3, 6):
sys.exit("Freqtrade requires Python version >= 3.6")

13
freqtrade/misc.py
View File

@@ -12,6 +12,7 @@ from typing.io import IO
import numpy as np
import rapidjson
logger = logging.getLogger(__name__)
@@ -41,7 +42,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray:
return dates.dt.to_pydatetime()
def file_dump_json(filename: Path, data: Any, is_zip: bool = False) -> None:
def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None:
"""
Dump JSON data into a file
:param filename: file to create
@@ -52,12 +53,14 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False) -> None:
if is_zip:
if filename.suffix != '.gz':
filename = filename.with_suffix('.gz')
logger.info(f'dumping json to "{filename}"')
if log:
logger.info(f'dumping json to "{filename}"')
with gzip.open(filename, 'w') as fp:
rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)
with gzip.open(filename, 'w') as fpz:
rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
else:
logger.info(f'dumping json to "{filename}"')
if log:
logger.info(f'dumping json to "{filename}"')
with open(filename, 'w') as fp:
rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)

263
freqtrade/optimize/backtesting.py
View File

@@ -4,31 +4,39 @@
This module contains the backtesting logic
"""
import logging
from collections import defaultdict
from copy import deepcopy
from datetime import datetime, timedelta
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
import arrow
from pandas import DataFrame
from freqtrade.configuration import (TimeRange, remove_credentials,
validate_config_consistency)
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
show_backtest_results,
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_stats)
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
logger = logging.getLogger(__name__)
# Indexes for backtest tuples
DATE_IDX = 0
BUY_IDX = 1
OPEN_IDX = 2
CLOSE_IDX = 3
SELL_IDX = 4
LOW_IDX = 5
HIGH_IDX = 6
class BacktestResult(NamedTuple):
"""
@@ -116,7 +124,7 @@ class Backtesting:
"""
Load strategy into backtesting
"""
self.strategy = strategy
self.strategy: IStrategy = strategy
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
@@ -148,12 +156,14 @@ class Backtesting:
return data, timerange
def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]:
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
"""
Helper function to convert a processed dataframes into lists for performance reasons.
Used by backtest() - so keep this optimized for performance.
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
data: Dict = {}
# Create dict with data
@@ -173,10 +183,10 @@ class Backtesting:
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = [x for x in df_analyzed.itertuples()]
data[pair] = [x for x in df_analyzed.itertuples(index=False, name=None)]
return data
def _get_close_rate(self, sell_row, trade: Trade, sell: SellCheckTuple,
def _get_close_rate(self, sell_row: Tuple, trade: Trade, sell: SellCheckTuple,
trade_dur: int) -> float:
"""
Get close rate for backtesting result
@@ -187,12 +197,12 @@ class Backtesting:
return trade.stop_loss
elif sell.sell_type == (SellType.ROI):
roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur)
if roi is not None:
if roi is not None and roi_entry is not None:
if roi == -1 and roi_entry % self.timeframe_min == 0:
# When forceselling with ROI=-1, the roi time will always be equal to trade_dur.
# If that entry is a multiple of the timeframe (so on candle open)
# - we'll use open instead of close
return sell_row.open
return sell_row[OPEN_IDX]
# - (Expected abs profit + open_rate + open_fee) / (fee_close -1)
close_rate = - (trade.open_rate * roi + trade.open_rate *
@@ -200,91 +210,79 @@ class Backtesting:
if (trade_dur > 0 and trade_dur == roi_entry
and roi_entry % self.timeframe_min == 0
and sell_row.open > close_rate):
and sell_row[OPEN_IDX] > close_rate):
# new ROI entry came into effect.
# use Open rate if open_rate > calculated sell rate
return sell_row.open
return sell_row[OPEN_IDX]
# Use the maximum between close_rate and low as we
# cannot sell outside of a candle.
# Applies when a new ROI setting comes in place and the whole candle is above that.
return max(close_rate, sell_row.low)
return max(close_rate, sell_row[LOW_IDX])
else:
# This should not be reached...
return sell_row.open
return sell_row[OPEN_IDX]
else:
return sell_row.open
return sell_row[OPEN_IDX]
def _get_sell_trade_entry(
self, pair: str, buy_row: DataFrame,
partial_ohlcv: List, trade_count_lock: Dict,
stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]:
def _get_sell_trade_entry(self, trade: Trade, sell_row: Tuple) -> Optional[BacktestResult]:
trade = Trade(
pair=pair,
open_rate=buy_row.open,
open_date=buy_row.date,
stake_amount=stake_amount,
amount=round(stake_amount / buy_row.open, 8),
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
)
logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.")
# calculate win/lose forwards from buy point
for sell_row in partial_ohlcv:
if max_open_trades > 0:
# Increase trade_count_lock for every iteration
trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], sell_row[DATE_IDX],
sell_row[BUY_IDX], sell_row[SELL_IDX],
low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
if sell.sell_flag:
trade_dur = int((sell_row[DATE_IDX] - trade.open_date).total_seconds() // 60)
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, sell_row.buy,
sell_row.sell, low=sell_row.low, high=sell_row.high)
if sell.sell_flag:
trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60)
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_ratio(rate=closerate),
profit_abs=trade.calc_profit(rate=closerate),
open_date=buy_row.date,
open_rate=buy_row.open,
open_fee=self.fee,
close_date=sell_row.date,
close_rate=closerate,
close_fee=self.fee,
amount=trade.amount,
trade_duration=trade_dur,
open_at_end=False,
sell_reason=sell.sell_type
)
if partial_ohlcv:
# no sell condition found - trade stil open at end of backtest period
sell_row = partial_ohlcv[-1]
bt_res = BacktestResult(pair=pair,
profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open),
open_date=buy_row.date,
open_rate=buy_row.open,
open_fee=self.fee,
close_date=sell_row.date,
close_rate=sell_row.open,
close_fee=self.fee,
amount=trade.amount,
trade_duration=int((
sell_row.date - buy_row.date).total_seconds() // 60),
open_at_end=True,
sell_reason=SellType.FORCE_SELL
)
logger.debug(f"{pair} - Force selling still open trade, "
f"profit percent: {bt_res.profit_percent}, "
f"profit abs: {bt_res.profit_abs}")
return bt_res
return BacktestResult(pair=trade.pair,
profit_percent=trade.calc_profit_ratio(rate=closerate),
profit_abs=trade.calc_profit(rate=closerate),
open_date=trade.open_date,
open_rate=trade.open_rate,
open_fee=self.fee,
close_date=sell_row[DATE_IDX],
close_rate=closerate,
close_fee=self.fee,
amount=trade.amount,
trade_duration=trade_dur,
open_at_end=False,
sell_reason=sell.sell_type
)
return None
def handle_left_open(self, open_trades: Dict[str, List[Trade]],
data: Dict[str, List[Tuple]]) -> List[BacktestResult]:
"""
Handling of left open trades at the end of backtesting
"""
trades = []
for pair in open_trades.keys():
if len(open_trades[pair]) > 0:
for trade in open_trades[pair]:
sell_row = data[pair][-1]
trade_entry = BacktestResult(pair=trade.pair,
profit_percent=trade.calc_profit_ratio(
rate=sell_row[OPEN_IDX]),
profit_abs=trade.calc_profit(sell_row[OPEN_IDX]),
open_date=trade.open_date,
open_rate=trade.open_rate,
open_fee=self.fee,
close_date=sell_row[DATE_IDX],
close_rate=sell_row[OPEN_IDX],
close_fee=self.fee,
amount=trade.amount,
trade_duration=int((
sell_row[DATE_IDX] - trade.open_date
).total_seconds() // 60),
open_at_end=True,
sell_reason=SellType.FORCE_SELL
)
trades.append(trade_entry)
return trades
def backtest(self, processed: Dict, stake_amount: float,
start_date: arrow.Arrow, end_date: arrow.Arrow,
start_date: datetime, end_date: datetime,
max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame:
"""
Implement backtesting functionality
@@ -306,19 +304,21 @@ class Backtesting:
f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}"
)
trades = []
trade_count_lock: Dict = {}
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
lock_pair_until: Dict = {}
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = {}
tmp = start_date + timedelta(minutes=self.timeframe_min)
open_trades: Dict[str, List] = defaultdict(list)
open_trade_count = 0
# Loop timerange and get candle for each pair at that point in time
while tmp < end_date:
while tmp <= end_date:
open_trade_count_start = open_trade_count
for i, pair in enumerate(data):
if pair not in indexes:
@@ -332,42 +332,52 @@ class Backtesting:
continue
# Waits until the time-counter reaches the start of the data for this pair.
if row.date > tmp.datetime:
if row[DATE_IDX] > tmp:
continue
indexes[pair] += 1
if row.buy == 0 or row.sell == 1:
continue # skip rows where no buy signal or that would immediately sell off
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
# don't open on the last row
if ((position_stacking or len(open_trades[pair]) == 0)
and max_open_trades > 0 and open_trade_count_start < max_open_trades
and tmp != end_date
and row[BUY_IDX] == 1 and row[SELL_IDX] != 1):
# Enter trade
trade = Trade(
pair=pair,
open_rate=row[OPEN_IDX],
open_date=row[DATE_IDX],
stake_amount=stake_amount,
amount=round(stake_amount / row[OPEN_IDX], 8),
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
)
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behaviour - not sure if this is correct
# Prevents buying if the trade-slot was freed in this candle
open_trade_count_start += 1
open_trade_count += 1
# logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.")
open_trades[pair].append(trade)
if (not position_stacking and pair in lock_pair_until
and row.date <= lock_pair_until[pair]):
# without positionstacking, we can only have one open trade per pair.
continue
if max_open_trades > 0:
# Check if max_open_trades has already been reached for the given date
if not trade_count_lock.get(row.date, 0) < max_open_trades:
continue
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
# since indexes has been incremented before, we need to go one step back to
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(pair, row, data[pair][indexes[pair]-1:],
trade_count_lock, stake_amount,
max_open_trades)
if trade_entry:
logger.debug(f"{pair} - Locking pair till "
f"close_date={trade_entry.close_date}")
lock_pair_until[pair] = trade_entry.close_date
trades.append(trade_entry)
else:
# Set lock_pair_until to end of testing period if trade could not be closed
lock_pair_until[pair] = end_date.datetime
for trade in open_trades[pair]:
# since indexes has been incremented before, we need to go one step back to
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(trade, row)
# Sell occured
if trade_entry:
# logger.debug(f"{pair} - Backtesting sell {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
trades.append(trade_entry)
# Move time one configured time_interval ahead.
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
return DataFrame.from_records(trades, columns=BacktestResult._fields)
def start(self) -> None:
@@ -380,12 +390,6 @@ class Backtesting:
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
# Use max_open_trades in backtesting, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
max_open_trades = self.config['max_open_trades']
else:
logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
max_open_trades = 0
position_stacking = self.config.get('position_stacking', False)
data, timerange = self.load_bt_data()
@@ -395,6 +399,15 @@ class Backtesting:
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
self._set_strategy(strat)
# Use max_open_trades in backtesting, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
# Must come from strategy config, as the strategy may modify this setting.
max_open_trades = self.strategy.config['max_open_trades']
else:
logger.info(
'Ignoring max_open_trades (--disable-max-market-positions was used) ...')
max_open_trades = 0
# need to reprocess data every time to populate signals
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
@@ -407,17 +420,21 @@ class Backtesting:
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
# Execute backtest and print results
all_results[self.strategy.get_strategy_name()] = self.backtest(
results = self.backtest(
processed=preprocessed,
stake_amount=self.config['stake_amount'],
start_date=min_date,
end_date=max_date,
start_date=min_date.datetime,
end_date=max_date.datetime,
max_open_trades=max_open_trades,
position_stacking=position_stacking,
)
all_results[self.strategy.get_strategy_name()] = {
'results': results,
'config': self.strategy.config,
}
stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date)
stats = generate_backtest_stats(self.config, data, all_results,
min_date=min_date, max_date=max_date)
if self.config.get('export', False):
store_backtest_stats(self.config['exportfilename'], stats)

8
freqtrade/optimize/default_hyperopt_loss.py
View File

@@ -1,5 +1,5 @@
"""
DefaultHyperOptLoss
ShortTradeDurHyperOptLoss
This module defines the default HyperoptLoss class which is being used for
Hyperoptimization.
"""
@@ -26,7 +26,7 @@ EXPECTED_MAX_PROFIT = 3.0
MAX_ACCEPTED_TRADE_DURATION = 300
class DefaultHyperOptLoss(IHyperOptLoss):
class ShortTradeDurHyperOptLoss(IHyperOptLoss):
"""
Defines the default loss function for hyperopt
"""
@@ -50,3 +50,7 @@ class DefaultHyperOptLoss(IHyperOptLoss):
duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1)
result = trade_loss + profit_loss + duration_loss
return result
# Create an alias for This to allow the legacy Method to work as well.
DefaultHyperOptLoss = ShortTradeDurHyperOptLoss

4
freqtrade/optimize/edge_cli.py
View File

@@ -7,12 +7,12 @@ import logging
from typing import Any, Dict
from freqtrade import constants
from freqtrade.configuration import (TimeRange, remove_credentials,
validate_config_consistency)
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.edge import Edge
from freqtrade.optimize.optimize_reports import generate_edge_table
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
logger = logging.getLogger(__name__)

70
freqtrade/optimize/hyperopt.py
View File

@@ -10,6 +10,7 @@ import logging
import random
import warnings
from collections import OrderedDict
from datetime import datetime
from math import ceil
from operator import itemgetter
from pathlib import Path
@@ -21,24 +22,22 @@ import rapidjson
import tabulate
from colorama import Fore, Style
from colorama import init as colorama_init
from joblib import (Parallel, cpu_count, delayed, dump, load,
wrap_non_picklable_objects)
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
from pandas import DataFrame, isna, json_normalize
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.history import get_timerange
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural, round_dict
from freqtrade.misc import file_dump_json, plural, round_dict
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import \
IHyperOptLoss # noqa: F401
from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
HyperOptResolver)
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
from freqtrade.strategy import IStrategy
# Suppress scikit-learn FutureWarnings from skopt
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
@@ -77,19 +76,16 @@ class Hyperopt:
self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config)
self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self.results_file = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
'hyperopt_results' / f'hyperopt_results_{time_now}.pickle')
self.data_pickle_file = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_tickerdata.pkl')
self.total_epochs = config.get('epochs', 0)
self.current_best_loss = 100
if not self.config.get('hyperopt_continue'):
self.clean_hyperopt()
else:
logger.info("Continuing on previous hyperopt results.")
self.clean_hyperopt()
self.num_epochs_saved = 0
@@ -98,14 +94,14 @@ class Hyperopt:
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = \
self.custom_hyperopt.populate_indicators # type: ignore
self.backtesting.strategy.advise_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
self.backtesting.strategy.advise_buy = \
self.custom_hyperopt.populate_buy_trend # type: ignore
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.backtesting.strategy.advise_sell = \
self.custom_hyperopt.populate_sell_trend # type: ignore
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
@@ -165,6 +161,10 @@ class Hyperopt:
self.num_epochs_saved = num_epochs
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
# Store hyperopt filename
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
log=False)
@staticmethod
def _read_results(results_file: Path) -> List:
@@ -262,6 +262,11 @@ class Hyperopt:
),
default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
params_result += f"minimal_roi = {minimal_roi_result}"
elif space == 'trailing':
for k, v in space_params.items():
params_result += f'{k} = {v}\n'
else:
params_result += f"{space}_params = {pformat(space_params, indent=4)}"
params_result = params_result.replace("}", "\n}").replace("{", "{\n ")
@@ -324,8 +329,9 @@ class Hyperopt:
'results_metrics.avg_profit', 'results_metrics.total_profit',
'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', 'W/D/L', 'Avg profit', 'Total profit',
'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'Objective',
'is_initial_point', 'is_best']
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '* '
trials.loc[trials['is_best'], 'Best'] = 'Best'
@@ -502,16 +508,16 @@ class Hyperopt:
params_details = self._get_params_details(params_dict)
if self.has_space('roi'):
self.backtesting.strategy.minimal_roi = \
self.custom_hyperopt.generate_roi_table(params_dict)
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.custom_hyperopt.generate_roi_table(params_dict))
if self.has_space('buy'):
self.backtesting.strategy.advise_buy = \
self.custom_hyperopt.buy_strategy_generator(params_dict)
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.buy_strategy_generator(params_dict))
if self.has_space('sell'):
self.backtesting.strategy.advise_sell = \
self.custom_hyperopt.sell_strategy_generator(params_dict)
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.sell_strategy_generator(params_dict))
if self.has_space('stoploss'):
self.backtesting.strategy.stoploss = params_dict['stoploss']
@@ -532,8 +538,8 @@ class Hyperopt:
backtesting_results = self.backtesting.backtest(
processed=processed,
stake_amount=self.config['stake_amount'],
start_date=min_date,
end_date=max_date,
start_date=min_date.datetime,
end_date=max_date.datetime,
max_open_trades=self.max_open_trades,
position_stacking=self.position_stacking,
)
@@ -574,7 +580,7 @@ class Hyperopt:
'wins': wins,
'draws': draws,
'losses': losses,
'winsdrawslosses': f"{wins}/{draws}/{losses}",
'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}",
'avg_profit': backtesting_results.profit_percent.mean() * 100.0,
'median_profit': backtesting_results.profit_percent.median() * 100.0,
'total_profit': backtesting_results.profit_abs.sum(),
@@ -656,8 +662,6 @@ class Hyperopt:
self.backtesting.strategy.dp = None # type: ignore
IStrategy.dp = None # type: ignore
self.epochs = self.load_previous_results(self.results_file)
cpus = cpu_count()
logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
config_jobs = self.config.get('hyperopt_jobs', -1)

1
freqtrade/optimize/hyperopt_interface.py
View File

@@ -13,6 +13,7 @@ from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import round_dict
logger = logging.getLogger(__name__)

2
freqtrade/optimize/hyperopt_loss_sharpe.py
View File

@@ -6,8 +6,8 @@ Hyperoptimization.
"""
from datetime import datetime
from pandas import DataFrame
import numpy as np
from pandas import DataFrame
from freqtrade.optimize.hyperopt import IHyperOptLoss

2
freqtrade/optimize/hyperopt_loss_sortino.py
View File

@@ -6,8 +6,8 @@ Hyperoptimization.
"""
from datetime import datetime
from pandas import DataFrame
import numpy as np
from pandas import DataFrame
from freqtrade.optimize.hyperopt import IHyperOptLoss

53
freqtrade/optimize/optimize_reports.py
View File

@@ -1,17 +1,18 @@
import logging
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Union
from arrow import Arrow
from pandas import DataFrame
from numpy import int64
from pandas import DataFrame
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.data.btanalysis import calculate_max_drawdown, calculate_market_change
from freqtrade.data.btanalysis import calculate_market_change, calculate_max_drawdown
from freqtrade.misc import file_dump_json
logger = logging.getLogger(__name__)
@@ -122,7 +123,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
profit_mean = result['profit_percent'].mean()
profit_sum = result["profit_percent"].sum()
profit_percent_tot = round(result['profit_percent'].sum() * 100.0 / max_open_trades, 2)
profit_percent_tot = result['profit_percent'].sum() / max_open_trades
tabular_data.append(
{
@@ -136,25 +137,25 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
'profit_sum': profit_sum,
'profit_sum_pct': round(profit_sum * 100, 2),
'profit_total_abs': result['profit_abs'].sum(),
'profit_total_pct': profit_percent_tot,
'profit_total': profit_percent_tot,
'profit_total_pct': round(profit_percent_tot * 100, 2),
}
)
return tabular_data
def generate_strategy_metrics(stake_currency: str, max_open_trades: int,
all_results: Dict) -> List[Dict]:
def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
"""
Generate summary per strategy
:param stake_currency: stake-currency - used to correctly name headers
:param max_open_trades: Maximum allowed open trades used for backtest
:param all_results: Dict of <Strategyname: BacktestResult> containing results for all strategies
:return: List of Dicts containing the metrics per Strategy
"""
tabular_data = []
for strategy, results in all_results.items():
tabular_data.append(_generate_result_line(results, max_open_trades, strategy))
tabular_data.append(_generate_result_line(
results['results'], results['config']['max_open_trades'], strategy)
)
return tabular_data
@@ -218,25 +219,29 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
}
def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
all_results: Dict[str, DataFrame],
def generate_backtest_stats(btdata: Dict[str, DataFrame],
all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
min_date: Arrow, max_date: Arrow
) -> Dict[str, Any]:
"""
:param config: Configuration object used for backtest
:param btdata: Backtest data
:param all_results: backtest result - dictionary with { Strategy: results}.
:param all_results: backtest result - dictionary in the form:
{ Strategy: {'results: results, 'config: config}}.
:param min_date: Backtest start date
:param max_date: Backtest end date
:return:
Dictionary containing results per strategy and a stratgy summary.
"""
stake_currency = config['stake_currency']
max_open_trades = config['max_open_trades']
result: Dict[str, Any] = {'strategy': {}}
market_change = calculate_market_change(btdata, 'close')
for strategy, results in all_results.items():
for strategy, content in all_results.items():
results: Dict[str, DataFrame] = content['results']
if not isinstance(results, DataFrame):
continue
config = content['config']
max_open_trades = config['max_open_trades']
stake_currency = config['stake_currency']
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
max_open_trades=max_open_trades,
@@ -276,6 +281,16 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
# Parameters relevant for backtesting
'stoploss': config['stoploss'],
'trailing_stop': config.get('trailing_stop', False),
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
'minimal_roi': config['minimal_roi'],
'use_sell_signal': config['ask_strategy']['use_sell_signal'],
'sell_profit_only': config['ask_strategy']['sell_profit_only'],
'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
**daily_stats,
}
result['strategy'][strategy] = strat_stats
@@ -299,9 +314,7 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
'drawdown_end_ts': 0,
})
strategy_results = generate_strategy_metrics(stake_currency=stake_currency,
max_open_trades=max_open_trades,
all_results=all_results)
strategy_results = generate_strategy_metrics(all_results=all_results)
result['strategy_comparison'] = strategy_results

3
freqtrade/pairlist/AgeFilter.py
View File

@@ -2,9 +2,10 @@
Minimum age (days listed) pair list filter
"""
import logging
import arrow
from typing import Any, Dict
import arrow
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.pairlist.IPairList import IPairList

2
freqtrade/pairlist/IPairList.py
View File

@@ -36,7 +36,7 @@ class IPairList(ABC):
self._pairlist_pos = pairlist_pos
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._last_refresh = 0
self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period)
self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period)
@property
def name(self) -> str:

3
freqtrade/pairlist/PrecisionFilter.py
View File

@@ -4,8 +4,9 @@ Precision pair list filter
import logging
from typing import Any, Dict
from freqtrade.pairlist.IPairList import IPairList
from freqtrade.exceptions import OperationalException
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)

2
freqtrade/pairlist/pairlistmanager.py
View File

@@ -7,10 +7,10 @@ from typing import Dict, List
from cachetools import TTLCache, cached
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.pairlist.IPairList import IPairList
from freqtrade.resolvers import PairListResolver
from freqtrade.constants import ListPairsWithTimeframes
logger = logging.getLogger(__name__)

4
freqtrade/persistence/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# flake8: noqa: F401
from freqtrade.persistence.models import (Order, PairLock, Trade, clean_dry_run_db, cleanup_db,
init_db)

150
freqtrade/persistence/migrations.py Normal file
View File

@@ -0,0 +1,150 @@
import logging
from typing import List
from sqlalchemy import inspect
logger = logging.getLogger(__name__)
def get_table_names_for_table(inspector, tabletype):
return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
def has_column(columns: List, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def get_backup_name(tabs, backup_prefix: str):
table_back_name = backup_prefix
for i, table_back_name in enumerate(tabs):
table_back_name = f'{backup_prefix}{i}'
logger.debug(f'trying {table_back_name}')
return table_back_name
def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
fee_close = get_column_def(cols, 'fee_close', 'fee')
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
else:
timeframe = get_column_def(cols, 'timeframe', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
timeframe, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
def migrate_open_orders_to_trades(engine):
engine.execute("""
insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
select id ft_trade_id, pair ft_pair, open_order_id,
case when close_rate_requested is null then 'buy'
else 'sell' end ft_order_side, 1 ft_is_open
from trades
where open_order_id is not null
union all
select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
'stoploss' ft_order_side, 1 ft_is_open
from trades
where stoploss_order_id is not null
""")
def check_migrate(engine, decl_base, previous_tables) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = get_table_names_for_table(inspector, 'trades')
table_back_name = get_backup_name(tabs, 'trades_bak')
# Check for latest column
if not has_column(cols, 'amount_requested'):
logger.info(f'Running database migration for trades - backup: {table_back_name}')
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
if 'orders' not in previous_tables:
logger.info('Moving open orders to Orders table.')
migrate_open_orders_to_trades(engine)
else:
pass
# Empty for now - as there is only one iteration of the orders table so far.
# table_back_name = get_backup_name(tabs, 'orders_bak')

405
freqtrade/persistence.py → freqtrade/persistence/models.py
View File

@@ -7,17 +7,21 @@ from decimal import Decimal
from typing import Any, Dict, List, Optional
import arrow
from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
create_engine, desc, func, inspect)
from sqlalchemy.exc import NoSuchModuleError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Query
from sqlalchemy.orm import Query, relationship
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.schema import UniqueConstraint
from freqtrade.exceptions import OperationalException
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.misc import safe_value_fallback
from freqtrade.persistence.migrations import check_migrate
logger = logging.getLogger(__name__)
@@ -26,7 +30,7 @@ _DECL_BASE: Any = declarative_base()
_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls'
def init(db_url: str, clean_open_orders: bool = False) -> None:
def init_db(db_url: str, clean_open_orders: bool = False) -> None:
"""
Initializes this module with the given config,
registers all known command handlers
@@ -57,122 +61,22 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
# We should use the scoped_session object - not a seperately initialized version
Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.query = Trade.session.query_property()
# Copy session attributes to order object too
Order.session = Trade.session
Order.query = Order.session.query_property()
PairLock.session = Trade.session
PairLock.query = PairLock.session.query_property()
previous_tables = inspect(engine).get_table_names()
_DECL_BASE.metadata.create_all(engine)
check_migrate(engine)
check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
# Clean dry_run DB if the db is not in-memory
if clean_open_orders and db_url != 'sqlite://':
clean_dry_run_db()
def has_column(columns: List, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def check_migrate(engine) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = inspector.get_table_names()
table_back_name = 'trades_bak'
for i, table_back_name in enumerate(tabs):
table_back_name = f'trades_bak{i}'
logger.debug(f'trying {table_back_name}')
# Check for latest column
if not has_column(cols, 'amount_requested'):
logger.info(f'Running database migration - backup available as {table_back_name}')
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
fee_close = get_column_def(cols, 'fee_close', 'fee')
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
else:
timeframe = get_column_def(cols, 'timeframe', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
_DECL_BASE.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
timeframe, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
def cleanup() -> None:
def cleanup_db() -> None:
"""
Flushes all pending operations to disk.
:return: None
@@ -191,13 +95,117 @@ def clean_dry_run_db() -> None:
trade.open_order_id = None
class Order(_DECL_BASE):
"""
Order database model
Keeps a record of all orders placed on the exchange
One to many relationship with Trades:
- One trade can have many orders
- One Order can only be associated with one Trade
Mirrors CCXT Order structure
"""
__tablename__ = 'orders'
# Uniqueness should be ensured over pair, order_id
# its likely that order_id is unique per Pair on some exchanges.
__table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
id = Column(Integer, primary_key=True)
ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
trade = relationship("Trade", back_populates="orders")
ft_order_side = Column(String, nullable=False)
ft_pair = Column(String, nullable=False)
ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
order_id = Column(String, nullable=False, index=True)
status = Column(String, nullable=True)
symbol = Column(String, nullable=True)
order_type = Column(String, nullable=True)
side = Column(String, nullable=True)
price = Column(Float, nullable=True)
amount = Column(Float, nullable=True)
filled = Column(Float, nullable=True)
remaining = Column(Float, nullable=True)
cost = Column(Float, nullable=True)
order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
order_filled_date = Column(DateTime, nullable=True)
order_update_date = Column(DateTime, nullable=True)
def __repr__(self):
return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
f'side={self.side}, order_type={self.order_type}, status={self.status})')
def update_from_ccxt_object(self, order):
"""
Update Order from ccxt response
Only updates if fields are available from ccxt -
"""
if self.order_id != str(order['id']):
raise DependencyException("Order-id's don't match")
self.status = order.get('status', self.status)
self.symbol = order.get('symbol', self.symbol)
self.order_type = order.get('type', self.order_type)
self.side = order.get('side', self.side)
self.price = order.get('price', self.price)
self.amount = order.get('amount', self.amount)
self.filled = order.get('filled', self.filled)
self.remaining = order.get('remaining', self.remaining)
self.cost = order.get('cost', self.cost)
if 'timestamp' in order and order['timestamp'] is not None:
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
self.ft_is_open = True
if self.status in ('closed', 'canceled', 'cancelled'):
self.ft_is_open = False
if order.get('filled', 0) > 0:
self.order_filled_date = arrow.utcnow().datetime
self.order_update_date = arrow.utcnow().datetime
@staticmethod
def update_orders(orders: List['Order'], order: Dict[str, Any]):
"""
Get all non-closed orders - useful when trying to batch-update orders
"""
filtered_orders = [o for o in orders if o.order_id == order.get('id')]
if filtered_orders:
oobj = filtered_orders[0]
oobj.update_from_ccxt_object(order)
else:
logger.warning(f"Did not find order for {order}.")
@staticmethod
def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order':
"""
Parse an order from a ccxt object and return a new order Object.
"""
o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair)
o.update_from_ccxt_object(order)
return o
@staticmethod
def get_open_orders() -> List['Order']:
"""
"""
return Order.query.filter(Order.ft_is_open.is_(True)).all()
class Trade(_DECL_BASE):
"""
Class used to define a trade structure
Trade database model.
Also handles updating and querying trades
"""
__tablename__ = 'trades'
id = Column(Integer, primary_key=True)
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
exchange = Column(String, nullable=False)
pair = Column(String, nullable=False, index=True)
is_open = Column(Boolean, nullable=False, default=True, index=True)
@@ -247,7 +255,7 @@ class Trade(_DECL_BASE):
self.recalc_open_trade_price()
def __repr__(self):
open_since = self.open_date.strftime('%Y-%m-%d %H:%M:%S') if self.is_open else 'closed'
open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
f'open_rate={self.open_rate:.8f}, open_since={open_since})')
@@ -273,7 +281,7 @@ class Trade(_DECL_BASE):
'fee_close_currency': self.fee_close_currency,
'open_date_hum': arrow.get(self.open_date).humanize(),
'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
'open_rate': self.open_rate,
'open_rate_requested': self.open_rate_requested,
@@ -281,7 +289,7 @@ class Trade(_DECL_BASE):
'close_date_hum': (arrow.get(self.close_date).humanize()
if self.close_date else None),
'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
if self.close_date else None),
'close_timestamp': int(self.close_date.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
@@ -297,7 +305,7 @@ class Trade(_DECL_BASE):
'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None,
'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
'stoploss_order_id': self.stoploss_order_id,
'stoploss_last_update': (self.stoploss_last_update.strftime("%Y-%m-%d %H:%M:%S")
'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT)
if self.stoploss_last_update else None),
'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
@@ -380,19 +388,22 @@ class Trade(_DECL_BASE):
self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price'))
self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount'))
self.recalc_open_trade_price()
logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
if self.is_open:
logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.')
self.open_order_id = None
elif order_type in ('market', 'limit') and order['side'] == 'sell':
if self.is_open:
logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.')
self.close(safe_value_fallback(order, 'average', 'price'))
logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self)
elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'):
self.stoploss_order_id = None
self.close_rate_requested = self.stop_loss
logger.info('%s is hit for %s.', order_type.upper(), self)
if self.is_open:
logger.info(f'{order_type.upper()} is hit for {self}.')
self.close(order['average'])
else:
raise ValueError(f'Unknown order type: {order_type}')
cleanup()
cleanup_db()
def close(self, rate: float) -> None:
"""
@@ -402,7 +413,7 @@ class Trade(_DECL_BASE):
self.close_rate = Decimal(rate)
self.close_profit = self.calc_profit_ratio()
self.close_profit_abs = self.calc_profit()
self.close_date = datetime.utcnow()
self.close_date = self.close_date or datetime.utcnow()
self.is_open = False
self.sell_order_status = 'closed'
self.open_order_id = None
@@ -440,6 +451,17 @@ class Trade(_DECL_BASE):
else:
return False
def update_order(self, order: Dict) -> None:
Order.update_orders(self.orders, order)
def delete(self) -> None:
for order in self.orders:
Order.session.delete(order)
Trade.session.delete(self)
Trade.session.flush()
def _calc_open_trade_price(self) -> float:
"""
Calculate the open_rate including open_fee.
@@ -506,6 +528,21 @@ class Trade(_DECL_BASE):
profit_ratio = (close_trade_price / self.open_trade_price) - 1
return float(f"{profit_ratio:.8f}")
def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]:
"""
Finds latest order for this orderside and status
:param order_side: Side of the order (either 'buy' or 'sell')
:param is_open: Only search for open orders?
:return: latest Order object if it exists, else None
"""
orders = [o for o in self.orders if o.side == order_side]
if is_open is not None:
orders = [o for o in orders if o.ft_is_open == is_open]
if len(orders) > 0:
return orders[-1]
else:
return None
@staticmethod
def get_trades(trade_filter=None) -> Query:
"""
@@ -537,6 +574,26 @@ class Trade(_DECL_BASE):
"""
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def get_open_trades_without_assigned_fees():
"""
Returns all open trades which don't have open fees set correctly
"""
return Trade.get_trades([Trade.fee_open_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(True),
]).all()
@staticmethod
def get_sold_trades_without_assigned_fees():
"""
Returns all closed trades which don't have fees set correctly
"""
return Trade.get_trades([Trade.fee_close_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(False),
]).all()
@staticmethod
def total_open_trades_stakes() -> float:
"""
@@ -601,3 +658,105 @@ class Trade(_DECL_BASE):
trade.stop_loss = None
trade.adjust_stop_loss(trade.open_rate, desired_stoploss)
logger.info(f"New stoploss: {trade.stop_loss}.")
class PairLock(_DECL_BASE):
"""
Pair Locks database model.
"""
__tablename__ = 'pairlocks'
id = Column(Integer, primary_key=True)
pair = Column(String, nullable=False, index=True)
reason = Column(String, nullable=True)
# Time the pair was locked (start time)
lock_time = Column(DateTime, nullable=False)
# Time until the pair is locked (end time)
lock_end_time = Column(DateTime, nullable=False, index=True)
active = Column(Boolean, nullable=False, default=True, index=True)
def __repr__(self):
lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, '
f'lock_end_time={lock_end_time})')
@staticmethod
def lock_pair(pair: str, until: datetime, reason: str = None) -> None:
lock = PairLock(
pair=pair,
lock_time=datetime.now(timezone.utc),
lock_end_time=until,
reason=reason,
active=True
)
PairLock.session.add(lock)
PairLock.session.flush()
@staticmethod
def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List['PairLock']:
"""
Get all locks for this pair
:param pair: Pair to check for. Returns all current locks if pair is empty
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.utcnow()
"""
if not now:
now = datetime.now(timezone.utc)
filters = [func.datetime(PairLock.lock_end_time) >= now,
# Only active locks
PairLock.active.is_(True), ]
if pair:
filters.append(PairLock.pair == pair)
return PairLock.query.filter(
*filters
).all()
@staticmethod
def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
"""
Release all locks for this pair.
:param pair: Pair to unlock
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.utcnow()
"""
if not now:
now = datetime.now(timezone.utc)
logger.info(f"Releasing all locks for {pair}.")
locks = PairLock.get_pair_locks(pair, now)
for lock in locks:
lock.active = False
PairLock.session.flush()
@staticmethod
def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
"""
:param pair: Pair to check for
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.utcnow()
"""
if not now:
now = datetime.now(timezone.utc)
return PairLock.query.filter(
PairLock.pair == pair,
func.datetime(PairLock.lock_end_time) >= now,
# Only active locks
PairLock.active.is_(True),
).scalar() is not None
def to_json(self) -> Dict[str, Any]:
return {
'pair': self.pair,
'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT),
'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000),
'lock_end_time': self.lock_end_time.strftime(DATETIME_PRINT_FORMAT),
'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc
).timestamp() * 1000),
'reason': self.reason,
'active': self.active,
}

12
freqtrade/plot/plotting.py
View File

@@ -5,11 +5,8 @@ from typing import Any, Dict, List
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.data.btanalysis import (calculate_max_drawdown,
combine_dataframes_with_mean,
create_cum_profit,
extract_trades_of_period,
load_trades)
from freqtrade.data.btanalysis import (calculate_max_drawdown, combine_dataframes_with_mean,
create_cum_profit, extract_trades_of_period, load_trades)
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import load_data
@@ -19,13 +16,14 @@ from freqtrade.misc import pair_to_filename
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.strategy import IStrategy
logger = logging.getLogger(__name__)
try:
from plotly.subplots import make_subplots
from plotly.offline import plot
import plotly.graph_objects as go
from plotly.offline import plot
from plotly.subplots import make_subplots
except ImportError:
logger.exception("Module plotly not found \n Please install using `pip3 install plotly`")
exit(1)

16
freqtrade/resolvers/__init__.py
View File

@@ -1,6 +1,12 @@
from freqtrade.resolvers.iresolver import IResolver # noqa: F401
from freqtrade.resolvers.exchange_resolver import ExchangeResolver # noqa: F401
# flake8: noqa: F401
# isort: off
from freqtrade.resolvers.iresolver import IResolver
from freqtrade.resolvers.exchange_resolver import ExchangeResolver
# isort: on
# Don't import HyperoptResolver to avoid loading the whole Optimize tree
# from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401
from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401
from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401
# from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver
from freqtrade.resolvers.pairlist_resolver import PairListResolver
from freqtrade.resolvers.strategy_resolver import StrategyResolver

3
freqtrade/resolvers/exchange_resolver.py
View File

@@ -3,10 +3,11 @@ This module loads custom exchanges
"""
import logging
from freqtrade.exchange import Exchange, MAP_EXCHANGE_CHILDCLASS
import freqtrade.exchange as exchanges
from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, Exchange
from freqtrade.resolvers import IResolver
logger = logging.getLogger(__name__)

15
freqtrade/resolvers/hyperopt_resolver.py
View File

@@ -7,12 +7,13 @@ import logging
from pathlib import Path
from typing import Dict
from freqtrade.constants import DEFAULT_HYPEROPT_LOSS, USERPATH_HYPEROPTS
from freqtrade.constants import USERPATH_HYPEROPTS
from freqtrade.exceptions import OperationalException
from freqtrade.optimize.hyperopt_interface import IHyperOpt
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss
from freqtrade.resolvers import IResolver
logger = logging.getLogger(__name__)
@@ -69,10 +70,10 @@ class HyperOptLossResolver(IResolver):
:param config: configuration dictionary
"""
# Verify the hyperopt_loss is in the configuration, otherwise fallback to the
# default hyperopt loss
hyperoptloss_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS
hyperoptloss_name = config.get('hyperopt_loss')
if not hyperoptloss_name:
raise OperationalException("No Hyperopt loss set. Please use `--hyperopt-loss` to "
"specify the Hyperopt-Loss class to use.")
hyperoptloss = HyperOptLossResolver.load_object(hyperoptloss_name,
config, kwargs={},
extra_dir=config.get('hyperopt_path'))
@@ -81,8 +82,4 @@ class HyperOptLossResolver(IResolver):
hyperoptloss.__class__.ticker_interval = str(config['timeframe'])
hyperoptloss.__class__.timeframe = str(config['timeframe'])
if not hasattr(hyperoptloss, 'hyperopt_loss_function'):
raise OperationalException(
f"Found HyperoptLoss class {hyperoptloss_name} does not "
"implement `hyperopt_loss_function`.")
return hyperoptloss

38
freqtrade/resolvers/iresolver.py
View File

@@ -11,6 +11,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
@@ -50,7 +51,8 @@ class IResolver:
:param object_name: Class name of the object
:param enum_failed: If True, will return None for modules which fail.
Otherwise, failing modules are skipped.
:return: generator containing matching objects
:return: generator containing tuple of matching objects
Tuple format: [Object, source]
"""
# Generate spec based on absolute path
@@ -66,14 +68,16 @@ class IResolver:
return iter([None])
valid_objects_gen = (
obj for name, obj in inspect.getmembers(module, inspect.isclass)
if ((object_name is None or object_name == name) and
issubclass(obj, cls.object_type) and obj is not cls.object_type)
(obj, inspect.getsource(module)) for
name, obj in inspect.getmembers(
module, inspect.isclass) if ((object_name is None or object_name == name)
and issubclass(obj, cls.object_type)
and obj is not cls.object_type)
)
return valid_objects_gen
@classmethod
def _search_object(cls, directory: Path, object_name: str
def _search_object(cls, directory: Path, *, object_name: str, add_source: bool = False
) -> Union[Tuple[Any, Path], Tuple[None, None]]:
"""
Search for the objectname in the given directory
@@ -92,11 +96,14 @@ class IResolver:
obj = next(cls._get_valid_object(module_path, object_name), None)
if obj:
return (obj, module_path)
obj[0].__file__ = str(entry)
if add_source:
obj[0].__source__ = obj[1]
return (obj[0], module_path)
return (None, None)
@classmethod
def _load_object(cls, paths: List[Path], object_name: str,
def _load_object(cls, paths: List[Path], *, object_name: str, add_source: bool = False,
kwargs: dict = {}) -> Optional[Any]:
"""
Try to load object from path list.
@@ -105,7 +112,8 @@ class IResolver:
for _path in paths:
try:
(module, module_path) = cls._search_object(directory=_path,
object_name=object_name)
object_name=object_name,
add_source=add_source)
if module:
logger.info(
f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} "
@@ -117,7 +125,7 @@ class IResolver:
return None
@classmethod
def load_object(cls, object_name: str, config: dict, kwargs: dict,
def load_object(cls, object_name: str, config: dict, *, kwargs: dict,
extra_dir: Optional[str] = None) -> Any:
"""
Search and loads the specified object as configured in hte child class.
@@ -132,10 +140,10 @@ class IResolver:
user_subdir=cls.user_subdir,
extra_dir=extra_dir)
pairlist = cls._load_object(paths=abs_paths, object_name=object_name,
kwargs=kwargs)
if pairlist:
return pairlist
found_object = cls._load_object(paths=abs_paths, object_name=object_name,
kwargs=kwargs)
if found_object:
return found_object
raise OperationalException(
f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist "
"or contains Python code errors."
@@ -163,8 +171,8 @@ class IResolver:
for obj in cls._get_valid_object(module_path, object_name=None,
enum_failed=enum_failed):
objects.append(
{'name': obj.__name__ if obj is not None else '',
'class': obj,
{'name': obj[0].__name__ if obj is not None else '',
'class': obj[0] if obj is not None else None,
'location': entry,
})
return objects

1
freqtrade/resolvers/pairlist_resolver.py
View File

@@ -9,6 +9,7 @@ from pathlib import Path
from freqtrade.pairlist.IPairList import IPairList
from freqtrade.resolvers import IResolver
logger = logging.getLogger(__name__)

8
freqtrade/resolvers/strategy_resolver.py
View File

@@ -11,12 +11,12 @@ from inspect import getfullargspec
from pathlib import Path
from typing import Any, Dict, Optional
from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES,
USERPATH_STRATEGIES)
from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.resolvers import IResolver
from freqtrade.strategy.interface import IStrategy
logger = logging.getLogger(__name__)
@@ -174,7 +174,9 @@ class StrategyResolver(IResolver):
strategy = StrategyResolver._load_object(paths=abs_paths,
object_name=strategy_name,
kwargs={'config': config})
add_source=True,
kwargs={'config': config},
)
if strategy:
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)

5
freqtrade/rpc/__init__.py
View File

@@ -1,2 +1,3 @@
from .rpc import RPC, RPCMessageType, RPCException # noqa
from .rpc_manager import RPCManager # noqa
# flake8: noqa: F401
from .rpc import RPC, RPCException, RPCMessageType
from .rpc_manager import RPCManager

238
freqtrade/rpc/api_server.py
View File

@@ -1,40 +1,43 @@
import logging
import threading
from copy import deepcopy
from datetime import date, datetime
from ipaddress import IPv4Address
from pathlib import Path
from typing import Any, Callable, Dict
from arrow import Arrow
from flask import Flask, jsonify, request
from flask.json import JSONEncoder
from flask_cors import CORS
from flask_jwt_extended import (JWTManager, create_access_token,
create_refresh_token, get_jwt_identity,
jwt_refresh_token_required,
from flask_jwt_extended import (JWTManager, create_access_token, create_refresh_token,
get_jwt_identity, jwt_refresh_token_required,
verify_jwt_in_request_optional)
from werkzeug.security import safe_str_cmp
from werkzeug.serving import make_server
from freqtrade.__init__ import __version__
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.constants import DATETIME_PRINT_FORMAT, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.persistence import Trade
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
from freqtrade.rpc.rpc import RPC, RPCException
logger = logging.getLogger(__name__)
BASE_URI = "/api/v1"
class ArrowJSONEncoder(JSONEncoder):
class FTJSONEncoder(JSONEncoder):
def default(self, obj):
try:
if isinstance(obj, Arrow):
return obj.for_json()
elif isinstance(obj, date):
return obj.strftime("%Y-%m-%d")
elif isinstance(obj, datetime):
return obj.strftime(DATETIME_PRINT_FORMAT)
elif isinstance(obj, date):
return obj.strftime("%Y-%m-%d")
iterable = iter(obj)
except TypeError:
pass
@@ -108,7 +111,7 @@ class ApiServer(RPC):
'jwt_secret_key', 'super-secret')
self.jwt = JWTManager(self.app)
self.app.json_encoder = ArrowJSONEncoder
self.app.json_encoder = FTJSONEncoder
self.app.teardown_appcontext(shutdown_session)
@@ -160,16 +163,12 @@ class ApiServer(RPC):
"""
pass
def rest_dump(self, return_value):
""" Helper function to jsonify object for a webserver """
return jsonify(return_value)
def rest_error(self, error_msg):
return jsonify({"error": error_msg}), 502
def rest_error(self, error_msg, error_code=502):
return jsonify({"error": error_msg}), error_code
def register_rest_rpc_urls(self):
"""
Registers flask app URLs that are calls to functonality in rpc.rpc.
Registers flask app URLs that are calls to functionality in rpc.rpc.
First two arguments passed are /URL and 'Label'
Label can be used as a shortcut when refactoring
@@ -193,6 +192,7 @@ class ApiServer(RPC):
self.app.add_url_rule(f'{BASE_URI}/balance', 'balance',
view_func=self._balance, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/count', 'count', view_func=self._count, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/locks', 'locks', view_func=self._locks, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/daily', 'daily', view_func=self._daily, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/edge', 'edge', view_func=self._edge, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET'])
@@ -212,6 +212,20 @@ class ApiServer(RPC):
view_func=self._trades, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/trades/<int:tradeid>', 'trades_delete',
view_func=self._trades_delete, methods=['DELETE'])
self.app.add_url_rule(f'{BASE_URI}/pair_candles', 'pair_candles',
view_func=self._analysed_candles, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/pair_history', 'pair_history',
view_func=self._analysed_history, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/plot_config', 'plot_config',
view_func=self._plot_config, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/strategies', 'strategies',
view_func=self._list_strategies, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/strategy/<string:strategy>', 'strategy',
view_func=self._get_strategy, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/available_pairs', 'pairs',
view_func=self._list_available_pairs, methods=['GET'])
# Combined actions and infos
self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist,
methods=['GET', 'POST'])
@@ -222,15 +236,12 @@ class ApiServer(RPC):
self.app.add_url_rule(f'{BASE_URI}/forcesell', 'forcesell', view_func=self._forcesell,
methods=['POST'])
# TODO: Implement the following
# help (?)
@require_login
def page_not_found(self, error):
"""
Return "404 not found", 404.
"""
return self.rest_dump({
return jsonify({
'status': 'error',
'reason': f"There's no API call for {request.base_url}.",
'code': 404
@@ -250,7 +261,7 @@ class ApiServer(RPC):
'access_token': create_access_token(identity=keystuff),
'refresh_token': create_refresh_token(identity=keystuff),
}
return self.rest_dump(ret)
return jsonify(ret)
return jsonify({"error": "Unauthorized"}), 401
@@ -265,7 +276,7 @@ class ApiServer(RPC):
new_token = create_access_token(identity=current_user, fresh=False)
ret = {'access_token': new_token}
return self.rest_dump(ret)
return jsonify(ret)
@require_login
@rpc_catch_errors
@@ -275,7 +286,7 @@ class ApiServer(RPC):
Starts TradeThread in bot if stopped.
"""
msg = self._rpc_start()
return self.rest_dump(msg)
return jsonify(msg)
@require_login
@rpc_catch_errors
@@ -285,7 +296,7 @@ class ApiServer(RPC):
Stops TradeThread in bot if running
"""
msg = self._rpc_stop()
return self.rest_dump(msg)
return jsonify(msg)
@require_login
@rpc_catch_errors
@@ -295,14 +306,14 @@ class ApiServer(RPC):
Sets max_open_trades to 0 and gracefully sells all open trades
"""
msg = self._rpc_stopbuy()
return self.rest_dump(msg)
return jsonify(msg)
@rpc_catch_errors
def _ping(self):
"""
simple poing version
simple ping version
"""
return self.rest_dump({"status": "pong"})
return jsonify({"status": "pong"})
@require_login
@rpc_catch_errors
@@ -310,7 +321,7 @@ class ApiServer(RPC):
"""
Prints the bot's version
"""
return self.rest_dump({"version": __version__})
return jsonify({"version": __version__})
@require_login
@rpc_catch_errors
@@ -318,7 +329,7 @@ class ApiServer(RPC):
"""
Prints the bot's version
"""
return self.rest_dump(self._rpc_show_config())
return jsonify(self._rpc_show_config(self._config))
@require_login
@rpc_catch_errors
@@ -328,7 +339,7 @@ class ApiServer(RPC):
Triggers a config file reload
"""
msg = self._rpc_reload_config()
return self.rest_dump(msg)
return jsonify(msg)
@require_login
@rpc_catch_errors
@@ -338,7 +349,16 @@ class ApiServer(RPC):
Returns the number of trades running
"""
msg = self._rpc_count()
return self.rest_dump(msg)
return jsonify(msg)
@require_login
@rpc_catch_errors
def _locks(self):
"""
Handler for /locks.
Returns the currently active locks.
"""
return jsonify(self._rpc_locks())
@require_login
@rpc_catch_errors
@@ -356,7 +376,7 @@ class ApiServer(RPC):
self._config.get('fiat_display_currency', '')
)
return self.rest_dump(stats)
return jsonify(stats)
@require_login
@rpc_catch_errors
@@ -368,7 +388,7 @@ class ApiServer(RPC):
limit: Only get a certain number of records
"""
limit = int(request.args.get('limit', 0)) or None
return self.rest_dump(self._rpc_get_logs(limit))
return jsonify(self._rpc_get_logs(limit))
@require_login
@rpc_catch_errors
@@ -379,7 +399,7 @@ class ApiServer(RPC):
"""
stats = self._rpc_edge()
return self.rest_dump(stats)
return jsonify(stats)
@require_login
@rpc_catch_errors
@@ -395,7 +415,7 @@ class ApiServer(RPC):
self._config.get('fiat_display_currency')
)
return self.rest_dump(stats)
return jsonify(stats)
@require_login
@rpc_catch_errors
@@ -408,7 +428,7 @@ class ApiServer(RPC):
"""
stats = self._rpc_performance()
return self.rest_dump(stats)
return jsonify(stats)
@require_login
@rpc_catch_errors
@@ -420,9 +440,9 @@ class ApiServer(RPC):
"""
try:
results = self._rpc_trade_status()
return self.rest_dump(results)
return jsonify(results)
except RPCException:
return self.rest_dump([])
return jsonify([])
@require_login
@rpc_catch_errors
@@ -434,7 +454,7 @@ class ApiServer(RPC):
"""
results = self._rpc_balance(self._config['stake_currency'],
self._config.get('fiat_display_currency', ''))
return self.rest_dump(results)
return jsonify(results)
@require_login
@rpc_catch_errors
@@ -446,7 +466,7 @@ class ApiServer(RPC):
"""
limit = int(request.args.get('limit', 0))
results = self._rpc_trade_history(limit)
return self.rest_dump(results)
return jsonify(results)
@require_login
@rpc_catch_errors
@@ -459,7 +479,7 @@ class ApiServer(RPC):
tradeid: Numeric trade-id assigned to the trade.
"""
result = self._rpc_delete(tradeid)
return self.rest_dump(result)
return jsonify(result)
@require_login
@rpc_catch_errors
@@ -468,7 +488,7 @@ class ApiServer(RPC):
Handler for /whitelist.
"""
results = self._rpc_whitelist()
return self.rest_dump(results)
return jsonify(results)
@require_login
@rpc_catch_errors
@@ -478,7 +498,7 @@ class ApiServer(RPC):
"""
add = request.json.get("blacklist", None) if request.method == 'POST' else None
results = self._rpc_blacklist(add)
return self.rest_dump(results)
return jsonify(results)
@require_login
@rpc_catch_errors
@@ -490,9 +510,9 @@ class ApiServer(RPC):
price = request.json.get("price", None)
trade = self._rpc_forcebuy(asset, price)
if trade:
return self.rest_dump(trade.to_json())
return jsonify(trade.to_json())
else:
return self.rest_dump({"status": f"Error buying pair {asset}."})
return jsonify({"status": f"Error buying pair {asset}."})
@require_login
@rpc_catch_errors
@@ -502,4 +522,132 @@ class ApiServer(RPC):
"""
tradeid = request.json.get("tradeid")
results = self._rpc_forcesell(tradeid)
return self.rest_dump(results)
return jsonify(results)
@require_login
@rpc_catch_errors
def _analysed_candles(self):
"""
Handler for /pair_candles.
Returns the dataframe the bot is using during live/dry operations.
Takes the following get arguments:
get:
parameters:
- pair: Pair
- timeframe: Timeframe to get data for (should be aligned to strategy.timeframe)
- limit: Limit return length to the latest X candles
"""
pair = request.args.get("pair")
timeframe = request.args.get("timeframe")
limit = request.args.get("limit", type=int)
if not pair or not timeframe:
return self.rest_error("Mandatory parameter missing.", 400)
results = self._rpc_analysed_dataframe(pair, timeframe, limit)
return jsonify(results)
@require_login
@rpc_catch_errors
def _analysed_history(self):
"""
Handler for /pair_history.
Returns the dataframe of a given timerange
Takes the following get arguments:
get:
parameters:
- pair: Pair
- timeframe: Timeframe to get data for (should be aligned to strategy.timeframe)
- strategy: Strategy to use - Must exist in configured strategy-path!
- timerange: timerange in the format YYYYMMDD-YYYYMMDD (YYYYMMDD- or (-YYYYMMDD))
are als possible. If omitted uses all available data.
"""
pair = request.args.get("pair")
timeframe = request.args.get("timeframe")
timerange = request.args.get("timerange")
strategy = request.args.get("strategy")
if not pair or not timeframe or not timerange or not strategy:
return self.rest_error("Mandatory parameter missing.", 400)
config = deepcopy(self._config)
config.update({
'strategy': strategy,
})
results = RPC._rpc_analysed_history_full(config, pair, timeframe, timerange)
return jsonify(results)
@require_login
@rpc_catch_errors
def _plot_config(self):
"""
Handler for /plot_config.
"""
return jsonify(self._rpc_plot_config())
@require_login
@rpc_catch_errors
def _list_strategies(self):
directory = Path(self._config.get(
'strategy_path', self._config['user_data_dir'] / USERPATH_STRATEGIES))
from freqtrade.resolvers.strategy_resolver import StrategyResolver
strategy_objs = StrategyResolver.search_all_objects(directory, False)
strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
return jsonify({'strategies': [x['name'] for x in strategy_objs]})
@require_login
@rpc_catch_errors
def _get_strategy(self, strategy: str):
"""
Get a single strategy
get:
parameters:
- strategy: Only get this strategy
"""
config = deepcopy(self._config)
from freqtrade.resolvers.strategy_resolver import StrategyResolver
try:
strategy_obj = StrategyResolver._load_strategy(strategy, config,
extra_dir=config.get('strategy_path'))
except OperationalException:
return self.rest_error("Strategy not found.", 404)
return jsonify({
'strategy': strategy_obj.get_strategy_name(),
'code': strategy_obj.__source__,
})
@require_login
@rpc_catch_errors
def _list_available_pairs(self):
"""
Handler for /available_pairs.
Returns an object, with pairs, available pair length and pair_interval combinations
Takes the following get arguments:
get:
parameters:
- stake_currency: Filter on this stake currency
- timeframe: Timeframe to get data for Filter elements to this timeframe
"""
timeframe = request.args.get("timeframe")
stake_currency = request.args.get("stake_currency")
from freqtrade.data.history import get_datahandler
dh = get_datahandler(self._config['datadir'], self._config.get('dataformat_ohlcv', None))
pair_interval = dh.ohlcv_get_available_data(self._config['datadir'])
if timeframe:
pair_interval = [pair for pair in pair_interval if pair[1] == timeframe]
if stake_currency:
pair_interval = [pair for pair in pair_interval if pair[0].endswith(stake_currency)]
pair_interval = sorted(pair_interval, key=lambda x: x[0])
pairs = list({x[0] for x in pair_interval})
result = {
'length': len(pairs),
'pairs': pairs,
'pair_interval': pair_interval,
}
return jsonify(result)

115
freqtrade/rpc/rpc.py
View File

@@ -9,25 +9,29 @@ from math import isnan
from typing import Any, Dict, List, Optional, Tuple, Union
import arrow
from numpy import NAN, mean
from numpy import NAN, int64, mean
from pandas import DataFrame
from freqtrade.constants import CANCEL_REASON
from freqtrade.configuration.timerange import TimeRange
from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT
from freqtrade.data.history import load_data
from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
from freqtrade.misc import shorten_date
from freqtrade.persistence import Trade
from freqtrade.persistence import PairLock, Trade
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
from freqtrade.state import State
from freqtrade.strategy.interface import SellType
logger = logging.getLogger(__name__)
class RPCMessageType(Enum):
STATUS_NOTIFICATION = 'status'
WARNING_NOTIFICATION = 'warning'
CUSTOM_NOTIFICATION = 'custom'
STARTUP_NOTIFICATION = 'startup'
BUY_NOTIFICATION = 'buy'
BUY_CANCEL_NOTIFICATION = 'buy_cancel'
SELL_NOTIFICATION = 'sell'
@@ -36,6 +40,9 @@ class RPCMessageType(Enum):
def __repr__(self):
return self.value
def __str__(self):
return self.value
class RPCException(Exception):
"""
@@ -86,13 +93,12 @@ class RPC:
def send_msg(self, msg: Dict[str, str]) -> None:
""" Sends a message to all registered rpc modules """
def _rpc_show_config(self) -> Dict[str, Any]:
def _rpc_show_config(self, config) -> Dict[str, Any]:
"""
Return a dict of config options.
Explicitly does NOT return the full config to avoid leakage of sensitive
information via rpc.
"""
config = self._freqtrade.config
val = {
'dry_run': config['dry_run'],
'stake_currency': config['stake_currency'],
@@ -113,7 +119,7 @@ class RPC:
'forcebuy_enabled': config.get('forcebuy_enable', False),
'ask_strategy': config.get('ask_strategy', {}),
'bid_strategy': config.get('bid_strategy', {}),
'state': str(self._freqtrade.state)
'state': str(self._freqtrade.state) if self._freqtrade else '',
}
return val
@@ -562,8 +568,7 @@ class RPC:
except (ExchangeError):
pass
Trade.session.delete(trade)
Trade.session.flush()
trade.delete()
self._freqtrade.wallets.update()
return {
'result': 'success',
@@ -594,6 +599,17 @@ class RPC:
'total_stake': sum((trade.open_rate * trade.amount) for trade in trades)
}
def _rpc_locks(self) -> Dict[str, Any]:
""" Returns the current locks"""
if self._freqtrade.state != State.RUNNING:
raise RPCException('trader is not running')
locks = PairLock.get_pair_locks(None)
return {
'lock_count': len(locks),
'locks': [lock.to_json() for lock in locks]
}
def _rpc_whitelist(self) -> Dict:
""" Returns the currently active whitelist"""
res = {'method': self._freqtrade.pairlists.name_list,
@@ -633,7 +649,7 @@ class RPC:
buffer = bufferHandler.buffer[-limit:]
else:
buffer = bufferHandler.buffer
records = [[datetime.fromtimestamp(r.created).strftime("%Y-%m-%d %H:%M:%S"),
records = [[datetime.fromtimestamp(r.created).strftime(DATETIME_PRINT_FORMAT),
r.created * 1000, r.name, r.levelname,
r.message + ('\n' + r.exc_text if r.exc_text else '')]
for r in buffer]
@@ -650,3 +666,82 @@ class RPC:
if not self._freqtrade.edge:
raise RPCException('Edge is not enabled.')
return self._freqtrade.edge.accepted_pairs()
@staticmethod
def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame,
last_analyzed: datetime) -> Dict[str, Any]:
has_content = len(dataframe) != 0
buy_signals = 0
sell_signals = 0
if has_content:
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
# Move open to seperate column when signal for easy plotting
if 'buy' in dataframe.columns:
buy_mask = (dataframe['buy'] == 1)
buy_signals = int(buy_mask.sum())
dataframe.loc[buy_mask, '_buy_signal_open'] = dataframe.loc[buy_mask, 'open']
if 'sell' in dataframe.columns:
sell_mask = (dataframe['sell'] == 1)
sell_signals = int(sell_mask.sum())
dataframe.loc[sell_mask, '_sell_signal_open'] = dataframe.loc[sell_mask, 'open']
dataframe = dataframe.replace({NAN: None})
res = {
'pair': pair,
'timeframe': timeframe,
'timeframe_ms': timeframe_to_msecs(timeframe),
'strategy': strategy,
'columns': list(dataframe.columns),
'data': dataframe.values.tolist(),
'length': len(dataframe),
'buy_signals': buy_signals,
'sell_signals': sell_signals,
'last_analyzed': last_analyzed,
'last_analyzed_ts': int(last_analyzed.timestamp()),
'data_start': '',
'data_start_ts': 0,
'data_stop': '',
'data_stop_ts': 0,
}
if has_content:
res.update({
'data_start': str(dataframe.iloc[0]['date']),
'data_start_ts': int(dataframe.iloc[0]['__date_ts']),
'data_stop': str(dataframe.iloc[-1]['date']),
'data_stop_ts': int(dataframe.iloc[-1]['__date_ts']),
})
return res
def _rpc_analysed_dataframe(self, pair: str, timeframe: str, limit: int) -> Dict[str, Any]:
_data, last_analyzed = self._freqtrade.dataprovider.get_analyzed_dataframe(
pair, timeframe)
_data = _data.copy()
if limit:
_data = _data.iloc[-limit:]
return self._convert_dataframe_to_dict(self._freqtrade.config['strategy'],
pair, timeframe, _data, last_analyzed)
@staticmethod
def _rpc_analysed_history_full(config, pair: str, timeframe: str,
timerange: str) -> Dict[str, Any]:
timerange_parsed = TimeRange.parse_timerange(timerange)
_data = load_data(
datadir=config.get("datadir"),
pairs=[pair],
timeframe=timeframe,
timerange=timerange_parsed,
data_format=config.get('dataformat_ohlcv', 'json'),
)
from freqtrade.resolvers.strategy_resolver import StrategyResolver
strategy = StrategyResolver.load_strategy(config)
df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair})
return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe,
df_analyzed, arrow.Arrow.utcnow().datetime)
def _rpc_plot_config(self) -> Dict[str, Any]:
return self._freqtrade.strategy.plot_config

7
freqtrade/rpc/rpc_manager.py
View File

@@ -6,6 +6,7 @@ from typing import Any, Dict, List
from freqtrade.rpc import RPC, RPCMessageType
logger = logging.getLogger(__name__)
@@ -59,7 +60,7 @@ class RPCManager:
try:
mod.send_msg(msg)
except NotImplementedError:
logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.")
logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.")
def startup_messages(self, config: Dict[str, Any], pairlist) -> None:
if config['dry_run']:
@@ -76,7 +77,7 @@ class RPCManager:
exchange_name = config['exchange']['name']
strategy_name = config.get('strategy', '')
self.send_msg({
'type': RPCMessageType.CUSTOM_NOTIFICATION,
'type': RPCMessageType.STARTUP_NOTIFICATION,
'status': f'*Exchange:* `{exchange_name}`\n'
f'*Stake per trade:* `{stake_amount} {stake_currency}`\n'
f'*Minimum ROI:* `{minimal_roi}`\n'
@@ -85,7 +86,7 @@ class RPCManager:
f'*Strategy:* `{strategy_name}`'
})
self.send_msg({
'type': RPCMessageType.STATUS_NOTIFICATION,
'type': RPCMessageType.STARTUP_NOTIFICATION,
'status': f'Searching for {stake_currency} pairs to buy and sell '
f'based on {pairlist.short_desc()}'
})

50
freqtrade/rpc/telegram.py
View File

@@ -5,9 +5,9 @@ This module manage Telegram communication
"""
import json
import logging
import arrow
from typing import Any, Callable, Dict, List
import arrow
from tabulate import tabulate
from telegram import ParseMode, ReplyKeyboardMarkup, Update
from telegram.error import NetworkError, TelegramError
@@ -18,6 +18,7 @@ from freqtrade.__init__ import __version__
from freqtrade.rpc import RPC, RPCException, RPCMessageType
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
logger = logging.getLogger(__name__)
logger.debug('Included module rpc.telegram ...')
@@ -99,6 +100,7 @@ class Telegram(RPC):
CommandHandler('performance', self._performance),
CommandHandler('daily', self._daily),
CommandHandler('count', self._count),
CommandHandler('locks', self._locks),
CommandHandler(['reload_config', 'reload_conf'], self._reload_config),
CommandHandler(['show_config', 'show_conf'], self._show_config),
CommandHandler('stopbuy', self._stopbuy),
@@ -133,6 +135,13 @@ class Telegram(RPC):
def send_msg(self, msg: Dict[str, Any]) -> None:
""" Send a message to telegram channel """
noti = self._config['telegram'].get('notification_settings', {}
).get(str(msg['type']), 'on')
if noti == 'off':
logger.info(f"Notification '{msg['type']}' not sent.")
# Notification disabled
return
if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
if self._fiat_converter:
msg['stake_amount_fiat'] = self._fiat_converter.convert_amount(
@@ -191,13 +200,13 @@ class Telegram(RPC):
elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION:
message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
elif msg['type'] == RPCMessageType.CUSTOM_NOTIFICATION:
elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION:
message = '{status}'.format(**msg)
else:
raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
self._send_msg(message)
self._send_msg(message, disable_notification=(noti == 'silent'))
def _get_sell_emoji(self, msg):
"""
@@ -601,6 +610,26 @@ class Telegram(RPC):
except RPCException as e:
self._send_msg(str(e))
@authorized_only
def _locks(self, update: Update, context: CallbackContext) -> None:
"""
Handler for /locks.
Returns the currently active locks
"""
try:
locks = self._rpc_locks()
message = tabulate([[
lock['pair'],
lock['lock_end_time'],
lock['reason']] for lock in locks['locks']],
headers=['Pair', 'Until', 'Reason'],
tablefmt='simple')
message = "<pre>{}</pre>".format(message)
logger.debug(message)
self._send_msg(message, parse_mode=ParseMode.HTML)
except RPCException as e:
self._send_msg(str(e))
@authorized_only
def _whitelist(self, update: Update, context: CallbackContext) -> None:
"""
@@ -712,8 +741,8 @@ class Telegram(RPC):
"*/delete <trade_id>:* `Instantly delete the given trade in the database`\n"
"*/performance:* `Show performance of each finished trade grouped by pair`\n"
"*/daily <n>:* `Shows profit or loss per day, over the last n days`\n"
"*/count:* `Show number of trades running compared to allowed number of trades`"
"\n"
"*/count:* `Show number of active trades compared to allowed number of trades`\n"
"*/locks:* `Show currently locked pairs`\n"
"*/balance:* `Show account balance per currency`\n"
"*/stopbuy:* `Stops buying, but handles open trades gracefully` \n"
"*/reload_config:* `Reload configuration file` \n"
@@ -804,7 +833,7 @@ class Telegram(RPC):
:param update: message update
:return: None
"""
val = self._rpc_show_config()
val = self._rpc_show_config(self._freqtrade.config)
if val['trailing_stop']:
sl_info = (
f"*Initial Stoploss:* `{val['stoploss']}`\n"
@@ -830,7 +859,8 @@ class Telegram(RPC):
f"*Current state:* `{val['state']}`"
)
def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:
def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN,
disable_notification: bool = False) -> None:
"""
Send given markdown message
:param msg: message
@@ -851,7 +881,8 @@ class Telegram(RPC):
self._config['telegram']['chat_id'],
text=msg,
parse_mode=parse_mode,
reply_markup=reply_markup
reply_markup=reply_markup,
disable_notification=disable_notification,
)
except NetworkError as network_err:
# Sometimes the telegram server resets the current connection,
@@ -864,7 +895,8 @@ class Telegram(RPC):
self._config['telegram']['chat_id'],
text=msg,
parse_mode=parse_mode,
reply_markup=reply_markup
reply_markup=reply_markup,
disable_notification=disable_notification,
)
except TelegramError as telegram_err:
logger.warning(

8
freqtrade/rpc/webhook.py
View File

@@ -2,9 +2,9 @@
This module manages webhook communication
"""
import logging
from typing import Any, Dict
from typing import Any, Dict
from requests import post, RequestException
from requests import RequestException, post
from freqtrade.rpc import RPC, RPCMessageType
@@ -48,13 +48,13 @@ class Webhook(RPC):
elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
valuedict = self._config['webhook'].get('webhooksellcancel', None)
elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION,
RPCMessageType.CUSTOM_NOTIFICATION,
RPCMessageType.STARTUP_NOTIFICATION,
RPCMessageType.WARNING_NOTIFICATION):
valuedict = self._config['webhook'].get('webhookstatus', None)
else:
raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
if not valuedict:
logger.info("Message type %s not configured for webhooks", msg['type'])
logger.info("Message type '%s' not configured for webhooks", msg['type'])
return
payload = {key: value.format(**msg) for (key, value) in valuedict.items()}

4
freqtrade/strategy/__init__.py
View File

@@ -1,5 +1,5 @@
# flake8: noqa: F401
from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_prev_date,
timeframe_to_seconds, timeframe_to_next_date, timeframe_to_msecs)
from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date,
timeframe_to_prev_date, timeframe_to_seconds)
from freqtrade.strategy.interface import IStrategy
from freqtrade.strategy.strategy_helper import merge_informative_pair

25
freqtrade/strategy/interface.py
View File

@@ -17,10 +17,11 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException, StrategyError
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.persistence import Trade
from freqtrade.persistence import PairLock, Trade
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
logger = logging.getLogger(__name__)
@@ -122,6 +123,8 @@ class IStrategy(ABC):
# and wallets - access to the current balance.
dp: Optional[DataProvider] = None
wallets: Optional[Wallets] = None
# container variable for strategy source code
__source__: str = ''
# Definition of plot_config. See plotting documentation for more details.
plot_config: Dict = {}
@@ -130,7 +133,6 @@ class IStrategy(ABC):
self.config = config
# Dict to determine if analysis is necessary
self._last_candle_seen_per_pair: Dict[str, datetime] = {}
self._pair_locked_until: Dict[str, datetime] = {}
@abstractmethod
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
@@ -275,7 +277,7 @@ class IStrategy(ABC):
"""
return self.__class__.__name__
def lock_pair(self, pair: str, until: datetime) -> None:
def lock_pair(self, pair: str, until: datetime, reason: str = None) -> None:
"""
Locks pair until a given timestamp happens.
Locked pairs are not analyzed, and are prevented from opening new trades.
@@ -284,9 +286,9 @@ class IStrategy(ABC):
:param pair: Pair to lock
:param until: datetime in UTC until the pair should be blocked from opening new trades.
Needs to be timezone aware `datetime.now(timezone.utc)`
:param reason: Optional string explaining why the pair was locked.
"""
if pair not in self._pair_locked_until or self._pair_locked_until[pair] < until:
self._pair_locked_until[pair] = until
PairLock.lock_pair(pair, until, reason)
def unlock_pair(self, pair: str) -> None:
"""
@@ -295,8 +297,7 @@ class IStrategy(ABC):
manually from within the strategy, to allow an easy way to unlock pairs.
:param pair: Unlock pair to allow trading again
"""
if pair in self._pair_locked_until:
del self._pair_locked_until[pair]
PairLock.unlock_pair(pair, datetime.now(timezone.utc))
def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
"""
@@ -308,15 +309,13 @@ class IStrategy(ABC):
:param candle_date: Date of the last candle. Optional, defaults to current date
:returns: locking state of the pair in question.
"""
if pair not in self._pair_locked_until:
return False
if not candle_date:
return self._pair_locked_until[pair] >= datetime.now(timezone.utc)
# Simple call ...
return PairLock.is_pair_locked(pair, candle_date)
else:
# Locking should happen until a new candle arrives
lock_time = timeframe_to_next_date(self.timeframe, candle_date)
# lock_time = candle_date + timedelta(minutes=timeframe_to_minutes(self.timeframe))
return self._pair_locked_until[pair] > lock_time
return PairLock.is_pair_locked(pair, lock_time)
def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""

1
freqtrade/strategy/strategy_helper.py
View File

@@ -1,4 +1,5 @@
import pandas as pd
from freqtrade.exchange import timeframe_to_minutes

1
freqtrade/strategy/strategy_wrapper.py
View File

@@ -2,6 +2,7 @@ import logging
from freqtrade.exceptions import StrategyError
logger = logging.getLogger(__name__)

1
freqtrade/templates/sample_hyperopt.py
View File

@@ -1,4 +1,5 @@
# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
# isort: skip_file
# --- Do not remove these libs ---
from functools import reduce

2
freqtrade/templates/sample_hyperopt_advanced.py
View File

@@ -1,5 +1,5 @@
# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
# isort: skip_file
# --- Do not remove these libs ---
from functools import reduce
from typing import Any, Callable, Dict, List

3
freqtrade/templates/sample_hyperopt_loss.py
View File

@@ -1,10 +1,11 @@
from math import exp
from datetime import datetime
from math import exp
from pandas import DataFrame
from freqtrade.optimize.hyperopt import IHyperOptLoss
# Define some constants:
# set TARGET_TRADES to suit your number concurrent trades so its realistic

2
freqtrade/templates/sample_strategy.py
View File

@@ -1,5 +1,5 @@
# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
# isort: skip_file
# --- Do not remove these libs ---
import numpy as np # noqa
import pandas as pd # noqa

3
freqtrade/vendor/qtpylib/indicators.py vendored
View File

@@ -19,14 +19,15 @@
# limitations under the License.
#
import warnings
import sys
import warnings
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
from pandas.core.base import PandasObject
# =============================================
# check min, python version
if sys.version_info < (3, 4):

6
freqtrade/wallets.py
View File

@@ -2,6 +2,7 @@
""" Wallet """
import logging
from copy import deepcopy
from typing import Any, Dict, NamedTuple
import arrow
@@ -9,6 +10,7 @@ import arrow
from freqtrade.exchange import Exchange
from freqtrade.persistence import Trade
logger = logging.getLogger(__name__)
@@ -93,6 +95,10 @@ class Wallets:
balances[currency].get('used', None),
balances[currency].get('total', None)
)
# Remove currencies no longer in get_balances output
for currency in deepcopy(self._wallets):
if currency not in balances:
del self._wallets[currency]
def update(self, require_update: bool = True) -> None:
"""

1
freqtrade/worker.py
View File

@@ -15,6 +15,7 @@ from freqtrade.exceptions import OperationalException, TemporaryError
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.state import State
logger = logging.getLogger(__name__)