adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into interface_ordertimeoutcallback

Browse Source
This commit is contained in:
Matthias
2020-03-15 14:56:14 +01:00
parent 0f2d771634 3b89e7d393
commit a1bbeaa668
101 changed files with 2205 additions and 1228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
freqtrade/commands/arguments.py
View File

@@ -69,7 +69,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
"hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
"hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit",
"hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit",
"print_colorized", "print_json", "hyperopt_list_no_details"]
"print_colorized", "print_json", "hyperopt_list_no_details",
"export_csv"]
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperopt_show_no_header"]
@@ -296,7 +297,7 @@ class Arguments:
# Add convert-data subcommand
convert_data_cmd = subparsers.add_parser(
'convert-data',
help='Convert OHLCV data from one format to another.',
help='Convert candle (OHLCV) data from one format to another.',
parents=[_common_parser],
)
convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=True))
@@ -305,7 +306,7 @@ class Arguments:
# Add convert-trade-data subcommand
convert_trade_data_cmd = subparsers.add_parser(
'convert-trade-data',
help='Convert trade-data from one format to another.',
help='Convert trade data from one format to another.',
parents=[_common_parser],
)
convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))

2
freqtrade/commands/build_config_commands.py
View File

@@ -76,7 +76,7 @@ def ask_user_config() -> Dict[str, Any]:
{
"type": "text",
"name": "ticker_interval",
"message": "Please insert your ticker interval:",
"message": "Please insert your timeframe (ticker interval):",
"default": "5m",
},
{

12
freqtrade/commands/cli_options.py
View File

@@ -221,6 +221,13 @@ AVAILABLE_CLI_OPTIONS = {
action='store_true',
default=False,
),
"export_csv": Arg(
'--export-csv',
help='Export to CSV-File.'
' This will disable table print.'
' Example: --export-csv hyperopt.csv',
metavar='FILE',
),
"hyperopt_jobs": Arg(
'-j', '--job-workers',
help='The number of concurrently running jobs for hyperoptimization '
@@ -257,7 +264,8 @@ AVAILABLE_CLI_OPTIONS = {
help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). '
'Different functions can generate completely different results, '
'since the target for optimization is different. Built-in Hyperopt-loss-functions are: '
'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily.'
'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, '
'SortinoHyperOptLoss, SortinoHyperOptLossDaily.'
'(default: `%(default)s`).',
metavar='NAME',
default=constants.DEFAULT_HYPEROPT_LOSS,
@@ -347,7 +355,7 @@ AVAILABLE_CLI_OPTIONS = {
),
"dataformat_ohlcv": Arg(
'--data-format-ohlcv',
help='Storage format for downloaded ohlcv data. (default: `%(default)s`).',
help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
choices=constants.AVAILABLE_DATAHANDLERS,
default='json'
),

92
freqtrade/commands/hyperopt_commands.py
View File

@@ -21,6 +21,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
print_colorized = config.get('print_colorized', False)
print_json = config.get('print_json', False)
export_csv = config.get('export_csv', None)
no_details = config.get('hyperopt_list_no_details', False)
no_header = False
@@ -46,26 +47,26 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
trials = _hyperopt_filter_trials(trials, filteroptions)
# TODO: fetch the interval for epochs to print from the cli option
epoch_start, epoch_stop = 0, None
if print_colorized:
colorama_init(autoreset=True)
try:
# Human-friendly indexes used here (starting from 1)
for val in trials[epoch_start:epoch_stop]:
Hyperopt.print_results_explanation(val, total_epochs,
not filteroptions['only_best'], print_colorized)
except KeyboardInterrupt:
print('User interrupted..')
if not export_csv:
try:
Hyperopt.print_result_table(config, trials, total_epochs,
not filteroptions['only_best'], print_colorized, 0)
except KeyboardInterrupt:
print('User interrupted..')
if trials and not no_details:
sorted_trials = sorted(trials, key=itemgetter('loss'))
results = sorted_trials[0]
Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
if trials and export_csv:
Hyperopt.export_csv_file(
config, trials, total_epochs, not filteroptions['only_best'], export_csv
)
def start_hyperopt_show(args: Dict[str, Any]) -> None:
"""
@@ -75,6 +76,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
print_json = config.get('print_json', False)
no_header = config.get('hyperopt_show_no_header', False)
trials_file = (config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
n = config.get('hyperopt_show_index', -1)
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
@@ -87,10 +94,6 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None)
}
no_header = config.get('hyperopt_show_no_header', False)
trials_file = (config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
# Previous evaluations
trials = Hyperopt.load_previous_results(trials_file)
@@ -99,20 +102,17 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
trials = _hyperopt_filter_trials(trials, filteroptions)
trials_epochs = len(trials)
n = config.get('hyperopt_show_index', -1)
if n > trials_epochs:
raise OperationalException(
f"The index of the epoch to show should be less than {trials_epochs + 1}.")
f"The index of the epoch to show should be less than {trials_epochs + 1}.")
if n < -trials_epochs:
raise OperationalException(
f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
# Translate epoch index from human-readable format to pythonic
if n > 0:
n -= 1
print_json = config.get('print_json', False)
if trials:
val = trials[n]
Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
@@ -129,52 +129,52 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List:
trials = [x for x in trials if x['results_metrics']['profit'] > 0]
if filteroptions['filter_min_trades'] > 0:
trials = [
x for x in trials
if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades']
]
x for x in trials
if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades']
]
if filteroptions['filter_max_trades'] > 0:
trials = [
x for x in trials
if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
]
x for x in trials
if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
]
if filteroptions['filter_min_avg_time'] is not None:
trials = [x for x in trials if x['results_metrics']['trade_count'] > 0]
trials = [
x for x in trials
if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']
]
x for x in trials
if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']
]
if filteroptions['filter_max_avg_time'] is not None:
trials = [x for x in trials if x['results_metrics']['trade_count'] > 0]
trials = [
x for x in trials
if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
]
x for x in trials
if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
]
if filteroptions['filter_min_avg_profit'] is not None:
trials = [x for x in trials if x['results_metrics']['trade_count'] > 0]
trials = [
x for x in trials
if x['results_metrics']['avg_profit']
> filteroptions['filter_min_avg_profit']
]
x for x in trials
if x['results_metrics']['avg_profit']
> filteroptions['filter_min_avg_profit']
]
if filteroptions['filter_max_avg_profit'] is not None:
trials = [x for x in trials if x['results_metrics']['trade_count'] > 0]
trials = [
x for x in trials
if x['results_metrics']['avg_profit']
< filteroptions['filter_max_avg_profit']
]
x for x in trials
if x['results_metrics']['avg_profit']
< filteroptions['filter_max_avg_profit']
]
if filteroptions['filter_min_total_profit'] is not None:
trials = [x for x in trials if x['results_metrics']['trade_count'] > 0]
trials = [
x for x in trials
if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']
]
x for x in trials
if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']
]
if filteroptions['filter_max_total_profit'] is not None:
trials = [x for x in trials if x['results_metrics']['trade_count'] > 0]
trials = [
x for x in trials
if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
]
x for x in trials
if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
]
logger.info(f"{len(trials)} " +
("best " if filteroptions['only_best'] else "") +

4
freqtrade/commands/list_commands.py
View File

@@ -58,7 +58,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
else yellow + "DUPLICATE NAME" + reset)
} for s in objs]
print(tabulate(objss_to_print, headers='keys', tablefmt='pipe'))
print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right'))
def start_list_strategies(args: Dict[str, Any]) -> None:
@@ -192,7 +192,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
else:
# print data as a table, with the human-readable summary
print(f"{summary_str}:")
print(tabulate(tabular_data, headers='keys', tablefmt='pipe'))
print(tabulate(tabular_data, headers='keys', tablefmt='psql', stralign='right'))
elif not (args.get('print_one_column', False) or
args.get('list_pairs_print_json', False) or
args.get('print_csv', False)):

13
freqtrade/commands/optimize_commands.py
View File

@@ -17,10 +17,15 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
"""
config = setup_utils_configuration(args, method)
if method == RunMode.BACKTEST:
if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
raise DependencyException('stake amount could not be "%s" for backtesting' %
constants.UNLIMITED_STAKE_AMOUNT)
no_unlimited_runmodes = {
RunMode.BACKTEST: 'backtesting',
RunMode.HYPEROPT: 'hyperoptimization',
}
if (method in no_unlimited_runmodes.keys() and
config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT):
raise DependencyException(
f'The value of `stake_amount` cannot be set as "{constants.UNLIMITED_STAKE_AMOUNT}" '
f'for {no_unlimited_runmodes[method]}')
return config

12
freqtrade/configuration/config_validation.py
View File

@@ -150,15 +150,3 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None:
if (pl.get('method') == 'StaticPairList'
and not conf.get('exchange', {}).get('pair_whitelist')):
raise OperationalException("StaticPairList requires pair_whitelist to be set.")
if pl.get('method') == 'StaticPairList':
stake = conf['stake_currency']
invalid_pairs = []
for pair in conf['exchange'].get('pair_whitelist'):
if not pair.endswith(f'/{stake}'):
invalid_pairs.append(pair)
if invalid_pairs:
raise OperationalException(
f"Stake-currency '{stake}' not compatible with pair-whitelist. "
f"Please remove the following pairs: {invalid_pairs}")

18
freqtrade/configuration/configuration.py
View File

@@ -96,6 +96,8 @@ class Configuration:
# Keep a copy of the original configuration file
config['original_config'] = deepcopy(config)
self._process_logging_options(config)
self._process_runmode(config)
self._process_common_options(config)
@@ -146,8 +148,6 @@ class Configuration:
def _process_common_options(self, config: Dict[str, Any]) -> None:
self._process_logging_options(config)
# Set strategy if not specified in config and or if it's non default
if self.args.get("strategy") or not config.get('strategy'):
config.update({'strategy': self.args.get("strategy")})
@@ -167,10 +167,6 @@ class Configuration:
if 'sd_notify' in self.args and self.args["sd_notify"]:
config['internals'].update({'sd_notify': True})
self._args_to_config(config, argname='dry_run',
logstring='Parameter --dry-run detected, '
'overriding dry_run to: {} ...')
def _process_datadir_options(self, config: Dict[str, Any]) -> None:
"""
Extract information for sys.argv and load directory configurations
@@ -200,6 +196,7 @@ class Configuration:
if self.args.get('exportfilename'):
self._args_to_config(config, argname='exportfilename',
logstring='Storing backtest results to {} ...')
config['exportfilename'] = Path(config['exportfilename'])
else:
config['exportfilename'] = (config['user_data_dir']
/ 'backtest_results/backtest-result.json')
@@ -286,6 +283,9 @@ class Configuration:
self._args_to_config(config, argname='print_json',
logstring='Parameter --print-json detected ...')
self._args_to_config(config, argname='export_csv',
logstring='Parameter --export-csv detected: {}')
self._args_to_config(config, argname='hyperopt_jobs',
logstring='Parameter -j/--job-workers detected: {}')
@@ -376,10 +376,14 @@ class Configuration:
def _process_runmode(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='dry_run',
logstring='Parameter --dry-run detected, '
'overriding dry_run to: {} ...')
if not self.runmode:
# Handle real mode, infer dry/live from config
self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE
logger.info(f"Runmode set to {self.runmode}.")
logger.info(f"Runmode set to {self.runmode.value}.")
config.update({'runmode': self.runmode})

2
freqtrade/configuration/timerange.py
View File

@@ -45,7 +45,7 @@ class TimeRange:
"""
Adjust startts by <startup_candles> candles.
Applies only if no startup-candles have been available.
:param timeframe_secs: Ticker timeframe in seconds e.g. `timeframe_to_seconds('5m')`
:param timeframe_secs: Timeframe in seconds e.g. `timeframe_to_seconds('5m')`
:param startup_candles: Number of candles to move start-date forward
:param min_date: Minimum data date loaded. Key kriterium to decide if start-time
has to be moved

26
freqtrade/constants.py
View File

@@ -15,6 +15,7 @@ UNLIMITED_STAKE_AMOUNT = 'unlimited'
DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05
REQUIRED_ORDERTIF = ['buy', 'sell']
REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
ORDERBOOK_SIDES = ['ask', 'bid']
ORDERTYPE_POSSIBILITIES = ['limit', 'market']
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
@@ -42,7 +43,7 @@ SUPPORTED_FIAT = [
"EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY",
"KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN",
"RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD",
"BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT"
"BTC", "ETH", "XRP", "LTC", "BCH"
]
MINIMAL_CONFIG = {
@@ -113,15 +114,16 @@ CONF_SCHEMA = {
'minimum': 0,
'maximum': 1,
'exclusiveMaximum': False,
'use_order_book': {'type': 'boolean'},
'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1},
'check_depth_of_market': {
'type': 'object',
'properties': {
'enabled': {'type': 'boolean'},
'bids_to_ask_delta': {'type': 'number', 'minimum': 0},
}
},
},
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'},
'use_order_book': {'type': 'boolean'},
'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1},
'check_depth_of_market': {
'type': 'object',
'properties': {
'enabled': {'type': 'boolean'},
'bids_to_ask_delta': {'type': 'number', 'minimum': 0},
}
},
},
'required': ['ask_last_balance']
@@ -129,6 +131,7 @@ CONF_SCHEMA = {
'ask_strategy': {
'type': 'object',
'properties': {
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'ask'},
'use_order_book': {'type': 'boolean'},
'order_book_min': {'type': 'integer', 'minimum': 1},
'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50},
@@ -251,7 +254,6 @@ CONF_SCHEMA = {
'type': 'array',
'items': {
'type': 'string',
'pattern': '^[0-9A-Z]+/[0-9A-Z]+$'
},
'uniqueItems': True
},
@@ -259,7 +261,6 @@ CONF_SCHEMA = {
'type': 'array',
'items': {
'type': 'string',
'pattern': '^[0-9A-Z]+/[0-9A-Z]+$'
},
'uniqueItems': True
},
@@ -301,6 +302,7 @@ SCHEMA_TRADE_REQUIRED = [
'last_stake_amount_min_ratio',
'dry_run',
'dry_run_wallet',
'ask_strategy',
'bid_strategy',
'unfilledtimeout',
'stoploss',

45
freqtrade/data/btanalysis.py
View File

@@ -3,7 +3,7 @@ Helpers when analyzing backtest data
"""
import logging
from pathlib import Path
from typing import Dict, Union
from typing import Dict, Union, Tuple
import numpy as np
import pandas as pd
@@ -129,16 +129,20 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
return trades
def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame:
def load_trades(source: str, db_url: str, exportfilename: Path) -> pd.DataFrame:
"""
Based on configuration option "trade_source":
* loads data from DB (using `db_url`)
* loads data from backtestfile (using `exportfilename`)
:param source: "DB" or "file" - specify source to load from
:param db_url: sqlalchemy formatted url to a database
:param exportfilename: Json file generated by backtesting
:return: DataFrame containing trades
"""
if source == "DB":
return load_trades_from_db(db_url)
elif source == "file":
return load_backtest_data(Path(exportfilename))
return load_backtest_data(exportfilename)
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
@@ -151,17 +155,17 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p
return trades
def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
Combine multiple dataframes "column"
:param tickers: Dict of Dataframes, dict key should be pair.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
"""
df_comb = pd.concat([tickers[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in tickers], axis=1)
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)
@@ -188,3 +192,28 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
# FFill to get continuous
df[col_name] = df[col_name].ffill()
return df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time',
value_col: str = 'profitperc'
) -> Tuple[float, pd.Timestamp, pd.Timestamp]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_time and profitperc)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_time')
:param value_col: Column in DataFrame to use for values (defaults to 'profitperc')
:return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col)
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
high_date = profit_results.loc[max_drawdown_df['high_value'].idxmax(), date_col]
low_date = profit_results.loc[max_drawdown_df['drawdown'].idxmin(), date_col]
return abs(min(max_drawdown_df['drawdown'])), high_date, low_date

55
freqtrade/data/converter.py
View File

@@ -13,12 +13,12 @@ from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
logger = logging.getLogger(__name__)
def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *,
fill_missing: bool = True,
drop_incomplete: bool = True) -> DataFrame:
def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *,
fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame:
"""
Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe
:param ticker: ticker list, as returned by exchange.async_get_candle_history
Converts a list with candle (OHLCV) data (in format returned by ccxt.fetch_ohlcv)
to a Dataframe
:param ohlcv: list with candle (OHLCV) data, as returned by exchange.async_get_candle_history
:param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data
:param pair: Pair this data is for (used to warn if fillup was necessary)
:param fill_missing: fill up missing candles with 0 candles
@@ -26,21 +26,18 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *,
:param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete
:return: DataFrame
"""
logger.debug("Parsing tickerlist to dataframe")
logger.debug(f"Converting candle (OHLCV) data to dataframe for pair {pair}.")
cols = DEFAULT_DATAFRAME_COLUMNS
frame = DataFrame(ticker, columns=cols)
df = DataFrame(ohlcv, columns=cols)
frame['date'] = to_datetime(frame['date'],
unit='ms',
utc=True,
infer_datetime_format=True)
df['date'] = to_datetime(df['date'], unit='ms', utc=True, infer_datetime_format=True)
# Some exchanges return int values for volume and even for ohlc.
# Some exchanges return int values for Volume and even for OHLC.
# Convert them since TA-LIB indicators used in the strategy assume floats
# and fail with exception...
frame = frame.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float',
'volume': 'float'})
return clean_ohlcv_dataframe(frame, timeframe, pair,
df = df.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float',
'volume': 'float'})
return clean_ohlcv_dataframe(df, timeframe, pair,
fill_missing=fill_missing,
drop_incomplete=drop_incomplete)
@@ -49,11 +46,11 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
fill_missing: bool = True,
drop_incomplete: bool = True) -> DataFrame:
"""
Clense a ohlcv dataframe by
Clense a OHLCV dataframe by
* Grouping it by date (removes duplicate tics)
* dropping last candles if requested
* Filling up missing data (if requested)
:param data: DataFrame containing ohlcv data.
:param data: DataFrame containing candle (OHLCV) data.
:param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data
:param pair: Pair this data is for (used to warn if fillup was necessary)
:param fill_missing: fill up missing candles with 0 candles
@@ -88,16 +85,16 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
"""
from freqtrade.exchange import timeframe_to_minutes
ohlc_dict = {
ohlcv_dict = {
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
}
ticker_minutes = timeframe_to_minutes(timeframe)
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to create "NAN" values
df = dataframe.resample(f'{ticker_minutes}min', on='date').agg(ohlc_dict)
df = dataframe.resample(f'{timeframe_minutes}min', on='date').agg(ohlcv_dict)
# Forwardfill close for missing columns
df['close'] = df['close'].fillna(method='ffill')
@@ -159,20 +156,20 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
"""
Converts trades list to ohlcv list
Converts trades list to OHLCV list
TODO: This should get a dedicated test
:param trades: List of trades, as returned by ccxt.fetch_trades.
:param timeframe: Ticker timeframe to resample data to
:return: ohlcv Dataframe.
:param timeframe: Timeframe to resample data to
:return: OHLCV Dataframe.
"""
from freqtrade.exchange import timeframe_to_minutes
ticker_minutes = timeframe_to_minutes(timeframe)
timeframe_minutes = timeframe_to_minutes(timeframe)
df = pd.DataFrame(trades)
df['datetime'] = pd.to_datetime(df['datetime'])
df = df.set_index('datetime')
df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
df_new['date'] = df_new.index
# Drop 0 volume rows
df_new = df_new.dropna()
@@ -206,7 +203,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
"""
Convert ohlcv from one format to another format.
Convert OHLCV from one format to another
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
@@ -216,7 +213,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
src = get_datahandler(config['datadir'], convert_from)
trg = get_datahandler(config['datadir'], convert_to)
timeframes = config.get('timeframes', [config.get('ticker_interval')])
logger.info(f"Converting OHLCV for timeframe {timeframes}")
logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}")
if 'pairs' not in config:
config['pairs'] = []
@@ -224,7 +221,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
for timeframe in timeframes:
config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'],
timeframe))
logger.info(f"Converting OHLCV for {config['pairs']}")
logger.info(f"Converting candle (OHLCV) data for {config['pairs']}")
for timeframe in timeframes:
for pair in config['pairs']:

14
freqtrade/data/dataprovider.py
View File

@@ -1,7 +1,7 @@
"""
Dataprovider
Responsible to provide data to the bot
including Klines, tickers, historic data
including ticker and orderbook data, live and historical candle (OHLCV) data
Common Interface for bot and strategy to access data.
"""
import logging
@@ -43,10 +43,10 @@ class DataProvider:
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
"""
Get ohlcv data for the given pair as DataFrame
Get candle (OHLCV) data for the given pair as DataFrame
Please use the `available_pairs` method to verify which pairs are currently cached.
:param pair: pair to get the data for
:param timeframe: Ticker timeframe to get data for
:param timeframe: Timeframe to get data for
:param copy: copy dataframe before returning if True.
Use False only for read-only operations (where the dataframe is not modified)
"""
@@ -58,7 +58,7 @@ class DataProvider:
def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
"""
Get stored historic ohlcv data
Get stored historical candle (OHLCV) data
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
"""
@@ -69,17 +69,17 @@ class DataProvider:
def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
"""
Return pair ohlcv data, either live or cached historical -- depending
Return pair candle (OHLCV) data, either live or cached historical -- depending
on the runmode.
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:return: Dataframe for this pair
"""
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
# Get live ohlcv data.
# Get live OHLCV data.
data = self.ohlcv(pair=pair, timeframe=timeframe)
else:
# Get historic ohlcv data (cached on disk).
# Get historical OHLCV data (cached on disk).
data = self.historic_ohlcv(pair=pair, timeframe=timeframe)
if len(data) == 0:
logger.warning(f"No data found for ({pair}, {timeframe}).")

22
freqtrade/data/history/history_utils.py
View File

@@ -9,7 +9,7 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
@@ -28,10 +28,10 @@ def load_pair_history(pair: str,
data_handler: IDataHandler = None,
) -> DataFrame:
"""
Load cached ticker history for the given pair.
Load cached ohlcv history for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param datadir: Path to the data storage location.
:param data_format: Format of the data. Ignored if data_handler is set.
:param timerange: Limit data to be loaded to this timerange
@@ -63,10 +63,10 @@ def load_data(datadir: Path,
data_format: str = 'json',
) -> Dict[str, DataFrame]:
"""
Load ticker history data for a list of pairs.
Load ohlcv history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param timerange: Limit data to be loaded to this timerange
:param fill_up_missing: Fill missing values with "No action"-candles
@@ -104,10 +104,10 @@ def refresh_data(datadir: Path,
timerange: Optional[TimeRange] = None,
) -> None:
"""
Refresh ticker history data for a list of pairs.
Refresh ohlcv history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param exchange: Exchange object
:param timerange: Limit data to be loaded to this timerange
@@ -165,7 +165,7 @@ def _download_pair_history(datadir: Path,
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Ticker Timeframe (e.g 5m)
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:return: bool with success state
"""
@@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path,
days=-30).float_timestamp) * 1000
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = parse_ticker_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
if data.empty:
data = new_dataframe
else:
@@ -362,7 +362,7 @@ def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,
:param pair: pair used for log output.
:param min_date: start-date of the data
:param max_date: end-date of the data
:param timeframe_min: ticker Timeframe in minutes
:param timeframe_min: Timeframe in minutes
"""
# total difference in minutes / timeframe-minutes
expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_min)

42
freqtrade/data/history/idatahandler.py
View File

@@ -55,7 +55,7 @@ class IDataHandler(ABC):
Implements the loading and conversion to a Pandas dataframe.
Timerange trimming and dataframe validation happens outside of this method.
:param pair: Pair to load data
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
@@ -67,7 +67,7 @@ class IDataHandler(ABC):
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:return: True when deleted, false if file did not exist.
"""
@@ -129,10 +129,10 @@ class IDataHandler(ABC):
warn_no_data: bool = True
) -> DataFrame:
"""
Load cached ticker history for the given pair.
Load cached candle (OHLCV) data for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange
:param fill_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
@@ -147,12 +147,7 @@ class IDataHandler(ABC):
pairdf = self._ohlcv_load(pair, timeframe,
timerange=timerange_startup)
if pairdf.empty:
if warn_no_data:
logger.warning(
f'No history data for pair: "{pair}", timeframe: {timeframe}. '
'Use `freqtrade download-data` to download the data'
)
if self._check_empty_df(pairdf, pair, timeframe, warn_no_data):
return pairdf
else:
enddate = pairdf.iloc[-1]['date']
@@ -160,13 +155,30 @@ class IDataHandler(ABC):
if timerange_startup:
self._validate_pairdata(pair, pairdf, timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
if self._check_empty_df(pairdf, pair, timeframe, warn_no_data):
return pairdf
# incomplete candles should only be dropped if we didn't trim the end beforehand.
return clean_ohlcv_dataframe(pairdf, timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete and
enddate == pairdf.iloc[-1]['date']))
pairdf = clean_ohlcv_dataframe(pairdf, timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete and
enddate == pairdf.iloc[-1]['date']))
self._check_empty_df(pairdf, pair, timeframe, warn_no_data)
return pairdf
def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, warn_no_data: bool):
"""
Warn on empty dataframe
"""
if pairdf.empty:
if warn_no_data:
logger.warning(
f'No history data for pair: "{pair}", timeframe: {timeframe}. '
'Use `freqtrade download-data` to download the data'
)
return True
return False
def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange):
"""

6
freqtrade/data/history/jsondatahandler.py
View File

@@ -60,7 +60,7 @@ class JsonDataHandler(IDataHandler):
Implements the loading and conversion to a Pandas dataframe.
Timerange trimming and dataframe validation happens outside of this method.
:param pair: Pair to load data
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
@@ -71,6 +71,8 @@ class JsonDataHandler(IDataHandler):
return DataFrame(columns=self._columns)
pairdata = read_json(filename, orient='values')
pairdata.columns = self._columns
pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',
'low': 'float', 'close': 'float', 'volume': 'float'})
pairdata['date'] = to_datetime(pairdata['date'],
unit='ms',
utc=True,
@@ -81,7 +83,7 @@ class JsonDataHandler(IDataHandler):
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)

30
freqtrade/edge/edge_positioning.py
View File

@@ -119,7 +119,7 @@ class Edge:
logger.critical("No data found. Edge is stopped ...")
return False
preprocessed = self.strategy.tickerdata_to_dataframe(data)
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
# Print timeframe
min_date, max_date = history.get_timerange(preprocessed)
@@ -137,10 +137,10 @@ class Edge:
pair_data = pair_data.sort_values(by=['date'])
pair_data = pair_data.reset_index(drop=True)
ticker_data = self.strategy.advise_sell(
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range)
trades += self._find_trades_for_stoploss_range(df_analyzed, pair, self._stoploss_range)
# If no trade found then exit
if len(trades) == 0:
@@ -246,7 +246,8 @@ class Edge:
# we set stake amount to an arbitrary amount.
# as it doesn't change the calculation.
# all returned values are relative. they are percentages.
# all returned values are relative.
# they are defined as ratios.
stake = 0.015
fee = self.fee
open_fee = fee / 2
@@ -269,8 +270,8 @@ class Edge:
result['sell_fee'] = result['sell_sum'] * close_fee
result['sell_take'] = result['sell_sum'] - result['sell_fee']
# profit_percent
result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']
# profit_ratio
result['profit_ratio'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']
# Absolute profit
result['profit_abs'] = result['sell_take'] - result['buy_spend']
@@ -316,7 +317,7 @@ class Edge:
}
# Group by (pair and stoploss) by applying above aggregator
df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg(
df = results.groupby(['pair', 'stoploss'])[['profit_abs', 'trade_duration']].agg(
groupby_aggregator).reset_index(col_level=1)
# Dropping level 0 as we don't need it
@@ -358,11 +359,11 @@ class Edge:
# Returning a list of pairs in order of "expectancy"
return final
def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range):
buy_column = ticker_data['buy'].values
sell_column = ticker_data['sell'].values
date_column = ticker_data['date'].values
ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values
def _find_trades_for_stoploss_range(self, df, pair, stoploss_range):
buy_column = df['buy'].values
sell_column = df['sell'].values
date_column = df['date'].values
ohlc_columns = df[['open', 'high', 'low', 'close']].values
result: list = []
for stoploss in stoploss_range:
@@ -399,9 +400,8 @@ class Edge:
# trade opens in reality on the next candle
open_trade_index += 1
stop_price_percentage = stoploss + 1
open_price = ohlc_columns[open_trade_index, 0]
stop_price = (open_price * stop_price_percentage)
stop_price = (open_price * (stoploss + 1))
# Searching for the index where stoploss is hit
stop_index = utf1st.find_1st(
@@ -441,7 +441,7 @@ class Edge:
trade = {'pair': pair,
'stoploss': stoploss,
'profit_percent': '',
'profit_ratio': '',
'profit_abs': '',
'open_time': date_column[open_trade_index],
'close_time': date_column[exit_index],

137
freqtrade/exchange/exchange.py
View File

@@ -18,7 +18,7 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE,
TRUNCATE, decimal_to_precision)
from pandas import DataFrame
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.data.converter import ohlcv_to_dataframe
from freqtrade.exceptions import (DependencyException, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
@@ -66,8 +66,6 @@ class Exchange:
self._config.update(config)
self._cached_ticker: Dict[str, Any] = {}
# Holds last candle refreshed time of each pair
self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {}
# Timestamp of last markets refresh
@@ -228,6 +226,18 @@ class Exchange:
markets = self.markets
return sorted(set([x['quote'] for _, x in markets.items()]))
def get_pair_quote_currency(self, pair: str) -> str:
"""
Return a pair's quote currency
"""
return self.markets.get(pair, {}).get('quote', '')
def get_pair_base_currency(self, pair: str) -> str:
"""
Return a pair's quote currency
"""
return self.markets.get(pair, {}).get('base', '')
def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame:
if pair_interval in self._klines:
return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
@@ -300,7 +310,7 @@ class Exchange:
if not self.markets:
logger.warning('Unable to validate pairs (assuming they are correct).')
return
invalid_pairs = []
for pair in pairs:
# Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
# TODO: add a support for having coins in BTC/USDT format
@@ -322,6 +332,13 @@ class Exchange:
logger.warning(f"Pair {pair} is restricted for some users on this exchange."
f"Please check if you are impacted by this restriction "
f"on the exchange and eventually remove {pair} from your whitelist.")
if (self._config['stake_currency'] and
self.get_pair_quote_currency(pair) != self._config['stake_currency']):
invalid_pairs.append(pair)
if invalid_pairs:
raise OperationalException(
f"Stake-currency '{self._config['stake_currency']}' not compatible with "
f"pair-whitelist. Please remove the following pairs: {invalid_pairs}")
def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str:
"""
@@ -334,7 +351,7 @@ class Exchange:
def validate_timeframes(self, timeframe: Optional[str]) -> None:
"""
Checks if ticker interval from config is a supported timeframe on the exchange
Check if timeframe from config is a supported timeframe on the exchange
"""
if not hasattr(self._api, "timeframes") or self._api.timeframes is None:
# If timeframes attribute is missing (or is None), the exchange probably
@@ -347,7 +364,7 @@ class Exchange:
if timeframe and (timeframe not in self.timeframes):
raise OperationalException(
f"Invalid ticker interval '{timeframe}'. This exchange supports: {self.timeframes}")
f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}")
if timeframe and timeframe_to_minutes(timeframe) < 1:
raise OperationalException(
@@ -582,7 +599,7 @@ class Exchange:
return self._api.fetch_tickers()
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching tickers in batch.'
f'Exchange {self._api.name} does not support fetching tickers in batch. '
f'Message: {e}') from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
@@ -591,39 +608,28 @@ class Exchange:
raise OperationalException(e) from e
@retrier
def fetch_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict:
if refresh or pair not in self._cached_ticker.keys():
try:
if pair not in self._api.markets or not self._api.markets[pair].get('active'):
raise DependencyException(f"Pair {pair} not available")
data = self._api.fetch_ticker(pair)
try:
self._cached_ticker[pair] = {
'bid': float(data['bid']),
'ask': float(data['ask']),
}
except KeyError:
logger.debug("Could not cache ticker data for %s", pair)
return data
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
else:
logger.info("returning cached ticker-data for %s", pair)
return self._cached_ticker[pair]
def fetch_ticker(self, pair: str) -> dict:
try:
if pair not in self._api.markets or not self._api.markets[pair].get('active'):
raise DependencyException(f"Pair {pair} not available")
data = self._api.fetch_ticker(pair)
return data
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
def get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int) -> List:
"""
Gets candle history using asyncio and returns the list of candles.
Handles all async doing.
Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call.
Get candle history using asyncio and returns the list of candles.
Handles all async work for this.
Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call.
:param pair: Pair to download
:param timeframe: Ticker Timeframe to get
:param timeframe: Timeframe to get data for
:param since_ms: Timestamp in milliseconds to get history from
:returns List of tickers
:returns List with candle (OHLCV) data
"""
return asyncio.get_event_loop().run_until_complete(
self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
@@ -643,26 +649,27 @@ class Exchange:
pair, timeframe, since) for since in
range(since_ms, arrow.utcnow().timestamp * 1000, one_call)]
tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
results = await asyncio.gather(*input_coroutines, return_exceptions=True)
# Combine tickers
# Combine gathered results
data: List = []
for p, timeframe, ticker in tickers:
for p, timeframe, res in results:
if p == pair:
data.extend(ticker)
data.extend(res)
# Sort data again after extending the result - above calls return in "async order"
data = sorted(data, key=lambda x: x[0])
logger.info("downloaded %s with length %s.", pair, len(data))
logger.info("Downloaded data for %s with length %s.", pair, len(data))
return data
def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]:
"""
Refresh in-memory ohlcv asynchronously and set `_klines` with the result
Refresh in-memory OHLCV asynchronously and set `_klines` with the result
Loops asynchronously over pair_list and downloads all pairs async (semi-parallel).
Only used in the dataprovider.refresh() method.
:param pair_list: List of 2 element tuples containing pair, interval to refresh
:return: Returns a List of ticker-dataframes.
:return: TODO: return value is only used in the tests, get rid of it
"""
logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list))
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
input_coroutines = []
@@ -673,15 +680,15 @@ class Exchange:
input_coroutines.append(self._async_get_candle_history(pair, timeframe))
else:
logger.debug(
"Using cached ohlcv data for pair %s, timeframe %s ...",
"Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
pair, timeframe
)
tickers = asyncio.get_event_loop().run_until_complete(
results = asyncio.get_event_loop().run_until_complete(
asyncio.gather(*input_coroutines, return_exceptions=True))
# handle caching
for res in tickers:
for res in results:
if isinstance(res, Exception):
logger.warning("Async code raised an exception: %s", res.__class__.__name__)
continue
@@ -692,13 +699,14 @@ class Exchange:
if ticks:
self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
self._klines[(pair, timeframe)] = parse_ticker_dataframe(
self._klines[(pair, timeframe)] = ohlcv_to_dataframe(
ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
return tickers
return results
def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool:
# Calculating ticker interval in seconds
# Timeframe in seconds
interval_in_sec = timeframe_to_seconds(timeframe)
return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0)
@@ -708,11 +716,11 @@ class Exchange:
async def _async_get_candle_history(self, pair: str, timeframe: str,
since_ms: Optional[int] = None) -> Tuple[str, str, List]:
"""
Asynchronously gets candle histories using fetch_ohlcv
Asynchronously get candle history data using fetch_ohlcv
returns tuple: (pair, timeframe, ohlcv_list)
"""
try:
# fetch ohlcv asynchronously
# Fetch OHLCV asynchronously
s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else ''
logger.debug(
"Fetching pair %s, interval %s, since %s %s...",
@@ -722,9 +730,9 @@ class Exchange:
data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe,
since=since_ms)
# Because some exchange sort Tickers ASC and other DESC.
# Ex: Bittrex returns a list of tickers ASC (oldest first, newest last)
# when GDAX returns a list of tickers DESC (newest first, oldest last)
# Some exchanges sort OHLCV in ASC order and others in DESC.
# Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last)
# while GDAX returns the list of OHLCV in DESC order (newest first, oldest last)
# Only sort if necessary to save computing time
try:
if data and data[0][0] > data[-1][0]:
@@ -737,14 +745,15 @@ class Exchange:
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching historical candlestick data.'
f'Message: {e}') from e
f'Exchange {self._api.name} does not support fetching historical '
f'candle (OHLCV) data. Message: {e}') from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(f'Could not load ticker history for pair {pair} due to '
f'{e.__class__.__name__}. Message: {e}') from e
raise TemporaryError(f'Could not fetch historical candle (OHLCV) data '
f'for pair {pair} due to {e.__class__.__name__}. '
f'Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(f'Could not fetch ticker data for pair {pair}. '
f'Msg: {e}') from e
raise OperationalException(f'Could not fetch historical candle (OHLCV) data '
f'for pair {pair}. Message: {e}') from e
@retrier_async
async def _async_fetch_trades(self, pair: str,
@@ -877,14 +886,14 @@ class Exchange:
until: Optional[int] = None,
from_id: Optional[str] = None) -> Tuple[str, List]:
"""
Gets candle history using asyncio and returns the list of candles.
Handles all async doing.
Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call.
Get trade history data using asyncio.
Handles all async work and returns the list of candles.
Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call.
:param pair: Pair to download
:param since: Timestamp in milliseconds to get history from
:param until: Timestamp in milliseconds. Defaults to current timestamp if not defined.
:param from_id: Download data starting with ID (if id is known)
:returns List of tickers
:returns List of trade data
"""
if not self.exchange_has("fetchTrades"):
raise OperationalException("This exchange does not suport downloading Trades.")
@@ -1018,7 +1027,7 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non
def is_exchange_officially_supported(exchange_name: str) -> bool:
return exchange_name in ['bittrex', 'binance']
return exchange_name in ['bittrex', 'binance', 'kraken']
def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:

132
freqtrade/freqtradebot.py
View File

@@ -6,11 +6,11 @@ import logging
import traceback
from datetime import datetime
from math import isclose
from os import getpid
from threading import Lock
from typing import Any, Dict, List, Optional, Tuple
import arrow
from cachetools import TTLCache
from requests.exceptions import RequestException
from freqtrade import __version__, constants, persistence
@@ -53,9 +53,8 @@ class FreqtradeBot:
# Init objects
self.config = config
self._heartbeat_msg = 0
self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60)
self._sell_rate_cache = TTLCache(maxsize=100, ttl=5)
self._buy_rate_cache = TTLCache(maxsize=100, ttl=5)
self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)
@@ -160,11 +159,6 @@ class FreqtradeBot:
self.check_handle_timedout()
Trade.session.flush()
if (self.heartbeat_interval
and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)):
logger.info(f"Bot heartbeat. PID={getpid()}")
self._heartbeat_msg = arrow.utcnow().timestamp
def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]:
"""
Refresh whitelist from pairlist or edge and extend it with trades.
@@ -179,8 +173,8 @@ class FreqtradeBot:
_whitelist = self.edge.adjust(_whitelist)
if trades:
# Extend active-pair whitelist with pairs from open trades
# It ensures that tickers are downloaded for open trades
# Extend active-pair whitelist with pairs of open trades
# It ensures that candle (OHLCV) data are downloaded for open trades as well
_whitelist.extend([trade.pair for trade in trades if trade.pair not in _whitelist])
return _whitelist
@@ -235,35 +229,43 @@ class FreqtradeBot:
return trades_created
def get_buy_rate(self, pair: str, refresh: bool, tick: Dict = None) -> float:
def get_buy_rate(self, pair: str, refresh: bool) -> float:
"""
Calculates bid target between current ask price and last price
:param pair: Pair to get rate for
:param refresh: allow cached data
:return: float: Price
"""
config_bid_strategy = self.config.get('bid_strategy', {})
if 'use_order_book' in config_bid_strategy and\
config_bid_strategy.get('use_order_book', False):
logger.info('Getting price from order book')
order_book_top = config_bid_strategy.get('order_book_top', 1)
if not refresh:
rate = self._buy_rate_cache.get(pair)
# Check if cache has been invalidated
if rate:
logger.info(f"Using cached buy rate for {pair}.")
return rate
bid_strategy = self.config.get('bid_strategy', {})
if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
logger.info(
f"Getting price from order book {bid_strategy['price_side'].capitalize()} side."
)
order_book_top = bid_strategy.get('order_book_top', 1)
order_book = self.exchange.get_order_book(pair, order_book_top)
logger.debug('order_book %s', order_book)
# top 1 = index 0
order_book_rate = order_book['bids'][order_book_top - 1][0]
logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate)
order_book_rate = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
logger.info(f'...top {order_book_top} order book buy rate {order_book_rate:.8f}')
used_rate = order_book_rate
else:
if not tick:
logger.info('Using Last Ask / Last Price')
ticker = self.exchange.fetch_ticker(pair, refresh)
else:
ticker = tick
if ticker['ask'] < ticker['last']:
ticker_rate = ticker['ask']
else:
logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
ticker = self.exchange.fetch_ticker(pair)
ticker_rate = ticker[bid_strategy['price_side']]
if ticker['last'] and ticker_rate > ticker['last']:
balance = self.config['bid_strategy']['ask_last_balance']
ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
used_rate = ticker_rate
self._buy_rate_cache[pair] = used_rate
return used_rate
def get_trade_stake_amount(self, pair: str) -> float:
@@ -567,7 +569,7 @@ class FreqtradeBot:
"""
Sends rpc notification when a buy cancel occured.
"""
current_rate = self.get_buy_rate(trade.pair, True)
current_rate = self.get_buy_rate(trade.pair, False)
msg = {
'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
@@ -616,23 +618,43 @@ class FreqtradeBot:
return trades_closed
def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1,
order_book_min: int = 1):
"""
Helper generator to query orderbook in loop (used for early sell-order placing)
"""
order_book = self.exchange.get_order_book(pair, order_book_max)
for i in range(order_book_min, order_book_max + 1):
yield order_book[side][i - 1][0]
def get_sell_rate(self, pair: str, refresh: bool) -> float:
"""
Get sell rate - either using get-ticker bid or first bid based on orderbook
Get sell rate - either using ticker bid or first bid based on orderbook
The orderbook portion is only used for rpc messaging, which would otherwise fail
for BitMex (has no bid/ask in fetch_ticker)
or remain static in any other case since it's not updating.
:param pair: Pair to get rate for
:param refresh: allow cached data
:return: Bid rate
"""
config_ask_strategy = self.config.get('ask_strategy', {})
if config_ask_strategy.get('use_order_book', False):
logger.debug('Using order book to get sell rate')
if not refresh:
rate = self._sell_rate_cache.get(pair)
# Check if cache has been invalidated
if rate:
logger.info(f"Using cached sell rate for {pair}.")
return rate
order_book = self.exchange.get_order_book(pair, 1)
rate = order_book['bids'][0][0]
ask_strategy = self.config.get('ask_strategy', {})
if ask_strategy.get('use_order_book', False):
# This code is only used for notifications, selling uses the generator directly
logger.info(
f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
)
rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s"))
else:
rate = self.exchange.fetch_ticker(pair, refresh)['bid']
rate = self.exchange.fetch_ticker(pair)[ask_strategy['price_side']]
self._sell_rate_cache[pair] = rate
return rate
def handle_trade(self, trade: Trade) -> bool:
@@ -650,7 +672,7 @@ class FreqtradeBot:
config_ask_strategy = self.config.get('ask_strategy', {})
if (config_ask_strategy.get('use_sell_signal', True) or
config_ask_strategy.get('ignore_roi_if_buy_signal')):
config_ask_strategy.get('ignore_roi_if_buy_signal', False)):
(buy, sell) = self.strategy.get_signal(
trade.pair, self.strategy.ticker_interval,
self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval))
@@ -661,12 +683,13 @@ class FreqtradeBot:
order_book_min = config_ask_strategy.get('order_book_min', 1)
order_book_max = config_ask_strategy.get('order_book_max', 1)
order_book = self.exchange.get_order_book(trade.pair, order_book_max)
order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s",
order_book_min=order_book_min,
order_book_max=order_book_max)
for i in range(order_book_min, order_book_max + 1):
order_book_rate = order_book['asks'][i - 1][0]
logger.debug(' order book asks top %s: %0.8f', i, order_book_rate)
sell_rate = order_book_rate
sell_rate = next(order_book)
logger.debug(f" order book {config_ask_strategy['price_side']} top {i}: "
f"{sell_rate:0.8f}")
if self._check_and_execute_sell(trade, sell_rate, buy, sell):
return True
@@ -960,8 +983,8 @@ class FreqtradeBot:
"""
# Update wallets to ensure amounts tied up in a stoploss is now free!
self.wallets.update()
wallet_amount = self.wallets.get_free(pair.split('/')[0])
trade_base_currency = self.exchange.get_pair_base_currency(pair)
wallet_amount = self.wallets.get_free(trade_base_currency)
logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}")
if wallet_amount >= amount:
return amount
@@ -1032,10 +1055,10 @@ class FreqtradeBot:
"""
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
# Use cached ticker here - it was updated seconds ago.
# Use cached rates here - it was updated seconds ago.
current_rate = self.get_sell_rate(trade.pair, False)
profit_percent = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_percent > 0 else "loss"
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"
msg = {
'type': RPCMessageType.SELL_NOTIFICATION,
@@ -1048,7 +1071,7 @@ class FreqtradeBot:
'open_rate': trade.open_rate,
'current_rate': current_rate,
'profit_amount': profit_trade,
'profit_percent': profit_percent,
'profit_ratio': profit_ratio,
'sell_reason': trade.sell_reason,
'open_date': trade.open_date,
'close_date': trade.close_date or datetime.utcnow(),
@@ -1070,9 +1093,9 @@ class FreqtradeBot:
"""
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
current_rate = self.get_sell_rate(trade.pair, True)
profit_percent = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_percent > 0 else "loss"
current_rate = self.get_sell_rate(trade.pair, False)
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"
msg = {
'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
@@ -1085,7 +1108,7 @@ class FreqtradeBot:
'open_rate': trade.open_rate,
'current_rate': current_rate,
'profit_amount': profit_trade,
'profit_percent': profit_percent,
'profit_ratio': profit_ratio,
'sell_reason': trade.sell_reason,
'open_date': trade.open_date,
'close_date': trade.close_date,
@@ -1147,12 +1170,13 @@ class FreqtradeBot:
if trade.fee_open == 0 or order['status'] == 'open':
return order_amount
trade_base_currency = self.exchange.get_pair_base_currency(trade.pair)
# use fee from order-dict if possible
if ('fee' in order and order['fee'] is not None and
(order['fee'].keys() >= {'currency', 'cost'})):
if (order['fee']['currency'] is not None and
order['fee']['cost'] is not None and
trade.pair.startswith(order['fee']['currency'])):
trade_base_currency == order['fee']['currency']):
new_amount = order_amount - order['fee']['cost']
logger.info("Applying fee on amount for %s (from %s to %s) from Order",
trade, order['amount'], new_amount)
@@ -1174,7 +1198,7 @@ class FreqtradeBot:
# only applies if fee is in quote currency!
if (exectrade['fee']['currency'] is not None and
exectrade['fee']['cost'] is not None and
trade.pair.startswith(exectrade['fee']['currency'])):
trade_base_currency == exectrade['fee']['currency']):
fee_abs += exectrade['fee']['cost']
if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):

12
freqtrade/misc.py
View File

@@ -81,13 +81,13 @@ def file_load_json(file):
gzipfile = file
# Try gzip file first, otherwise regular json file.
if gzipfile.is_file():
logger.debug('Loading ticker data from file %s', gzipfile)
with gzip.open(gzipfile) as tickerdata:
pairdata = json_load(tickerdata)
logger.debug(f"Loading historical data from file {gzipfile}")
with gzip.open(gzipfile) as datafile:
pairdata = json_load(datafile)
elif file.is_file():
logger.debug('Loading ticker data from file %s', file)
with open(file) as tickerdata:
pairdata = json_load(tickerdata)
logger.debug(f"Loading historical data from file {file}")
with open(file) as datafile:
pairdata = json_load(datafile)
else:
return None
return pairdata

97
freqtrade/optimize/backtesting.py
View File

@@ -88,8 +88,8 @@ class Backtesting:
validate_config_consistency(self.config)
if "ticker_interval" not in self.config:
raise OperationalException("Ticker-interval needs to be set in either configuration "
"or as cli argument `--ticker-interval 5m`")
raise OperationalException("Timeframe (ticker interval) needs to be set in either "
"configuration or as cli argument `--ticker-interval 5m`")
self.timeframe = str(self.config.get('ticker_interval'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
@@ -151,32 +151,33 @@ class Backtesting:
logger.info(f'Dumping backtest results to {recordfilename}')
file_dump_json(recordfilename, records)
def _get_ticker_list(self, processed: Dict) -> Dict[str, DataFrame]:
def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]:
"""
Helper function to convert a processed tickerlist into a list for performance reasons.
Helper function to convert a processed dataframes into lists for performance reasons.
Used by backtest() - so keep this optimized for performance.
"""
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
ticker: Dict = {}
# Create ticker dict
data: Dict = {}
# Create dict with data
for pair, pair_data in processed.items():
pair_data.loc[:, 'buy'] = 0 # cleanup from previous run
pair_data.loc[:, 'sell'] = 0 # cleanup from previous run
ticker_data = self.strategy.advise_sell(
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
# to avoid using data from future, we buy/sell with signal from previous candle
ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
# To avoid using data from future, we use buy/sell signals shifted
# from the previous candle
df_analyzed.loc[:, 'buy'] = df_analyzed['buy'].shift(1)
df_analyzed.loc[:, 'sell'] = df_analyzed['sell'].shift(1)
ticker_data.drop(ticker_data.head(1).index, inplace=True)
df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
ticker[pair] = [x for x in ticker_data.itertuples()]
return ticker
data[pair] = [x for x in df_analyzed.itertuples()]
return data
def _get_close_rate(self, sell_row, trade: Trade, sell: SellCheckTuple,
trade_dur: int) -> float:
@@ -220,7 +221,7 @@ class Backtesting:
def _get_sell_trade_entry(
self, pair: str, buy_row: DataFrame,
partial_ticker: List, trade_count_lock: Dict,
partial_ohlcv: List, trade_count_lock: Dict,
stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]:
trade = Trade(
@@ -235,7 +236,7 @@ class Backtesting:
)
logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.")
# calculate win/lose forwards from buy point
for sell_row in partial_ticker:
for sell_row in partial_ohlcv:
if max_open_trades > 0:
# Increase trade_count_lock for every iteration
trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
@@ -259,9 +260,9 @@ class Backtesting:
close_rate=closerate,
sell_reason=sell.sell_type
)
if partial_ticker:
if partial_ohlcv:
# no sell condition found - trade stil open at end of backtest period
sell_row = partial_ticker[-1]
sell_row = partial_ohlcv[-1]
bt_res = BacktestResult(pair=pair,
profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open),
@@ -308,8 +309,9 @@ class Backtesting:
trades = []
trade_count_lock: Dict = {}
# Dict of ticker-lists for performance (looping lists is a lot faster than dataframes)
ticker: Dict = self._get_ticker_list(processed)
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
lock_pair_until: Dict = {}
# Indexes per pair, so some pairs are allowed to have a missing start.
@@ -319,12 +321,12 @@ class Backtesting:
# Loop timerange and get candle for each pair at that point in time
while tmp < end_date:
for i, pair in enumerate(ticker):
for i, pair in enumerate(data):
if pair not in indexes:
indexes[pair] = 0
try:
row = ticker[pair][indexes[pair]]
row = data[pair][indexes[pair]]
except IndexError:
# missing Data for one pair at the end.
# Warnings for this are shown during data loading
@@ -352,7 +354,7 @@ class Backtesting:
# since indexes has been incremented before, we need to go one step back to
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][indexes[pair]-1:],
trade_entry = self._get_sell_trade_entry(pair, row, data[pair][indexes[pair]-1:],
trade_count_lock, stake_amount,
max_open_trades)
@@ -395,7 +397,7 @@ class Backtesting:
self._set_strategy(strat)
# need to reprocess data every time to populate signals
preprocessed = self.strategy.tickerdata_to_dataframe(data)
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
@@ -419,32 +421,41 @@ class Backtesting:
for strategy, results in all_results.items():
if self.config.get('export', False):
self._store_backtest_result(Path(self.config['exportfilename']), results,
self._store_backtest_result(self.config['exportfilename'], results,
strategy if len(self.strategylist) > 1 else None)
print(f"Result for strategy {strategy}")
print(' BACKTESTING REPORT '.center(133, '='))
print(generate_text_table(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results))
table = generate_text_table(data, stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results)
if isinstance(table, str):
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
print(' SELL REASON STATS '.center(133, '='))
print(generate_text_table_sell_reason(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results))
table = generate_text_table_sell_reason(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results)
if isinstance(table, str):
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
print(' LEFT OPEN TRADES REPORT '.center(133, '='))
print(generate_text_table(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results.loc[results.open_at_end], skip_nan=True))
table = generate_text_table(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results.loc[results.open_at_end], skip_nan=True)
if isinstance(table, str):
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str):
print('=' * len(table.splitlines()[0]))
print()
if len(all_results) > 1:
# Print Strategy summary table
print(' STRATEGY SUMMARY '.center(133, '='))
print(generate_text_table_strategy(self.config['stake_currency'],
self.config['max_open_trades'],
all_results=all_results))
table = generate_text_table_strategy(self.config['stake_currency'],
self.config['max_open_trades'],
all_results=all_results)
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
print(table)
print('=' * len(table.splitlines()[0]))
print('\nFor more details, please look at the detail tables above')

178
freqtrade/optimize/hyperopt.py
View File

@@ -9,6 +9,7 @@ import logging
import random
import sys
import warnings
from math import ceil
from collections import OrderedDict
from operator import itemgetter
from pathlib import Path
@@ -20,7 +21,10 @@ from colorama import Fore, Style
from colorama import init as colorama_init
from joblib import (Parallel, cpu_count, delayed, dump, load,
wrap_non_picklable_objects)
from pandas import DataFrame
from pandas import DataFrame, json_normalize, isna
import tabulate
from os import path
import io
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.history import get_timerange
@@ -73,8 +77,8 @@ class Hyperopt:
self.trials_file = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
self.tickerdata_pickle = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_tickerdata.pkl')
self.data_pickle_file = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_tickerdata.pkl')
self.total_epochs = config.get('epochs', 0)
self.current_best_loss = 100
@@ -115,6 +119,7 @@ class Hyperopt:
self.config['ask_strategy']['use_sell_signal'] = True
self.print_all = self.config.get('print_all', False)
self.hyperopt_table_header = 0
self.print_colorized = self.config.get('print_colorized', False)
self.print_json = self.config.get('print_json', False)
@@ -127,7 +132,7 @@ class Hyperopt:
"""
Remove hyperopt pickle files to restart hyperopt.
"""
for f in [self.tickerdata_pickle, self.trials_file]:
for f in [self.data_pickle_file, self.trials_file]:
p = Path(f)
if p.is_file():
logger.info(f"Removing `{p}`.")
@@ -152,7 +157,7 @@ class Hyperopt:
"""
num_trials = len(self.trials)
if num_trials > self.num_trials_saved:
logger.info(f"Saving {num_trials} {plural(num_trials, 'epoch')}.")
logger.debug(f"Saving {num_trials} {plural(num_trials, 'epoch')}.")
dump(self.trials, self.trials_file)
self.num_trials_saved = num_trials
if final:
@@ -271,8 +276,10 @@ class Hyperopt:
if not self.print_all:
# Separate the results explanation string from dots
print("\n")
self.print_results_explanation(results, self.total_epochs, self.print_all,
self.print_colorized)
self.print_result_table(self.config, results, self.total_epochs,
self.print_all, self.print_colorized,
self.hyperopt_table_header)
self.hyperopt_table_header = 2
@staticmethod
def print_results_explanation(results, total_epochs, highlight_best: bool,
@@ -296,6 +303,142 @@ class Hyperopt:
f"{results['results_explanation']} " +
f"Objective: {results['loss']:.5f}")
@staticmethod
def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
print_colorized: bool, remove_header: int) -> None:
"""
Log result table
"""
if not results:
return
tabulate.PRESERVE_WHITESPACE = True
trials = json_normalize(results, max_level=1)
trials['Best'] = ''
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.total_profit',
'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit',
'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '*'
trials.loc[trials['is_best'], 'Best'] = 'Best'
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
trials['Trades'] = trials['Trades'].astype(str)
trials['Epoch'] = trials['Epoch'].apply(
lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
)
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
)
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
)
trials['Objective'] = trials['Objective'].apply(
lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
)
trials['Profit'] = trials.apply(
lambda x: '{:,.8f} {} {}'.format(
x['Total profit'], config['stake_currency'],
'({:,.2f}%)'.format(x['Profit']).rjust(10, ' ')
).rjust(25+len(config['stake_currency']))
if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])),
axis=1
)
trials = trials.drop(columns=['Total profit'])
if print_colorized:
for i in range(len(trials)):
if trials.loc[i]['is_profit']:
for j in range(len(trials.loc[i])-3):
trials.iat[i, j] = "{}{}{}".format(Fore.GREEN,
str(trials.loc[i][j]), Fore.RESET)
if trials.loc[i]['is_best'] and highlight_best:
for j in range(len(trials.loc[i])-3):
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
str(trials.loc[i][j]), Style.RESET_ALL)
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
if remove_header > 0:
table = tabulate.tabulate(
trials.to_dict(orient='list'), tablefmt='orgtbl',
headers='keys', stralign="right"
)
table = table.split("\n", remove_header)[remove_header]
elif remove_header < 0:
table = tabulate.tabulate(
trials.to_dict(orient='list'), tablefmt='psql',
headers='keys', stralign="right"
)
table = "\n".join(table.split("\n")[0:remove_header])
else:
table = tabulate.tabulate(
trials.to_dict(orient='list'), tablefmt='psql',
headers='keys', stralign="right"
)
print(table)
@staticmethod
def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
csv_file: str) -> None:
"""
Log result to csv-file
"""
if not results:
return
# Verification for overwrite
if path.isfile(csv_file):
logger.error("CSV-File already exists!")
return
try:
io.open(csv_file, 'w+').close()
except IOError:
logger.error("Filed to create CSV-File!")
return
trials = json_normalize(results, max_level=1)
trials['Best'] = ''
trials['Stake currency'] = config['stake_currency']
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '*'
trials.loc[trials['is_best'], 'Best'] = 'Best'
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
trials['Epoch'] = trials['Epoch'].astype(str)
trials['Trades'] = trials['Trades'].astype(str)
trials['Total profit'] = trials['Total profit'].apply(
lambda x: '{:,.8f}'.format(x) if x != 0.0 else ""
)
trials['Profit'] = trials['Profit'].apply(
lambda x: '{:,.2f}'.format(x) if not isna(x) else ""
)
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: '{:,.2f}%'.format(x) if not isna(x) else ""
)
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: '{:,.1f} m'.format(x) if not isna(x) else ""
)
trials['Objective'] = trials['Objective'].apply(
lambda x: '{:,.5f}'.format(x) if x != 100000 else ""
)
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8')
print("CSV-File created!")
def has_space(self, space: str) -> bool:
"""
Tell if the space value is contained in the configuration
@@ -369,7 +512,7 @@ class Hyperopt:
self.backtesting.strategy.trailing_only_offset_is_reached = \
d['trailing_only_offset_is_reached']
processed = load(self.tickerdata_pickle)
processed = load(self.data_pickle_file)
min_date, max_date = get_timerange(processed)
@@ -482,10 +625,10 @@ class Hyperopt:
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
data, timerange = self.backtesting.load_bt_data()
preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
@@ -496,7 +639,7 @@ class Hyperopt:
'Hyperopting with data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
dump(preprocessed, self.tickerdata_pickle)
dump(preprocessed, self.data_pickle_file)
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange = None # type: ignore
@@ -518,16 +661,21 @@ class Hyperopt:
with Parallel(n_jobs=config_jobs) as parallel:
jobs = parallel._effective_n_jobs()
logger.info(f'Effective number of parallel workers used: {jobs}')
EVALS = max(self.total_epochs // jobs, 1)
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
asked = self.opt.ask(n_points=jobs)
# Correct the number of epochs to be processed for the last
# iteration (should not exceed self.total_epochs in total)
n_rest = (i + 1) * jobs - self.total_epochs
current_jobs = jobs - n_rest if n_rest > 0 else jobs
asked = self.opt.ask(n_points=current_jobs)
f_val = self.run_optimizer_parallel(parallel, asked, i)
self.opt.tell(asked, [v['loss'] for v in f_val])
self.fix_optimizer_models_list()
for j in range(jobs):
for j, val in enumerate(f_val):
# Use human-friendly indexes here (starting from 1)
current = i * jobs + j + 1
val = f_val[j]
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")

2
freqtrade/optimize/hyperopt_loss_sharpe.py
View File

@@ -36,7 +36,7 @@ class SharpeHyperOptLoss(IHyperOptLoss):
expected_returns_mean = total_profit.sum() / days_period
up_stdev = np.std(total_profit)
if (np.std(total_profit) != 0.):
if up_stdev != 0:
sharp_ratio = expected_returns_mean / up_stdev * np.sqrt(365)
else:
# Define high (negative) sharpe ratio to be clear that this is NOT optimal.

2
freqtrade/optimize/hyperopt_loss_sharpe_daily.py
View File

@@ -51,7 +51,7 @@ class SharpeHyperOptLossDaily(IHyperOptLoss):
expected_returns_mean = total_profit.mean()
up_stdev = total_profit.std()
if (up_stdev != 0.):
if up_stdev != 0:
sharp_ratio = expected_returns_mean / up_stdev * math.sqrt(days_in_year)
else:
# Define high (negative) sharpe ratio to be clear that this is NOT optimal.

49
freqtrade/optimize/hyperopt_loss_sortino.py Normal file
View File

@@ -0,0 +1,49 @@
"""
SortinoHyperOptLoss
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
from datetime import datetime
from pandas import DataFrame
import numpy as np
from freqtrade.optimize.hyperopt import IHyperOptLoss
class SortinoHyperOptLoss(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation uses the Sortino Ratio calculation.
"""
@staticmethod
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
*args, **kwargs) -> float:
"""
Objective function, returns smaller number for more optimal results.
Uses Sortino Ratio calculation.
"""
total_profit = results["profit_percent"]
days_period = (max_date - min_date).days
# adding slippage of 0.1% per trade
total_profit = total_profit - 0.0005
expected_returns_mean = total_profit.sum() / days_period
results['downside_returns'] = 0
results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent']
down_stdev = np.std(results['downside_returns'])
if down_stdev != 0:
sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365)
else:
# Define high (negative) sortino ratio to be clear that this is NOT optimal.
sortino_ratio = -20.
# print(expected_returns_mean, down_stdev, sortino_ratio)
return -sortino_ratio

70
freqtrade/optimize/hyperopt_loss_sortino_daily.py Normal file
View File

@@ -0,0 +1,70 @@
"""
SortinoHyperOptLossDaily
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
import math
from datetime import datetime
from pandas import DataFrame, date_range
from freqtrade.optimize.hyperopt import IHyperOptLoss
class SortinoHyperOptLossDaily(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation uses the Sortino Ratio calculation.
"""
@staticmethod
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
*args, **kwargs) -> float:
"""
Objective function, returns smaller number for more optimal results.
Uses Sortino Ratio calculation.
Sortino Ratio calculated as described in
http://www.redrockcapital.com/Sortino__A__Sharper__Ratio_Red_Rock_Capital.pdf
"""
resample_freq = '1D'
slippage_per_trade_ratio = 0.0005
days_in_year = 365
minimum_acceptable_return = 0.0
# apply slippage per trade to profit_percent
results.loc[:, 'profit_percent_after_slippage'] = \
results['profit_percent'] - slippage_per_trade_ratio
# create the index within the min_date and end max_date
t_index = date_range(start=min_date, end=max_date, freq=resample_freq,
normalize=True)
sum_daily = (
results.resample(resample_freq, on='close_time').agg(
{"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
)
total_profit = sum_daily["profit_percent_after_slippage"] - minimum_acceptable_return
expected_returns_mean = total_profit.mean()
sum_daily['downside_returns'] = 0
sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit
total_downside = sum_daily['downside_returns']
# Here total_downside contains min(0, P - MAR) values,
# where P = sum_daily["profit_percent_after_slippage"]
down_stdev = math.sqrt((total_downside**2).sum() / len(total_downside))
if down_stdev != 0:
sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year)
else:
# Define high (negative) sortino ratio to be clear that this is NOT optimal.
sortino_ratio = -20.
# print(t_index, sum_daily, total_profit)
# print(minimum_acceptable_return, expected_returns_mean, down_stdev, sortino_ratio)
return -sortino_ratio

8
freqtrade/optimize/optimize_reports.py
View File

@@ -66,7 +66,7 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra
])
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def generate_text_table_sell_reason(
@@ -112,7 +112,7 @@ def generate_text_table_sell_reason(
profit_percent_tot,
]
)
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
return tabulate(tabular_data, headers=headers, tablefmt="orgtbl", stralign="right")
def generate_text_table_strategy(stake_currency: str, max_open_trades: str,
@@ -146,7 +146,7 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str,
])
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def generate_edge_table(results: dict) -> str:
@@ -172,4 +172,4 @@ def generate_edge_table(results: dict) -> str:
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore

28
freqtrade/pairlist/IPairList.py
View File

@@ -67,21 +67,37 @@ class IPairList(ABC):
"""
@staticmethod
def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]:
def verify_blacklist(pairlist: List[str], blacklist: List[str],
aswarning: bool) -> List[str]:
"""
Verify and remove items from pairlist - returning a filtered pairlist.
Logs a warning or info depending on `aswarning`.
Pairlists explicitly using this method shall use `aswarning=False`!
:param pairlist: Pairlist to validate
:param blacklist: Blacklist to validate pairlist against
:param aswarning: Log message as Warning or info
:return: pairlist - blacklisted pairs
"""
for pair in deepcopy(pairlist):
if pair in blacklist:
logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...")
if aswarning:
logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...")
else:
logger.info(f"Pair {pair} in your blacklist. Removing it from whitelist...")
pairlist.remove(pair)
return pairlist
def _verify_blacklist(self, pairlist: List[str]) -> List[str]:
def _verify_blacklist(self, pairlist: List[str], aswarning: bool = True) -> List[str]:
"""
Proxy method to verify_blacklist for easy access for child classes.
Logs a warning or info depending on `aswarning`.
Pairlists explicitly using this method shall use aswarning=False!
:param pairlist: Pairlist to validate
:param aswarning: Log message as Warning or info.
:return: pairlist - blacklisted pairs
"""
return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist)
return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist,
aswarning=aswarning)
def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
"""
@@ -99,7 +115,8 @@ class IPairList(ABC):
logger.warning(f"Pair {pair} is not compatible with exchange "
f"{self._exchange.name}. Removing it from whitelist..")
continue
if not pair.endswith(self._config['stake_currency']):
if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
logger.warning(f"Pair {pair} is not compatible with your stake currency "
f"{self._config['stake_currency']}. Removing it from whitelist..")
continue
@@ -112,6 +129,5 @@ class IPairList(ABC):
if pair not in sanitized_whitelist:
sanitized_whitelist.append(pair)
sanitized_whitelist = self._verify_blacklist(sanitized_whitelist)
# We need to remove pairs that are unknown
return sanitized_whitelist

6
freqtrade/pairlist/VolumePairList.py
View File

@@ -91,9 +91,9 @@ class VolumePairList(IPairList):
if self._pairlist_pos == 0:
# If VolumePairList is the first in the list, use fresh pairlist
# check length so that we make sure that '/' is actually in the string
# Check if pair quote currency equals to the stake currency.
filtered_tickers = [v for k, v in tickers.items()
if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency
if (self._exchange.get_pair_quote_currency(k) == base_currency
and v[key] is not None)]
else:
# If other pairlist is in front, use the incomming pairlist.
@@ -106,7 +106,7 @@ class VolumePairList(IPairList):
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
pairs = self._verify_blacklist(pairs)
pairs = self._verify_blacklist(pairs, aswarning=False)
# Limit to X number of pairs
pairs = pairs[:self._number_pairs]
logger.info(f"Searching {self._number_pairs} pairs: {pairs}")

2
freqtrade/pairlist/pairlistmanager.py
View File

@@ -91,6 +91,6 @@ class PairListManager():
pairlist = pl.filter_pairlist(pairlist, tickers)
# Validation against blacklist happens after the pairlists to ensure blacklist is respected.
pairlist = IPairList.verify_blacklist(pairlist, self.blacklist)
pairlist = IPairList.verify_blacklist(pairlist, self.blacklist, True)
self._whitelist = pairlist

4
freqtrade/persistence.py
View File

@@ -405,8 +405,8 @@ class Trade(_DECL_BASE):
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
)
profit_percent = (close_trade_price / self.open_trade_price) - 1
return float(f"{profit_percent:.8f}")
profit_ratio = (close_trade_price / self.open_trade_price) - 1
return float(f"{profit_ratio:.8f}")
@staticmethod
def get_trades(trade_filter=None) -> Query:

58
freqtrade/plot/plotting.py
View File

@@ -5,7 +5,8 @@ from typing import Any, Dict, List
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.data.btanalysis import (combine_tickers_with_mean,
from freqtrade.data.btanalysis import (calculate_max_drawdown,
combine_dataframes_with_mean,
create_cum_profit,
extract_trades_of_period, load_trades)
from freqtrade.data.converter import trim_dataframe
@@ -28,7 +29,7 @@ except ImportError:
def init_plotscript(config):
"""
Initialize objects needed for plotting
:return: Dict with tickers, trades and pairs
:return: Dict with candle (OHLCV) data, trades and pairs
"""
if "pairs" in config:
@@ -39,7 +40,7 @@ def init_plotscript(config):
# Set timerange to use
timerange = TimeRange.parse_timerange(config.get("timerange"))
tickers = load_data(
data = load_data(
datadir=config.get("datadir"),
pairs=pairs,
timeframe=config.get('ticker_interval', '5m'),
@@ -52,7 +53,7 @@ def init_plotscript(config):
exportfilename=config.get('exportfilename'),
)
trades = trim_dataframe(trades, timerange, 'open_time')
return {"tickers": tickers,
return {"ohlcv": data,
"trades": trades,
"pairs": pairs,
}
@@ -111,6 +112,36 @@ def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> make_sub
return fig
def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> make_subplots:
"""
Add scatter points indicating max drawdown
"""
try:
max_drawdown, highdate, lowdate = calculate_max_drawdown(trades)
drawdown = go.Scatter(
x=[highdate, lowdate],
y=[
df_comb.loc[highdate, 'cum_profit'],
df_comb.loc[lowdate, 'cum_profit'],
],
mode='markers',
name=f"Max drawdown {max_drawdown:.2f}%",
text=f"Max drawdown {max_drawdown:.2f}%",
marker=dict(
symbol='square-open',
size=9,
line=dict(width=2),
color='green'
)
)
fig.add_trace(drawdown, row, 1)
except ValueError:
logger.warning("No trades found - not plotting max drawdown.")
return fig
def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
"""
Add trades to "fig"
@@ -337,10 +368,10 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
return fig
def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
trades: pd.DataFrame, timeframe: str) -> go.Figure:
# Combine close-values for all pairs, rename columns to "pair"
df_comb = combine_tickers_with_mean(tickers, "close")
df_comb = combine_dataframes_with_mean(data, "close")
# Add combined cumulative profit
df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe)
@@ -364,6 +395,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
fig.add_trace(avgclose, 1, 1)
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
fig = add_max_drawdown(fig, 2, trades, df_comb)
for pair in pairs:
profit_col = f'cum_profit_{pair}'
@@ -407,7 +439,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
"""
From configuration provided
- Initializes plot-script
- Get tickers data
- Get candle (OHLCV) data
- Generate Dafaframes populated with indicators and signals based on configured strategy
- Load trades excecuted during the selected period
- Generate Plotly plot objects
@@ -419,19 +451,17 @@ def load_and_plot_trades(config: Dict[str, Any]):
plot_elements = init_plotscript(config)
trades = plot_elements['trades']
pair_counter = 0
for pair, data in plot_elements["tickers"].items():
for pair, data in plot_elements["ohlcv"].items():
pair_counter += 1
logger.info("analyse pair %s", pair)
tickers = {}
tickers[pair] = data
dataframe = strategy.analyze_ticker(tickers[pair], {'pair': pair})
df_analyzed = strategy.analyze_ticker(data, {'pair': pair})
trades_pair = trades.loc[trades['pair'] == pair]
trades_pair = extract_trades_of_period(dataframe, trades_pair)
trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
fig = generate_candlestick_graph(
pair=pair,
data=dataframe,
data=df_analyzed,
trades=trades_pair,
indicators1=config.get("indicators1", []),
indicators2=config.get("indicators2", []),
@@ -462,7 +492,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
# Create an average close price of all the pairs that were involved.
# this could be useful to gauge the overall market trend
fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"],
fig = generate_profit_graph(plot_elements["pairs"], plot_elements["ohlcv"],
trades, config.get('ticker_interval', '5m'))
store_plot_file(fig, filename='freqtrade-profit-plot.html',
directory=config['user_data_dir'] / "plot", auto_open=True)

52
freqtrade/rpc/fiat_convert.py
View File

@@ -7,7 +7,7 @@ import logging
import time
from typing import Dict, List
from coinmarketcap import Market
from pycoingecko import CoinGeckoAPI
from freqtrade.constants import SUPPORTED_FIAT
@@ -38,8 +38,8 @@ class CryptoFiat:
# Private attributes
self._expiration = 0.0
self.crypto_symbol = crypto_symbol.upper()
self.fiat_symbol = fiat_symbol.upper()
self.crypto_symbol = crypto_symbol.lower()
self.fiat_symbol = fiat_symbol.lower()
self.set_price(price=price)
def set_price(self, price: float) -> None:
@@ -67,17 +67,20 @@ class CryptoToFiatConverter:
This object is also a Singleton
"""
__instance = None
_coinmarketcap: Market = None
_coingekko: CoinGeckoAPI = None
_cryptomap: Dict = {}
def __new__(cls):
"""
This class is a singleton - cannot be instantiated twice.
"""
if CryptoToFiatConverter.__instance is None:
CryptoToFiatConverter.__instance = object.__new__(cls)
try:
CryptoToFiatConverter._coinmarketcap = Market()
CryptoToFiatConverter._coingekko = CoinGeckoAPI()
except BaseException:
CryptoToFiatConverter._coinmarketcap = None
CryptoToFiatConverter._coingekko = None
return CryptoToFiatConverter.__instance
def __init__(self) -> None:
@@ -86,14 +89,12 @@ class CryptoToFiatConverter:
def _load_cryptomap(self) -> None:
try:
coinlistings = self._coinmarketcap.listings()
self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])),
coinlistings["data"]))
except (BaseException) as exception:
coinlistings = self._coingekko.get_coins_list()
# Create mapping table from synbol to coingekko_id
self._cryptomap = {x['symbol']: x['id'] for x in coinlistings}
except (Exception) as exception:
logger.error(
"Could not load FIAT Cryptocurrency map for the following problem: %s",
type(exception).__name__
)
f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float:
"""
@@ -115,8 +116,8 @@ class CryptoToFiatConverter:
:param fiat_symbol: FIAT currency you want to convert to (e.g USD)
:return: Price in FIAT
"""
crypto_symbol = crypto_symbol.upper()
fiat_symbol = fiat_symbol.upper()
crypto_symbol = crypto_symbol.lower()
fiat_symbol = fiat_symbol.lower()
# Check if the fiat convertion you want is supported
if not self._is_supported_fiat(fiat=fiat_symbol):
@@ -170,15 +171,13 @@ class CryptoToFiatConverter:
:return: bool, True supported, False not supported
"""
fiat = fiat.upper()
return fiat in SUPPORTED_FIAT
return fiat.upper() in SUPPORTED_FIAT
def _find_price(self, crypto_symbol: str, fiat_symbol: str) -> float:
"""
Call CoinMarketCap API to retrieve the price in the FIAT
:param crypto_symbol: Crypto-currency you want to convert (e.g BTC)
:param fiat_symbol: FIAT currency you want to convert to (e.g USD)
Call CoinGekko API to retrieve the price in the FIAT
:param crypto_symbol: Crypto-currency you want to convert (e.g btc)
:param fiat_symbol: FIAT currency you want to convert to (e.g usd)
:return: float, price of the crypto-currency in Fiat
"""
# Check if the fiat convertion you want is supported
@@ -195,12 +194,13 @@ class CryptoToFiatConverter:
return 0.0
try:
_gekko_id = self._cryptomap[crypto_symbol]
return float(
self._coinmarketcap.ticker(
currency=self._cryptomap[crypto_symbol],
convert=fiat_symbol
)['data']['quotes'][fiat_symbol.upper()]['price']
self._coingekko.get_price(
ids=_gekko_id,
vs_currencies=fiat_symbol
)[_gekko_id][fiat_symbol]
)
except BaseException as exception:
except Exception as exception:
logger.error("Error in _find_price: %s", exception)
return 0.0

26
freqtrade/rpc/rpc.py
View File

@@ -155,9 +155,9 @@ class RPC:
current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
except DependencyException:
current_rate = NAN
trade_perc = (100 * trade.calc_profit_ratio(current_rate))
trade_percent = (100 * trade.calc_profit_ratio(current_rate))
trade_profit = trade.calc_profit(current_rate)
profit_str = f'{trade_perc:.2f}%'
profit_str = f'{trade_percent:.2f}%'
if self._fiat_converter:
fiat_profit = self._fiat_converter.convert_amount(
trade_profit,
@@ -232,9 +232,9 @@ class RPC:
trades = Trade.get_trades().order_by(Trade.id).all()
profit_all_coin = []
profit_all_perc = []
profit_all_ratio = []
profit_closed_coin = []
profit_closed_perc = []
profit_closed_ratio = []
durations = []
for trade in trades:
@@ -246,21 +246,21 @@ class RPC:
durations.append((trade.close_date - trade.open_date).total_seconds())
if not trade.is_open:
profit_percent = trade.calc_profit_ratio()
profit_ratio = trade.calc_profit_ratio()
profit_closed_coin.append(trade.calc_profit())
profit_closed_perc.append(profit_percent)
profit_closed_ratio.append(profit_ratio)
else:
# Get current rate
try:
current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
except DependencyException:
current_rate = NAN
profit_percent = trade.calc_profit_ratio(rate=current_rate)
profit_ratio = trade.calc_profit_ratio(rate=current_rate)
profit_all_coin.append(
trade.calc_profit(rate=trade.close_rate or current_rate)
)
profit_all_perc.append(profit_percent)
profit_all_ratio.append(profit_ratio)
best_pair = Trade.get_best_pair()
@@ -271,7 +271,7 @@ class RPC:
# Prepare data to display
profit_closed_coin_sum = round(sum(profit_closed_coin), 8)
profit_closed_percent = (round(mean(profit_closed_perc) * 100, 2) if profit_closed_perc
profit_closed_percent = (round(mean(profit_closed_ratio) * 100, 2) if profit_closed_ratio
else 0.0)
profit_closed_fiat = self._fiat_converter.convert_amount(
profit_closed_coin_sum,
@@ -280,7 +280,7 @@ class RPC:
) if self._fiat_converter else 0
profit_all_coin_sum = round(sum(profit_all_coin), 8)
profit_all_percent = round(mean(profit_all_perc) * 100, 2) if profit_all_perc else 0.0
profit_all_percent = round(mean(profit_all_ratio) * 100, 2) if profit_all_ratio else 0.0
profit_all_fiat = self._fiat_converter.convert_amount(
profit_all_coin_sum,
stake_currency,
@@ -460,9 +460,9 @@ class RPC:
if self._freqtrade.state != State.RUNNING:
raise RPCException('trader is not running')
# Check pair is in stake currency
# Check if pair quote currency equals to the stake currency.
stake_currency = self._freqtrade.config.get('stake_currency')
if not pair.endswith(stake_currency):
if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency:
raise RPCException(
f'Wrong pair selected. Please pairs with stake {stake_currency} pairs only')
# check if valid pair
@@ -517,7 +517,7 @@ class RPC:
if add:
stake_currency = self._freqtrade.config.get('stake_currency')
for pair in add:
if (pair.endswith(stake_currency)
if (self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency
and pair not in self._freqtrade.pairlists.blacklist):
self._freqtrade.pairlists.blacklist.append(pair)

2
freqtrade/rpc/telegram.py
View File

@@ -148,7 +148,7 @@ class Telegram(RPC):
elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
msg['amount'] = round(msg['amount'], 8)
msg['profit_percent'] = round(msg['profit_percent'] * 100, 2)
msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
msg['duration'] = msg['close_date'].replace(
microsecond=0) - msg['open_date'].replace(microsecond=0)
msg['duration_min'] = msg['duration'].total_seconds() / 60

35
freqtrade/strategy/interface.py
View File

@@ -60,7 +60,7 @@ class IStrategy(ABC):
Attributes you can use:
minimal_roi -> Dict: Minimal ROI designed for the strategy
stoploss -> float: optimal stoploss designed for the strategy
ticker_interval -> str: value of the ticker interval to use for the strategy
ticker_interval -> str: value of the timeframe (ticker interval) to use with the strategy
"""
# Strategy interface version
# Default to version 2
@@ -126,7 +126,7 @@ class IStrategy(ABC):
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Populate indicators that will be used in the Buy and Sell strategy
:param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe()
:param dataframe: DataFrame with data from the exchange
:param metadata: Additional information, like the currently traded pair
:return: a Dataframe with all mandatory indicators for the strategies
"""
@@ -237,11 +237,11 @@ class IStrategy(ABC):
def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Parses the given ticker history and returns a populated DataFrame
Parses the given candle (OHLCV) data and returns a populated DataFrame
add several TA indicators and buy signal to it
:param dataframe: Dataframe containing ticker data
:param dataframe: Dataframe containing data from exchange
:param metadata: Metadata dictionary with additional data (e.g. 'pair')
:return: DataFrame with ticker data and indicator data
:return: DataFrame of candle (OHLCV) data with indicator data and signals added
"""
logger.debug("TA Analysis Launched")
dataframe = self.advise_indicators(dataframe, metadata)
@@ -251,12 +251,12 @@ class IStrategy(ABC):
def _analyze_ticker_internal(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Parses the given ticker history and returns a populated DataFrame
Parses the given candle (OHLCV) data and returns a populated DataFrame
add several TA indicators and buy signal to it
WARNING: Used internally only, may skip analysis if `process_only_new_candles` is set.
:param dataframe: Dataframe containing ticker data
:param dataframe: Dataframe containing data from exchange
:param metadata: Metadata dictionary with additional data (e.g. 'pair')
:return: DataFrame with ticker data and indicator data
:return: DataFrame of candle (OHLCV) data with indicator data and signals added
"""
pair = str(metadata.get('pair'))
@@ -288,7 +288,7 @@ class IStrategy(ABC):
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty ticker history for pair %s', pair)
logger.warning('Empty candle (OHLCV) data for pair %s', pair)
return False, False
try:
@@ -296,7 +296,7 @@ class IStrategy(ABC):
self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair})
except StrategyError as error:
logger.warning(f"Unable to analyze ticker for pair {pair}: {error}")
logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
return False, False
@@ -393,7 +393,7 @@ class IStrategy(ABC):
"""
Based on current profit of the trade and configured (trailing) stoploss,
decides to sell or not
:param current_profit: current profit in percent
:param current_profit: current profit as ratio
"""
stop_loss_value = force_stoploss if force_stoploss else self.stoploss
@@ -456,8 +456,9 @@ class IStrategy(ABC):
def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool:
"""
Based on trade duration, current price and ROI configuration, decides whether bot should
sell. Requires current_profit to be in percent!!
Based on trade duration, current profit of the trade and ROI configuration,
decides whether bot should sell.
:param current_profit: current profit as ratio
:return: True if bot should sell at current rate
"""
# Check if time matches and current rate is above threshold
@@ -468,19 +469,19 @@ class IStrategy(ABC):
else:
return current_profit > roi
def tickerdata_to_dataframe(self, tickerdata: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
"""
Creates a dataframe and populates indicators for given ticker data
Creates a dataframe and populates indicators for given candle (OHLCV) data
Used by optimize operations only, not during dry / live runs.
"""
return {pair: self.advise_indicators(pair_data, {'pair': pair})
for pair, pair_data in tickerdata.items()}
for pair, pair_data in data.items()}
def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Populate indicators that will be used in the Buy and Sell strategy
This method should not be overridden.
:param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe()
:param dataframe: Dataframe with data from the exchange
:param metadata: Additional information, like the currently traded pair
:return: a Dataframe with all mandatory indicators for the strategies
"""

4
freqtrade/templates/base_config.json.j2
View File

@@ -11,6 +11,7 @@
"sell": 30
},
"bid_strategy": {
"price_side": "bid",
"ask_last_balance": 0.0,
"use_order_book": false,
"order_book_top": 1,
@@ -20,9 +21,10 @@
}
},
"ask_strategy": {
"price_side": "ask",
"use_order_book": false,
"order_book_min": 1,
"order_book_max": 9,
"order_book_max": 1,
"use_sell_signal": true,
"sell_profit_only": false,
"ignore_roi_if_buy_signal": false

21
freqtrade/templates/base_hyperopt.py.j2
View File

@@ -21,7 +21,7 @@ class {{ hyperopt }}(IHyperOpt):
"""
This is a Hyperopt template to get you started.
More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md
More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/
You should:
- Add any lib you need to build your hyperopt.
@@ -29,11 +29,14 @@ class {{ hyperopt }}(IHyperOpt):
You must keep:
- The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator.
The roi_space, generate_roi_table, stoploss_space methods are no longer required to be
copied in every custom hyperopt. However, you may override them if you need the
'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade.
Sample implementation of these methods can be found in
https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py
The methods roi_space, generate_roi_table and stoploss_space are not required
and are provided by default.
However, you may override them if you need 'roi' and 'stoploss' spaces that
differ from the defaults offered by Freqtrade.
Sample implementation of these methods will be copied to `user_data/hyperopts` when
creating the user-data directory using `freqtrade create-userdir --userdir user_data`,
or is available online under the following URL:
https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py.
"""
@staticmethod
@@ -63,6 +66,9 @@ class {{ hyperopt }}(IHyperOpt):
dataframe['close'], dataframe['sar']
))
# Check that the candle had volume
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
@@ -108,6 +114,9 @@ class {{ hyperopt }}(IHyperOpt):
dataframe['sar'], dataframe['close']
))
# Check that the candle had volume
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),

2
freqtrade/templates/base_strategy.py.j2
View File

@@ -99,7 +99,7 @@ class {{ strategy }}(IStrategy):
Performance Note: For the best performance be frugal on the number of indicators
you are using. Let uncomment only the indicator you are using in your strategies
or your hyperopt configuration, otherwise you will waste your memory and CPU usage.
:param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe()
:param dataframe: Dataframe with data from the exchange
:param metadata: Additional information, like the currently traded pair
:return: a Dataframe with all mandatory indicators for the strategies
"""

25
freqtrade/templates/sample_hyperopt.py
View File

@@ -20,23 +20,28 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib
class SampleHyperOpt(IHyperOpt):
"""
This is a sample Hyperopt to inspire you.
Feel free to customize it.
More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md
More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/
You should:
- Rename the class name to some unique name.
- Add any methods you want to build your hyperopt.
- Add any lib you need to build your hyperopt.
An easier way to get a new hyperopt file is by using
`freqtrade new-hyperopt --hyperopt MyCoolHyperopt`.
You must keep:
- The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator.
The roi_space, generate_roi_table, stoploss_space methods are no longer required to be
copied in every custom hyperopt. However, you may override them if you need the
'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade.
Sample implementation of these methods can be found in
https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py
The methods roi_space, generate_roi_table and stoploss_space are not required
and are provided by default.
However, you may override them if you need 'roi' and 'stoploss' spaces that
differ from the defaults offered by Freqtrade.
Sample implementation of these methods will be copied to `user_data/hyperopts` when
creating the user-data directory using `freqtrade create-userdir --userdir user_data`,
or is available online under the following URL:
https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py.
"""
@staticmethod
@@ -73,6 +78,9 @@ class SampleHyperOpt(IHyperOpt):
dataframe['close'], dataframe['sar']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
@@ -133,6 +141,9 @@ class SampleHyperOpt(IHyperOpt):
dataframe['sar'], dataframe['close']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),

13
freqtrade/templates/sample_hyperopt_advanced.py
View File

@@ -22,7 +22,7 @@ class AdvancedSampleHyperOpt(IHyperOpt):
This is a sample hyperopt to inspire you.
Feel free to customize it.
More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md
More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/
You should:
- Rename the class name to some unique name.
@@ -32,8 +32,9 @@ class AdvancedSampleHyperOpt(IHyperOpt):
You must keep:
- The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator.
The roi_space, generate_roi_table, stoploss_space methods are no longer required to be
copied in every custom hyperopt. However, you may override them if you need the
The methods roi_space, generate_roi_table and stoploss_space are not required
and are provided by default.
However, you may override them if you need the
'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade.
This sample illustrates how to override these methods.
@@ -92,6 +93,9 @@ class AdvancedSampleHyperOpt(IHyperOpt):
dataframe['close'], dataframe['sar']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
@@ -152,6 +156,9 @@ class AdvancedSampleHyperOpt(IHyperOpt):
dataframe['sar'], dataframe['close']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),

126
freqtrade/templates/sample_strategy.py
View File

@@ -116,7 +116,7 @@ class SampleStrategy(IStrategy):
Performance Note: For the best performance be frugal on the number of indicators
you are using. Let uncomment only the indicator you are using in your strategies
or your hyperopt configuration, otherwise you will waste your memory and CPU usage.
:param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe()
:param dataframe: Dataframe with data from the exchange
:param metadata: Additional information, like the currently traded pair
:return: a Dataframe with all mandatory indicators for the strategies
"""
@@ -124,24 +124,70 @@ class SampleStrategy(IStrategy):
# Momentum Indicators
# ------------------------------------
# RSI
dataframe['rsi'] = ta.RSI(dataframe)
# ADX
dataframe['adx'] = ta.ADX(dataframe)
# # Plus Directional Indicator / Movement
# dataframe['plus_dm'] = ta.PLUS_DM(dataframe)
# dataframe['plus_di'] = ta.PLUS_DI(dataframe)
# # Minus Directional Indicator / Movement
# dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # Aroon, Aroon Oscillator
# aroon = ta.AROON(dataframe)
# dataframe['aroonup'] = aroon['aroonup']
# dataframe['aroondown'] = aroon['aroondown']
# dataframe['aroonosc'] = ta.AROONOSC(dataframe)
# # Awesome oscillator
# # Awesome Oscillator
# dataframe['ao'] = qtpylib.awesome_oscillator(dataframe)
# # Commodity Channel Index: values Oversold:<-100, Overbought:>100
# # Keltner Channel
# keltner = qtpylib.keltner_channel(dataframe)
# dataframe["kc_upperband"] = keltner["upper"]
# dataframe["kc_lowerband"] = keltner["lower"]
# dataframe["kc_middleband"] = keltner["mid"]
# dataframe["kc_percent"] = (
# (dataframe["close"] - dataframe["kc_lowerband"]) /
# (dataframe["kc_upperband"] - dataframe["kc_lowerband"])
# )
# dataframe["kc_width"] = (
# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"]
# )
# # Ultimate Oscillator
# dataframe['uo'] = ta.ULTOSC(dataframe)
# # Commodity Channel Index: values [Oversold:-100, Overbought:100]
# dataframe['cci'] = ta.CCI(dataframe)
# RSI
dataframe['rsi'] = ta.RSI(dataframe)
# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy)
# rsi = 0.1 * (dataframe['rsi'] - 50)
# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1)
# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy)
# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1)
# # Stochastic Slow
# stoch = ta.STOCH(dataframe)
# dataframe['slowd'] = stoch['slowd']
# dataframe['slowk'] = stoch['slowk']
# Stochastic Fast
stoch_fast = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch_fast['fastd']
dataframe['fastk'] = stoch_fast['fastk']
# # Stochastic RSI
# stoch_rsi = ta.STOCHRSI(dataframe)
# dataframe['fastd_rsi'] = stoch_rsi['fastd']
# dataframe['fastk_rsi'] = stoch_rsi['fastk']
# MACD
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
@@ -151,60 +197,58 @@ class SampleStrategy(IStrategy):
# MFI
dataframe['mfi'] = ta.MFI(dataframe)
# # Minus Directional Indicator / Movement
# dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # Plus Directional Indicator / Movement
# dataframe['plus_dm'] = ta.PLUS_DM(dataframe)
# dataframe['plus_di'] = ta.PLUS_DI(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # ROC
# dataframe['roc'] = ta.ROC(dataframe)
# # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy)
# rsi = 0.1 * (dataframe['rsi'] - 50)
# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1)
# # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy)
# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1)
# # Stoch
# stoch = ta.STOCH(dataframe)
# dataframe['slowd'] = stoch['slowd']
# dataframe['slowk'] = stoch['slowk']
# Stoch fast
stoch_fast = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch_fast['fastd']
dataframe['fastk'] = stoch_fast['fastk']
# # Stoch RSI
# stoch_rsi = ta.STOCHRSI(dataframe)
# dataframe['fastd_rsi'] = stoch_rsi['fastd']
# dataframe['fastk_rsi'] = stoch_rsi['fastk']
# Overlap Studies
# ------------------------------------
# Bollinger bands
# Bollinger Bands
bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
dataframe['bb_lowerband'] = bollinger['lower']
dataframe['bb_middleband'] = bollinger['mid']
dataframe['bb_upperband'] = bollinger['upper']
dataframe["bb_percent"] = (
(dataframe["close"] - dataframe["bb_lowerband"]) /
(dataframe["bb_upperband"] - dataframe["bb_lowerband"])
)
dataframe["bb_width"] = (
(dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"]
)
# Bollinger Bands - Weighted (EMA based instead of SMA)
# weighted_bollinger = qtpylib.weighted_bollinger_bands(
# qtpylib.typical_price(dataframe), window=20, stds=2
# )
# dataframe["wbb_upperband"] = weighted_bollinger["upper"]
# dataframe["wbb_lowerband"] = weighted_bollinger["lower"]
# dataframe["wbb_middleband"] = weighted_bollinger["mid"]
# dataframe["wbb_percent"] = (
# (dataframe["close"] - dataframe["wbb_lowerband"]) /
# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"])
# )
# dataframe["wbb_width"] = (
# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) /
# dataframe["wbb_middleband"]
# )
# # EMA - Exponential Moving Average
# dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3)
# dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5)
# dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21)
# dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
# dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
# # SMA - Simple Moving Average
# dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3)
# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5)
# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10)
# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21)
# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50)
# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100)
# SAR Parabol
# Parabolic SAR
dataframe['sar'] = ta.SAR(dataframe)
# TEMA - Triple Exponential Moving Average
@@ -264,7 +308,7 @@ class SampleStrategy(IStrategy):
# # Chart type
# # ------------------------------------
# # Heikinashi stategy
# # Heikin Ashi Strategy
# heikinashi = qtpylib.heikinashi(dataframe)
# dataframe['ha_open'] = heikinashi['open']
# dataframe['ha_close'] = heikinashi['close']

6
freqtrade/templates/strategy_analysis_example.ipynb
View File

@@ -190,7 +190,6 @@
"# Analyze the above\n",
"parallel_trades = analyze_trade_parallelism(trades, '5m')\n",
"\n",
"\n",
"parallel_trades.plot()"
]
},
@@ -212,11 +211,14 @@
"from freqtrade.plot.plotting import generate_candlestick_graph\n",
"# Limit graph period to keep plotly quick and reactive\n",
"\n",
"# Filter trades to one pair\n",
"trades_red = trades.loc[trades['pair'] == pair]\n",
"\n",
"data_red = data['2019-06-01':'2019-06-10']\n",
"# Generate candlestick graph\n",
"graph = generate_candlestick_graph(pair=pair,\n",
" data=data_red,\n",
" trades=trades,\n",
" trades=trades_red,\n",
" indicators1=['sma20', 'ema50', 'ema55'],\n",
" indicators2=['rsi', 'macd', 'macdsignal', 'macdhist']\n",
" )\n",

123
freqtrade/templates/subtemplates/indicators_full.j2
View File

@@ -2,24 +2,70 @@
# Momentum Indicators
# ------------------------------------
# RSI
dataframe['rsi'] = ta.RSI(dataframe)
# ADX
dataframe['adx'] = ta.ADX(dataframe)
# # Plus Directional Indicator / Movement
# dataframe['plus_dm'] = ta.PLUS_DM(dataframe)
# dataframe['plus_di'] = ta.PLUS_DI(dataframe)
# # Minus Directional Indicator / Movement
# dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # Aroon, Aroon Oscillator
# aroon = ta.AROON(dataframe)
# dataframe['aroonup'] = aroon['aroonup']
# dataframe['aroondown'] = aroon['aroondown']
# dataframe['aroonosc'] = ta.AROONOSC(dataframe)
# # Awesome oscillator
# # Awesome Oscillator
# dataframe['ao'] = qtpylib.awesome_oscillator(dataframe)
# # Commodity Channel Index: values Oversold:<-100, Overbought:>100
# # Keltner Channel
# keltner = qtpylib.keltner_channel(dataframe)
# dataframe["kc_upperband"] = keltner["upper"]
# dataframe["kc_lowerband"] = keltner["lower"]
# dataframe["kc_middleband"] = keltner["mid"]
# dataframe["kc_percent"] = (
# (dataframe["close"] - dataframe["kc_lowerband"]) /
# (dataframe["kc_upperband"] - dataframe["kc_lowerband"])
# )
# dataframe["kc_width"] = (
# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"]
# )
# # Ultimate Oscillator
# dataframe['uo'] = ta.ULTOSC(dataframe)
# # Commodity Channel Index: values [Oversold:-100, Overbought:100]
# dataframe['cci'] = ta.CCI(dataframe)
# RSI
dataframe['rsi'] = ta.RSI(dataframe)
# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy)
# rsi = 0.1 * (dataframe['rsi'] - 50)
# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1)
# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy)
# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1)
# # Stochastic Slow
# stoch = ta.STOCH(dataframe)
# dataframe['slowd'] = stoch['slowd']
# dataframe['slowk'] = stoch['slowk']
# Stochastic Fast
stoch_fast = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch_fast['fastd']
dataframe['fastk'] = stoch_fast['fastk']
# # Stochastic RSI
# stoch_rsi = ta.STOCHRSI(dataframe)
# dataframe['fastd_rsi'] = stoch_rsi['fastd']
# dataframe['fastk_rsi'] = stoch_rsi['fastk']
# MACD
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
@@ -29,60 +75,57 @@ dataframe['macdhist'] = macd['macdhist']
# MFI
dataframe['mfi'] = ta.MFI(dataframe)
# # Minus Directional Indicator / Movement
# dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # Plus Directional Indicator / Movement
# dataframe['plus_dm'] = ta.PLUS_DM(dataframe)
# dataframe['plus_di'] = ta.PLUS_DI(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # ROC
# dataframe['roc'] = ta.ROC(dataframe)
# # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy)
# rsi = 0.1 * (dataframe['rsi'] - 50)
# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1)
# # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy)
# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1)
# # Stoch
# stoch = ta.STOCH(dataframe)
# dataframe['slowd'] = stoch['slowd']
# dataframe['slowk'] = stoch['slowk']
# Stoch fast
stoch_fast = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch_fast['fastd']
dataframe['fastk'] = stoch_fast['fastk']
# # Stoch RSI
# stoch_rsi = ta.STOCHRSI(dataframe)
# dataframe['fastd_rsi'] = stoch_rsi['fastd']
# dataframe['fastk_rsi'] = stoch_rsi['fastk']
# Overlap Studies
# ------------------------------------
# Bollinger bands
# Bollinger Bands
bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
dataframe['bb_lowerband'] = bollinger['lower']
dataframe['bb_middleband'] = bollinger['mid']
dataframe['bb_upperband'] = bollinger['upper']
dataframe["bb_percent"] = (
(dataframe["close"] - dataframe["bb_lowerband"]) /
(dataframe["bb_upperband"] - dataframe["bb_lowerband"])
)
dataframe["bb_width"] = (
(dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"]
)
# Bollinger Bands - Weighted (EMA based instead of SMA)
# weighted_bollinger = qtpylib.weighted_bollinger_bands(
# qtpylib.typical_price(dataframe), window=20, stds=2
# )
# dataframe["wbb_upperband"] = weighted_bollinger["upper"]
# dataframe["wbb_lowerband"] = weighted_bollinger["lower"]
# dataframe["wbb_middleband"] = weighted_bollinger["mid"]
# dataframe["wbb_percent"] = (
# (dataframe["close"] - dataframe["wbb_lowerband"]) /
# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"])
# )
# dataframe["wbb_width"] = (
# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"]
# )
# # EMA - Exponential Moving Average
# dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3)
# dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5)
# dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21)
# dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
# dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
# # SMA - Simple Moving Average
# dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3)
# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5)
# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10)
# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21)
# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50)
# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100)
# SAR Parabol
# Parabolic SAR
dataframe['sar'] = ta.SAR(dataframe)
# TEMA - Triple Exponential Moving Average
@@ -142,7 +185,7 @@ dataframe['htleadsine'] = hilbert['leadsine']
# # Chart type
# # ------------------------------------
# # Heikinashi stategy
# # Heikin Ashi Strategy
# heikinashi = qtpylib.heikinashi(dataframe)
# dataframe['ha_open'] = heikinashi['open']
# dataframe['ha_close'] = heikinashi['close']

2
freqtrade/wallets.py
View File

@@ -74,7 +74,7 @@ class Wallets:
)
for trade in open_trades:
curr = trade.pair.split('/')[0]
curr = self._exchange.get_pair_base_currency(trade.pair)
_wallets[curr] = Wallet(
curr,
trade.amount,

62
freqtrade/worker.py
View File

@@ -4,6 +4,7 @@ Main Freqtrade worker class.
import logging
import time
import traceback
from os import getpid
from typing import Any, Callable, Dict, Optional
import sdnotify
@@ -26,12 +27,15 @@ class Worker:
"""
Init all variables and objects the bot needs to work
"""
logger.info('Starting worker %s', __version__)
logger.info(f"Starting worker {__version__}")
self._args = args
self._config = config
self._init(False)
self.last_throttle_start_time: float = 0
self._heartbeat_msg: float = 0
# Tell systemd that we completed initialization phase
if self._sd_notify:
logger.debug("sd_notify: READY=1")
@@ -48,10 +52,10 @@ class Worker:
# Init the instance of the bot
self.freqtrade = FreqtradeBot(self._config)
self._throttle_secs = self._config.get('internals', {}).get(
'process_throttle_secs',
constants.PROCESS_THROTTLE_SECS
)
internals_config = self._config.get('internals', {})
self._throttle_secs = internals_config.get('process_throttle_secs',
constants.PROCESS_THROTTLE_SECS)
self._heartbeat_interval = internals_config.get('heartbeat_interval', 60)
self._sd_notify = sdnotify.SystemdNotifier() if \
self._config.get('internals', {}).get('sd_notify', False) else None
@@ -63,31 +67,33 @@ class Worker:
if state == State.RELOAD_CONF:
self._reconfigure()
def _worker(self, old_state: Optional[State], throttle_secs: Optional[float] = None) -> State:
def _worker(self, old_state: Optional[State]) -> State:
"""
Trading routine that must be run at each loop
The main routine that runs each throttling iteration and handles the states.
:param old_state: the previous service state from the previous call
:return: current service state
"""
state = self.freqtrade.state
if throttle_secs is None:
throttle_secs = self._throttle_secs
# Log state transition
if state != old_state:
self.freqtrade.notify_status(f'{state.name.lower()}')
logger.info('Changing state to: %s', state.name)
logger.info(f"Changing state to: {state.name}")
if state == State.RUNNING:
self.freqtrade.startup()
# Reset heartbeat timestamp to log the heartbeat message at
# first throttling iteration when the state changes
self._heartbeat_msg = 0
if state == State.STOPPED:
# Ping systemd watchdog before sleeping in the stopped state
if self._sd_notify:
logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.")
self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.")
time.sleep(throttle_secs)
self._throttle(func=self._process_stopped, throttle_secs=self._throttle_secs)
elif state == State.RUNNING:
# Ping systemd watchdog before throttling
@@ -95,28 +101,40 @@ class Worker:
logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.")
self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.")
self._throttle(func=self._process, min_secs=throttle_secs)
self._throttle(func=self._process_running, throttle_secs=self._throttle_secs)
if self._heartbeat_interval:
now = time.time()
if (now - self._heartbeat_msg) > self._heartbeat_interval:
logger.info(f"Bot heartbeat. PID={getpid()}, "
f"version='{__version__}', state='{state.name}'")
self._heartbeat_msg = now
return state
def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:
def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any:
"""
Throttles the given callable that it
takes at least `min_secs` to finish execution.
:param func: Any callable
:param min_secs: minimum execution time in seconds
:return: Any
:param throttle_secs: throttling interation execution time limit in seconds
:return: Any (result of execution of func)
"""
start = time.time()
self.last_throttle_start_time = time.time()
logger.debug("========================================")
result = func(*args, **kwargs)
end = time.time()
duration = max(min_secs - (end - start), 0.0)
logger.debug('Throttling %s for %.2f seconds', func.__name__, duration)
time.sleep(duration)
time_passed = time.time() - self.last_throttle_start_time
sleep_duration = max(throttle_secs - time_passed, 0.0)
logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, "
f"last iteration took {time_passed:.2f} s.")
time.sleep(sleep_duration)
return result
def _process(self) -> None:
logger.debug("========================================")
def _process_stopped(self) -> None:
# Maybe do here something in the future...
pass
def _process_running(self) -> None:
try:
self.freqtrade.process()
except TemporaryError as error: