adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into volumeList_enhanced_filter

Browse Source
This commit is contained in:
Matthias
2019-11-09 15:34:47 +01:00
parent 86a5dfa62e 3f65c31883
commit 748fe94603
52 changed files with 958 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/configuration/__init__.py
View File

@@ -1,4 +1,5 @@
from freqtrade.configuration.arguments import Arguments # noqa: F401
from freqtrade.configuration.check_exchange import check_exchange, remove_credentials # noqa: F401
from freqtrade.configuration.timerange import TimeRange # noqa: F401
from freqtrade.configuration.configuration import Configuration # noqa: F401
from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401

16
freqtrade/configuration/check_exchange.py
View File

@@ -10,6 +10,19 @@ from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
def remove_credentials(config: Dict[str, Any]):
"""
Removes exchange keys from the configuration and specifies dry-run
Used for backtesting / hyperopt / edge and utils.
Modifies the input dict!
"""
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
config['exchange']['password'] = ''
config['exchange']['uid'] = ''
config['dry_run'] = True
def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
"""
Check if the exchange name in the config file is supported by Freqtrade
@@ -21,7 +34,8 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
and thus is not known for the Freqtrade at all.
"""
if config['runmode'] in [RunMode.PLOT] and not config.get('exchange', {}).get('name'):
if (config['runmode'] in [RunMode.PLOT, RunMode.UTIL_NO_EXCHANGE, RunMode.OTHER]
and not config.get('exchange', {}).get('name')):
# Skip checking exchange in plot mode, since it requires no exchange
return True
logger.info("Checking exchange...")

3
freqtrade/configuration/config_validation.py
View File

@@ -118,7 +118,8 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None:
"""
Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does.
"""
if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT]:
if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT,
RunMode.UTIL_NO_EXCHANGE, RunMode.UTIL_EXCHANGE]:
return
for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]):

46
freqtrade/configuration/configuration.py
View File

@@ -17,7 +17,7 @@ from freqtrade.configuration.directory_operations import (create_datadir,
from freqtrade.configuration.load_config import load_config_file
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts, json_load
from freqtrade.state import RunMode
from freqtrade.state import RunMode, TRADING_MODES, NON_UTIL_MODES
logger = logging.getLogger(__name__)
@@ -101,14 +101,16 @@ class Configuration:
# Keep a copy of the original configuration file
config['original_config'] = deepcopy(config)
self._process_runmode(config)
self._process_common_options(config)
self._process_trading_options(config)
self._process_optimize_options(config)
self._process_plot_options(config)
self._process_runmode(config)
# Check if the exchange set by the user is supported
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
@@ -133,6 +135,22 @@ class Configuration:
setup_logging(config)
def _process_trading_options(self, config: Dict[str, Any]) -> None:
if config['runmode'] not in TRADING_MODES:
return
if config.get('dry_run', False):
logger.info('Dry run is enabled')
if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]:
# Default to in-memory db for dry_run if not specified
config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL
else:
if not config.get('db_url', None):
config['db_url'] = constants.DEFAULT_DB_PROD_URL
logger.info('Dry run is disabled')
logger.info(f'Using DB: "{config["db_url"]}"')
def _process_common_options(self, config: Dict[str, Any]) -> None:
self._process_logging_options(config)
@@ -149,25 +167,9 @@ class Configuration:
config.update({'db_url': self.args["db_url"]})
logger.info('Parameter --db-url detected ...')
if config.get('dry_run', False):
logger.info('Dry run is enabled')
if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]:
# Default to in-memory db for dry_run if not specified
config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL
else:
if not config.get('db_url', None):
config['db_url'] = constants.DEFAULT_DB_PROD_URL
logger.info('Dry run is disabled')
logger.info(f'Using DB: "{config["db_url"]}"')
if config.get('forcebuy_enable', False):
logger.warning('`forcebuy` RPC message enabled.')
# Setting max_open_trades to infinite if -1
if config.get('max_open_trades') == -1:
config['max_open_trades'] = float('inf')
# Support for sd_notify
if 'sd_notify' in self.args and self.args["sd_notify"]:
config['internals'].update({'sd_notify': True})
@@ -215,6 +217,10 @@ class Configuration:
self._args_to_config(config, argname='position_stacking',
logstring='Parameter --enable-position-stacking detected ...')
# Setting max_open_trades to infinite if -1
if config.get('max_open_trades') == -1:
config['max_open_trades'] = float('inf')
if 'use_max_market_positions' in self.args and not self.args["use_max_market_positions"]:
config.update({'use_max_market_positions': False})
logger.info('Parameter --disable-max-market-positions detected ...')
@@ -223,7 +229,7 @@ class Configuration:
config.update({'max_open_trades': self.args["max_open_trades"]})
logger.info('Parameter --max_open_trades detected, '
'overriding max_open_trades to: %s ...', config.get('max_open_trades'))
else:
elif config['runmode'] in NON_UTIL_MODES:
logger.info('Using max_open_trades: %s ...', config.get('max_open_trades'))
self._args_to_config(config, argname='stake_amount',

41
freqtrade/data/btanalysis.py
View File

@@ -7,7 +7,7 @@ from typing import Dict
import numpy as np
import pandas as pd
import pytz
from datetime import timezone
from freqtrade import persistence
from freqtrade.misc import json_load
@@ -52,16 +52,18 @@ def load_backtest_data(filename) -> pd.DataFrame:
return df
def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int) -> pd.DataFrame:
def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataFrame:
"""
Find overlapping trades by expanding each trade once per period it was open
and then counting overlaps
and then counting overlaps.
:param results: Results Dataframe - can be loaded
:param freq: Frequency used for the backtest
:param max_open_trades: parameter max_open_trades used during backtest run
:return: dataframe with open-counts per time-period in freq
:param timeframe: Timeframe used for backtest
:return: dataframe with open-counts per time-period in timeframe
"""
dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq))
from freqtrade.exchange import timeframe_to_minutes
timeframe_min = timeframe_to_minutes(timeframe)
dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time,
freq=f"{timeframe_min}min"))
for row in results[['open_time', 'close_time']].iterrows()]
deltas = [len(x) for x in dates]
dates = pd.Series(pd.concat(dates).values, name='date')
@@ -69,8 +71,23 @@ def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int
df2 = pd.concat([dates, df2], axis=1)
df2 = df2.set_index('date')
df_final = df2.resample(freq)[['pair']].count()
return df_final[df_final['pair'] > max_open_trades]
df_final = df2.resample(f"{timeframe_min}min")[['pair']].count()
df_final = df_final.rename({'pair': 'open_trades'}, axis=1)
return df_final
def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
max_open_trades: int) -> pd.DataFrame:
"""
Find overlapping trades by expanding each trade once per period it was open
and then counting overlaps
:param results: Results Dataframe - can be loaded
:param timeframe: Frequency used for the backtest
:param max_open_trades: parameter max_open_trades used during backtest run
:return: dataframe with open-counts per time-period in freq
"""
df_final = analyze_trade_parallelism(results, timeframe)
return df_final[df_final['open_trades'] > max_open_trades]
def load_trades_from_db(db_url: str) -> pd.DataFrame:
@@ -89,8 +106,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
"stop_loss", "initial_stop_loss", "strategy", "ticker_interval"]
trades = pd.DataFrame([(t.pair,
t.open_date.replace(tzinfo=pytz.UTC),
t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None,
t.open_date.replace(tzinfo=timezone.utc),
t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
t.calc_profit(), t.calc_profit_percent(),
t.open_rate, t.close_rate, t.amount,
(round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
@@ -106,7 +123,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
t.stop_loss, t.initial_stop_loss,
t.strategy, t.ticker_interval
)
for t in Trade.query.all()],
for t in Trade.get_trades().all()],
columns=columns)
return trades

10
freqtrade/data/history.py
View File

@@ -9,12 +9,11 @@ Includes:
import logging
import operator
from copy import deepcopy
from datetime import datetime
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import arrow
import pytz
from pandas import DataFrame
from freqtrade import OperationalException, misc
@@ -56,10 +55,10 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame:
Trim dataframe based on given timerange
"""
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc)
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
df = df.loc[df['date'] >= start, :]
if timerange.stoptype == 'date':
stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc)
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
df = df.loc[df['date'] <= stop, :]
return df
@@ -148,7 +147,6 @@ def load_pair_history(pair: str,
timerange_startup = deepcopy(timerange)
if startup_candles > 0 and timerange_startup:
logger.info('Using indicator startup period: %s ...', startup_candles)
timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles)
# The user forced the refresh of pairs
@@ -204,6 +202,8 @@ def load_data(datadir: Path,
exchange and refresh_pairs are then not needed here nor in load_pair_history.
"""
result: Dict[str, DataFrame] = {}
if startup_candles > 0 and timerange:
logger.info(f'Using indicator startup period: {startup_candles} ...')
for pair in pairs:
hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,

3
freqtrade/exchange/__init__.py
View File

@@ -1,4 +1,5 @@
from freqtrade.exchange.exchange import Exchange, MAP_EXCHANGE_CHILDCLASS # noqa: F401
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS # noqa: F401
from freqtrade.exchange.exchange import Exchange # noqa: F401
from freqtrade.exchange.exchange import (get_exchange_bad_reason, # noqa: F401
is_exchange_bad,
is_exchange_known_ccxt,

124
freqtrade/exchange/common.py Normal file
View File

@@ -0,0 +1,124 @@
import logging
from freqtrade import DependencyException, TemporaryError
logger = logging.getLogger(__name__)
API_RETRY_COUNT = 4
BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "
"Details in https://github.com/freqtrade/freqtrade/issues/1983",
"hitbtc": "This API cannot be used with Freqtrade. "
"Use `hitbtc2` exchange id to access this exchange.",
**dict.fromkeys([
'adara',
'anxpro',
'bigone',
'coinbase',
'coinexchange',
'coinmarketcap',
'lykke',
'xbtce',
], "Does not provide timeframes. ccxt fetchOHLCV: False"),
**dict.fromkeys([
'bcex',
'bit2c',
'bitbay',
'bitflyer',
'bitforex',
'bithumb',
'bitso',
'bitstamp1',
'bl3p',
'braziliex',
'btcbox',
'btcchina',
'btctradeim',
'btctradeua',
'bxinth',
'chilebit',
'coincheck',
'coinegg',
'coinfalcon',
'coinfloor',
'coingi',
'coinmate',
'coinone',
'coinspot',
'coolcoin',
'crypton',
'deribit',
'exmo',
'exx',
'flowbtc',
'foxbit',
'fybse',
# 'hitbtc',
'ice3x',
'independentreserve',
'indodax',
'itbit',
'lakebtc',
'latoken',
'liquid',
'livecoin',
'luno',
'mixcoins',
'negociecoins',
'nova',
'paymium',
'southxchange',
'stronghold',
'surbitcoin',
'therock',
'tidex',
'vaultoro',
'vbtc',
'virwox',
'yobit',
'zaif',
], "Does not provide timeframes. ccxt fetchOHLCV: emulated"),
}
MAP_EXCHANGE_CHILDCLASS = {
'binanceus': 'binance',
'binanceje': 'binance',
}
def retrier_async(f):
async def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
try:
return await f(*args, **kwargs)
except (TemporaryError, DependencyException) as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
if count > 0:
count -= 1
kwargs.update({'count': count})
logger.warning('retrying %s() still for %s times', f.__name__, count)
return await wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
raise ex
return wrapper
def retrier(f):
def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
try:
return f(*args, **kwargs)
except (TemporaryError, DependencyException) as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
if count > 0:
count -= 1
kwargs.update({'count': count})
logger.warning('retrying %s() still for %s times', f.__name__, count)
return wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
raise ex
return wrapper

142
freqtrade/exchange/exchange.py
View File

@@ -14,137 +14,18 @@ from typing import Any, Dict, List, Optional, Tuple
import arrow
import ccxt
import ccxt.async_support as ccxt_async
from ccxt.base.decimal_to_precision import ROUND_UP, ROUND_DOWN
from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP
from pandas import DataFrame
from freqtrade import (DependencyException, InvalidOrderException,
OperationalException, TemporaryError, constants)
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
API_RETRY_COUNT = 4
BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "
"Details in https://github.com/freqtrade/freqtrade/issues/1983",
"hitbtc": "This API cannot be used with Freqtrade. "
"Use `hitbtc2` exchange id to access this exchange.",
**dict.fromkeys([
'adara',
'anxpro',
'bigone',
'coinbase',
'coinexchange',
'coinmarketcap',
'lykke',
'xbtce',
], "Does not provide timeframes. ccxt fetchOHLCV: False"),
**dict.fromkeys([
'bcex',
'bit2c',
'bitbay',
'bitflyer',
'bitforex',
'bithumb',
'bitso',
'bitstamp1',
'bl3p',
'braziliex',
'btcbox',
'btcchina',
'btctradeim',
'btctradeua',
'bxinth',
'chilebit',
'coincheck',
'coinegg',
'coinfalcon',
'coinfloor',
'coingi',
'coinmate',
'coinone',
'coinspot',
'coolcoin',
'crypton',
'deribit',
'exmo',
'exx',
'flowbtc',
'foxbit',
'fybse',
# 'hitbtc',
'ice3x',
'independentreserve',
'indodax',
'itbit',
'lakebtc',
'latoken',
'liquid',
'livecoin',
'luno',
'mixcoins',
'negociecoins',
'nova',
'paymium',
'southxchange',
'stronghold',
'surbitcoin',
'therock',
'tidex',
'vaultoro',
'vbtc',
'virwox',
'yobit',
'zaif',
], "Does not provide timeframes. ccxt fetchOHLCV: emulated"),
}
MAP_EXCHANGE_CHILDCLASS = {
'binanceus': 'binance',
'binanceje': 'binance',
}
def retrier_async(f):
async def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
try:
return await f(*args, **kwargs)
except (TemporaryError, DependencyException) as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
if count > 0:
count -= 1
kwargs.update({'count': count})
logger.warning('retrying %s() still for %s times', f.__name__, count)
return await wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
raise ex
return wrapper
def retrier(f):
def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
try:
return f(*args, **kwargs)
except (TemporaryError, DependencyException) as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
if count > 0:
count -= 1
kwargs.update({'count': count})
logger.warning('retrying %s() still for %s times', f.__name__, count)
return wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
raise ex
return wrapper
class Exchange:
_config: Dict = {}
@@ -994,6 +875,22 @@ class Exchange:
@retrier
def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List:
"""
Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id.
The "since" argument passed in is coming from the database and is in UTC,
as timezone-native datetime object.
From the python documentation:
> Naive datetime instances are assumed to represent local time
Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the
transformation from local timezone to UTC.
This works for timezones UTC+ since then the result will contain trades from a few hours
instead of from the last 5 seconds, however fails for UTC- timezones,
since we're then asking for trades with a "since" argument in the future.
:param order_id order_id: Order-id as given when creating the order
:param pair: Pair the order is for
:param since: datetime object of the order creation time. Assumes object is in UTC.
"""
if self._config['dry_run']:
return []
if not self.exchange_has('fetchMyTrades'):
@@ -1001,7 +898,8 @@ class Exchange:
try:
# Allow 5s offset to catch slight time offsets (discovered in #1185)
# since needs to be int in milliseconds
my_trades = self._api.fetch_my_trades(pair, int((since.timestamp() - 5) * 1000))
my_trades = self._api.fetch_my_trades(
pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000))
matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
return matched_trades

9
freqtrade/freqtradebot.py
View File

@@ -318,8 +318,7 @@ class FreqtradeBot:
(bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0):
if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market):
buycount += self.execute_buy(_pair, stake_amount)
else:
continue
continue
buycount += self.execute_buy(_pair, stake_amount)
@@ -633,8 +632,8 @@ class FreqtradeBot:
Force-sells the pair (using EmergencySell reason) in case of Problems creating the order.
:return: True if the order succeeded, and False in case of problems.
"""
# Limit price threshold: As limit price should always be below price
LIMIT_PRICE_PCT = 0.99
# Limit price threshold: As limit price should always be below stop-price
LIMIT_PRICE_PCT = self.strategy.order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
try:
stoploss_order = self.exchange.stoploss_limit(pair=trade.pair, amount=trade.amount,
@@ -767,7 +766,7 @@ class FreqtradeBot:
buy_timeout_threshold = arrow.utcnow().shift(minutes=-buy_timeout).datetime
sell_timeout_threshold = arrow.utcnow().shift(minutes=-sell_timeout).datetime
for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all():
for trade in Trade.get_open_order_trades():
try:
# FIXME: Somehow the query above returns results
# where the open_order_id is in fact None.

4
freqtrade/loggers.py
View File

@@ -33,8 +33,8 @@ def setup_logging(config: Dict[str, Any]) -> None:
# Log level
verbosity = config['verbosity']
# Log to stdout, not stderr
log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stdout)]
# Log to stderr
log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)]
if config.get('logfile'):
log_handlers.append(RotatingFileHandler(config['logfile'],

13
freqtrade/optimize/backtesting.py
View File

@@ -10,9 +10,10 @@ from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional
from pandas import DataFrame
from tabulate import tabulate
from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.configuration import TimeRange, remove_credentials
from freqtrade.data import history
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
@@ -21,7 +22,6 @@ from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
from freqtrade.strategy.interface import IStrategy, SellType
from tabulate import tabulate
logger = logging.getLogger(__name__)
@@ -57,11 +57,7 @@ class Backtesting:
self.config = config
# Reset keys for backtesting
self.config['exchange']['key'] = ''
self.config['exchange']['secret'] = ''
self.config['exchange']['password'] = ''
self.config['exchange']['uid'] = ''
self.config['dry_run'] = True
remove_credentials(self.config)
self.strategylist: List[IStrategy] = []
self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
@@ -245,7 +241,8 @@ class Backtesting:
ticker: Dict = {}
# Create ticker dict
for pair, pair_data in processed.items():
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
pair_data.loc[:, 'buy'] = 0 # cleanup from previous run
pair_data.loc[:, 'sell'] = 0 # cleanup from previous run
ticker_data = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()

17
freqtrade/optimize/edge_cli.py
View File

@@ -4,12 +4,13 @@
This module contains the edge backtesting interface
"""
import logging
from typing import Dict, Any
from tabulate import tabulate
from freqtrade import constants
from freqtrade.edge import Edge
from typing import Any, Dict
from freqtrade.configuration import TimeRange
from tabulate import tabulate
from freqtrade import constants
from freqtrade.configuration import TimeRange, remove_credentials
from freqtrade.edge import Edge
from freqtrade.exchange import Exchange
from freqtrade.resolvers import StrategyResolver
@@ -29,12 +30,8 @@ class EdgeCli:
self.config = config
# Reset keys for edge
self.config['exchange']['key'] = ''
self.config['exchange']['secret'] = ''
self.config['exchange']['password'] = ''
self.config['exchange']['uid'] = ''
remove_credentials(self.config)
self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
self.config['dry_run'] = True
self.exchange = Exchange(self.config)
self.strategy = StrategyResolver(self.config).strategy

18
freqtrade/optimize/hyperopt.py
View File

@@ -4,9 +4,9 @@
This module contains the hyperopt logic
"""
import locale
import logging
import sys
from collections import OrderedDict
from operator import itemgetter
from pathlib import Path
@@ -14,10 +14,10 @@ from pprint import pprint
from typing import Any, Dict, List, Optional
import rapidjson
from colorama import init as colorama_init
from colorama import Fore, Style
from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count
from colorama import init as colorama_init
from joblib import (Parallel, cpu_count, delayed, dump, load,
wrap_non_picklable_objects)
from pandas import DataFrame
from skopt import Optimizer
from skopt.space import Dimension
@@ -28,8 +28,8 @@ from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F4
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4
from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver, HyperOptLossResolver
from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
HyperOptResolver)
logger = logging.getLogger(__name__)
@@ -216,7 +216,7 @@ class Hyperopt:
if print_all:
print(log_str)
else:
print('\n' + log_str)
print(f'\n{log_str}')
else:
print('.', end='')
sys.stdout.flush()
@@ -335,7 +335,9 @@ class Hyperopt:
return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. '
f'Total profit {total_profit: 11.8f} {stake_cur} '
f'({profit: 7.2f}Σ%). Avg duration {duration:5.1f} mins.')
f'({profit: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). '
f'Avg duration {duration:5.1f} mins.'
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
def get_optimizer(self, dimensions, cpu_count) -> Optimizer:
return Optimizer(

12
freqtrade/optimize/hyperopt_interface.py
View File

@@ -5,10 +5,9 @@ This module defines the interface to apply for hyperopts
import logging
import math
from abc import ABC, abstractmethod
from abc import ABC
from typing import Dict, Any, Callable, List
from pandas import DataFrame
from skopt.space import Dimension, Integer, Real
from freqtrade import OperationalException
@@ -42,15 +41,6 @@ class IHyperOpt(ABC):
# Assign ticker_interval to be used in hyperopt
IHyperOpt.ticker_interval = str(config['ticker_interval'])
@staticmethod
@abstractmethod
def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Populate indicators that will be used in the Buy and Sell strategy.
:param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe().
:return: A Dataframe with all mandatory indicators for the strategies.
"""
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
"""

77
freqtrade/persistence.py
View File

@@ -8,17 +8,16 @@ from typing import Any, Dict, List, Optional
import arrow
from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
create_engine, inspect)
create_engine, desc, func, inspect)
from sqlalchemy.exc import NoSuchModuleError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Query
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from sqlalchemy import func
from sqlalchemy.pool import StaticPool
from freqtrade import OperationalException
logger = logging.getLogger(__name__)
@@ -52,9 +51,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
raise OperationalException(f"Given value for db_url: '{db_url}' "
f"is no valid database URL! (See {_SQL_DOCS_URL})")
session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.session = session()
Trade.query = session.query_property()
# https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
# Scoped sessions proxy requests to the appropriate thread-local session.
# We should use the scoped_session object - not a seperately initialized version
Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.query = Trade.session.query_property()
_DECL_BASE.metadata.create_all(engine)
check_migrate(engine)
@@ -393,6 +394,37 @@ class Trade(_DECL_BASE):
profit_percent = (close_trade_price / open_trade_price) - 1
return float(f"{profit_percent:.8f}")
@staticmethod
def get_trades(trade_filter=None) -> Query:
"""
Helper function to query Trades using filters.
:param trade_filter: Optional filter to apply to trades
Can be either a Filter object, or a List of filters
e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
e.g. `(trade_filter=Trade.id == trade_id)`
:return: unsorted query object
"""
if trade_filter is not None:
if not isinstance(trade_filter, list):
trade_filter = [trade_filter]
return Trade.query.filter(*trade_filter)
else:
return Trade.query
@staticmethod
def get_open_trades() -> List[Any]:
"""
Query trades from persistence layer
"""
return Trade.get_trades(Trade.is_open.is_(True)).all()
@staticmethod
def get_open_order_trades():
"""
Returns all open trades
"""
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def total_open_trades_stakes() -> float:
"""
@@ -405,11 +437,38 @@ class Trade(_DECL_BASE):
return total_open_stake_amount or 0
@staticmethod
def get_open_trades() -> List[Any]:
def get_overall_performance() -> List[Dict[str, Any]]:
"""
Query trades from persistence layer
Returns List of dicts containing all Trades, including profit and trade count
"""
return Trade.query.filter(Trade.is_open.is_(True)).all()
pair_rates = Trade.session.query(
Trade.pair,
func.sum(Trade.close_profit).label('profit_sum'),
func.count(Trade.pair).label('count')
).filter(Trade.is_open.is_(False))\
.group_by(Trade.pair) \
.order_by(desc('profit_sum')) \
.all()
return [
{
'pair': pair,
'profit': rate,
'count': count
}
for pair, rate, count in pair_rates
]
@staticmethod
def get_best_pair():
"""
Get best pair with closed trade.
"""
best_pair = Trade.session.query(
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
).filter(Trade.is_open.is_(False)) \
.group_by(Trade.pair) \
.order_by(desc('profit_sum')).first()
return best_pair
@staticmethod
def stoploss_reinitialization(desired_stoploss):

3
freqtrade/resolvers/hyperopt_resolver.py
View File

@@ -34,6 +34,9 @@ class HyperOptResolver(IResolver):
self.hyperopt = self._load_hyperopt(hyperopt_name, config,
extra_dir=config.get('hyperopt_path'))
if not hasattr(self.hyperopt, 'populate_indicators'):
logger.warning("Hyperopt class does not provide populate_indicators() method. "
"Using populate_indicators from the strategy.")
if not hasattr(self.hyperopt, 'populate_buy_trend'):
logger.warning("Hyperopt class does not provide populate_buy_trend() method. "
"Using populate_buy_trend from the strategy.")

49
freqtrade/rpc/rpc.py
View File

@@ -9,7 +9,6 @@ from enum import Enum
from typing import Dict, Any, List, Optional
import arrow
import sqlalchemy as sql
from numpy import mean, NAN
from pandas import DataFrame
@@ -154,12 +153,11 @@ class RPC:
for day in range(0, timescale):
profitday = today - timedelta(days=day)
trades = Trade.query \
.filter(Trade.is_open.is_(False)) \
.filter(Trade.close_date >= profitday)\
.filter(Trade.close_date < (profitday + timedelta(days=1)))\
.order_by(Trade.close_date)\
.all()
trades = Trade.get_trades(trade_filter=[
Trade.is_open.is_(False),
Trade.close_date >= profitday,
Trade.close_date < (profitday + timedelta(days=1))
]).order_by(Trade.close_date).all()
curdayprofit = sum(trade.calc_profit() for trade in trades)
profit_days[profitday] = {
'amount': f'{curdayprofit:.8f}',
@@ -192,7 +190,7 @@ class RPC:
def _rpc_trade_statistics(
self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
""" Returns cumulative profit statistics """
trades = Trade.query.order_by(Trade.id).all()
trades = Trade.get_trades().order_by(Trade.id).all()
profit_all_coin = []
profit_all_perc = []
@@ -225,11 +223,7 @@ class RPC:
)
profit_all_perc.append(profit_percent)
best_pair = Trade.session.query(
Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum')
).filter(Trade.is_open.is_(False)) \
.group_by(Trade.pair) \
.order_by(sql.text('profit_sum DESC')).first()
best_pair = Trade.get_best_pair()
if not best_pair:
raise RPCException('no closed trade')
@@ -389,11 +383,8 @@ class RPC:
return {'result': 'Created sell orders for all open trades.'}
# Query for trade
trade = Trade.query.filter(
sql.and_(
Trade.id == trade_id,
Trade.is_open.is_(True)
)
trade = Trade.get_trades(
trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ]
).first()
if not trade:
logger.warning('forcesell: Invalid argument received')
@@ -423,7 +414,7 @@ class RPC:
# check if valid pair
# check if pair already has an open pair
trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first()
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first()
if trade:
raise RPCException(f'position for {pair} already open - id: {trade.id}')
@@ -432,28 +423,20 @@ class RPC:
# execute buy
if self._freqtrade.execute_buy(pair, stakeamount, price):
trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first()
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first()
return trade
else:
return None
def _rpc_performance(self) -> List[Dict]:
def _rpc_performance(self) -> List[Dict[str, Any]]:
"""
Handler for performance.
Shows a performance statistic from finished trades
"""
pair_rates = Trade.session.query(Trade.pair,
sql.func.sum(Trade.close_profit).label('profit_sum'),
sql.func.count(Trade.pair).label('count')) \
.filter(Trade.is_open.is_(False)) \
.group_by(Trade.pair) \
.order_by(sql.text('profit_sum DESC')) \
.all()
return [
{'pair': pair, 'profit': round(rate * 100, 2), 'count': count}
for pair, rate, count in pair_rates
]
pair_rates = Trade.get_overall_performance()
# Round and convert to %
[x.update({'profit': round(x['profit'] * 100, 2)}) for x in pair_rates]
return pair_rates
def _rpc_count(self) -> Dict[str, float]:
""" Returns the number of trades running """

9
freqtrade/state.py
View File

@@ -25,5 +25,12 @@ class RunMode(Enum):
BACKTEST = "backtest"
EDGE = "edge"
HYPEROPT = "hyperopt"
UTIL_EXCHANGE = "util_exchange"
UTIL_NO_EXCHANGE = "util_no_exchange"
PLOT = "plot"
OTHER = "other" # Used for plotting scripts and test
OTHER = "other"
TRADING_MODES = [RunMode.LIVE, RunMode.DRY_RUN]
OPTIMIZE_MODES = [RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT]
NON_UTIL_MODES = TRADING_MODES + OPTIMIZE_MODES

12
freqtrade/utils.py
View File

@@ -10,7 +10,7 @@ import rapidjson
from tabulate import tabulate
from freqtrade import OperationalException
from freqtrade.configuration import Configuration, TimeRange
from freqtrade.configuration import Configuration, TimeRange, remove_credentials
from freqtrade.configuration.directory_operations import create_userdata_dir
from freqtrade.data.history import (convert_trades_to_ohlcv,
refresh_backtest_ohlcv_data,
@@ -33,10 +33,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
configuration = Configuration(args, method)
config = configuration.get_config()
config['exchange']['dry_run'] = True
# Ensure we do not use Exchange credentials
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
remove_credentials(config)
return config
@@ -74,7 +72,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
"""
Download data (former download_backtest_data.py script)
"""
config = setup_utils_configuration(args, RunMode.OTHER)
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
timerange = TimeRange()
if 'days' in config:
@@ -123,7 +121,7 @@ def start_list_timeframes(args: Dict[str, Any]) -> None:
"""
Print ticker intervals (timeframes) available on Exchange
"""
config = setup_utils_configuration(args, RunMode.OTHER)
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
# Do not use ticker_interval set in the config
config['ticker_interval'] = None
@@ -144,7 +142,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
:param pairs_only: if True print only pairs, otherwise print all instruments (markets)
:return: None
"""
config = setup_utils_configuration(args, RunMode.OTHER)
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
# Init exchange
exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange