adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into split_btanalysis_load_trades

Browse Source
This commit is contained in:
Matthias 2019-06-24 07:15:14 +02:00
commit eba7327058
15 changed files with 197 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/installation.md
View File

@ -6,7 +6,7 @@ This page explains how to prepare your environment for running the bot.
Before running your bot in production you will need to setup few
external API. In production mode, the bot will require valid Exchange API
credentials. We also reccomend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended).
credentials. We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended).
- [Setup your exchange account](#setup-your-exchange-account)

39
freqtrade/arguments.py
View File

@ -87,9 +87,9 @@ class Arguments(object):
)
parser.add_argument(
'-c', '--config',
help="Specify configuration file (default: %(default)s). "
"Multiple --config options may be used. "
"Can be set to '-' to read config from stdin.",
help=f'Specify configuration file (default: {constants.DEFAULT_CONFIG}). '
f'Multiple --config options may be used. '
f'Can be set to `-` to read config from stdin.',
dest='config',
action='append',
metavar='PATH',
@ -122,9 +122,9 @@ class Arguments(object):
)
parser.add_argument(
'--dynamic-whitelist',
help='Dynamically generate and update whitelist'
' based on 24h BaseVolume (default: %(const)s).'
' DEPRECATED.',
help='Dynamically generate and update whitelist '
'based on 24h BaseVolume (default: %(const)s). '
'DEPRECATED.',
dest='dynamic_whitelist',
const=constants.DYNAMIC_WHITELIST,
type=int,
@ -133,8 +133,8 @@ class Arguments(object):
)
parser.add_argument(
'--db-url',
help='Override trades database URL, this is useful if dry_run is enabled'
' or in custom deployments (default: %(default)s).',
help=f'Override trades database URL, this is useful if dry_run is enabled '
f'or in custom deployments (default: {constants.DEFAULT_DB_DRYRUN_URL}.',
dest='db_url',
metavar='PATH',
)
@ -228,10 +228,10 @@ class Arguments(object):
)
parser.add_argument(
'--export-filename',
help='Save backtest results to this filename \
requires --export to be set as well\
Example --export-filename=user_data/backtest_data/backtest_today.json\
(default: %(default)s)',
help='Save backtest results to this filename '
'requires --export to be set as well. '
'Example --export-filename=user_data/backtest_data/backtest_today.json '
'(default: %(default)s)',
default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'),
dest='exportfilename',
metavar='PATH',
@ -246,8 +246,8 @@ class Arguments(object):
parser.add_argument(
'--stoplosses',
help='Defines a range of stoploss against which edge will assess the strategy '
'the format is "min,max,step" (without any space).'
'example: --stoplosses=-0.01,-0.1,-0.001',
'the format is "min,max,step" (without any space). '
'Example: --stoplosses=-0.01,-0.1,-0.001',
dest='stoploss_range',
)
@ -289,8 +289,8 @@ class Arguments(object):
)
parser.add_argument(
'-s', '--spaces',
help='Specify which parameters to hyperopt. Space separate list. \
Default: %(default)s.',
help='Specify which parameters to hyperopt. Space separate list. '
'Default: %(default)s.',
choices=['all', 'buy', 'sell', 'roi', 'stoploss'],
default='all',
nargs='+',
@ -467,13 +467,14 @@ class Arguments(object):
)
parser.add_argument(
'--exchange',
help='Exchange name (default: %(default)s). Only valid if no config is provided.',
help=f'Exchange name (default: {constants.DEFAULT_EXCHANGE}). '
f'Only valid if no config is provided.',
dest='exchange',
)
parser.add_argument(
'-t', '--timeframes',
help='Specify which tickers to download. Space separated list. \
Default: %(default)s.',
help=f'Specify which tickers to download. Space separated list. '
f'Default: {constants.DEFAULT_DOWNLOAD_TICKER_INTERVALS}.',
choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
'6h', '8h', '12h', '1d', '3d', '1w'],
nargs='+',

2
freqtrade/constants.py
View File

@ -4,6 +4,7 @@
bot constants
"""
DEFAULT_CONFIG = 'config.json'
DEFAULT_EXCHANGE = 'bittrex'
DYNAMIC_WHITELIST = 20 # pairs
PROCESS_THROTTLE_SECS = 5 # sec
DEFAULT_TICKER_INTERVAL = 5 # min
@ -21,6 +22,7 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market']
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList']
DRY_RUN_WALLET = 999.9
DEFAULT_DOWNLOAD_TICKER_INTERVALS = '1m 5m'
TICKER_INTERVALS = [
'1m', '3m', '5m', '15m', '30m',

4
freqtrade/data/btanalysis.py
View File

@ -23,7 +23,7 @@ def load_backtest_data(filename) -> pd.DataFrame:
"""
Load backtest data file.
:param filename: pathlib.Path object, or string pointing to the file.
:return a dataframe with the analysis results
:return: a dataframe with the analysis results
"""
if isinstance(filename, str):
filename = Path(filename)
@ -77,7 +77,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
"""
Load trades from a DB (using dburl)
:param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
:returns: Dataframe containing Trades
:return: Dataframe containing Trades
"""
trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
persistence.init(db_url, clean_open_orders=False)

2
freqtrade/data/history.py
View File

@ -63,7 +63,7 @@ def load_tickerdata_file(
timerange: Optional[TimeRange] = None) -> Optional[list]:
"""
Load a pair from file, either .json.gz or .json
:return tickerlist or None if unsuccesful
:return: tickerlist or None if unsuccesful
"""
filename = pair_data_filename(datadir, pair, ticker_interval)
pairdata = misc.file_load_json(filename)

19
freqtrade/freqtradebot.py
View File

@ -53,8 +53,7 @@ class FreqtradeBot(object):
self.rpc: RPCManager = RPCManager(self)
exchange_name = self.config.get('exchange', {}).get('name').title()
self.exchange = ExchangeResolver(exchange_name, self.config).exchange
self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
self.wallets = Wallets(self.config, self.exchange)
self.dataprovider = DataProvider(self.config, self.exchange)
@ -691,13 +690,22 @@ class FreqtradeBot(object):
# cancelling the current stoploss on exchange first
logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s})'
'in order to add another one ...', order['id'])
if self.exchange.cancel_order(order['id'], trade.pair):
try:
self.exchange.cancel_order(order['id'], trade.pair)
except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {order['id']} "
f"for pair {trade.pair}")
try:
# creating the new one
stoploss_order_id = self.exchange.stoploss_limit(
pair=trade.pair, amount=trade.amount,
stop_price=trade.stop_loss, rate=trade.stop_loss * 0.99
)['id']
trade.stoploss_order_id = str(stoploss_order_id)
except DependencyException:
logger.exception(f"Could create trailing stoploss order "
f"for pair {trade.pair}.")
def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool:
if self.edge:
@ -843,7 +851,10 @@ class FreqtradeBot(object):
# First cancelling stoploss on exchange ...
if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
self.exchange.cancel_order(trade.stoploss_order_id, trade.pair)
try:
self.exchange.cancel_order(trade.stoploss_order_id, trade.pair)
except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}")
# Execute sell and update trade record
order_id = self.exchange.sell(pair=str(trade.pair),

3
freqtrade/optimize/backtesting.py
View File

@ -63,8 +63,7 @@ class Backtesting(object):
self.config['dry_run'] = True
self.strategylist: List[IStrategy] = []
exchange_name = self.config.get('exchange', {}).get('name').title()
self.exchange = ExchangeResolver(exchange_name, self.config).exchange
self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
self.fee = self.exchange.get_fee()
if self.config.get('runmode') != RunMode.HYPEROPT:

1
freqtrade/resolvers/exchange_resolver.py
View File

@ -22,6 +22,7 @@ class ExchangeResolver(IResolver):
Load the custom class from config parameter
:param config: configuration dictionary
"""
exchange_name = exchange_name.title()
try:
self.exchange = self._load_exchange(exchange_name, kwargs={'config': config})
except ImportError:

4
freqtrade/strategy/interface.py
View File

@ -158,7 +158,7 @@ class IStrategy(ABC):
"""
Parses the given ticker history and returns a populated DataFrame
add several TA indicators and buy signal to it
:return DataFrame with ticker data and indicator data
:return: DataFrame with ticker data and indicator data
"""
pair = str(metadata.get('pair'))
@ -351,7 +351,7 @@ class IStrategy(ABC):
"""
Based an earlier trade and current price and ROI configuration, decides whether bot should
sell. Requires current_profit to be in percent!!
:return True if bot should sell at current rate
:return: True if bot should sell at current rate
"""
# Check if time matches and current rate is above threshold

17
freqtrade/tests/conftest.py
View File

@ -5,6 +5,7 @@ import re
from copy import deepcopy
from datetime import datetime
from functools import reduce
from pathlib import Path
from typing import List
from unittest.mock import MagicMock, PropertyMock
@ -60,7 +61,7 @@ def get_patched_exchange(mocker, config, api_mock=None, id='bittrex') -> Exchang
patch_exchange(mocker, api_mock, id)
config["exchange"]["name"] = id
try:
exchange = ExchangeResolver(id.title(), config).exchange
exchange = ExchangeResolver(id, config).exchange
except ImportError:
exchange = Exchange(config)
return exchange
@ -110,11 +111,23 @@ def patch_freqtradebot(mocker, config) -> None:
def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
"""
This function patches _init_modules() to not call dependencies
:param mocker: a Mocker object to apply patches
:param config: Config to pass to the bot
:return: FreqtradeBot
"""
patch_freqtradebot(mocker, config)
return FreqtradeBot(config)
def get_patched_worker(mocker, config) -> Worker:
"""
This function patches _init_modules() to not call dependencies
:param mocker: a Mocker object to apply patches
:param config: Config to pass to the bot
:return: Worker
"""
patch_freqtradebot(mocker, config)
return Worker(args=None, config=config)
@ -865,7 +878,7 @@ def tickers():
@pytest.fixture
def result():
with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
with Path('freqtrade/tests/testdata/UNITTEST_BTC-1m.json').open('r') as data_file:
return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True)
# FIX:

4
freqtrade/tests/exchange/test_exchange.py
View File

@ -124,14 +124,14 @@ def test_exchange_resolver(default_conf, mocker, caplog):
caplog.record_tuples)
caplog.clear()
exchange = ExchangeResolver('Kraken', default_conf).exchange
exchange = ExchangeResolver('kraken', default_conf).exchange
assert isinstance(exchange, Exchange)
assert isinstance(exchange, Kraken)
assert not isinstance(exchange, Binance)
assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.",
caplog.record_tuples)
exchange = ExchangeResolver('Binance', default_conf).exchange
exchange = ExchangeResolver('binance', default_conf).exchange
assert isinstance(exchange, Exchange)
assert isinstance(exchange, Binance)
assert not isinstance(exchange, Kraken)

8
freqtrade/tests/strategy/test_strategy.py
View File

@ -75,14 +75,10 @@ def test_load_strategy_byte64(result):
def test_load_strategy_invalid_directory(result, caplog):
resolver = StrategyResolver()
extra_dir = path.join('some', 'path')
extra_dir = Path.cwd() / 'some/path'
resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir)
assert (
'freqtrade.resolvers.strategy_resolver',
logging.WARNING,
'Path "{}" does not exist'.format(extra_dir),
) in caplog.record_tuples
assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog.record_tuples)
assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})

143
freqtrade/tests/test_freqtradebot.py
View File

@ -19,47 +19,13 @@ from freqtrade.persistence import Trade
from freqtrade.rpc import RPCMessageType
from freqtrade.state import State
from freqtrade.strategy.interface import SellCheckTuple, SellType
from freqtrade.tests.conftest import (log_has, log_has_re, patch_edge,
patch_exchange, patch_get_signal,
patch_wallet)
from freqtrade.tests.conftest import (get_patched_freqtradebot,
get_patched_worker, log_has, log_has_re,
patch_edge, patch_exchange,
patch_get_signal, patch_wallet)
from freqtrade.worker import Worker
# Functions for recurrent object patching
def patch_freqtradebot(mocker, config) -> None:
"""
This function patches _init_modules() to not call dependencies
:param mocker: a Mocker object to apply patches
:param config: Config to pass to the bot
:return: None
"""
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
patch_exchange(mocker)
def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
"""
This function patches _init_modules() to not call dependencies
:param mocker: a Mocker object to apply patches
:param config: Config to pass to the bot
:return: FreqtradeBot
"""
patch_freqtradebot(mocker, config)
return FreqtradeBot(config)
def get_patched_worker(mocker, config) -> Worker:
"""
This function patches _init_modules() to not call dependencies
:param mocker: a Mocker object to apply patches
:param config: Config to pass to the bot
:return: Worker
"""
patch_freqtradebot(mocker, config)
return Worker(args=None, config=config)
def patch_RPCManager(mocker) -> MagicMock:
"""
This function mock RPC manager to avoid repeating this code in almost every tests
@ -1176,6 +1142,77 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog,
stop_price=0.00002344 * 0.95)
def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, caplog,
markets, limit_buy_order,
limit_sell_order) -> None:
# When trailing stoploss is set
stoploss_limit = MagicMock(return_value={'id': 13434334})
patch_exchange(mocker)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_ticker=MagicMock(return_value={
'bid': 0.00001172,
'ask': 0.00001173,
'last': 0.00001172
}),
buy=MagicMock(return_value={'id': limit_buy_order['id']}),
sell=MagicMock(return_value={'id': limit_sell_order['id']}),
get_fee=fee,
markets=PropertyMock(return_value=markets),
stoploss_limit=stoploss_limit
)
# enabling TSL
default_conf['trailing_stop'] = True
freqtrade = get_patched_freqtradebot(mocker, default_conf)
# enabling stoploss on exchange
freqtrade.strategy.order_types['stoploss_on_exchange'] = True
# setting stoploss
freqtrade.strategy.stoploss = -0.05
# setting stoploss_on_exchange_interval to 60 seconds
freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 60
patch_get_signal(freqtrade)
freqtrade.create_trade()
trade = Trade.query.first()
trade.is_open = True
trade.open_order_id = None
trade.stoploss_order_id = "abcd"
trade.stop_loss = 0.2
trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime.replace(tzinfo=None)
stoploss_order_hanging = {
'id': "abcd",
'status': 'open',
'type': 'stop_loss_limit',
'price': 3,
'average': 2,
'info': {
'stopPrice': '0.1'
}
}
mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hanging)
freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging)
assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/BTC.*",
caplog.record_tuples)
# Still try to create order
assert stoploss_limit.call_count == 1
# Fail creating stoploss order
caplog.clear()
cancel_mock = mocker.patch("freqtrade.exchange.Exchange.cancel_order", MagicMock())
mocker.patch("freqtrade.exchange.Exchange.stoploss_limit", side_effect=DependencyException())
freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging)
assert cancel_mock.call_count == 1
assert log_has_re(r"Could create trailing stoploss order for pair ETH/BTC\..*",
caplog.record_tuples)
def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
markets, limit_buy_order, limit_sell_order) -> None:
@ -2108,6 +2145,36 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe
} == last_msg
def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee,
markets, caplog) -> None:
freqtrade = get_patched_freqtradebot(mocker, default_conf)
mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
sellmock = MagicMock()
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
_load_markets=MagicMock(return_value={}),
get_ticker=ticker,
get_fee=fee,
markets=PropertyMock(return_value=markets),
sell=sellmock
)
freqtrade.strategy.order_types['stoploss_on_exchange'] = True
patch_get_signal(freqtrade)
freqtrade.create_trade()
trade = Trade.query.first()
Trade.session = MagicMock()
freqtrade.config['dry_run'] = False
trade.stoploss_order_id = "abcd"
freqtrade.execute_sell(trade=trade, limit=1234,
sell_reason=SellType.STOP_LOSS)
assert sellmock.call_count == 1
assert log_has('Could not cancel stoploss order abcd', caplog.record_tuples)
def test_execute_sell_with_stoploss_on_exchange(default_conf,
ticker, fee, ticker_sell_up,
markets, mocker) -> None:

51
scripts/plot_dataframe.py
View File

@ -31,7 +31,7 @@ from typing import Any, Dict, List
import pandas as pd
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.arguments import Arguments
from freqtrade.data import history
from freqtrade.data.btanalysis import (extract_trades_of_period,
load_backtest_data, load_trades_from_db)
@ -43,38 +43,6 @@ from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange,
datadir: Path, refresh_pairs: bool, live: bool):
"""
Get tickers data for each pairs on live or local, option defined in args
:return: dictionary of tickers. output format: {'pair': tickersdata}
"""
ticker_interval = strategy.ticker_interval
tickers = history.load_data(
datadir=datadir,
pairs=pairs,
ticker_interval=ticker_interval,
refresh_pairs=refresh_pairs,
timerange=timerange,
exchange=exchange,
live=live,
)
# No ticker found, impossible to download, len mismatch
for pair, data in tickers.copy().items():
logger.debug("checking tickers data of pair: %s", pair)
logger.debug("data.empty: %s", data.empty)
logger.debug("len(data): %s", len(data))
if data.empty:
del tickers[pair]
logger.info(
'An issue occured while retreiving data of %s pair, please retry '
'using -l option for live or --refresh-pairs-cached', pair)
return tickers
def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame:
"""
Get tickers then Populate strategy indicators and signals, then return the full dataframe
@ -100,8 +68,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]):
-Generate plot files
:return: None
"""
exchange_name = config.get('exchange', {}).get('name').title()
exchange = ExchangeResolver(exchange_name, config).exchange
exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange
strategy = StrategyResolver(config).strategy
if "pairs" in config:
@ -113,10 +80,16 @@ def analyse_and_plot_pairs(config: Dict[str, Any]):
timerange = Arguments.parse_timerange(config["timerange"])
ticker_interval = strategy.ticker_interval
tickers = get_tickers_data(strategy, exchange, pairs, timerange,
datadir=Path(str(config.get("datadir"))),
refresh_pairs=config.get('refresh_pairs', False),
live=config.get("live", False))
tickers = history.load_data(
datadir=Path(str(config.get("datadir"))),
pairs=pairs,
ticker_interval=config['ticker_interval'],
refresh_pairs=config.get('refresh_pairs', False),
timerange=timerange,
exchange=exchange,
live=config.get("live", False),
)
pair_counter = 0
for pair, data in tickers.items():
pair_counter += 1

32
scripts/rest_client.py
View File

@ -65,14 +65,14 @@ class FtRestClient():
def start(self):
"""
Start the bot if it's in stopped state.
:returns: json object
:return: json object
"""
return self._post("start")
def stop(self):
"""
Stop the bot. Use start to restart
:returns: json object
:return: json object
"""
return self._post("stop")
@ -80,77 +80,77 @@ class FtRestClient():
"""
Stop buying (but handle sells gracefully).
use reload_conf to reset
:returns: json object
:return: json object
"""
return self._post("stopbuy")
def reload_conf(self):
"""
Reload configuration
:returns: json object
:return: json object
"""
return self._post("reload_conf")
def balance(self):
"""
Get the account balance
:returns: json object
:return: json object
"""
return self._get("balance")
def count(self):
"""
Returns the amount of open trades
:returns: json object
:return: json object
"""
return self._get("count")
def daily(self, days=None):
"""
Returns the amount of open trades
:returns: json object
:return: json object
"""
return self._get("daily", params={"timescale": days} if days else None)
def edge(self):
"""
Returns information about edge
:returns: json object
:return: json object
"""
return self._get("edge")
def profit(self):
"""
Returns the profit summary
:returns: json object
:return: json object
"""
return self._get("profit")
def performance(self):
"""
Returns the performance of the different coins
:returns: json object
:return: json object
"""
return self._get("performance")
def status(self):
"""
Get the status of open trades
:returns: json object
:return: json object
"""
return self._get("status")
def version(self):
"""
Returns the version of the bot
:returns: json object containing the version
:return: json object containing the version
"""
return self._get("version")
def whitelist(self):
"""
Show the current whitelist
:returns: json object
:return: json object
"""
return self._get("whitelist")
@ -158,7 +158,7 @@ class FtRestClient():
"""
Show the current blacklist
:param add: List of coins to add (example: "BNB/BTC")
:returns: json object
:return: json object
"""
if not args:
return self._get("blacklist")
@ -170,7 +170,7 @@ class FtRestClient():
Buy an asset
:param pair: Pair to buy (ETH/BTC)
:param price: Optional - price to buy
:returns: json object of the trade
:return: json object of the trade
"""
data = {"pair": pair,
"price": price
@ -181,7 +181,7 @@ class FtRestClient():
"""
Force-sell a trade
:param tradeid: Id of the trade (can be received via status command)
:returns: json object
:return: json object
"""
return self._post("forcesell", data={"tradeid": tradeid})