adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into dataprovider-add-ticker

Browse Source
This commit is contained in:
hroff-1902
2020-05-14 13:22:52 +03:00
parent 63b55658ac d957f3dd2d
commit 3079e18239
53 changed files with 886 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
freqtrade/commands/__init__.py
View File

@@ -19,7 +19,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges,
start_list_hyperopts,
start_list_markets,
start_list_strategies,
start_list_timeframes)
start_list_timeframes,
start_show_trades)
from freqtrade.commands.optimize_commands import (start_backtesting,
start_edge, start_hyperopt)
from freqtrade.commands.pairlist_commands import start_test_pairlist

15
freqtrade/commands/arguments.py
View File

@@ -64,6 +64,8 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "ticker_interval"]
ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
"hyperopt_list_min_trades", "hyperopt_list_max_trades",
"hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
@@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies",
"list-hyperopts", "hyperopt-list", "hyperopt-show",
"plot-dataframe", "plot-profit"]
"plot-dataframe", "plot-profit", "show-trades"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]
@@ -163,7 +165,7 @@ class Arguments:
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config,
start_new_hyperopt, start_new_strategy,
start_plot_dataframe, start_plot_profit,
start_plot_dataframe, start_plot_profit, start_show_trades,
start_backtesting, start_hyperopt, start_edge,
start_test_pairlist, start_trading)
@@ -330,6 +332,15 @@ class Arguments:
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
# Add show-trades subcommand
show_trades = subparsers.add_parser(
'show-trades',
help='Show trades.',
parents=[_common_parser],
)
show_trades.set_defaults(func=start_show_trades)
self._build_args(optionlist=ARGS_SHOW_TRADES, parser=show_trades)
# Add hyperopt-list subcommand
hyperopt_list_cmd = subparsers.add_parser(
'hyperopt-list',

7
freqtrade/commands/cli_options.py
View File

@@ -217,7 +217,7 @@ AVAILABLE_CLI_OPTIONS = {
),
"print_json": Arg(
'--print-json',
help='Print best result detailization in JSON format.',
help='Print output in JSON format.',
action='store_true',
default=False,
),
@@ -425,6 +425,11 @@ AVAILABLE_CLI_OPTIONS = {
choices=["DB", "file"],
default="file",
),
"trade_ids": Arg(
'--trade-ids',
help='Specify the list of trade ids.',
nargs='+',
),
# hyperopt-list, hyperopt-show
"hyperopt_list_profitable": Arg(
'--profitable',

27
freqtrade/commands/list_commands.py
View File

@@ -197,3 +197,30 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
args.get('list_pairs_print_json', False) or
args.get('print_csv', False)):
print(f"{summary_str}.")
def start_show_trades(args: Dict[str, Any]) -> None:
"""
Show trades
"""
from freqtrade.persistence import init, Trade
import json
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if 'db_url' not in config:
raise OperationalException("--db-url is required for this command.")
logger.info(f'Using DB: "{config["db_url"]}"')
init(config['db_url'], clean_open_orders=False)
tfilter = []
if config.get('trade_ids'):
tfilter.append(Trade.id.in_(config['trade_ids']))
trades = Trade.get_trades(tfilter).all()
logger.info(f"Printing {len(trades)} Trades: ")
if config.get('print_json', False):
print(json.dumps([trade.to_json() for trade in trades], indent=4))
else:
for trade in trades:
print(trade)

3
freqtrade/commands/trade_commands.py
View File

@@ -18,6 +18,9 @@ def start_trading(args: Dict[str, Any]) -> int:
try:
worker = Worker(args)
worker.run()
except Exception as e:
logger.error(str(e))
logger.exception("Fatal exception!")
except KeyboardInterrupt:
logger.info('SIGINT received, aborting ...')
finally:

4
freqtrade/configuration/configuration.py
View File

@@ -351,8 +351,12 @@ class Configuration:
self._args_to_config(config, argname='indicators2',
logstring='Using indicators2: {}')
self._args_to_config(config, argname='trade_ids',
logstring='Filtering on trade_ids: {}')
self._args_to_config(config, argname='plot_limit',
logstring='Limiting plot to: {}')
self._args_to_config(config, argname='trade_source',
logstring='Using trades from: {}')

3
freqtrade/constants.py
View File

@@ -24,6 +24,9 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
DRY_RUN_WALLET = 1000
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons
DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
# Don't modify sequence of DEFAULT_TRADES_COLUMNS
# it has wide consequences for stored trades files
DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
USERPATH_HYPEROPTS = 'hyperopts'
USERPATH_STRATEGIES = 'strategies'

36
freqtrade/data/converter.py
View File

@@ -1,14 +1,17 @@
"""
Functions to convert data from one format to another
"""
import itertools
import logging
from datetime import datetime, timezone
from typing import Any, Dict
from operator import itemgetter
from typing import Any, Dict, List
import pandas as pd
from pandas import DataFrame, to_datetime
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
DEFAULT_TRADES_COLUMNS)
logger = logging.getLogger(__name__)
@@ -154,7 +157,27 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
return frame
def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
def trades_remove_duplicates(trades: List[List]) -> List[List]:
"""
Removes duplicates from the trades list.
Uses itertools.groupby to avoid converting to pandas.
Tests show it as being pretty efficient on lists of 4M Lists.
:param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns
:return: same format as above, but with duplicates removed
"""
return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))]
def trades_dict_to_list(trades: List[Dict]) -> List[List]:
"""
Convert fetch_trades result into a List (to be more memory efficient).
:param trades: List of trades, as returned by ccxt.fetch_trades.
:return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns
"""
return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades]
def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame:
"""
Converts trades list to OHLCV list
TODO: This should get a dedicated test
@@ -164,9 +187,10 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
"""
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
df = pd.DataFrame(trades)
df['datetime'] = pd.to_datetime(df['datetime'])
df = df.set_index('datetime')
df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms',
utc=True,)
df = df.set_index('timestamp')
df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()

18
freqtrade/data/dataprovider.py
View File

@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from freqtrade.data.history import load_pair_history
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.state import RunMode
@@ -18,9 +19,10 @@ logger = logging.getLogger(__name__)
class DataProvider:
def __init__(self, config: dict, exchange: Exchange) -> None:
def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
self._config = config
self._exchange = exchange
self._pairlists = pairlists
def refresh(self,
pairlist: List[Tuple[str, str]],
@@ -115,3 +117,17 @@ class DataProvider:
can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other".
"""
return RunMode(self._config.get('runmode', RunMode.OTHER))
def current_whitelist(self) -> List[str]:
"""
fetch latest available whitelist.
Useful when you have a large whitelist and need to call each pair as an informative pair.
As available pairs does not show whitelist until after informative pairs have been cached.
:return: list of pairs in whitelist
"""
if self._pairlists:
return self._pairlists.whitelist
else:
raise OperationalException("Dataprovider was not initialized with a pairlist provider.")

36
freqtrade/data/history/history_utils.py
View File

@@ -9,10 +9,13 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv
from freqtrade.data.converter import (ohlcv_to_dataframe,
trades_remove_duplicates,
trades_to_ohlcv)
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.misc import format_ms_time
logger = logging.getLogger(__name__)
@@ -257,27 +260,40 @@ def _download_trades_history(exchange: Exchange,
"""
try:
since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
since = timerange.startts * 1000 if \
(timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000
trades = data_handler.trades_load(pair)
from_id = trades[-1]['id'] if trades else None
# TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp
# DEFAULT_TRADES_COLUMNS: 1 -> id
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
from_id = trades[-1][1] if trades else None
if trades and since < trades[-1][0]:
# Reset since to the last available point
# - 5 seconds (to ensure we're getting all trades)
since = trades[-1][0] - (5 * 1000)
logger.info(f"Using last trade date -5s - Downloading trades for {pair} "
f"since: {format_ms_time(since)}.")
logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}")
logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}")
logger.info(f"Current Amount of trades: {len(trades)}")
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000,
since=since,
from_id=from_id,
)
trades.extend(new_trades[1])
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_remove_duplicates(trades)
data_handler.trades_store(pair, data=trades)
logger.debug("New Start: %s", trades[0]['datetime'])
logger.debug("New End: %s", trades[-1]['datetime'])
logger.debug(f"New Start: {format_ms_time(trades[0][0])}")
logger.debug(f"New End: {format_ms_time(trades[-1][0])}")
logger.info(f"New Amount of trades: {len(trades)}")
return True

30
freqtrade/data/history/idatahandler.py
View File

@@ -8,16 +8,20 @@ from abc import ABC, abstractclassmethod, abstractmethod
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional, Type
from typing import List, Optional, Type
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
from freqtrade.data.converter import (clean_ohlcv_dataframe,
trades_remove_duplicates, trim_dataframe)
from freqtrade.exchange import timeframe_to_seconds
logger = logging.getLogger(__name__)
# Type for trades list
TradeList = List[List]
class IDataHandler(ABC):
@@ -89,23 +93,25 @@ class IDataHandler(ABC):
"""
@abstractmethod
def trades_store(self, pair: str, data: List[Dict]) -> None:
def trades_store(self, pair: str, data: TradeList) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Dicts containing trade data
:param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS
"""
@abstractmethod
def trades_append(self, pair: str, data: List[Dict]):
def trades_append(self, pair: str, data: TradeList):
"""
Append data to existing files
:param pair: Pair - used for filename
:param data: List of Dicts containing trade data
:param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS
"""
@abstractmethod
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]:
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
"""
Load a pair from file, either .json.gz or .json
:param pair: Load trades for this pair
@@ -121,6 +127,16 @@ class IDataHandler(ABC):
:return: True when deleted, false if file did not exist.
"""
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
"""
Load a pair from file, either .json.gz or .json
Removes duplicates in the process.
:param pair: Load trades for this pair
:param timerange: Timerange to load trades for - currently not implemented
:return: List of trades
"""
return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
def ohlcv_load(self, pair, timeframe: str,
timerange: Optional[TimeRange] = None,
fill_missing: bool = True,

26
freqtrade/data/history/jsondatahandler.py
View File

@@ -1,6 +1,7 @@
import logging
import re
from pathlib import Path
from typing import Dict, List, Optional
from typing import List, Optional
import numpy as np
from pandas import DataFrame, read_json, to_datetime
@@ -8,8 +9,11 @@ from pandas import DataFrame, read_json, to_datetime
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import trades_dict_to_list
from .idatahandler import IDataHandler
from .idatahandler import IDataHandler, TradeList
logger = logging.getLogger(__name__)
class JsonDataHandler(IDataHandler):
@@ -113,24 +117,26 @@ class JsonDataHandler(IDataHandler):
# Check if regex found something and only return these results to avoid exceptions.
return [match[0].replace('_', '/') for match in _tmp if match]
def trades_store(self, pair: str, data: List[Dict]) -> None:
def trades_store(self, pair: str, data: TradeList) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Dicts containing trade data
:param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS
"""
filename = self._pair_trades_filename(self._datadir, pair)
misc.file_dump_json(filename, data, is_zip=self._use_zip)
def trades_append(self, pair: str, data: List[Dict]):
def trades_append(self, pair: str, data: TradeList):
"""
Append data to existing files
:param pair: Pair - used for filename
:param data: List of Dicts containing trade data
:param data: List of Lists containing trade data,
column sequence as in DEFAULT_TRADES_COLUMNS
"""
raise NotImplementedError()
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]:
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
"""
Load a pair from file, either .json.gz or .json
# TODO: respect timerange ...
@@ -140,9 +146,15 @@ class JsonDataHandler(IDataHandler):
"""
filename = self._pair_trades_filename(self._datadir, pair)
tradesdata = misc.file_load_json(filename)
if not tradesdata:
return []
if isinstance(tradesdata[0], dict):
# Convert trades dict to list
logger.info("Old trades format detected - converting")
tradesdata = trades_dict_to_list(tradesdata)
pass
return tradesdata
def trades_purge(self, pair: str) -> bool:

19
freqtrade/edge/edge_positioning.py
View File

@@ -238,20 +238,9 @@ class Edge:
:param result Dataframe
:return: result Dataframe
"""
# stake and fees
# stake = 0.015
# 0.05% is 0.0005
# fee = 0.001
# we set stake amount to an arbitrary amount.
# as it doesn't change the calculation.
# all returned values are relative.
# they are defined as ratios.
# We set stake amount to an arbitrary amount, as it doesn't change the calculation.
# All returned values are relative, they are defined as ratios.
stake = 0.015
fee = self.fee
open_fee = fee / 2
close_fee = fee / 2
result['trade_duration'] = result['close_time'] - result['open_time']
@@ -262,12 +251,12 @@ class Edge:
# Buy Price
result['buy_vol'] = stake / result['open_rate'] # How many target are we buying
result['buy_fee'] = stake * open_fee
result['buy_fee'] = stake * self.fee
result['buy_spend'] = stake + result['buy_fee'] # How much we're spending
# Sell price
result['sell_sum'] = result['buy_vol'] * result['close_rate']
result['sell_fee'] = result['sell_sum'] * close_fee
result['sell_fee'] = result['sell_sum'] * self.fee
result['sell_take'] = result['sell_sum'] - result['sell_fee']
# profit_ratio

39
freqtrade/exchange/exchange.py
View File

@@ -18,13 +18,12 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE,
TRUNCATE, decimal_to_precision)
from pandas import DataFrame
from freqtrade.data.converter import ohlcv_to_dataframe
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.exceptions import (DependencyException, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
from freqtrade.misc import deep_merge_dicts
CcxtModuleType = Any
@@ -769,7 +768,7 @@ class Exchange:
@retrier_async
async def _async_fetch_trades(self, pair: str,
since: Optional[int] = None,
params: Optional[dict] = None) -> List[Dict]:
params: Optional[dict] = None) -> List[List]:
"""
Asyncronously gets trade history using fetch_trades.
Handles exchange errors, does one call to the exchange.
@@ -789,7 +788,7 @@ class Exchange:
'(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
)
trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
return trades
return trades_dict_to_list(trades)
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching historical trade data.'
@@ -803,7 +802,7 @@ class Exchange:
async def _async_get_trade_history_id(self, pair: str,
until: int,
since: Optional[int] = None,
from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
from_id: Optional[str] = None) -> Tuple[str, List[List]]:
"""
Asyncronously gets trade history using fetch_trades
use this when exchange uses id-based iteration (check `self._trades_pagination`)
@@ -814,7 +813,7 @@ class Exchange:
returns tuple: (pair, trades-list)
"""
trades: List[Dict] = []
trades: List[List] = []
if not from_id:
# Fetch first elements using timebased method to get an ID to paginate on
@@ -823,7 +822,9 @@ class Exchange:
# e.g. Binance returns the "last 1000" candles within a 1h time interval
# - so we will miss the first trades.
t = await self._async_fetch_trades(pair, since=since)
from_id = t[-1]['id']
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp
# DEFAULT_TRADES_COLUMNS: 1 -> id
from_id = t[-1][1]
trades.extend(t[:-1])
while True:
t = await self._async_fetch_trades(pair,
@@ -831,21 +832,21 @@ class Exchange:
if len(t):
# Skip last id since its the key for the next call
trades.extend(t[:-1])
if from_id == t[-1]['id'] or t[-1]['timestamp'] > until:
if from_id == t[-1][1] or t[-1][0] > until:
logger.debug(f"Stopping because from_id did not change. "
f"Reached {t[-1]['timestamp']} > {until}")
f"Reached {t[-1][0]} > {until}")
# Reached the end of the defined-download period - add last trade as well.
trades.extend(t[-1:])
break
from_id = t[-1]['id']
from_id = t[-1][1]
else:
break
return (pair, trades)
async def _async_get_trade_history_time(self, pair: str, until: int,
since: Optional[int] = None) -> Tuple[str, List]:
since: Optional[int] = None) -> Tuple[str, List[List]]:
"""
Asyncronously gets trade history using fetch_trades,
when the exchange uses time-based iteration (check `self._trades_pagination`)
@@ -855,16 +856,18 @@ class Exchange:
returns tuple: (pair, trades-list)
"""
trades: List[Dict] = []
trades: List[List] = []
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp
# DEFAULT_TRADES_COLUMNS: 1 -> id
while True:
t = await self._async_fetch_trades(pair, since=since)
if len(t):
since = t[-1]['timestamp']
since = t[-1][1]
trades.extend(t)
# Reached the end of the defined-download period
if until and t[-1]['timestamp'] > until:
if until and t[-1][0] > until:
logger.debug(
f"Stopping because until was reached. {t[-1]['timestamp']} > {until}")
f"Stopping because until was reached. {t[-1][0]} > {until}")
break
else:
break
@@ -874,7 +877,7 @@ class Exchange:
async def _async_get_trade_history(self, pair: str,
since: Optional[int] = None,
until: Optional[int] = None,
from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
from_id: Optional[str] = None) -> Tuple[str, List[List]]:
"""
Async wrapper handling downloading trades using either time or id based methods.
"""
@@ -1041,9 +1044,9 @@ class Exchange:
return matched_trades
except ccxt.NetworkError as e:
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not get trades due to networking error. Message: {e}') from e
f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e

34
freqtrade/freqtradebot.py
View File

@@ -54,8 +54,11 @@ class FreqtradeBot:
# Init objects
self.config = config
self._sell_rate_cache = TTLCache(maxsize=100, ttl=5)
self._buy_rate_cache = TTLCache(maxsize=100, ttl=5)
# Cache values for 1800 to avoid frequent polling of the exchange for prices
# Caching only applies to RPC methods, so prices for open trades are still
# refreshed once every iteration.
self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800)
self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800)
self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)
@@ -68,15 +71,15 @@ class FreqtradeBot:
self.wallets = Wallets(self.config, self.exchange)
self.dataprovider = DataProvider(self.config, self.exchange)
self.pairlists = PairListManager(self.exchange, self.config)
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)
# Attach Dataprovider to Strategy baseclass
IStrategy.dp = self.dataprovider
# Attach Wallets to Strategy baseclass
IStrategy.wallets = self.wallets
self.pairlists = PairListManager(self.exchange, self.config)
# Initializing Edge only if enabled
self.edge = Edge(self.config, self.exchange, self.strategy) if \
self.config.get('edge', {}).get('enabled', False) else None
@@ -898,7 +901,8 @@ class FreqtradeBot:
Buy timeout - cancel order
:return: True if order was fully cancelled
"""
if order['status'] != 'canceled':
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in ('canceled', 'closed'):
reason = "cancelled due to timeout"
corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
@@ -909,7 +913,10 @@ class FreqtradeBot:
logger.info('Buy order %s for %s.', reason, trade)
if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
# Using filled to determine the filled amount
filled_amount = safe_value_fallback(corder, order, 'filled', 'filled')
if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
logger.info('Buy order fully cancelled. Removing %s from database.', trade)
# if trade is not partially completed, just delete the trade
Trade.session.delete(trade)
@@ -921,8 +928,7 @@ class FreqtradeBot:
# cancel_order may not contain the full order dict, so we need to fallback
# to the order dict aquired before cancelling.
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = order['amount'] - safe_value_fallback(corder, order,
'remaining', 'remaining')
trade.amount = filled_amount
trade.stake_amount = trade.amount * trade.open_rate
self.update_trade_state(trade, corder, trade.amount)
@@ -943,8 +949,12 @@ class FreqtradeBot:
if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
if not self.exchange.check_order_canceled_empty(order):
reason = "cancelled due to timeout"
# if trade is not partially completed, just delete the trade
self.exchange.cancel_order(trade.open_order_id, trade.pair)
try:
# if trade is not partially completed, just delete the trade
self.exchange.cancel_order(trade.open_order_id, trade.pair)
except InvalidOrderException:
logger.exception(f"Could not cancel sell order {trade.open_order_id}")
return 'error cancelling order'
logger.info('Sell order %s for %s.', reason, trade)
else:
reason = "cancelled on exchange"
@@ -982,7 +992,7 @@ class FreqtradeBot:
if wallet_amount >= amount:
return amount
elif wallet_amount > amount * 0.98:
logger.info(f"{pair} - Falling back to wallet-amount.")
logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.")
return wallet_amount
else:
raise DependencyException(

21
freqtrade/optimize/hyperopt.py
View File

@@ -387,12 +387,19 @@ class Hyperopt:
trials = json_normalize(results, max_level=1)
trials['Best'] = ''
trials['Stake currency'] = config['stake_currency']
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
trials = trials[base_metrics + param_metrics]
base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
param_columns = list(results[0]['params_dict'].keys())
trials.columns = base_columns + param_columns
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '*'
trials.loc[trials['is_best'], 'Best'] = 'Best'
@@ -648,7 +655,7 @@ class Hyperopt:
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
]
with progressbar.ProgressBar(
maxval=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
widgets=widgets
) as pbar:
EVALS = ceil(self.total_epochs / jobs)

56
freqtrade/rpc/api_server.py
View File

@@ -2,11 +2,16 @@ import logging
import threading
from datetime import date, datetime
from ipaddress import IPv4Address
from typing import Dict, Callable, Any
from typing import Any, Callable, Dict
from arrow import Arrow
from flask import Flask, jsonify, request
from flask.json import JSONEncoder
from flask_jwt_extended import (JWTManager, create_access_token,
create_refresh_token, get_jwt_identity,
jwt_refresh_token_required,
verify_jwt_in_request_optional)
from werkzeug.security import safe_str_cmp
from werkzeug.serving import make_server
from freqtrade.__init__ import __version__
@@ -38,9 +43,9 @@ class ArrowJSONEncoder(JSONEncoder):
def require_login(func: Callable[[Any, Any], Any]):
def func_wrapper(obj, *args, **kwargs):
verify_jwt_in_request_optional()
auth = request.authorization
if auth and obj.check_auth(auth.username, auth.password):
if get_jwt_identity() or auth and obj.check_auth(auth.username, auth.password):
return func(obj, *args, **kwargs)
else:
return jsonify({"error": "Unauthorized"}), 401
@@ -70,8 +75,8 @@ class ApiServer(RPC):
"""
def check_auth(self, username, password):
return (username == self._config['api_server'].get('username') and
password == self._config['api_server'].get('password'))
return (safe_str_cmp(username, self._config['api_server'].get('username')) and
safe_str_cmp(password, self._config['api_server'].get('password')))
def __init__(self, freqtrade) -> None:
"""
@@ -83,6 +88,12 @@ class ApiServer(RPC):
self._config = freqtrade.config
self.app = Flask(__name__)
# Setup the Flask-JWT-Extended extension
self.app.config['JWT_SECRET_KEY'] = self._config['api_server'].get(
'jwt_secret_key', 'super-secret')
self.jwt = JWTManager(self.app)
self.app.json_encoder = ArrowJSONEncoder
# Register application handling
@@ -148,6 +159,10 @@ class ApiServer(RPC):
self.app.register_error_handler(404, self.page_not_found)
# Actions to control the bot
self.app.add_url_rule(f'{BASE_URI}/token/login', 'login',
view_func=self._token_login, methods=['POST'])
self.app.add_url_rule(f'{BASE_URI}/token/refresh', 'token_refresh',
view_func=self._token_refresh, methods=['POST'])
self.app.add_url_rule(f'{BASE_URI}/start', 'start',
view_func=self._start, methods=['POST'])
self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST'])
@@ -199,6 +214,37 @@ class ApiServer(RPC):
'code': 404
}), 404
@require_login
@rpc_catch_errors
def _token_login(self):
"""
Handler for /token/login
Returns a JWT token
"""
auth = request.authorization
if auth and self.check_auth(auth.username, auth.password):
keystuff = {'u': auth.username}
ret = {
'access_token': create_access_token(identity=keystuff),
'refresh_token': create_refresh_token(identity=keystuff),
}
return self.rest_dump(ret)
return jsonify({"error": "Unauthorized"}), 401
@jwt_refresh_token_required
@rpc_catch_errors
def _token_refresh(self):
"""
Handler for /token/refresh
Returns a JWT token based on a JWT refresh token
"""
current_user = get_jwt_identity()
new_token = create_access_token(identity=current_user, fresh=False)
ret = {'access_token': new_token}
return self.rest_dump(ret)
@require_login
@rpc_catch_errors
def _start(self):

3
freqtrade/rpc/rpc.py
View File

@@ -94,6 +94,7 @@ class RPC:
'dry_run': config['dry_run'],
'stake_currency': config['stake_currency'],
'stake_amount': config['stake_amount'],
'max_open_trades': config['max_open_trades'],
'minimal_roi': config['minimal_roi'].copy(),
'stoploss': config['stoploss'],
'trailing_stop': config['trailing_stop'],
@@ -103,6 +104,8 @@ class RPC:
'ticker_interval': config['ticker_interval'],
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),
'state': str(self._freqtrade.state)
}
return val

6
freqtrade/rpc/telegram.py
View File

@@ -579,7 +579,7 @@ class Telegram(RPC):
"*/whitelist:* `Show current whitelist` \n" \
"*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " \
"to the blacklist.` \n" \
"*/edge:* `Shows validated pairs by Edge if it is enabeld` \n" \
"*/edge:* `Shows validated pairs by Edge if it is enabled` \n" \
"*/help:* `This help message`\n" \
"*/version:* `Show version`"
@@ -621,10 +621,12 @@ class Telegram(RPC):
f"*Mode:* `{'Dry-run' if val['dry_run'] else 'Live'}`\n"
f"*Exchange:* `{val['exchange']}`\n"
f"*Stake per trade:* `{val['stake_amount']} {val['stake_currency']}`\n"
f"*Max open Trades:* `{val['max_open_trades']}`\n"
f"*Minimum ROI:* `{val['minimal_roi']}`\n"
f"{sl_info}"
f"*Ticker Interval:* `{val['ticker_interval']}`\n"
f"*Strategy:* `{val['strategy']}`"
f"*Strategy:* `{val['strategy']}`\n"
f"*Current state:* `{val['state']}`"
)
def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:

3
freqtrade/state.py
View File

@@ -14,6 +14,9 @@ class State(Enum):
STOPPED = 2
RELOAD_CONF = 3
def __str__(self):
return f"{self.name.lower()}"
class RunMode(Enum):
"""