adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into arrow_deprecation_timestamp

Browse Source
This commit is contained in:
Matthias 2020-10-20 20:01:54 +02:00
commit 7a092271c5
31 changed files with 291 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -140,6 +140,10 @@ information about the bot, we encourage you to join our slack channel.
- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE).
To those interested to check out the newly created discord channel. Click [here](https://discord.gg/MA9v74M)
P.S. currently since discord channel is relatively new, answers to questions might be slightly delayed as currently the user base quite small.
### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue)
If you discover a bug in the bot, please

4
config.json.example
View File

@ -5,15 +5,15 @@
"tradable_balance_ratio": 0.99,
"fiat_display_currency": "USD",
"timeframe": "5m",
"dry_run": false,
"dry_run": true,
"cancel_open_orders_on_exit": false,
"unfilledtimeout": {
"buy": 10,
"sell": 30
},
"bid_strategy": {
"ask_last_balance": 0.0,
"use_order_book": false,
"ask_last_balance": 0.0,
"order_book_top": 1,
"check_depth_of_market": {
"enabled": false,

2
config_full.json.example
View File

@ -7,7 +7,7 @@
"amount_reserve_percent": 0.05,
"amend_last_stake_amount": false,
"last_stake_amount_min_ratio": 0.5,
"dry_run": false,
"dry_run": true,
"cancel_open_orders_on_exit": false,
"timeframe": "5m",
"trailing_stop": false,

5
config_kraken.json.example
View File

@ -27,12 +27,11 @@
"use_sell_signal": true,
"sell_profit_only": false,
"ignore_roi_if_buy_signal": false
},
"exchange": {
"name": "kraken",
"key": "",
"secret": "",
"key": "your_exchange_key",
"secret": "your_exchange_key",
"ccxt_config": {"enableRateLimit": true},
"ccxt_async_config": {
"enableRateLimit": true,

4
docs/index.md
View File

@ -64,6 +64,10 @@ For any questions not covered by the documentation or for further information ab
Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) to join the Freqtrade Slack channel.
To those interested to check out the newly created discord channel. Click [here](https://discord.gg/MA9v74M)
P.S. currently since discord channel is relatively new, answers to questions might be slightly delayed as currently the user base quite small.
## Ready to try?
Begin by reading our installation guide [for docker](docker.md), or for [installation without docker](installation.md).

2
docs/requirements-docs.txt
View File

@ -1,3 +1,3 @@
mkdocs-material==6.0.2
mkdocs-material==6.1.0
mdx_truly_sane_lists==1.2
pymdown-extensions==8.0.1

4
freqtrade/commands/list_commands.py
View File

@ -205,14 +205,14 @@ def start_show_trades(args: Dict[str, Any]) -> None:
"""
import json
from freqtrade.persistence import Trade, init
from freqtrade.persistence import Trade, init_db
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if 'db_url' not in config:
raise OperationalException("--db-url is required for this command.")
logger.info(f'Using DB: "{config["db_url"]}"')
init(config['db_url'], clean_open_orders=False)
init_db(config['db_url'], clean_open_orders=False)
tfilter = []
if config.get('trade_ids'):

5
freqtrade/data/btanalysis.py
View File

@ -9,10 +9,9 @@ from typing import Any, Dict, Optional, Tuple, Union
import numpy as np
import pandas as pd
from freqtrade import persistence
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.misc import json_load
from freqtrade.persistence import Trade
from freqtrade.persistence import Trade, init_db
logger = logging.getLogger(__name__)
@ -218,7 +217,7 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
Can also serve as protection to load the correct result.
:return: Dataframe containing Trades
"""
persistence.init(db_url, clean_open_orders=False)
init_db(db_url, clean_open_orders=False)
columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
"open_rate", "close_rate", "amount", "trade_duration", "sell_reason",

1
freqtrade/exchange/__init__.py
View File

@ -5,6 +5,7 @@ from freqtrade.exchange.exchange import Exchange
# isort: on
from freqtrade.exchange.bibox import Bibox
from freqtrade.exchange.binance import Binance
from freqtrade.exchange.bittrex import Bittrex
from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
get_exchange_bad_reason, is_exchange_bad,
is_exchange_known_ccxt, is_exchange_officially_supported,

13
freqtrade/exchange/binance.py
View File

@ -20,20 +20,9 @@ class Binance(Exchange):
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"trades_pagination": "id",
"trades_pagination_arg": "fromId",
"l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
}
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""
get order book level 2 from exchange
20180619: binance support limits but only on specific range
"""
limit_range = [5, 10, 20, 50, 100, 500, 1000]
# get next-higher step in the limit_range list
limit = min(list(filter(lambda x: limit <= x, limit_range)))
return super().fetch_l2_order_book(pair, limit)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)

23
freqtrade/exchange/bittrex.py Normal file
View File

@ -0,0 +1,23 @@
""" Bittrex exchange subclass """
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Bittrex(Exchange):
"""
Bittrex exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
_ft_has: Dict = {
"l2_limit_range": [1, 25, 500],
}

15
freqtrade/exchange/exchange.py
View File

@ -53,7 +53,7 @@ class Exchange:
"ohlcv_partial_candle": True,
"trades_pagination": "time", # Possible are "time" or "id"
"trades_pagination_arg": "since",
"l2_limit_range": None,
}
_ft_has: Dict = {}
@ -1069,6 +1069,16 @@ class Exchange:
return self.fetch_stoploss_order(order_id, pair)
return self.fetch_order(order_id, pair)
@staticmethod
def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]):
"""
Get next greater value in the list.
Used by fetch_l2_order_book if the api only supports a limited range
"""
if not limit_range:
return limit
return min([x for x in limit_range if limit <= x] + [max(limit_range)])
@retrier
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""
@ -1077,9 +1087,10 @@ class Exchange:
Returns a dict in the format
{'asks': [price, volume], 'bids': [price, volume]}
"""
limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'])
try:
return self._api.fetch_l2_order_book(pair, limit)
return self._api.fetch_l2_order_book(pair, limit1)
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching order book.'

8
freqtrade/freqtradebot.py
View File

@ -12,7 +12,7 @@ from typing import Any, Dict, List, Optional
import arrow
from cachetools import TTLCache
from freqtrade import __version__, constants, persistence
from freqtrade import __version__, constants
from freqtrade.configuration import validate_config_consistency
from freqtrade.data.converter import order_book_to_dataframe
from freqtrade.data.dataprovider import DataProvider
@ -22,7 +22,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.misc import safe_value_fallback, safe_value_fallback2
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Order, Trade
from freqtrade.persistence import Order, Trade, cleanup_db, init_db
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State
@ -68,7 +68,7 @@ class FreqtradeBot:
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
persistence.init(self.config.get('db_url', None), clean_open_orders=self.config['dry_run'])
init_db(self.config.get('db_url', None), clean_open_orders=self.config['dry_run'])
self.wallets = Wallets(self.config, self.exchange)
@ -123,7 +123,7 @@ class FreqtradeBot:
self.check_for_open_trades()
self.rpc.cleanup()
persistence.cleanup()
cleanup_db()
def startup(self) -> None:
"""

185
freqtrade/optimize/backtesting.py
View File

@ -4,11 +4,11 @@
This module contains the backtesting logic
"""
import logging
from collections import defaultdict
from copy import deepcopy
from datetime import datetime, timedelta
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
import arrow
from pandas import DataFrame
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
@ -28,6 +28,15 @@ from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
logger = logging.getLogger(__name__)
# Indexes for backtest tuples
DATE_IDX = 0
BUY_IDX = 1
OPEN_IDX = 2
CLOSE_IDX = 3
SELL_IDX = 4
LOW_IDX = 5
HIGH_IDX = 6
class BacktestResult(NamedTuple):
"""
@ -115,7 +124,7 @@ class Backtesting:
"""
Load strategy into backtesting
"""
self.strategy = strategy
self.strategy: IStrategy = strategy
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
@ -147,12 +156,14 @@ class Backtesting:
return data, timerange
def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]:
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
"""
Helper function to convert a processed dataframes into lists for performance reasons.
Used by backtest() - so keep this optimized for performance.
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
data: Dict = {}
# Create dict with data
@ -172,10 +183,10 @@ class Backtesting:
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = [x for x in df_analyzed.itertuples()]
data[pair] = [x for x in df_analyzed.itertuples(index=False, name=None)]
return data
def _get_close_rate(self, sell_row, trade: Trade, sell: SellCheckTuple,
def _get_close_rate(self, sell_row: Tuple, trade: Trade, sell: SellCheckTuple,
trade_dur: int) -> float:
"""
Get close rate for backtesting result
@ -186,12 +197,12 @@ class Backtesting:
return trade.stop_loss
elif sell.sell_type == (SellType.ROI):
roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur)
if roi is not None:
if roi is not None and roi_entry is not None:
if roi == -1 and roi_entry % self.timeframe_min == 0:
# When forceselling with ROI=-1, the roi time will always be equal to trade_dur.
# If that entry is a multiple of the timeframe (so on candle open)
# - we'll use open instead of close
return sell_row.open
return sell_row[OPEN_IDX]
# - (Expected abs profit + open_rate + open_fee) / (fee_close -1)
close_rate = - (trade.open_rate * roi + trade.open_rate *
@ -199,57 +210,38 @@ class Backtesting:
if (trade_dur > 0 and trade_dur == roi_entry
and roi_entry % self.timeframe_min == 0
and sell_row.open > close_rate):
and sell_row[OPEN_IDX] > close_rate):
# new ROI entry came into effect.
# use Open rate if open_rate > calculated sell rate
return sell_row.open
return sell_row[OPEN_IDX]
# Use the maximum between close_rate and low as we
# cannot sell outside of a candle.
# Applies when a new ROI setting comes in place and the whole candle is above that.
return max(close_rate, sell_row.low)
return max(close_rate, sell_row[LOW_IDX])
else:
# This should not be reached...
return sell_row.open
return sell_row[OPEN_IDX]
else:
return sell_row.open
return sell_row[OPEN_IDX]
def _get_sell_trade_entry(
self, pair: str, buy_row: DataFrame,
partial_ohlcv: List, trade_count_lock: Dict,
stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]:
def _get_sell_trade_entry(self, trade: Trade, sell_row: Tuple) -> Optional[BacktestResult]:
trade = Trade(
pair=pair,
open_rate=buy_row.open,
open_date=buy_row.date,
stake_amount=stake_amount,
amount=round(stake_amount / buy_row.open, 8),
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
)
logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.")
# calculate win/lose forwards from buy point
for sell_row in partial_ohlcv:
if max_open_trades > 0:
# Increase trade_count_lock for every iteration
trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, sell_row.buy,
sell_row.sell, low=sell_row.low, high=sell_row.high)
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], sell_row[DATE_IDX],
sell_row[BUY_IDX], sell_row[SELL_IDX],
low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
if sell.sell_flag:
trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60)
trade_dur = int((sell_row[DATE_IDX] - trade.open_date).total_seconds() // 60)
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
return BacktestResult(pair=pair,
return BacktestResult(pair=trade.pair,
profit_percent=trade.calc_profit_ratio(rate=closerate),
profit_abs=trade.calc_profit(rate=closerate),
open_date=buy_row.date,
open_rate=buy_row.open,
open_date=trade.open_date,
open_rate=trade.open_rate,
open_fee=self.fee,
close_date=sell_row.date,
close_date=sell_row[DATE_IDX],
close_rate=closerate,
close_fee=self.fee,
amount=trade.amount,
@ -257,33 +249,40 @@ class Backtesting:
open_at_end=False,
sell_reason=sell.sell_type
)
if partial_ohlcv:
# no sell condition found - trade stil open at end of backtest period
sell_row = partial_ohlcv[-1]
bt_res = BacktestResult(pair=pair,
profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open),
open_date=buy_row.date,
open_rate=buy_row.open,
return None
def handle_left_open(self, open_trades: Dict[str, List[Trade]],
data: Dict[str, List[Tuple]]) -> List[BacktestResult]:
"""
Handling of left open trades at the end of backtesting
"""
trades = []
for pair in open_trades.keys():
if len(open_trades[pair]) > 0:
for trade in open_trades[pair]:
sell_row = data[pair][-1]
trade_entry = BacktestResult(pair=trade.pair,
profit_percent=trade.calc_profit_ratio(
rate=sell_row[OPEN_IDX]),
profit_abs=trade.calc_profit(sell_row[OPEN_IDX]),
open_date=trade.open_date,
open_rate=trade.open_rate,
open_fee=self.fee,
close_date=sell_row.date,
close_rate=sell_row.open,
close_date=sell_row[DATE_IDX],
close_rate=sell_row[OPEN_IDX],
close_fee=self.fee,
amount=trade.amount,
trade_duration=int((
sell_row.date - buy_row.date).total_seconds() // 60),
sell_row[DATE_IDX] - trade.open_date
).total_seconds() // 60),
open_at_end=True,
sell_reason=SellType.FORCE_SELL
)
logger.debug(f"{pair} - Force selling still open trade, "
f"profit percent: {bt_res.profit_percent}, "
f"profit abs: {bt_res.profit_abs}")
return bt_res
return None
trades.append(trade_entry)
return trades
def backtest(self, processed: Dict, stake_amount: float,
start_date: arrow.Arrow, end_date: arrow.Arrow,
start_date: datetime, end_date: datetime,
max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame:
"""
Implement backtesting functionality
@ -305,19 +304,21 @@ class Backtesting:
f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}"
)
trades = []
trade_count_lock: Dict = {}
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
lock_pair_until: Dict = {}
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = {}
tmp = start_date + timedelta(minutes=self.timeframe_min)
open_trades: Dict[str, List] = defaultdict(list)
open_trade_count = 0
# Loop timerange and get candle for each pair at that point in time
while tmp < end_date:
while tmp <= end_date:
open_trade_count_start = open_trade_count
for i, pair in enumerate(data):
if pair not in indexes:
@ -331,42 +332,52 @@ class Backtesting:
continue
# Waits until the time-counter reaches the start of the data for this pair.
if row.date > tmp.datetime:
if row[DATE_IDX] > tmp:
continue
indexes[pair] += 1
if row.buy == 0 or row.sell == 1:
continue # skip rows where no buy signal or that would immediately sell off
if (not position_stacking and pair in lock_pair_until
and row.date <= lock_pair_until[pair]):
# without positionstacking, we can only have one open trade per pair.
continue
if max_open_trades > 0:
# Check if max_open_trades has already been reached for the given date
if not trade_count_lock.get(row.date, 0) < max_open_trades:
continue
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
# max_open_trades must be respected
# don't open on the last row
if ((position_stacking or len(open_trades[pair]) == 0)
and max_open_trades > 0 and open_trade_count_start < max_open_trades
and tmp != end_date
and row[BUY_IDX] == 1 and row[SELL_IDX] != 1):
# Enter trade
trade = Trade(
pair=pair,
open_rate=row[OPEN_IDX],
open_date=row[DATE_IDX],
stake_amount=stake_amount,
amount=round(stake_amount / row[OPEN_IDX], 8),
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
)
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behaviour - not sure if this is correct
# Prevents buying if the trade-slot was freed in this candle
open_trade_count_start += 1
open_trade_count += 1
# logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.")
open_trades[pair].append(trade)
for trade in open_trades[pair]:
# since indexes has been incremented before, we need to go one step back to
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(pair, row, data[pair][indexes[pair]-1:],
trade_count_lock, stake_amount,
max_open_trades)
trade_entry = self._get_sell_trade_entry(trade, row)
# Sell occured
if trade_entry:
logger.debug(f"{pair} - Locking pair till "
f"close_date={trade_entry.close_date}")
lock_pair_until[pair] = trade_entry.close_date
# logger.debug(f"{pair} - Backtesting sell {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
trades.append(trade_entry)
else:
# Set lock_pair_until to end of testing period if trade could not be closed
lock_pair_until[pair] = end_date.datetime
# Move time one configured time_interval ahead.
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
return DataFrame.from_records(trades, columns=BacktestResult._fields)
def start(self) -> None:
@ -412,8 +423,8 @@ class Backtesting:
results = self.backtest(
processed=preprocessed,
stake_amount=self.config['stake_amount'],
start_date=min_date,
end_date=max_date,
start_date=min_date.datetime,
end_date=max_date.datetime,
max_open_trades=max_open_trades,
position_stacking=position_stacking,
)

28
freqtrade/optimize/hyperopt.py
View File

@ -94,14 +94,14 @@ class Hyperopt:
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = \
self.custom_hyperopt.populate_indicators # type: ignore
self.backtesting.strategy.advise_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
self.backtesting.strategy.advise_buy = \
self.custom_hyperopt.populate_buy_trend # type: ignore
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.backtesting.strategy.advise_sell = \
self.custom_hyperopt.populate_sell_trend # type: ignore
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
@ -508,16 +508,16 @@ class Hyperopt:
params_details = self._get_params_details(params_dict)
if self.has_space('roi'):
self.backtesting.strategy.minimal_roi = \
self.custom_hyperopt.generate_roi_table(params_dict)
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.custom_hyperopt.generate_roi_table(params_dict))
if self.has_space('buy'):
self.backtesting.strategy.advise_buy = \
self.custom_hyperopt.buy_strategy_generator(params_dict)
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.buy_strategy_generator(params_dict))
if self.has_space('sell'):
self.backtesting.strategy.advise_sell = \
self.custom_hyperopt.sell_strategy_generator(params_dict)
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.sell_strategy_generator(params_dict))
if self.has_space('stoploss'):
self.backtesting.strategy.stoploss = params_dict['stoploss']
@ -538,8 +538,8 @@ class Hyperopt:
backtesting_results = self.backtesting.backtest(
processed=processed,
stake_amount=self.config['stake_amount'],
start_date=min_date,
end_date=max_date,
start_date=min_date.datetime,
end_date=max_date.datetime,
max_open_trades=self.max_open_trades,
position_stacking=self.position_stacking,
)

2
freqtrade/persistence/__init__.py
View File

@ -1,3 +1,3 @@
# flake8: noqa: F401
from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup, init
from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup_db, init_db

10
freqtrade/persistence/models.py
View File

@ -29,7 +29,7 @@ _DECL_BASE: Any = declarative_base()
_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls'
def init(db_url: str, clean_open_orders: bool = False) -> None:
def init_db(db_url: str, clean_open_orders: bool = False) -> None:
"""
Initializes this module with the given config,
registers all known command handlers
@ -72,7 +72,7 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
clean_dry_run_db()
def cleanup() -> None:
def cleanup_db() -> None:
"""
Flushes all pending operations to disk.
:return: None
@ -167,12 +167,12 @@ class Order(_DECL_BASE):
"""
Get all non-closed orders - useful when trying to batch-update orders
"""
filtered_orders = [o for o in orders if o.order_id == order['id']]
filtered_orders = [o for o in orders if o.order_id == order.get('id')]
if filtered_orders:
oobj = filtered_orders[0]
oobj.update_from_ccxt_object(order)
else:
logger.warning(f"Did not find order for {order['id']}.")
logger.warning(f"Did not find order for {order}.")
@staticmethod
def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order':
@ -399,7 +399,7 @@ class Trade(_DECL_BASE):
self.close(order['average'])
else:
raise ValueError(f'Unknown order type: {order_type}')
cleanup()
cleanup_db()
def close(self, rate: float) -> None:
"""

2
freqtrade/rpc/api_server.py
View File

@ -563,7 +563,7 @@ class ApiServer(RPC):
config.update({
'strategy': strategy,
})
results = self._rpc_analysed_history_full(config, pair, timeframe, timerange)
results = RPC._rpc_analysed_history_full(config, pair, timeframe, timerange)
return jsonify(results)
@require_login

10
freqtrade/rpc/rpc.py
View File

@ -656,8 +656,9 @@ class RPC:
raise RPCException('Edge is not enabled.')
return self._freqtrade.edge.accepted_pairs()
def _convert_dataframe_to_dict(self, strategy: str, pair: str, timeframe: str,
dataframe: DataFrame, last_analyzed: datetime) -> Dict[str, Any]:
@staticmethod
def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame,
last_analyzed: datetime) -> Dict[str, Any]:
has_content = len(dataframe) != 0
buy_signals = 0
sell_signals = 0
@ -711,7 +712,8 @@ class RPC:
return self._convert_dataframe_to_dict(self._freqtrade.config['strategy'],
pair, timeframe, _data, last_analyzed)
def _rpc_analysed_history_full(self, config, pair: str, timeframe: str,
@staticmethod
def _rpc_analysed_history_full(config, pair: str, timeframe: str,
timerange: str) -> Dict[str, Any]:
timerange_parsed = TimeRange.parse_timerange(timerange)
@ -726,7 +728,7 @@ class RPC:
strategy = StrategyResolver.load_strategy(config)
df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair})
return self._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe,
return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe,
df_analyzed, arrow.Arrow.utcnow().datetime)
def _rpc_plot_config(self) -> Dict[str, Any]:

2
requirements-dev.txt
View File

@ -13,7 +13,7 @@ pytest-asyncio==0.14.0
pytest-cov==2.10.1
pytest-mock==3.3.1
pytest-random-order==1.0.4
isort==5.6.3
isort==5.6.4
# Convert jupyter notebooks to markdown documents
nbconvert==6.0.7

2
requirements-hyperopt.txt
View File

@ -2,7 +2,7 @@
-r requirements.txt
# Required for hyperopt
scipy==1.5.2
scipy==1.5.3
scikit-learn==0.23.2
scikit-optimize==0.8.1
filelock==3.0.12

12
requirements.txt
View File

@ -1,11 +1,11 @@
numpy==1.19.2
pandas==1.1.3
ccxt==1.36.2
ccxt==1.36.66
multidict==4.7.6
aiohttp==3.6.3
SQLAlchemy==1.3.19
python-telegram-bot==12.8
SQLAlchemy==1.3.20
python-telegram-bot==13.0
arrow==0.17.0
cachetools==4.1.1
requests==2.24.0
@ -34,7 +34,7 @@ flask-jwt-extended==3.24.1
flask-cors==3.0.9
# Support for colorized terminal output
colorama==0.4.3
colorama==0.4.4
# Building config files interactively
questionary==1.6.0
prompt-toolkit==3.0.7
questionary==1.7.0
prompt-toolkit==3.0.8

2
tests/commands/test_commands.py
View File

@ -1150,7 +1150,7 @@ def test_start_list_data(testdatadir, capsys):
@pytest.mark.usefixtures("init_persistence")
def test_show_trades(mocker, fee, capsys, caplog):
mocker.patch("freqtrade.persistence.init")
mocker.patch("freqtrade.persistence.init_db")
create_mock_trades(fee)
args = [
"show-trades",

10
tests/conftest.py
View File

@ -13,13 +13,13 @@ import numpy as np
import pytest
from telegram import Chat, Message, Update
from freqtrade import constants, persistence
from freqtrade import constants
from freqtrade.commands import Arguments
from freqtrade.data.converter import ohlcv_to_dataframe
from freqtrade.edge import Edge, PairInfo
from freqtrade.exchange import Exchange
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Trade
from freqtrade.persistence import Trade, init_db
from freqtrade.resolvers import ExchangeResolver
from freqtrade.worker import Worker
from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4,
@ -131,7 +131,7 @@ def patch_freqtradebot(mocker, config) -> None:
:return: None
"""
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
persistence.init(config['db_url'])
init_db(config['db_url'])
patch_exchange(mocker)
mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock())
mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock())
@ -219,7 +219,7 @@ def patch_coingekko(mocker) -> None:
@pytest.fixture(scope='function')
def init_persistence(default_conf):
persistence.init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
@pytest.fixture(scope="function")
@ -297,7 +297,7 @@ def default_conf(testdatadir):
@pytest.fixture
def update():
_update = Update(0)
_update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0))
_update.message = Message(0, datetime.utcnow(), Chat(0, 0))
return _update

2
tests/data/test_btanalysis.py
View File

@ -114,7 +114,7 @@ def test_load_trades_from_db(default_conf, fee, mocker):
create_mock_trades(fee)
# remove init so it does not init again
init_mock = mocker.patch('freqtrade.persistence.init', MagicMock())
init_mock = mocker.patch('freqtrade.data.btanalysis.init_db', MagicMock())
trades = load_trades_from_db(db_url=default_conf['db_url'])
assert init_mock.call_count == 1

2
tests/data/test_dataprovider.py
View File

@ -132,7 +132,7 @@ def test_orderbook(mocker, default_conf, order_book_l2):
res = dp.orderbook('ETH/BTC', 5)
assert order_book_l2.call_count == 1
assert order_book_l2.call_args_list[0][0][0] == 'ETH/BTC'
assert order_book_l2.call_args_list[0][0][1] == 5
assert order_book_l2.call_args_list[0][0][1] >= 5
assert type(res) is dict
assert 'bids' in res

46
tests/exchange/test_exchange.py
View File

@ -11,7 +11,7 @@ from pandas import DataFrame
from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange import Binance, Exchange, Kraken
from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
calculate_backoff)
from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs,
@ -148,11 +148,19 @@ def test_exchange_resolver(default_conf, mocker, caplog):
mocker.patch('freqtrade.exchange.Exchange.validate_pairs')
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
exchange = ExchangeResolver.load_exchange('Bittrex', default_conf)
exchange = ExchangeResolver.load_exchange('huobi', default_conf)
assert isinstance(exchange, Exchange)
assert log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog)
caplog.clear()
exchange = ExchangeResolver.load_exchange('Bittrex', default_conf)
assert isinstance(exchange, Exchange)
assert isinstance(exchange, Bittrex)
assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.",
caplog)
caplog.clear()
exchange = ExchangeResolver.load_exchange('kraken', default_conf)
assert isinstance(exchange, Exchange)
assert isinstance(exchange, Kraken)
@ -1439,6 +1447,27 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog):
assert log_has("Async code raised an exception: TypeError", caplog)
def test_get_next_limit_in_list():
limit_range = [5, 10, 20, 50, 100, 500, 1000]
assert Exchange.get_next_limit_in_list(1, limit_range) == 5
assert Exchange.get_next_limit_in_list(5, limit_range) == 5
assert Exchange.get_next_limit_in_list(6, limit_range) == 10
assert Exchange.get_next_limit_in_list(9, limit_range) == 10
assert Exchange.get_next_limit_in_list(10, limit_range) == 10
assert Exchange.get_next_limit_in_list(11, limit_range) == 20
assert Exchange.get_next_limit_in_list(19, limit_range) == 20
assert Exchange.get_next_limit_in_list(21, limit_range) == 50
assert Exchange.get_next_limit_in_list(51, limit_range) == 100
assert Exchange.get_next_limit_in_list(1000, limit_range) == 1000
# Going over the limit ...
assert Exchange.get_next_limit_in_list(1001, limit_range) == 1000
assert Exchange.get_next_limit_in_list(2000, limit_range) == 1000
assert Exchange.get_next_limit_in_list(21, None) == 21
assert Exchange.get_next_limit_in_list(100, None) == 100
assert Exchange.get_next_limit_in_list(1000, None) == 1000
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name):
default_conf['exchange']['name'] = exchange_name
@ -1451,6 +1480,19 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name)
assert 'asks' in order_book
assert len(order_book['bids']) == 10
assert len(order_book['asks']) == 10
assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC'
for val in [1, 5, 10, 12, 20, 50, 100]:
api_mock.fetch_l2_order_book.reset_mock()
order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val)
assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC'
# Not all exchanges support all limits for orderbook
if not exchange._ft_has['l2_limit_range'] or val in exchange._ft_has['l2_limit_range']:
assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val
else:
next_limit = exchange.get_next_limit_in_list(val, exchange._ft_has['l2_limit_range'])
assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == next_limit
@pytest.mark.parametrize("exchange_name", EXCHANGES)

17
tests/rpc/test_rpc_telegram.py
View File

@ -82,7 +82,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None:
assert log_has(message_str, caplog)
def test_cleanup(default_conf, mocker) -> None:
def test_cleanup(default_conf, mocker, ) -> None:
updater_mock = MagicMock()
updater_mock.stop = MagicMock()
mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock)
@ -92,13 +92,9 @@ def test_cleanup(default_conf, mocker) -> None:
assert telegram._updater.stop.call_count == 1
def test_authorized_only(default_conf, mocker, caplog) -> None:
def test_authorized_only(default_conf, mocker, caplog, update) -> None:
patch_exchange(mocker)
chat = Chat(0, 0)
update = Update(randint(1, 100))
update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat)
default_conf['telegram']['enabled'] = False
bot = FreqtradeBot(default_conf)
patch_get_signal(bot, (True, False))
@ -114,7 +110,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
chat = Chat(0xdeadbeef, 0)
update = Update(randint(1, 100))
update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat)
update.message = Message(randint(1, 100), datetime.utcnow(), chat)
default_conf['telegram']['enabled'] = False
bot = FreqtradeBot(default_conf)
@ -127,12 +123,9 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None:
assert not log_has('Exception occurred within Telegram module', caplog)
def test_authorized_only_exception(default_conf, mocker, caplog) -> None:
def test_authorized_only_exception(default_conf, mocker, caplog, update) -> None:
patch_exchange(mocker)
update = Update(randint(1, 100))
update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0))
default_conf['telegram']['enabled'] = False
bot = FreqtradeBot(default_conf)
@ -146,7 +139,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None:
assert log_has('Exception occurred within Telegram module', caplog)
def test_telegram_status(default_conf, update, mocker, fee, ticker,) -> None:
def test_telegram_status(default_conf, update, mocker) -> None:
update.message.chat.id = "123"
default_conf['telegram']['enabled'] = False
default_conf['telegram']['chat_id'] = "123"

5
tests/test_freqtradebot.py
View File

@ -15,8 +15,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
InvalidOrderException, OperationalException, PricingError,
TemporaryError)
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Trade
from freqtrade.persistence.models import Order
from freqtrade.persistence import Order, Trade
from freqtrade.rpc import RPCMessageType
from freqtrade.state import RunMode, State
from freqtrade.strategy.interface import SellCheckTuple, SellType
@ -66,7 +65,7 @@ def test_process_stopped(mocker, default_conf) -> None:
def test_bot_cleanup(mocker, default_conf, caplog) -> None:
mock_cleanup = mocker.patch('freqtrade.persistence.cleanup')
mock_cleanup = mocker.patch('freqtrade.freqtradebot.cleanup_db')
coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders')
freqtrade = get_patched_freqtradebot(mocker, default_conf)
freqtrade.cleanup()

10
tests/test_main.py
View File

@ -65,7 +65,7 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None:
mocker.patch('freqtrade.worker.Worker._worker', MagicMock(side_effect=Exception))
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
args = ['trade', '-c', 'config.json.example']
@ -83,7 +83,7 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
args = ['trade', '-c', 'config.json.example']
@ -104,7 +104,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
args = ['trade', '-c', 'config.json.example']
@ -155,7 +155,7 @@ def test_main_reload_config(mocker, default_conf, caplog) -> None:
reconfigure_mock = mocker.patch('freqtrade.worker.Worker._reconfigure', MagicMock())
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
args = Arguments(['trade', '-c', 'config.json.example']).get_parsed_arg()
worker = Worker(args=args, config=default_conf)
@ -178,7 +178,7 @@ def test_reconfigure(mocker, default_conf) -> None:
mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
args = Arguments(['trade', '-c', 'config.json.example']).get_parsed_arg()
worker = Worker(args=args, config=default_conf)

20
tests/test_persistence.py
View File

@ -8,13 +8,13 @@ from sqlalchemy import create_engine
from freqtrade import constants
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.persistence import Order, Trade, clean_dry_run_db, init
from freqtrade.persistence import Order, Trade, clean_dry_run_db, init_db
from tests.conftest import create_mock_trades, log_has, log_has_re
def test_init_create_session(default_conf):
# Check if init create a session
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert hasattr(Trade, 'session')
assert 'scoped_session' in type(Trade.session).__name__
@ -24,7 +24,7 @@ def test_init_custom_db_url(default_conf, mocker):
default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'})
create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert create_engine_mock.call_count == 1
assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite'
@ -33,7 +33,7 @@ def test_init_invalid_db_url(default_conf):
# Update path to a value other than default, but still in-memory
default_conf.update({'db_url': 'unknown:///some.url'})
with pytest.raises(OperationalException, match=r'.*no valid database URL*'):
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
def test_init_prod_db(default_conf, mocker):
@ -42,7 +42,7 @@ def test_init_prod_db(default_conf, mocker):
create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert create_engine_mock.call_count == 1
assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite'
@ -53,7 +53,7 @@ def test_init_dryrun_db(default_conf, mocker):
create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert create_engine_mock.call_count == 1
assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.dryrun.sqlite'
@ -482,7 +482,7 @@ def test_migrate_old(mocker, default_conf, fee):
engine.execute(insert_table_old)
engine.execute(insert_table_old2)
# Run init to test migration
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert len(Trade.query.filter(Trade.id == 1).all()) == 1
trade = Trade.query.filter(Trade.id == 1).first()
@ -581,7 +581,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
engine.execute("create table trades_bak1 as select * from trades")
# Run init to test migration
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert len(Trade.query.filter(Trade.id == 1).all()) == 1
trade = Trade.query.filter(Trade.id == 1).first()
@ -661,7 +661,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
engine.execute(insert_table_old)
# Run init to test migration
init(default_conf['db_url'], default_conf['dry_run'])
init_db(default_conf['db_url'], default_conf['dry_run'])
assert len(Trade.query.filter(Trade.id == 1).all()) == 1
trade = Trade.query.filter(Trade.id == 1).first()
@ -904,7 +904,7 @@ def test_to_json(default_conf, fee):
def test_stoploss_reinitialization(default_conf, fee):
init(default_conf['db_url'])
init_db(default_conf['db_url'])
trade = Trade(
pair='ETH/BTC',
stake_amount=0.001,