Merge branch 'develop' into pr/eatrisno/4308

freqtrade/rpc/__init__.py

@@ -1,3 +1,3 @@
 # flake8: noqa: F401
-from .rpc import RPC, RPCException, RPCHandler, RPCMessageType
+from .rpc import RPC, RPCException, RPCHandler
 from .rpc_manager import RPCManager

freqtrade/rpc/api_server/api_schemas.py

@@ -1,5 +1,5 @@
 from datetime import date, datetime
-from typing import Any, Dict, List, Optional, TypeVar, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -57,19 +57,18 @@ class Count(BaseModel):
 class PerformanceEntry(BaseModel):
     pair: str
     profit: float
+    profit_abs: float
     count: int
 
 
 class Profit(BaseModel):
     profit_closed_coin: float
-    profit_closed_percent: float
     profit_closed_percent_mean: float
     profit_closed_ratio_mean: float
     profit_closed_percent_sum: float
     profit_closed_ratio_sum: float
     profit_closed_fiat: float
     profit_all_coin: float
-    profit_all_percent: float
     profit_all_percent_mean: float
     profit_all_ratio_mean: float
     profit_all_percent_sum: float
@@ -113,7 +112,7 @@ class Daily(BaseModel):
 
 
 class ShowConfig(BaseModel):
-    dry_run: str
+    dry_run: bool
     stake_currency: str
     stake_amount: Union[float, str]
     max_open_trades: int
@@ -153,13 +152,11 @@ class TradeSchema(BaseModel):
     fee_close: Optional[float]
     fee_close_cost: Optional[float]
     fee_close_currency: Optional[str]
-    open_date_hum: str
     open_date: str
     open_timestamp: int
     open_rate: float
     open_rate_requested: Optional[float]
     open_trade_value: float
-    close_date_hum: Optional[str]
     close_date: Optional[str]
     close_timestamp: Optional[int]
     close_rate: Optional[float]
@@ -170,6 +167,7 @@ class TradeSchema(BaseModel):
     profit_ratio: Optional[float]
     profit_pct: Optional[float]
     profit_abs: Optional[float]
+    profit_fiat: Optional[float]
     sell_reason: Optional[str]
     sell_order_status: Optional[str]
     stop_loss_abs: Optional[float]
@@ -192,7 +190,6 @@ class OpenTradeSchema(TradeSchema):
     stoploss_current_dist_ratio: Optional[float]
     stoploss_entry_dist: Optional[float]
     stoploss_entry_dist_ratio: Optional[float]
-    base_currency: str
     current_profit: float
     current_profit_abs: float
     current_profit_pct: float
@@ -203,12 +200,15 @@ class OpenTradeSchema(TradeSchema):
 class TradeResponse(BaseModel):
     trades: List[TradeSchema]
     trades_count: int
+    total_trades: int
 
 
-ForceBuyResponse = TypeVar('ForceBuyResponse', TradeSchema, StatusMsg)
+class ForceBuyResponse(BaseModel):
+    __root__: Union[TradeSchema, StatusMsg]
 
 
 class LockModel(BaseModel):
+    id: int
     active: bool
     lock_end_time: str
     lock_end_timestamp: int
@@ -223,6 +223,11 @@ class Locks(BaseModel):
     locks: List[LockModel]
 
 
+class DeleteLockRequest(BaseModel):
+    pair: Optional[str]
+    lockid: Optional[int]
+
+
 class Logs(BaseModel):
     log_count: int
     logs: List[List]
@@ -264,10 +269,11 @@ class DeleteTrade(BaseModel):
 
 class PlotConfig_(BaseModel):
     main_plot: Dict[str, Any]
-    subplots: Optional[Dict[str, Any]]
+    subplots: Dict[str, Any]
 
 
-PlotConfig = TypeVar('PlotConfig', PlotConfig_, Dict)
+class PlotConfig(BaseModel):
+    __root__: Union[PlotConfig_, Dict]
 
 
 class StrategyListResponse(BaseModel):

freqtrade/rpc/api_server/api_v1.py

@@ -11,13 +11,13 @@ from freqtrade.data.history import get_datahandler
 from freqtrade.exceptions import OperationalException
 from freqtrade.rpc import RPC
 from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload,
-                                                  BlacklistResponse, Count, Daily, DeleteTrade,
-                                                  ForceBuyPayload, ForceBuyResponse,
-                                                  ForceSellPayload, Locks, Logs, OpenTradeSchema,
-                                                  PairHistory, PerformanceEntry, Ping, PlotConfig,
-                                                  Profit, ResultMsg, ShowConfig, Stats, StatusMsg,
-                                                  StrategyListResponse, StrategyResponse,
-                                                  TradeResponse, Version, WhitelistResponse)
+                                                  BlacklistResponse, Count, Daily,
+                                                  DeleteLockRequest, DeleteTrade, ForceBuyPayload,
+                                                  ForceBuyResponse, ForceSellPayload, Locks, Logs,
+                                                  OpenTradeSchema, PairHistory, PerformanceEntry,
+                                                  Ping, PlotConfig, Profit, ResultMsg, ShowConfig,
+                                                  Stats, StatusMsg, StrategyListResponse,
+                                                  StrategyResponse, Version, WhitelistResponse)
 from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
 from freqtrade.rpc.rpc import RPCException
 
@@ -82,9 +82,19 @@ def status(rpc: RPC = Depends(get_rpc)):
         return []
 
 
-@router.get('/trades', response_model=TradeResponse, tags=['info', 'trading'])
-def trades(limit: int = 0, rpc: RPC = Depends(get_rpc)):
-    return rpc._rpc_trade_history(limit)
+# Using the responsemodel here will cause a ~100% increase in response time (from 1s to 2s)
+# on big databases. Correct response model: response_model=TradeResponse,
+@router.get('/trades', tags=['info', 'trading'])
+def trades(limit: int = 500, offset: int = 0, rpc: RPC = Depends(get_rpc)):
+    return rpc._rpc_trade_history(limit, offset=offset, order_by_id=True)
+
+
+@router.get('/trade/{tradeid}', response_model=OpenTradeSchema, tags=['info', 'trading'])
+def trade(tradeid: int = 0, rpc: RPC = Depends(get_rpc)):
+    try:
+        return rpc._rpc_trade_status([tradeid])[0]
+    except (RPCException, KeyError):
+        raise HTTPException(status_code=404, detail='Trade not found.')
 
 
 @router.delete('/trades/{tradeid}', response_model=DeleteTrade, tags=['info', 'trading'])
@@ -111,9 +121,9 @@ def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)):
     trade = rpc._rpc_forcebuy(payload.pair, payload.price)
 
     if trade:
-        return trade.to_json()
+        return ForceBuyResponse.parse_obj(trade.to_json())
     else:
-        return {"status": f"Error buying pair {payload.pair}."}
+        return ForceBuyResponse.parse_obj({"status": f"Error buying pair {payload.pair}."})
 
 
 @router.post('/forcesell', response_model=ResultMsg, tags=['trading'])
@@ -136,11 +146,21 @@ def whitelist(rpc: RPC = Depends(get_rpc)):
     return rpc._rpc_whitelist()
 
 
-@router.get('/locks', response_model=Locks, tags=['info'])
+@router.get('/locks', response_model=Locks, tags=['info', 'locks'])
 def locks(rpc: RPC = Depends(get_rpc)):
     return rpc._rpc_locks()
 
 
+@router.delete('/locks/{lockid}', response_model=Locks, tags=['info', 'locks'])
+def delete_lock(lockid: int, rpc: RPC = Depends(get_rpc)):
+    return rpc._rpc_delete_lock(lockid=lockid)
+
+
+@router.post('/locks/delete', response_model=Locks, tags=['info', 'locks'])
+def delete_lock_pair(payload: DeleteLockRequest, rpc: RPC = Depends(get_rpc)):
+    return rpc._rpc_delete_lock(lockid=payload.lockid, pair=payload.pair)
+
+
 @router.get('/logs', response_model=Logs, tags=['info'])
 def logs(limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)):
     return rpc._rpc_get_logs(limit)
@@ -183,7 +203,7 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str,
 
 @router.get('/plot_config', response_model=PlotConfig, tags=['candle data'])
 def plot_config(rpc: RPC = Depends(get_rpc)):
-    return rpc._rpc_plot_config()
+    return PlotConfig.parse_obj(rpc._rpc_plot_config())
 
 
 @router.get('/strategies', response_model=StrategyListResponse, tags=['strategy'])

freqtrade/rpc/api_server/uvicorn_threaded.py

@@ -8,12 +8,33 @@ import uvicorn
 class UvicornServer(uvicorn.Server):
     """
     Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742
+
+    Removed install_signal_handlers() override based on changes from this commit:
+        https://github.com/encode/uvicorn/commit/ce2ef45a9109df8eae038c0ec323eb63d644cbc6
+
+    Cannot rely on asyncio.get_event_loop() to create new event loop because of this check:
+        https://github.com/python/cpython/blob/4d7f11e05731f67fd2c07ec2972c6cb9861d52be/Lib/asyncio/events.py#L638
+
+    Fix by overriding run() and forcing creation of new event loop if uvloop is available
     """
-    def install_signal_handlers(self):
+
+    def run(self, sockets=None):
+        import asyncio
+
         """
-        In the parent implementation, this starts the thread, therefore we must patch it away here.
+        Parent implementation calls self.config.setup_event_loop(),
+            but we need to create uvloop event loop manually
         """
-        pass
+        try:
+            import uvloop  # noqa
+        except ImportError:  # pragma: no cover
+            from uvicorn.loops.asyncio import asyncio_setup
+            asyncio_setup()
+        else:
+            asyncio.set_event_loop(uvloop.new_event_loop())
+
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(self.serve(sockets=sockets))
 
     @contextlib.contextmanager
     def run_in_thread(self):

freqtrade/rpc/api_server/web_ui.py

@@ -10,7 +10,12 @@ router_ui = APIRouter()
 
 @router_ui.get('/favicon.ico', include_in_schema=False)
 async def favicon():
-    return FileResponse(Path(__file__).parent / 'ui/favicon.ico')
+    return FileResponse(str(Path(__file__).parent / 'ui/favicon.ico'))
+
+
+@router_ui.get('/fallback_file.html', include_in_schema=False)
+async def fallback():
+    return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html'))
 
 
 @router_ui.get('/{rest_of_path:path}', include_in_schema=False)

freqtrade/rpc/api_server/webserver.py

@@ -2,6 +2,7 @@ import logging
 from ipaddress import IPv4Address
 from typing import Any, Dict
 
+import rapidjson
 import uvicorn
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -14,6 +15,17 @@ from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler
 logger = logging.getLogger(__name__)
 
 
+class FTJSONResponse(JSONResponse):
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        """
+        Use rapidjson for responses
+        Handles NaN and Inf / -Inf in a javascript way by default.
+        """
+        return rapidjson.dumps(content).encode("utf-8")
+
+
 class ApiServer(RPCHandler):
 
     _rpc: RPC
@@ -32,6 +44,7 @@ class ApiServer(RPCHandler):
         self.app = FastAPI(title="Freqtrade API",
                            docs_url='/docs' if api_config.get('enable_openapi', False) else None,
                            redoc_url=None,
+                           default_response_class=FTJSONResponse,
                            )
         self.configure_app(self.app, self._config)
 

freqtrade/rpc/fiat_convert.py

@@ -3,11 +3,13 @@ Module that define classes to convert Crypto-currency to FIAT
 e.g BTC to USD
 """
 
+import datetime
 import logging
-import time
-from typing import Dict, List
+from typing import Dict
 
+from cachetools.ttl import TTLCache
 from pycoingecko import CoinGeckoAPI
+from requests.exceptions import RequestException
 
 from freqtrade.constants import SUPPORTED_FIAT
 
@@ -15,51 +17,6 @@ from freqtrade.constants import SUPPORTED_FIAT
 logger = logging.getLogger(__name__)
 
 
-class CryptoFiat:
-    """
-    Object to describe what is the price of Crypto-currency in a FIAT
-    """
-    # Constants
-    CACHE_DURATION = 6 * 60 * 60  # 6 hours
-
-    def __init__(self, crypto_symbol: str, fiat_symbol: str, price: float) -> None:
-        """
-        Create an object that will contains the price for a crypto-currency in fiat
-        :param crypto_symbol: Crypto-currency you want to convert (e.g BTC)
-        :param fiat_symbol: FIAT currency you want to convert to (e.g USD)
-        :param price: Price in FIAT
-        """
-
-        # Public attributes
-        self.crypto_symbol = None
-        self.fiat_symbol = None
-        self.price = 0.0
-
-        # Private attributes
-        self._expiration = 0.0
-
-        self.crypto_symbol = crypto_symbol.lower()
-        self.fiat_symbol = fiat_symbol.lower()
-        self.set_price(price=price)
-
-    def set_price(self, price: float) -> None:
-        """
-        Set the price of the Crypto-currency in FIAT and set the expiration time
-        :param price: Price of the current Crypto currency in the fiat
-        :return: None
-        """
-        self.price = price
-        self._expiration = time.time() + self.CACHE_DURATION
-
-    def is_expired(self) -> bool:
-        """
-        Return if the current price is still valid or needs to be refreshed
-        :return: bool, true the price is expired and needs to be refreshed, false the price is
-         still valid
-        """
-        return self._expiration - time.time() <= 0
-
-
 class CryptoToFiatConverter:
     """
     Main class to initiate Crypto to FIAT.
@@ -70,6 +27,7 @@ class CryptoToFiatConverter:
     _coingekko: CoinGeckoAPI = None
 
     _cryptomap: Dict = {}
+    _backoff: float = 0.0
 
     def __new__(cls):
         """
@@ -84,14 +42,29 @@ class CryptoToFiatConverter:
         return CryptoToFiatConverter.__instance
 
     def __init__(self) -> None:
-        self._pairs: List[CryptoFiat] = []
+        # Timeout: 6h
+        self._pair_price: TTLCache = TTLCache(maxsize=500, ttl=6 * 60 * 60)
+
         self._load_cryptomap()
 
     def _load_cryptomap(self) -> None:
         try:
             coinlistings = self._coingekko.get_coins_list()
-            # Create mapping table from synbol to coingekko_id
+            # Create mapping table from symbol to coingekko_id
             self._cryptomap = {x['symbol']: x['id'] for x in coinlistings}
+        except RequestException as request_exception:
+            if "429" in str(request_exception):
+                logger.warning(
+                    "Too many requests for Coingecko API, backing off and trying again later.")
+                # Set backoff timestamp to 60 seconds in the future
+                self._backoff = datetime.datetime.now().timestamp() + 60
+                return
+            # If the request is not a 429 error we want to raise the normal error
+            logger.error(
+                "Could not load FIAT Cryptocurrency map for the following problem: {}".format(
+                  request_exception
+                )
+            )
         except (Exception) as exception:
             logger.error(
                 f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
@@ -118,49 +91,31 @@ class CryptoToFiatConverter:
         """
         crypto_symbol = crypto_symbol.lower()
         fiat_symbol = fiat_symbol.lower()
+        inverse = False
 
+        if crypto_symbol == 'usd':
+            # usd corresponds to "uniswap-state-dollar" for coingecko.
+            # We'll therefore need to "swap" the currencies
+            logger.info(f"reversing Rates {crypto_symbol}, {fiat_symbol}")
+            crypto_symbol = fiat_symbol
+            fiat_symbol = 'usd'
+            inverse = True
+
+        symbol = f"{crypto_symbol}/{fiat_symbol}"
         # Check if the fiat convertion you want is supported
         if not self._is_supported_fiat(fiat=fiat_symbol):
             raise ValueError(f'The fiat {fiat_symbol} is not supported.')
 
-        # Get the pair that interest us and return the price in fiat
-        for pair in self._pairs:
-            if pair.crypto_symbol == crypto_symbol and pair.fiat_symbol == fiat_symbol:
-                # If the price is expired we refresh it, avoid to call the API all the time
-                if pair.is_expired():
-                    pair.set_price(
-                        price=self._find_price(
-                            crypto_symbol=pair.crypto_symbol,
-                            fiat_symbol=pair.fiat_symbol
-                        )
-                    )
+        price = self._pair_price.get(symbol, None)
 
-                # return the last price we have for this pair
-                return pair.price
-
-        # The pair does not exist, so we create it and return the price
-        return self._add_pair(
-            crypto_symbol=crypto_symbol,
-            fiat_symbol=fiat_symbol,
-            price=self._find_price(
+        if not price:
+            price = self._find_price(
                 crypto_symbol=crypto_symbol,
                 fiat_symbol=fiat_symbol
             )
-        )
-
-    def _add_pair(self, crypto_symbol: str, fiat_symbol: str, price: float) -> float:
-        """
-        :param crypto_symbol: Crypto-currency you want to convert (e.g BTC)
-        :param fiat_symbol: FIAT currency you want to convert to (e.g USD)
-        :return: price in FIAT
-        """
-        self._pairs.append(
-            CryptoFiat(
-                crypto_symbol=crypto_symbol,
-                fiat_symbol=fiat_symbol,
-                price=price
-            )
-        )
+            if inverse and price != 0.0:
+                price = 1 / price
+            self._pair_price[symbol] = price
 
         return price
 
@@ -188,6 +143,15 @@ class CryptoToFiatConverter:
         if crypto_symbol == fiat_symbol:
             return 1.0
 
+        if self._cryptomap == {}:
+            if self._backoff <= datetime.datetime.now().timestamp():
+                self._load_cryptomap()
+                # return 0.0 if we still dont have data to check, no reason to proceed
+                if self._cryptomap == {}:
+                    return 0.0
+            else:
+                return 0.0
+
         if crypto_symbol not in self._cryptomap:
             # return 0 for unsupported stake currencies (fiat-convert should not break the bot)
             logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)

freqtrade/rpc/rpc.py

@@ -3,8 +3,7 @@ This module contains class to define a RPC communications
 """
 import logging
 from abc import abstractmethod
-from datetime import date, datetime, timedelta
-from enum import Enum
+from datetime import date, datetime, timedelta, timezone
 from math import isnan
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -15,36 +14,21 @@ from pandas import DataFrame
 from freqtrade.configuration.timerange import TimeRange
 from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT
 from freqtrade.data.history import load_data
+from freqtrade.enums import SellType, State
 from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
 from freqtrade.misc import shorten_date
 from freqtrade.persistence import PairLocks, Trade
+from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
-from freqtrade.state import State
-from freqtrade.strategy.interface import SellType
+from freqtrade.strategy.interface import SellCheckTuple
 
 
 logger = logging.getLogger(__name__)
 
 
-class RPCMessageType(Enum):
-    STATUS_NOTIFICATION = 'status'
-    WARNING_NOTIFICATION = 'warning'
-    STARTUP_NOTIFICATION = 'startup'
-    BUY_NOTIFICATION = 'buy'
-    BUY_CANCEL_NOTIFICATION = 'buy_cancel'
-    SELL_NOTIFICATION = 'sell'
-    SELL_CANCEL_NOTIFICATION = 'sell_cancel'
-
-    def __repr__(self):
-        return self.value
-
-    def __str__(self):
-        return self.value
-
-
 class RPCException(Exception):
     """
     Should be raised with a rpc-formatted message in an _rpc_* method
@@ -166,12 +150,24 @@ class RPC:
                 if trade.open_order_id:
                     order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
                 # calculate profit and send message to user
-                try:
-                    current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                except (ExchangeError, PricingError):
-                    current_rate = NAN
+                if trade.is_open:
+                    try:
+                        current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
+                    except (ExchangeError, PricingError):
+                        current_rate = NAN
+                else:
+                    current_rate = trade.close_rate
                 current_profit = trade.calc_profit_ratio(current_rate)
                 current_profit_abs = trade.calc_profit(current_rate)
+                current_profit_fiat: Optional[float] = None
+                # Calculate fiat profit
+                if self._fiat_converter:
+                    current_profit_fiat = self._fiat_converter.convert_amount(
+                        current_profit_abs,
+                        self._freqtrade.config['stake_currency'],
+                        self._freqtrade.config['fiat_display_currency']
+                    )
+
                 # Calculate guaranteed profit (in case of trailing stop)
                 stoploss_entry_dist = trade.calc_profit(trade.stop_loss)
                 stoploss_entry_dist_ratio = trade.calc_profit_ratio(trade.stop_loss)
@@ -190,6 +186,7 @@ class RPC:
                     profit_ratio=current_profit,
                     profit_pct=round(current_profit * 100, 2),
                     profit_abs=current_profit_abs,
+                    profit_fiat=current_profit_fiat,
 
                     stoploss_current_dist=stoploss_current_dist,
                     stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8),
@@ -204,16 +201,17 @@ class RPC:
             return results
 
     def _rpc_status_table(self, stake_currency: str,
-                          fiat_display_currency: str) -> Tuple[List, List]:
+                          fiat_display_currency: str) -> Tuple[List, List, float]:
         trades = Trade.get_open_trades()
         if not trades:
             raise RPCException('no active trade')
         else:
             trades_list = []
+            fiat_profit_sum = NAN
             for trade in trades:
                 # calculate profit and send message to user
                 try:
-                    current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
+                    current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
                 except (PricingError, ExchangeError):
                     current_rate = NAN
                 trade_percent = (100 * trade.calc_profit_ratio(current_rate))
@@ -227,6 +225,8 @@ class RPC:
                     )
                     if fiat_profit and not isnan(fiat_profit):
                         profit_str += f" ({fiat_profit:.2f})"
+                        fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \
+                            else fiat_profit_sum + fiat_profit
                 trades_list.append([
                     trade.id,
                     trade.pair + ('*' if (trade.open_order_id is not None
@@ -240,7 +240,7 @@ class RPC:
                 profitcol += " (" + fiat_display_currency + ")"
 
             columns = ['ID', 'Pair', 'Since', profitcol]
-            return trades_list, columns
+            return trades_list, columns, fiat_profit_sum
 
     def _rpc_daily_profit(
             self, timescale: int,
@@ -284,19 +284,22 @@ class RPC:
             'data': data
         }
 
-    def _rpc_trade_history(self, limit: int) -> Dict:
+    def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict:
         """ Returns the X last trades """
-        if limit > 0:
+        order_by = Trade.id if order_by_id else Trade.close_date.desc()
+        if limit:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
-                Trade.id.desc()).limit(limit)
+                order_by).limit(limit).offset(offset)
         else:
-            trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(Trade.id.desc()).all()
+            trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
+                Trade.close_date.desc()).all()
 
         output = [trade.to_json() for trade in trades]
 
         return {
             "trades": output,
-            "trades_count": len(output)
+            "trades_count": len(output),
+            "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(),
         }
 
     def _rpc_stats(self) -> Dict[str, Any]:
@@ -333,9 +336,10 @@ class RPC:
         return {'sell_reasons': sell_reasons, 'durations': durations}
 
     def _rpc_trade_statistics(
-            self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
+            self, stake_currency: str, fiat_display_currency: str,
+            start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]:
         """ Returns cumulative profit statistics """
-        trades = Trade.get_trades().order_by(Trade.id).all()
+        trades = Trade.get_trades([Trade.open_date >= start_date]).order_by(Trade.id).all()
 
         profit_all_coin = []
         profit_all_ratio = []
@@ -364,7 +368,7 @@ class RPC:
             else:
                 # Get current rate
                 try:
-                    current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
+                    current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
                 except (PricingError, ExchangeError):
                     current_rate = NAN
                 profit_ratio = trade.calc_profit_ratio(rate=current_rate)
@@ -401,14 +405,12 @@ class RPC:
         num = float(len(durations) or 1)
         return {
             'profit_closed_coin': profit_closed_coin_sum,
-            'profit_closed_percent': round(profit_closed_ratio_mean * 100, 2),  # DEPRECATED
             'profit_closed_percent_mean': round(profit_closed_ratio_mean * 100, 2),
             'profit_closed_ratio_mean': profit_closed_ratio_mean,
             'profit_closed_percent_sum': round(profit_closed_ratio_sum * 100, 2),
             'profit_closed_ratio_sum': profit_closed_ratio_sum,
             'profit_closed_fiat': profit_closed_fiat,
             'profit_all_coin': profit_all_coin_sum,
-            'profit_all_percent': round(profit_all_ratio_mean * 100, 2),  # DEPRECATED
             'profit_all_percent_mean': round(profit_all_ratio_mean * 100, 2),
             'profit_all_ratio_mean': profit_all_ratio_mean,
             'profit_all_percent_sum': round(profit_all_ratio_sum * 100, 2),
@@ -432,7 +434,7 @@ class RPC:
         output = []
         total = 0.0
         try:
-            tickers = self._freqtrade.exchange.get_tickers()
+            tickers = self._freqtrade.exchange.get_tickers(cached=True)
         except (ExchangeError):
             raise RPCException('Error getting current tickers.')
 
@@ -536,8 +538,9 @@ class RPC:
 
             if not fully_canceled:
                 # Get current rate and execute sell
-                current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL)
+                current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
+                sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
+                self._freqtrade.execute_sell(trade, current_rate, sell_reason)
         # ---- EOF def _exec_forcesell ----
 
         if self._freqtrade.state != State.RUNNING:
@@ -548,7 +551,7 @@ class RPC:
                 # Execute sell for all open orders
                 for trade in Trade.get_open_trades():
                     _exec_forcesell(trade)
-                Trade.session.flush()
+                Trade.commit()
                 self._freqtrade.wallets.update()
                 return {'result': 'Created sell orders for all open trades.'}
 
@@ -561,7 +564,7 @@ class RPC:
                 raise RPCException('invalid argument')
 
             _exec_forcesell(trade)
-            Trade.session.flush()
+            Trade.commit()
             self._freqtrade.wallets.update()
             return {'result': f'Created sell order for trade {trade_id}.'}
 
@@ -590,11 +593,11 @@ class RPC:
             raise RPCException(f'position for {pair} already open - id: {trade.id}')
 
         # gen stake amount
-        stakeamount = self._freqtrade.wallets.get_trade_stake_amount(
-            pair, self._freqtrade.get_free_open_trades())
+        stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
 
         # execute buy
-        if self._freqtrade.execute_buy(pair, stakeamount, price):
+        if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True):
+            Trade.commit()
             trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
             return trade
         else:
@@ -663,7 +666,7 @@ class RPC:
         }
 
     def _rpc_locks(self) -> Dict[str, Any]:
-        """ Returns the  current locks"""
+        """ Returns the  current locks """
 
         locks = PairLocks.get_pair_locks(None)
         return {
@@ -671,6 +674,24 @@ class RPC:
             'locks': [lock.to_json() for lock in locks]
         }
 
+    def _rpc_delete_lock(self, lockid: Optional[int] = None,
+                         pair: Optional[str] = None) -> Dict[str, Any]:
+        """ Delete specific lock(s) """
+        locks = []
+
+        if pair:
+            locks = PairLocks.get_pair_locks(pair)
+        if lockid:
+            locks = PairLock.query.filter(PairLock.id == lockid).all()
+
+        for lock in locks:
+            lock.active = False
+            lock.lock_end_time = datetime.now(timezone.utc)
+
+        PairLock.query.session.commit()
+
+        return self._rpc_locks()
+
     def _rpc_whitelist(self) -> Dict:
         """ Returns the currently active whitelist"""
         res = {'method': self._freqtrade.pairlists.name_list,
@@ -809,5 +830,7 @@ class RPC:
                                               df_analyzed, arrow.Arrow.utcnow().datetime)
 
     def _rpc_plot_config(self) -> Dict[str, Any]:
-
+        if (self._freqtrade.strategy.plot_config and
+                'subplots' not in self._freqtrade.strategy.plot_config):
+            self._freqtrade.strategy.plot_config['subplots'] = {}
         return self._freqtrade.strategy.plot_config

freqtrade/rpc/rpc_manager.py

@@ -4,7 +4,8 @@ This module contains class to manage RPC communications (Telegram, Slack, ...)
 import logging
 from typing import Any, Dict, List
 
-from freqtrade.rpc import RPC, RPCHandler, RPCMessageType
+from freqtrade.enums import RPCMessageType
+from freqtrade.rpc import RPC, RPCHandler
 
 
 logger = logging.getLogger(__name__)
@@ -67,7 +68,7 @@ class RPCManager:
     def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None:
         if config['dry_run']:
             self.send_msg({
-                'type': RPCMessageType.WARNING_NOTIFICATION,
+                'type': RPCMessageType.WARNING,
                 'status': 'Dry run is enabled. All trades are simulated.'
             })
         stake_currency = config['stake_currency']
@@ -79,7 +80,7 @@ class RPCManager:
         exchange_name = config['exchange']['name']
         strategy_name = config.get('strategy', '')
         self.send_msg({
-            'type': RPCMessageType.STARTUP_NOTIFICATION,
+            'type': RPCMessageType.STARTUP,
             'status': f'*Exchange:* `{exchange_name}`\n'
                       f'*Stake per trade:* `{stake_amount} {stake_currency}`\n'
                       f'*Minimum ROI:* `{minimal_roi}`\n'
@@ -88,13 +89,13 @@ class RPCManager:
                       f'*Strategy:* `{strategy_name}`'
         })
         self.send_msg({
-            'type': RPCMessageType.STARTUP_NOTIFICATION,
+            'type': RPCMessageType.STARTUP,
             'status': f'Searching for {stake_currency} pairs to buy and sell '
                       f'based on {pairlist.short_desc()}'
         })
         if len(protections.name_list) > 0:
             prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()])
             self.send_msg({
-                'type': RPCMessageType.STARTUP_NOTIFICATION,
+                'type': RPCMessageType.STARTUP,
                 'status': f'Using Protections: \n{prots}'
             })

freqtrade/rpc/telegram.py

@@ -5,21 +5,27 @@ This module manage Telegram communication
 """
 import json
 import logging
-from datetime import timedelta, datetime
+import re
+from datetime import date, datetime, timedelta
+from html import escape
 from itertools import chain
-from typing import Any, Callable, Dict, List, Union
+from math import isnan
+from typing import Any, Callable, Dict, List, Union, cast
 
 import arrow
 from tabulate import tabulate
-from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update, InlineKeyboardButton, InlineKeyboardMarkup
-from telegram.error import NetworkError, TelegramError
-from telegram.ext import CallbackContext, CommandHandler, Updater, CallbackQueryHandler
+from telegram import (InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ParseMode,
+                      ReplyKeyboardMarkup, Update)
+from telegram.error import BadRequest, NetworkError, TelegramError
+from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater
 from telegram.utils.helpers import escape_markdown
 
 from freqtrade.__init__ import __version__
+from freqtrade.constants import DUST_PER_COIN
+from freqtrade.enums import RPCMessageType
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_coin_value
-from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType
+from freqtrade.misc import chunks, round_coin_value
+from freqtrade.rpc import RPC, RPCException, RPCHandler
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +51,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]:
             cchat_id = int(update.callback_query.message.chat.id)
         else:
             cchat_id = int(update.message.chat_id)
-            
+
         chat_id = int(self._config['telegram']['chat_id'])
         if cchat_id != chat_id:
             logger.info(
@@ -89,7 +95,7 @@ class Telegram(RPCHandler):
         Validates the keyboard configuration from telegram config
         section.
         """
-        self._keyboard: List[List[Union[str, KeyboardButton]]] = [
+        self._keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = [
             ['/daily', '/profit', '/balance'],
             ['/status', '/status table', '/performance'],
             ['/count', '/start', '/stop', '/help']
@@ -99,23 +105,27 @@ class Telegram(RPCHandler):
         # TODO: DRY! - its not good to list all valid cmds here. But otherwise
         #       this needs refacoring of the whole telegram module (same
         #       problem in _help()).
-        valid_keys: List[str] = ['/start', '/stop', '/status', '/status table',
-                                 '/trades', '/profit', '/performance', '/daily',
-                                 '/stats', '/count', '/locks', '/balance',
-                                 '/stopbuy', '/reload_config', '/show_config',
-                                 '/logs', '/whitelist', '/blacklist', '/edge',
-                                 '/help', '/version']
+        valid_keys: List[str] = [r'/start$', r'/stop$', r'/status$', r'/status table$',
+                                 r'/trades$', r'/performance$', r'/daily$', r'/daily \d+$',
+                                 r'/profit$', r'/profit \d+',
+                                 r'/stats$', r'/count$', r'/locks$', r'/balance$',
+                                 r'/stopbuy$', r'/reload_config$', r'/show_config$',
+                                 r'/logs$', r'/whitelist$', r'/blacklist$', r'/edge$',
+                                 r'/forcebuy$', r'/help$', r'/version$']
+        # Create keys for generation
+        valid_keys_print = [k.replace('$', '') for k in valid_keys]
 
         # custom keyboard specified in config.json
         cust_keyboard = self._config['telegram'].get('keyboard', [])
         if cust_keyboard:
+            combined = "(" + ")|(".join(valid_keys) + ")"
             # check for valid shortcuts
             invalid_keys = [b for b in chain.from_iterable(cust_keyboard)
-                            if b not in valid_keys]
+                            if not re.match(combined, b)]
             if len(invalid_keys):
                 err_msg = ('config.telegram.keyboard: Invalid commands for '
                            f'custom Telegram keyboard: {invalid_keys}'
-                           f'\nvalid commands are: {valid_keys}')
+                           f'\nvalid commands are: {valid_keys_print}')
                 raise OperationalException(err_msg)
             else:
                 self._keyboard = cust_keyboard
@@ -147,6 +157,7 @@ class Telegram(RPCHandler):
             CommandHandler('daily', self._daily),
             CommandHandler('count', self._count),
             CommandHandler('locks', self._locks),
+            CommandHandler(['unlock', 'delete_locks'], self._delete_locks),
             CommandHandler(['reload_config', 'reload_conf'], self._reload_config),
             CommandHandler(['show_config', 'show_conf'], self._show_config),
             CommandHandler('stopbuy', self._stopbuy),
@@ -163,7 +174,8 @@ class Telegram(RPCHandler):
             CallbackQueryHandler(self._profit, pattern='update_profit'),
             CallbackQueryHandler(self._balance, pattern='update_balance'),
             CallbackQueryHandler(self._performance, pattern='update_performance'),
-            CallbackQueryHandler(self._count, pattern='update_count')
+            CallbackQueryHandler(self._count, pattern='update_count'),
+            CallbackQueryHandler(self._forcebuy_inline),
         ]
         for handle in handles:
             self._updater.dispatcher.add_handler(handle)
@@ -172,10 +184,10 @@ class Telegram(RPCHandler):
             self._updater.dispatcher.add_handler(handle)
 
         self._updater.start_polling(
-            clean=True,
             bootstrap_retries=-1,
             timeout=30,
             read_latency=60,
+            drop_pending_updates=True,
         )
         logger.info(
             'rpc.telegram is listening for following commands: %s',
@@ -189,79 +201,111 @@ class Telegram(RPCHandler):
         """
         self._updater.stop()
 
+    def _format_buy_msg(self, msg: Dict[str, Any]) -> str:
+        if self._rpc._fiat_converter:
+            msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
+                msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
+        else:
+            msg['stake_amount_fiat'] = 0
+
+        message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
+                   f" (#{msg['trade_id']})\n"
+                   f"*Amount:* `{msg['amount']:.8f}`\n"
+                   f"*Open Rate:* `{msg['limit']:.8f}`\n"
+                   f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
+                   f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
+
+        if msg.get('fiat_currency', None):
+            message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
+        message += ")`"
+        return message
+
+    def _format_sell_msg(self, msg: Dict[str, Any]) -> str:
+        msg['amount'] = round(msg['amount'], 8)
+        msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
+        msg['duration'] = msg['close_date'].replace(
+            microsecond=0) - msg['open_date'].replace(microsecond=0)
+        msg['duration_min'] = msg['duration'].total_seconds() / 60
+
+        msg['emoji'] = self._get_sell_emoji(msg)
+
+        # Check if all sell properties are available.
+        # This might not be the case if the message origin is triggered by /forcesell
+        if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency'])
+                and self._rpc._fiat_converter):
+            msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount(
+                msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
+            msg['profit_extra'] = (' ({gain}: {profit_amount:.8f} {stake_currency}'
+                                   ' / {profit_fiat:.3f} {fiat_currency})').format(**msg)
+        else:
+            msg['profit_extra'] = ''
+
+        message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n"
+                   "*Profit:* `{profit_percent:.2f}%{profit_extra}`\n"
+                   "*Sell Reason:* `{sell_reason}`\n"
+                   "*Duration:* `{duration} ({duration_min:.1f} min)`\n"
+                   "*Amount:* `{amount:.8f}`\n"
+                   "*Open Rate:* `{open_rate:.8f}`\n"
+                   "*Current Rate:* `{current_rate:.8f}`\n"
+                   "*Close Rate:* `{limit:.8f}`").format(**msg)
+
+        return message
+
     def send_msg(self, msg: Dict[str, Any]) -> None:
         """ Send a message to telegram channel """
 
-        noti = self._config['telegram'].get('notification_settings', {}
-                                            ).get(str(msg['type']), 'on')
+        default_noti = 'on'
+
+        msg_type = msg['type']
+        noti = ''
+        if msg_type == RPCMessageType.SELL:
+            sell_noti = self._config['telegram'] \
+              .get('notification_settings', {}).get(str(msg_type), {})
+            # For backward compatibility sell still can be string
+            if isinstance(sell_noti, str):
+                noti = sell_noti
+            else:
+                noti = sell_noti.get(str(msg['sell_reason']), default_noti)
+        else:
+            noti = self._config['telegram'] \
+              .get('notification_settings', {}).get(str(msg_type), default_noti)
+
         if noti == 'off':
-            logger.info(f"Notification '{msg['type']}' not sent.")
+            logger.info(f"Notification '{msg_type}' not sent.")
             # Notification disabled
             return
 
-        if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
-            if self._rpc._fiat_converter:
-                msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
-                    msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
-            else:
-                msg['stake_amount_fiat'] = 0
+        if msg_type == RPCMessageType.BUY:
+            message = self._format_buy_msg(msg)
 
-            message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}\n"
-                       f"*Amount:* `{msg['amount']:.8f}`\n"
-                       f"*Open Rate:* `{msg['limit']:.8f}`\n"
-                       f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
-                       f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
-
-            if msg.get('fiat_currency', None):
-                message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
-            message += ")`"
-
-        elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
+        elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
+            msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell'
             message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling open buy Order for {pair}. Reason: {reason}.".format(**msg))
+                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
+                       "Reason: {reason}.".format(**msg))
 
-        elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
-            msg['amount'] = round(msg['amount'], 8)
-            msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
-            msg['duration'] = msg['close_date'].replace(
-                microsecond=0) - msg['open_date'].replace(microsecond=0)
-            msg['duration_min'] = msg['duration'].total_seconds() / 60
+        elif msg_type == RPCMessageType.BUY_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Buy order for {pair} (#{trade_id}) filled "
+                       "for {open_rate}.".format(**msg))
+        elif msg_type == RPCMessageType.SELL_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Sell order for {pair} (#{trade_id}) filled "
+                       "for {close_rate}.".format(**msg))
+        elif msg_type == RPCMessageType.SELL:
+            message = self._format_sell_msg(msg)
 
-            msg['emoji'] = self._get_sell_emoji(msg)
-
-            message = ("{emoji} *{exchange}:* Selling {pair}\n"
-                       "*Amount:* `{amount:.8f}`\n"
-                       "*Open Rate:* `{open_rate:.8f}`\n"
-                       "*Current Rate:* `{current_rate:.8f}`\n"
-                       "*Close Rate:* `{limit:.8f}`\n"
-                       "*Sell Reason:* `{sell_reason}`\n"
-                       "*Duration:* `{duration} ({duration_min:.1f} min)`\n"
-                       "*Profit:* `{profit_percent:.2f}%`").format(**msg)
-
-            # Check if all sell properties are available.
-            # This might not be the case if the message origin is triggered by /forcesell
-            if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency'])
-                    and self._rpc._fiat_converter):
-                msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount(
-                    msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
-                message += (' `({gain}: {profit_amount:.8f} {stake_currency}'
-                            ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg)
-
-        elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
-            message = ("\N{WARNING SIGN} *{exchange}:* Cancelling Open Sell Order "
-                       "for {pair}. Reason: {reason}").format(**msg)
-
-        elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION:
+        elif msg_type == RPCMessageType.STATUS:
             message = '*Status:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION:
+        elif msg_type == RPCMessageType.WARNING:
             message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION:
+        elif msg_type == RPCMessageType.STARTUP:
             message = '{status}'.format(**msg)
 
         else:
-            raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
+            raise NotImplementedError('Unknown message type: {}'.format(msg_type))
 
         self._send_msg(message, disable_notification=(noti == 'silent'))
 
@@ -305,6 +349,7 @@ class Telegram(RPCHandler):
 
             messages = []
             for r in results:
+                r['open_date_hum'] = arrow.get(r['open_date']).humanize()
                 lines = [
                     "*Trade ID:* `{trade_id}` `(since {open_date_hum})`",
                     "*Current Pair:* {pair}",
@@ -351,15 +396,41 @@ class Telegram(RPCHandler):
         :return: None
         """
         try:
-            statlist, head = self._rpc._rpc_status_table(
-                self._config['stake_currency'], self._config.get('fiat_display_currency', ''))
+            fiat_currency = self._config.get('fiat_display_currency', '')
+            statlist, head, fiat_profit_sum = self._rpc._rpc_status_table(
+                self._config['stake_currency'], fiat_currency)
 
-            message = tabulate(statlist, headers=head, tablefmt='simple')
-            if(update.callback_query):
-                query = update.callback_query
-                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=f"<pre>{message}</pre>", parse_mode=ParseMode.HTML, callback_path="update_status_table", reload_able=True)
-            else:
-                self._send_msg(f"<pre>{message}</pre>", reload_able=True, callback_path="update_status_table", parse_mode=ParseMode.HTML)
+            show_total = not isnan(fiat_profit_sum) and len(statlist) > 1
+            max_trades_per_msg = 50
+            """
+            Calculate the number of messages of 50 trades per message
+            0.99 is used to make sure that there are no extra (empty) messages
+            As an example with 50 trades, there will be int(50/50 + 0.99) = 1 message
+            """
+            messages_count = max(int(len(statlist) / max_trades_per_msg + 0.99), 1)
+            for i in range(0, messages_count):
+                trades = statlist[i * max_trades_per_msg:(i + 1) * max_trades_per_msg]
+                if show_total and i == messages_count - 1:
+                    # append total line
+                    trades.append(["Total", "", "", f"{fiat_profit_sum:.2f} {fiat_currency}"])
+
+                message = tabulate(trades,
+                                   headers=head,
+                                   tablefmt='simple')
+                if show_total and i == messages_count - 1:
+                    # insert separators line between Total
+                    lines = message.split("\n")
+                    message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]])
+                if(messages_count == 1 and update.callback_query):
+                    query = update.callback_query
+                    self._update_msg(chat_id=query.message.chat_id,
+                                     message_id=query.message.message_id,
+                                     msg=f"<pre>{message}</pre>",
+                                     parse_mode=ParseMode.HTML,
+                                     callback_path="update_status_table", reload_able=True)
+                else:
+                    self._send_msg(f"<pre>{message}</pre>", reload_able=True,
+                                   callback_path="update_status_table", parse_mode=ParseMode.HTML)
         except RPCException as e:
             self._send_msg(str(e))
 
@@ -399,9 +470,12 @@ class Telegram(RPCHandler):
             message = f'<b>Daily Profit over the last {timescale} days</b>:\n<pre>{stats_tab}</pre>'
             if(update.callback_query):
                 query = update.callback_query
-                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True)
+                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id,
+                                 msg=message, parse_mode=ParseMode.HTML,
+                                 callback_path="update_daily", reload_able=True)
             else:
-                self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True)
+                self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily",
+                               reload_able=True)
         except RPCException as e:
             self._send_msg(str(e))
 
@@ -417,9 +491,20 @@ class Telegram(RPCHandler):
         stake_cur = self._config['stake_currency']
         fiat_disp_cur = self._config.get('fiat_display_currency', '')
 
+        start_date = datetime.fromtimestamp(0)
+        timescale = None
+        try:
+            if context.args:
+                timescale = int(context.args[0])
+                today_start = datetime.combine(date.today(), datetime.min.time())
+                start_date = today_start - timedelta(days=timescale)
+        except (TypeError, ValueError, IndexError):
+            pass
+
         stats = self._rpc._rpc_trade_statistics(
             stake_cur,
-            fiat_disp_cur)
+            fiat_disp_cur,
+            start_date)
         profit_closed_coin = stats['profit_closed_coin']
         profit_closed_percent_mean = stats['profit_closed_percent_mean']
         profit_closed_percent_sum = stats['profit_closed_percent_sum']
@@ -447,22 +532,25 @@ class Telegram(RPCHandler):
             else:
                 markdown_msg = "`No closed trade` \n"
 
-            markdown_msg += (f"*ROI:* All trades\n"
-                             f"∙ `{round_coin_value(profit_all_coin, stake_cur)} "
-                             f"({profit_all_percent_mean:.2f}%) "
-                             f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
-                             f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n"
-                             f"*Total Trade Count:* `{trade_count}`\n"
-                             f"*First Trade opened:* `{first_trade_date}`\n"
-                             f"*Latest Trade opened:* `{latest_trade_date}\n`"
-                             f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`"
-                             )
+            markdown_msg += (
+                f"*ROI:* All trades\n"
+                f"∙ `{round_coin_value(profit_all_coin, stake_cur)} "
+                f"({profit_all_percent_mean:.2f}%) "
+                f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
+                f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n"
+                f"*Total Trade Count:* `{trade_count}`\n"
+                f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* "
+                f"`{first_trade_date}`\n"
+                f"*Latest Trade opened:* `{latest_trade_date}\n`"
+                f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`"
+                )
             if stats['closed_trade_count'] > 0:
                 markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n"
                                  f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`")
         if(update.callback_query):
             query = update.callback_query
-            self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=markdown_msg, callback_path="update_profit", reload_able=True)
+            self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id,
+                             msg=markdown_msg, callback_path="update_profit", reload_able=True)
         else:
             self._send_msg(msg=markdown_msg, callback_path="update_profit", reload_able=True)
 
@@ -515,6 +603,10 @@ class Telegram(RPCHandler):
             result = self._rpc._rpc_balance(self._config['stake_currency'],
                                             self._config.get('fiat_display_currency', ''))
 
+            balance_dust_level = self._config['telegram'].get('balance_dust_level', 0.0)
+            if not balance_dust_level:
+                balance_dust_level = DUST_PER_COIN.get(self._config['stake_currency'], 1.0)
+
             output = ''
             if self._config['dry_run']:
                 output += (
@@ -524,7 +616,7 @@ class Telegram(RPCHandler):
                     f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n"
                 )
             for curr in result['currencies']:
-                if curr['est_stake'] > 0.0001:
+                if curr['est_stake'] > balance_dust_level:
                     curr_output = (
                         f"*{curr['currency']}:*\n"
                         f"\t`Available: {curr['free']:.8f}`\n"
@@ -533,7 +625,8 @@ class Telegram(RPCHandler):
                         f"\t`Est. {curr['stake']}: "
                         f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n")
                 else:
-                    curr_output = f"*{curr['currency']}:* not showing <1$ amount \n"
+                    curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} "
+                                   f"{curr['stake']} amount \n")
 
                 # Handle overflowing messsage length
                 if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH:
@@ -548,7 +641,8 @@ class Telegram(RPCHandler):
                        f"{round_coin_value(result['value'], result['symbol'], False)}`\n")
             if(update.callback_query):
                 query = update.callback_query
-                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=output, callback_path="update_balance", reload_able=True)
+                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id,
+                                 msg=output, callback_path="update_balance", reload_able=True)
             else:
                 self._send_msg(msg=output, callback_path="update_balance", reload_able=True)
         except RPCException as e:
@@ -623,6 +717,25 @@ class Telegram(RPCHandler):
         except RPCException as e:
             self._send_msg(str(e))
 
+    def _forcebuy_action(self, pair, price=None):
+        try:
+            self._rpc._rpc_forcebuy(pair, price)
+        except RPCException as e:
+            self._send_msg(str(e))
+
+    def _forcebuy_inline(self, update: Update, _: CallbackContext) -> None:
+        if update.callback_query:
+            query = update.callback_query
+            pair = query.data
+            query.answer()
+            query.edit_message_text(text=f"Force Buying: {pair}")
+            self._forcebuy_action(pair)
+
+    @staticmethod
+    def _layout_inline_keyboard(buttons: List[InlineKeyboardButton],
+                                cols=3) -> List[List[InlineKeyboardButton]]:
+        return [buttons[i:i + cols] for i in range(0, len(buttons), cols)]
+
     @authorized_only
     def _forcebuy(self, update: Update, context: CallbackContext) -> None:
         """
@@ -635,10 +748,13 @@ class Telegram(RPCHandler):
         if context.args:
             pair = context.args[0]
             price = float(context.args[1]) if len(context.args) > 1 else None
-            try:
-                self._rpc._rpc_forcebuy(pair, price)
-            except RPCException as e:
-                self._send_msg(str(e))
+            self._forcebuy_action(pair, price)
+        else:
+            whitelist = self._rpc._rpc_whitelist()['whitelist']
+            pairs = [InlineKeyboardButton(text=pair, callback_data=pair) for pair in whitelist]
+
+            self._send_msg(msg="Which pair?",
+                           keyboard=self._layout_inline_keyboard(pairs))
 
     @authorized_only
     def _trades(self, update: Update, context: CallbackContext) -> None:
@@ -659,13 +775,13 @@ class Telegram(RPCHandler):
                 nrecent
             )
             trades_tab = tabulate(
-                [[arrow.get(trade['open_date']).humanize(),
-                  trade['pair'],
+                [[arrow.get(trade['close_date']).humanize(),
+                  trade['pair'] + " (#" + str(trade['trade_id']) + ")",
                   f"{(100 * trade['close_profit']):.2f}% ({trade['close_profit_abs']})"]
                  for trade in trades['trades']],
                 headers=[
-                    'Open Date',
-                    'Pair',
+                    'Close Date',
+                    'Pair (ID)',
                     f'Profit ({stake_cur})',
                 ],
                 tablefmt='simple')
@@ -708,18 +824,30 @@ class Telegram(RPCHandler):
         """
         try:
             trades = self._rpc._rpc_performance()
-            stats = '\n'.join('{index}.\t<code>{pair}\t{profit:.2f}% ({count})</code>'.format(
-                index=i + 1,
-                pair=trade['pair'],
-                profit=trade['profit'],
-                count=trade['count']
-            ) for i, trade in enumerate(trades))
-            message = '<b>Performance:</b>\n{}'.format(stats)
-            if(update.callback_query):
+            output = "<b>Performance:</b>\n"
+            sent_messages = 0
+            for i, trade in enumerate(trades):
+                stat_line = (
+                    f"{i+1}.\t <code>{trade['pair']}\t"
+                    f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} "
+                    f"({trade['profit']:.2f}%) "
+                    f"({trade['count']})</code>\n")
+
+                if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
+                    self._send_msg(output, parse_mode=ParseMode.HTML)
+                    output = stat_line
+                    sent_messages += 1
+                else:
+                    output += stat_line
+
+            if(sent_messages == 0 and update.callback_query):
                 query = update.callback_query
-                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True)
+                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id,
+                                 msg=output, parse_mode=ParseMode.HTML,
+                                 callback_path="update_performance", reload_able=True)
             else:
-                self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True)
+                self._send_msg(msg=output, parse_mode=ParseMode.HTML,
+                               callback_path="update_performance", reload_able=True)
         except RPCException as e:
             self._send_msg(str(e))
 
@@ -741,9 +869,12 @@ class Telegram(RPCHandler):
             logger.debug(message)
             if(update.callback_query):
                 query = update.callback_query
-                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True)
+                self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id,
+                                 msg=message, parse_mode=ParseMode.HTML,
+                                 callback_path="update_count", reload_able=True)
             else:
-                self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True)
+                self._send_msg(msg=message, parse_mode=ParseMode.HTML,
+                               callback_path="update_count", reload_able=True)
         except RPCException as e:
             self._send_msg(str(e))
 
@@ -753,19 +884,39 @@ class Telegram(RPCHandler):
         Handler for /locks.
         Returns the currently active locks
         """
-        try:
-            locks = self._rpc._rpc_locks()
+        rpc_locks = self._rpc._rpc_locks()
+        if not rpc_locks['locks']:
+            self._send_msg('No active locks.', parse_mode=ParseMode.HTML)
+
+        for locks in chunks(rpc_locks['locks'], 25):
             message = tabulate([[
+                lock['id'],
                 lock['pair'],
                 lock['lock_end_time'],
-                lock['reason']] for lock in locks['locks']],
-                headers=['Pair', 'Until', 'Reason'],
+                lock['reason']] for lock in locks],
+                headers=['ID', 'Pair', 'Until', 'Reason'],
                 tablefmt='simple')
-            message = "<pre>{}</pre>".format(message)
+            message = f"<pre>{escape(message)}</pre>"
             logger.debug(message)
             self._send_msg(message, parse_mode=ParseMode.HTML)
-        except RPCException as e:
-            self._send_msg(str(e))
+
+    @authorized_only
+    def _delete_locks(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /delete_locks.
+        Returns the currently active locks
+        """
+        arg = context.args[0] if context.args and len(context.args) > 0 else None
+        lockid = None
+        pair = None
+        if arg:
+            try:
+                lockid = int(arg)
+            except ValueError:
+                pair = arg
+
+        self._rpc._rpc_delete_lock(lockid=lockid, pair=pair)
+        self._locks(update, context)
 
     @authorized_only
     def _whitelist(self, update: Update, context: CallbackContext) -> None:
@@ -847,9 +998,17 @@ class Telegram(RPCHandler):
         """
         try:
             edge_pairs = self._rpc._rpc_edge()
-            edge_pairs_tab = tabulate(edge_pairs, headers='keys', tablefmt='simple')
-            message = f'<b>Edge only validated following pairs:</b>\n<pre>{edge_pairs_tab}</pre>'
-            self._send_msg(message, parse_mode=ParseMode.HTML)
+            if not edge_pairs:
+                message = '<b>Edge only validated following pairs:</b>'
+                self._send_msg(message, parse_mode=ParseMode.HTML)
+
+            for chunk in chunks(edge_pairs, 25):
+                edge_pairs_tab = tabulate(chunk, headers='keys', tablefmt='simple')
+                message = (f'<b>Edge only validated following pairs:</b>\n'
+                           f'<pre>{edge_pairs_tab}</pre>')
+
+                self._send_msg(message, parse_mode=ParseMode.HTML)
+
         except RPCException as e:
             self._send_msg(str(e))
 
@@ -873,7 +1032,8 @@ class Telegram(RPCHandler):
                    "                `pending buy orders are marked with an asterisk (*)`\n"
                    "                `pending sell orders are marked with a double asterisk (**)`\n"
                    "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n"
-                   "*/profit:* `Lists cumulative profit from all finished trades`\n"
+                   "*/profit [<n>]:* `Lists cumulative profit from all finished trades, "
+                   "over the last n days`\n"
                    "*/forcesell <trade_id>|all:* `Instantly sells the given trade or all trades, "
                    "regardless of profit`\n"
                    f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}"
@@ -884,6 +1044,7 @@ class Telegram(RPCHandler):
                    "Avg. holding durationsfor buys and sells.`\n"
                    "*/count:* `Show number of active trades compared to allowed number of trades`\n"
                    "*/locks:* `Show currently locked pairs`\n"
+                   "*/unlock <pair|id>:* `Unlock this Pair (or this lock id if it's numeric)`\n"
                    "*/balance:* `Show account balance per currency`\n"
                    "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n"
                    "*/reload_config:* `Reload configuration file` \n"
@@ -945,12 +1106,15 @@ class Telegram(RPCHandler):
             f"*Current state:* `{val['state']}`"
         )
 
-    def _update_msg(self, chat_id: str, message_id: str, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
+    def _update_msg(self, chat_id: str, message_id: str, msg: str, callback_path: str = "",
+                    reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
         if reload_able:
-            reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]])
+            reply_markup = InlineKeyboardMarkup([
+                [InlineKeyboardButton("Refresh", callback_data=callback_path)],
+                ])
         else:
             reply_markup = InlineKeyboardMarkup([[]])
-        msg+="\nUpdated: {}".format(datetime.now().ctime())
+        msg += "\nUpdated: {}".format(datetime.now().ctime())
         try:
             try:
                 self._updater.bot.edit_message_text(
@@ -974,7 +1138,11 @@ class Telegram(RPCHandler):
                 telegram_err.message
             )
 
-    def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, callback_path: str = "", reload_able: bool = False) -> None:
+    def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN,
+                  disable_notification: bool = False,
+                  keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None,
+                  callback_path: str = "",
+                  reload_able: bool = False) -> None:
         """
         Send given markdown message
         :param msg: message
@@ -982,10 +1150,14 @@ class Telegram(RPCHandler):
         :param parse_mode: telegram parse mode
         :return: None
         """
-        if reload_able and self._config['telegram'].get('reload',True):
-            reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]])
+        if reload_able and self._config['telegram'].get('reload', True):
+            reply_markup = InlineKeyboardMarkup([
+                [InlineKeyboardButton("Refresh", callback_data=callback_path)]])
         else:
-            reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True)
+            if keyboard is not None:
+                reply_markup = InlineKeyboardMarkup(keyboard, resize_keyboard=True)
+            else:
+                reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True)
         try:
             try:
                 self._updater.bot.send_message(

freqtrade/rpc/webhook.py

@@ -6,7 +6,8 @@ from typing import Any, Dict
 
 from requests import RequestException, post
 
-from freqtrade.rpc import RPC, RPCHandler, RPCMessageType
+from freqtrade.enums import RPCMessageType
+from freqtrade.rpc import RPC, RPCHandler
 
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,12 @@ class Webhook(RPCHandler):
 
         self._url = self._config['webhook']['url']
 
+        self._format = self._config['webhook'].get('format', 'form')
+
+        if self._format != 'form' and self._format != 'json':
+            raise NotImplementedError('Unknown webhook format `{}`, possible values are '
+                                      '`form` (default) and `json`'.format(self._format))
+
     def cleanup(self) -> None:
         """
         Cleanup pending module resources.
@@ -39,17 +46,21 @@ class Webhook(RPCHandler):
         """ Send a message to telegram channel """
         try:
 
-            if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
+            if msg['type'] == RPCMessageType.BUY:
                 valuedict = self._config['webhook'].get('webhookbuy', None)
-            elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.BUY_CANCEL:
                 valuedict = self._config['webhook'].get('webhookbuycancel', None)
-            elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.BUY_FILL:
+                valuedict = self._config['webhook'].get('webhookbuyfill', None)
+            elif msg['type'] == RPCMessageType.SELL:
                 valuedict = self._config['webhook'].get('webhooksell', None)
-            elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.SELL_FILL:
+                valuedict = self._config['webhook'].get('webhooksellfill', None)
+            elif msg['type'] == RPCMessageType.SELL_CANCEL:
                 valuedict = self._config['webhook'].get('webhooksellcancel', None)
-            elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION,
-                                 RPCMessageType.STARTUP_NOTIFICATION,
-                                 RPCMessageType.WARNING_NOTIFICATION):
+            elif msg['type'] in (RPCMessageType.STATUS,
+                                 RPCMessageType.STARTUP,
+                                 RPCMessageType.WARNING):
                 valuedict = self._config['webhook'].get('webhookstatus', None)
             else:
                 raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
@@ -66,7 +77,14 @@ class Webhook(RPCHandler):
     def _send_msg(self, payload: dict) -> None:
         """do the actual call to the webhook"""
 
+        if self._format == 'form':
+            kwargs = {'data': payload}
+        elif self._format == 'json':
+            kwargs = {'json': payload}
+        else:
+            raise NotImplementedError('Unknown format: {}'.format(self._format))
+
         try:
-            post(self._url, data=payload)
+            post(self._url, **kwargs)
         except RequestException as exc:
             logger.warning("Could not call webhook url. Exception: %s", exc)