Merge branch 'develop' of https://github.com/freqtrade/freqtrade into custom_order_price

docs/configuration.md

@@ -105,7 +105,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `ask_strategy.order_book_top` | Bot will use the top N rate in Order Book "price_side" to sell. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Asks](#sell-price-with-orderbook-enabled)<br>*Defaults to `1`.* <br> **Datatype:** Positive Integer
 | `use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `sell_profit_only` | Wait until the bot reaches `sell_profit_offset` before taking a sell decision. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
-| `sell_profit_offset` | Sell-signal is only active above this value. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
+| `sell_profit_offset` | Sell-signal is only active above this value. Only active in combination with `sell_profit_only=True`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
 | `ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
 | `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used. <br> **Datatype:** Integer
 | `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Dict

docs/exchanges.md

@@ -105,7 +105,7 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll
 
 ## Kucoin
 
-Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
+Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
 
 ```json
 "exchange": {

docs/requirements-docs.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.2
-mkdocs-material==7.2.2
+mkdocs-material==7.2.4
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2

docs/strategy_analysis_example.md

@@ -228,7 +228,7 @@ graph = generate_candlestick_graph(pair=pair,
 # Show graph inline
 # graph.show()
 
-# Render graph in a seperate window
+# Render graph in a separate window
 graph.show(renderer="browser")
 
 ```

freqtrade/data/btanalysis.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
                        "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
 
-# Mid-term format, crated by BacktestResult Named Tuple
+# Mid-term format, created by BacktestResult Named Tuple
 BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
                        'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
                        'fee_close', 'amount', 'profit_abs', 'profit_ratio']

freqtrade/data/converter.py

@@ -242,7 +242,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
     :param config: Config dictionary
     :param convert_from: Source format
     :param convert_to: Target format
-    :param erase: Erase souce data (does not apply if source and target format are identical)
+    :param erase: Erase source data (does not apply if source and target format are identical)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
@@ -267,7 +267,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
     :param config: Config dictionary
     :param convert_from: Source format
     :param convert_to: Target format
-    :param erase: Erase souce data (does not apply if source and target format are identical)
+    :param erase: Erase source data (does not apply if source and target format are identical)
     """
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)

freqtrade/exchange/exchange.py

@@ -1497,7 +1497,7 @@ class Exchange:
         :returns List of trade data
         """
         if not self.exchange_has("fetchTrades"):
-            raise OperationalException("This exchange does not suport downloading Trades.")
+            raise OperationalException("This exchange does not support downloading Trades.")
 
         return asyncio.get_event_loop().run_until_complete(
             self._async_get_trade_history(pair=pair, since=since,

freqtrade/freqtradebot.py

@@ -983,7 +983,7 @@ class FreqtradeBot(LoggingMixin):
             # if trade is partially complete, edit the stake details for the trade
             # and close the order
             # cancel_order may not contain the full order dict, so we need to fallback
-            # to the order dict aquired before cancelling.
+            # to the order dict acquired before cancelling.
             # we need to fall back to the values from order if corder does not contain these keys.
             trade.amount = filled_amount
             trade.stake_amount = trade.amount * trade.open_rate

freqtrade/plot/plotting.py

@@ -538,7 +538,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
     - Initializes plot-script
     - Get candle (OHLCV) data
     - Generate Dafaframes populated with indicators and signals based on configured strategy
-    - Load trades excecuted during the selected period
+    - Load trades executed during the selected period
     - Generate Plotly plot objects
     - Generate plot files
     :return: None

freqtrade/plugins/pairlist/IPairList.py

@@ -150,18 +150,20 @@ class IPairList(LoggingMixin, ABC):
         for pair in pairlist:
             # pair is not in the generated dynamic market or has the wrong stake currency
             if pair not in markets:
-                logger.warning(f"Pair {pair} is not compatible with exchange "
-                               f"{self._exchange.name}. Removing it from whitelist..")
+                self.log_once(f"Pair {pair} is not compatible with exchange "
+                              f"{self._exchange.name}. Removing it from whitelist..",
+                              logger.warning)
                 continue
 
             if not self._exchange.market_is_tradable(markets[pair]):
-                logger.warning(f"Pair {pair} is not tradable with Freqtrade."
-                               "Removing it from whitelist..")
+                self.log_once(f"Pair {pair} is not tradable with Freqtrade."
+                              "Removing it from whitelist..", logger.warning)
                 continue
 
             if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
-                logger.warning(f"Pair {pair} is not compatible with your stake currency "
-                               f"{self._config['stake_currency']}. Removing it from whitelist..")
+                self.log_once(f"Pair {pair} is not compatible with your stake currency "
+                              f"{self._config['stake_currency']}. Removing it from whitelist..",
+                              logger.warning)
                 continue
 
             # Check if market is active

freqtrade/plugins/pairlist/VolumePairList.py

@@ -126,7 +126,7 @@ class VolumePairList(IPairList):
             pairlist = [s['symbol'] for s in filtered_tickers]
 
             pairlist = self.filter_pairlist(pairlist, tickers)
-            self._pair_cache['pairlist'] = pairlist
+            self._pair_cache['pairlist'] = pairlist.copy()
 
         return pairlist
 

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -120,5 +120,6 @@ class RangeStabilityFilter(IPairList):
                         logger.info)
                     result = False
             self._pair_cache[pair] = result
-
+        else:
+            self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
         return result

freqtrade/rpc/api_server/api_v1.py

@@ -223,11 +223,11 @@ def list_strategies(config=Depends(get_config)):
 @router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
 def get_strategy(strategy: str, config=Depends(get_config)):
 
-    config = deepcopy(config)
+    config_ = deepcopy(config)
     from freqtrade.resolvers.strategy_resolver import StrategyResolver
     try:
-        strategy_obj = StrategyResolver._load_strategy(strategy, config,
-                                                       extra_dir=config.get('strategy_path'))
+        strategy_obj = StrategyResolver._load_strategy(strategy, config_,
+                                                       extra_dir=config_.get('strategy_path'))
     except OperationalException:
         raise HTTPException(status_code=404, detail='Strategy not found')
 

freqtrade/rpc/api_server/web_ui.py

@@ -29,6 +29,16 @@ async def ui_version():
     }
 
 
+def is_relative_to(path, base) -> bool:
+    # Helper function simulating behaviour of is_relative_to, which was only added in python 3.9
+    try:
+        path.relative_to(base)
+        return True
+    except ValueError:
+        pass
+    return False
+
+
 @router_ui.get('/{rest_of_path:path}', include_in_schema=False)
 async def index_html(rest_of_path: str):
     """
@@ -37,8 +47,11 @@ async def index_html(rest_of_path: str):
     if rest_of_path.startswith('api') or rest_of_path.startswith('.'):
         raise HTTPException(status_code=404, detail="Not Found")
     uibase = Path(__file__).parent / 'ui/installed/'
-    if (uibase / rest_of_path).is_file():
-        return FileResponse(str(uibase / rest_of_path))
+    filename = uibase / rest_of_path
+    # It's security relevant to check "relative_to".
+    # Without this, Directory-traversal is possible.
+    if filename.is_file() and is_relative_to(filename, uibase):
+        return FileResponse(str(filename))
 
     index_file = uibase / 'index.html'
     if not index_file.is_file():

freqtrade/rpc/rpc.py

@@ -776,7 +776,7 @@ class RPC:
         if has_content:
 
             dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
-            # Move open to seperate column when signal for easy plotting
+            # Move open to separate column when signal for easy plotting
             if 'buy' in dataframe.columns:
                 buy_mask = (dataframe['buy'] == 1)
                 buy_signals = int(buy_mask.sum())

requirements-dev.txt

@@ -6,7 +6,7 @@
 coveralls==3.2.0
 flake8==3.9.2
 flake8-type-annotations==0.1.0
-flake8-tidy-imports==4.3.0
+flake8-tidy-imports==4.4.1
 mypy==0.910
 pytest==6.2.4
 pytest-asyncio==0.15.1

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.1.0
+plotly==5.2.1
 

requirements.txt

@@ -1,7 +1,7 @@
-numpy==1.21.1
-pandas==1.3.1
+numpy==1.21.2
+pandas==1.3.2
 
-ccxt==1.54.74
+ccxt==1.55.6
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.7
 aiohttp==3.7.4.post0
@@ -32,7 +32,7 @@ sdnotify==0.3.2
 
 # API Server
 fastapi==0.68.0
-uvicorn==0.14.0
+uvicorn==0.15.0
 pyjwt==2.1.0
 aiofiles==0.7.0
 

tests/data/test_converter.py

@@ -119,7 +119,7 @@ def test_ohlcv_fill_up_missing_data2(caplog):
     # 3rd candle has been filled
     row = data2.loc[2, :]
     assert row['volume'] == 0
-    # close shoult match close of previous candle
+    # close should match close of previous candle
     assert row['close'] == data.loc[1, 'close']
     assert row['open'] == row['close']
     assert row['high'] == row['close']

tests/data/test_dataprovider.py

@@ -66,7 +66,7 @@ def test_historic_ohlcv_dataformat(mocker, default_conf, ohlcv_history):
     hdf5loadmock.assert_not_called()
     jsonloadmock.assert_called_once()
 
-    # Swiching to dataformat hdf5
+    # Switching to dataformat hdf5
     hdf5loadmock.reset_mock()
     jsonloadmock.reset_mock()
     default_conf["dataformat_ohlcv"] = "hdf5"

tests/data/test_history.py

@@ -200,15 +200,15 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None:
     assert start_ts == test_data[0][0] - 1000
 
     # timeframe starts in the center of the cached data
-    # should return the chached data w/o the last item
+    # should return the cached data w/o the last item
     timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
     data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler)
 
     assert_frame_equal(data, test_data_df.iloc[:-1])
     assert test_data[-2][0] <= start_ts < test_data[-1][0]
 
-    # timeframe starts after the chached data
-    # should return the chached data w/o the last item
+    # timeframe starts after the cached data
+    # should return the cached data w/o the last item
     timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0)
     data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler)
     assert_frame_equal(data, test_data_df.iloc[:-1])

tests/exchange/test_exchange.py

@@ -2182,7 +2182,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange
     pair = 'ETH/BTC'
 
     with pytest.raises(OperationalException,
-                       match="This exchange does not suport downloading Trades."):
+                       match="This exchange does not support downloading Trades."):
         exchange.get_historic_trades(pair, since=trades_history[0][0],
                                      until=trades_history[-1][0])
 

tests/rpc/test_rpc_apiserver.py

@@ -109,6 +109,11 @@ def test_api_ui_fallback(botclient):
     rc = client_get(client, "/something")
     assert rc.status_code == 200
 
+    # Test directory traversal
+    rc = client_get(client, '%2F%2F%2Fetc/passwd')
+    assert rc.status_code == 200
+    assert '`freqtrade install-ui`' in rc.text
+
 
 def test_api_ui_version(botclient, mocker):
     ftbot, client = botclient

tests/strategy/test_interface.py

@@ -630,7 +630,7 @@ def test_strategy_safe_wrapper_error(caplog, error):
     assert ret
 
     caplog.clear()
-    # Test supressing error
+    # Test suppressing error
     ret = strategy_safe_wrapper(failing_method, message='DeadBeef', supress_error=True)()
     assert log_has_re(r'DeadBeef.*', caplog)