Bump ruff from 0.0.251 to 0.0.252

Bumps [ruff](https://github.com/charliermarsh/ruff) from 0.0.251 to 0.0.252.
- [Release notes](https://github.com/charliermarsh/ruff/releases)
- [Changelog](https://github.com/charliermarsh/ruff/blob/main/BREAKING_CHANGES.md)
- [Commits](https://github.com/charliermarsh/ruff/compare/v0.0.251...v0.0.252)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/ci.yml

@@ -90,14 +90,14 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 6 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Flake8
-      run: |
-        flake8
-
     - name: Sort imports (isort)
       run: |
         isort --check .
 
+    - name: Run Ruff
+      run: |
+        ruff check --format=github .
+
     - name: Mypy
       run: |
         mypy freqtrade scripts tests
@@ -186,14 +186,14 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Flake8
-      run: |
-        flake8
-
     - name: Sort imports (isort)
       run: |
         isort --check .
 
+    - name: Run Ruff
+      run: |
+        ruff check --format=github .
+
     - name: Mypy
       run: |
         mypy freqtrade scripts
@@ -248,9 +248,9 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Flake8
+    - name: Run Ruff
       run: |
-        flake8
+        ruff check --format=github .
 
     - name: Mypy
       run: |

.pre-commit-config.yaml

@@ -27,6 +27,12 @@ repos:
         name: isort (python)
         # stages: [push]
 
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.251'
+    hooks:
+      - id: ruff
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:

CONTRIBUTING.md

@@ -45,16 +45,17 @@ pytest tests/test_<file_name>.py::test_<method_name>
 
 ### 2. Test if your code is PEP8 compliant
 
-#### Run Flake8
+#### Run Ruff
 
 ```bash
-flake8 freqtrade tests scripts
+ruff .
 ```
 
-We receive a lot of code that fails the `flake8` checks.
+We receive a lot of code that fails the `ruff` checks.
 To help with that, we encourage you to install the git pre-commit 
 hook that will warn you when you try to commit code that fails these checks.
-Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html).
+
+you can manually run pre-commit with `pre-commit run -a`.
 
 ##### Additional styles applied
 

docs/developer.md

@@ -24,7 +24,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
 Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
 
-This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
+This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`.
 
 Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing.
 This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine.

environment.yml

@@ -41,7 +41,6 @@ dependencies:
     # 2/4 req dev
 
     - coveralls
-    - flake8
     - mypy
     - pytest
     - pytest-asyncio
@@ -70,6 +69,6 @@ dependencies:
         - tables
         - pytest-random-order
         - ccxt
-        - flake8-tidy-imports
+        - ruff
         - -e .
     # - python-rapidjso

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.2'
+__version__ = '2023.3.dev'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/data_commands.py

@@ -5,7 +5,7 @@ from datetime import datetime, timedelta
 from typing import Any, Dict, List
 
 from freqtrade.configuration import TimeRange, setup_utils_configuration
-from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
 from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
@@ -20,15 +20,24 @@ from freqtrade.util.binance_mig import migrate_binance_futures_data
 logger = logging.getLogger(__name__)
 
 
+def _data_download_sanity(config: Config) -> None:
+    if 'days' in config and 'timerange' in config:
+        raise OperationalException("--days and --timerange are mutually exclusive. "
+                                   "You can only specify one or the other.")
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+
 def start_download_data(args: Dict[str, Any]) -> None:
     """
     Download data (former download_backtest_data.py script)
     """
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
-    if 'days' in config and 'timerange' in config:
-        raise OperationalException("--days and --timerange are mutually exclusive. "
-                                   "You can only specify one or the other.")
+    _data_download_sanity(config)
     timerange = TimeRange()
     if 'days' in config:
         time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
@@ -40,11 +49,6 @@ def start_download_data(args: Dict[str, Any]) -> None:
     # Remove stake-currency to skip checks which are not relevant for datadownload
     config['stake_currency'] = ''
 
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
-
     pairs_not_available: List[str] = []
 
     # Init exchange

freqtrade/enums/candletype.py

@@ -13,6 +13,9 @@ class CandleType(str, Enum):
     FUNDING_RATE = "funding_rate"
     # BORROW_RATE = "borrow_rate"  # * unimplemented
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
     @staticmethod
     def from_string(value: str) -> 'CandleType':
         if not value:

freqtrade/enums/rpcmessagetype.py

@@ -37,5 +37,8 @@ class RPCRequestType(str, Enum):
     WHITELIST = 'whitelist'
     ANALYZED_DF = 'analyzed_df'
 
+    def __str__(self):
+        return self.value
+
 
 NO_ECHO_MESSAGES = (RPCMessageType.ANALYZED_DF, RPCMessageType.WHITELIST, RPCMessageType.NEW_CANDLE)

freqtrade/enums/signaltype.py

@@ -10,6 +10,9 @@ class SignalType(Enum):
     ENTER_SHORT = "enter_short"
     EXIT_SHORT = "exit_short"
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
 
 class SignalTagType(Enum):
     """
@@ -18,7 +21,13 @@ class SignalTagType(Enum):
     ENTER_TAG = "enter_tag"
     EXIT_TAG = "exit_tag"
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
 
 class SignalDirection(str, Enum):
     LONG = 'long'
     SHORT = 'short'
+
+    def __str__(self):
+        return f"{self.name.lower()}"

freqtrade/exchange/exchange.py

@@ -1961,7 +1961,8 @@ class Exchange:
                           cache: bool, drop_incomplete: bool) -> DataFrame:
         # keeping last candle time as last refreshed time of the pair
         if ticks and cache:
-            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
+            idx = -2 if drop_incomplete and len(ticks) > 1 else -1
+            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[idx][0] // 1000
         # keeping parsed dataframe in cache
         ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
                                       drop_incomplete=drop_incomplete)
@@ -2034,7 +2035,9 @@ class Exchange:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
         plr = self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0) + interval_in_sec
-        return plr < arrow.utcnow().int_timestamp
+        # current,active candle open date
+        now = int(timeframe_to_prev_date(timeframe).timestamp())
+        return plr < now
 
     @retrier_async
     async def _async_get_candle_history(

freqtrade/exchange/kucoin.py

@@ -64,6 +64,7 @@ class Kucoin(Exchange):
         # ccxt returns status = 'closed' at the moment - which is information ccxt invented.
         # Since we rely on status heavily, we must set it to 'open' here.
         # ref: https://github.com/ccxt/ccxt/pull/16674, (https://github.com/ccxt/ccxt/pull/16553)
+        if not self._config['dry_run']:
             res['type'] = ordertype
             res['status'] = 'open'
         return res

freqtrade/freqai/data_drawer.py

@@ -570,12 +570,12 @@ class FreqaiDataDrawer:
 
             for pair in dk.all_pairs:
                 for tf in feat_params.get("include_timeframes"):
-
+                    hist_df = history_data[pair][tf]
                     # check if newest candle is already appended
                     df_dp = strategy.dp.get_pair_dataframe(pair, tf)
                     if len(df_dp.index) == 0:
                         continue
-                    if str(history_data[pair][tf].iloc[-1]["date"]) == str(
+                    if str(hist_df.iloc[-1]["date"]) == str(
                         df_dp.iloc[-1:]["date"].iloc[-1]
                     ):
                         continue
@@ -583,21 +583,30 @@ class FreqaiDataDrawer:
                     try:
                         index = (
                             df_dp.loc[
-                                df_dp["date"] == history_data[pair][tf].iloc[-1]["date"]
+                                df_dp["date"] == hist_df.iloc[-1]["date"]
                             ].index[0]
                             + 1
                         )
                     except IndexError:
+                        if hist_df.iloc[-1]['date'] < df_dp['date'].iloc[0]:
+                            raise OperationalException("In memory historical data is older than "
+                                                       f"oldest DataProvider candle for {pair} on "
+                                                       f"timeframe {tf}")
+                        else:
+                            index = -1
                             logger.warning(
-                            f"Unable to update pair history for {pair}. "
-                            "If this does not resolve itself after 1 additional candle, "
-                            "please report the error to #freqai discord channel"
+                                f"No common dates in historical data and dataprovider for {pair}. "
+                                f"Appending latest dataprovider candle to historical data "
+                                "but please be aware that there is likely a gap in the historical "
+                                "data. \n"
+                                f"Historical data ends at {hist_df.iloc[-1]['date']} "
+                                f"while dataprovider starts at {df_dp['date'].iloc[0]} and"
+                                f"ends at {df_dp['date'].iloc[0]}."
                             )
-                        return
 
                     history_data[pair][tf] = pd.concat(
                         [
-                            history_data[pair][tf],
+                            hist_df,
                             df_dp.iloc[index:],
                         ],
                         ignore_index=True,

freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py

@@ -34,6 +34,11 @@ class ReinforcementLearner_multiproc(ReinforcementLearner):
         train_df = data_dictionary["train_features"]
         test_df = data_dictionary["test_features"]
 
+        if self.train_env:
+            self.train_env.close()
+        if self.eval_env:
+            self.eval_env.close()
+
         env_info = self.pack_env_dict(dk.pair)
 
         env_id = "train_env"

pyproject.toml

@@ -56,3 +56,24 @@ exclude = [
   "build_helpers/*.py",
 ]
 ignore = ["freqtrade/vendor/**"]
+
+
+[tool.ruff]
+line-length = 100
+extend-exclude = [".env"]
+target-version = "py38"
+extend-select = [
+  "C90",    # mccabe
+  # "N",    # pep8-naming
+  # "UP",   # pyupgrade
+  "TID",    # flake8-tidy-imports
+  # "EXE",  # flake8-executable
+  "YTT",    # flake8-2020
+  # "DTZ",  # flake8-datetimez
+  # "RSE",  # flake8-raise
+  # "TCH",  # flake8-type-checking
+  # "PTH",  # flake8-use-pathlib
+]
+
+[tool.ruff.mccabe]
+max-complexity = 12

requirements-dev.txt

@@ -7,10 +7,9 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-flake8==6.0.0
-flake8-tidy-imports==4.8.0
+ruff==0.0.252
 mypy==1.0.1
-pre-commit==3.0.4
+pre-commit==3.1.0
 pytest==7.2.1
 pytest-asyncio==0.20.3
 pytest-cov==4.0.0

requirements.txt

@@ -35,7 +35,7 @@ sdnotify==0.3.2
 
 # API Server
 fastapi==0.92.0
-pydantic==1.10.4
+pydantic==1.10.5
 uvicorn==0.20.0
 pyjwt==2.6.0
 aiofiles==23.1.0

setup.py

@@ -32,8 +32,6 @@ hdf5 = [
 
 develop = [
     'coveralls',
-    'flake8',
-    'flake8-tidy-imports',
     'mypy',
     'pytest',
     'pytest-asyncio',

tests/exchange/test_bybit.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 from unittest.mock import MagicMock
 
 from freqtrade.enums.marginmode import MarginMode
@@ -55,3 +56,19 @@ async def test_bybit_fetch_funding_rate(default_conf, mocker):
     kwargs = api_mock.fetch_funding_rate_history.call_args_list[0][1]
     assert kwargs['params'] == {'until': since_ms_end}
     assert kwargs['since'] == since_ms
+
+
+def test_bybit_get_funding_fees(default_conf, mocker):
+    now = datetime.now(timezone.utc)
+    exchange = get_patched_exchange(mocker, default_conf, id='bybit')
+    exchange._fetch_and_calculate_funding_fees = MagicMock()
+    exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now)
+    assert exchange._fetch_and_calculate_funding_fees.call_count == 0
+
+    default_conf['trading_mode'] = 'futures'
+    default_conf['margin_mode'] = 'isolated'
+    exchange = get_patched_exchange(mocker, default_conf, id='bybit')
+    exchange._fetch_and_calculate_funding_fees = MagicMock()
+    exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now)
+
+    assert exchange._fetch_and_calculate_funding_fees.call_count == 1

tests/exchange/test_exchange.py

@@ -27,7 +27,7 @@ from tests.conftest import (generate_test_data_raw, get_mock_coro, get_patched_e
 
 
 # Make sure to always keep one exchange here which is NOT subclassed!!
-EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate']
+EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit']
 
 get_entry_rate_data = [
     ('other', 20, 19, 10, 0.0, 20),  # Full ask side
@@ -1269,7 +1269,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, fill
                           fetch_l2_order_book=order_book_l2_usd,
                           )
 
-    order = exchange.create_dry_run_order(
+    order = exchange.create_order(
         pair='LTC/USDT',
         ordertype='limit',
         side=side,
@@ -1332,7 +1332,7 @@ def test_create_dry_run_order_market_fill(default_conf, mocker, side, rate, amou
                           fetch_l2_order_book=order_book_l2_usd,
                           )
 
-    order = exchange.create_dry_run_order(
+    order = exchange.create_order(
         pair='LTC/USDT',
         ordertype='market',
         side=side,
@@ -1425,9 +1425,10 @@ def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice,
     assert order['amount'] == 0.01
 
 
-def test_buy_dry_run(default_conf, mocker):
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_buy_dry_run(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
-    exchange = get_patched_exchange(mocker, default_conf)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
 
     order = exchange.create_order(pair='ETH/BTC', ordertype='limit', side="buy",
                                   amount=1, rate=200, leverage=1.0,
@@ -2215,7 +2216,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert len(res[pair1]) == 99
     assert len(res[pair2]) == 99
     assert exchange._klines
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
     exchange._api_async.fetch_ohlcv.reset_mock()
 
     # Returned from cache
@@ -2224,7 +2225,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert len(res) == 2
     assert len(res[pair1]) == 99
     assert len(res[pair2]) == 99
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
 
     # Move time 1 candle further but result didn't change yet
     time_machine.move_to(start + timedelta(hours=101))
@@ -2234,7 +2235,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert len(res[pair1]) == 99
     assert len(res[pair2]) == 99
     assert res[pair2].at[0, 'open']
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
     refresh_pior = exchange._pairs_last_refresh_time[pair1]
 
     # New candle on exchange - return 100 candles - but skip one candle so we actually get 2 candles
@@ -2252,8 +2253,8 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert res[pair2].at[0, 'open']
     assert refresh_pior != exchange._pairs_last_refresh_time[pair1]
 
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
-    assert exchange._pairs_last_refresh_time[pair2] == ohlcv[-1][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair2] == ohlcv[-2][0] // 1000
     exchange._api_async.fetch_ohlcv.reset_mock()
 
     # Retry same call - from cache
@@ -5015,7 +5016,7 @@ def test_get_max_leverage_futures(default_conf, mocker, leverage_tiers):
         exchange.get_max_leverage("BTC/USDT:USDT", 1000000000.01)
 
 
-@pytest.mark.parametrize("exchange_name", ['bittrex', 'binance', 'kraken', 'gate', 'okx'])
+@pytest.mark.parametrize("exchange_name", ['bittrex', 'binance', 'kraken', 'gate', 'okx', 'bybit'])
 def test__get_params(mocker, default_conf, exchange_name):
     api_mock = MagicMock()
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
@@ -5036,6 +5037,9 @@ def test__get_params(mocker, default_conf, exchange_name):
         params2['tdMode'] = 'isolated'
         params2['posSide'] = 'net'
 
+    if exchange_name == 'bybit':
+        params2['position_idx'] = 0
+
     assert exchange._get_params(
         side="buy",
         ordertype='market',

tests/exchange/test_kucoin.py

@@ -125,3 +125,45 @@ def test_stoploss_adjust_kucoin(mocker, default_conf):
     # Test with invalid order case
     order['stopPrice'] = None
     assert exchange.stoploss_adjust(1501, order, 'sell')
+
+
+@pytest.mark.parametrize("side", ["buy", "sell"])
+@pytest.mark.parametrize("ordertype,rate", [
+    ("market", None),
+    ("market", 200),
+    ("limit", 200),
+    ("stop_loss_limit", 200)
+])
+def test_kucoin_create_order(default_conf, mocker, side, ordertype, rate):
+    api_mock = MagicMock()
+    order_id = 'test_prod_{}_{}'.format(side, randint(0, 10 ** 6))
+    api_mock.create_order = MagicMock(return_value={
+        'id': order_id,
+        'info': {
+            'foo': 'bar'
+        },
+        'symbol': 'XRP/USDT',
+        'amount': 1
+    })
+    default_conf['dry_run'] = False
+    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
+    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id='kucoin')
+    exchange._set_leverage = MagicMock()
+    exchange.set_margin_mode = MagicMock()
+
+    order = exchange.create_order(
+        pair='XRP/USDT',
+        ordertype=ordertype,
+        side=side,
+        amount=1,
+        rate=rate,
+        leverage=1.0
+    )
+
+    assert 'id' in order
+    assert 'info' in order
+    assert order['id'] == order_id
+    assert order['amount'] == 1
+    # Status must be faked to open for kucoin.
+    assert order['status'] == 'open'