Merge pull request #8224 from freqtrade/new_release

New release 2023.2

.github/workflows/ci.yml

@@ -90,14 +90,14 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 6 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
+    - name: Flake8
+      run: |
+        flake8
+
     - name: Sort imports (isort)
       run: |
         isort --check .
 
-    - name: Run Ruff
-      run: |
-        ruff check --format=github .
-
     - name: Mypy
       run: |
         mypy freqtrade scripts tests
@@ -186,14 +186,14 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
+    - name: Flake8
+      run: |
+        flake8
+
     - name: Sort imports (isort)
       run: |
         isort --check .
 
-    - name: Run Ruff
-      run: |
-        ruff check --format=github .
-
     - name: Mypy
       run: |
         mypy freqtrade scripts
@@ -248,9 +248,9 @@ jobs:
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
-    - name: Run Ruff
+    - name: Flake8
       run: |
-        ruff check --format=github .
+        flake8
 
     - name: Mypy
       run: |

.pre-commit-config.yaml

@@ -27,12 +27,6 @@ repos:
         name: isort (python)
         # stages: [push]
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: 'v0.0.251'
-    hooks:
-      - id: ruff
-
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:

CONTRIBUTING.md

@@ -45,17 +45,16 @@ pytest tests/test_<file_name>.py::test_<method_name>
 
 ### 2. Test if your code is PEP8 compliant
 
-#### Run Ruff
+#### Run Flake8
 
 ```bash
-ruff .
+flake8 freqtrade tests scripts
 ```
 
-We receive a lot of code that fails the `ruff` checks.
+We receive a lot of code that fails the `flake8` checks.
 To help with that, we encourage you to install the git pre-commit 
-hook that will warn you when you try to commit code that fails these checks.
-
-you can manually run pre-commit with `pre-commit run -a`.
+hook that will warn you when you try to commit code that fails these checks. 
+Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html).
 
 ##### Additional styles applied
 

docs/developer.md

@@ -24,7 +24,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
 Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
 
-This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`.
+This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
 
 Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing.
 This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine.

environment.yml

@@ -41,6 +41,7 @@ dependencies:
     # 2/4 req dev
 
     - coveralls
+    - flake8
     - mypy
     - pytest
     - pytest-asyncio
@@ -69,6 +70,6 @@ dependencies:
         - tables
         - pytest-random-order
         - ccxt
-        - ruff
+        - flake8-tidy-imports
         - -e .
     # - python-rapidjso

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.3.dev'
+__version__ = '2023.2'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/data_commands.py

@@ -5,7 +5,7 @@ from datetime import datetime, timedelta
 from typing import Any, Dict, List
 
 from freqtrade.configuration import TimeRange, setup_utils_configuration
-from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
 from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
@@ -20,24 +20,15 @@ from freqtrade.util.binance_mig import migrate_binance_futures_data
 logger = logging.getLogger(__name__)
 
 
-def _data_download_sanity(config: Config) -> None:
-    if 'days' in config and 'timerange' in config:
-        raise OperationalException("--days and --timerange are mutually exclusive. "
-                                   "You can only specify one or the other.")
-
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
-
-
 def start_download_data(args: Dict[str, Any]) -> None:
     """
     Download data (former download_backtest_data.py script)
     """
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
-    _data_download_sanity(config)
+    if 'days' in config and 'timerange' in config:
+        raise OperationalException("--days and --timerange are mutually exclusive. "
+                                   "You can only specify one or the other.")
     timerange = TimeRange()
     if 'days' in config:
         time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
@@ -49,6 +40,11 @@ def start_download_data(args: Dict[str, Any]) -> None:
     # Remove stake-currency to skip checks which are not relevant for datadownload
     config['stake_currency'] = ''
 
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
     pairs_not_available: List[str] = []
 
     # Init exchange

freqtrade/enums/candletype.py

@@ -13,9 +13,6 @@ class CandleType(str, Enum):
     FUNDING_RATE = "funding_rate"
     # BORROW_RATE = "borrow_rate"  # * unimplemented
 
-    def __str__(self):
-        return f"{self.name.lower()}"
-
     @staticmethod
     def from_string(value: str) -> 'CandleType':
         if not value:

freqtrade/enums/rpcmessagetype.py

@@ -37,8 +37,5 @@ class RPCRequestType(str, Enum):
     WHITELIST = 'whitelist'
     ANALYZED_DF = 'analyzed_df'
 
-    def __str__(self):
-        return self.value
-
 
 NO_ECHO_MESSAGES = (RPCMessageType.ANALYZED_DF, RPCMessageType.WHITELIST, RPCMessageType.NEW_CANDLE)

freqtrade/enums/signaltype.py

@@ -10,9 +10,6 @@ class SignalType(Enum):
     ENTER_SHORT = "enter_short"
     EXIT_SHORT = "exit_short"
 
-    def __str__(self):
-        return f"{self.name.lower()}"
-
 
 class SignalTagType(Enum):
     """
@@ -21,13 +18,7 @@ class SignalTagType(Enum):
     ENTER_TAG = "enter_tag"
     EXIT_TAG = "exit_tag"
 
-    def __str__(self):
-        return f"{self.name.lower()}"
-
 
 class SignalDirection(str, Enum):
     LONG = 'long'
     SHORT = 'short'
-
-    def __str__(self):
-        return f"{self.name.lower()}"

freqtrade/exchange/exchange.py

@@ -1961,8 +1961,7 @@ class Exchange:
                           cache: bool, drop_incomplete: bool) -> DataFrame:
         # keeping last candle time as last refreshed time of the pair
         if ticks and cache:
-            idx = -2 if drop_incomplete and len(ticks) > 1 else -1
-            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[idx][0] // 1000
+            self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
         # keeping parsed dataframe in cache
         ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
                                       drop_incomplete=drop_incomplete)
@@ -2035,9 +2034,7 @@ class Exchange:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
         plr = self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0) + interval_in_sec
-        # current,active candle open date
-        now = int(timeframe_to_prev_date(timeframe).timestamp())
-        return plr < now
+        return plr < arrow.utcnow().int_timestamp
 
     @retrier_async
     async def _async_get_candle_history(

freqtrade/exchange/kucoin.py

@@ -64,7 +64,6 @@ class Kucoin(Exchange):
         # ccxt returns status = 'closed' at the moment - which is information ccxt invented.
         # Since we rely on status heavily, we must set it to 'open' here.
         # ref: https://github.com/ccxt/ccxt/pull/16674, (https://github.com/ccxt/ccxt/pull/16553)
-        if not self._config['dry_run']:
-            res['type'] = ordertype
-            res['status'] = 'open'
+        res['type'] = ordertype
+        res['status'] = 'open'
         return res

freqtrade/freqai/data_drawer.py

@@ -570,12 +570,12 @@ class FreqaiDataDrawer:
 
             for pair in dk.all_pairs:
                 for tf in feat_params.get("include_timeframes"):
-                    hist_df = history_data[pair][tf]
+
                     # check if newest candle is already appended
                     df_dp = strategy.dp.get_pair_dataframe(pair, tf)
                     if len(df_dp.index) == 0:
                         continue
-                    if str(hist_df.iloc[-1]["date"]) == str(
+                    if str(history_data[pair][tf].iloc[-1]["date"]) == str(
                         df_dp.iloc[-1:]["date"].iloc[-1]
                     ):
                         continue
@@ -583,30 +583,21 @@ class FreqaiDataDrawer:
                     try:
                         index = (
                             df_dp.loc[
-                                df_dp["date"] == hist_df.iloc[-1]["date"]
+                                df_dp["date"] == history_data[pair][tf].iloc[-1]["date"]
                             ].index[0]
                             + 1
                         )
                     except IndexError:
-                        if hist_df.iloc[-1]['date'] < df_dp['date'].iloc[0]:
-                            raise OperationalException("In memory historical data is older than "
-                                                       f"oldest DataProvider candle for {pair} on "
-                                                       f"timeframe {tf}")
-                        else:
-                            index = -1
-                            logger.warning(
-                                f"No common dates in historical data and dataprovider for {pair}. "
-                                f"Appending latest dataprovider candle to historical data "
-                                "but please be aware that there is likely a gap in the historical "
-                                "data. \n"
-                                f"Historical data ends at {hist_df.iloc[-1]['date']} "
-                                f"while dataprovider starts at {df_dp['date'].iloc[0]} and"
-                                f"ends at {df_dp['date'].iloc[0]}."
-                            )
+                        logger.warning(
+                            f"Unable to update pair history for {pair}. "
+                            "If this does not resolve itself after 1 additional candle, "
+                            "please report the error to #freqai discord channel"
+                        )
+                        return
 
                     history_data[pair][tf] = pd.concat(
                         [
-                            hist_df,
+                            history_data[pair][tf],
                             df_dp.iloc[index:],
                         ],
                         ignore_index=True,

freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py

@@ -34,11 +34,6 @@ class ReinforcementLearner_multiproc(ReinforcementLearner):
         train_df = data_dictionary["train_features"]
         test_df = data_dictionary["test_features"]
 
-        if self.train_env:
-            self.train_env.close()
-        if self.eval_env:
-            self.eval_env.close()
-
         env_info = self.pack_env_dict(dk.pair)
 
         env_id = "train_env"

pyproject.toml

@@ -56,24 +56,3 @@ exclude = [
   "build_helpers/*.py",
 ]
 ignore = ["freqtrade/vendor/**"]
-
-
-[tool.ruff]
-line-length = 100
-extend-exclude = [".env"]
-target-version = "py38"
-extend-select = [
-  "C90",    # mccabe
-  # "N",    # pep8-naming
-  # "UP",   # pyupgrade
-  "TID",    # flake8-tidy-imports
-  # "EXE",  # flake8-executable
-  "YTT",    # flake8-2020
-  # "DTZ",  # flake8-datetimez
-  # "RSE",  # flake8-raise
-  # "TCH",  # flake8-type-checking
-  # "PTH",  # flake8-use-pathlib
-]
-
-[tool.ruff.mccabe]
-max-complexity = 12

requirements-dev.txt

@@ -7,9 +7,10 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-ruff==0.0.252
+flake8==6.0.0
+flake8-tidy-imports==4.8.0
 mypy==1.0.1
-pre-commit==3.1.0
+pre-commit==3.0.4
 pytest==7.2.1
 pytest-asyncio==0.20.3
 pytest-cov==4.0.0

requirements.txt

@@ -35,7 +35,7 @@ sdnotify==0.3.2
 
 # API Server
 fastapi==0.92.0
-pydantic==1.10.5
+pydantic==1.10.4
 uvicorn==0.20.0
 pyjwt==2.6.0
 aiofiles==23.1.0

setup.py

@@ -32,6 +32,8 @@ hdf5 = [
 
 develop = [
     'coveralls',
+    'flake8',
+    'flake8-tidy-imports',
     'mypy',
     'pytest',
     'pytest-asyncio',

tests/exchange/test_bybit.py

@@ -1,4 +1,3 @@
-from datetime import datetime, timezone
 from unittest.mock import MagicMock
 
 from freqtrade.enums.marginmode import MarginMode
@@ -56,19 +55,3 @@ async def test_bybit_fetch_funding_rate(default_conf, mocker):
     kwargs = api_mock.fetch_funding_rate_history.call_args_list[0][1]
     assert kwargs['params'] == {'until': since_ms_end}
     assert kwargs['since'] == since_ms
-
-
-def test_bybit_get_funding_fees(default_conf, mocker):
-    now = datetime.now(timezone.utc)
-    exchange = get_patched_exchange(mocker, default_conf, id='bybit')
-    exchange._fetch_and_calculate_funding_fees = MagicMock()
-    exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now)
-    assert exchange._fetch_and_calculate_funding_fees.call_count == 0
-
-    default_conf['trading_mode'] = 'futures'
-    default_conf['margin_mode'] = 'isolated'
-    exchange = get_patched_exchange(mocker, default_conf, id='bybit')
-    exchange._fetch_and_calculate_funding_fees = MagicMock()
-    exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now)
-
-    assert exchange._fetch_and_calculate_funding_fees.call_count == 1

tests/exchange/test_exchange.py

@@ -27,7 +27,7 @@ from tests.conftest import (generate_test_data_raw, get_mock_coro, get_patched_e
 
 
 # Make sure to always keep one exchange here which is NOT subclassed!!
-EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit']
+EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate']
 
 get_entry_rate_data = [
     ('other', 20, 19, 10, 0.0, 20),  # Full ask side
@@ -1269,7 +1269,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, fill
                           fetch_l2_order_book=order_book_l2_usd,
                           )
 
-    order = exchange.create_order(
+    order = exchange.create_dry_run_order(
         pair='LTC/USDT',
         ordertype='limit',
         side=side,
@@ -1332,7 +1332,7 @@ def test_create_dry_run_order_market_fill(default_conf, mocker, side, rate, amou
                           fetch_l2_order_book=order_book_l2_usd,
                           )
 
-    order = exchange.create_order(
+    order = exchange.create_dry_run_order(
         pair='LTC/USDT',
         ordertype='market',
         side=side,
@@ -1425,10 +1425,9 @@ def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice,
     assert order['amount'] == 0.01
 
 
-@pytest.mark.parametrize("exchange_name", EXCHANGES)
-def test_buy_dry_run(default_conf, mocker, exchange_name):
+def test_buy_dry_run(default_conf, mocker):
     default_conf['dry_run'] = True
-    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    exchange = get_patched_exchange(mocker, default_conf)
 
     order = exchange.create_order(pair='ETH/BTC', ordertype='limit', side="buy",
                                   amount=1, rate=200, leverage=1.0,
@@ -2216,7 +2215,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert len(res[pair1]) == 99
     assert len(res[pair2]) == 99
     assert exchange._klines
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
     exchange._api_async.fetch_ohlcv.reset_mock()
 
     # Returned from cache
@@ -2225,7 +2224,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert len(res) == 2
     assert len(res[pair1]) == 99
     assert len(res[pair2]) == 99
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
 
     # Move time 1 candle further but result didn't change yet
     time_machine.move_to(start + timedelta(hours=101))
@@ -2235,7 +2234,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert len(res[pair1]) == 99
     assert len(res[pair2]) == 99
     assert res[pair2].at[0, 'open']
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
     refresh_pior = exchange._pairs_last_refresh_time[pair1]
 
     # New candle on exchange - return 100 candles - but skip one candle so we actually get 2 candles
@@ -2253,8 +2252,8 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach
     assert res[pair2].at[0, 'open']
     assert refresh_pior != exchange._pairs_last_refresh_time[pair1]
 
-    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-2][0] // 1000
-    assert exchange._pairs_last_refresh_time[pair2] == ohlcv[-2][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair1] == ohlcv[-1][0] // 1000
+    assert exchange._pairs_last_refresh_time[pair2] == ohlcv[-1][0] // 1000
     exchange._api_async.fetch_ohlcv.reset_mock()
 
     # Retry same call - from cache
@@ -5016,7 +5015,7 @@ def test_get_max_leverage_futures(default_conf, mocker, leverage_tiers):
         exchange.get_max_leverage("BTC/USDT:USDT", 1000000000.01)
 
 
-@pytest.mark.parametrize("exchange_name", ['bittrex', 'binance', 'kraken', 'gate', 'okx', 'bybit'])
+@pytest.mark.parametrize("exchange_name", ['bittrex', 'binance', 'kraken', 'gate', 'okx'])
 def test__get_params(mocker, default_conf, exchange_name):
     api_mock = MagicMock()
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
@@ -5037,9 +5036,6 @@ def test__get_params(mocker, default_conf, exchange_name):
         params2['tdMode'] = 'isolated'
         params2['posSide'] = 'net'
 
-    if exchange_name == 'bybit':
-        params2['position_idx'] = 0
-
     assert exchange._get_params(
         side="buy",
         ordertype='market',

tests/exchange/test_kucoin.py

@@ -125,45 +125,3 @@ def test_stoploss_adjust_kucoin(mocker, default_conf):
     # Test with invalid order case
     order['stopPrice'] = None
     assert exchange.stoploss_adjust(1501, order, 'sell')
-
-
-@pytest.mark.parametrize("side", ["buy", "sell"])
-@pytest.mark.parametrize("ordertype,rate", [
-    ("market", None),
-    ("market", 200),
-    ("limit", 200),
-    ("stop_loss_limit", 200)
-])
-def test_kucoin_create_order(default_conf, mocker, side, ordertype, rate):
-    api_mock = MagicMock()
-    order_id = 'test_prod_{}_{}'.format(side, randint(0, 10 ** 6))
-    api_mock.create_order = MagicMock(return_value={
-        'id': order_id,
-        'info': {
-            'foo': 'bar'
-        },
-        'symbol': 'XRP/USDT',
-        'amount': 1
-    })
-    default_conf['dry_run'] = False
-    mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
-    exchange = get_patched_exchange(mocker, default_conf, api_mock, id='kucoin')
-    exchange._set_leverage = MagicMock()
-    exchange.set_margin_mode = MagicMock()
-
-    order = exchange.create_order(
-        pair='XRP/USDT',
-        ordertype=ordertype,
-        side=side,
-        amount=1,
-        rate=rate,
-        leverage=1.0
-    )
-
-    assert 'id' in order
-    assert 'info' in order
-    assert order['id'] == order_id
-    assert order['amount'] == 1
-    # Status must be faked to open for kucoin.
-    assert order['status'] == 'open'