diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4a8336b9..ccf9d5098 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,9 +15,9 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.11.4 + - types-requests==2.28.11.5 - types-tabulate==0.9.0.0 - - types-python-dateutil==2.8.19.3 + - types-python-dateutil==2.8.19.4 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/docs/configuration.md b/docs/configuration.md index ce4453561..83b23425c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Using proxy with Freqtrade To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values. +This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests. ``` bash export HTTP_PROXY="http://addr:port" @@ -672,17 +673,20 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` -#### Proxy just exchange requests +#### Proxy exchange requests -To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration. +To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration. ``` json -"ccxt_config": { +{ + "exchange": { + "ccxt_config": { "aiohttp_proxy": "http://addr:port", "proxies": { - "http": "http://addr:port", - "https": "http://addr:port" + "http": "http://addr:port", + "https": "http://addr:port" }, + } } ``` diff --git a/docs/producer-consumer.md b/docs/producer-consumer.md index b69406edf..88e34d0d6 100644 --- a/docs/producer-consumer.md +++ b/docs/producer-consumer.md @@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect "name": "default", // This can be any name you'd like, default is "default" "host": "127.0.0.1", // The host from your producer's api_server config "port": 8080, // The port from your producer's api_server config + "secure": false, // Use a secure websockets connection, default false "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config } ], @@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.
**Datatype:** string | `producers.host` | **Required.** The hostname or IP address from your producer.
**Datatype:** string | `producers.port` | **Required.** The port matching the above host.
**Datatype:** string +| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.
**Datatype:** string | `producers.ws_token` | **Required.** `ws_token` as configured on the producer.
**Datatype:** string | | **Optional settings** | `wait_timeout` | Timeout until we ping again if no message is received.
*Defaults to `300`.*
**Datatype:** Integer - in seconds. diff --git a/docs/rest-api.md b/docs/rest-api.md index c7d762648..62ad586dd 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` +#### Reverse Proxy setup + +When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is): + +Please make sure to replace `` (and the subsequent port) with the IP and Port matching your configuration/setup. + +``` +http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + #... + + server { + #... + + location / { + proxy_http_version 1.1; + proxy_pass http://:8080; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + } + } +} +``` + +To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets. + +- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) +- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) + +!!! Tip "SSL certificates" + You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies. + While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel). + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 6cbc1f36e..ba43e1328 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -512,6 +512,7 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 65535 }, + 'secure': {'type': 'boolean', 'default': False}, 'ws_token': {'type': 'string'}, }, 'required': ['name', 'host', 'ws_token'] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2e2638126..77b099d80 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin): trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, stoploss_order=True) - # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') self._notify_exit(trade, "stoploss", True) + self.handle_protections(trade.pair, trade.trade_direction) return True if trade.open_order_id or not trade.is_open: @@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin): trade.close_rate_requested = limit trade.exit_reason = exit_reason - if not sub_trade_amt: - # Lock pair for one candle to prevent immediate re-trading - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') - self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): @@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin): self._notify_enter(trade, order, fill=True, sub_trade=sub_trade) def handle_protections(self, pair: str, side: LongShort) -> None: + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock') prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 49d33d46f..2d2c7513a 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union from typing.io import IO from urllib.parse import urlparse -import pandas +import orjson +import pandas as pd import rapidjson from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN @@ -256,29 +257,37 @@ def parse_db_uri_for_logging(uri: str): return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') -def dataframe_to_json(dataframe: pandas.DataFrame) -> str: +def dataframe_to_json(dataframe: pd.DataFrame) -> str: """ Serialize a DataFrame for transmission over the wire using JSON :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - return dataframe.to_json(orient='split') + # https://github.com/pandas-dev/pandas/issues/24889 + # https://github.com/pandas-dev/pandas/issues/40443 + # We need to convert to a dict to avoid mem leak + def default(z): + if isinstance(z, pd.Timestamp): + return z.timestamp() * 1e3 + raise TypeError + + return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8') -def json_to_dataframe(data: str) -> pandas.DataFrame: +def json_to_dataframe(data: str) -> pd.DataFrame: """ Deserialize JSON into a DataFrame :param data: A JSON string :returns: A pandas DataFrame from the JSON string """ - dataframe = pandas.read_json(data, orient='split') + dataframe = pd.read_json(data, orient='split') if 'date' in dataframe.columns: - dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) + dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True) return dataframe -def remove_entry_exit_signals(dataframe: pandas.DataFrame): +def remove_entry_exit_signals(dataframe: pd.DataFrame): """ Remove Entry and Exit signals from a DataFrame diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 6464ae44e..ec4907e67 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -194,6 +194,9 @@ class ApiServer(RPCHandler): try: while True: logger.debug("Getting queue messages...") + if (qsize := async_queue.qsize()) > 20: + # If the queue becomes too big for too long, this may indicate a problem. + logger.warning(f"Queue size now {qsize}") # Get data from queue message: WSMessageSchemaType = await async_queue.get() logger.debug(f"Found message of type: {message.get('type')}") diff --git a/freqtrade/rpc/api_server/ws/channel.py b/freqtrade/rpc/api_server/ws/channel.py index 88b4db9ba..4eef738d4 100644 --- a/freqtrade/rpc/api_server/ws/channel.py +++ b/freqtrade/rpc/api_server/ws/channel.py @@ -77,21 +77,24 @@ class WebSocketChannel: # until self.drain_timeout for the relay to drain the outgoing queue # We can't use asyncio.wait_for here because the queue may have been created with a # different eventloop - start = time.time() - while self.queue.full(): - await asyncio.sleep(1) - if (time.time() - start) > self.drain_timeout: + if not self.is_closed(): + start = time.time() + while self.queue.full(): + await asyncio.sleep(1) + if (time.time() - start) > self.drain_timeout: + return False + + # If for some reason the queue is still full, just return False + try: + self.queue.put_nowait(data) + except asyncio.QueueFull: return False - # If for some reason the queue is still full, just return False - try: - self.queue.put_nowait(data) - except asyncio.QueueFull: + # If we got here everything is ok + return True + else: return False - # If we got here everything is ok - return True - async def recv(self): """ Receive data on the wrapped websocket @@ -109,14 +112,14 @@ class WebSocketChannel: Close the WebSocketChannel """ + self._closed.set() + self._relay_task.cancel() + try: await self.raw_websocket.close() except Exception: pass - self._closed.set() - self._relay_task.cancel() - def is_closed(self) -> bool: """ Closed flag diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index b978407e4..6078efd07 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -31,6 +31,7 @@ class Producer(TypedDict): name: str host: str port: int + secure: bool ws_token: str @@ -180,7 +181,8 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + scheme = 'wss' if producer.get('secure', False) else 'ws' + ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad async with websockets.connect( diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a0824bcc1..1d3f36844 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -774,6 +774,9 @@ class RPC: is_short = trade.is_short if not self._freqtrade.strategy.position_adjustment_enable: raise RPCException(f'position for {pair} already open - id: {trade.id}') + if trade.open_order_id is not None: + raise RPCException(f'position for {pair} already open - id: {trade.id} ' + f'and has open order {trade.open_order_id}') else: if Trade.get_open_trade_count() >= self._config['max_open_trades']: raise RPCException("Maximum number of trades is reached.") diff --git a/requirements-dev.txt b/requirements-dev.txt index 0dfe74a36..b46c244b5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,7 +9,7 @@ coveralls==3.3.1 flake8==5.0.4 flake8-tidy-imports==4.8.0 -mypy==0.990 +mypy==0.991 pre-commit==2.20.0 pytest==7.2.0 pytest-asyncio==0.20.2 @@ -20,14 +20,14 @@ isort==5.10.1 # For datetime mocking time-machine==2.8.2 # fastapi testing -httpx==0.23.0 +httpx==0.23.1 # Convert jupyter notebooks to markdown documents -nbconvert==7.2.4 +nbconvert==7.2.5 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.11.4 +types-requests==2.28.11.5 types-tabulate==0.9.0.0 -types-python-dateutil==2.8.19.3 +types-python-dateutil==2.8.19.4 diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..a9555b90c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -numpy==1.23.4 +numpy==1.23.5 pandas==1.5.1 pandas-ta==0.3.14b -ccxt==2.1.75 +ccxt==2.1.96 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.3; platform_machine != 'armv7l' @@ -30,7 +30,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.1 +orjson==3.8.2 # Notify systemd sdnotify==0.3.2 @@ -38,7 +38,7 @@ sdnotify==0.3.2 # API Server fastapi==0.87.0 pydantic==1.10.2 -uvicorn==0.19.0 +uvicorn==0.20.0 pyjwt==2.6.0 aiofiles==22.1.0 psutil==5.9.4 diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..5d27f512e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -199,6 +199,7 @@ async def create_client( host, port, token, + scheme='ws', name='default', protocol=ClientProtocol(), sleep_time=10, @@ -211,13 +212,14 @@ async def create_client( :param host: The host :param port: The port :param token: The websocket auth token + :param scheme: `ws` for most connections, `wss` for ssl :param name: The name of the producer :param **kwargs: Any extra kwargs passed to websockets.connect """ while 1: try: - websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" logger.info(f"Attempting to connect to {name} @ {host}:{port}") async with websockets.connect(websocket_url, **kwargs) as ws: @@ -304,6 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], + 'wss' if producer.get('secure', False) else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, diff --git a/setup.sh b/setup.sh index f57e820af..4cb504853 100755 --- a/setup.sh +++ b/setup.sh @@ -83,7 +83,7 @@ function updateenv() { dev=$REPLY if [[ $REPLY =~ ^[Yy]$ ]] then - REQUIREMENTS_FREQAI="-r requirements-freqai.txt" + REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517" read -p "Do you also want dependencies for freqai-rl (~700mb additional space required) [y/N]? " dev=$REPLY if [[ $REPLY =~ ^[Yy]$ ]] diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a719496e5..e61ad8532 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees( assert order1['fee']['rate'] == fee -@pytest.mark.parametrize("side,startprice,endprice", [ - ("buy", 25.563, 25.566), - ("sell", 25.566, 25.563) +@pytest.mark.parametrize("side,price,filled", [ + # order_book_l2_usd spread: + # best ask: 25.566 + # best bid: 25.563 + ("buy", 25.563, False), + ("buy", 25.566, True), + ("sell", 25.566, False), + ("sell", 25.563, True), ]) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled, exchange_name, order_book_l2_usd): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, ordertype='limit', side=side, amount=1, - rate=startprice, + rate=price, leverage=1.0 ) assert order_book_l2_usd.call_count == 1 @@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order["side"] == side assert order["type"] == "limit" assert order["symbol"] == "LTC/USDT" + assert order["average"] == price + assert order['status'] == 'open' if not filled else 'closed' order_book_l2_usd.reset_mock() + # fetch order again... order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_book_l2_usd.call_count == 1 - assert order_closed['status'] == 'open' - assert not order['fee'] - assert order_closed['filled'] == 0 + assert order_book_l2_usd.call_count == (1 if not filled else 0) + assert order_closed['status'] == ('open' if not filled else 'closed') + assert order_closed['filled'] == (0 if not filled else 1) order_book_l2_usd.reset_mock() - order_closed['price'] = endprice - - order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_closed['status'] == 'closed' - assert order['fee'] - assert order_closed['filled'] == 1 - assert order_closed['filled'] == order_closed['amount'] # Empty orderbook test mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index ef6c8b204..8828b6f33 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1066,6 +1066,11 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) assert trade.stake_amount == 0.05 assert trade.buy_tag == 'force_entry' + assert trade.open_order_id == 'mocked_limit_buy' + + freqtradebot.strategy.position_adjustment_enable = True + with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'): + rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) # Test not buying pair = 'XRP/BTC'