diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4a8336b9..ccf9d5098 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,9 +15,9 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.11.4 + - types-requests==2.28.11.5 - types-tabulate==0.9.0.0 - - types-python-dateutil==2.8.19.3 + - types-python-dateutil==2.8.19.4 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/docs/configuration.md b/docs/configuration.md index ce4453561..83b23425c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Using proxy with Freqtrade To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values. +This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests. ``` bash export HTTP_PROXY="http://addr:port" @@ -672,17 +673,20 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` -#### Proxy just exchange requests +#### Proxy exchange requests -To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration. +To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration. ``` json -"ccxt_config": { +{ + "exchange": { + "ccxt_config": { "aiohttp_proxy": "http://addr:port", "proxies": { - "http": "http://addr:port", - "https": "http://addr:port" + "http": "http://addr:port", + "https": "http://addr:port" }, + } } ``` diff --git a/docs/producer-consumer.md b/docs/producer-consumer.md index b69406edf..88e34d0d6 100644 --- a/docs/producer-consumer.md +++ b/docs/producer-consumer.md @@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect "name": "default", // This can be any name you'd like, default is "default" "host": "127.0.0.1", // The host from your producer's api_server config "port": 8080, // The port from your producer's api_server config + "secure": false, // Use a secure websockets connection, default false "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config } ], @@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.
**Datatype:** string | `producers.host` | **Required.** The hostname or IP address from your producer.
**Datatype:** string | `producers.port` | **Required.** The port matching the above host.
**Datatype:** string +| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.
**Datatype:** string | `producers.ws_token` | **Required.** `ws_token` as configured on the producer.
**Datatype:** string | | **Optional settings** | `wait_timeout` | Timeout until we ping again if no message is received.
*Defaults to `300`.*
**Datatype:** Integer - in seconds. diff --git a/docs/rest-api.md b/docs/rest-api.md index c7d762648..62ad586dd 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` +#### Reverse Proxy setup + +When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is): + +Please make sure to replace `` (and the subsequent port) with the IP and Port matching your configuration/setup. + +``` +http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + #... + + server { + #... + + location / { + proxy_http_version 1.1; + proxy_pass http://:8080; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + } + } +} +``` + +To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets. + +- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) +- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) + +!!! Tip "SSL certificates" + You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies. + While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel). + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 534d06fd4..cfac98ebd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -512,6 +512,7 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 65535 }, + 'secure': {'type': 'boolean', 'default': False}, 'ws_token': {'type': 'string'}, }, 'required': ['name', 'host', 'ws_token'] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2e2638126..77b099d80 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin): trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, stoploss_order=True) - # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') self._notify_exit(trade, "stoploss", True) + self.handle_protections(trade.pair, trade.trade_direction) return True if trade.open_order_id or not trade.is_open: @@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin): trade.close_rate_requested = limit trade.exit_reason = exit_reason - if not sub_trade_amt: - # Lock pair for one candle to prevent immediate re-trading - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') - self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): @@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin): self._notify_enter(trade, order, fill=True, sub_trade=sub_trade) def handle_protections(self, pair: str, side: LongShort) -> None: + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock') prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index b978407e4..6078efd07 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -31,6 +31,7 @@ class Producer(TypedDict): name: str host: str port: int + secure: bool ws_token: str @@ -180,7 +181,8 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + scheme = 'wss' if producer.get('secure', False) else 'ws' + ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad async with websockets.connect( diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..ca76e5aee 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 flake8==5.0.4 flake8-tidy-imports==4.8.0 -mypy==0.990 +mypy==0.991 pre-commit==2.20.0 pytest==7.2.0 pytest-asyncio==0.20.2 @@ -19,14 +19,14 @@ isort==5.10.1 # For datetime mocking time-machine==2.8.2 # fastapi testing -httpx==0.23.0 +httpx==0.23.1 # Convert jupyter notebooks to markdown documents -nbconvert==7.2.4 +nbconvert==7.2.5 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.11.4 +types-requests==2.28.11.5 types-tabulate==0.9.0.0 -types-python-dateutil==2.8.19.3 +types-python-dateutil==2.8.19.4 diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..a9555b90c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -numpy==1.23.4 +numpy==1.23.5 pandas==1.5.1 pandas-ta==0.3.14b -ccxt==2.1.75 +ccxt==2.1.96 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.3; platform_machine != 'armv7l' @@ -30,7 +30,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.1 +orjson==3.8.2 # Notify systemd sdnotify==0.3.2 @@ -38,7 +38,7 @@ sdnotify==0.3.2 # API Server fastapi==0.87.0 pydantic==1.10.2 -uvicorn==0.19.0 +uvicorn==0.20.0 pyjwt==2.6.0 aiofiles==22.1.0 psutil==5.9.4 diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..5d27f512e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -199,6 +199,7 @@ async def create_client( host, port, token, + scheme='ws', name='default', protocol=ClientProtocol(), sleep_time=10, @@ -211,13 +212,14 @@ async def create_client( :param host: The host :param port: The port :param token: The websocket auth token + :param scheme: `ws` for most connections, `wss` for ssl :param name: The name of the producer :param **kwargs: Any extra kwargs passed to websockets.connect """ while 1: try: - websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" logger.info(f"Attempting to connect to {name} @ {host}:{port}") async with websockets.connect(websocket_url, **kwargs) as ws: @@ -304,6 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], + 'wss' if producer.get('secure', False) else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, diff --git a/setup.sh b/setup.sh index 1a4a285a3..fceab0074 100755 --- a/setup.sh +++ b/setup.sh @@ -82,7 +82,7 @@ function updateenv() { dev=$REPLY if [[ $REPLY =~ ^[Yy]$ ]] then - REQUIREMENTS_FREQAI="-r requirements-freqai.txt" + REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517" fi ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI} diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a719496e5..e61ad8532 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees( assert order1['fee']['rate'] == fee -@pytest.mark.parametrize("side,startprice,endprice", [ - ("buy", 25.563, 25.566), - ("sell", 25.566, 25.563) +@pytest.mark.parametrize("side,price,filled", [ + # order_book_l2_usd spread: + # best ask: 25.566 + # best bid: 25.563 + ("buy", 25.563, False), + ("buy", 25.566, True), + ("sell", 25.566, False), + ("sell", 25.563, True), ]) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled, exchange_name, order_book_l2_usd): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, ordertype='limit', side=side, amount=1, - rate=startprice, + rate=price, leverage=1.0 ) assert order_book_l2_usd.call_count == 1 @@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order["side"] == side assert order["type"] == "limit" assert order["symbol"] == "LTC/USDT" + assert order["average"] == price + assert order['status'] == 'open' if not filled else 'closed' order_book_l2_usd.reset_mock() + # fetch order again... order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_book_l2_usd.call_count == 1 - assert order_closed['status'] == 'open' - assert not order['fee'] - assert order_closed['filled'] == 0 + assert order_book_l2_usd.call_count == (1 if not filled else 0) + assert order_closed['status'] == ('open' if not filled else 'closed') + assert order_closed['filled'] == (0 if not filled else 1) order_book_l2_usd.reset_mock() - order_closed['price'] = endprice - - order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_closed['status'] == 'closed' - assert order['fee'] - assert order_closed['filled'] == 1 - assert order_closed['filled'] == order_closed['amount'] # Empty orderbook test mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',