merge develop into feat/freqai-rl-dev

This commit is contained in:
robcaulk 2022-11-24 18:50:11 +01:00
commit 4894d772ed
17 changed files with 132 additions and 64 deletions

View File

@ -15,9 +15,9 @@ repos:
additional_dependencies: additional_dependencies:
- types-cachetools==5.2.1 - types-cachetools==5.2.1
- types-filelock==3.2.7 - types-filelock==3.2.7
- types-requests==2.28.11.4 - types-requests==2.28.11.5
- types-tabulate==0.9.0.0 - types-tabulate==0.9.0.0
- types-python-dateutil==2.8.19.3 - types-python-dateutil==2.8.19.4
# stages: [push] # stages: [push]
- repo: https://github.com/pycqa/isort - repo: https://github.com/pycqa/isort

View File

@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
### Using proxy with Freqtrade ### Using proxy with Freqtrade
To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values. To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests.
``` bash ``` bash
export HTTP_PROXY="http://addr:port" export HTTP_PROXY="http://addr:port"
@ -672,11 +673,13 @@ export HTTPS_PROXY="http://addr:port"
freqtrade freqtrade
``` ```
#### Proxy just exchange requests #### Proxy exchange requests
To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration. To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
``` json ``` json
{
"exchange": {
"ccxt_config": { "ccxt_config": {
"aiohttp_proxy": "http://addr:port", "aiohttp_proxy": "http://addr:port",
"proxies": { "proxies": {
@ -684,6 +687,7 @@ To use a proxy just for exchange connections (skips/ignores telegram and coingec
"https": "http://addr:port" "https": "http://addr:port"
}, },
} }
}
``` ```
## Next step ## Next step

View File

@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
"name": "default", // This can be any name you'd like, default is "default" "name": "default", // This can be any name you'd like, default is "default"
"host": "127.0.0.1", // The host from your producer's api_server config "host": "127.0.0.1", // The host from your producer's api_server config
"port": 8080, // The port from your producer's api_server config "port": 8080, // The port from your producer's api_server config
"secure": false, // Use a secure websockets connection, default false
"ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config
} }
], ],
@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
| `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string
| `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string | `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string
| `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string | `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string
| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.<br> **Datatype:** string
| `producers.ws_token` | **Required.** `ws_token` as configured on the producer.<br> **Datatype:** string | `producers.ws_token` | **Required.** `ws_token` as configured on the producer.<br> **Datatype:** string
| | **Optional settings** | | **Optional settings**
| `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds. | `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds.

View File

@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th
} }
``` ```
#### Reverse Proxy setup
When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is):
Please make sure to replace `<freqtrade_listen_ip>` (and the subsequent port) with the IP and Port matching your configuration/setup.
```
http {
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
#...
server {
#...
location / {
proxy_http_version 1.1;
proxy_pass http://<freqtrade_listen_ip>:8080;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
}
}
}
```
To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets.
- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/)
- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy)
!!! Tip "SSL certificates"
You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies.
While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel).
### OpenAPI interface ### OpenAPI interface
To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.

View File

@ -512,6 +512,7 @@ CONF_SCHEMA = {
'minimum': 0, 'minimum': 0,
'maximum': 65535 'maximum': 65535
}, },
'secure': {'type': 'boolean', 'default': False},
'ws_token': {'type': 'string'}, 'ws_token': {'type': 'string'},
}, },
'required': ['name', 'host', 'ws_token'] 'required': ['name', 'host', 'ws_token']

View File

@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin):
trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
stoploss_order=True) stoploss_order=True)
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_exit(trade, "stoploss", True) self._notify_exit(trade, "stoploss", True)
self.handle_protections(trade.pair, trade.trade_direction)
return True return True
if trade.open_order_id or not trade.is_open: if trade.open_order_id or not trade.is_open:
@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin):
trade.close_rate_requested = limit trade.close_rate_requested = limit
trade.exit_reason = exit_reason trade.exit_reason = exit_reason
if not sub_trade_amt:
# Lock pair for one candle to prevent immediate re-trading
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
# In case of market sell orders the order can be closed immediately # In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') in ('closed', 'expired'): if order.get('status', 'unknown') in ('closed', 'expired'):
@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin):
self._notify_enter(trade, order, fill=True, sub_trade=sub_trade) self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
def handle_protections(self, pair: str, side: LongShort) -> None: def handle_protections(self, pair: str, side: LongShort) -> None:
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock')
prot_trig = self.protections.stop_per_pair(pair, side=side) prot_trig = self.protections.stop_per_pair(pair, side=side)
if prot_trig: if prot_trig:
msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }

View File

@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union
from typing.io import IO from typing.io import IO
from urllib.parse import urlparse from urllib.parse import urlparse
import pandas import orjson
import pandas as pd
import rapidjson import rapidjson
from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@ -256,29 +257,37 @@ def parse_db_uri_for_logging(uri: str):
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
def dataframe_to_json(dataframe: pandas.DataFrame) -> str: def dataframe_to_json(dataframe: pd.DataFrame) -> str:
""" """
Serialize a DataFrame for transmission over the wire using JSON Serialize a DataFrame for transmission over the wire using JSON
:param dataframe: A pandas DataFrame :param dataframe: A pandas DataFrame
:returns: A JSON string of the pandas DataFrame :returns: A JSON string of the pandas DataFrame
""" """
return dataframe.to_json(orient='split') # https://github.com/pandas-dev/pandas/issues/24889
# https://github.com/pandas-dev/pandas/issues/40443
# We need to convert to a dict to avoid mem leak
def default(z):
if isinstance(z, pd.Timestamp):
return z.timestamp() * 1e3
raise TypeError
return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
def json_to_dataframe(data: str) -> pandas.DataFrame: def json_to_dataframe(data: str) -> pd.DataFrame:
""" """
Deserialize JSON into a DataFrame Deserialize JSON into a DataFrame
:param data: A JSON string :param data: A JSON string
:returns: A pandas DataFrame from the JSON string :returns: A pandas DataFrame from the JSON string
""" """
dataframe = pandas.read_json(data, orient='split') dataframe = pd.read_json(data, orient='split')
if 'date' in dataframe.columns: if 'date' in dataframe.columns:
dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
return dataframe return dataframe
def remove_entry_exit_signals(dataframe: pandas.DataFrame): def remove_entry_exit_signals(dataframe: pd.DataFrame):
""" """
Remove Entry and Exit signals from a DataFrame Remove Entry and Exit signals from a DataFrame

View File

@ -194,6 +194,9 @@ class ApiServer(RPCHandler):
try: try:
while True: while True:
logger.debug("Getting queue messages...") logger.debug("Getting queue messages...")
if (qsize := async_queue.qsize()) > 20:
# If the queue becomes too big for too long, this may indicate a problem.
logger.warning(f"Queue size now {qsize}")
# Get data from queue # Get data from queue
message: WSMessageSchemaType = await async_queue.get() message: WSMessageSchemaType = await async_queue.get()
logger.debug(f"Found message of type: {message.get('type')}") logger.debug(f"Found message of type: {message.get('type')}")

View File

@ -77,6 +77,7 @@ class WebSocketChannel:
# until self.drain_timeout for the relay to drain the outgoing queue # until self.drain_timeout for the relay to drain the outgoing queue
# We can't use asyncio.wait_for here because the queue may have been created with a # We can't use asyncio.wait_for here because the queue may have been created with a
# different eventloop # different eventloop
if not self.is_closed():
start = time.time() start = time.time()
while self.queue.full(): while self.queue.full():
await asyncio.sleep(1) await asyncio.sleep(1)
@ -91,6 +92,8 @@ class WebSocketChannel:
# If we got here everything is ok # If we got here everything is ok
return True return True
else:
return False
async def recv(self): async def recv(self):
""" """
@ -109,14 +112,14 @@ class WebSocketChannel:
Close the WebSocketChannel Close the WebSocketChannel
""" """
self._closed.set()
self._relay_task.cancel()
try: try:
await self.raw_websocket.close() await self.raw_websocket.close()
except Exception: except Exception:
pass pass
self._closed.set()
self._relay_task.cancel()
def is_closed(self) -> bool: def is_closed(self) -> bool:
""" """
Closed flag Closed flag

View File

@ -31,6 +31,7 @@ class Producer(TypedDict):
name: str name: str
host: str host: str
port: int port: int
secure: bool
ws_token: str ws_token: str
@ -180,7 +181,8 @@ class ExternalMessageConsumer:
host, port = producer['host'], producer['port'] host, port = producer['host'], producer['port']
token = producer['ws_token'] token = producer['ws_token']
name = producer['name'] name = producer['name']
ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" scheme = 'wss' if producer.get('secure', False) else 'ws'
ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
# This will raise InvalidURI if the url is bad # This will raise InvalidURI if the url is bad
async with websockets.connect( async with websockets.connect(

View File

@ -774,6 +774,9 @@ class RPC:
is_short = trade.is_short is_short = trade.is_short
if not self._freqtrade.strategy.position_adjustment_enable: if not self._freqtrade.strategy.position_adjustment_enable:
raise RPCException(f'position for {pair} already open - id: {trade.id}') raise RPCException(f'position for {pair} already open - id: {trade.id}')
if trade.open_order_id is not None:
raise RPCException(f'position for {pair} already open - id: {trade.id} '
f'and has open order {trade.open_order_id}')
else: else:
if Trade.get_open_trade_count() >= self._config['max_open_trades']: if Trade.get_open_trade_count() >= self._config['max_open_trades']:
raise RPCException("Maximum number of trades is reached.") raise RPCException("Maximum number of trades is reached.")

View File

@ -9,7 +9,7 @@
coveralls==3.3.1 coveralls==3.3.1
flake8==5.0.4 flake8==5.0.4
flake8-tidy-imports==4.8.0 flake8-tidy-imports==4.8.0
mypy==0.990 mypy==0.991
pre-commit==2.20.0 pre-commit==2.20.0
pytest==7.2.0 pytest==7.2.0
pytest-asyncio==0.20.2 pytest-asyncio==0.20.2
@ -20,14 +20,14 @@ isort==5.10.1
# For datetime mocking # For datetime mocking
time-machine==2.8.2 time-machine==2.8.2
# fastapi testing # fastapi testing
httpx==0.23.0 httpx==0.23.1
# Convert jupyter notebooks to markdown documents # Convert jupyter notebooks to markdown documents
nbconvert==7.2.4 nbconvert==7.2.5
# mypy types # mypy types
types-cachetools==5.2.1 types-cachetools==5.2.1
types-filelock==3.2.7 types-filelock==3.2.7
types-requests==2.28.11.4 types-requests==2.28.11.5
types-tabulate==0.9.0.0 types-tabulate==0.9.0.0
types-python-dateutil==2.8.19.3 types-python-dateutil==2.8.19.4

View File

@ -1,8 +1,8 @@
numpy==1.23.4 numpy==1.23.5
pandas==1.5.1 pandas==1.5.1
pandas-ta==0.3.14b pandas-ta==0.3.14b
ccxt==2.1.75 ccxt==2.1.96
# Pin cryptography for now due to rust build errors with piwheels # Pin cryptography for now due to rust build errors with piwheels
cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.1; platform_machine == 'armv7l'
cryptography==38.0.3; platform_machine != 'armv7l' cryptography==38.0.3; platform_machine != 'armv7l'
@ -30,7 +30,7 @@ py_find_1st==1.1.5
# Load ticker files 30% faster # Load ticker files 30% faster
python-rapidjson==1.9 python-rapidjson==1.9
# Properly format api responses # Properly format api responses
orjson==3.8.1 orjson==3.8.2
# Notify systemd # Notify systemd
sdnotify==0.3.2 sdnotify==0.3.2
@ -38,7 +38,7 @@ sdnotify==0.3.2
# API Server # API Server
fastapi==0.87.0 fastapi==0.87.0
pydantic==1.10.2 pydantic==1.10.2
uvicorn==0.19.0 uvicorn==0.20.0
pyjwt==2.6.0 pyjwt==2.6.0
aiofiles==22.1.0 aiofiles==22.1.0
psutil==5.9.4 psutil==5.9.4

View File

@ -199,6 +199,7 @@ async def create_client(
host, host,
port, port,
token, token,
scheme='ws',
name='default', name='default',
protocol=ClientProtocol(), protocol=ClientProtocol(),
sleep_time=10, sleep_time=10,
@ -211,13 +212,14 @@ async def create_client(
:param host: The host :param host: The host
:param port: The port :param port: The port
:param token: The websocket auth token :param token: The websocket auth token
:param scheme: `ws` for most connections, `wss` for ssl
:param name: The name of the producer :param name: The name of the producer
:param **kwargs: Any extra kwargs passed to websockets.connect :param **kwargs: Any extra kwargs passed to websockets.connect
""" """
while 1: while 1:
try: try:
websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
logger.info(f"Attempting to connect to {name} @ {host}:{port}") logger.info(f"Attempting to connect to {name} @ {host}:{port}")
async with websockets.connect(websocket_url, **kwargs) as ws: async with websockets.connect(websocket_url, **kwargs) as ws:
@ -304,6 +306,7 @@ async def _main(args):
producer['host'], producer['host'],
producer['port'], producer['port'],
producer['ws_token'], producer['ws_token'],
'wss' if producer.get('secure', False) else 'ws',
producer['name'], producer['name'],
sleep_time=sleep_time, sleep_time=sleep_time,
ping_timeout=ping_timeout, ping_timeout=ping_timeout,

View File

@ -83,7 +83,7 @@ function updateenv() {
dev=$REPLY dev=$REPLY
if [[ $REPLY =~ ^[Yy]$ ]] if [[ $REPLY =~ ^[Yy]$ ]]
then then
REQUIREMENTS_FREQAI="-r requirements-freqai.txt" REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517"
read -p "Do you also want dependencies for freqai-rl (~700mb additional space required) [y/N]? " read -p "Do you also want dependencies for freqai-rl (~700mb additional space required) [y/N]? "
dev=$REPLY dev=$REPLY
if [[ $REPLY =~ ^[Yy]$ ]] if [[ $REPLY =~ ^[Yy]$ ]]

View File

@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees(
assert order1['fee']['rate'] == fee assert order1['fee']['rate'] == fee
@pytest.mark.parametrize("side,startprice,endprice", [ @pytest.mark.parametrize("side,price,filled", [
("buy", 25.563, 25.566), # order_book_l2_usd spread:
("sell", 25.566, 25.563) # best ask: 25.566
# best bid: 25.563
("buy", 25.563, False),
("buy", 25.566, True),
("sell", 25.566, False),
("sell", 25.563, True),
]) ])
@pytest.mark.parametrize("exchange_name", EXCHANGES) @pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled,
exchange_name, order_book_l2_usd): exchange_name, order_book_l2_usd):
default_conf['dry_run'] = True default_conf['dry_run'] = True
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
ordertype='limit', ordertype='limit',
side=side, side=side,
amount=1, amount=1,
rate=startprice, rate=price,
leverage=1.0 leverage=1.0
) )
assert order_book_l2_usd.call_count == 1 assert order_book_l2_usd.call_count == 1
@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
assert order["side"] == side assert order["side"] == side
assert order["type"] == "limit" assert order["type"] == "limit"
assert order["symbol"] == "LTC/USDT" assert order["symbol"] == "LTC/USDT"
assert order["average"] == price
assert order['status'] == 'open' if not filled else 'closed'
order_book_l2_usd.reset_mock() order_book_l2_usd.reset_mock()
# fetch order again...
order_closed = exchange.fetch_dry_run_order(order['id']) order_closed = exchange.fetch_dry_run_order(order['id'])
assert order_book_l2_usd.call_count == 1 assert order_book_l2_usd.call_count == (1 if not filled else 0)
assert order_closed['status'] == 'open' assert order_closed['status'] == ('open' if not filled else 'closed')
assert not order['fee'] assert order_closed['filled'] == (0 if not filled else 1)
assert order_closed['filled'] == 0
order_book_l2_usd.reset_mock() order_book_l2_usd.reset_mock()
order_closed['price'] = endprice
order_closed = exchange.fetch_dry_run_order(order['id'])
assert order_closed['status'] == 'closed'
assert order['fee']
assert order_closed['filled'] == 1
assert order_closed['filled'] == order_closed['amount']
# Empty orderbook test # Empty orderbook test
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',

View File

@ -1066,6 +1066,11 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
assert trade.stake_amount == 0.05 assert trade.stake_amount == 0.05
assert trade.buy_tag == 'force_entry' assert trade.buy_tag == 'force_entry'
assert trade.open_order_id == 'mocked_limit_buy'
freqtradebot.strategy.position_adjustment_enable = True
with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'):
rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
# Test not buying # Test not buying
pair = 'XRP/BTC' pair = 'XRP/BTC'