merge develop into feat/freqai-rl-dev
This commit is contained in:
robcaulk
commit
4894d772ed
|
|
@ -15,9 +15,9 @@ repos:
|
|||
additional_dependencies:
|
||||
- types-cachetools==5.2.1
|
||||
- types-filelock==3.2.7
|
||||
- types-requests==2.28.11.4
|
||||
- types-requests==2.28.11.5
|
||||
- types-tabulate==0.9.0.0
|
||||
- types-python-dateutil==2.8.19.3
|
||||
- types-python-dateutil==2.8.19.4
|
||||
# stages: [push]
|
||||
|
||||
- repo: https://github.com/pycqa/isort
|
||||
|
|
|
|||
|
|
@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
|
|||
### Using proxy with Freqtrade
|
||||
|
||||
To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
|
||||
This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests.
|
||||
|
||||
``` bash
|
||||
export HTTP_PROXY="http://addr:port"
|
||||
|
|
@ -672,17 +673,20 @@ export HTTPS_PROXY="http://addr:port"
|
|||
freqtrade
|
||||
```
|
||||
|
||||
#### Proxy just exchange requests
|
||||
#### Proxy exchange requests
|
||||
|
||||
To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration.
|
||||
To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
|
||||
|
||||
``` json
|
||||
"ccxt_config": {
|
||||
{
|
||||
"exchange": {
|
||||
"ccxt_config": {
|
||||
"aiohttp_proxy": "http://addr:port",
|
||||
"proxies": {
|
||||
"http": "http://addr:port",
|
||||
"https": "http://addr:port"
|
||||
"http": "http://addr:port",
|
||||
"https": "http://addr:port"
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
|
|||
"name": "default", // This can be any name you'd like, default is "default"
|
||||
"host": "127.0.0.1", // The host from your producer's api_server config
|
||||
"port": 8080, // The port from your producer's api_server config
|
||||
"secure": false, // Use a secure websockets connection, default false
|
||||
"ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config
|
||||
}
|
||||
],
|
||||
|
|
@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
|
|||
| `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string
|
||||
| `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string
|
||||
| `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string
|
||||
| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.<br> **Datatype:** string
|
||||
| `producers.ws_token` | **Required.** `ws_token` as configured on the producer.<br> **Datatype:** string
|
||||
| | **Optional settings**
|
||||
| `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds.
|
||||
|
|
|
|||
|
|
@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th
|
|||
}
|
||||
```
|
||||
|
||||
#### Reverse Proxy setup
|
||||
|
||||
When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is):
|
||||
|
||||
Please make sure to replace `<freqtrade_listen_ip>` (and the subsequent port) with the IP and Port matching your configuration/setup.
|
||||
|
||||
```
|
||||
http {
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
#...
|
||||
|
||||
server {
|
||||
#...
|
||||
|
||||
location / {
|
||||
proxy_http_version 1.1;
|
||||
proxy_pass http://<freqtrade_listen_ip>:8080;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets.
|
||||
|
||||
- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/)
|
||||
- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy)
|
||||
|
||||
!!! Tip "SSL certificates"
|
||||
You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies.
|
||||
While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel).
|
||||
|
||||
### OpenAPI interface
|
||||
|
||||
To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.
|
||||
|
|
|
|||
|
|
@ -512,6 +512,7 @@ CONF_SCHEMA = {
|
|||
'minimum': 0,
|
||||
'maximum': 65535
|
||||
},
|
||||
'secure': {'type': 'boolean', 'default': False},
|
||||
'ws_token': {'type': 'string'},
|
||||
},
|
||||
'required': ['name', 'host', 'ws_token']
|
||||
|
|
|
|||
|
|
@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin):
|
|||
trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
|
||||
self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
|
||||
stoploss_order=True)
|
||||
# Lock pair for one candle to prevent immediate rebuys
|
||||
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
|
||||
reason='Auto lock')
|
||||
self._notify_exit(trade, "stoploss", True)
|
||||
self.handle_protections(trade.pair, trade.trade_direction)
|
||||
return True
|
||||
|
||||
if trade.open_order_id or not trade.is_open:
|
||||
|
|
@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin):
|
|||
trade.close_rate_requested = limit
|
||||
trade.exit_reason = exit_reason
|
||||
|
||||
if not sub_trade_amt:
|
||||
# Lock pair for one candle to prevent immediate re-trading
|
||||
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
|
||||
reason='Auto lock')
|
||||
|
||||
self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
|
||||
# In case of market sell orders the order can be closed immediately
|
||||
if order.get('status', 'unknown') in ('closed', 'expired'):
|
||||
|
|
@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin):
|
|||
self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
|
||||
|
||||
def handle_protections(self, pair: str, side: LongShort) -> None:
|
||||
# Lock pair for one candle to prevent immediate rebuys
|
||||
self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock')
|
||||
prot_trig = self.protections.stop_per_pair(pair, side=side)
|
||||
if prot_trig:
|
||||
msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union
|
|||
from typing.io import IO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pandas
|
||||
import orjson
|
||||
import pandas as pd
|
||||
import rapidjson
|
||||
|
||||
from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
|
||||
|
|
@ -256,29 +257,37 @@ def parse_db_uri_for_logging(uri: str):
|
|||
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
|
||||
|
||||
|
||||
def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
|
||||
def dataframe_to_json(dataframe: pd.DataFrame) -> str:
|
||||
"""
|
||||
Serialize a DataFrame for transmission over the wire using JSON
|
||||
:param dataframe: A pandas DataFrame
|
||||
:returns: A JSON string of the pandas DataFrame
|
||||
"""
|
||||
return dataframe.to_json(orient='split')
|
||||
# https://github.com/pandas-dev/pandas/issues/24889
|
||||
# https://github.com/pandas-dev/pandas/issues/40443
|
||||
# We need to convert to a dict to avoid mem leak
|
||||
def default(z):
|
||||
if isinstance(z, pd.Timestamp):
|
||||
return z.timestamp() * 1e3
|
||||
raise TypeError
|
||||
|
||||
return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
|
||||
|
||||
|
||||
def json_to_dataframe(data: str) -> pandas.DataFrame:
|
||||
def json_to_dataframe(data: str) -> pd.DataFrame:
|
||||
"""
|
||||
Deserialize JSON into a DataFrame
|
||||
:param data: A JSON string
|
||||
:returns: A pandas DataFrame from the JSON string
|
||||
"""
|
||||
dataframe = pandas.read_json(data, orient='split')
|
||||
dataframe = pd.read_json(data, orient='split')
|
||||
if 'date' in dataframe.columns:
|
||||
dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
|
||||
dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
|
||||
|
||||
return dataframe
|
||||
|
||||
|
||||
def remove_entry_exit_signals(dataframe: pandas.DataFrame):
|
||||
def remove_entry_exit_signals(dataframe: pd.DataFrame):
|
||||
"""
|
||||
Remove Entry and Exit signals from a DataFrame
|
||||
|
||||
|
|
|
|||
|
|
@ -194,6 +194,9 @@ class ApiServer(RPCHandler):
|
|||
try:
|
||||
while True:
|
||||
logger.debug("Getting queue messages...")
|
||||
if (qsize := async_queue.qsize()) > 20:
|
||||
# If the queue becomes too big for too long, this may indicate a problem.
|
||||
logger.warning(f"Queue size now {qsize}")
|
||||
# Get data from queue
|
||||
message: WSMessageSchemaType = await async_queue.get()
|
||||
logger.debug(f"Found message of type: {message.get('type')}")
|
||||
|
|
|
|||
|
|
@ -77,21 +77,24 @@ class WebSocketChannel:
|
|||
# until self.drain_timeout for the relay to drain the outgoing queue
|
||||
# We can't use asyncio.wait_for here because the queue may have been created with a
|
||||
# different eventloop
|
||||
start = time.time()
|
||||
while self.queue.full():
|
||||
await asyncio.sleep(1)
|
||||
if (time.time() - start) > self.drain_timeout:
|
||||
if not self.is_closed():
|
||||
start = time.time()
|
||||
while self.queue.full():
|
||||
await asyncio.sleep(1)
|
||||
if (time.time() - start) > self.drain_timeout:
|
||||
return False
|
||||
|
||||
# If for some reason the queue is still full, just return False
|
||||
try:
|
||||
self.queue.put_nowait(data)
|
||||
except asyncio.QueueFull:
|
||||
return False
|
||||
|
||||
# If for some reason the queue is still full, just return False
|
||||
try:
|
||||
self.queue.put_nowait(data)
|
||||
except asyncio.QueueFull:
|
||||
# If we got here everything is ok
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
# If we got here everything is ok
|
||||
return True
|
||||
|
||||
async def recv(self):
|
||||
"""
|
||||
Receive data on the wrapped websocket
|
||||
|
|
@ -109,14 +112,14 @@ class WebSocketChannel:
|
|||
Close the WebSocketChannel
|
||||
"""
|
||||
|
||||
self._closed.set()
|
||||
self._relay_task.cancel()
|
||||
|
||||
try:
|
||||
await self.raw_websocket.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self._closed.set()
|
||||
self._relay_task.cancel()
|
||||
|
||||
def is_closed(self) -> bool:
|
||||
"""
|
||||
Closed flag
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ class Producer(TypedDict):
|
|||
name: str
|
||||
host: str
|
||||
port: int
|
||||
secure: bool
|
||||
ws_token: str
|
||||
|
||||
|
||||
|
|
@ -180,7 +181,8 @@ class ExternalMessageConsumer:
|
|||
host, port = producer['host'], producer['port']
|
||||
token = producer['ws_token']
|
||||
name = producer['name']
|
||||
ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
|
||||
scheme = 'wss' if producer.get('secure', False) else 'ws'
|
||||
ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
|
||||
|
||||
# This will raise InvalidURI if the url is bad
|
||||
async with websockets.connect(
|
||||
|
|
|
|||
|
|
@ -774,6 +774,9 @@ class RPC:
|
|||
is_short = trade.is_short
|
||||
if not self._freqtrade.strategy.position_adjustment_enable:
|
||||
raise RPCException(f'position for {pair} already open - id: {trade.id}')
|
||||
if trade.open_order_id is not None:
|
||||
raise RPCException(f'position for {pair} already open - id: {trade.id} '
|
||||
f'and has open order {trade.open_order_id}')
|
||||
else:
|
||||
if Trade.get_open_trade_count() >= self._config['max_open_trades']:
|
||||
raise RPCException("Maximum number of trades is reached.")
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
coveralls==3.3.1
|
||||
flake8==5.0.4
|
||||
flake8-tidy-imports==4.8.0
|
||||
mypy==0.990
|
||||
mypy==0.991
|
||||
pre-commit==2.20.0
|
||||
pytest==7.2.0
|
||||
pytest-asyncio==0.20.2
|
||||
|
|
@ -20,14 +20,14 @@ isort==5.10.1
|
|||
# For datetime mocking
|
||||
time-machine==2.8.2
|
||||
# fastapi testing
|
||||
httpx==0.23.0
|
||||
httpx==0.23.1
|
||||
|
||||
# Convert jupyter notebooks to markdown documents
|
||||
nbconvert==7.2.4
|
||||
nbconvert==7.2.5
|
||||
|
||||
# mypy types
|
||||
types-cachetools==5.2.1
|
||||
types-filelock==3.2.7
|
||||
types-requests==2.28.11.4
|
||||
types-requests==2.28.11.5
|
||||
types-tabulate==0.9.0.0
|
||||
types-python-dateutil==2.8.19.3
|
||||
types-python-dateutil==2.8.19.4
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
numpy==1.23.4
|
||||
numpy==1.23.5
|
||||
pandas==1.5.1
|
||||
pandas-ta==0.3.14b
|
||||
|
||||
ccxt==2.1.75
|
||||
ccxt==2.1.96
|
||||
# Pin cryptography for now due to rust build errors with piwheels
|
||||
cryptography==38.0.1; platform_machine == 'armv7l'
|
||||
cryptography==38.0.3; platform_machine != 'armv7l'
|
||||
|
|
@ -30,7 +30,7 @@ py_find_1st==1.1.5
|
|||
# Load ticker files 30% faster
|
||||
python-rapidjson==1.9
|
||||
# Properly format api responses
|
||||
orjson==3.8.1
|
||||
orjson==3.8.2
|
||||
|
||||
# Notify systemd
|
||||
sdnotify==0.3.2
|
||||
|
|
@ -38,7 +38,7 @@ sdnotify==0.3.2
|
|||
# API Server
|
||||
fastapi==0.87.0
|
||||
pydantic==1.10.2
|
||||
uvicorn==0.19.0
|
||||
uvicorn==0.20.0
|
||||
pyjwt==2.6.0
|
||||
aiofiles==22.1.0
|
||||
psutil==5.9.4
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ async def create_client(
|
|||
host,
|
||||
port,
|
||||
token,
|
||||
scheme='ws',
|
||||
name='default',
|
||||
protocol=ClientProtocol(),
|
||||
sleep_time=10,
|
||||
|
|
@ -211,13 +212,14 @@ async def create_client(
|
|||
:param host: The host
|
||||
:param port: The port
|
||||
:param token: The websocket auth token
|
||||
:param scheme: `ws` for most connections, `wss` for ssl
|
||||
:param name: The name of the producer
|
||||
:param **kwargs: Any extra kwargs passed to websockets.connect
|
||||
"""
|
||||
|
||||
while 1:
|
||||
try:
|
||||
websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
|
||||
websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
|
||||
logger.info(f"Attempting to connect to {name} @ {host}:{port}")
|
||||
|
||||
async with websockets.connect(websocket_url, **kwargs) as ws:
|
||||
|
|
@ -304,6 +306,7 @@ async def _main(args):
|
|||
producer['host'],
|
||||
producer['port'],
|
||||
producer['ws_token'],
|
||||
'wss' if producer.get('secure', False) else 'ws',
|
||||
producer['name'],
|
||||
sleep_time=sleep_time,
|
||||
ping_timeout=ping_timeout,
|
||||
|
|
|
|||
2
setup.sh
2
setup.sh
|
|
@ -83,7 +83,7 @@ function updateenv() {
|
|||
dev=$REPLY
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]
|
||||
then
|
||||
REQUIREMENTS_FREQAI="-r requirements-freqai.txt"
|
||||
REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517"
|
||||
read -p "Do you also want dependencies for freqai-rl (~700mb additional space required) [y/N]? "
|
||||
dev=$REPLY
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]
|
||||
|
|
|
|||
|
|
@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees(
|
|||
assert order1['fee']['rate'] == fee
|
||||
|
||||
|
||||
@pytest.mark.parametrize("side,startprice,endprice", [
|
||||
("buy", 25.563, 25.566),
|
||||
("sell", 25.566, 25.563)
|
||||
@pytest.mark.parametrize("side,price,filled", [
|
||||
# order_book_l2_usd spread:
|
||||
# best ask: 25.566
|
||||
# best bid: 25.563
|
||||
("buy", 25.563, False),
|
||||
("buy", 25.566, True),
|
||||
("sell", 25.566, False),
|
||||
("sell", 25.563, True),
|
||||
])
|
||||
@pytest.mark.parametrize("exchange_name", EXCHANGES)
|
||||
def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice,
|
||||
def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled,
|
||||
exchange_name, order_book_l2_usd):
|
||||
default_conf['dry_run'] = True
|
||||
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
|
||||
|
|
@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
|
|||
ordertype='limit',
|
||||
side=side,
|
||||
amount=1,
|
||||
rate=startprice,
|
||||
rate=price,
|
||||
leverage=1.0
|
||||
)
|
||||
assert order_book_l2_usd.call_count == 1
|
||||
|
|
@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
|
|||
assert order["side"] == side
|
||||
assert order["type"] == "limit"
|
||||
assert order["symbol"] == "LTC/USDT"
|
||||
assert order["average"] == price
|
||||
assert order['status'] == 'open' if not filled else 'closed'
|
||||
order_book_l2_usd.reset_mock()
|
||||
|
||||
# fetch order again...
|
||||
order_closed = exchange.fetch_dry_run_order(order['id'])
|
||||
assert order_book_l2_usd.call_count == 1
|
||||
assert order_closed['status'] == 'open'
|
||||
assert not order['fee']
|
||||
assert order_closed['filled'] == 0
|
||||
assert order_book_l2_usd.call_count == (1 if not filled else 0)
|
||||
assert order_closed['status'] == ('open' if not filled else 'closed')
|
||||
assert order_closed['filled'] == (0 if not filled else 1)
|
||||
|
||||
order_book_l2_usd.reset_mock()
|
||||
order_closed['price'] = endprice
|
||||
|
||||
order_closed = exchange.fetch_dry_run_order(order['id'])
|
||||
assert order_closed['status'] == 'closed'
|
||||
assert order['fee']
|
||||
assert order_closed['filled'] == 1
|
||||
assert order_closed['filled'] == order_closed['amount']
|
||||
|
||||
# Empty orderbook test
|
||||
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',
|
||||
|
|
|
|||
|
|
@ -1066,6 +1066,11 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
|
|||
trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
|
||||
assert trade.stake_amount == 0.05
|
||||
assert trade.buy_tag == 'force_entry'
|
||||
assert trade.open_order_id == 'mocked_limit_buy'
|
||||
|
||||
freqtradebot.strategy.position_adjustment_enable = True
|
||||
with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'):
|
||||
rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
|
||||
|
||||
# Test not buying
|
||||
pair = 'XRP/BTC'
|
||||
|
|
|
|||
Loading…
Reference in New Issue