adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge develop into feat/freqai-rl-dev

Browse Source
This commit is contained in:
robcaulk 2022-11-24 18:50:11 +01:00
commit 4894d772ed
17 changed files with 132 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.pre-commit-config.yaml
View File

@ -15,9 +15,9 @@ repos:
additional_dependencies:
- types-cachetools==5.2.1
- types-filelock==3.2.7
- types-requests==2.28.11.4
- types-requests==2.28.11.5
- types-tabulate==0.9.0.0
- types-python-dateutil==2.8.19.3
- types-python-dateutil==2.8.19.4
# stages: [push]
- repo: https://github.com/pycqa/isort

8
docs/configuration.md
View File

@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
### Using proxy with Freqtrade
To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests.
``` bash
export HTTP_PROXY="http://addr:port"
@ -672,11 +673,13 @@ export HTTPS_PROXY="http://addr:port"
freqtrade
```
#### Proxy just exchange requests
#### Proxy exchange requests
To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration.
To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
``` json
{
"exchange": {
"ccxt_config": {
"aiohttp_proxy": "http://addr:port",
"proxies": {
@ -684,6 +687,7 @@ To use a proxy just for exchange connections (skips/ignores telegram and coingec
"https": "http://addr:port"
},
}
}
```
## Next step

2
docs/producer-consumer.md
View File

@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
"name": "default", // This can be any name you'd like, default is "default"
"host": "127.0.0.1", // The host from your producer's api_server config
"port": 8080, // The port from your producer's api_server config
"secure": false, // Use a secure websockets connection, default false
"ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config
}
],
@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
| `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string
| `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string
| `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string
| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.<br> **Datatype:** string
| `producers.ws_token` | **Required.** `ws_token` as configured on the producer.<br> **Datatype:** string
| | **Optional settings**
| `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds.

38
docs/rest-api.md
View File

@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th
}
```
#### Reverse Proxy setup
When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is):
Please make sure to replace `<freqtrade_listen_ip>` (and the subsequent port) with the IP and Port matching your configuration/setup.
```
http {
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
#...
server {
#...
location / {
proxy_http_version 1.1;
proxy_pass http://<freqtrade_listen_ip>:8080;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
}
}
}
```
To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets.
- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/)
- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy)
!!! Tip "SSL certificates"
You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies.
While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel).
### OpenAPI interface
To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.

1
freqtrade/constants.py
View File

@ -512,6 +512,7 @@ CONF_SCHEMA = {
'minimum': 0,
'maximum': 65535
},
'secure': {'type': 'boolean', 'default': False},
'ws_token': {'type': 'string'},
},
'required': ['name', 'host', 'ws_token']

11
freqtrade/freqtradebot.py
View File

@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin):
trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
stoploss_order=True)
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_exit(trade, "stoploss", True)
self.handle_protections(trade.pair, trade.trade_direction)
return True
if trade.open_order_id or not trade.is_open:
@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin):
trade.close_rate_requested = limit
trade.exit_reason = exit_reason
if not sub_trade_amt:
# Lock pair for one candle to prevent immediate re-trading
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') in ('closed', 'expired'):
@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin):
self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
def handle_protections(self, pair: str, side: LongShort) -> None:
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock')
prot_trig = self.protections.stop_per_pair(pair, side=side)
if prot_trig:
msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }

23
freqtrade/misc.py
View File

@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union
from typing.io import IO
from urllib.parse import urlparse
import pandas
import orjson
import pandas as pd
import rapidjson
from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@ -256,29 +257,37 @@ def parse_db_uri_for_logging(uri: str):
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
def dataframe_to_json(dataframe: pd.DataFrame) -> str:
"""
Serialize a DataFrame for transmission over the wire using JSON
:param dataframe: A pandas DataFrame
:returns: A JSON string of the pandas DataFrame
"""
return dataframe.to_json(orient='split')
# https://github.com/pandas-dev/pandas/issues/24889
# https://github.com/pandas-dev/pandas/issues/40443
# We need to convert to a dict to avoid mem leak
def default(z):
if isinstance(z, pd.Timestamp):
return z.timestamp() * 1e3
raise TypeError
return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
def json_to_dataframe(data: str) -> pandas.DataFrame:
def json_to_dataframe(data: str) -> pd.DataFrame:
"""
Deserialize JSON into a DataFrame
:param data: A JSON string
:returns: A pandas DataFrame from the JSON string
"""
dataframe = pandas.read_json(data, orient='split')
dataframe = pd.read_json(data, orient='split')
if 'date' in dataframe.columns:
dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
return dataframe
def remove_entry_exit_signals(dataframe: pandas.DataFrame):
def remove_entry_exit_signals(dataframe: pd.DataFrame):
"""
Remove Entry and Exit signals from a DataFrame

3
freqtrade/rpc/api_server/webserver.py
View File

@ -194,6 +194,9 @@ class ApiServer(RPCHandler):
try:
while True:
logger.debug("Getting queue messages...")
if (qsize := async_queue.qsize()) > 20:
# If the queue becomes too big for too long, this may indicate a problem.
logger.warning(f"Queue size now {qsize}")
# Get data from queue
message: WSMessageSchemaType = await async_queue.get()
logger.debug(f"Found message of type: {message.get('type')}")

9
freqtrade/rpc/api_server/ws/channel.py
View File

@ -77,6 +77,7 @@ class WebSocketChannel:
# until self.drain_timeout for the relay to drain the outgoing queue
# We can't use asyncio.wait_for here because the queue may have been created with a
# different eventloop
if not self.is_closed():
start = time.time()
while self.queue.full():
await asyncio.sleep(1)
@ -91,6 +92,8 @@ class WebSocketChannel:
# If we got here everything is ok
return True
else:
return False
async def recv(self):
"""
@ -109,14 +112,14 @@ class WebSocketChannel:
Close the WebSocketChannel
"""
self._closed.set()
self._relay_task.cancel()
try:
await self.raw_websocket.close()
except Exception:
pass
self._closed.set()
self._relay_task.cancel()
def is_closed(self) -> bool:
"""
Closed flag

4
freqtrade/rpc/external_message_consumer.py
View File

@ -31,6 +31,7 @@ class Producer(TypedDict):
name: str
host: str
port: int
secure: bool
ws_token: str
@ -180,7 +181,8 @@ class ExternalMessageConsumer:
host, port = producer['host'], producer['port']
token = producer['ws_token']
name = producer['name']
ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
scheme = 'wss' if producer.get('secure', False) else 'ws'
ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
# This will raise InvalidURI if the url is bad
async with websockets.connect(

3
freqtrade/rpc/rpc.py
View File

@ -774,6 +774,9 @@ class RPC:
is_short = trade.is_short
if not self._freqtrade.strategy.position_adjustment_enable:
raise RPCException(f'position for {pair} already open - id: {trade.id}')
if trade.open_order_id is not None:
raise RPCException(f'position for {pair} already open - id: {trade.id} '
f'and has open order {trade.open_order_id}')
else:
if Trade.get_open_trade_count() >= self._config['max_open_trades']:
raise RPCException("Maximum number of trades is reached.")

10
requirements-dev.txt
View File

@ -9,7 +9,7 @@
coveralls==3.3.1
flake8==5.0.4
flake8-tidy-imports==4.8.0
mypy==0.990
mypy==0.991
pre-commit==2.20.0
pytest==7.2.0
pytest-asyncio==0.20.2
@ -20,14 +20,14 @@ isort==5.10.1
# For datetime mocking
time-machine==2.8.2
# fastapi testing
httpx==0.23.0
httpx==0.23.1
# Convert jupyter notebooks to markdown documents
nbconvert==7.2.4
nbconvert==7.2.5
# mypy types
types-cachetools==5.2.1
types-filelock==3.2.7
types-requests==2.28.11.4
types-requests==2.28.11.5
types-tabulate==0.9.0.0
types-python-dateutil==2.8.19.3
types-python-dateutil==2.8.19.4

8
requirements.txt
View File

@ -1,8 +1,8 @@
numpy==1.23.4
numpy==1.23.5
pandas==1.5.1
pandas-ta==0.3.14b
ccxt==2.1.75
ccxt==2.1.96
# Pin cryptography for now due to rust build errors with piwheels
cryptography==38.0.1; platform_machine == 'armv7l'
cryptography==38.0.3; platform_machine != 'armv7l'
@ -30,7 +30,7 @@ py_find_1st==1.1.5
# Load ticker files 30% faster
python-rapidjson==1.9
# Properly format api responses
orjson==3.8.1
orjson==3.8.2
# Notify systemd
sdnotify==0.3.2
@ -38,7 +38,7 @@ sdnotify==0.3.2
# API Server
fastapi==0.87.0
pydantic==1.10.2
uvicorn==0.19.0
uvicorn==0.20.0
pyjwt==2.6.0
aiofiles==22.1.0
psutil==5.9.4

5
scripts/ws_client.py
View File

@ -199,6 +199,7 @@ async def create_client(
host,
port,
token,
scheme='ws',
name='default',
protocol=ClientProtocol(),
sleep_time=10,
@ -211,13 +212,14 @@ async def create_client(
:param host: The host
:param port: The port
:param token: The websocket auth token
:param scheme: `ws` for most connections, `wss` for ssl
:param name: The name of the producer
:param **kwargs: Any extra kwargs passed to websockets.connect
"""
while 1:
try:
websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
logger.info(f"Attempting to connect to {name} @ {host}:{port}")
async with websockets.connect(websocket_url, **kwargs) as ws:
@ -304,6 +306,7 @@ async def _main(args):
producer['host'],
producer['port'],
producer['ws_token'],
'wss' if producer.get('secure', False) else 'ws',
producer['name'],
sleep_time=sleep_time,
ping_timeout=ping_timeout,

2
setup.sh
View File

@ -83,7 +83,7 @@ function updateenv() {
dev=$REPLY
if [[ $REPLY =~ ^[Yy]$ ]]
then
REQUIREMENTS_FREQAI="-r requirements-freqai.txt"
REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517"
read -p "Do you also want dependencies for freqai-rl (~700mb additional space required) [y/N]? "
dev=$REPLY
if [[ $REPLY =~ ^[Yy]$ ]]

32
tests/exchange/test_exchange.py
View File

@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees(
assert order1['fee']['rate'] == fee
@pytest.mark.parametrize("side,startprice,endprice", [
("buy", 25.563, 25.566),
("sell", 25.566, 25.563)
@pytest.mark.parametrize("side,price,filled", [
# order_book_l2_usd spread:
# best ask: 25.566
# best bid: 25.563
("buy", 25.563, False),
("buy", 25.566, True),
("sell", 25.566, False),
("sell", 25.563, True),
])
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice,
def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled,
exchange_name, order_book_l2_usd):
default_conf['dry_run'] = True
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
ordertype='limit',
side=side,
amount=1,
rate=startprice,
rate=price,
leverage=1.0
)
assert order_book_l2_usd.call_count == 1
@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
assert order["side"] == side
assert order["type"] == "limit"
assert order["symbol"] == "LTC/USDT"
assert order["average"] == price
assert order['status'] == 'open' if not filled else 'closed'
order_book_l2_usd.reset_mock()
# fetch order again...
order_closed = exchange.fetch_dry_run_order(order['id'])
assert order_book_l2_usd.call_count == 1
assert order_closed['status'] == 'open'
assert not order['fee']
assert order_closed['filled'] == 0
assert order_book_l2_usd.call_count == (1 if not filled else 0)
assert order_closed['status'] == ('open' if not filled else 'closed')
assert order_closed['filled'] == (0 if not filled else 1)
order_book_l2_usd.reset_mock()
order_closed['price'] = endprice
order_closed = exchange.fetch_dry_run_order(order['id'])
assert order_closed['status'] == 'closed'
assert order['fee']
assert order_closed['filled'] == 1
assert order_closed['filled'] == order_closed['amount']
# Empty orderbook test
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',

5
tests/rpc/test_rpc.py
View File

@ -1066,6 +1066,11 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
assert trade.stake_amount == 0.05
assert trade.buy_tag == 'force_entry'
assert trade.open_order_id == 'mocked_limit_buy'
freqtradebot.strategy.position_adjustment_enable = True
with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'):
rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
# Test not buying
pair = 'XRP/BTC'