From efbeabf14101d205b9035000c02197914f909b2b Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 8 Jun 2018 18:35:01 +0000 Subject: [PATCH 1/9] Added Local RPC client - added only "Daily" call so far, submitting for early review/feedback This depends on zerorpc as a requirement. simple examples here: http://www.zerorpc.io/ Installed with pip3 install zerorpc localRCP is enabled/disabled from within config.json e.g "localrpc": { "enabled": true }, The server is enabled from within existing rpc manager and makes use of the existing superclass (RPC) Though making use of the existing hardwork done in rpc.py It *should be easy to add the other Telegram calls into local_rpy_server.py The server is wrapped in a thread to be non-blocking The server and client accept serialised calls or not, used in daily to return json The client can be used from command line or in a python script As example, from cmdline for last 3 days Daily /Users/creslin/PycharmProjects/freqtrade_new/.env/bin/zerorpc tcp://127.0.0.1:4242 daily 3 connecting to "tcp://127.0.0.1:4242" False ('[\n' ' [\n' ' "2018-06-08",\n' ' "0.00000000 BTC",\n' ' "0.000 USDT",\n' ' "0 trade"\n' ' ],\n' ' [\n' ' "2018-06-07",\n' ' "0.00000000 BTC",\n' ' "0.000 USDT",\n' ' "0 trade"\n' ' ],\n' ' [\n' ' "2018-06-06",\n' ' "0.00000000 BTC",\n' ' "0.000 USDT",\n' ' "0 trade"\n' ' ]\n' ']') Programitcally this would be: import zerorpc c = zerorpc.Client() c.connect("tcp://127.0.0.1:4242") for item in c.daily(3): print item --- freqtrade/rpc/local_rpc_server.py | 83 +++++++++++++++++++++++++++++++ freqtrade/rpc/rpc_manager.py | 8 +++ requirements.txt | 3 ++ 3 files changed, 94 insertions(+) create mode 100644 freqtrade/rpc/local_rpc_server.py diff --git a/freqtrade/rpc/local_rpc_server.py b/freqtrade/rpc/local_rpc_server.py new file mode 100644 index 000000000..50b6b3a6d --- /dev/null +++ b/freqtrade/rpc/local_rpc_server.py @@ -0,0 +1,83 @@ +import threading +import time +import zerorpc +import logging +import json + +from freqtrade.rpc.rpc import RPC + + +logger = logging.getLogger(__name__) + +class LocalRPCControls(object): + """ + zeroRPC - allows local cmdline calls to super class in rpc.py + as used by Telegram.py + """ + + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + self.freqtrade = freqtrade + self._config = freqtrade.config + + # # Example of calling none serialed call + # # without decorator - left if as template while in dev for me + # def add_42(self, n): + # """ Add 42 to an integer argument to make it cooler, and return the + # result. """ + # n = int(n) + # r = n + 42 + # s = str(r) + # return s + + @zerorpc.stream + def daily(self, timescale): + logger.info("LocalRPC - Daily Command Called") + timescale = int(timescale) + + (error, stats) = RPC.rpc_daily_profit(self, timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + + #Everything in stats to a string, serialised, then back to client. + stats = json.dumps(stats, indent=4, sort_keys=True, default=str) + return(error, stats) + +class LocalRPCSuperWrap(RPC): + """ + Telegram, this class send messages to Telegram + """ + def __init__(self, freqtrade) -> None: + """ + Init the LocalRPCServer call, and init the super class RPC + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + super().__init__(freqtrade) + """ Constructor + :type interval: int + :param interval: Check interval, in seconds + """ + self.interval = int(1) + + thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple + thread.daemon = True # Daemonize thread + thread.start() # Start the execution + + def run(self, freqtrade): + """ Method that runs forever """ + self._config = freqtrade.config + + # TODO add IP address / port to bind to in config.json and use in below. + while True: + # Do something + logger.info('Starting Local RPC Listener') + s = zerorpc.Server(LocalRPCControls(freqtrade)) + s.bind("tcp://0.0.0.0:4242") + s.run() + time.sleep(self.interval) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 58e9bf2b9..b89e8a0d5 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -3,8 +3,10 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) """ from typing import Any, List import logging +import time from freqtrade.rpc.telegram import Telegram +from freqtrade.rpc.local_rpc_server import LocalRPCSuperWrap logger = logging.getLogger(__name__) @@ -36,6 +38,12 @@ class RPCManager(object): self.registered_modules.append('telegram') self.telegram = Telegram(self.freqtrade) + # Added another RPC client - for cmdline local client. + # Uses existing superclass RPC build for Telegram + if self.freqtrade.config['localrpc'].get('enabled', False): + self.localRPC = LocalRPCSuperWrap(self.freqtrade) + time.sleep(1) + def cleanup(self) -> None: """ Stops all enabled rpc modules diff --git a/requirements.txt b/requirements.txt index 5f5183321..4f4924225 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,6 @@ coinmarketcap==5.0.3 # Required for plotting data #plotly==2.3.0 + +#Added for localRCP +zerorpc== \ No newline at end of file From e5e64a7035172cce6e09d4eaf8bbc014bf0a5557 Mon Sep 17 00:00:00 2001 From: creslinux Date: Wed, 13 Jun 2018 22:18:49 +0000 Subject: [PATCH 2/9] Implemented local restful flask service and provided cmdline client Added only the "Daily" call so far, submitting for early review/feedback Called as example "./rest_client.py daily 3" This depends on listed as requirements. Flask==1.0.2 flask-jsonpify==1.5.0 (will do later) flask-restful==0.3.6 TODO: make loading optional, cleanly unload on close unit tests, take feedback, tidy output, add other Telegram functions, onwards local rest server is enabled/disabled from within config.json. E.g "localrest": { "enabled": true }, The server is enabled from within existing rpc manager and makes use of the existing superclass (RPC) Through making use of the existing hard work done in rpc.py It *should be easy to add the other Telegram calls into local_rpc_server.py The server is wrapped in a thread to be non-blocking The server and client accept serialised calls or not, used in daily to return json The client can be used from command line or in a python client script As example, from cmdline for last 3 days Daily DannyMBP:rpc creslin$ ./rest_client.py daily 3 [ [ "2018-06-13", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-12", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-11", "0.00000000 USDT", "0.000 USD", "0 trade" ] ] --- ...cal_rpc_server.py => local_rest_server.py} | 62 +++++++++---------- freqtrade/rpc/rest_client.py | 23 +++++++ freqtrade/rpc/rpc_manager.py | 6 +- requirements.txt | 6 +- 4 files changed, 61 insertions(+), 36 deletions(-) rename freqtrade/rpc/{local_rpc_server.py => local_rest_server.py} (57%) create mode 100755 freqtrade/rpc/rest_client.py diff --git a/freqtrade/rpc/local_rpc_server.py b/freqtrade/rpc/local_rest_server.py similarity index 57% rename from freqtrade/rpc/local_rpc_server.py rename to freqtrade/rpc/local_rest_server.py index 50b6b3a6d..940cc5942 100644 --- a/freqtrade/rpc/local_rpc_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -1,19 +1,19 @@ import threading -import time -import zerorpc import logging import json +from flask import Flask, request +from flask_restful import Resource, Api +from json import dumps from freqtrade.rpc.rpc import RPC logger = logging.getLogger(__name__) -class LocalRPCControls(object): - """ - zeroRPC - allows local cmdline calls to super class in rpc.py - as used by Telegram.py - """ + +class Daily(Resource): + # called by http://127.0.0.1:/daily?timescale=7 + # where 7 is the number of days to report back with. def __init__(self, freqtrade) -> None: """ @@ -24,18 +24,9 @@ class LocalRPCControls(object): self.freqtrade = freqtrade self._config = freqtrade.config - # # Example of calling none serialed call - # # without decorator - left if as template while in dev for me - # def add_42(self, n): - # """ Add 42 to an integer argument to make it cooler, and return the - # result. """ - # n = int(n) - # r = n + 42 - # s = str(r) - # return s - @zerorpc.stream - def daily(self, timescale): + def get(self): + timescale = request.args.get('timescale') logger.info("LocalRPC - Daily Command Called") timescale = int(timescale) @@ -43,18 +34,21 @@ class LocalRPCControls(object): self._config['stake_currency'], self._config['fiat_display_currency'] ) + if error == False: + stats = dumps(stats, indent=4, sort_keys=True, default=str) + return stats + else: + json.dumps(error) + return error - #Everything in stats to a string, serialised, then back to client. - stats = json.dumps(stats, indent=4, sort_keys=True, default=str) - return(error, stats) -class LocalRPCSuperWrap(RPC): +class LocalRestSuperWrap(RPC): """ - Telegram, this class send messages to Telegram + This class is for REST cmd line client """ def __init__(self, freqtrade) -> None: """ - Init the LocalRPCServer call, and init the super class RPC + Init the LocalRestServer call, and init the super class RPC :param freqtrade: Instance of a freqtrade bot :return: None """ @@ -73,11 +67,17 @@ class LocalRPCSuperWrap(RPC): """ Method that runs forever """ self._config = freqtrade.config + # TODO add IP address / port to bind to in config.json and use in below. - while True: - # Do something - logger.info('Starting Local RPC Listener') - s = zerorpc.Server(LocalRPCControls(freqtrade)) - s.bind("tcp://0.0.0.0:4242") - s.run() - time.sleep(self.interval) + logger.info('Starting Local Rest Server') + + my_freqtrade = freqtrade + app = Flask(__name__) + api = Api(app) + + # Our resources for restful apps go here, pass freqtrade object across + api.add_resource(Daily, '/daily', methods=['GET'], + resource_class_kwargs={'freqtrade': my_freqtrade}) # Route for returning daily + + #run the server + app.run(port='5002') diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py new file mode 100755 index 000000000..5b39b7a0b --- /dev/null +++ b/freqtrade/rpc/rest_client.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +""" +Simple command line client into RPC commands +Can be used as an alternate to Telegram +""" + +from requests import get +from sys import argv + +if len(argv) == 1: + print('\nThis script accepts the following arguments') + print('- daily (int) - Where int is the number of days to report back. daily 3') + print('- there will be more....\n') + +if len(argv) == 3 and argv[1] == "daily": + if str.isnumeric(argv[2]): + get_url = 'http://localhost:5002/daily?timescale=' + argv[2] + d=get(get_url).json() + print(d) + else: + print("\nThe second argument to daily must be an integer, 1,2,3 etc") + + diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index b89e8a0d5..2dfb119fe 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -6,7 +6,7 @@ import logging import time from freqtrade.rpc.telegram import Telegram -from freqtrade.rpc.local_rpc_server import LocalRPCSuperWrap +from freqtrade.rpc.local_rest_server import LocalRestSuperWrap logger = logging.getLogger(__name__) @@ -40,8 +40,8 @@ class RPCManager(object): # Added another RPC client - for cmdline local client. # Uses existing superclass RPC build for Telegram - if self.freqtrade.config['localrpc'].get('enabled', False): - self.localRPC = LocalRPCSuperWrap(self.freqtrade) + if self.freqtrade.config['rest_cmd'].get('enabled', False): + self.localRPC = LocalRestSuperWrap(self.freqtrade) time.sleep(1) def cleanup(self) -> None: diff --git a/requirements.txt b/requirements.txt index 4f4924225..34373b22f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,5 +24,7 @@ coinmarketcap==5.0.3 # Required for plotting data #plotly==2.3.0 -#Added for localRCP -zerorpc== \ No newline at end of file +#Added for local rest client +Flask==1.0.2 +flask-jsonpify==1.5.0 +flask-restful==0.3.6 From 6bb1ad288e6d74b749e11ec7d8490ebe1ab1b466 Mon Sep 17 00:00:00 2001 From: creslinux Date: Wed, 13 Jun 2018 22:26:21 +0000 Subject: [PATCH 3/9] Implemented local restful flask service and provided cmdline client Added only the "Daily" call so far, submitting for early review/feedback Called as example "./rest_client.py daily 3" This depends on listed as requirements. Flask==1.0.2 flask-jsonpify==1.5.0 (will do later) flask-restful==0.3.6 TODO: make loading optional, cleanly unload on close unit tests, take feedback, tidy output, add other Telegram functions, onwards local rest server is enabled/disabled from within config.json. E.g "localrest": { "enabled": true }, The server is enabled from within existing rpc manager and makes use of the existing superclass (RPC) Through making use of the existing hard work done in rpc.py It *should be easy to add the other Telegram calls into local_rpc_server.py The server is wrapped in a thread to be non-blocking The server and client accept serialised calls or not, used in daily to return json The client can be used from command line or in a python client script As example, from cmdline for last 3 days Daily DannyMBP:rpc creslin$ ./rest_client.py daily 3 [ [ "2018-06-13", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-12", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-11", "0.00000000 USDT", "0.000 USD", "0 trade" ] ] --- freqtrade/rpc/local_rest_server.py | 2 +- freqtrade/rpc/rest_client.py | 2 +- freqtrade/rpc/rpc_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/local_rest_server.py index 940cc5942..0c9fad16b 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class Daily(Resource): # called by http://127.0.0.1:/daily?timescale=7 - # where 7 is the number of days to report back with. + # where 7 is the number of days to report back with def __init__(self, freqtrade) -> None: """ diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 5b39b7a0b..928e1798c 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3. """ Simple command line client into RPC commands Can be used as an alternate to Telegram diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 2dfb119fe..ef71ff3a9 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,5 +1,5 @@ """ -This module contains class to manage RPC communications (Telegram, Slack, ...) +This module contains class to manage RPC communications (Telegram, Slack, ....) """ from typing import Any, List import logging From faf0a4973d4cdf9921874dad7e61044336973ea0 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Wed, 13 Jun 2018 22:41:43 +0000 Subject: [PATCH 4/9] Update rest_client.py --- freqtrade/rpc/rest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 928e1798c..5b39b7a0b 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3. +#!/usr/bin/env python3 """ Simple command line client into RPC commands Can be used as an alternate to Telegram From fb60f684f7f3385bd2dedf8807e9f2ffae5b730d Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 15:38:26 +0000 Subject: [PATCH 5/9] Updated Stop and start calls added Along with refactoring to base line and use decorators. Also modules loaded optionally if enabled in config or not binds to ip / port set from config.json with warning if not localhost TODO: - use argparse in client, and generally clean client up - create unit test - documentation - extend to other RCP commands, after feedback --- freqtrade/rpc/local_rest_server.py | 122 ++++++++++++++++++----------- freqtrade/rpc/rest_client.py | 30 +++++++ freqtrade/rpc/rpc_manager.py | 65 ++++++--------- 3 files changed, 131 insertions(+), 86 deletions(-) diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/local_rest_server.py index 0c9fad16b..24d9a6594 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -5,43 +5,11 @@ import json from flask import Flask, request from flask_restful import Resource, Api from json import dumps -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) - -class Daily(Resource): - # called by http://127.0.0.1:/daily?timescale=7 - # where 7 is the number of days to report back with - - def __init__(self, freqtrade) -> None: - """ - Initializes all enabled rpc modules - :param freqtrade: Instance of a freqtrade bot - :return: None - """ - self.freqtrade = freqtrade - self._config = freqtrade.config - - - def get(self): - timescale = request.args.get('timescale') - logger.info("LocalRPC - Daily Command Called") - timescale = int(timescale) - - (error, stats) = RPC.rpc_daily_profit(self, timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - if error == False: - stats = dumps(stats, indent=4, sort_keys=True, default=str) - return stats - else: - json.dumps(error) - return error - - class LocalRestSuperWrap(RPC): """ This class is for REST cmd line client @@ -61,23 +29,85 @@ class LocalRestSuperWrap(RPC): thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple thread.daemon = True # Daemonize thread - thread.start() # Start the execution + thread.start() # Start the execution + def run(self, freqtrade): """ Method that runs forever """ self._config = freqtrade.config - - - # TODO add IP address / port to bind to in config.json and use in below. - logger.info('Starting Local Rest Server') - - my_freqtrade = freqtrade app = Flask(__name__) - api = Api(app) - # Our resources for restful apps go here, pass freqtrade object across - api.add_resource(Daily, '/daily', methods=['GET'], - resource_class_kwargs={'freqtrade': my_freqtrade}) # Route for returning daily + """ + Define the application routes here + each Telegram command should have a like local substitute + """ + @app.route("/") + def hello(): + # For simple rest server testing via browser + # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status + # /status /table /performance /count, + # /start /stop /help' - #run the server - app.run(port='5002') + rest_cmds ='Commands implemented:
' \ + '/daily?timescale=7' \ + '
' \ + '/stop' \ + '
' \ + '/start' + return rest_cmds + + @app.route('/daily', methods=['GET']) + def daily(): + try: + timescale = request.args.get('timescale') + logger.info("LocalRPC - Daily Command Called") + timescale = int(timescale) + + stats = self._rpc_daily_profit(timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + + stats = dumps(stats, indent=4, sort_keys=True, default=str) + return stats + except RPCException as e: + return e + + @app.route('/start', methods=['GET']) + def start(): + """ + Handler for /start. + Starts TradeThread + """ + msg = self._rpc_start() + print("msg is", msg) + return msg + + @app.route('/stop', methods=['GET']) + def stop(): + """ + Handler for /stop. + Stops TradeThread + """ + msg = self._rpc_stop() + print("msg is", msg) + return msg + + """ + Section to handle configuration and running of the Rest serve + also to check and warn if not bound to 127.0.0.1 as a security risk + """ + + rest_ip = self._config['rest_cmd_line']['listen_ip_address'] + rest_port = self._config['rest_cmd_line']['listen_port'] + + if rest_ip != "127.0.0.1": + i=0 + while i < 10: + logger.info("SECURITY WARNING - Local Rest Server listening to external connections") + logger.info("SECURITY WARNING - This is insecure please set to 127.0.0.1 in config.json") + i += 1 + + # Run the Server + logger.info('Starting Local Rest Server') + app.run(host=rest_ip, port=rest_port) diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 928e1798c..c5007970f 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -4,12 +4,20 @@ Simple command line client into RPC commands Can be used as an alternate to Telegram """ +import time from requests import get from sys import argv + + +#TODO - use argparse to clean this up +#TODO - use IP and Port from config.json not hardcode + if len(argv) == 1: print('\nThis script accepts the following arguments') print('- daily (int) - Where int is the number of days to report back. daily 3') + print('- start - this will start the trading thread') + print('- stop - this will start the trading thread') print('- there will be more....\n') if len(argv) == 3 and argv[1] == "daily": @@ -20,4 +28,26 @@ if len(argv) == 3 and argv[1] == "daily": else: print("\nThe second argument to daily must be an integer, 1,2,3 etc") +if len(argv) == 2 and argv[1] == "start": + get_url = 'http://localhost:5002/start' + d = get(get_url).text + print(d) + + if "already" not in d: + time.sleep(2) + d = get(get_url).text + print(d) + +if len(argv) == 2 and argv[1] == "stop": + get_url = 'http://localhost:5002/stop' + d = get(get_url).text + print(d) + + if "already" not in d: + time.sleep(2) + d = get(get_url).text + print(d) + + + diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index ef71ff3a9..21e54f488 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,13 +1,10 @@ """ -This module contains class to manage RPC communications (Telegram, Slack, ....) +This module contains class to manage RPC communications (Telegram, Slack, Rest ...) """ -from typing import Any, List import logging -import time - -from freqtrade.rpc.telegram import Telegram -from freqtrade.rpc.local_rest_server import LocalRestSuperWrap +from typing import List +from freqtrade.rpc.rpc import RPC logger = logging.getLogger(__name__) @@ -17,42 +14,29 @@ class RPCManager(object): Class to manage RPC objects (Telegram, Slack, ...) """ def __init__(self, freqtrade) -> None: - """ - Initializes all enabled rpc modules - :param config: config to use - :return: None - """ - self.freqtrade = freqtrade + """ Initializes all enabled rpc modules """ + self.registered_modules: List[RPC] = [] - self.registered_modules: List[str] = [] - self.telegram: Any = None - self._init() - - def _init(self) -> None: - """ - Init RPC modules - :return: - """ - if self.freqtrade.config['telegram'].get('enabled', False): + # Enable telegram + if freqtrade.config['telegram'].get('enabled', False): logger.info('Enabling rpc.telegram ...') - self.registered_modules.append('telegram') - self.telegram = Telegram(self.freqtrade) + from freqtrade.rpc.telegram import Telegram + self.registered_modules.append(Telegram(freqtrade)) - # Added another RPC client - for cmdline local client. - # Uses existing superclass RPC build for Telegram - if self.freqtrade.config['rest_cmd'].get('enabled', False): - self.localRPC = LocalRestSuperWrap(self.freqtrade) - time.sleep(1) + # Enable local rest server for cmd line control + if freqtrade.config['rest_cmd_line'].get('enabled', False): + logger.info('Enabling rpc.local_rest_server ...') + from freqtrade.rpc.local_rest_server import LocalRestSuperWrap + self.registered_modules.append(LocalRestSuperWrap(freqtrade)) def cleanup(self) -> None: - """ - Stops all enabled rpc modules - :return: None - """ - if 'telegram' in self.registered_modules: - logger.info('Cleaning up rpc.telegram ...') - self.registered_modules.remove('telegram') - self.telegram.cleanup() + """ Stops all enabled rpc modules """ + logger.info('Cleaning up rpc modules ...') + while self.registered_modules: + mod = self.registered_modules.pop() + logger.debug('Cleaning up rpc.%s ...', mod.name) + mod.cleanup() + del mod def send_msg(self, msg: str) -> None: """ @@ -60,6 +44,7 @@ class RPCManager(object): :param msg: message :return: None """ - logger.info(msg) - if 'telegram' in self.registered_modules: - self.telegram.send_msg(msg) + logger.info('Sending rpc message: %s', msg) + for mod in self.registered_modules: + logger.debug('Forwarding message to rpc.%s', mod.name) + mod.send_msg(msg) From 9bbc186b1636237189deccac89c4d5b0e3aed19e Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 15:44:12 +0000 Subject: [PATCH 6/9] Updated Stop and start calls added Along with refactoring to base line and use decorators. Also modules loaded optionally if enabled in config or not binds to ip / port set from config.json with warning if not localhost TODO: - use argparse in client, and generally clean client up - create unit test - documentation - extend to other RCP commands, after feedback --- freqtrade/rpc/local_rest_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/local_rest_server.py index 24d9a6594..92bb17be8 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -95,7 +95,7 @@ class LocalRestSuperWrap(RPC): """ Section to handle configuration and running of the Rest serve - also to check and warn if not bound to 127.0.0.1 as a security risk + also to check and warn if not bound to 127.0.0.1 as a security risk. """ rest_ip = self._config['rest_cmd_line']['listen_ip_address'] From 7caf8a46d485c9be1116658eb58bf89f31b3e082 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 15:44:52 +0000 Subject: [PATCH 7/9] Updated Stop and start calls added Along with refactoring to base line and use decorators. Also modules loaded optionally if enabled in config or not binds to ip / port set from config.json with warning if not localhost TODO: - use argparse in client, and generally clean client up - create unit test - documentation - extend to other RCP commands, after feedback --- freqtrade/rpc/rpc_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 21e54f488..d796b2454 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,5 +1,5 @@ """ -This module contains class to manage RPC communications (Telegram, Slack, Rest ...) +This module contains class to manage RPC communications (Telegram, Slack, Rest ....) """ import logging from typing import List From 40db83c24c803876951f71ed93814398de74923f Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 20:19:15 +0000 Subject: [PATCH 8/9] Added json validation of formats to check IPv4 refactored files and calls to be api_server worked down satisfying review comments of last commit --- freqtrade/configuration.py | 8 +++-- freqtrade/constants.py | 13 +++++++ .../{local_rest_server.py => api_server.py} | 35 ++++++++----------- freqtrade/rpc/rpc_manager.py | 11 +++--- 4 files changed, 39 insertions(+), 28 deletions(-) rename freqtrade/rpc/{local_rest_server.py => api_server.py} (78%) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 2a9e8fbd8..76726b293 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -6,7 +6,7 @@ import json import logging from argparse import Namespace from typing import Optional, Dict, Any -from jsonschema import Draft4Validator, validate +from jsonschema import Draft4Validator, validate, draft4_format_checker from jsonschema.exceptions import ValidationError, best_match import ccxt @@ -209,7 +209,7 @@ class Configuration(object): :return: Returns the config if valid, otherwise throw an exception """ try: - validate(conf, constants.CONF_SCHEMA) + validate(conf, constants.CONF_SCHEMA, format_checker=draft4_format_checker) return conf except ValidationError as exception: logger.critical( @@ -217,7 +217,9 @@ class Configuration(object): exception ) raise ValidationError( - best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message + best_match(Draft4Validator(constants.CONF_SCHEMA, + format_checker=draft4_format_checker) + .iter_errors(conf)).message ) def get_config(self) -> Dict[str, Any]: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5be01f977..8560bf8a3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -85,6 +85,19 @@ CONF_SCHEMA = { }, 'required': ['enabled', 'token', 'chat_id'] }, + 'api_server': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'listen_ip_address': { "format": "ipv4"}, + 'listen_port': { + 'type': 'integer', + "minimum": 1024, + "maximum": 65535 + }, + }, + 'required': ['enabled', 'listen_ip_address', 'listen_port'] + }, 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'internals': { diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/api_server.py similarity index 78% rename from freqtrade/rpc/local_rest_server.py rename to freqtrade/rpc/api_server.py index 92bb17be8..2a1ad0b8c 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/api_server.py @@ -6,26 +6,25 @@ from flask import Flask, request from flask_restful import Resource, Api from json import dumps from freqtrade.rpc.rpc import RPC, RPCException +from ipaddress import IPv4Address logger = logging.getLogger(__name__) +app = Flask(__name__) -class LocalRestSuperWrap(RPC): +class ApiServerSuperWrap(RPC): """ - This class is for REST cmd line client + This class is for REST calls across api server """ def __init__(self, freqtrade) -> None: """ - Init the LocalRestServer call, and init the super class RPC + Init the api server, and init the super class RPC :param freqtrade: Instance of a freqtrade bot :return: None """ super().__init__(freqtrade) - """ Constructor - :type interval: int - :param interval: Check interval, in seconds - """ - self.interval = int(1) + + self.interval = 1 thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple thread.daemon = True # Daemonize thread @@ -35,11 +34,10 @@ class LocalRestSuperWrap(RPC): def run(self, freqtrade): """ Method that runs forever """ self._config = freqtrade.config - app = Flask(__name__) """ - Define the application routes here - each Telegram command should have a like local substitute + Define the application routes here + each Telegram command should have a like local substitute """ @app.route("/") def hello(): @@ -97,17 +95,14 @@ class LocalRestSuperWrap(RPC): Section to handle configuration and running of the Rest serve also to check and warn if not bound to 127.0.0.1 as a security risk. """ + rest_ip = self._config['api_server']['listen_ip_address'] + rest_port = self._config['api_server']['listen_port'] - rest_ip = self._config['rest_cmd_line']['listen_ip_address'] - rest_port = self._config['rest_cmd_line']['listen_port'] - - if rest_ip != "127.0.0.1": - i=0 - while i < 10: - logger.info("SECURITY WARNING - Local Rest Server listening to external connections") - logger.info("SECURITY WARNING - This is insecure please set to 127.0.0.1 in config.json") - i += 1 + if not IPv4Address(rest_ip).is_loopback : + logger.info("SECURITY WARNING - Local Rest Server listening to external connections") + logger.info("SECURITY WARNING - This is insecure please set to your loopback, e.g 127.0.0.1 in config.json") # Run the Server logger.info('Starting Local Rest Server') app.run(host=rest_ip, port=rest_port) + diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index d796b2454..9dddcdb80 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -23,11 +23,12 @@ class RPCManager(object): from freqtrade.rpc.telegram import Telegram self.registered_modules.append(Telegram(freqtrade)) - # Enable local rest server for cmd line control - if freqtrade.config['rest_cmd_line'].get('enabled', False): - logger.info('Enabling rpc.local_rest_server ...') - from freqtrade.rpc.local_rest_server import LocalRestSuperWrap - self.registered_modules.append(LocalRestSuperWrap(freqtrade)) + # Enable local rest api server for cmd line control + if freqtrade.config['api_server'].get('enabled', False): + logger.info('Enabling rpc.api_server') + from freqtrade.rpc.api_server import ApiServerSuperWrap + self.registered_modules.append(ApiServerSuperWrap(freqtrade)) + def cleanup(self) -> None: """ Stops all enabled rpc modules """ From 05226f6defc0defee8c2e636727b08529bd40b16 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 15 Jun 2018 09:14:17 +0000 Subject: [PATCH 9/9] Moved from decorators to app.add_url_rule This has the benefit of creating a label which may be helpful if later refactoring. This change misses the main thrust of requests from both Gcarq and Shusso to better layout the code Im running into a challenge with 'self' not being available, or able to be passed in either to decorators or view_func This may simply be how I've instantiated an RPC in the wuperwrap or im within a thread - my very limited exposure to programming is at play! After moving code around lots of ways to no success and google not being helpful im committing for further feeback --- freqtrade/rpc/api_server.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 2a1ad0b8c..04150b4d7 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -1,9 +1,9 @@ import threading import logging -import json +# import json from flask import Flask, request -from flask_restful import Resource, Api +# from flask_restful import Resource, Api from json import dumps from freqtrade.rpc.rpc import RPC, RPCException from ipaddress import IPv4Address @@ -36,10 +36,10 @@ class ApiServerSuperWrap(RPC): self._config = freqtrade.config """ - Define the application routes here + Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute """ - @app.route("/") + # @app.route("/") def hello(): # For simple rest server testing via browser # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status @@ -54,7 +54,6 @@ class ApiServerSuperWrap(RPC): '/start' return rest_cmds - @app.route('/daily', methods=['GET']) def daily(): try: timescale = request.args.get('timescale') @@ -71,29 +70,36 @@ class ApiServerSuperWrap(RPC): except RPCException as e: return e - @app.route('/start', methods=['GET']) def start(): """ Handler for /start. Starts TradeThread """ msg = self._rpc_start() - print("msg is", msg) return msg - @app.route('/stop', methods=['GET']) def stop(): """ Handler for /stop. Stops TradeThread """ msg = self._rpc_stop() - print("msg is", msg) return msg + ## defines the url rules available on the api server + ''' + First two arguments passed are /URL and 'Label' + Label can be used as a shortcut when refactoring + ''' + app.add_url_rule('/', 'hello', view_func=hello, methods=['GET']) + app.add_url_rule('/stop', 'stop', view_func=stop, methods=['GET']) + app.add_url_rule('/start', 'start', view_func=start, methods=['GET']) + app.add_url_rule('/daily', 'daily', view_func=daily, methods=['GET']) + + """ - Section to handle configuration and running of the Rest serve - also to check and warn if not bound to 127.0.0.1 as a security risk. + Section to handle configuration and running of the Rest server + also to check and warn if not bound to a loopback, warn on security risk. """ rest_ip = self._config['api_server']['listen_ip_address'] rest_port = self._config['api_server']['listen_port'] @@ -104,5 +110,9 @@ class ApiServerSuperWrap(RPC): # Run the Server logger.info('Starting Local Rest Server') - app.run(host=rest_ip, port=rest_port) + try: + app.run(host=rest_ip, port=rest_port) + except: + logger.exception("Api server failed to start, exception message is:") +