From 5fdb7612d04610dd42ee275fb64542735552a7b4 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 8 Jun 2018 18:35:01 +0000 Subject: [PATCH 01/26] Added Local RPC client - added only "Daily" call so far, submitting for early review/feedback This depends on zerorpc as a requirement. simple examples here: http://www.zerorpc.io/ Installed with pip3 install zerorpc localRCP is enabled/disabled from within config.json e.g "localrpc": { "enabled": true }, The server is enabled from within existing rpc manager and makes use of the existing superclass (RPC) Though making use of the existing hardwork done in rpc.py It *should be easy to add the other Telegram calls into local_rpy_server.py The server is wrapped in a thread to be non-blocking The server and client accept serialised calls or not, used in daily to return json The client can be used from command line or in a python script As example, from cmdline for last 3 days Daily /Users/creslin/PycharmProjects/freqtrade_new/.env/bin/zerorpc tcp://127.0.0.1:4242 daily 3 connecting to "tcp://127.0.0.1:4242" False ('[\n' ' [\n' ' "2018-06-08",\n' ' "0.00000000 BTC",\n' ' "0.000 USDT",\n' ' "0 trade"\n' ' ],\n' ' [\n' ' "2018-06-07",\n' ' "0.00000000 BTC",\n' ' "0.000 USDT",\n' ' "0 trade"\n' ' ],\n' ' [\n' ' "2018-06-06",\n' ' "0.00000000 BTC",\n' ' "0.000 USDT",\n' ' "0 trade"\n' ' ]\n' ']') Programitcally this would be: import zerorpc c = zerorpc.Client() c.connect("tcp://127.0.0.1:4242") for item in c.daily(3): print item --- freqtrade/rpc/local_rpc_server.py | 83 +++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 freqtrade/rpc/local_rpc_server.py diff --git a/freqtrade/rpc/local_rpc_server.py b/freqtrade/rpc/local_rpc_server.py new file mode 100644 index 000000000..50b6b3a6d --- /dev/null +++ b/freqtrade/rpc/local_rpc_server.py @@ -0,0 +1,83 @@ +import threading +import time +import zerorpc +import logging +import json + +from freqtrade.rpc.rpc import RPC + + +logger = logging.getLogger(__name__) + +class LocalRPCControls(object): + """ + zeroRPC - allows local cmdline calls to super class in rpc.py + as used by Telegram.py + """ + + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + self.freqtrade = freqtrade + self._config = freqtrade.config + + # # Example of calling none serialed call + # # without decorator - left if as template while in dev for me + # def add_42(self, n): + # """ Add 42 to an integer argument to make it cooler, and return the + # result. """ + # n = int(n) + # r = n + 42 + # s = str(r) + # return s + + @zerorpc.stream + def daily(self, timescale): + logger.info("LocalRPC - Daily Command Called") + timescale = int(timescale) + + (error, stats) = RPC.rpc_daily_profit(self, timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + + #Everything in stats to a string, serialised, then back to client. + stats = json.dumps(stats, indent=4, sort_keys=True, default=str) + return(error, stats) + +class LocalRPCSuperWrap(RPC): + """ + Telegram, this class send messages to Telegram + """ + def __init__(self, freqtrade) -> None: + """ + Init the LocalRPCServer call, and init the super class RPC + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + super().__init__(freqtrade) + """ Constructor + :type interval: int + :param interval: Check interval, in seconds + """ + self.interval = int(1) + + thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple + thread.daemon = True # Daemonize thread + thread.start() # Start the execution + + def run(self, freqtrade): + """ Method that runs forever """ + self._config = freqtrade.config + + # TODO add IP address / port to bind to in config.json and use in below. + while True: + # Do something + logger.info('Starting Local RPC Listener') + s = zerorpc.Server(LocalRPCControls(freqtrade)) + s.bind("tcp://0.0.0.0:4242") + s.run() + time.sleep(self.interval) From dcbdbecae06da5dbdedaf537040a1d92c3d57bf0 Mon Sep 17 00:00:00 2001 From: creslinux Date: Wed, 13 Jun 2018 22:18:49 +0000 Subject: [PATCH 02/26] Implemented local restful flask service and provided cmdline client Added only the "Daily" call so far, submitting for early review/feedback Called as example "./rest_client.py daily 3" This depends on listed as requirements. Flask==1.0.2 flask-jsonpify==1.5.0 (will do later) flask-restful==0.3.6 TODO: make loading optional, cleanly unload on close unit tests, take feedback, tidy output, add other Telegram functions, onwards local rest server is enabled/disabled from within config.json. E.g "localrest": { "enabled": true }, The server is enabled from within existing rpc manager and makes use of the existing superclass (RPC) Through making use of the existing hard work done in rpc.py It *should be easy to add the other Telegram calls into local_rpc_server.py The server is wrapped in a thread to be non-blocking The server and client accept serialised calls or not, used in daily to return json The client can be used from command line or in a python client script As example, from cmdline for last 3 days Daily DannyMBP:rpc creslin$ ./rest_client.py daily 3 [ [ "2018-06-13", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-12", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-11", "0.00000000 USDT", "0.000 USD", "0 trade" ] ] --- ...cal_rpc_server.py => local_rest_server.py} | 62 +++++++++---------- freqtrade/rpc/rest_client.py | 23 +++++++ 2 files changed, 54 insertions(+), 31 deletions(-) rename freqtrade/rpc/{local_rpc_server.py => local_rest_server.py} (57%) create mode 100755 freqtrade/rpc/rest_client.py diff --git a/freqtrade/rpc/local_rpc_server.py b/freqtrade/rpc/local_rest_server.py similarity index 57% rename from freqtrade/rpc/local_rpc_server.py rename to freqtrade/rpc/local_rest_server.py index 50b6b3a6d..940cc5942 100644 --- a/freqtrade/rpc/local_rpc_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -1,19 +1,19 @@ import threading -import time -import zerorpc import logging import json +from flask import Flask, request +from flask_restful import Resource, Api +from json import dumps from freqtrade.rpc.rpc import RPC logger = logging.getLogger(__name__) -class LocalRPCControls(object): - """ - zeroRPC - allows local cmdline calls to super class in rpc.py - as used by Telegram.py - """ + +class Daily(Resource): + # called by http://127.0.0.1:/daily?timescale=7 + # where 7 is the number of days to report back with. def __init__(self, freqtrade) -> None: """ @@ -24,18 +24,9 @@ class LocalRPCControls(object): self.freqtrade = freqtrade self._config = freqtrade.config - # # Example of calling none serialed call - # # without decorator - left if as template while in dev for me - # def add_42(self, n): - # """ Add 42 to an integer argument to make it cooler, and return the - # result. """ - # n = int(n) - # r = n + 42 - # s = str(r) - # return s - @zerorpc.stream - def daily(self, timescale): + def get(self): + timescale = request.args.get('timescale') logger.info("LocalRPC - Daily Command Called") timescale = int(timescale) @@ -43,18 +34,21 @@ class LocalRPCControls(object): self._config['stake_currency'], self._config['fiat_display_currency'] ) + if error == False: + stats = dumps(stats, indent=4, sort_keys=True, default=str) + return stats + else: + json.dumps(error) + return error - #Everything in stats to a string, serialised, then back to client. - stats = json.dumps(stats, indent=4, sort_keys=True, default=str) - return(error, stats) -class LocalRPCSuperWrap(RPC): +class LocalRestSuperWrap(RPC): """ - Telegram, this class send messages to Telegram + This class is for REST cmd line client """ def __init__(self, freqtrade) -> None: """ - Init the LocalRPCServer call, and init the super class RPC + Init the LocalRestServer call, and init the super class RPC :param freqtrade: Instance of a freqtrade bot :return: None """ @@ -73,11 +67,17 @@ class LocalRPCSuperWrap(RPC): """ Method that runs forever """ self._config = freqtrade.config + # TODO add IP address / port to bind to in config.json and use in below. - while True: - # Do something - logger.info('Starting Local RPC Listener') - s = zerorpc.Server(LocalRPCControls(freqtrade)) - s.bind("tcp://0.0.0.0:4242") - s.run() - time.sleep(self.interval) + logger.info('Starting Local Rest Server') + + my_freqtrade = freqtrade + app = Flask(__name__) + api = Api(app) + + # Our resources for restful apps go here, pass freqtrade object across + api.add_resource(Daily, '/daily', methods=['GET'], + resource_class_kwargs={'freqtrade': my_freqtrade}) # Route for returning daily + + #run the server + app.run(port='5002') diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py new file mode 100755 index 000000000..5b39b7a0b --- /dev/null +++ b/freqtrade/rpc/rest_client.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +""" +Simple command line client into RPC commands +Can be used as an alternate to Telegram +""" + +from requests import get +from sys import argv + +if len(argv) == 1: + print('\nThis script accepts the following arguments') + print('- daily (int) - Where int is the number of days to report back. daily 3') + print('- there will be more....\n') + +if len(argv) == 3 and argv[1] == "daily": + if str.isnumeric(argv[2]): + get_url = 'http://localhost:5002/daily?timescale=' + argv[2] + d=get(get_url).json() + print(d) + else: + print("\nThe second argument to daily must be an integer, 1,2,3 etc") + + From ec252ff7741f16299d715892a8edc5b674beb088 Mon Sep 17 00:00:00 2001 From: creslinux Date: Wed, 13 Jun 2018 22:26:21 +0000 Subject: [PATCH 03/26] Implemented local restful flask service and provided cmdline client Added only the "Daily" call so far, submitting for early review/feedback Called as example "./rest_client.py daily 3" This depends on listed as requirements. Flask==1.0.2 flask-jsonpify==1.5.0 (will do later) flask-restful==0.3.6 TODO: make loading optional, cleanly unload on close unit tests, take feedback, tidy output, add other Telegram functions, onwards local rest server is enabled/disabled from within config.json. E.g "localrest": { "enabled": true }, The server is enabled from within existing rpc manager and makes use of the existing superclass (RPC) Through making use of the existing hard work done in rpc.py It *should be easy to add the other Telegram calls into local_rpc_server.py The server is wrapped in a thread to be non-blocking The server and client accept serialised calls or not, used in daily to return json The client can be used from command line or in a python client script As example, from cmdline for last 3 days Daily DannyMBP:rpc creslin$ ./rest_client.py daily 3 [ [ "2018-06-13", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-12", "0.00000000 USDT", "0.000 USD", "0 trade" ], [ "2018-06-11", "0.00000000 USDT", "0.000 USD", "0 trade" ] ] --- freqtrade/rpc/local_rest_server.py | 2 +- freqtrade/rpc/rest_client.py | 2 +- freqtrade/rpc/rpc_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/local_rest_server.py index 940cc5942..0c9fad16b 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class Daily(Resource): # called by http://127.0.0.1:/daily?timescale=7 - # where 7 is the number of days to report back with. + # where 7 is the number of days to report back with def __init__(self, freqtrade) -> None: """ diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 5b39b7a0b..928e1798c 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3. """ Simple command line client into RPC commands Can be used as an alternate to Telegram diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 252bbcdd8..33e432cec 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,5 +1,5 @@ """ -This module contains class to manage RPC communications (Telegram, Slack, ...) +This module contains class to manage RPC communications (Telegram, Slack, ....) """ import logging from typing import List From 6cd4414874a35784daa3780acb630db430a11edc Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Wed, 13 Jun 2018 22:41:43 +0000 Subject: [PATCH 04/26] Update rest_client.py --- freqtrade/rpc/rest_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 928e1798c..5b39b7a0b 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3. +#!/usr/bin/env python3 """ Simple command line client into RPC commands Can be used as an alternate to Telegram From cbfa9e835521ec07f99a5485dd1250eb67fa0df8 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 15:38:26 +0000 Subject: [PATCH 05/26] Updated Stop and start calls added Along with refactoring to base line and use decorators. Also modules loaded optionally if enabled in config or not binds to ip / port set from config.json with warning if not localhost TODO: - use argparse in client, and generally clean client up - create unit test - documentation - extend to other RCP commands, after feedback --- freqtrade/rpc/local_rest_server.py | 122 ++++++++++++++++++----------- freqtrade/rpc/rest_client.py | 30 +++++++ 2 files changed, 106 insertions(+), 46 deletions(-) diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/local_rest_server.py index 0c9fad16b..24d9a6594 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -5,43 +5,11 @@ import json from flask import Flask, request from flask_restful import Resource, Api from json import dumps -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) - -class Daily(Resource): - # called by http://127.0.0.1:/daily?timescale=7 - # where 7 is the number of days to report back with - - def __init__(self, freqtrade) -> None: - """ - Initializes all enabled rpc modules - :param freqtrade: Instance of a freqtrade bot - :return: None - """ - self.freqtrade = freqtrade - self._config = freqtrade.config - - - def get(self): - timescale = request.args.get('timescale') - logger.info("LocalRPC - Daily Command Called") - timescale = int(timescale) - - (error, stats) = RPC.rpc_daily_profit(self, timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - if error == False: - stats = dumps(stats, indent=4, sort_keys=True, default=str) - return stats - else: - json.dumps(error) - return error - - class LocalRestSuperWrap(RPC): """ This class is for REST cmd line client @@ -61,23 +29,85 @@ class LocalRestSuperWrap(RPC): thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple thread.daemon = True # Daemonize thread - thread.start() # Start the execution + thread.start() # Start the execution + def run(self, freqtrade): """ Method that runs forever """ self._config = freqtrade.config - - - # TODO add IP address / port to bind to in config.json and use in below. - logger.info('Starting Local Rest Server') - - my_freqtrade = freqtrade app = Flask(__name__) - api = Api(app) - # Our resources for restful apps go here, pass freqtrade object across - api.add_resource(Daily, '/daily', methods=['GET'], - resource_class_kwargs={'freqtrade': my_freqtrade}) # Route for returning daily + """ + Define the application routes here + each Telegram command should have a like local substitute + """ + @app.route("/") + def hello(): + # For simple rest server testing via browser + # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status + # /status /table /performance /count, + # /start /stop /help' - #run the server - app.run(port='5002') + rest_cmds ='Commands implemented:
' \ + '/daily?timescale=7' \ + '
' \ + '/stop' \ + '
' \ + '/start' + return rest_cmds + + @app.route('/daily', methods=['GET']) + def daily(): + try: + timescale = request.args.get('timescale') + logger.info("LocalRPC - Daily Command Called") + timescale = int(timescale) + + stats = self._rpc_daily_profit(timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + + stats = dumps(stats, indent=4, sort_keys=True, default=str) + return stats + except RPCException as e: + return e + + @app.route('/start', methods=['GET']) + def start(): + """ + Handler for /start. + Starts TradeThread + """ + msg = self._rpc_start() + print("msg is", msg) + return msg + + @app.route('/stop', methods=['GET']) + def stop(): + """ + Handler for /stop. + Stops TradeThread + """ + msg = self._rpc_stop() + print("msg is", msg) + return msg + + """ + Section to handle configuration and running of the Rest serve + also to check and warn if not bound to 127.0.0.1 as a security risk + """ + + rest_ip = self._config['rest_cmd_line']['listen_ip_address'] + rest_port = self._config['rest_cmd_line']['listen_port'] + + if rest_ip != "127.0.0.1": + i=0 + while i < 10: + logger.info("SECURITY WARNING - Local Rest Server listening to external connections") + logger.info("SECURITY WARNING - This is insecure please set to 127.0.0.1 in config.json") + i += 1 + + # Run the Server + logger.info('Starting Local Rest Server') + app.run(host=rest_ip, port=rest_port) diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 5b39b7a0b..fa63cf4f7 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -4,12 +4,20 @@ Simple command line client into RPC commands Can be used as an alternate to Telegram """ +import time from requests import get from sys import argv + + +#TODO - use argparse to clean this up +#TODO - use IP and Port from config.json not hardcode + if len(argv) == 1: print('\nThis script accepts the following arguments') print('- daily (int) - Where int is the number of days to report back. daily 3') + print('- start - this will start the trading thread') + print('- stop - this will start the trading thread') print('- there will be more....\n') if len(argv) == 3 and argv[1] == "daily": @@ -20,4 +28,26 @@ if len(argv) == 3 and argv[1] == "daily": else: print("\nThe second argument to daily must be an integer, 1,2,3 etc") +if len(argv) == 2 and argv[1] == "start": + get_url = 'http://localhost:5002/start' + d = get(get_url).text + print(d) + + if "already" not in d: + time.sleep(2) + d = get(get_url).text + print(d) + +if len(argv) == 2 and argv[1] == "stop": + get_url = 'http://localhost:5002/stop' + d = get(get_url).text + print(d) + + if "already" not in d: + time.sleep(2) + d = get(get_url).text + print(d) + + + From b101a2608a7d8232d8eadf26e02b4930f6589d5b Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 15:44:12 +0000 Subject: [PATCH 06/26] Updated Stop and start calls added Along with refactoring to base line and use decorators. Also modules loaded optionally if enabled in config or not binds to ip / port set from config.json with warning if not localhost TODO: - use argparse in client, and generally clean client up - create unit test - documentation - extend to other RCP commands, after feedback --- freqtrade/rpc/local_rest_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/local_rest_server.py index 24d9a6594..92bb17be8 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/local_rest_server.py @@ -95,7 +95,7 @@ class LocalRestSuperWrap(RPC): """ Section to handle configuration and running of the Rest serve - also to check and warn if not bound to 127.0.0.1 as a security risk + also to check and warn if not bound to 127.0.0.1 as a security risk. """ rest_ip = self._config['rest_cmd_line']['listen_ip_address'] From 9aa08ec3c195631f59653eeb78b04ffa7005c785 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 14 Jun 2018 20:19:15 +0000 Subject: [PATCH 07/26] Added json validation of formats to check IPv4 refactored files and calls to be api_server worked down satisfying review comments of last commit --- freqtrade/configuration.py | 8 +++-- freqtrade/constants.py | 13 +++++++ .../{local_rest_server.py => api_server.py} | 35 ++++++++----------- 3 files changed, 33 insertions(+), 23 deletions(-) rename freqtrade/rpc/{local_rest_server.py => api_server.py} (78%) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 7c3a5eb4b..c994eb8c2 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -6,7 +6,7 @@ import json import logging from argparse import Namespace from typing import Optional, Dict, Any -from jsonschema import Draft4Validator, validate +from jsonschema import Draft4Validator, validate, draft4_format_checker from jsonschema.exceptions import ValidationError, best_match import ccxt @@ -202,7 +202,7 @@ class Configuration(object): :return: Returns the config if valid, otherwise throw an exception """ try: - validate(conf, constants.CONF_SCHEMA) + validate(conf, constants.CONF_SCHEMA, format_checker=draft4_format_checker) return conf except ValidationError as exception: logger.critical( @@ -210,7 +210,9 @@ class Configuration(object): exception ) raise ValidationError( - best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message + best_match(Draft4Validator(constants.CONF_SCHEMA, + format_checker=draft4_format_checker) + .iter_errors(conf)).message ) def get_config(self) -> Dict[str, Any]: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..28f8ec6f3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -92,6 +92,19 @@ CONF_SCHEMA = { }, 'required': ['enabled', 'token', 'chat_id'] }, + 'api_server': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'listen_ip_address': { "format": "ipv4"}, + 'listen_port': { + 'type': 'integer', + "minimum": 1024, + "maximum": 65535 + }, + }, + 'required': ['enabled', 'listen_ip_address', 'listen_port'] + }, 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'internals': { diff --git a/freqtrade/rpc/local_rest_server.py b/freqtrade/rpc/api_server.py similarity index 78% rename from freqtrade/rpc/local_rest_server.py rename to freqtrade/rpc/api_server.py index 92bb17be8..2a1ad0b8c 100644 --- a/freqtrade/rpc/local_rest_server.py +++ b/freqtrade/rpc/api_server.py @@ -6,26 +6,25 @@ from flask import Flask, request from flask_restful import Resource, Api from json import dumps from freqtrade.rpc.rpc import RPC, RPCException +from ipaddress import IPv4Address logger = logging.getLogger(__name__) +app = Flask(__name__) -class LocalRestSuperWrap(RPC): +class ApiServerSuperWrap(RPC): """ - This class is for REST cmd line client + This class is for REST calls across api server """ def __init__(self, freqtrade) -> None: """ - Init the LocalRestServer call, and init the super class RPC + Init the api server, and init the super class RPC :param freqtrade: Instance of a freqtrade bot :return: None """ super().__init__(freqtrade) - """ Constructor - :type interval: int - :param interval: Check interval, in seconds - """ - self.interval = int(1) + + self.interval = 1 thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple thread.daemon = True # Daemonize thread @@ -35,11 +34,10 @@ class LocalRestSuperWrap(RPC): def run(self, freqtrade): """ Method that runs forever """ self._config = freqtrade.config - app = Flask(__name__) """ - Define the application routes here - each Telegram command should have a like local substitute + Define the application routes here + each Telegram command should have a like local substitute """ @app.route("/") def hello(): @@ -97,17 +95,14 @@ class LocalRestSuperWrap(RPC): Section to handle configuration and running of the Rest serve also to check and warn if not bound to 127.0.0.1 as a security risk. """ + rest_ip = self._config['api_server']['listen_ip_address'] + rest_port = self._config['api_server']['listen_port'] - rest_ip = self._config['rest_cmd_line']['listen_ip_address'] - rest_port = self._config['rest_cmd_line']['listen_port'] - - if rest_ip != "127.0.0.1": - i=0 - while i < 10: - logger.info("SECURITY WARNING - Local Rest Server listening to external connections") - logger.info("SECURITY WARNING - This is insecure please set to 127.0.0.1 in config.json") - i += 1 + if not IPv4Address(rest_ip).is_loopback : + logger.info("SECURITY WARNING - Local Rest Server listening to external connections") + logger.info("SECURITY WARNING - This is insecure please set to your loopback, e.g 127.0.0.1 in config.json") # Run the Server logger.info('Starting Local Rest Server') app.run(host=rest_ip, port=rest_port) + From d7dfa00cb3c93f62754df2493fc6b38316e66292 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 15 Jun 2018 09:14:17 +0000 Subject: [PATCH 08/26] Moved from decorators to app.add_url_rule This has the benefit of creating a label which may be helpful if later refactoring. This change misses the main thrust of requests from both Gcarq and Shusso to better layout the code Im running into a challenge with 'self' not being available, or able to be passed in either to decorators or view_func This may simply be how I've instantiated an RPC in the wuperwrap or im within a thread - my very limited exposure to programming is at play! After moving code around lots of ways to no success and google not being helpful im committing for further feeback --- freqtrade/rpc/api_server.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 2a1ad0b8c..04150b4d7 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -1,9 +1,9 @@ import threading import logging -import json +# import json from flask import Flask, request -from flask_restful import Resource, Api +# from flask_restful import Resource, Api from json import dumps from freqtrade.rpc.rpc import RPC, RPCException from ipaddress import IPv4Address @@ -36,10 +36,10 @@ class ApiServerSuperWrap(RPC): self._config = freqtrade.config """ - Define the application routes here + Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute """ - @app.route("/") + # @app.route("/") def hello(): # For simple rest server testing via browser # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status @@ -54,7 +54,6 @@ class ApiServerSuperWrap(RPC): '/start' return rest_cmds - @app.route('/daily', methods=['GET']) def daily(): try: timescale = request.args.get('timescale') @@ -71,29 +70,36 @@ class ApiServerSuperWrap(RPC): except RPCException as e: return e - @app.route('/start', methods=['GET']) def start(): """ Handler for /start. Starts TradeThread """ msg = self._rpc_start() - print("msg is", msg) return msg - @app.route('/stop', methods=['GET']) def stop(): """ Handler for /stop. Stops TradeThread """ msg = self._rpc_stop() - print("msg is", msg) return msg + ## defines the url rules available on the api server + ''' + First two arguments passed are /URL and 'Label' + Label can be used as a shortcut when refactoring + ''' + app.add_url_rule('/', 'hello', view_func=hello, methods=['GET']) + app.add_url_rule('/stop', 'stop', view_func=stop, methods=['GET']) + app.add_url_rule('/start', 'start', view_func=start, methods=['GET']) + app.add_url_rule('/daily', 'daily', view_func=daily, methods=['GET']) + + """ - Section to handle configuration and running of the Rest serve - also to check and warn if not bound to 127.0.0.1 as a security risk. + Section to handle configuration and running of the Rest server + also to check and warn if not bound to a loopback, warn on security risk. """ rest_ip = self._config['api_server']['listen_ip_address'] rest_port = self._config['api_server']['listen_port'] @@ -104,5 +110,9 @@ class ApiServerSuperWrap(RPC): # Run the Server logger.info('Starting Local Rest Server') - app.run(host=rest_ip, port=rest_port) + try: + app.run(host=rest_ip, port=rest_port) + except: + logger.exception("Api server failed to start, exception message is:") + From 75de61843b14abf8fdb7f2a10ac1c81770393f2c Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 02:55:27 +0200 Subject: [PATCH 09/26] fix flake8 warnings --- freqtrade/constants.py | 2 +- freqtrade/rpc/api_server.py | 1 + freqtrade/rpc/rest_client.py | 10 ++-------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 28f8ec6f3..8b7d29f16 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -96,7 +96,7 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'enabled': {'type': 'boolean'}, - 'listen_ip_address': { "format": "ipv4"}, + 'listen_ip_address': {"format": "ipv4"}, 'listen_port': { 'type': 'integer', "minimum": 1024, diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 04150b4d7..9810782ef 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -12,6 +12,7 @@ from ipaddress import IPv4Address logger = logging.getLogger(__name__) app = Flask(__name__) + class ApiServerSuperWrap(RPC): """ This class is for REST calls across api server diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index fa63cf4f7..1e54bbafd 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -8,10 +8,8 @@ import time from requests import get from sys import argv - - -#TODO - use argparse to clean this up -#TODO - use IP and Port from config.json not hardcode +# TODO - use argparse to clean this up +# TODO - use IP and Port from config.json not hardcode if len(argv) == 1: print('\nThis script accepts the following arguments') @@ -47,7 +45,3 @@ if len(argv) == 2 and argv[1] == "stop": time.sleep(2) d = get(get_url).text print(d) - - - - From 446c62100662840e3c9bfb3074a8db62e15850e5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 02:56:10 +0200 Subject: [PATCH 10/26] move endpoint definitions to class scope --- freqtrade/rpc/api_server.py | 126 +++++++++++++++++------------------- 1 file changed, 60 insertions(+), 66 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 9810782ef..32d402719 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -25,78 +25,23 @@ class ApiServerSuperWrap(RPC): """ super().__init__(freqtrade) - self.interval = 1 - - thread = threading.Thread(target=self.run, args=(freqtrade,)) # extra comma as ref ! Tuple - thread.daemon = True # Daemonize thread - thread.start() # Start the execution - - - def run(self, freqtrade): - """ Method that runs forever """ self._config = freqtrade.config - """ - Define the application methods here, called by app.add_url_rule - each Telegram command should have a like local substitute - """ - # @app.route("/") - def hello(): - # For simple rest server testing via browser - # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status - # /status /table /performance /count, - # /start /stop /help' + thread = threading.Thread(target=self.run, daemon=True) + thread.start() - rest_cmds ='Commands implemented:
' \ - '/daily?timescale=7' \ - '
' \ - '/stop' \ - '
' \ - '/start' - return rest_cmds + def run(self): + """ Method that runs forever """ - def daily(): - try: - timescale = request.args.get('timescale') - logger.info("LocalRPC - Daily Command Called") - timescale = int(timescale) - - stats = self._rpc_daily_profit(timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] - ) - - stats = dumps(stats, indent=4, sort_keys=True, default=str) - return stats - except RPCException as e: - return e - - def start(): - """ - Handler for /start. - Starts TradeThread - """ - msg = self._rpc_start() - return msg - - def stop(): - """ - Handler for /stop. - Stops TradeThread - """ - msg = self._rpc_stop() - return msg - - ## defines the url rules available on the api server + # defines the url rules available on the api server ''' First two arguments passed are /URL and 'Label' Label can be used as a shortcut when refactoring ''' - app.add_url_rule('/', 'hello', view_func=hello, methods=['GET']) - app.add_url_rule('/stop', 'stop', view_func=stop, methods=['GET']) - app.add_url_rule('/start', 'start', view_func=start, methods=['GET']) - app.add_url_rule('/daily', 'daily', view_func=daily, methods=['GET']) - + app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) + app.add_url_rule('/stop', 'stop', view_func=self.stop, methods=['GET']) + app.add_url_rule('/start', 'start', view_func=self.start, methods=['GET']) + app.add_url_rule('/daily', 'daily', view_func=self.daily, methods=['GET']) """ Section to handle configuration and running of the Rest server @@ -105,7 +50,8 @@ class ApiServerSuperWrap(RPC): rest_ip = self._config['api_server']['listen_ip_address'] rest_port = self._config['api_server']['listen_port'] - if not IPv4Address(rest_ip).is_loopback : + logger.info('Starting HTTP Server at {}:{}'.format(rest_ip, rest_port)) + if not IPv4Address(rest_ip).is_loopback: logger.info("SECURITY WARNING - Local Rest Server listening to external connections") logger.info("SECURITY WARNING - This is insecure please set to your loopback, e.g 127.0.0.1 in config.json") @@ -113,7 +59,55 @@ class ApiServerSuperWrap(RPC): logger.info('Starting Local Rest Server') try: app.run(host=rest_ip, port=rest_port) - except: + except Exception: logger.exception("Api server failed to start, exception message is:") + """ + Define the application methods here, called by app.add_url_rule + each Telegram command should have a like local substitute + """ + def hello(self): + # For simple rest server testing via browser + # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status + # /status /table /performance /count, + # /start /stop /help' + rest_cmds = 'Commands implemented:
' \ + '/daily?timescale=7' \ + '
' \ + '/stop' \ + '
' \ + '/start' + return rest_cmds + + def daily(self): + try: + timescale = request.args.get('timescale') + logger.info("LocalRPC - Daily Command Called") + timescale = int(timescale) + + stats = self._rpc_daily_profit(timescale, + self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + + stats = dumps(stats, indent=4, sort_keys=True, default=str) + return stats + except RPCException as e: + return e + + def start(self): + """ + Handler for /start. + Starts TradeThread + """ + msg = self._rpc_start() + return msg + + def stop(self): + """ + Handler for /stop. + Stops TradeThread + """ + msg = self._rpc_stop() + return msg From 913cb80a33186e246a28559a614de71e5d4284ce Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 03:32:45 +0200 Subject: [PATCH 11/26] convert start, stop and reload_conf to return a dict --- freqtrade/rpc/api_server.py | 6 ++--- freqtrade/rpc/rest_client.py | 2 +- freqtrade/rpc/rpc.py | 29 ++++++++++++++++-------- freqtrade/rpc/telegram.py | 6 ++--- freqtrade/tests/rpc/test_rpc.py | 9 ++++---- freqtrade/tests/rpc/test_rpc_telegram.py | 4 ++-- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 32d402719..26520949d 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -2,7 +2,7 @@ import threading import logging # import json -from flask import Flask, request +from flask import Flask, request, jsonify # from flask_restful import Resource, Api from json import dumps from freqtrade.rpc.rpc import RPC, RPCException @@ -102,7 +102,7 @@ class ApiServerSuperWrap(RPC): Starts TradeThread """ msg = self._rpc_start() - return msg + return jsonify(msg) def stop(self): """ @@ -110,4 +110,4 @@ class ApiServerSuperWrap(RPC): Stops TradeThread """ msg = self._rpc_stop() - return msg + return jsonify(msg) diff --git a/freqtrade/rpc/rest_client.py b/freqtrade/rpc/rest_client.py index 1e54bbafd..cabedebb8 100755 --- a/freqtrade/rpc/rest_client.py +++ b/freqtrade/rpc/rest_client.py @@ -21,7 +21,7 @@ if len(argv) == 1: if len(argv) == 3 and argv[1] == "daily": if str.isnumeric(argv[2]): get_url = 'http://localhost:5002/daily?timescale=' + argv[2] - d=get(get_url).json() + d = get(get_url).json() print(d) else: print("\nThe second argument to daily must be an integer, 1,2,3 etc") diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ee6ecb770..e60315dc9 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -26,7 +26,17 @@ class RPCException(Exception): raise RPCException('*Status:* `no active trade`') """ - pass + def __init__(self, message: str) -> None: + super().__init__(self) + self.message = message + + def __str__(self): + return self.message + + def __json__(self): + return { + 'msg': self.message + } class RPC(object): @@ -287,28 +297,27 @@ class RPC(object): value = fiat.convert_amount(total, 'BTC', symbol) return output, total, symbol, value - def _rpc_start(self) -> str: + def _rpc_start(self) -> Dict[str, str]: """ Handler for start """ if self._freqtrade.state == State.RUNNING: - return '*Status:* `already running`' + return {'status': 'already running'} self._freqtrade.state = State.RUNNING - return '`Starting trader ...`' + return {'status': 'starting trader ...'} - def _rpc_stop(self) -> str: + def _rpc_stop(self) -> Dict[str, str]: """ Handler for stop """ if self._freqtrade.state == State.RUNNING: self._freqtrade.state = State.STOPPED - return '`Stopping trader ...`' + return {'status': 'stopping trader ...'} - return '*Status:* `already stopped`' + return {'status': 'already stopped'} - def _rpc_reload_conf(self) -> str: + def _rpc_reload_conf(self) -> Dict[str, str]: """ Handler for reload_conf. """ self._freqtrade.state = State.RELOAD_CONF - return '*Status:* `Reloading config ...`' + return {'status': 'reloading config ...'} - # FIX: no test for this!!!! def _rpc_forcesell(self, trade_id) -> None: """ Handler for forcesell . diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4dd23971b..a6538a32b 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -266,7 +266,7 @@ class Telegram(RPC): :return: None """ msg = self._rpc_start() - self._send_msg(msg, bot=bot) + self._send_msg('Status: `{status}`'.format(**msg), bot=bot) @authorized_only def _stop(self, bot: Bot, update: Update) -> None: @@ -278,7 +278,7 @@ class Telegram(RPC): :return: None """ msg = self._rpc_stop() - self._send_msg(msg, bot=bot) + self._send_msg('Status: `{status}`'.format(**msg), bot=bot) @authorized_only def _reload_conf(self, bot: Bot, update: Update) -> None: @@ -290,7 +290,7 @@ class Telegram(RPC): :return: None """ msg = self._rpc_reload_conf() - self._send_msg(msg, bot=bot) + self._send_msg('Status: `{status}`'.format(**msg), bot=bot) @authorized_only def _forcesell(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 11db7ffb3..07c635010 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -357,11 +357,11 @@ def test_rpc_start(mocker, default_conf) -> None: freqtradebot.state = State.STOPPED result = rpc._rpc_start() - assert '`Starting trader ...`' in result + assert {'status': 'starting trader ...'} == result assert freqtradebot.state == State.RUNNING result = rpc._rpc_start() - assert '*Status:* `already running`' in result + assert {'status': 'already running'} == result assert freqtradebot.state == State.RUNNING @@ -383,11 +383,12 @@ def test_rpc_stop(mocker, default_conf) -> None: freqtradebot.state = State.RUNNING result = rpc._rpc_stop() - assert '`Stopping trader ...`' in result + assert {'status': 'stopping trader ...'} == result assert freqtradebot.state == State.STOPPED result = rpc._rpc_stop() - assert '*Status:* `already stopped`' in result + + assert {'status': 'already stopped'} == result assert freqtradebot.state == State.STOPPED diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b2cca9b9a..b70375b48 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -663,7 +663,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: telegram._stop(bot=MagicMock(), update=update) assert freqtradebot.state == State.STOPPED assert msg_mock.call_count == 1 - assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] + assert 'stopping trader' in msg_mock.call_args_list[0][0][0] def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: @@ -707,7 +707,7 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: telegram._reload_conf(bot=MagicMock(), update=update) assert freqtradebot.state == State.RELOAD_CONF assert msg_mock.call_count == 1 - assert 'Reloading config' in msg_mock.call_args_list[0][0][0] + assert 'reloading config' in msg_mock.call_args_list[0][0][0] def test_forcesell_handle(default_conf, update, ticker, fee, From 62afba1bebb3ad46af5654ddba991914f6a41a3b Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 03:37:19 +0200 Subject: [PATCH 12/26] remove markdown formatting from exception string --- freqtrade/rpc/rpc.py | 22 +++++++++++----------- freqtrade/tests/rpc/test_rpc.py | 12 ++++++------ freqtrade/tests/rpc/test_rpc_telegram.py | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index e60315dc9..098c7160f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -72,9 +72,9 @@ class RPC(object): # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self._freqtrade.state != State.RUNNING: - raise RPCException('*Status:* `trader is not running`') + raise RPCException('trader is not running') elif not trades: - raise RPCException('*Status:* `no active trade`') + raise RPCException('no active trade') else: result = [] for trade in trades: @@ -118,9 +118,9 @@ class RPC(object): def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self._freqtrade.state != State.RUNNING: - raise RPCException('*Status:* `trader is not running`') + raise RPCException('trader is not running') elif not trades: - raise RPCException('*Status:* `no active order`') + raise RPCException('no active order') else: trades_list = [] for trade in trades: @@ -145,7 +145,7 @@ class RPC(object): profit_days: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): - raise RPCException('*Daily [n]:* `must be an integer greater than 0`') + raise RPCException('timescale must be an integer greater than 0') fiat = self._freqtrade.fiat_converter for day in range(0, timescale): @@ -225,7 +225,7 @@ class RPC(object): .order_by(sql.text('profit_sum DESC')).first() if not best_pair: - raise RPCException('*Status:* `no closed trade`') + raise RPCException('no closed trade') bp_pair, bp_rate = best_pair @@ -290,7 +290,7 @@ class RPC(object): } ) if total == 0.0: - raise RPCException('`All balances are zero.`') + raise RPCException('all balances are zero') fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency @@ -351,7 +351,7 @@ class RPC(object): # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: - raise RPCException('`trader is not running`') + raise RPCException('trader is not running') if trade_id == 'all': # Execute sell for all open orders @@ -368,7 +368,7 @@ class RPC(object): ).first() if not trade: logger.warning('forcesell: Invalid argument received') - raise RPCException('Invalid argument.') + raise RPCException('invalid argument') _exec_forcesell(trade) Trade.session.flush() @@ -379,7 +379,7 @@ class RPC(object): Shows a performance statistic from finished trades """ if self._freqtrade.state != State.RUNNING: - raise RPCException('`trader is not running`') + raise RPCException('trader is not running') pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), @@ -396,6 +396,6 @@ class RPC(object): def _rpc_count(self) -> List[Trade]: """ Returns the number of trades running """ if self._freqtrade.state != State.RUNNING: - raise RPCException('`trader is not running`') + raise RPCException('trader is not running') return Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 07c635010..56c7f2adc 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -91,11 +91,11 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*\*Status:\* `trader is not running``*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_status_table() freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*\*Status:\* `no active order`*'): + with pytest.raises(RPCException, match=r'.*no active order*'): rpc._rpc_status_table() freqtradebot.create_trade() @@ -421,11 +421,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_forcesell(None) freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*Invalid argument.*'): + with pytest.raises(RPCException, match=r'.*invalid argument*'): rpc._rpc_forcesell(None) rpc._rpc_forcesell('all') @@ -436,10 +436,10 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: rpc._rpc_forcesell('1') freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_forcesell(None) - with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_forcesell('all') freqtradebot.state = State.RUNNING diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b70375b48..3e4f19cc3 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -597,7 +597,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: telegram._balance(bot=MagicMock(), update=update) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 - assert '`All balances are zero.`' in result + assert 'all balances are zero' in result def test_start_handle(default_conf, update, mocker) -> None: @@ -865,7 +865,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: update.message.text = '/forcesell' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'Invalid argument' in msg_mock.call_args_list[0][0][0] + assert 'invalid argument' in msg_mock.call_args_list[0][0][0] # Invalid argument msg_mock.reset_mock() @@ -873,7 +873,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: update.message.text = '/forcesell 123456' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'Invalid argument.' in msg_mock.call_args_list[0][0][0] + assert 'invalid argument' in msg_mock.call_args_list[0][0][0] def test_performance_handle(default_conf, update, ticker, fee, From 37c70d2d3e7488a6d041abd342eade9af3abcea5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 03:54:10 +0200 Subject: [PATCH 13/26] return dict from _rpc_status and handle rendering in module impl --- freqtrade/rpc/telegram.py | 18 ++++++++++++-- freqtrade/tests/rpc/test_rpc.py | 31 +++++++++++------------- freqtrade/tests/rpc/test_rpc_telegram.py | 16 ++++++++++-- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a6538a32b..c045ffa1d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -136,8 +136,22 @@ class Telegram(RPC): return try: - for trade_msg in self._rpc_trade_status(): - self._send_msg(trade_msg, bot=bot) + results = self._rpc_trade_status() + messages = [ + "*Trade ID:* `{trade_id}`\n" + "*Current Pair:* [{pair}]({market_url})\n" + "*Open Since:* `{date}`\n" + "*Amount:* `{amount}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Close Rate:* `{close_rate}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Profit:* `{close_profit}`\n" + "*Current Profit:* `{current_profit:.2f}%`\n" + "*Open Order:* `{open_order}`".format(**result) + for result in results + ] + for msg in messages: + self._send_msg(msg, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 56c7f2adc..10ce5d6c7 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -52,24 +52,21 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: rpc._rpc_trade_status() freqtradebot.create_trade() - trades = rpc._rpc_trade_status() - trade = trades[0] + results = rpc._rpc_trade_status() - result_message = [ - '*Trade ID:* `1`\n' - '*Current Pair:* ' - '[ETH/BTC](https://bittrex.com/Market/Index?MarketName=BTC-ETH)\n' - '*Open Since:* `just now`\n' - '*Amount:* `90.99181074`\n' - '*Open Rate:* `0.00001099`\n' - '*Close Rate:* `None`\n' - '*Current Rate:* `0.00001098`\n' - '*Close Profit:* `None`\n' - '*Current Profit:* `-0.59%`\n' - '*Open Order:* `(limit buy rem=0.00000000)`' - ] - assert trades == result_message - assert trade.find('[ETH/BTC]') >= 0 + assert { + 'trade_id': 1, + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'date': 'just now', + 'open_rate': 1.099e-05, + 'close_rate': None, + 'current_rate': 1.098e-05, + 'amount': 90.99181074, + 'close_profit': None, + 'current_profit': -0.59, + 'open_order': '(limit buy rem=0.00000000)' + } == results[0] def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 3e4f19cc3..19f1a01a4 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -209,7 +209,19 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - _rpc_trade_status=MagicMock(return_value=[1, 2, 3]), + _rpc_trade_status=MagicMock(return_value=[{ + 'trade_id': 1, + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'date': 'just now', + 'open_rate': 1.099e-05, + 'close_rate': None, + 'current_rate': 1.098e-05, + 'amount': 90.99181074, + 'close_profit': None, + 'current_profit': -0.59, + 'open_order': '(limit buy rem=0.00000000)' + }]), _status_table=status_table, _send_msg=msg_mock ) @@ -223,7 +235,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: freqtradebot.create_trade() telegram._status(bot=MagicMock(), update=update) - assert msg_mock.call_count == 3 + assert msg_mock.call_count == 1 update.message.text = MagicMock() update.message.text.replace = MagicMock(return_value='table 2 3') From 2df03778329805e51c6ba776e71e9a5478aae123 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 04:08:51 +0200 Subject: [PATCH 14/26] refactor _rpc_balance --- freqtrade/rpc/rpc.py | 27 +++++++++++++++------------ freqtrade/rpc/telegram.py | 9 ++++----- freqtrade/tests/rpc/test_rpc.py | 21 +++++++++++---------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 098c7160f..9bf92aa18 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -5,7 +5,7 @@ import logging from abc import abstractmethod from datetime import datetime, timedelta, date from decimal import Decimal -from typing import Dict, Tuple, Any, List +from typing import Dict, Any, List import arrow import sqlalchemy as sql @@ -263,7 +263,7 @@ class RPC(object): 'best_rate': round(bp_rate * 100, 2), } - def _rpc_balance(self, fiat_display_currency: str) -> Tuple[List[Dict], float, str, float]: + def _rpc_balance(self, fiat_display_currency: str) -> Dict: """ Returns current account balance per crypto """ output = [] total = 0.0 @@ -280,22 +280,25 @@ class RPC(object): rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] est_btc: float = rate * balance['total'] total = total + est_btc - output.append( - { - 'currency': coin, - 'available': balance['free'], - 'balance': balance['total'], - 'pending': balance['used'], - 'est_btc': est_btc - } - ) + output.append({ + 'currency': coin, + 'available': balance['free'], + 'balance': balance['total'], + 'pending': balance['used'], + 'est_btc': est_btc, + }) if total == 0.0: raise RPCException('all balances are zero') fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) - return output, total, symbol, value + return { + 'currencies': output, + 'total': total, + 'symbol': symbol, + 'value': value, + } def _rpc_start(self) -> Dict[str, str]: """ Handler for start """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c045ffa1d..1d186f9df 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -253,10 +253,9 @@ class Telegram(RPC): def _balance(self, bot: Bot, update: Update) -> None: """ Handler for /balance """ try: - currencys, total, symbol, value = \ - self._rpc_balance(self._config['fiat_display_currency']) + result = self._rpc_balance(self._config['fiat_display_currency']) output = '' - for currency in currencys: + for currency in result['currencies']: output += "*{currency}:*\n" \ "\t`Available: {available: .8f}`\n" \ "\t`Balance: {balance: .8f}`\n" \ @@ -264,8 +263,8 @@ class Telegram(RPC): "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) output += "\n*Estimated Value*:\n" \ - "\t`BTC: {0: .8f}`\n" \ - "\t`{1}: {2: .2f}`\n".format(total, symbol, value) + "\t`BTC: {total: .8f}`\n" \ + "\t`{symbol}: {value: .2f}`\n".format(**result) self._send_msg(output, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 10ce5d6c7..ae8c46d7e 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -324,16 +324,17 @@ def test_rpc_balance_handle(default_conf, mocker): freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) - output, total, symbol, value = rpc._rpc_balance(default_conf['fiat_display_currency']) - assert prec_satoshi(total, 12) - assert prec_satoshi(value, 180000) - assert 'USD' in symbol - assert len(output) == 1 - assert 'BTC' in output[0]['currency'] - assert prec_satoshi(output[0]['available'], 10) - assert prec_satoshi(output[0]['balance'], 12) - assert prec_satoshi(output[0]['pending'], 2) - assert prec_satoshi(output[0]['est_btc'], 12) + result = rpc._rpc_balance(default_conf['fiat_display_currency']) + assert prec_satoshi(result['total'], 12) + assert prec_satoshi(result['value'], 180000) + assert 'USD' == result['symbol'] + assert result['currencies'] == [{ + 'currency': 'BTC', + 'available': 10.0, + 'balance': 12.0, + 'pending': 2.0, + 'est_btc': 12.0, + }] def test_rpc_start(mocker, default_conf) -> None: From a92f0c21250179e050104d07212d18f84d4d93f4 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 09:27:28 +0200 Subject: [PATCH 15/26] api_server: fix flake8 warnings and implement missing methods --- freqtrade/rpc/api_server.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 26520949d..8927584fe 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -53,7 +53,8 @@ class ApiServerSuperWrap(RPC): logger.info('Starting HTTP Server at {}:{}'.format(rest_ip, rest_port)) if not IPv4Address(rest_ip).is_loopback: logger.info("SECURITY WARNING - Local Rest Server listening to external connections") - logger.info("SECURITY WARNING - This is insecure please set to your loopback, e.g 127.0.0.1 in config.json") + logger.info("SECURITY WARNING - This is insecure please set to your loopback," + "e.g 127.0.0.1 in config.json") # Run the Server logger.info('Starting Local Rest Server') @@ -62,10 +63,24 @@ class ApiServerSuperWrap(RPC): except Exception: logger.exception("Api server failed to start, exception message is:") + def cleanup(self) -> None: + # TODO: implement me + raise NotImplementedError + + @property + def name(self) -> str: + # TODO: implement me + raise NotImplementedError + + def send_msg(self, msg: str) -> None: + # TODO: implement me + raise NotImplementedError + """ Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute """ + def hello(self): # For simple rest server testing via browser # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status From 4abfcd246217660f79a1aa2db2152ef9cc51644d Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 09:28:13 +0200 Subject: [PATCH 16/26] remove _rpc_status_table and reuse _rpc_status instead --- freqtrade/rpc/telegram.py | 48 +++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1d186f9df..90913b4a5 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -6,12 +6,15 @@ This module manage Telegram communication import logging from typing import Any, Callable +import arrow +from pandas import DataFrame from tabulate import tabulate from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ +from freqtrade.misc import shorten_date from freqtrade.rpc.rpc import RPC, RPCException logger = logging.getLogger(__name__) @@ -137,21 +140,25 @@ class Telegram(RPC): try: results = self._rpc_trade_status() - messages = [ - "*Trade ID:* `{trade_id}`\n" - "*Current Pair:* [{pair}]({market_url})\n" - "*Open Since:* `{date}`\n" - "*Amount:* `{amount}`\n" - "*Open Rate:* `{open_rate:.8f}`\n" - "*Close Rate:* `{close_rate}`\n" - "*Current Rate:* `{current_rate:.8f}`\n" - "*Close Profit:* `{close_profit}`\n" - "*Current Profit:* `{current_profit:.2f}%`\n" - "*Open Order:* `{open_order}`".format(**result) - for result in results - ] + messages = [] + for result in results: + result['open_date'] = arrow.get(result['open_date']).humanize() + messages.append( + "*Trade ID:* `{trade_id}`\n" + "*Current Pair:* [{pair}]({market_url})\n" + "*Open Since:* `{open_date}`\n" + "*Amount:* `{amount}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Close Rate:* `{close_rate}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Profit:* `{close_profit}`\n" + "*Current Profit:* `{current_profit:.2f}%`\n" + "*Open Order:* `{open_order}`".format(**result) + ) + for msg in messages: self._send_msg(msg, bot=bot) + except RPCException as e: self._send_msg(str(e), bot=bot) @@ -165,7 +172,20 @@ class Telegram(RPC): :return: None """ try: - df_statuses = self._rpc_status_table() + + results = self._rpc_trade_status() + data = [ + [ + result['trade_id'], + result['pair'], + shorten_date(arrow.get(result['open_date']).humanize(only_distance=True)), + result['current_profit'], + ] for result in results + ] + columns = ['ID', 'Pair', 'Since', 'Profit'] + df_statuses = DataFrame.from_records(data, columns=columns) + df_statuses = df_statuses.set_index(columns[0]) + message = tabulate(df_statuses, headers='keys', tablefmt='simple') self._send_msg("
{}
".format(message), parse_mode=ParseMode.HTML) except RPCException as e: From ce251a965e0e37e59970cefb409195816b0178eb Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 09:08:39 +0000 Subject: [PATCH 17/26] Moved registering application urls out of the run def and into their own Added 404 handling Split registration of URLs that use rpc.rpc and others into own def. Seems logical to be able to register separately for later use. --- freqtrade/rpc/api_server.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 8927584fe..945e870bc 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -27,13 +27,19 @@ class ApiServerSuperWrap(RPC): self._config = freqtrade.config + # Register application handling + self.register_rest_other() + self.register_rest_rpc_urls() + thread = threading.Thread(target=self.run, daemon=True) thread.start() - def run(self): - """ Method that runs forever """ + def register_rest_other(self): + #Added as a placeholder for app URLs that are not implemented in rpc.rpc + app.register_error_handler(404, self.page_not_found) - # defines the url rules available on the api server + def register_rest_rpc_urls(self): + # register the url rules available on the api server ''' First two arguments passed are /URL and 'Label' Label can be used as a shortcut when refactoring @@ -43,6 +49,9 @@ class ApiServerSuperWrap(RPC): app.add_url_rule('/start', 'start', view_func=self.start, methods=['GET']) app.add_url_rule('/daily', 'daily', view_func=self.daily, methods=['GET']) + def run(self): + """ Method that runs forever """ + """ Section to handle configuration and running of the Rest server also to check and warn if not bound to a loopback, warn on security risk. @@ -81,6 +90,12 @@ class ApiServerSuperWrap(RPC): each Telegram command should have a like local substitute """ + def page_not_found(self, error): + # return "404 not found", 404 + return jsonify({'status': 'error', + 'reason': '''There's no API call for %s''' % request.base_url, + 'code': 404}), 404 + def hello(self): # For simple rest server testing via browser # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status From f590f514f30e6895a1ed6cd0ff4e11a3f6f81fb1 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 09:19:34 +0000 Subject: [PATCH 18/26] moved default page "/" index into self_register_other() out of the block of URLs that call rpc.rcp functionality. --- freqtrade/rpc/api_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 945e870bc..7c8d33231 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -37,14 +37,16 @@ class ApiServerSuperWrap(RPC): def register_rest_other(self): #Added as a placeholder for app URLs that are not implemented in rpc.rpc app.register_error_handler(404, self.page_not_found) + app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) def register_rest_rpc_urls(self): # register the url rules available on the api server + # This is where to register rest urls that make use of + # rpc.rpc functions ''' First two arguments passed are /URL and 'Label' Label can be used as a shortcut when refactoring ''' - app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) app.add_url_rule('/stop', 'stop', view_func=self.stop, methods=['GET']) app.add_url_rule('/start', 'start', view_func=self.start, methods=['GET']) app.add_url_rule('/daily', 'daily', view_func=self.daily, methods=['GET']) From 2ac6a225befa4c8be6cdf6cdb27d693ccd8e4fa0 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 09:22:25 +0000 Subject: [PATCH 19/26] flake 8 fix --- freqtrade/rpc/api_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 7c8d33231..3b51126a9 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -35,7 +35,7 @@ class ApiServerSuperWrap(RPC): thread.start() def register_rest_other(self): - #Added as a placeholder for app URLs that are not implemented in rpc.rpc + # Added as a placeholder for app URLs that are not implemented in rpc.rpc app.register_error_handler(404, self.page_not_found) app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) From dc1aebf391a6e4eec485da43fc62069f0df6ed30 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 09:48:51 +0000 Subject: [PATCH 20/26] Updated def comments to be __docstring__ compatible --- freqtrade/rpc/api_server.py | 40 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 3b51126a9..3a8391b98 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -35,24 +35,27 @@ class ApiServerSuperWrap(RPC): thread.start() def register_rest_other(self): - # Added as a placeholder for app URLs that are not implemented in rpc.rpc + """ + Registers flask app URLs that are not calls to functionality in rpc.rpc. + :return: + """ app.register_error_handler(404, self.page_not_found) app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) def register_rest_rpc_urls(self): - # register the url rules available on the api server - # This is where to register rest urls that make use of - # rpc.rpc functions - ''' + """ + Registers flask app URLs that are calls to functonality in rpc.rpc. + First two arguments passed are /URL and 'Label' Label can be used as a shortcut when refactoring - ''' + :return: + """ app.add_url_rule('/stop', 'stop', view_func=self.stop, methods=['GET']) app.add_url_rule('/start', 'start', view_func=self.start, methods=['GET']) app.add_url_rule('/daily', 'daily', view_func=self.daily, methods=['GET']) def run(self): - """ Method that runs forever """ + """ Method that runs flask app in its own thread forever """ """ Section to handle configuration and running of the Rest server @@ -93,17 +96,19 @@ class ApiServerSuperWrap(RPC): """ def page_not_found(self, error): - # return "404 not found", 404 + # Return "404 not found", 404. return jsonify({'status': 'error', 'reason': '''There's no API call for %s''' % request.base_url, 'code': 404}), 404 def hello(self): - # For simple rest server testing via browser - # cmds = 'Try uri:/daily?timescale=7 /profit /balance /status - # /status /table /performance /count, - # /start /stop /help' + """ + None critical but helpful default index page. + That lists URLs added to the flask server. + This may be deprecated at any time. + :return: index.html + """ rest_cmds = 'Commands implemented:
' \ '/daily?timescale=7' \ '
' \ @@ -113,6 +118,11 @@ class ApiServerSuperWrap(RPC): return rest_cmds def daily(self): + """ + Returns the last X days trading stats summary. + + :return: stats + """ try: timescale = request.args.get('timescale') logger.info("LocalRPC - Daily Command Called") @@ -131,7 +141,8 @@ class ApiServerSuperWrap(RPC): def start(self): """ Handler for /start. - Starts TradeThread + + Starts TradeThread in bot if stopped. """ msg = self._rpc_start() return jsonify(msg) @@ -139,7 +150,8 @@ class ApiServerSuperWrap(RPC): def stop(self): """ Handler for /stop. - Stops TradeThread + + Stops TradeThread in bot if running """ msg = self._rpc_stop() return jsonify(msg) From ab2cf1bbfee0a124cfe3aa5537587692a9cfa306 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 11:53:09 +0000 Subject: [PATCH 21/26] Added api server shutdown function, and exposed on HTTP as /stop_api url This will stop the running app gracefully - processing current api calls then shutting the werkzueg (run) listening server. Have also called this from the cleanup placeholder. I'm not sure this is what is intended by cleanup def. By which I mean there may be a thread left running with no app within - not sure how to check this just yet. tidied excessive logging. --- freqtrade/rpc/api_server.py | 49 +++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 3a8391b98..d89bee5f9 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -41,6 +41,7 @@ class ApiServerSuperWrap(RPC): """ app.register_error_handler(404, self.page_not_found) app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) + app.add_url_rule('/stop_api', 'stop_api', view_func=self.stop_api, methods=['GET']) def register_rest_rpc_urls(self): """ @@ -77,10 +78,6 @@ class ApiServerSuperWrap(RPC): except Exception: logger.exception("Api server failed to start, exception message is:") - def cleanup(self) -> None: - # TODO: implement me - raise NotImplementedError - @property def name(self) -> str: # TODO: implement me @@ -90,11 +87,43 @@ class ApiServerSuperWrap(RPC): # TODO: implement me raise NotImplementedError + def shutdown_api_server(self): + """ + Stop the running flask application + + Records the shutdown in logger.info + :return: + """ + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running the Flask Werkzeug Server') + if func is not None: + logger.info('Stopping the Local Rest Server') + func() + return + + def cleanup(self) -> None: + """ + Stops the running application server + + Does not stop the thread,this may not be the desired outcome of cleanup. TBC + :return: + """ + self.shutdown_api_server() + # def cleanup(self) -> None: + # # TODO: implement me + # raise NotImplementedError + """ Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute """ + def stop_api(self): + """ For calling shutdown_api_server over via api server HTTP""" + self.shutdown_api_server() + return 'Api Server shutting down... ' + def page_not_found(self, error): # Return "404 not found", 404. return jsonify({'status': 'error', @@ -110,11 +139,16 @@ class ApiServerSuperWrap(RPC): :return: index.html """ rest_cmds = 'Commands implemented:
' \ - '/daily?timescale=7' \ + 'Show 7 days of stats' \ '
' \ - '/stop' \ + 'Stop the Trade thread' \ '
' \ - '/start' + 'Start the Traded thread' \ + '
' \ + ' 404 page does not exist' \ + '
' \ + '
' \ + 'Shut down the api server - be sure' return rest_cmds def daily(self): @@ -125,7 +159,6 @@ class ApiServerSuperWrap(RPC): """ try: timescale = request.args.get('timescale') - logger.info("LocalRPC - Daily Command Called") timescale = int(timescale) stats = self._rpc_daily_profit(timescale, From 60b24a8aa7d6bc12629a64dded880fee99bc5439 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 12:17:00 +0000 Subject: [PATCH 22/26] Added api server shutdown function, and exposed on HTTP as /stop_api url This will stop the running app gracefully - processing current api calls then shutting the werkzueg (run) listening server. Have also called this from the cleanup placeholder. I'm not sure this is what is intended by cleanup def. By which I mean there may be a thread left running with no app within - not sure how to check this just yet. tidied excessive logging. --- freqtrade/rpc/api_server.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index d89bee5f9..40eb39943 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -1,3 +1,4 @@ +import json import threading import logging # import json @@ -13,7 +14,7 @@ logger = logging.getLogger(__name__) app = Flask(__name__) -class ApiServerSuperWrap(RPC): +class ApiServer(RPC): """ This class is for REST calls across api server """ @@ -78,14 +79,8 @@ class ApiServerSuperWrap(RPC): except Exception: logger.exception("Api server failed to start, exception message is:") - @property - def name(self) -> str: - # TODO: implement me - raise NotImplementedError - def send_msg(self, msg: str) -> None: - # TODO: implement me - raise NotImplementedError + pass def shutdown_api_server(self): """ @@ -111,14 +106,12 @@ class ApiServerSuperWrap(RPC): """ self.shutdown_api_server() # def cleanup(self) -> None: - # # TODO: implement me - # raise NotImplementedError + # pass """ Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute """ - def stop_api(self): """ For calling shutdown_api_server over via api server HTTP""" self.shutdown_api_server() @@ -178,7 +171,7 @@ class ApiServerSuperWrap(RPC): Starts TradeThread in bot if stopped. """ msg = self._rpc_start() - return jsonify(msg) + return json.dumps(msg) def stop(self): """ @@ -187,4 +180,4 @@ class ApiServerSuperWrap(RPC): Stops TradeThread in bot if running """ msg = self._rpc_stop() - return jsonify(msg) + return json.dumps(msg) From 0b061fda97821ae64aa7203a9d95b1ac8cf05a38 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 12:38:04 +0000 Subject: [PATCH 23/26] removed change to cleanup() in api_server.py --- freqtrade/rpc/api_server.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 40eb39943..5f82b307f 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -79,6 +79,9 @@ class ApiServer(RPC): except Exception: logger.exception("Api server failed to start, exception message is:") + def cleanup(self) -> None: + pass + def send_msg(self, msg: str) -> None: pass @@ -97,17 +100,6 @@ class ApiServer(RPC): func() return - def cleanup(self) -> None: - """ - Stops the running application server - - Does not stop the thread,this may not be the desired outcome of cleanup. TBC - :return: - """ - self.shutdown_api_server() - # def cleanup(self) -> None: - # pass - """ Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute From 63c16b7f838c58b00e3c520bbbe1a0f7de17c736 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 23 Jun 2018 14:04:15 +0000 Subject: [PATCH 24/26] Moved routes that do not need access to rpc.rpc self into their own common file. This is to reduce file size and separate api server routes with privilege to access rpc.rpc defs and those that do not need access, so should not. --- freqtrade/rpc/api_server.py | 72 ++++++----------------------- freqtrade/rpc/api_server_common.py | 74 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 57 deletions(-) create mode 100644 freqtrade/rpc/api_server_common.py diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 5f82b307f..e1897ff3d 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -1,22 +1,32 @@ import json import threading import logging -# import json -from flask import Flask, request, jsonify -# from flask_restful import Resource, Api +from flask import request from json import dumps from freqtrade.rpc.rpc import RPC, RPCException from ipaddress import IPv4Address +from freqtrade.rpc.api_server_common import MyApiApp logger = logging.getLogger(__name__) -app = Flask(__name__) +""" +api server routes that do not need access to rpc.rpc +are held within api_server_common.api_server +""" +app = MyApiApp(__name__) class ApiServer(RPC): """ - This class is for REST calls across api server + This class runs api server and provides rpc.rpc functionality to it + + This class starts a none blocking thread the api server runs within + Any routes that require access to rpc.rpc defs are held within this + class. + + Any routes that do not require access to rpc.rcp should be registered + in api_server_common.MyApiApp """ def __init__(self, freqtrade) -> None: """ @@ -29,21 +39,11 @@ class ApiServer(RPC): self._config = freqtrade.config # Register application handling - self.register_rest_other() self.register_rest_rpc_urls() thread = threading.Thread(target=self.run, daemon=True) thread.start() - def register_rest_other(self): - """ - Registers flask app URLs that are not calls to functionality in rpc.rpc. - :return: - """ - app.register_error_handler(404, self.page_not_found) - app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) - app.add_url_rule('/stop_api', 'stop_api', view_func=self.stop_api, methods=['GET']) - def register_rest_rpc_urls(self): """ Registers flask app URLs that are calls to functonality in rpc.rpc. @@ -85,21 +85,6 @@ class ApiServer(RPC): def send_msg(self, msg: str) -> None: pass - def shutdown_api_server(self): - """ - Stop the running flask application - - Records the shutdown in logger.info - :return: - """ - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running the Flask Werkzeug Server') - if func is not None: - logger.info('Stopping the Local Rest Server') - func() - return - """ Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute @@ -109,33 +94,6 @@ class ApiServer(RPC): self.shutdown_api_server() return 'Api Server shutting down... ' - def page_not_found(self, error): - # Return "404 not found", 404. - return jsonify({'status': 'error', - 'reason': '''There's no API call for %s''' % request.base_url, - 'code': 404}), 404 - - def hello(self): - """ - None critical but helpful default index page. - - That lists URLs added to the flask server. - This may be deprecated at any time. - :return: index.html - """ - rest_cmds = 'Commands implemented:
' \ - 'Show 7 days of stats' \ - '
' \ - 'Stop the Trade thread' \ - '
' \ - 'Start the Traded thread' \ - '
' \ - ' 404 page does not exist' \ - '
' \ - '
' \ - 'Shut down the api server - be sure' - return rest_cmds - def daily(self): """ Returns the last X days trading stats summary. diff --git a/freqtrade/rpc/api_server_common.py b/freqtrade/rpc/api_server_common.py new file mode 100644 index 000000000..19338a825 --- /dev/null +++ b/freqtrade/rpc/api_server_common.py @@ -0,0 +1,74 @@ +import logging +import flask +from flask import request, jsonify + +logger = logging.getLogger(__name__) + + +class MyApiApp(flask.Flask): + def __init__(self, import_name): + """ + Contains common rest routes and resource that do not need + to access to rpc.rpc functionality + """ + super(MyApiApp, self).__init__(import_name) + + """ + Registers flask app URLs that are not calls to functionality in rpc.rpc. + :return: + """ + self.before_request(self.my_preprocessing) + self.register_error_handler(404, self.page_not_found) + self.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) + self.add_url_rule('/stop_api', 'stop_api', view_func=self.stop_api, methods=['GET']) + + def my_preprocessing(self): + # Do stuff to flask.request + pass + + def page_not_found(self, error): + # Return "404 not found", 404. + return jsonify({'status': 'error', + 'reason': '''There's no API call for %s''' % request.base_url, + 'code': 404}), 404 + + def hello(self): + """ + None critical but helpful default index page. + + That lists URLs added to the flask server. + This may be deprecated at any time. + :return: index.html + """ + rest_cmds = 'Commands implemented:
' \ + 'Show 7 days of stats' \ + '
' \ + 'Stop the Trade thread' \ + '
' \ + 'Start the Traded thread' \ + '
' \ + ' 404 page does not exist' \ + '
' \ + '
' \ + 'Shut down the api server - be sure' + return rest_cmds + + def stop_api(self): + """ For calling shutdown_api_server over via api server HTTP""" + self.shutdown_api_server() + return 'Api Server shutting down... ' + + def shutdown_api_server(self): + """ + Stop the running flask application + + Records the shutdown in logger.info + :return: + """ + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running the Flask Werkzeug Server') + if func is not None: + logger.info('Stopping the Local Rest Server') + func() + return From 95ba016558f8a0167df5240c5959885933905079 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 25 Jun 2018 19:55:28 +0000 Subject: [PATCH 25/26] rpc.py Added RPCExcpetion if trades is empty on trade.query api_server Return e on exception created server 500 error. Returned text for client, and sent e to logger Added profit and status table functions. Will look to add unit tests for these two --- freqtrade/rpc/api_server.py | 116 +++++++++++++++++++++++++++++------- freqtrade/rpc/rpc.py | 88 +++++++++------------------ 2 files changed, 122 insertions(+), 82 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index e1897ff3d..e117c1362 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -1,33 +1,27 @@ import json import threading import logging +# import json +from typing import Dict -from flask import request +from flask import Flask, request +# from flask_restful import Resource, Api from json import dumps from freqtrade.rpc.rpc import RPC, RPCException from ipaddress import IPv4Address -from freqtrade.rpc.api_server_common import MyApiApp logger = logging.getLogger(__name__) -""" -api server routes that do not need access to rpc.rpc -are held within api_server_common.api_server -""" -app = MyApiApp(__name__) +app = Flask(__name__) class ApiServer(RPC): """ + This class is for REST calls across api server This class runs api server and provides rpc.rpc functionality to it - This class starts a none blocking thread the api server runs within - Any routes that require access to rpc.rpc defs are held within this - class. - - Any routes that do not require access to rpc.rcp should be registered - in api_server_common.MyApiApp - """ + This class starts a none blocking thread the api server runs within\ + """ def __init__(self, freqtrade) -> None: """ Init the api server, and init the super class RPC @@ -39,11 +33,20 @@ class ApiServer(RPC): self._config = freqtrade.config # Register application handling + self.register_rest_other() self.register_rest_rpc_urls() thread = threading.Thread(target=self.run, daemon=True) thread.start() + def register_rest_other(self): + """ + Registers flask app URLs that are not calls to functionality in rpc.rpc. + :return: + """ + app.register_error_handler(404, self.page_not_found) + app.add_url_rule('/', 'hello', view_func=self.hello, methods=['GET']) + def register_rest_rpc_urls(self): """ Registers flask app URLs that are calls to functonality in rpc.rpc. @@ -55,6 +58,9 @@ class ApiServer(RPC): app.add_url_rule('/stop', 'stop', view_func=self.stop, methods=['GET']) app.add_url_rule('/start', 'start', view_func=self.start, methods=['GET']) app.add_url_rule('/daily', 'daily', view_func=self.daily, methods=['GET']) + app.add_url_rule('/profit', 'profit', view_func=self.profit, methods=['GET']) + app.add_url_rule('/status_table', 'status_table', + view_func=self.status_table, methods=['GET']) def run(self): """ Method that runs flask app in its own thread forever """ @@ -82,17 +88,47 @@ class ApiServer(RPC): def cleanup(self) -> None: pass - def send_msg(self, msg: str) -> None: + def send_msg(self, msg: Dict[str, str]) -> None: pass """ Define the application methods here, called by app.add_url_rule each Telegram command should have a like local substitute """ - def stop_api(self): - """ For calling shutdown_api_server over via api server HTTP""" - self.shutdown_api_server() - return 'Api Server shutting down... ' + + def page_not_found(self, error): + """ + Return "404 not found", 404. + """ + return json.dumps({ + 'status': 'error', + 'reason': '''There's no API call for %s''' % request.base_url, + 'code': 404 + }), 404 + + def hello(self): + """ + None critical but helpful default index page. + + That lists URLs added to the flask server. + This may be deprecated at any time. + :return: index.html + """ + rest_cmds = 'Commands implemented:
' \ + 'Show 7 days of stats' \ + '
' \ + 'Stop the Trade thread' \ + '
' \ + 'Start the Traded thread' \ + '
' \ + 'Show profit summary' \ + '
' \ + 'Show status table - Open trades' \ + '
' \ + ' 404 page does not exist' \ + '
' + + return rest_cmds def daily(self): """ @@ -102,6 +138,7 @@ class ApiServer(RPC): """ try: timescale = request.args.get('timescale') + logger.info("LocalRPC - Daily Command Called") timescale = int(timescale) stats = self._rpc_daily_profit(timescale, @@ -109,10 +146,45 @@ class ApiServer(RPC): self._config['fiat_display_currency'] ) - stats = dumps(stats, indent=4, sort_keys=True, default=str) - return stats + return json.dumps(stats, indent=4, sort_keys=True, default=str) except RPCException as e: - return e + logger.exception("API Error querying daily:", e) + return "Error querying daily" + + def profit(self): + """ + Handler for /profit. + + Returns a cumulative profit statistics + :return: stats + """ + try: + logger.info("LocalRPC - Profit Command Called") + + stats = self._rpc_trade_statistics(self._config['stake_currency'], + self._config['fiat_display_currency'] + ) + + return json.dumps(stats, indent=4, sort_keys=True, default=str) + except RPCException as e: + logger.exception("API Error calling profit", e) + return "Error querying closed trades - maybe there are none" + + def status_table(self): + """ + Handler for /status table. + + Returns the current TradeThread status in table format + :return: results + """ + try: + results = self._rpc_trade_status() + return json.dumps(results, indent=4, sort_keys=True, default=str) + + except RPCException as e: + logger.exception("API Error calling status table", e) + return "Error querying open trades - maybe there are none." + def start(self): """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9bf92aa18..ce3d62ed7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -5,14 +5,12 @@ import logging from abc import abstractmethod from datetime import datetime, timedelta, date from decimal import Decimal -from typing import Dict, Any, List +from typing import Dict, List, Any import arrow import sqlalchemy as sql from numpy import mean, nan_to_num -from pandas import DataFrame -from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State @@ -51,20 +49,20 @@ class RPC(object): """ self._freqtrade = freqtrade + @property + def name(self) -> str: + """ Returns the lowercase name of the implementation """ + return self.__class__.__name__.lower() + @abstractmethod def cleanup(self) -> None: """ Cleanup pending module resources """ - @property @abstractmethod - def name(self) -> str: - """ Returns the lowercase name of this module """ - - @abstractmethod - def send_msg(self, msg: str) -> None: + def send_msg(self, msg: Dict[str, str]) -> None: """ Sends a message to all registered rpc modules """ - def _rpc_trade_status(self) -> List[str]: + def _rpc_trade_status(self) -> List[Dict]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function @@ -76,7 +74,7 @@ class RPC(object): elif not trades: raise RPCException('no active trade') else: - result = [] + results = [] for trade in trades: order = None if trade.open_order_id: @@ -87,56 +85,23 @@ class RPC(object): fmt_close_profit = '{:.2f}%'.format( round(trade.close_profit * 100, 2) ) if trade.close_profit else None - message = "*Trade ID:* `{trade_id}`\n" \ - "*Current Pair:* [{pair}]({market_url})\n" \ - "*Open Since:* `{date}`\n" \ - "*Amount:* `{amount}`\n" \ - "*Open Rate:* `{open_rate:.8f}`\n" \ - "*Close Rate:* `{close_rate}`\n" \ - "*Current Rate:* `{current_rate:.8f}`\n" \ - "*Close Profit:* `{close_profit}`\n" \ - "*Current Profit:* `{current_profit:.2f}%`\n" \ - "*Open Order:* `{open_order}`"\ - .format( - trade_id=trade.id, - pair=trade.pair, - market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} {} rem={:.8f})'.format( - order['type'], order['side'], order['remaining'] - ) if order else None, - ) - result.append(message) - return result - def _rpc_status_table(self) -> DataFrame: - trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self._freqtrade.state != State.RUNNING: - raise RPCException('trader is not running') - elif not trades: - raise RPCException('no active order') - else: - trades_list = [] - for trade in trades: - # calculate profit and send message to user - current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] - trades_list.append([ - trade.id, - trade.pair, - shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) - ]) - - columns = ['ID', 'Pair', 'Since', 'Profit'] - df_statuses = DataFrame.from_records(trades_list, columns=columns) - df_statuses = df_statuses.set_index(columns[0]) - return df_statuses + results.append(dict( + trade_id=trade.id, + pair=trade.pair, + market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), + open_date=arrow.get(trade.open_date), + open_rate=trade.open_rate, + close_rate=trade.close_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + close_profit=fmt_close_profit, + current_profit=round(current_profit * 100, 2), + open_order='({} {} rem={:.8f})'.format( + order['type'], order['side'], order['remaining'] + ) if order else None, + )) + return results def _rpc_daily_profit( self, timescale: int, @@ -190,6 +155,9 @@ class RPC(object): """ Returns cumulative profit statistics """ trades = Trade.query.order_by(Trade.id).all() + if not trades: + raise RPCException('No trades found') + profit_all_coin = [] profit_all_percent = [] profit_closed_coin = [] From b7b9fd1fd511867b5cc3532959237ea6b5eb19c3 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 25 Jun 2018 20:06:53 +0000 Subject: [PATCH 26/26] flake 8 fix --- freqtrade/rpc/api_server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index e117c1362..7ef96a8b5 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -6,7 +6,6 @@ from typing import Dict from flask import Flask, request # from flask_restful import Resource, Api -from json import dumps from freqtrade.rpc.rpc import RPC, RPCException from ipaddress import IPv4Address @@ -162,8 +161,8 @@ class ApiServer(RPC): logger.info("LocalRPC - Profit Command Called") stats = self._rpc_trade_statistics(self._config['stake_currency'], - self._config['fiat_display_currency'] - ) + self._config['fiat_display_currency'] + ) return json.dumps(stats, indent=4, sort_keys=True, default=str) except RPCException as e: @@ -185,7 +184,6 @@ class ApiServer(RPC): logger.exception("API Error calling status table", e) return "Error querying open trades - maybe there are none." - def start(self): """ Handler for /start.