From e37e7bd914ab4a1073bab451ecd4a9dfc55bda8c Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 24 May 2018 14:05:29 -0700 Subject: [PATCH] added support for backtest ranges to collect more data and initial support for pagination --- freqtrade/aws/backtesting_lambda.py | 20 ++++- freqtrade/aws/strategy.py | 23 ++++- freqtrade/tests/aws/test_backtest.py | 94 +++++++++++++++++++++ freqtrade/tests/aws/test_strategy_lambda.py | 2 +- serverless.yml | 14 +++ 5 files changed, 145 insertions(+), 8 deletions(-) diff --git a/freqtrade/aws/backtesting_lambda.py b/freqtrade/aws/backtesting_lambda.py index d006f8a50..130bca827 100644 --- a/freqtrade/aws/backtesting_lambda.py +++ b/freqtrade/aws/backtesting_lambda.py @@ -57,13 +57,17 @@ def backtest(event, context): ) - print(response) + today = datetime.datetime.today() + yesterday = today - datetime.timedelta(days=1) + + if 'from' in event['body']: + yesterday = datetime.datetime.strptime(event['body']['from'], '%Y%m%d') + if 'till' in event['body']: + yesterday = datetime.datetime.strptime(event['body']['till'], '%Y%m%d') + try: if "Items" in response and len(response['Items']) > 0: - today = datetime.datetime.today() - yesterday = today - datetime.timedelta(days=1) - content = response['Items'][0]['content'] configuration = { "max_open_trades": 1, @@ -189,6 +193,14 @@ def cron(event, context): "name": i['name'] } + # triggered over html, let's provide + # a date range for the backtesting + if 'pathParameters' in event: + if 'from' in event['pathParameters']: + message['from'] = event['pathParameters']['from'] + if 'till' in event['pathParameters']: + message['till'] = event['pathParameters']['till'] + serialized = json.dumps(message, use_decimal=True) # submit item to queue for routing to the correct persistence diff --git a/freqtrade/aws/strategy.py b/freqtrade/aws/strategy.py index d6b7f4fcc..9b2e0a42d 100644 --- a/freqtrade/aws/strategy.py +++ b/freqtrade/aws/strategy.py @@ -26,6 +26,7 @@ def names(event, context): response = table.scan() result = response['Items'] + # no pagination here while 'LastEvaluatedKey' in response: for i in response['Items']: result.append(i) @@ -36,6 +37,10 @@ def names(event, context): # map results and hide informations data = list(map(lambda x: {'name': x['name'], 'public': x['public'], 'user': x['user']}, result)) + # keep in a result object, so we can later add pagination to it + data = { + "result": data + } return { "statusCode": 200, "body": json.dumps(data) @@ -304,13 +309,25 @@ def get_trades(event, context): if "Items" in response and len(response['Items']) > 0: + # preparation for pagination + # TODO include in parameters an optional + # start key ExclusiveStartKey=response['LastEvaluatedKey'] + + data = { + "result": response['Items'], + "paginationKey": response.get('LastEvaluatedKey') + } + return { "statusCode": response['ResponseMetadata']['HTTPStatusCode'], - "body": json.dumps(response['Items']) + "body": json.dumps(data) } else: return { - "statusCode": response['ResponseMetadata']['HTTPStatusCode'], - "body": json.dumps(response) + "statusCode": 404, + "body": json.dumps({ + "error": "sorry this query did not produce any results", + "event": event + }) } diff --git a/freqtrade/tests/aws/test_backtest.py b/freqtrade/tests/aws/test_backtest.py index 18b0fc690..909771b27 100644 --- a/freqtrade/tests/aws/test_backtest.py +++ b/freqtrade/tests/aws/test_backtest.py @@ -8,6 +8,100 @@ from freqtrade.aws.backtesting_lambda import backtest, cron from freqtrade.aws.strategy import submit, get_trades +def test_backtest_time_frame(lambda_context): + content = """# --- Do not remove these libs --- +from freqtrade.strategy.interface import IStrategy +from typing import Dict, List +from hyperopt import hp +from functools import reduce +from pandas import DataFrame +# -------------------------------- + +import talib.abstract as ta +import freqtrade.vendor.qtpylib.indicators as qtpylib + +class MyFancyTestStrategy(IStrategy): + minimal_roi = { + "0": 0.5 + } + stoploss = -0.2 + ticker_interval = '5m' + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + macd = ta.MACD(dataframe) + dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8) + dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21) + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['maShort'], dataframe['maMedium']) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['maMedium'], dataframe['maShort']) + ), + 'sell'] = 1 + return dataframe + + + """ + + request = { + "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG", + "description": "simple test strategy", + "name": "MyFancyTestStrategy", + "content": urlsafe_b64encode(content.encode('utf-8')), + "public": False + } + + # now we add an entry + submit({ + "body": json.dumps(request) + }, {}) + + # build sns request + request = { + "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG", + "name": "MyFancyTestStrategy", + "from": "20180501", + "till": "20180401" + + } + + data = json.loads(backtest({ + "Records": [ + { + "Sns": { + "Subject": "backtesting", + "Message": json.dumps(request) + } + }] + }, {})['body']) + + # evaluate that we now have trades in the database + # sadly not always a given at this tage + # due to the dynamic nature. Should pick a strategy for testing + # which generates a lot of trades + if len(data) > 0: + data = get_trades({ + 'pathParameters': { + 'user': "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG", + "name": "MyFancyTestStrategy", + 'stake': "USDT", + 'asset': "{}".format(data[0]['pair'].split("/")[0]) + } + }, {})['body'] + print(data) + assert len(json.loads(data)) > 0 + + def test_backtest(lambda_context): content = """# --- Do not remove these libs --- from freqtrade.strategy.interface import IStrategy diff --git a/freqtrade/tests/aws/test_strategy_lambda.py b/freqtrade/tests/aws/test_strategy_lambda.py index 413706819..3c349cac3 100644 --- a/freqtrade/tests/aws/test_strategy_lambda.py +++ b/freqtrade/tests/aws/test_strategy_lambda.py @@ -87,7 +87,7 @@ class TestStrategy(IStrategy): "body": json.dumps(request) }, {}) - assert (len(json.loads(aws.names({}, {})['body'])) == 2) + assert (len(json.loads(aws.names({}, {})['body']['result'])) == 2) # able to add a duplicated strategy, which should overwrite the existing strategy diff --git a/serverless.yml b/serverless.yml index 48df22fbb..51103e859 100644 --- a/serverless.yml +++ b/serverless.yml @@ -223,6 +223,7 @@ functions: events: - sns: ${self:custom.snsTopic} + environment: topic: ${self:custom.snsTopic} tradeTable: ${self:custom.tradeTable} @@ -241,6 +242,19 @@ functions: - schedule: rate: rate(5 minutes) enabled: false + + # manual trigger for a longer range + # TODO protect with api key + - http: + path: strategies/backtest/{from}/{till} + method: post + cors: true + request: + parameter: + paths: + from: true + till: true + environment: topic: ${self:custom.snsTopic} tradeTable: ${self:custom.tradeTable}