added support for backtest ranges to collect more data and initial support for pagination

freqtrade/aws/backtesting_lambda.py

@@ -57,13 +57,17 @@ def backtest(event, context):
 
                 )
 
-                print(response)
-                try:
-                    if "Items" in response and len(response['Items']) > 0:
-
                 today = datetime.datetime.today()
                 yesterday = today - datetime.timedelta(days=1)
 
+                if 'from' in event['body']:
+                    yesterday = datetime.datetime.strptime(event['body']['from'], '%Y%m%d')
+                if 'till' in event['body']:
+                    yesterday = datetime.datetime.strptime(event['body']['till'], '%Y%m%d')
+
+                try:
+                    if "Items" in response and len(response['Items']) > 0:
+
                         content = response['Items'][0]['content']
                         configuration = {
                             "max_open_trades": 1,
@@ -189,6 +193,14 @@ def cron(event, context):
                 "name": i['name']
             }
 
+            # triggered over html, let's provide
+            # a date range for the backtesting
+            if 'pathParameters' in event:
+                if 'from' in event['pathParameters']:
+                    message['from'] = event['pathParameters']['from']
+                if 'till' in event['pathParameters']:
+                    message['till'] = event['pathParameters']['till']
+
             serialized = json.dumps(message, use_decimal=True)
             # submit item to queue for routing to the correct persistence
 

freqtrade/aws/strategy.py

@@ -26,6 +26,7 @@ def names(event, context):
     response = table.scan()
     result = response['Items']
 
+    # no pagination here
     while 'LastEvaluatedKey' in response:
         for i in response['Items']:
             result.append(i)
@@ -36,6 +37,10 @@ def names(event, context):
     # map results and hide informations
     data = list(map(lambda x: {'name': x['name'], 'public': x['public'], 'user': x['user']}, result))
 
+    # keep in a result object, so we can later add pagination to it
+    data = {
+        "result": data
+    }
     return {
         "statusCode": 200,
         "body": json.dumps(data)
@@ -304,13 +309,25 @@ def get_trades(event, context):
 
     if "Items" in response and len(response['Items']) > 0:
 
+        # preparation for pagination
+        # TODO include in parameters an optional
+        # start key ExclusiveStartKey=response['LastEvaluatedKey']
+
+        data = {
+            "result": response['Items'],
+            "paginationKey": response.get('LastEvaluatedKey')
+        }
+
         return {
             "statusCode": response['ResponseMetadata']['HTTPStatusCode'],
-            "body": json.dumps(response['Items'])
+            "body": json.dumps(data)
         }
 
     else:
         return {
-            "statusCode": response['ResponseMetadata']['HTTPStatusCode'],
-            "body": json.dumps(response)
+            "statusCode": 404,
+            "body": json.dumps({
+                "error": "sorry this query did not produce any results",
+                "event": event
+            })
         }

freqtrade/tests/aws/test_backtest.py

@@ -8,6 +8,100 @@ from freqtrade.aws.backtesting_lambda import backtest, cron
 from freqtrade.aws.strategy import submit, get_trades
 
 
+def test_backtest_time_frame(lambda_context):
+    content = """# --- Do not remove these libs ---
+from freqtrade.strategy.interface import IStrategy
+from typing import Dict, List
+from hyperopt import hp
+from functools import reduce
+from pandas import DataFrame
+# --------------------------------
+
+import talib.abstract as ta
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyFancyTestStrategy(IStrategy):
+    minimal_roi = {
+        "0": 0.5
+    }
+    stoploss = -0.2
+    ticker_interval = '5m'
+
+    def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
+        macd = ta.MACD(dataframe)
+        dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8)
+        dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21)
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['maShort'], dataframe['maMedium'])
+            ),
+            'buy'] = 1
+
+        return dataframe
+
+    def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['maMedium'], dataframe['maShort'])
+            ),
+            'sell'] = 1
+        return dataframe
+
+
+        """
+
+    request = {
+        "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+        "description": "simple test strategy",
+        "name": "MyFancyTestStrategy",
+        "content": urlsafe_b64encode(content.encode('utf-8')),
+        "public": False
+    }
+
+    # now we add an entry
+    submit({
+        "body": json.dumps(request)
+    }, {})
+
+    # build sns request
+    request = {
+        "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+        "name": "MyFancyTestStrategy",
+        "from": "20180501",
+        "till": "20180401"
+
+    }
+
+    data = json.loads(backtest({
+        "Records": [
+            {
+                "Sns": {
+                    "Subject": "backtesting",
+                    "Message": json.dumps(request)
+                }
+            }]
+    }, {})['body'])
+
+    # evaluate that we now have trades in the database
+    # sadly not always a given at this tage
+    # due to the dynamic nature. Should pick a strategy for testing
+    # which generates a lot of trades
+    if len(data) > 0:
+        data = get_trades({
+            'pathParameters': {
+                'user': "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+                "name": "MyFancyTestStrategy",
+                'stake': "USDT",
+                'asset': "{}".format(data[0]['pair'].split("/")[0])
+            }
+        }, {})['body']
+        print(data)
+        assert len(json.loads(data)) > 0
+
+
 def test_backtest(lambda_context):
     content = """# --- Do not remove these libs ---
 from freqtrade.strategy.interface import IStrategy

freqtrade/tests/aws/test_strategy_lambda.py

@@ -87,7 +87,7 @@ class TestStrategy(IStrategy):
         "body": json.dumps(request)
     }, {})
 
-    assert (len(json.loads(aws.names({}, {})['body'])) == 2)
+    assert (len(json.loads(aws.names({}, {})['body']['result'])) == 2)
 
     # able to add a duplicated strategy, which should overwrite the existing strategy
 

serverless.yml

@@ -223,6 +223,7 @@ functions:
 
     events:
       - sns: ${self:custom.snsTopic}
+
     environment:
       topic: ${self:custom.snsTopic}
       tradeTable: ${self:custom.tradeTable}
@@ -241,6 +242,19 @@ functions:
       - schedule:
           rate: rate(5 minutes)
           enabled: false
+
+      # manual trigger for a longer range
+      # TODO protect with api key
+      - http:
+          path: strategies/backtest/{from}/{till}
+          method: post
+          cors: true
+          request:
+            parameter:
+              paths:
+                from: true
+                till: true
+
     environment:
       topic: ${self:custom.snsTopic}
       tradeTable: ${self:custom.tradeTable}