fixing backtesting and perisstence

freqtrade/aws/backtesting_lambda.py

@@ -1,13 +1,11 @@
 import logging
-
-import boto3
 import os
-
-from freqtrade.arguments import Arguments
-from freqtrade.configuration import Configuration
-from freqtrade.optimize.backtesting import Backtesting
+import boto3
 import simplejson as json
-from boto3.dynamodb.conditions import Key, Attr
+from boto3.dynamodb.conditions import Key
+
+from freqtrade.aws.tables import get_trade_table, get_strategy_table
+from freqtrade.optimize.backtesting import Backtesting
 
 db = boto3.resource('dynamodb')
 
@@ -30,6 +28,9 @@ def backtest(event, context):
                 'exchange' : name of the exchange we should be using
 
             }
+
+        it should be invoked by SNS only to avoid abuse of the system!
+
         :param context:
             standard AWS context, so pleaes ignore for now!
         :return:
@@ -40,13 +41,16 @@ def backtest(event, context):
         event['body'] = json.loads(event['body'])
         name = event['body']['name']
         user = event['body']['user']
+
+        # technically we can get all these from teh strategy table
         stake_currency = event['body']['stake_currency'].upper()
         asset = event['body']['asset']
         exchange = event['body']['exchange']
 
         assets = list(map(lambda x: "{}/{}".format(x, stake_currency).upper(), asset))
 
-        table = db.Table(os.environ['strategyTable'])
+        trade_table = get_trade_table()
+        table = get_strategy_table()
 
         response = table.query(
             KeyConditionExpression=Key('user').eq(user) &
@@ -104,27 +108,33 @@ def backtest(event, context):
 
             print("persist data in dynamo")
 
+            print(result)
             for index, row in result.iterrows():
-                if row['loss'] > 0 or row['profit'] > 0:
-                    item = {
-                        "id": "{}.{}:{}".format(user, name, row['pair']),
-                        "pair": row['pair'],
-                        "count_profit": row['profit'],
-                        "count_loss": row['loss'],
-                        "avg_duration": row['avg duration'],
-                        "avg profit": row['avg profit %'],
-                        "total profit": row['total profit {}'.format(stake_currency)]
+                data = {
+                    "id": "{}.{}:{}".format(user, name, row['currency']),
+                    "trade": "{} to {}".format(row['entry'].strftime('%Y-%m-%d %H:%M:%S'),
+                                               row['exit'].strftime('%Y-%m-%d %H:%M:%S')),
+                    "pair": row['currency'],
+                    "duration": row['duration'],
+                    "profit_percent": row['profit_percent'],
+                    "profit_stake": row['profit_BTC'],
+                    "entry_date": row['entry'].strftime('%Y-%m-%d %H:%M:%S'),
+                    "exit_date": row['exit'].strftime('%Y-%m-%d %H:%M:%S')
+                }
 
-                    }
-
-                print(item)
+                data = json.dumps(data, use_decimal=True)
+                data = json.loads(data, use_decimal=True)
+                print(data)
+                # persist data
+                trade_table.put_item(Item=data)
         else:
-            raise Exception("sorry we did not find any matching strategy for user {} and name {}".format(user, name))
+            raise Exception(
+                "sorry we did not find any matching strategy for user {} and name {}".format(user, name))
     else:
         raise Exception("no body provided")
 
 
-def submit(event, context):
+def cron(event, context):
     """
 
     this functions submits a new strategy to the backtesting queue
@@ -133,8 +143,56 @@ def submit(event, context):
     :param context:
     :return:
     """
+
+    # if topic exists, we just reuse it
+    client = boto3.client('sns')
+    topic_arn = client.create_topic(Name=os.environ['topic'])['TopicArn']
+
+    table = get_strategy_table()
+    response = table.scan()
+
+    def fetch(response, table):
+        """
+            fetches all strategies from the server
+            technically code duplications
+            TODO refacture
+        :param response:
+        :param table:
+        :return:
+        """
+
+        for i in response['Items']:
+            # fire a message to our queue
+
+            print(i)
+            message = {
+                "user": i['user'],
+                "name": i['name']
+            }
+
+            serialized = json.dumps(message, use_decimal=True)
+            # submit item to queue for routing to the correct persistence
+
+            result = client.publish(
+                TopicArn=topic_arn,
+                Message=json.dumps({'default': serialized}),
+                Subject="schedule backtesting",
+                MessageStructure='json'
+            )
+
+            print(result)
+
+        if 'LastEvaluatedKey' in response:
+            return table.scan(
+                ExclusiveStartKey=response['LastEvaluatedKey']
+            )
+        else:
+            return {}
+
+    # do/while simulation
+    response = fetch(response, table)
+
+    while 'LastEvaluatedKey' in response:
+        response = fetch(response, table)
+
     pass
-
-
-if __name__ == '__main__':
-    backtest({}, {})