updated backtesting to allow for more data aggregation

freqtrade/aws/backtesting_lambda.py

@@ -197,11 +197,19 @@ def run_backtest(configuration, name, user, interval, fromDate, till):
 
 
 def _store_aggregated_data(interval, name, result, timerange, user):
+    """
+    stores aggregated data for ease of access, yay for dynamodb data duplication...
+
+    :param interval:
+    :param name:
+    :param result:
+    :param timerange:
+    :param user:
+    :return:
+    """
     for row in result[1][2]:
         if row[1] > 0:
             data = {
-                "id": "aggregate:{}:{}:{}:test".format(row[0].upper(), interval, timerange),
-                "trade": "{}.{}".format(user, name),
                 "pair": row[0],
                 "trades": row[1],
                 "losses": row[6],
@@ -213,19 +221,40 @@ def _store_aggregated_data(interval, name, result, timerange, user):
                 "ticker": interval,
                 "days": timerange
             }
+            # aggregate by pair + interval + time range for each strategy
+            data['id'] = "aggregate:{}:{}:{}:test".format(row[0].upper(), interval, timerange)
+            data['trade'] = "{}.{}".format(user, name)
+            _submit_to_remote(data)
 
-            print(data)
+            # id: aggregate by strategy + user + range + pair
-            try:
+            # range: ticker
-                print(
+            # allows us to easily see on which ticker the strategy works best
-                    post("{}/trade".format(os.environ.get('BASE_URL', 'https://freq.isaac.international/dev')),
+            data['id'] = "aggregate:ticker:{}:{}:{}:{}:test".format(user, name, row[0].upper(), timerange),
-                         json=data))
+            data['trade'] = "{}".format(interval)
-            except Exception as e:
+
-                print("submission ignored: {}".format(e))
+            _submit_to_remote(data)
+
+            # id: aggregate by strategy + user + ticker + pair
+            # range: timerange
+            # allows us to easily see on which time range the strategy works best
+            data['id'] = "aggregate:timerange:{}:{}:{}:{}:test".format(user, name, row[0].upper(), interval),
+            data['trade'] = "{}".format(timerange)
+
+            _submit_to_remote(data)
+
+
+def _submit_to_remote(data):
+    try:
+        print(
+            post("{}/trade".format(os.environ.get('BASE_URL', 'https://freq.isaac.international/dev')),
+                 json=data))
+    except Exception as e:
+        print("submission ignored: {}".format(e))
 
 
 def _store_trade_data(interval, name, result, timerange, user):
     for index, row in result[0].iterrows():
-        data = {
+        _submit_to_remote({
             "id": "{}.{}:{}:{}:{}:test".format(user, name, interval, timerange, row['currency'].upper()),
             "trade": "{} to {}".format(row['entry'].strftime('%Y-%m-%d %H:%M:%S'),
                                        row['exit'].strftime('%Y-%m-%d %H:%M:%S')),
@@ -238,15 +267,7 @@ def _store_trade_data(interval, name, result, timerange, user):
             "strategy": name,
             "user": user
 
-        }
+        })
-
-        print(data)
-        try:
-            print(
-                post("{}/trade".format(os.environ.get('BASE_URL', 'https://freq.isaac.international/dev')),
-                     json=data))
-        except Exception as e:
-            print("submission ignored: {}".format(e))
 
 
 def generate_configuration(fromDate, till, name, refresh, user, remote=True):

serverless.yml

@@ -112,6 +112,7 @@ functions:
 
     environment:
       strategyTable: ${self:custom.strategyTable}
+    reservedConcurrency: 5
 
   #returns the source code of this given strategy
   #unless it's private
@@ -136,6 +137,7 @@ functions:
                         
     environment:
       strategyTable: ${self:custom.strategyTable}
+    reservedConcurrency: 5
 
   # loads the details of the specific strategy
   get:
@@ -154,6 +156,80 @@ functions:
 
     environment:
       strategyTable: ${self:custom.strategyTable}
+    reservedConcurrency: 5
+
+  # loads the aggregation report for the given strategy based on different tickers
+  get_aggregate_interval:
+    memorySize: 128
+    handler: freqtrade/aws/aggregate/strategy.ticker
+    events:
+      - http:
+          path: strategies/{user}/{name}/aggregate/ticker
+          method: get
+          cors: true
+          request:
+            parameter:
+              paths:
+                user: true
+                name: true
+
+    environment:
+      strategyTable: ${self:custom.strategyTable}
+      tradeTable: ${self:custom.tradeTable}
+    reservedConcurrency: 5
+
+  # loads the aggregation report for the given strategy based on different tickers
+  get_aggregate_timeframe:
+    memorySize: 128
+    handler: freqtrade/aws/aggregate/strategy.timeframe
+    events:
+      - http:
+          path: strategies/{user}/{name}/aggregate/timeframe
+          method: get
+          cors: true
+          request:
+            parameter:
+              paths:
+                user: true
+                name: true
+
+    environment:
+      strategyTable: ${self:custom.strategyTable}
+      tradeTable: ${self:custom.tradeTable}
+    reservedConcurrency: 5
+
+  #submits a new strategy to the system
+  submit:
+    memorySize: 128
+    handler: freqtrade/aws/strategy.submit
+    events:
+      - http:
+          path: strategies/submit
+          method: post
+          cors: true
+
+    environment:
+      topic: ${self:custom.snsTopic}
+      strategyTable: ${self:custom.strategyTable}
+      BASE_URL: ${self:custom.customDomain.domainName}/${self:custom.customDomain.stage}
+    reservedConcurrency: 5
+
+  #submits a new strategy to the system
+  submit_github:
+    memorySize: 128
+    handler: freqtrade/aws/strategy.submit_github
+    events:
+      - http:
+          path: strategies/submit/github
+          method: post
+          cors: true
+
+    environment:
+      topic: ${self:custom.snsTopic}
+      strategyTable: ${self:custom.strategyTable}
+    reservedConcurrency: 1
+
+### TRADE REQUESTS
 
   # loads all trades for a strategy and it's associated pairs
   trades:
@@ -175,22 +251,7 @@ functions:
     environment:
       strategyTable: ${self:custom.strategyTable}
       tradeTable: ${self:custom.tradeTable}
-
+    reservedConcurrency: 5
-
-  #submits a new strategy to the system
-  submit:
-    memorySize: 128
-    handler: freqtrade/aws/strategy.submit
-    events:
-      - http:
-          path: strategies/submit
-          method: post
-          cors: true
-
-    environment:
-      topic: ${self:custom.snsTopic}
-      strategyTable: ${self:custom.strategyTable}
-      BASE_URL: ${self:custom.customDomain.domainName}/${self:custom.customDomain.stage}
 
   # submits a new trade to the system
   trade:
@@ -204,19 +265,9 @@ functions:
 
     environment:
       tradeTopic: ${self:custom.snsTradeTopic}
+    reservedConcurrency: 5
 
-  # stores the received message in the trade table
+  # query aggregates by day and ticker for all strategies
-  trade-store:
-    memorySize: 128
-    handler: freqtrade/aws/trade.store
-
-    events:
-      - sns: ${self:custom.snsTradeTopic}
-
-    environment:
-      tradeTable: ${self:custom.tradeTable}
-
-  # stores the received message in the trade table
   trade-aggregate:
     memorySize: 128
     handler: freqtrade/aws/trade.get_aggregated_trades
@@ -234,21 +285,21 @@ functions:
     environment:
       tradeTable: ${self:custom.tradeTable}
 
-  #submits a new strategy to the system
+    reservedConcurrency: 5
-  submit_github:
+### SNS TRIGGERED FUNCTIONS
+
+  # stores the received message in the trade table
+  trade-store:
     memorySize: 128
-    handler: freqtrade/aws/strategy.submit_github
+    handler: freqtrade/aws/trade.store
+
     events:
-      - http:
+      - sns: ${self:custom.snsTradeTopic}
-          path: strategies/submit/github
-          method: post
-          cors: true
 
     environment:
-      topic: ${self:custom.snsTopic}
+      tradeTable: ${self:custom.tradeTable}
-      strategyTable: ${self:custom.strategyTable}
-
 
+    reservedConcurrency: 1
   #backtests the strategy
   #should be switched to utilze aws fargate instead
   #and running a container
@@ -275,7 +326,7 @@ functions:
 
     events:
       - schedule:
-          rate: rate(580 minutes)
+          rate: rate(1440 minutes)
           enabled: true
 
     environment: