From 9379e0b15a47db50341149dc16967ae7ba1f0096 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Wed, 6 Jun 2018 08:27:08 -0700 Subject: [PATCH] updated backtesting to allow for more data aggregation --- freqtrade/aws/aggregate/__init__.py | 0 freqtrade/aws/aggregate/strategy.py | 0 freqtrade/aws/backtesting_lambda.py | 59 +++++++++---- serverless.yml | 129 +++++++++++++++++++--------- 4 files changed, 130 insertions(+), 58 deletions(-) create mode 100644 freqtrade/aws/aggregate/__init__.py create mode 100644 freqtrade/aws/aggregate/strategy.py diff --git a/freqtrade/aws/aggregate/__init__.py b/freqtrade/aws/aggregate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/aws/aggregate/strategy.py b/freqtrade/aws/aggregate/strategy.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/aws/backtesting_lambda.py b/freqtrade/aws/backtesting_lambda.py index 15a117564..626ad2193 100644 --- a/freqtrade/aws/backtesting_lambda.py +++ b/freqtrade/aws/backtesting_lambda.py @@ -197,11 +197,19 @@ def run_backtest(configuration, name, user, interval, fromDate, till): def _store_aggregated_data(interval, name, result, timerange, user): + """ + stores aggregated data for ease of access, yay for dynamodb data duplication... + + :param interval: + :param name: + :param result: + :param timerange: + :param user: + :return: + """ for row in result[1][2]: if row[1] > 0: data = { - "id": "aggregate:{}:{}:{}:test".format(row[0].upper(), interval, timerange), - "trade": "{}.{}".format(user, name), "pair": row[0], "trades": row[1], "losses": row[6], @@ -213,19 +221,40 @@ def _store_aggregated_data(interval, name, result, timerange, user): "ticker": interval, "days": timerange } + # aggregate by pair + interval + time range for each strategy + data['id'] = "aggregate:{}:{}:{}:test".format(row[0].upper(), interval, timerange) + data['trade'] = "{}.{}".format(user, name) + _submit_to_remote(data) - print(data) - try: - print( - post("{}/trade".format(os.environ.get('BASE_URL', 'https://freq.isaac.international/dev')), - json=data)) - except Exception as e: - print("submission ignored: {}".format(e)) + # id: aggregate by strategy + user + range + pair + # range: ticker + # allows us to easily see on which ticker the strategy works best + data['id'] = "aggregate:ticker:{}:{}:{}:{}:test".format(user, name, row[0].upper(), timerange), + data['trade'] = "{}".format(interval) + + _submit_to_remote(data) + + # id: aggregate by strategy + user + ticker + pair + # range: timerange + # allows us to easily see on which time range the strategy works best + data['id'] = "aggregate:timerange:{}:{}:{}:{}:test".format(user, name, row[0].upper(), interval), + data['trade'] = "{}".format(timerange) + + _submit_to_remote(data) + + +def _submit_to_remote(data): + try: + print( + post("{}/trade".format(os.environ.get('BASE_URL', 'https://freq.isaac.international/dev')), + json=data)) + except Exception as e: + print("submission ignored: {}".format(e)) def _store_trade_data(interval, name, result, timerange, user): for index, row in result[0].iterrows(): - data = { + _submit_to_remote({ "id": "{}.{}:{}:{}:{}:test".format(user, name, interval, timerange, row['currency'].upper()), "trade": "{} to {}".format(row['entry'].strftime('%Y-%m-%d %H:%M:%S'), row['exit'].strftime('%Y-%m-%d %H:%M:%S')), @@ -238,15 +267,7 @@ def _store_trade_data(interval, name, result, timerange, user): "strategy": name, "user": user - } - - print(data) - try: - print( - post("{}/trade".format(os.environ.get('BASE_URL', 'https://freq.isaac.international/dev')), - json=data)) - except Exception as e: - print("submission ignored: {}".format(e)) + }) def generate_configuration(fromDate, till, name, refresh, user, remote=True): diff --git a/serverless.yml b/serverless.yml index 4185017e0..1bac69e0d 100644 --- a/serverless.yml +++ b/serverless.yml @@ -112,6 +112,7 @@ functions: environment: strategyTable: ${self:custom.strategyTable} + reservedConcurrency: 5 #returns the source code of this given strategy #unless it's private @@ -136,6 +137,7 @@ functions: environment: strategyTable: ${self:custom.strategyTable} + reservedConcurrency: 5 # loads the details of the specific strategy get: @@ -154,6 +156,80 @@ functions: environment: strategyTable: ${self:custom.strategyTable} + reservedConcurrency: 5 + + # loads the aggregation report for the given strategy based on different tickers + get_aggregate_interval: + memorySize: 128 + handler: freqtrade/aws/aggregate/strategy.ticker + events: + - http: + path: strategies/{user}/{name}/aggregate/ticker + method: get + cors: true + request: + parameter: + paths: + user: true + name: true + + environment: + strategyTable: ${self:custom.strategyTable} + tradeTable: ${self:custom.tradeTable} + reservedConcurrency: 5 + + # loads the aggregation report for the given strategy based on different tickers + get_aggregate_timeframe: + memorySize: 128 + handler: freqtrade/aws/aggregate/strategy.timeframe + events: + - http: + path: strategies/{user}/{name}/aggregate/timeframe + method: get + cors: true + request: + parameter: + paths: + user: true + name: true + + environment: + strategyTable: ${self:custom.strategyTable} + tradeTable: ${self:custom.tradeTable} + reservedConcurrency: 5 + + #submits a new strategy to the system + submit: + memorySize: 128 + handler: freqtrade/aws/strategy.submit + events: + - http: + path: strategies/submit + method: post + cors: true + + environment: + topic: ${self:custom.snsTopic} + strategyTable: ${self:custom.strategyTable} + BASE_URL: ${self:custom.customDomain.domainName}/${self:custom.customDomain.stage} + reservedConcurrency: 5 + + #submits a new strategy to the system + submit_github: + memorySize: 128 + handler: freqtrade/aws/strategy.submit_github + events: + - http: + path: strategies/submit/github + method: post + cors: true + + environment: + topic: ${self:custom.snsTopic} + strategyTable: ${self:custom.strategyTable} + reservedConcurrency: 1 + +### TRADE REQUESTS # loads all trades for a strategy and it's associated pairs trades: @@ -175,22 +251,7 @@ functions: environment: strategyTable: ${self:custom.strategyTable} tradeTable: ${self:custom.tradeTable} - - - #submits a new strategy to the system - submit: - memorySize: 128 - handler: freqtrade/aws/strategy.submit - events: - - http: - path: strategies/submit - method: post - cors: true - - environment: - topic: ${self:custom.snsTopic} - strategyTable: ${self:custom.strategyTable} - BASE_URL: ${self:custom.customDomain.domainName}/${self:custom.customDomain.stage} + reservedConcurrency: 5 # submits a new trade to the system trade: @@ -204,19 +265,9 @@ functions: environment: tradeTopic: ${self:custom.snsTradeTopic} + reservedConcurrency: 5 - # stores the received message in the trade table - trade-store: - memorySize: 128 - handler: freqtrade/aws/trade.store - - events: - - sns: ${self:custom.snsTradeTopic} - - environment: - tradeTable: ${self:custom.tradeTable} - - # stores the received message in the trade table + # query aggregates by day and ticker for all strategies trade-aggregate: memorySize: 128 handler: freqtrade/aws/trade.get_aggregated_trades @@ -234,21 +285,21 @@ functions: environment: tradeTable: ${self:custom.tradeTable} - #submits a new strategy to the system - submit_github: + reservedConcurrency: 5 +### SNS TRIGGERED FUNCTIONS + + # stores the received message in the trade table + trade-store: memorySize: 128 - handler: freqtrade/aws/strategy.submit_github + handler: freqtrade/aws/trade.store + events: - - http: - path: strategies/submit/github - method: post - cors: true + - sns: ${self:custom.snsTradeTopic} environment: - topic: ${self:custom.snsTopic} - strategyTable: ${self:custom.strategyTable} - + tradeTable: ${self:custom.tradeTable} + reservedConcurrency: 1 #backtests the strategy #should be switched to utilze aws fargate instead #and running a container @@ -275,7 +326,7 @@ functions: events: - schedule: - rate: rate(580 minutes) + rate: rate(1440 minutes) enabled: true environment: