From 9fd735a3a0e9c6c83da832e43e5ec34cd5ad8923 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Wed, 23 May 2018 00:25:38 -0700 Subject: [PATCH] fixing backtesting and perisstence --- freqtrade/aws/backtesting_lambda.py | 110 +++++++++++++---- freqtrade/aws/exchange.py | 14 +++ freqtrade/aws/schemas.py | 126 +++++++++++++------- freqtrade/aws/service/Persistence.py | 79 ------------ freqtrade/aws/service/Queue.py | 47 -------- freqtrade/aws/service/__init__.py | 0 freqtrade/aws/strategy.py | 14 +-- freqtrade/aws/tables.py | 93 +++++++++++++++ freqtrade/optimize/backtesting.py | 12 +- freqtrade/tests/aws/test_backtest.py | 65 +++++++++- freqtrade/tests/aws/test_strategy_lambda.py | 2 + freqtrade/tests/conftest.py | 29 +---- 12 files changed, 355 insertions(+), 236 deletions(-) create mode 100644 freqtrade/aws/exchange.py delete mode 100644 freqtrade/aws/service/Persistence.py delete mode 100644 freqtrade/aws/service/Queue.py delete mode 100644 freqtrade/aws/service/__init__.py create mode 100644 freqtrade/aws/tables.py diff --git a/freqtrade/aws/backtesting_lambda.py b/freqtrade/aws/backtesting_lambda.py index 261ffdea3..43730d025 100644 --- a/freqtrade/aws/backtesting_lambda.py +++ b/freqtrade/aws/backtesting_lambda.py @@ -1,13 +1,11 @@ import logging - -import boto3 import os - -from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration -from freqtrade.optimize.backtesting import Backtesting +import boto3 import simplejson as json -from boto3.dynamodb.conditions import Key, Attr +from boto3.dynamodb.conditions import Key + +from freqtrade.aws.tables import get_trade_table, get_strategy_table +from freqtrade.optimize.backtesting import Backtesting db = boto3.resource('dynamodb') @@ -30,6 +28,9 @@ def backtest(event, context): 'exchange' : name of the exchange we should be using } + + it should be invoked by SNS only to avoid abuse of the system! + :param context: standard AWS context, so pleaes ignore for now! :return: @@ -40,13 +41,16 @@ def backtest(event, context): event['body'] = json.loads(event['body']) name = event['body']['name'] user = event['body']['user'] + + # technically we can get all these from teh strategy table stake_currency = event['body']['stake_currency'].upper() asset = event['body']['asset'] exchange = event['body']['exchange'] assets = list(map(lambda x: "{}/{}".format(x, stake_currency).upper(), asset)) - table = db.Table(os.environ['strategyTable']) + trade_table = get_trade_table() + table = get_strategy_table() response = table.query( KeyConditionExpression=Key('user').eq(user) & @@ -104,27 +108,33 @@ def backtest(event, context): print("persist data in dynamo") + print(result) for index, row in result.iterrows(): - if row['loss'] > 0 or row['profit'] > 0: - item = { - "id": "{}.{}:{}".format(user, name, row['pair']), - "pair": row['pair'], - "count_profit": row['profit'], - "count_loss": row['loss'], - "avg_duration": row['avg duration'], - "avg profit": row['avg profit %'], - "total profit": row['total profit {}'.format(stake_currency)] + data = { + "id": "{}.{}:{}".format(user, name, row['currency']), + "trade": "{} to {}".format(row['entry'].strftime('%Y-%m-%d %H:%M:%S'), + row['exit'].strftime('%Y-%m-%d %H:%M:%S')), + "pair": row['currency'], + "duration": row['duration'], + "profit_percent": row['profit_percent'], + "profit_stake": row['profit_BTC'], + "entry_date": row['entry'].strftime('%Y-%m-%d %H:%M:%S'), + "exit_date": row['exit'].strftime('%Y-%m-%d %H:%M:%S') + } - } - - print(item) + data = json.dumps(data, use_decimal=True) + data = json.loads(data, use_decimal=True) + print(data) + # persist data + trade_table.put_item(Item=data) else: - raise Exception("sorry we did not find any matching strategy for user {} and name {}".format(user, name)) + raise Exception( + "sorry we did not find any matching strategy for user {} and name {}".format(user, name)) else: raise Exception("no body provided") -def submit(event, context): +def cron(event, context): """ this functions submits a new strategy to the backtesting queue @@ -133,8 +143,56 @@ def submit(event, context): :param context: :return: """ + + # if topic exists, we just reuse it + client = boto3.client('sns') + topic_arn = client.create_topic(Name=os.environ['topic'])['TopicArn'] + + table = get_strategy_table() + response = table.scan() + + def fetch(response, table): + """ + fetches all strategies from the server + technically code duplications + TODO refacture + :param response: + :param table: + :return: + """ + + for i in response['Items']: + # fire a message to our queue + + print(i) + message = { + "user": i['user'], + "name": i['name'] + } + + serialized = json.dumps(message, use_decimal=True) + # submit item to queue for routing to the correct persistence + + result = client.publish( + TopicArn=topic_arn, + Message=json.dumps({'default': serialized}), + Subject="schedule backtesting", + MessageStructure='json' + ) + + print(result) + + if 'LastEvaluatedKey' in response: + return table.scan( + ExclusiveStartKey=response['LastEvaluatedKey'] + ) + else: + return {} + + # do/while simulation + response = fetch(response, table) + + while 'LastEvaluatedKey' in response: + response = fetch(response, table) + pass - - -if __name__ == '__main__': - backtest({}, {}) diff --git a/freqtrade/aws/exchange.py b/freqtrade/aws/exchange.py new file mode 100644 index 000000000..672655ca3 --- /dev/null +++ b/freqtrade/aws/exchange.py @@ -0,0 +1,14 @@ + +def fetch_pairs (events, context): + """ + fetches the pairs for the given exchange name and currency + + requires: + + name: name of the exchange + stake_currency: name of the stake currency + + :param events: + :param context: + :return: + """ \ No newline at end of file diff --git a/freqtrade/aws/schemas.py b/freqtrade/aws/schemas.py index 7838b5b8f..9181366c9 100644 --- a/freqtrade/aws/schemas.py +++ b/freqtrade/aws/schemas.py @@ -1,55 +1,91 @@ # defines the schema to submit a new strategy to the system __SUBMIT_STRATEGY_SCHEMA__ = { - "$id": "http://example.com/example.json", - "type": "object", - "definitions": {}, - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "user": { - "$id": "/properties/user", - "type": "string", - "title": "the associated Isaac user", - "default": "", - "examples": [ - "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG" - ] - }, - "description": { - "$id": "/properties/description", - "type": "string", - "title": "a brief description", - "default": "", - "examples": [ - "simple test strategy" - ] - }, + "$id": "http://example.com/example.json", + "type": "object", + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "user": { + "$id": "/properties/user", + "type": "string", + "title": "The User Schema ", + "default": "", + "examples": [ + "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG" + ] + }, + "description": { + "$id": "/properties/description", + "type": "string", + "title": "The Description Schema ", + "default": "", + "examples": [ + "simple test strategy" + ] + }, + "exchange": { + "$id": "/properties/exchange", + "type": "object", + "properties": { "name": { - "$id": "/properties/name", + "$id": "/properties/exchange/properties/name", + "type": "string", + "title": "The Name Schema ", + "default": "", + "examples": [ + "binance" + ] + }, + "stake": { + "$id": "/properties/exchange/properties/stake", + "type": "string", + "title": "The Stake Schema ", + "default": "", + "examples": [ + "usdt" + ] + }, + "pairs": { + "$id": "/properties/exchange/properties/pairs", + "type": "array", + "items": { + "$id": "/properties/exchange/properties/pairs/items", "type": "string", - "title": "the name of your strategy", + "title": "The 0th Schema ", "default": "", "examples": [ - "TestStrategy" - ] - }, - "public": { - "$id": "/properties/public", - "type": "boolean", - "title": "Will this strategy be public", - "default": "false", - "examples": [ - "true", - "false" - ] - }, - "content": { - "$id": "/properties/content", - "type": "string", - "title": "The Content Schema ", - "default": "", - "examples": [ - "IyAtLS0gRG8gbm90IHJlbW92ZSB0aGVzZSBsaWJzIC0tLQpmcm9tIGZyZXF0cmFkZS5zdHJhdGVneS5pbnRlcmZhY2UgaW1wb3J0IElTdHJhdGVneQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdApmcm9tIGh5cGVyb3B0IGltcG9ydCBocApmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlCmZyb20gcGFuZGFzIGltcG9ydCBEYXRhRnJhbWUKIyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQoKaW1wb3J0IHRhbGliLmFic3RyYWN0IGFzIHRhCmltcG9ydCBmcmVxdHJhZGUudmVuZG9yLnF0cHlsaWIuaW5kaWNhdG9ycyBhcyBxdHB5bGliCgpjbGFzcyBUZXN0U3RyYXRlZ3koSVN0cmF0ZWd5KToKICAgIG1pbmltYWxfcm9pID0gewogICAgICAgICIwIjogMC41CiAgICB9CiAgICBzdG9wbG9zcyA9IC0wLjIKICAgIHRpY2tlcl9pbnRlcnZhbCA9ICc1bScKCiAgICBkZWYgcG9wdWxhdGVfaW5kaWNhdG9ycyhzZWxmLCBkYXRhZnJhbWU6IERhdGFGcmFtZSkgLT4gRGF0YUZyYW1lOgogICAgICAgIG1hY2QgPSB0YS5NQUNEKGRhdGFmcmFtZSkKICAgICAgICBkYXRhZnJhbWVbJ21hU2hvcnQnXSA9IHRhLkVNQShkYXRhZnJhbWUsIHRpbWVwZXJpb2Q9OCkKICAgICAgICBkYXRhZnJhbWVbJ21hTWVkaXVtJ10gPSB0YS5FTUEoZGF0YWZyYW1lLCB0aW1lcGVyaW9kPTIxKQogICAgICAgIHJldHVybiBkYXRhZnJhbWUKCiAgICBkZWYgcG9wdWxhdGVfYnV5X3RyZW5kKHNlbGYsIGRhdGFmcmFtZTogRGF0YUZyYW1lKSAtPiBEYXRhRnJhbWU6CiAgICAgICAgZGF0YWZyYW1lLmxvY1sKICAgICAgICAgICAgKAogICAgICAgICAgICAgICAgcXRweWxpYi5jcm9zc2VkX2Fib3ZlKGRhdGFmcmFtZVsnbWFTaG9ydCddLCBkYXRhZnJhbWVbJ21hTWVkaXVtJ10pCiAgICAgICAgICAgICksCiAgICAgICAgICAgICdidXknXSA9IDEKCiAgICAgICAgcmV0dXJuIGRhdGFmcmFtZQoKICAgIGRlZiBwb3B1bGF0ZV9zZWxsX3RyZW5kKHNlbGYsIGRhdGFmcmFtZTogRGF0YUZyYW1lKSAtPiBEYXRhRnJhbWU6CiAgICAgICAgZGF0YWZyYW1lLmxvY1sKICAgICAgICAgICAgKAogICAgICAgICAgICAgICAgcXRweWxpYi5jcm9zc2VkX2Fib3ZlKGRhdGFmcmFtZVsnbWFNZWRpdW0nXSwgZGF0YWZyYW1lWydtYVNob3J0J10pCiAgICAgICAgICAgICksCiAgICAgICAgICAgICdzZWxsJ10gPSAxCiAgICAgICAgcmV0dXJuIGRhdGFmcmFtZQoKCiAgICAgICAg" + "btc/usdt" ] + } } + } + }, + "name": { + "$id": "/properties/name", + "type": "string", + "title": "The Name Schema ", + "default": "", + "examples": [ + "MyFancyTestStrategy" + ] + }, + "content": { + "$id": "/properties/content", + "type": "string", + "title": "The Content Schema ", + "default": "", + "examples": [ + "IyAtLS0gRG8gbm90IHJlbW92ZSB0aGVzZSBsaWJzIC0tLQpmcm9tIGZyZXF0cmFkZS5zdHJhdGVneS5pbnRlcmZhY2UgaW1wb3J0IElTdHJhdGVneQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdApmcm9tIGh5cGVyb3B0IGltcG9ydCBocApmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlCmZyb20gcGFuZGFzIGltcG9ydCBEYXRhRnJhbWUKIyAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQoKaW1wb3J0IHRhbGliLmFic3RyYWN0IGFzIHRhCmltcG9ydCBmcmVxdHJhZGUudmVuZG9yLnF0cHlsaWIuaW5kaWNhdG9ycyBhcyBxdHB5bGliCgpjbGFzcyBNeUZhbmN5VGVzdFN0cmF0ZWd5KElTdHJhdGVneSk6CiAgICBtaW5pbWFsX3JvaSA9IHsKICAgICAgICAiMCI6IDAuNQogICAgfQogICAgc3RvcGxvc3MgPSAtMC4yCiAgICB0aWNrZXJfaW50ZXJ2YWwgPSAnNW0nCgogICAgZGVmIHBvcHVsYXRlX2luZGljYXRvcnMoc2VsZiwgZGF0YWZyYW1lOiBEYXRhRnJhbWUpIC0-IERhdGFGcmFtZToKICAgICAgICBtYWNkID0gdGEuTUFDRChkYXRhZnJhbWUpCiAgICAgICAgZGF0YWZyYW1lWydtYVNob3J0J10gPSB0YS5FTUEoZGF0YWZyYW1lLCB0aW1lcGVyaW9kPTgpCiAgICAgICAgZGF0YWZyYW1lWydtYU1lZGl1bSddID0gdGEuRU1BKGRhdGFmcmFtZSwgdGltZXBlcmlvZD0yMSkKICAgICAgICByZXR1cm4gZGF0YWZyYW1lCgogICAgZGVmIHBvcHVsYXRlX2J1eV90cmVuZChzZWxmLCBkYXRhZnJhbWU6IERhdGFGcmFtZSkgLT4gRGF0YUZyYW1lOgogICAgICAgIGRhdGFmcmFtZS5sb2NbCiAgICAgICAgICAgICgKICAgICAgICAgICAgICAgIHF0cHlsaWIuY3Jvc3NlZF9hYm92ZShkYXRhZnJhbWVbJ21hU2hvcnQnXSwgZGF0YWZyYW1lWydtYU1lZGl1bSddKQogICAgICAgICAgICApLAogICAgICAgICAgICAnYnV5J10gPSAxCgogICAgICAgIHJldHVybiBkYXRhZnJhbWUKCiAgICBkZWYgcG9wdWxhdGVfc2VsbF90cmVuZChzZWxmLCBkYXRhZnJhbWU6IERhdGFGcmFtZSkgLT4gRGF0YUZyYW1lOgogICAgICAgIGRhdGFmcmFtZS5sb2NbCiAgICAgICAgICAgICgKICAgICAgICAgICAgICAgIHF0cHlsaWIuY3Jvc3NlZF9hYm92ZShkYXRhZnJhbWVbJ21hTWVkaXVtJ10sIGRhdGFmcmFtZVsnbWFTaG9ydCddKQogICAgICAgICAgICApLAogICAgICAgICAgICAnc2VsbCddID0gMQogICAgICAgIHJldHVybiBkYXRhZnJhbWUKCgogICAgICAgIA==" + ] + }, + "public": { + "$id": "/properties/public", + "type": "boolean", + "title": "The Public Schema ", + "default": False, + "examples": [ + False + ] } + } } diff --git a/freqtrade/aws/service/Persistence.py b/freqtrade/aws/service/Persistence.py deleted file mode 100644 index 1d07be889..000000000 --- a/freqtrade/aws/service/Persistence.py +++ /dev/null @@ -1,79 +0,0 @@ -import boto3 -import simplejson as json -import decimal - - -class DecimalEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, decimal.Decimal): - if o % 1 > 0: - return float(o) - else: - return int(o) - return super(DecimalEncoder, self).default(o) - - -class Persistence: - """ - simplistic persistence framework - """ - - def __init__(self, table): - """ - creates a new object with the associated table - :param table: - """ - - self.table = table - self.db = boto3.resource('dynamodb') - - def list(self): - table = self.db.Table(self.table) - - response = table.scan() - result = response['Items'] - - while 'LastEvaluatedKey' in response: - for i in response['Items']: - result.append(i) - response = table.scan( - ExclusiveStartKey=response['LastEvaluatedKey'] - ) - - return result - - def load(self, sample): - """ - loads a given object from the database storage - :param sample: - :return: - """ - - table = self.db.Table(self.table) - result = table.get_item( - Key={ - 'id': sample - } - ) - - if 'Item' in result: - return result['Item'] - else: - return None - - def save(self, object): - """ - - saves and object to the database storage with the specific key - - :param object: - :return: - """ - - table = self.db.Table(self.table) - - # force serialization to deal with decimal number tag - data = json.dumps(object, use_decimal=True) - data = json.loads(data, use_decimal=True) - print(data) - return table.put_item(Item=data) diff --git a/freqtrade/aws/service/Queue.py b/freqtrade/aws/service/Queue.py deleted file mode 100644 index 7f333a24a..000000000 --- a/freqtrade/aws/service/Queue.py +++ /dev/null @@ -1,47 +0,0 @@ -import simplejson as json -import os -import boto3 - - -class Queue: - """ - abstraction of the underlaying queuing system to schedule a message to the backend for processing - """ - - def submit(self, object, routingKey): - """ - submits the given object to the queue associated with the - routing key. - The routing lambda function will than make sure it will be delivered to the right destination - - :param object: - :param routingKey: - :return: - """ - - # get topic refrence - client = boto3.client('sns') - - # if topic exists, we just reuse it - topic_arn = client.create_topic(Name=os.environ['topic'])['TopicArn'] - - serialized = json.dumps(object, use_decimal=True) - # submit item to queue for routing to the correct persistence - - result = client.publish( - TopicArn=topic_arn, - Message=json.dumps({'default': serialized}), - Subject="route:" + routingKey, - MessageStructure='json', - MessageAttributes={ - 'route': { - 'DataType': 'String', - 'StringValue': routingKey - } - }, - ) - - return { - "statusCode": result['ResponseMetadata']['HTTPStatusCode'], - "body": serialized - } diff --git a/freqtrade/aws/service/__init__.py b/freqtrade/aws/service/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/freqtrade/aws/strategy.py b/freqtrade/aws/strategy.py index 6fe649b66..309865524 100644 --- a/freqtrade/aws/strategy.py +++ b/freqtrade/aws/strategy.py @@ -8,6 +8,7 @@ from boto3.dynamodb.conditions import Key, Attr from jsonschema import validate from freqtrade.aws.schemas import __SUBMIT_STRATEGY_SCHEMA__ +from freqtrade.aws.tables import get_strategy_table from freqtrade.strategy.resolver import StrategyResolver import requests @@ -21,7 +22,7 @@ def names(event, context): :param context: :return: """ - table = db.Table(os.environ['strategyTable']) + table = get_strategy_table() response = table.scan() result = response['Items'] @@ -63,8 +64,7 @@ def get(event, context): assert 'user' in event['pathParameters'] assert 'name' in event['pathParameters'] - table = db.Table(os.environ['strategyTable']) - + table = get_strategy_table() response = table.query( KeyConditionExpression=Key('user').eq(event['pathParameters']['user']) & Key('name').eq(event['pathParameters']['name']) @@ -112,8 +112,7 @@ def code(event, context): user = event['path']['user'] name = event['path']['name'] - table = db.Table(os.environ['strategyTable']) - + table = get_strategy_table() response = table.query( KeyConditionExpression=Key('user').eq(user) & Key('name').eq(name) @@ -191,7 +190,7 @@ def __evaluate(data): # force serialization to deal with decimal number data = json.dumps(data, use_decimal=True) data = json.loads(data, use_decimal=True) - table = db.Table(os.environ['strategyTable']) + table = get_strategy_table() result = table.put_item(Item=data) return result @@ -223,7 +222,8 @@ def submit_github(event, context): # generate simple id - # submit it + # submit it - we should be able to support multiple repositories + # maybe another database table, where we can map these? try: __evaluate({ "name": x['path'].split("/")[-1].split(".py")[0], diff --git a/freqtrade/aws/tables.py b/freqtrade/aws/tables.py new file mode 100644 index 000000000..059783728 --- /dev/null +++ b/freqtrade/aws/tables.py @@ -0,0 +1,93 @@ +import os + +import boto3 +from boto.dynamodb2.exceptions import ResourceInUseException + +db = boto3.resource('dynamodb') + + +def get_trade_table(): + """ + provides access to the trade table and if it doesn't exists + creates it for us + :return: + """ + + table_name = os.environ['tradeTable'] + existing_tables = boto3.client('dynamodb').list_tables()['TableNames'] + if table_name not in existing_tables: + try: + db.create_table( + TableName=table_name, + KeySchema=[ + { + 'AttributeName': 'id', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'trade', + 'KeyType': 'RANGE' + } + ], + AttributeDefinitions=[ + { + 'AttributeName': 'id', + 'AttributeType': 'S' + }, { + 'AttributeName': 'trade', + 'AttributeType': 'S' + } + ], + + ProvisionedThroughput={ + 'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1 + } + ) + except ResourceInUseException as e: + print("table already exist {}".format(e)) + + return db.Table(table_name) + + +def get_strategy_table(): + """ + provides us access to the strategy table and if it doesn't exists creates it for us + :return: + """ + table_name = os.environ['strategyTable'] + existing_tables = boto3.client('dynamodb').list_tables()['TableNames'] + + existing_tables = boto3.client('dynamodb').list_tables()['TableNames'] + if table_name not in existing_tables: + try: + db.create_table( + TableName=os.environ[table_name], + KeySchema=[ + { + 'AttributeName': 'user', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'name', + 'KeyType': 'RANGE' + } + ], + AttributeDefinitions=[ + { + 'AttributeName': 'user', + 'AttributeType': 'S' + }, { + 'AttributeName': 'name', + 'AttributeType': 'S' + } + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1 + } + ) + except ResourceInUseException as e: + print("table already exist {}".format(e)) + + return db.Table(table_name) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9649f15c4..820bf71d9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -143,7 +143,9 @@ class Backtesting(object): pair, trade.calc_profit_percent(rate=sell_row.close), trade.calc_profit(rate=sell_row.close), - (sell_row.date - buy_row.date).seconds // 60 + (sell_row.date - buy_row.date).seconds // 60, + buy_row.date, + sell_row.date ), \ sell_row.date return None @@ -200,6 +202,7 @@ class Backtesting(object): if ret: row2, trade_entry, next_date = ret lock_pair_until = next_date + trades.append(trade_entry) if record: # Note, need to be json.dump friendly @@ -214,7 +217,8 @@ class Backtesting(object): if record and record.find('trades') >= 0: logger.info('Dumping backtest results') file_dump_json('backtest-result.json', records) - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'entry', 'exit'] + return DataFrame.from_records(trades, columns=labels) def start(self): @@ -287,8 +291,8 @@ class Backtesting(object): ) # return date for data storage - temp = self.aggregate(data, results) - return DataFrame(data=temp[2][:-1], columns=temp[1]) + self.aggregate(data, results) + return results def setup_configuration(args: Namespace) -> Dict[str, Any]: diff --git a/freqtrade/tests/aws/test_backtest.py b/freqtrade/tests/aws/test_backtest.py index 05c2026a5..a3447c07b 100644 --- a/freqtrade/tests/aws/test_backtest.py +++ b/freqtrade/tests/aws/test_backtest.py @@ -2,7 +2,7 @@ from base64 import urlsafe_b64encode import pytest import simplejson as json -from freqtrade.aws.backtesting_lambda import backtest +from freqtrade.aws.backtesting_lambda import backtest, cron from freqtrade.aws.strategy import submit @@ -73,3 +73,66 @@ class MyFancyTestStrategy(IStrategy): } backtest({"body": json.dumps(request)}, {}) + + +def test_cron(lambda_context): + """ test the scheduling to the queue""" + content = """# --- Do not remove these libs --- +from freqtrade.strategy.interface import IStrategy +from typing import Dict, List +from hyperopt import hp +from functools import reduce +from pandas import DataFrame +# -------------------------------- + +import talib.abstract as ta +import freqtrade.vendor.qtpylib.indicators as qtpylib + +class MyFancyTestStrategy(IStrategy): + minimal_roi = { + "0": 0.5 + } + stoploss = -0.2 + ticker_interval = '5m' + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + macd = ta.MACD(dataframe) + dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8) + dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21) + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['maShort'], dataframe['maMedium']) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['maMedium'], dataframe['maShort']) + ), + 'sell'] = 1 + return dataframe + + + """ + + request = { + "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG", + "description": "simple test strategy", + "name": "MyFancyTestStrategy", + "content": urlsafe_b64encode(content.encode('utf-8')), + "public": False + } + + # now we add an entry + submit({ + "body": json.dumps(request) + }, {}) + + print("evaluating cron job") + cron({}, {}) diff --git a/freqtrade/tests/aws/test_strategy_lambda.py b/freqtrade/tests/aws/test_strategy_lambda.py index ac74e40f3..62c6ad72e 100644 --- a/freqtrade/tests/aws/test_strategy_lambda.py +++ b/freqtrade/tests/aws/test_strategy_lambda.py @@ -99,6 +99,8 @@ class TestStrategy(IStrategy): "public": True } + print(json.dumps(request)) + aws.submit({ "body": json.dumps(request) }, {}) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 4ba531739..f31c9373b 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -609,33 +609,8 @@ def lambda_context(): client = session.client('sns') dynamodb = boto3.resource('dynamodb') os.environ["strategyTable"] = "StrategyTable" - - dynamodb.create_table( - TableName=os.environ["strategyTable"], - KeySchema=[ - { - 'AttributeName': 'user', - 'KeyType': 'HASH' - }, - { - 'AttributeName': 'name', - 'KeyType': 'RANGE' - } - ], - AttributeDefinitions=[ - { - 'AttributeName': 'user', - 'AttributeType': 'S' - }, { - 'AttributeName': 'name', - 'AttributeType': 'S' - } - ], - ProvisionedThroughput={ - 'ReadCapacityUnits': 1, - 'WriteCapacityUnits': 1 - } - ) + os.environ["tradeTable"] = "TradeTable" + os.environ["topic"] = "UnitTestTopic" import responses