diff --git a/freqtrade/aws/__init__.py b/freqtrade/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/aws/backtesting_lambda.py b/freqtrade/aws/backtesting_lambda.py new file mode 100644 index 000000000..4f1a9bb80 --- /dev/null +++ b/freqtrade/aws/backtesting_lambda.py @@ -0,0 +1,28 @@ +from freqtrade.optimize.backtesting import Backtesting + +def backtest(event, context): + """ + this method is running on the AWS server + and back tests this application for us + and stores the back testing results in a local database + + this event can be given as: + + :param event: + { + 'strategy' : 'url handle where we can find the strategy' + 'pair' : ' pair to backtest, BTC_ETH as example' + 'timeframe' : 'how long should we backtest for, 0-100 as example for the last 100 ticks' + } + :param context: + standard AWS context, so pleaes ignore for now! + :return: + no return + """ + + + backtesting = Backtesting() + backtesting.start() + + pass + diff --git a/freqtrade/tests/aws/__init__.py b/freqtrade/tests/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/aws/test_backtest.py b/freqtrade/tests/aws/test_backtest.py new file mode 100644 index 000000000..bd5ab4fce --- /dev/null +++ b/freqtrade/tests/aws/test_backtest.py @@ -0,0 +1,7 @@ +import pytest + +from freqtrade.aws.backtesting_lambda import backtest + + +def test_backtest(lambda_context): + backtest({}, {}) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index bb42bcff9..23cebb9a7 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -15,6 +15,10 @@ from freqtrade.analyze import Analyze from freqtrade import constants from freqtrade.freqtradebot import FreqtradeBot +import moto +import boto3 +import os + logging.getLogger('').setLevel(logging.INFO) @@ -585,3 +589,34 @@ def buy_order_fee(): 'status': 'closed', 'fee': None } + + +@pytest.fixture +def lambda_context(): + + # mock the different AWS features we need + sns = moto.mock_sns() + sns.start() + + dynamo = moto.mock_dynamodb2() + dynamo.start() + + lamb = moto.mock_lambda() + lamb.start() + + session = boto3.session.Session() + + client = session.client('sns') + + os.environ["topic"] = "UnitTestTopic" + os.environ["trackingTable"] = "UnitTrackingTable" + os.environ["acquisitionTable"] = "UnitAcquisitionTable" + os.environ["resultTable"] = "UnitResultTable" + + dynamodb = boto3.resource('dynamodb') + + # here we will define required tables later + yield + sns.stop() + dynamo.stop() + lamb.stop() diff --git a/requirements.txt b/requirements.txt index de0fd2181..8eb3a23fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,7 @@ coinmarketcap==4.2.1 # Required for plotting data #plotly==2.3.0 + +# required for the lambda module +moto +boto3 diff --git a/serverless.yml b/serverless.yml new file mode 100644 index 000000000..5c27c4e46 --- /dev/null +++ b/serverless.yml @@ -0,0 +1,198 @@ +service: stasis + +frameworkVersion: ">=1.1.0 <2.0.0" + +plugins: + - serverless-domain-manager + - serverless-python-requirements + +############################################################################################ +# configure out provider and the security guide lines +############################################################################################ +provider: + name: aws + runtime: python3.6 + region: us-west-2 + + #required permissions + iamRoleStatements: + - Effect: Allow + Action: + - dynamodb:Query + - dynamodb:Scan + - dynamodb:GetItem + - dynamodb:PutItem + - dynamodb:UpdateItem + - dynamodb:DeleteItem + Resource: "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/*" + - Effect: Allow + Action: + - SNS:* + Resource: { "Fn::Join" : [":", ["arn:aws:sns:${self:custom.region}", "*:*" ] ] } + + memorySize: 128 + timeout: 90 + versionFunctions: false + + logRetentionInDays: 3 + + #where to store out data, needs to be manually created! + deploymentBucket: + name: lambdas-stasis + +############################################################################################ +#custom configuration settings +############################################################################################ +custom: + stage: ${opt:stage, self:provider.stage} + region: ${opt:region, self:provider.region} + + snsTopic: "StasisQueue-${self:custom.stage}" + + trackingTable: "StasisTrackingTable-${self:custom.stage}" + acquisitionTable: "StasisMetaDataTable-${self:custom.stage}" + resultTable: "StasisResultTable-${self:custom.stage}" + + ### + # custom domain management + ### + + domains: + prod: api.metabolomics.us + test: test-api.metabolomics.us + dev: dev-api.metabolomics.us + + customDomain: + basePath: "stasis" + domainName: ${self:custom.domains.${self:custom.stage}} + stage: "${self:custom.stage}" + createRoute53Record: true + + pythonRequirements: + dockerizePip: non-linux +############################################################################################ +# this section defines all lambda function and triggers +############################################################################################ +functions: + + + #fetches an result record + resultGet: + handler: stasis/results/get.get + events: + - http: + path: result/{sample} + method: get + cors: true + request: + parameter: + paths: + sample: true + #defines to which topic we want to connect + environment: + resultTable: ${self:custom.resultTable} + topic: ${self:custom.snsTopic} + + #fetches an result record + resultCreate: + handler: stasis/results/create.create + events: + - http: + path: result + method: post + cors: true + + #defines to which topic we want to connect + environment: + resultTable: ${self:custom.resultTable} + topic: ${self:custom.snsTopic} + + + #creates a new tracking record in the system + trackingCreate: + handler: stasis/tracking/create.create + events: + - http: + path: tracking + method: post + cors: true + + #defines to which topic we want to connect + environment: + topic: ${self:custom.snsTopic} + + #fetches an existing tracking record from the system + trackingGet: + handler: stasis/tracking/get.get + events: + - http: + path: tracking/{sample} + method: get + cors: true + request: + parameter: + paths: + sample: true + + #defines to which topic we want to connect + environment: + trackingTable: ${self:custom.trackingTable} + + #fetches an acquisition record + acquisitionGet: + handler: stasis/acquisition/get.get + events: + - http: + path: acquisition/{sample} + method: get + cors: true + request: + parameter: + paths: + sample: true + #defines to which topic we want to connect + environment: + acquisitionTable: ${self:custom.acquisitionTable} + topic: ${self:custom.snsTopic} + + #fetches an acquisition record + acquisitionCreate: + handler: stasis/acquisition/create.create + events: + - http: + path: acquisition + method: post + cors: true + + #defines to which topic we want to connect + environment: + acquisitionTable: ${self:custom.acquisitionTable} + topic: ${self:custom.snsTopic} + + #imports data from MiniX + acquisitionCreateFromMinix: + handler: stasis/acquisition/create.fromMinix + events: + - http: + path: acquisition/import/minix + method: post + cors: true + + #defines to which topic we want to connect + environment: + acquisitionTable: ${self:custom.acquisitionTable} + topic: ${self:custom.snsTopic} + + + #monitors our event queue for received messages and routes them + #no public exposure + backtest: + memorySize: 128 + + handler: stasis/route/route.route + events: + - sns: ${self:custom.snsTopic} + environment: + trackingTable: ${self:custom.trackingTable} + acquisitionTable: ${self:custom.acquisitionTable} + topic: ${self:custom.snsTopic}