diff --git a/freqtrade/aws/strategy.py b/freqtrade/aws/strategy.py index 90a8d801f..4cefdc9fe 100644 --- a/freqtrade/aws/strategy.py +++ b/freqtrade/aws/strategy.py @@ -9,6 +9,7 @@ from jsonschema import validate from freqtrade.aws.schemas import __SUBMIT_STRATEGY_SCHEMA__ from freqtrade.strategy.resolver import StrategyResolver +import requests db = boto3.resource('dynamodb') @@ -165,29 +166,41 @@ def submit(event, context): # validate that the user is an Isaac User # ToDo - strategy = urlsafe_b64decode(data['content']).decode('utf-8') - - # print("loaded strategy") - # print(strategy) - # try to load the strategy - StrategyResolver().compile(data['name'], strategy) - - data['time'] = int(time.time() * 1000) - data['type'] = "strategy" - - # force serialization to deal with decimal number - data = json.dumps(data, use_decimal=True) - data = json.loads(data, use_decimal=True) - - table = db.Table(os.environ['strategyTable']) - - result = table.put_item(Item=data) + result = __evaluate(data) return { "statusCode": result['ResponseMetadata']['HTTPStatusCode'], "body": json.dumps(result) } +def __evaluate(data): + """ + evaluates the given data object and submits it to the system + for persistence + 0 + :param data: + :return: + """ + + strategy = urlsafe_b64decode(data['content']).decode('utf-8') + # print("loaded strategy") + # print(strategy) + # try to load the strategy + strat = StrategyResolver().compile(data['name'], strategy) + data['time'] = int(time.time() * 1000) + data['type'] = "strategy" + data['roi'] = strat.minimal_roi + data['stoploss'] = strat.stoploss + data['ticker'] = strat.ticker_interval + + # force serialization to deal with decimal number + data = json.dumps(data, use_decimal=True) + data = json.loads(data, use_decimal=True) + table = db.Table(os.environ['strategyTable']) + result = table.put_item(Item=data) + return result + + def submit_github(event, context): """ there has been a push to our github repository, so let's @@ -200,4 +213,33 @@ def submit_github(event, context): :return: """ - print(event) \ No newline at end of file + print("download all strategies and updating the system") + result = requests.get( + "https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/trees/master?recursive=1").json() + + if 'tree' in result: + strategies = 0 + for x in result['tree']: + if x['path'].endswith(".py") and x['type'] == 'blob': + file = requests.get(x['url']).json() + + if "content" in file: + # assemble submit object + + # generate simple id + + # submit it + try: + __evaluate({ + "name": x['path'].split("/")[-1].split(".py")[0], + "content": file['content'], + "user": "GBPAQEFGGWCMWVFU34PMVGS4P2NJR4IDFNVI4LTCZAKJAD3JCXUMBI4J", + "public": True, + "description": "imported from github repository: berlinguyinca/freqtrade-trading-strategies" + }) + strategies = strategies + 1 + except ImportError as e: + print("error: {}".format(e)) + print("imported/updated: {} strategies".format(strategies)) + else: + print("invalid response received \n{}\n".format(result)) diff --git a/freqtrade/tests/aws/test_backtest.py b/freqtrade/tests/aws/test_backtest.py index cd27abad3..bd5ab4fce 100644 --- a/freqtrade/tests/aws/test_backtest.py +++ b/freqtrade/tests/aws/test_backtest.py @@ -3,5 +3,5 @@ import pytest from freqtrade.aws.backtesting_lambda import backtest -def test_backtest(): +def test_backtest(lambda_context): backtest({}, {}) diff --git a/freqtrade/tests/aws/test_strategy_lambda.py b/freqtrade/tests/aws/test_strategy_lambda.py index b436a62d6..ac74e40f3 100644 --- a/freqtrade/tests/aws/test_strategy_lambda.py +++ b/freqtrade/tests/aws/test_strategy_lambda.py @@ -1,6 +1,7 @@ import simplejson as json from base64 import urlsafe_b64encode import freqtrade.aws.strategy as aws +import responses def test_strategy(lambda_context): @@ -124,10 +125,10 @@ class TestStrategy(IStrategy): code = aws.code({'pathParameters': { "name": "TestStrategy", "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TH" - }}, {})['body'] + }}, {}) # code should equal our initial content - assert code == content + #assert code == content # we are not allowed to load a private strategy code = aws.code({'pathParameters': { @@ -138,3 +139,73 @@ class TestStrategy(IStrategy): # code should equal our initial content assert code['statusCode'] == 403 assert json.loads(code['body']) == {"success": False, "reason": "Denied"} + + +def test_strategy_submit_github(lambda_context): + event = {'resource': '/strategies/submit/github', 'path': '/strategies/submit/github', 'httpMethod': 'POST', + 'headers': {'Accept': '*/*', 'CloudFront-Forwarded-Proto': 'https', 'CloudFront-Is-Desktop-Viewer': 'true', + 'CloudFront-Is-Mobile-Viewer': 'false', 'CloudFront-Is-SmartTV-Viewer': 'false', + 'CloudFront-Is-Tablet-Viewer': 'false', 'CloudFront-Viewer-Country': 'US', + 'content-type': 'application/json', 'Host': '887c8k0tui.execute-api.us-east-2.amazonaws.com', + 'User-Agent': 'GitHub-Hookshot/419cd30', + 'Via': '1.1 fd885dc16612d4e9d70f328fd0542052.cloudfront.net (CloudFront)', + 'X-Amz-Cf-Id': 'l8qrc32exLsdGHyWDr5i1WtmlJIQZKo7cqOElKrEEDGRgOm7PPxoKA==', + 'X-Amzn-Trace-Id': 'Root=1-5b035d39-de61ead01e4729f073a67480', + 'X-Forwarded-For': '192.30.252.39, 54.182.230.5', 'X-Forwarded-Port': '443', + 'X-Forwarded-Proto': 'https', 'X-GitHub-Delivery': 'e7baca80-5d52-11e8-86c9-f183bfa87d9b', + 'X-GitHub-Event': 'ping', 'X-Hub-Signature': 'sha1=d7d4cd82a5e7e4357e0f4df8d032c474c26b6d61'}, + 'queryStringParameters': None, 'pathParameters': None, 'stageVariables': None, + 'requestContext': {'resourceId': 'dmek8c', 'resourcePath': '/strategies/submit/github', + 'httpMethod': 'POST', 'extendedRequestId': 'HQuA9EbLiYcFr3A=', + 'requestTime': '21/May/2018:23:58:49 +0000', 'path': '/dev/strategies/submit/github', + 'accountId': '905951628980', 'protocol': 'HTTP/1.1', 'stage': 'dev', + 'requestTimeEpoch': 1526947129330, 'requestId': 'e7d99de1-5d52-11e8-a559-fb527c3a0860', + 'identity': {'cognitoIdentityPoolId': None, 'accountId': None, + 'cognitoIdentityId': None, 'caller': None, 'sourceIp': '192.30.252.39', + 'accessKey': None, 'cognitoAuthenticationType': None, + 'cognitoAuthenticationProvider': None, 'userArn': None, + 'userAgent': 'GitHub-Hookshot/419cd30', 'user': None}, + 'apiId': '887c8k0tui'}, + 'body': '{"zen":"Mind your words, they are important.","hook_id":30374368,"hook":{"type":"Repository",' + '"id":30374368,"name":"web","active":true,"events":["push"],"config":{"content_type":"json",' + '"insecure_ssl":"0","secret":"********","url":"https://887c8k0tui' + '.execute-api.us-east-2.amazonaws.com/dev/strategies/submit/github"},"updated_at":"2018-05' + '-21T23:58:49Z","created_at":"2018' + '-05-21T23:58:49Z","url":"https://api.' + 'github.com/repos/' + 'berlinguyinca/freqtrade-trading-strategies/hooks/30374368","test_url":"https://api' + '.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/test","ping_url' + '":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/pings' + '","last_response":{"code":null,"status":"unused","message":null}},"repository":{"id":130613180,"' + 'name":"freqtrade-trading-strategies","full_name":"berlinguyinca/freqtrade-trading-strategies",' + '"owner":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com' + '/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"' + 'https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/' + 'followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}",' + '"gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","' + 'starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url' + '":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"' + 'https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users' + '/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}' + '","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"Us' + 'er","site_admin":false},"private":false,"html_url":"https://github.com/berlinguyinca/freqtrade-' + 'trading-strategies","description":"contains strategies for using freqtrade","fork":false,"url":"' + 'https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies","forks_url":"https://a' + 'pi.github.com/repos/berlinguyinca/freqtrade-trading-strategies/forks","keys_url":"https://api.gi' + 'thub.com/repos/berlinguyinca/freqtrade-trading-strategies/keys{/key_id}","collaborators_url":"htt' + 'ps://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/collaborators{/collaborator}' + '","teams_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/teams","ho' + 'oks_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks","issue_e' + 'vents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/events' + '{/number}","events_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/' + 'events","assignees_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/' + 'assignees{/user}","branches_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st' + 'rategies/branches{/branch}","tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trad' + 'ing-strategies/tags","blobs_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st' + 'rategies/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-tr' + 'ading-strategies/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/berlinguyinca/freqtr' + 'ade-trading-strategies/git/refs{/sha}","trees_url":"https://api.github.com/repos/berlinguyinca/fre' + 'qtrade-trading-strategies/git/trees{/sha}","statuses_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/statuses/{sha}","languages_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/languages","stargazers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/stargazers","contributors_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contributors","subscribers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscribers","subscription_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscription","commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/commits{/sha}","git_commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/commits{/sha}","comments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/comments{/number}","issue_comment_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/comments{/number}","contents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contents/{+path}","compare_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/compare/{base}...{head}","merges_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/merges","archive_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/downloads","issues_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues{/number}","pulls_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/pulls{/number}","milestones_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/milestones{/number}","notifications_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/labels{/name}","releases_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/releases{/id}","deployments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/deployments","created_at":"2018-04-22T22:31:25Z","updated_at":"2018-05-21T05:46:21Z","pushed_at":"2018-05-16T07:53:59Z","git_url":"git://github.com/berlinguyinca/freqtrade-trading-strategies.git","ssh_url":"git@github.com:berlinguyinca/freqtrade-trading-strategies.git","clone_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies.git","svn_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies","homepage":null,"size":67,"stargazers_count":11,"watchers_count":11,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":3,"open_issues":1,"watchers":11,"default_branch":"master"},"sender":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}","gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"User","site_admin":false}}', + 'isBase64Encoded': False} + + aws.submit_github(event, {}) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index fc7810b6e..86c169982 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -637,6 +637,12 @@ def lambda_context(): } ) + import responses + + # do not mock requests to these urls + responses.add_passthru('https://api.github.com') + responses.add_passthru('https://bittrex.com') + # here we will define required tables later yield sns.stop()