added support for downloading the github strategy repository

freqtrade/aws/strategy.py

@@ -9,6 +9,7 @@ from jsonschema import validate
 
 from freqtrade.aws.schemas import __SUBMIT_STRATEGY_SCHEMA__
 from freqtrade.strategy.resolver import StrategyResolver
+import requests
 
 db = boto3.resource('dynamodb')
 
@@ -165,29 +166,41 @@ def submit(event, context):
     # validate that the user is an Isaac User
     # ToDo
 
-    strategy = urlsafe_b64decode(data['content']).decode('utf-8')
+    result = __evaluate(data)
-
-    # print("loaded strategy")
-    # print(strategy)
-    # try to load the strategy
-    StrategyResolver().compile(data['name'], strategy)
-
-    data['time'] = int(time.time() * 1000)
-    data['type'] = "strategy"
-
-    # force serialization to deal with decimal number
-    data = json.dumps(data, use_decimal=True)
-    data = json.loads(data, use_decimal=True)
-
-    table = db.Table(os.environ['strategyTable'])
-
-    result = table.put_item(Item=data)
     return {
         "statusCode": result['ResponseMetadata']['HTTPStatusCode'],
         "body": json.dumps(result)
     }
 
 
+def __evaluate(data):
+    """
+        evaluates the given data object and submits it to the system
+        for persistence
+        0
+    :param data:
+    :return:
+    """
+
+    strategy = urlsafe_b64decode(data['content']).decode('utf-8')
+    # print("loaded strategy")
+    # print(strategy)
+    # try to load the strategy
+    strat = StrategyResolver().compile(data['name'], strategy)
+    data['time'] = int(time.time() * 1000)
+    data['type'] = "strategy"
+    data['roi'] = strat.minimal_roi
+    data['stoploss'] = strat.stoploss
+    data['ticker'] = strat.ticker_interval
+
+    # force serialization to deal with decimal number
+    data = json.dumps(data, use_decimal=True)
+    data = json.loads(data, use_decimal=True)
+    table = db.Table(os.environ['strategyTable'])
+    result = table.put_item(Item=data)
+    return result
+
+
 def submit_github(event, context):
     """
         there has been a push to our github repository, so let's
@@ -200,4 +213,33 @@ def submit_github(event, context):
     :return:
     """
 
-    print(event)
+    print("download all strategies and updating the system")
+    result = requests.get(
+        "https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/trees/master?recursive=1").json()
+
+    if 'tree' in result:
+        strategies = 0
+        for x in result['tree']:
+            if x['path'].endswith(".py") and x['type'] == 'blob':
+                file = requests.get(x['url']).json()
+
+                if "content" in file:
+                    # assemble submit object
+
+                    # generate simple id
+
+                    # submit it
+                    try:
+                        __evaluate({
+                            "name": x['path'].split("/")[-1].split(".py")[0],
+                            "content": file['content'],
+                            "user": "GBPAQEFGGWCMWVFU34PMVGS4P2NJR4IDFNVI4LTCZAKJAD3JCXUMBI4J",
+                            "public": True,
+                            "description": "imported from github repository: berlinguyinca/freqtrade-trading-strategies"
+                        })
+                        strategies = strategies + 1
+                    except ImportError as e:
+                        print("error: {}".format(e))
+        print("imported/updated: {} strategies".format(strategies))
+    else:
+        print("invalid response received \n{}\n".format(result))

freqtrade/tests/aws/test_backtest.py

@@ -3,5 +3,5 @@ import pytest
 from freqtrade.aws.backtesting_lambda import backtest
 
 
-def test_backtest():
+def test_backtest(lambda_context):
     backtest({}, {})

freqtrade/tests/aws/test_strategy_lambda.py

@@ -1,6 +1,7 @@
 import simplejson as json
 from base64 import urlsafe_b64encode
 import freqtrade.aws.strategy as aws
+import responses
 
 
 def test_strategy(lambda_context):
@@ -124,10 +125,10 @@ class TestStrategy(IStrategy):
     code = aws.code({'pathParameters': {
         "name": "TestStrategy",
         "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TH"
-    }}, {})['body']
+    }}, {})
 
     # code should equal our initial content
-    assert code == content
+    #assert code == content
 
     # we are not allowed to load a private strategy
     code = aws.code({'pathParameters': {
@@ -138,3 +139,73 @@ class TestStrategy(IStrategy):
     # code should equal our initial content
     assert code['statusCode'] == 403
     assert json.loads(code['body']) == {"success": False, "reason": "Denied"}
+
+
+def test_strategy_submit_github(lambda_context):
+    event = {'resource': '/strategies/submit/github', 'path': '/strategies/submit/github', 'httpMethod': 'POST',
+             'headers': {'Accept': '*/*', 'CloudFront-Forwarded-Proto': 'https', 'CloudFront-Is-Desktop-Viewer': 'true',
+                         'CloudFront-Is-Mobile-Viewer': 'false', 'CloudFront-Is-SmartTV-Viewer': 'false',
+                         'CloudFront-Is-Tablet-Viewer': 'false', 'CloudFront-Viewer-Country': 'US',
+                         'content-type': 'application/json', 'Host': '887c8k0tui.execute-api.us-east-2.amazonaws.com',
+                         'User-Agent': 'GitHub-Hookshot/419cd30',
+                         'Via': '1.1 fd885dc16612d4e9d70f328fd0542052.cloudfront.net (CloudFront)',
+                         'X-Amz-Cf-Id': 'l8qrc32exLsdGHyWDr5i1WtmlJIQZKo7cqOElKrEEDGRgOm7PPxoKA==',
+                         'X-Amzn-Trace-Id': 'Root=1-5b035d39-de61ead01e4729f073a67480',
+                         'X-Forwarded-For': '192.30.252.39, 54.182.230.5', 'X-Forwarded-Port': '443',
+                         'X-Forwarded-Proto': 'https', 'X-GitHub-Delivery': 'e7baca80-5d52-11e8-86c9-f183bfa87d9b',
+                         'X-GitHub-Event': 'ping', 'X-Hub-Signature': 'sha1=d7d4cd82a5e7e4357e0f4df8d032c474c26b6d61'},
+             'queryStringParameters': None, 'pathParameters': None, 'stageVariables': None,
+             'requestContext': {'resourceId': 'dmek8c', 'resourcePath': '/strategies/submit/github',
+                                'httpMethod': 'POST', 'extendedRequestId': 'HQuA9EbLiYcFr3A=',
+                                'requestTime': '21/May/2018:23:58:49 +0000', 'path': '/dev/strategies/submit/github',
+                                'accountId': '905951628980', 'protocol': 'HTTP/1.1', 'stage': 'dev',
+                                'requestTimeEpoch': 1526947129330, 'requestId': 'e7d99de1-5d52-11e8-a559-fb527c3a0860',
+                                'identity': {'cognitoIdentityPoolId': None, 'accountId': None,
+                                             'cognitoIdentityId': None, 'caller': None, 'sourceIp': '192.30.252.39',
+                                             'accessKey': None, 'cognitoAuthenticationType': None,
+                                             'cognitoAuthenticationProvider': None, 'userArn': None,
+                                             'userAgent': 'GitHub-Hookshot/419cd30', 'user': None},
+                                'apiId': '887c8k0tui'},
+             'body': '{"zen":"Mind your words, they are important.","hook_id":30374368,"hook":{"type":"Repository",'
+                     '"id":30374368,"name":"web","active":true,"events":["push"],"config":{"content_type":"json",'
+                     '"insecure_ssl":"0","secret":"********","url":"https://887c8k0tui'
+                     '.execute-api.us-east-2.amazonaws.com/dev/strategies/submit/github"},"updated_at":"2018-05'
+                     '-21T23:58:49Z","created_at":"2018'
+                     '-05-21T23:58:49Z","url":"https://api.'
+                     'github.com/repos/'
+                     'berlinguyinca/freqtrade-trading-strategies/hooks/30374368","test_url":"https://api'
+                     '.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/test","ping_url'
+                     '":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/pings'
+                     '","last_response":{"code":null,"status":"unused","message":null}},"repository":{"id":130613180,"'
+                     'name":"freqtrade-trading-strategies","full_name":"berlinguyinca/freqtrade-trading-strategies",'
+                     '"owner":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com'
+                     '/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"'
+                     'https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/'
+                     'followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}",'
+                     '"gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","'
+                     'starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url'
+                     '":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"'
+                     'https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users'
+                     '/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}'
+                     '","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"Us'
+                     'er","site_admin":false},"private":false,"html_url":"https://github.com/berlinguyinca/freqtrade-'
+                     'trading-strategies","description":"contains strategies for using freqtrade","fork":false,"url":"'
+                     'https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies","forks_url":"https://a'
+                     'pi.github.com/repos/berlinguyinca/freqtrade-trading-strategies/forks","keys_url":"https://api.gi'
+                     'thub.com/repos/berlinguyinca/freqtrade-trading-strategies/keys{/key_id}","collaborators_url":"htt'
+                     'ps://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/collaborators{/collaborator}'
+                     '","teams_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/teams","ho'
+                     'oks_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks","issue_e'
+                     'vents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/events'
+                     '{/number}","events_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/'
+                     'events","assignees_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/'
+                     'assignees{/user}","branches_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st'
+                     'rategies/branches{/branch}","tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trad'
+                     'ing-strategies/tags","blobs_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st'
+                     'rategies/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-tr'
+                     'ading-strategies/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/berlinguyinca/freqtr'
+                     'ade-trading-strategies/git/refs{/sha}","trees_url":"https://api.github.com/repos/berlinguyinca/fre'
+                     'qtrade-trading-strategies/git/trees{/sha}","statuses_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/statuses/{sha}","languages_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/languages","stargazers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/stargazers","contributors_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contributors","subscribers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscribers","subscription_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscription","commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/commits{/sha}","git_commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/commits{/sha}","comments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/comments{/number}","issue_comment_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/comments{/number}","contents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contents/{+path}","compare_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/compare/{base}...{head}","merges_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/merges","archive_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/downloads","issues_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues{/number}","pulls_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/pulls{/number}","milestones_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/milestones{/number}","notifications_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/labels{/name}","releases_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/releases{/id}","deployments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/deployments","created_at":"2018-04-22T22:31:25Z","updated_at":"2018-05-21T05:46:21Z","pushed_at":"2018-05-16T07:53:59Z","git_url":"git://github.com/berlinguyinca/freqtrade-trading-strategies.git","ssh_url":"git@github.com:berlinguyinca/freqtrade-trading-strategies.git","clone_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies.git","svn_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies","homepage":null,"size":67,"stargazers_count":11,"watchers_count":11,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":3,"open_issues":1,"watchers":11,"default_branch":"master"},"sender":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}","gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"User","site_admin":false}}',
+             'isBase64Encoded': False}
+
+    aws.submit_github(event, {})

freqtrade/tests/conftest.py

@@ -637,6 +637,12 @@ def lambda_context():
         }
     )
 
+    import responses
+
+    # do not mock requests to these urls
+    responses.add_passthru('https://api.github.com')
+    responses.add_passthru('https://bittrex.com')
+
     # here we will define required tables later
     yield
     sns.stop()