Merge branch 'aws' of https://github.com/berlinguyinca/freqtrade into aws

freqtrade/aws/backtesting_lambda.py

@@ -1,8 +1,15 @@
 import logging
 
+import boto3
+import os
+
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.optimize.backtesting import Backtesting
+import simplejson as json
+from boto3.dynamodb.conditions import Key, Attr
+
+db = boto3.resource('dynamodb')
 
 
 def backtest(event, context):
@@ -29,25 +36,39 @@ def backtest(event, context):
             no return
     """
 
-    name = "TestStrategy"
+    if 'body' in event:
-    user = "12345678"
+        event['body'] = json.loads(event['body'])
-    stake_currency = "USDT"
+        name = event['body']['name']
-    asset = ["ETH", "BTC"]
+        user = event['body']['user']
-    exchange = "binance"
+        stake_currency = event['body']['stake_currency'].upper()
+        asset = event['body']['asset']
+        exchange = event['body']['exchange']
 
         assets = list(map(lambda x: "{}/{}".format(x, stake_currency).upper(), asset))
 
+        table = db.Table(os.environ['strategyTable'])
+
+        response = table.query(
+            KeyConditionExpression=Key('user').eq(user) &
+                                   Key('name').eq(name)
+
+        )
+
+        print(response)
+        if "Items" in response and len(response['Items']) > 0:
+
+            content = response['Items'][0]['content']
             configuration = {
                 "max_open_trades": 1,
                 "stake_currency": stake_currency,
-        "stake_amount": 0.001,
+                "stake_amount": 1,
                 "fiat_display_currency": "USD",
                 "unfilledtimeout": 600,
                 "bid_strategy": {
                     "ask_last_balance": 0.0
                 },
                 "exchange": {
-            "name": "bittrex",
+                    "name": exchange,
                     "enabled": True,
                     "key": "key",
                     "secret": "secret",
@@ -68,7 +89,8 @@ def backtest(event, context):
                     "process_throttle_secs": 5
                 },
                 'realistic_simulation': True,
-        "loglevel": logging.DEBUG
+                "loglevel": logging.DEBUG,
+                "strategy": "{}:{}".format(name, content)
 
             }
 
@@ -80,23 +102,26 @@ def backtest(event, context):
             result = backtesting.start()
             print("finished test")
 
-    print(result)
             print("persist data in dynamo")
 
             for index, row in result.iterrows():
+                if row['loss'] > 0 or row['profit'] > 0:
                     item = {
                         "id": "{}.{}:{}".format(user, name, row['pair']),
                         "pair": row['pair'],
-            "profit": row['profit'],
+                        "count_profit": row['profit'],
-            "loss": row['loss'],
+                        "count_loss": row['loss'],
-            "duration": row['avg duration'],
+                        "avg_duration": row['avg duration'],
                         "avg profit": row['avg profit %'],
                         "total profit": row['total profit {}'.format(stake_currency)]
 
                     }
 
                 print(item)
-    pass
+        else:
+            raise Exception("sorry we did not find any matching strategy for user {} and name {}".format(user, name))
+    else:
+        raise Exception("no body provided")
 
 
 def submit(event, context):

freqtrade/aws/strategy.py

@@ -9,6 +9,7 @@ from jsonschema import validate
 
 from freqtrade.aws.schemas import __SUBMIT_STRATEGY_SCHEMA__
 from freqtrade.strategy.resolver import StrategyResolver
+import requests
 
 db = boto3.resource('dynamodb')
 
@@ -94,11 +95,6 @@ def code(event, context):
     :return:
     """
 
-    print("event")
-    print(event)
-    print("context")
-    print(context)
-
     user = ""
     name = ""
 
@@ -164,29 +160,41 @@ def submit(event, context):
     # validate that the user is an Isaac User
     # ToDo
 
-    strategy = urlsafe_b64decode(data['content']).decode('utf-8')
+    result = __evaluate(data)
-
-    # print("loaded strategy")
-    # print(strategy)
-    # try to load the strategy
-    StrategyResolver().compile(data['name'], strategy)
-
-    data['time'] = int(time.time() * 1000)
-    data['type'] = "strategy"
-
-    # force serialization to deal with decimal number
-    data = json.dumps(data, use_decimal=True)
-    data = json.loads(data, use_decimal=True)
-
-    table = db.Table(os.environ['strategyTable'])
-
-    result = table.put_item(Item=data)
     return {
         "statusCode": result['ResponseMetadata']['HTTPStatusCode'],
         "body": json.dumps(result)
     }
 
 
+def __evaluate(data):
+    """
+        evaluates the given data object and submits it to the system
+        for persistence
+        0
+    :param data:
+    :return:
+    """
+
+    strategy = urlsafe_b64decode(data['content']).decode('utf-8')
+    # print("loaded strategy")
+    # print(strategy)
+    # try to load the strategy
+    strat = StrategyResolver().compile(data['name'], strategy)
+    data['time'] = int(time.time() * 1000)
+    data['type'] = "strategy"
+    data['roi'] = strat.minimal_roi
+    data['stoploss'] = strat.stoploss
+    data['ticker'] = strat.ticker_interval
+
+    # force serialization to deal with decimal number
+    data = json.dumps(data, use_decimal=True)
+    data = json.loads(data, use_decimal=True)
+    table = db.Table(os.environ['strategyTable'])
+    result = table.put_item(Item=data)
+    return result
+
+
 def submit_github(event, context):
     """
         there has been a push to our github repository, so let's
@@ -199,4 +207,33 @@ def submit_github(event, context):
     :return:
     """
 
-    print(event)
+    print("download all strategies and updating the system")
+    result = requests.get(
+        "https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/trees/master?recursive=1").json()
+
+    if 'tree' in result:
+        strategies = 0
+        for x in result['tree']:
+            if x['path'].endswith(".py") and x['type'] == 'blob':
+                file = requests.get(x['url']).json()
+
+                if "content" in file:
+                    # assemble submit object
+
+                    # generate simple id
+
+                    # submit it
+                    try:
+                        __evaluate({
+                            "name": x['path'].split("/")[-1].split(".py")[0],
+                            "content": file['content'],
+                            "user": "GBPAQEFGGWCMWVFU34PMVGS4P2NJR4IDFNVI4LTCZAKJAD3JCXUMBI4J",
+                            "public": True,
+                            "description": "imported from github repository: berlinguyinca/freqtrade-trading-strategies"
+                        })
+                        strategies = strategies + 1
+                    except ImportError as e:
+                        print("error: {}".format(e))
+        print("imported/updated: {} strategies".format(strategies))
+    else:
+        print("invalid response received \n{}\n".format(result))

freqtrade/strategy/resolver.py

@@ -6,6 +6,7 @@ This module load custom strategies
 import importlib.util
 import inspect
 import logging
+from base64 import urlsafe_b64decode
 from collections import OrderedDict
 from typing import Optional, Dict, Type
 
@@ -88,8 +89,26 @@ class StrategyResolver(object):
             # Add extra strategy directory on top of search paths
             abs_paths.insert(0, extra_dir)
 
-        try:
+        # check if the given strategy is provided as name, value pair
+        # where the value is the strategy encoded in base 64
+        if ":" in strategy_name:
+            strat = strategy_name.split(":")
+
+            if len(strat) == 2:
+                temp = Path(tempfile.mkdtemp("freq", "strategy"))
+                name = strat[0] + ".py"
+
+                temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8'))
+                temp.joinpath("__init__.py").touch()
+
+                strategy_name = os.path.splitext(name)[0]
+
+                # register temp path with the bot
+                abs_paths.insert(0, temp.absolute())
+
         # check if given strategy matches an url
+        else:
+            try:
                 logger.debug("requesting remote strategy from {}".format(strategy_name))
                 resp = requests.get(strategy_name, stream=True)
                 if resp.status_code == 200:

freqtrade/tests/aws/test_backtest.py

@@ -1,7 +1,75 @@
+from base64 import urlsafe_b64encode
+
 import pytest
-
+import simplejson as json
 from freqtrade.aws.backtesting_lambda import backtest
+from freqtrade.aws.strategy import submit
 
 
-def test_backtest():
+def test_backtest(lambda_context):
-    backtest({}, {})
+    content = """# --- Do not remove these libs ---
+from freqtrade.strategy.interface import IStrategy
+from typing import Dict, List
+from hyperopt import hp
+from functools import reduce
+from pandas import DataFrame
+# --------------------------------
+
+import talib.abstract as ta
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyFancyTestStrategy(IStrategy):
+    minimal_roi = {
+        "0": 0.5
+    }
+    stoploss = -0.2
+    ticker_interval = '5m'
+
+    def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
+        macd = ta.MACD(dataframe)
+        dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8)
+        dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21)
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['maShort'], dataframe['maMedium'])
+            ),
+            'buy'] = 1
+
+        return dataframe
+
+    def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['maMedium'], dataframe['maShort'])
+            ),
+            'sell'] = 1
+        return dataframe
+
+
+        """
+
+    request = {
+        "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+        "description": "simple test strategy",
+        "name": "MyFancyTestStrategy",
+        "content": urlsafe_b64encode(content.encode('utf-8')),
+        "public": False
+    }
+
+    # now we add an entry
+    submit({
+        "body": json.dumps(request)
+    }, {})
+
+    request = {
+        "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+        "name": "MyFancyTestStrategy",
+        "stake_currency": "usdt",
+        "asset": ["ETH", "BTC", "XRP", "LTC"],
+        "exchange": "binance"
+    }
+
+    backtest({"body": json.dumps(request)}, {})

freqtrade/tests/aws/test_strategy_lambda.py

@@ -1,6 +1,7 @@
 import simplejson as json
 from base64 import urlsafe_b64encode
 import freqtrade.aws.strategy as aws
+import responses
 
 
 def test_strategy(lambda_context):
@@ -124,10 +125,10 @@ class TestStrategy(IStrategy):
     code = aws.code({'pathParameters': {
         "name": "TestStrategy",
         "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TH"
-    }}, {})['body']
+    }}, {})
 
     # code should equal our initial content
-    assert code == content
+    #assert code == content
 
     # we are not allowed to load a private strategy
     code = aws.code({'pathParameters': {
@@ -138,3 +139,73 @@ class TestStrategy(IStrategy):
     # code should equal our initial content
     assert code['statusCode'] == 403
     assert json.loads(code['body']) == {"success": False, "reason": "Denied"}
+
+
+def test_strategy_submit_github(lambda_context):
+    event = {'resource': '/strategies/submit/github', 'path': '/strategies/submit/github', 'httpMethod': 'POST',
+             'headers': {'Accept': '*/*', 'CloudFront-Forwarded-Proto': 'https', 'CloudFront-Is-Desktop-Viewer': 'true',
+                         'CloudFront-Is-Mobile-Viewer': 'false', 'CloudFront-Is-SmartTV-Viewer': 'false',
+                         'CloudFront-Is-Tablet-Viewer': 'false', 'CloudFront-Viewer-Country': 'US',
+                         'content-type': 'application/json', 'Host': '887c8k0tui.execute-api.us-east-2.amazonaws.com',
+                         'User-Agent': 'GitHub-Hookshot/419cd30',
+                         'Via': '1.1 fd885dc16612d4e9d70f328fd0542052.cloudfront.net (CloudFront)',
+                         'X-Amz-Cf-Id': 'l8qrc32exLsdGHyWDr5i1WtmlJIQZKo7cqOElKrEEDGRgOm7PPxoKA==',
+                         'X-Amzn-Trace-Id': 'Root=1-5b035d39-de61ead01e4729f073a67480',
+                         'X-Forwarded-For': '192.30.252.39, 54.182.230.5', 'X-Forwarded-Port': '443',
+                         'X-Forwarded-Proto': 'https', 'X-GitHub-Delivery': 'e7baca80-5d52-11e8-86c9-f183bfa87d9b',
+                         'X-GitHub-Event': 'ping', 'X-Hub-Signature': 'sha1=d7d4cd82a5e7e4357e0f4df8d032c474c26b6d61'},
+             'queryStringParameters': None, 'pathParameters': None, 'stageVariables': None,
+             'requestContext': {'resourceId': 'dmek8c', 'resourcePath': '/strategies/submit/github',
+                                'httpMethod': 'POST', 'extendedRequestId': 'HQuA9EbLiYcFr3A=',
+                                'requestTime': '21/May/2018:23:58:49 +0000', 'path': '/dev/strategies/submit/github',
+                                'accountId': '905951628980', 'protocol': 'HTTP/1.1', 'stage': 'dev',
+                                'requestTimeEpoch': 1526947129330, 'requestId': 'e7d99de1-5d52-11e8-a559-fb527c3a0860',
+                                'identity': {'cognitoIdentityPoolId': None, 'accountId': None,
+                                             'cognitoIdentityId': None, 'caller': None, 'sourceIp': '192.30.252.39',
+                                             'accessKey': None, 'cognitoAuthenticationType': None,
+                                             'cognitoAuthenticationProvider': None, 'userArn': None,
+                                             'userAgent': 'GitHub-Hookshot/419cd30', 'user': None},
+                                'apiId': '887c8k0tui'},
+             'body': '{"zen":"Mind your words, they are important.","hook_id":30374368,"hook":{"type":"Repository",'
+                     '"id":30374368,"name":"web","active":true,"events":["push"],"config":{"content_type":"json",'
+                     '"insecure_ssl":"0","secret":"********","url":"https://887c8k0tui'
+                     '.execute-api.us-east-2.amazonaws.com/dev/strategies/submit/github"},"updated_at":"2018-05'
+                     '-21T23:58:49Z","created_at":"2018'
+                     '-05-21T23:58:49Z","url":"https://api.'
+                     'github.com/repos/'
+                     'berlinguyinca/freqtrade-trading-strategies/hooks/30374368","test_url":"https://api'
+                     '.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/test","ping_url'
+                     '":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/pings'
+                     '","last_response":{"code":null,"status":"unused","message":null}},"repository":{"id":130613180,"'
+                     'name":"freqtrade-trading-strategies","full_name":"berlinguyinca/freqtrade-trading-strategies",'
+                     '"owner":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com'
+                     '/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"'
+                     'https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/'
+                     'followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}",'
+                     '"gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","'
+                     'starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url'
+                     '":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"'
+                     'https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users'
+                     '/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}'
+                     '","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"Us'
+                     'er","site_admin":false},"private":false,"html_url":"https://github.com/berlinguyinca/freqtrade-'
+                     'trading-strategies","description":"contains strategies for using freqtrade","fork":false,"url":"'
+                     'https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies","forks_url":"https://a'
+                     'pi.github.com/repos/berlinguyinca/freqtrade-trading-strategies/forks","keys_url":"https://api.gi'
+                     'thub.com/repos/berlinguyinca/freqtrade-trading-strategies/keys{/key_id}","collaborators_url":"htt'
+                     'ps://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/collaborators{/collaborator}'
+                     '","teams_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/teams","ho'
+                     'oks_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks","issue_e'
+                     'vents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/events'
+                     '{/number}","events_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/'
+                     'events","assignees_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/'
+                     'assignees{/user}","branches_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st'
+                     'rategies/branches{/branch}","tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trad'
+                     'ing-strategies/tags","blobs_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st'
+                     'rategies/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-tr'
+                     'ading-strategies/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/berlinguyinca/freqtr'
+                     'ade-trading-strategies/git/refs{/sha}","trees_url":"https://api.github.com/repos/berlinguyinca/fre'
+                     'qtrade-trading-strategies/git/trees{/sha}","statuses_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/statuses/{sha}","languages_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/languages","stargazers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/stargazers","contributors_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contributors","subscribers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscribers","subscription_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscription","commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/commits{/sha}","git_commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/commits{/sha}","comments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/comments{/number}","issue_comment_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/comments{/number}","contents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contents/{+path}","compare_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/compare/{base}...{head}","merges_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/merges","archive_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/downloads","issues_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues{/number}","pulls_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/pulls{/number}","milestones_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/milestones{/number}","notifications_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/labels{/name}","releases_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/releases{/id}","deployments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/deployments","created_at":"2018-04-22T22:31:25Z","updated_at":"2018-05-21T05:46:21Z","pushed_at":"2018-05-16T07:53:59Z","git_url":"git://github.com/berlinguyinca/freqtrade-trading-strategies.git","ssh_url":"git@github.com:berlinguyinca/freqtrade-trading-strategies.git","clone_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies.git","svn_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies","homepage":null,"size":67,"stargazers_count":11,"watchers_count":11,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":3,"open_issues":1,"watchers":11,"default_branch":"master"},"sender":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}","gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"User","site_admin":false}}',
+             'isBase64Encoded': False}
+
+    aws.submit_github(event, {})

freqtrade/tests/conftest.py

@@ -637,6 +637,12 @@ def lambda_context():
         }
     )
 
+    import responses
+
+    # do not mock requests to these urls
+    responses.add_passthru('https://api.github.com')
+    responses.add_passthru('https://bittrex.com')
+    responses.add_passthru('https://api.binance.com')
     # here we will define required tables later
     yield
     sns.stop()