adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'aws' of https://github.com/berlinguyinca/freqtrade into aws

Browse Source
This commit is contained in:
EC2 Default User 2018-05-22 03:50:07 +00:00
commit b76e659768
6 changed files with 323 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
freqtrade/aws/backtesting_lambda.py
View File

@ -1,8 +1,15 @@
import logging
import boto3
import os
from freqtrade.arguments import Arguments
from freqtrade.configuration import Configuration
from freqtrade.optimize.backtesting import Backtesting
import simplejson as json
from boto3.dynamodb.conditions import Key, Attr
db = boto3.resource('dynamodb')
def backtest(event, context):
@ -29,25 +36,39 @@ def backtest(event, context):
no return
"""
name = "TestStrategy"
user = "12345678"
stake_currency = "USDT"
asset = ["ETH", "BTC"]
exchange = "binance"
if 'body' in event:
event['body'] = json.loads(event['body'])
name = event['body']['name']
user = event['body']['user']
stake_currency = event['body']['stake_currency'].upper()
asset = event['body']['asset']
exchange = event['body']['exchange']
assets = list(map(lambda x: "{}/{}".format(x, stake_currency).upper(), asset))
table = db.Table(os.environ['strategyTable'])
response = table.query(
KeyConditionExpression=Key('user').eq(user) &
Key('name').eq(name)
)
print(response)
if "Items" in response and len(response['Items']) > 0:
content = response['Items'][0]['content']
configuration = {
"max_open_trades": 1,
"stake_currency": stake_currency,
"stake_amount": 0.001,
"stake_amount": 1,
"fiat_display_currency": "USD",
"unfilledtimeout": 600,
"bid_strategy": {
"ask_last_balance": 0.0
},
"exchange": {
"name": "bittrex",
"name": exchange,
"enabled": True,
"key": "key",
"secret": "secret",
@ -68,7 +89,8 @@ def backtest(event, context):
"process_throttle_secs": 5
},
'realistic_simulation': True,
"loglevel": logging.DEBUG
"loglevel": logging.DEBUG,
"strategy": "{}:{}".format(name, content)
}
@ -80,23 +102,26 @@ def backtest(event, context):
result = backtesting.start()
print("finished test")
print(result)
print("persist data in dynamo")
for index, row in result.iterrows():
if row['loss'] > 0 or row['profit'] > 0:
item = {
"id": "{}.{}:{}".format(user, name, row['pair']),
"pair": row['pair'],
"profit": row['profit'],
"loss": row['loss'],
"duration": row['avg duration'],
"count_profit": row['profit'],
"count_loss": row['loss'],
"avg_duration": row['avg duration'],
"avg profit": row['avg profit %'],
"total profit": row['total profit {}'.format(stake_currency)]
}
print(item)
pass
else:
raise Exception("sorry we did not find any matching strategy for user {} and name {}".format(user, name))
else:
raise Exception("no body provided")
def submit(event, context):

83
freqtrade/aws/strategy.py
View File

@ -9,6 +9,7 @@ from jsonschema import validate
from freqtrade.aws.schemas import __SUBMIT_STRATEGY_SCHEMA__
from freqtrade.strategy.resolver import StrategyResolver
import requests
db = boto3.resource('dynamodb')
@ -94,11 +95,6 @@ def code(event, context):
:return:
"""
print("event")
print(event)
print("context")
print(context)
user = ""
name = ""
@ -164,29 +160,41 @@ def submit(event, context):
# validate that the user is an Isaac User
# ToDo
strategy = urlsafe_b64decode(data['content']).decode('utf-8')
# print("loaded strategy")
# print(strategy)
# try to load the strategy
StrategyResolver().compile(data['name'], strategy)
data['time'] = int(time.time() * 1000)
data['type'] = "strategy"
# force serialization to deal with decimal number
data = json.dumps(data, use_decimal=True)
data = json.loads(data, use_decimal=True)
table = db.Table(os.environ['strategyTable'])
result = table.put_item(Item=data)
result = __evaluate(data)
return {
"statusCode": result['ResponseMetadata']['HTTPStatusCode'],
"body": json.dumps(result)
}
def __evaluate(data):
"""
evaluates the given data object and submits it to the system
for persistence
0
:param data:
:return:
"""
strategy = urlsafe_b64decode(data['content']).decode('utf-8')
# print("loaded strategy")
# print(strategy)
# try to load the strategy
strat = StrategyResolver().compile(data['name'], strategy)
data['time'] = int(time.time() * 1000)
data['type'] = "strategy"
data['roi'] = strat.minimal_roi
data['stoploss'] = strat.stoploss
data['ticker'] = strat.ticker_interval
# force serialization to deal with decimal number
data = json.dumps(data, use_decimal=True)
data = json.loads(data, use_decimal=True)
table = db.Table(os.environ['strategyTable'])
result = table.put_item(Item=data)
return result
def submit_github(event, context):
"""
there has been a push to our github repository, so let's
@ -199,4 +207,33 @@ def submit_github(event, context):
:return:
"""
print(event)
print("download all strategies and updating the system")
result = requests.get(
"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/trees/master?recursive=1").json()
if 'tree' in result:
strategies = 0
for x in result['tree']:
if x['path'].endswith(".py") and x['type'] == 'blob':
file = requests.get(x['url']).json()
if "content" in file:
# assemble submit object
# generate simple id
# submit it
try:
__evaluate({
"name": x['path'].split("/")[-1].split(".py")[0],
"content": file['content'],
"user": "GBPAQEFGGWCMWVFU34PMVGS4P2NJR4IDFNVI4LTCZAKJAD3JCXUMBI4J",
"public": True,
"description": "imported from github repository: berlinguyinca/freqtrade-trading-strategies"
})
strategies = strategies + 1
except ImportError as e:
print("error: {}".format(e))
print("imported/updated: {} strategies".format(strategies))
else:
print("invalid response received \n{}\n".format(result))

21
freqtrade/strategy/resolver.py
View File

@ -6,6 +6,7 @@ This module load custom strategies
import importlib.util
import inspect
import logging
from base64 import urlsafe_b64decode
from collections import OrderedDict
from typing import Optional, Dict, Type
@ -88,8 +89,26 @@ class StrategyResolver(object):
# Add extra strategy directory on top of search paths
abs_paths.insert(0, extra_dir)
try:
# check if the given strategy is provided as name, value pair
# where the value is the strategy encoded in base 64
if ":" in strategy_name:
strat = strategy_name.split(":")
if len(strat) == 2:
temp = Path(tempfile.mkdtemp("freq", "strategy"))
name = strat[0] + ".py"
temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8'))
temp.joinpath("__init__.py").touch()
strategy_name = os.path.splitext(name)[0]
# register temp path with the bot
abs_paths.insert(0, temp.absolute())
# check if given strategy matches an url
else:
try:
logger.debug("requesting remote strategy from {}".format(strategy_name))
resp = requests.get(strategy_name, stream=True)
if resp.status_code == 200:

74
freqtrade/tests/aws/test_backtest.py
View File

@ -1,7 +1,75 @@
from base64 import urlsafe_b64encode
import pytest
import simplejson as json
from freqtrade.aws.backtesting_lambda import backtest
from freqtrade.aws.strategy import submit
def test_backtest():
backtest({}, {})
def test_backtest(lambda_context):
content = """# --- Do not remove these libs ---
from freqtrade.strategy.interface import IStrategy
from typing import Dict, List
from hyperopt import hp
from functools import reduce
from pandas import DataFrame
# --------------------------------
import talib.abstract as ta
import freqtrade.vendor.qtpylib.indicators as qtpylib
class MyFancyTestStrategy(IStrategy):
minimal_roi = {
"0": 0.5
}
stoploss = -0.2
ticker_interval = '5m'
def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
macd = ta.MACD(dataframe)
dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8)
dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21)
return dataframe
def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
dataframe.loc[
(
qtpylib.crossed_above(dataframe['maShort'], dataframe['maMedium'])
),
'buy'] = 1
return dataframe
def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame:
dataframe.loc[
(
qtpylib.crossed_above(dataframe['maMedium'], dataframe['maShort'])
),
'sell'] = 1
return dataframe
"""
request = {
"user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
"description": "simple test strategy",
"name": "MyFancyTestStrategy",
"content": urlsafe_b64encode(content.encode('utf-8')),
"public": False
}
# now we add an entry
submit({
"body": json.dumps(request)
}, {})
request = {
"user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
"name": "MyFancyTestStrategy",
"stake_currency": "usdt",
"asset": ["ETH", "BTC", "XRP", "LTC"],
"exchange": "binance"
}
backtest({"body": json.dumps(request)}, {})

75
freqtrade/tests/aws/test_strategy_lambda.py
View File

@ -1,6 +1,7 @@
import simplejson as json
from base64 import urlsafe_b64encode
import freqtrade.aws.strategy as aws
import responses
def test_strategy(lambda_context):
@ -124,10 +125,10 @@ class TestStrategy(IStrategy):
code = aws.code({'pathParameters': {
"name": "TestStrategy",
"user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TH"
}}, {})['body']
}}, {})
# code should equal our initial content
assert code == content
#assert code == content
# we are not allowed to load a private strategy
code = aws.code({'pathParameters': {
@ -138,3 +139,73 @@ class TestStrategy(IStrategy):
# code should equal our initial content
assert code['statusCode'] == 403
assert json.loads(code['body']) == {"success": False, "reason": "Denied"}
def test_strategy_submit_github(lambda_context):
event = {'resource': '/strategies/submit/github', 'path': '/strategies/submit/github', 'httpMethod': 'POST',
'headers': {'Accept': '*/*', 'CloudFront-Forwarded-Proto': 'https', 'CloudFront-Is-Desktop-Viewer': 'true',
'CloudFront-Is-Mobile-Viewer': 'false', 'CloudFront-Is-SmartTV-Viewer': 'false',
'CloudFront-Is-Tablet-Viewer': 'false', 'CloudFront-Viewer-Country': 'US',
'content-type': 'application/json', 'Host': '887c8k0tui.execute-api.us-east-2.amazonaws.com',
'User-Agent': 'GitHub-Hookshot/419cd30',
'Via': '1.1 fd885dc16612d4e9d70f328fd0542052.cloudfront.net (CloudFront)',
'X-Amz-Cf-Id': 'l8qrc32exLsdGHyWDr5i1WtmlJIQZKo7cqOElKrEEDGRgOm7PPxoKA==',
'X-Amzn-Trace-Id': 'Root=1-5b035d39-de61ead01e4729f073a67480',
'X-Forwarded-For': '192.30.252.39, 54.182.230.5', 'X-Forwarded-Port': '443',
'X-Forwarded-Proto': 'https', 'X-GitHub-Delivery': 'e7baca80-5d52-11e8-86c9-f183bfa87d9b',
'X-GitHub-Event': 'ping', 'X-Hub-Signature': 'sha1=d7d4cd82a5e7e4357e0f4df8d032c474c26b6d61'},
'queryStringParameters': None, 'pathParameters': None, 'stageVariables': None,
'requestContext': {'resourceId': 'dmek8c', 'resourcePath': '/strategies/submit/github',
'httpMethod': 'POST', 'extendedRequestId': 'HQuA9EbLiYcFr3A=',
'requestTime': '21/May/2018:23:58:49 +0000', 'path': '/dev/strategies/submit/github',
'accountId': '905951628980', 'protocol': 'HTTP/1.1', 'stage': 'dev',
'requestTimeEpoch': 1526947129330, 'requestId': 'e7d99de1-5d52-11e8-a559-fb527c3a0860',
'identity': {'cognitoIdentityPoolId': None, 'accountId': None,
'cognitoIdentityId': None, 'caller': None, 'sourceIp': '192.30.252.39',
'accessKey': None, 'cognitoAuthenticationType': None,
'cognitoAuthenticationProvider': None, 'userArn': None,
'userAgent': 'GitHub-Hookshot/419cd30', 'user': None},
'apiId': '887c8k0tui'},
'body': '{"zen":"Mind your words, they are important.","hook_id":30374368,"hook":{"type":"Repository",'
'"id":30374368,"name":"web","active":true,"events":["push"],"config":{"content_type":"json",'
'"insecure_ssl":"0","secret":"********","url":"https://887c8k0tui'
'.execute-api.us-east-2.amazonaws.com/dev/strategies/submit/github"},"updated_at":"2018-05'
'-21T23:58:49Z","created_at":"2018'
'-05-21T23:58:49Z","url":"https://api.'
'github.com/repos/'
'berlinguyinca/freqtrade-trading-strategies/hooks/30374368","test_url":"https://api'
'.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/test","ping_url'
'":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks/30374368/pings'
'","last_response":{"code":null,"status":"unused","message":null}},"repository":{"id":130613180,"'
'name":"freqtrade-trading-strategies","full_name":"berlinguyinca/freqtrade-trading-strategies",'
'"owner":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com'
'/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"'
'https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/'
'followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}",'
'"gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","'
'starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url'
'":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"'
'https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users'
'/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}'
'","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"Us'
'er","site_admin":false},"private":false,"html_url":"https://github.com/berlinguyinca/freqtrade-'
'trading-strategies","description":"contains strategies for using freqtrade","fork":false,"url":"'
'https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies","forks_url":"https://a'
'pi.github.com/repos/berlinguyinca/freqtrade-trading-strategies/forks","keys_url":"https://api.gi'
'thub.com/repos/berlinguyinca/freqtrade-trading-strategies/keys{/key_id}","collaborators_url":"htt'
'ps://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/collaborators{/collaborator}'
'","teams_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/teams","ho'
'oks_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/hooks","issue_e'
'vents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/events'
'{/number}","events_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/'
'events","assignees_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/'
'assignees{/user}","branches_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st'
'rategies/branches{/branch}","tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trad'
'ing-strategies/tags","blobs_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-st'
'rategies/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/berlinguyinca/freqtrade-tr'
'ading-strategies/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/berlinguyinca/freqtr'
'ade-trading-strategies/git/refs{/sha}","trees_url":"https://api.github.com/repos/berlinguyinca/fre'
'qtrade-trading-strategies/git/trees{/sha}","statuses_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/statuses/{sha}","languages_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/languages","stargazers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/stargazers","contributors_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contributors","subscribers_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscribers","subscription_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/subscription","commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/commits{/sha}","git_commits_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/git/commits{/sha}","comments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/comments{/number}","issue_comment_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues/comments{/number}","contents_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/contents/{+path}","compare_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/compare/{base}...{head}","merges_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/merges","archive_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/downloads","issues_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/issues{/number}","pulls_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/pulls{/number}","milestones_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/milestones{/number}","notifications_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/labels{/name}","releases_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/releases{/id}","deployments_url":"https://api.github.com/repos/berlinguyinca/freqtrade-trading-strategies/deployments","created_at":"2018-04-22T22:31:25Z","updated_at":"2018-05-21T05:46:21Z","pushed_at":"2018-05-16T07:53:59Z","git_url":"git://github.com/berlinguyinca/freqtrade-trading-strategies.git","ssh_url":"git@github.com:berlinguyinca/freqtrade-trading-strategies.git","clone_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies.git","svn_url":"https://github.com/berlinguyinca/freqtrade-trading-strategies","homepage":null,"size":67,"stargazers_count":11,"watchers_count":11,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":3,"open_issues":1,"watchers":11,"default_branch":"master"},"sender":{"login":"berlinguyinca","id":16364,"avatar_url":"https://avatars2.githubusercontent.com/u/16364?v=4","gravatar_id":"","url":"https://api.github.com/users/berlinguyinca","html_url":"https://github.com/berlinguyinca","followers_url":"https://api.github.com/users/berlinguyinca/followers","following_url":"https://api.github.com/users/berlinguyinca/following{/other_user}","gists_url":"https://api.github.com/users/berlinguyinca/gists{/gist_id}","starred_url":"https://api.github.com/users/berlinguyinca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berlinguyinca/subscriptions","organizations_url":"https://api.github.com/users/berlinguyinca/orgs","repos_url":"https://api.github.com/users/berlinguyinca/repos","events_url":"https://api.github.com/users/berlinguyinca/events{/privacy}","received_events_url":"https://api.github.com/users/berlinguyinca/received_events","type":"User","site_admin":false}}',
'isBase64Encoded': False}
aws.submit_github(event, {})

6
freqtrade/tests/conftest.py
View File

@ -637,6 +637,12 @@ def lambda_context():
}
)
import responses
# do not mock requests to these urls
responses.add_passthru('https://api.github.com')
responses.add_passthru('https://bittrex.com')
responses.add_passthru('https://api.binance.com')
# here we will define required tables later
yield
sns.stop()