From 71b246b9f10b3d0f00df53c1841650fe0ae9efa9 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 4 Mar 2018 17:55:17 +0200 Subject: [PATCH 01/35] DOC: added parameter to doc on exchange_bundle --- catalyst/exchange/exchange_bundle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 569fa6e55..16f08658f 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -843,6 +843,7 @@ def get_history_window_series_and_load(self, field: str data_frequency: str algo_end_dt: pd.Timestamp + force_auto_ingest: Returns ------- From 3335ef16a4585c4301c5d9c57888ad070a02681d Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 5 Mar 2018 22:19:46 +0200 Subject: [PATCH 02/35] BLD: adding changes done in order to add the option to call the instance (WIP) --- catalyst/__main__.py | 179 +++++++++++++++++++++++++++++++---- catalyst/utils/run_server.py | 40 ++++++++ 2 files changed, 202 insertions(+), 17 deletions(-) create mode 100644 catalyst/utils/run_server.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 036694869..529206239 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -14,6 +14,7 @@ from catalyst.exchange.utils.exchange_utils import delete_algo_folder from catalyst.utils.cli import Date, Timestamp from catalyst.utils.run_algo import _run, load_extensions +from catalyst.utils.run_server import run_server try: __IPYTHON__ @@ -505,6 +506,166 @@ def live(ctx, return perf +@main.command(name='serve-live') +@click.option( + '-f', + '--algofile', + default=None, + type=click.File('r'), + help='The file that contains the algorithm to run.', +) +@click.option( + '--capital-base', + type=float, + show_default=True, + help='The amount of capital (in base_currency) allocated to trading.', +) +@click.option( + '-t', + '--algotext', + help='The algorithm script to run.', +) +@click.option( + '-D', + '--define', + multiple=True, + help="Define a name to be bound in the namespace before executing" + " the algotext. For example '-Dname=value'. The value may be" + " any python expression. These are evaluated in order so they" + " may refer to previously defined names.", +) +@click.option( + '-o', + '--output', + default='-', + metavar='FILENAME', + show_default=True, + help="The location to write the perf data. If this is '-' the perf will" + " be written to stdout.", +) +@click.option( + '--print-algo/--no-print-algo', + is_flag=True, + default=False, + help='Print the algorithm to stdout.', +) +@ipython_only(click.option( + '--local-namespace/--no-local-namespace', + is_flag=True, + default=None, + help='Should the algorithm methods be resolved in the local namespace.' +)) +@click.option( + '-x', + '--exchange-name', + help='The name of the targeted exchange.', +) +@click.option( + '-n', + '--algo-namespace', + help='A label assigned to the algorithm for data storage purposes.' +) +@click.option( + '-c', + '--base-currency', + help='The base currency used to calculate statistics ' + '(e.g. usd, btc, eth).', +) +@click.option( + '-e', + '--end', + type=Date(tz='utc', as_timestamp=True), + help='An optional end date at which to stop the execution.', +) +@click.option( + '--live-graph/--no-live-graph', + is_flag=True, + default=False, + help='Display live graph.', +) +@click.option( + '--simulate-orders/--no-simulate-orders', + is_flag=True, + default=True, + help='Simulating orders enable the paper trading mode. No orders will be ' + 'sent to the exchange unless set to false.', +) +@click.option( + '--auth-aliases', + default=None, + help='Authentication file aliases for the specified exchanges. By default,' + 'each exchange uses the "auth.json" file in the exchange folder. ' + 'Specifying an "auth2" alias would use "auth2.json". It should be ' + 'specified like this: "[exchange_name],[alias],..." For example, ' + '"binance,auth2" or "binance,auth2,bittrex,auth2".', +) +@click.pass_context +def serve_live(ctx, + algofile, + capital_base, + algotext, + define, + output, + print_algo, + local_namespace, + exchange_name, + algo_namespace, + base_currency, + end, + live_graph, + auth_aliases, + simulate_orders): + """Trade live with the given algorithm on the server. + """ + if (algotext is not None) == (algofile is not None): + ctx.fail( + "must specify exactly one of '-f' / '--algofile' or" + " '-t' / '--algotext'", + ) + + if exchange_name is None: + ctx.fail("must specify an exchange name '-x'") + + if algo_namespace is None: + ctx.fail("must specify an algorithm name '-n' in live execution mode") + + if base_currency is None: + ctx.fail("must specify a base currency '-c' in live execution mode") + + if capital_base is None: + ctx.fail("must specify a capital base with '--capital-base'") + + if simulate_orders: + click.echo('Running in paper trading mode.', sys.stdout) + + else: + click.echo('Running in live trading mode.', sys.stdout) + + perf = run_server( + algofile=algofile, + algotext=algotext, + define=define, + capital_base=capital_base, + end=end, + output=output, + print_algo=print_algo, + live=False, + exchange=exchange_name, + algo_namespace=algo_namespace, + base_currency=base_currency, + live_graph=live_graph, + simulate_orders=simulate_orders, + auth_aliases=auth_aliases, + ) + + if output == '-': + click.echo(str(perf), sys.stdout) + elif output != os.devnull: # make the catalyst magic not write any data + perf.to_pickle(output) + + return perf + + @main.command(name='ingest-exchange') @click.option( '-x', @@ -767,18 +928,12 @@ def bundles(): @main.group() @click.pass_context def marketplace(ctx): - """Access the Enigma Data Marketplace to:\n - - Register and Publish new datasets (seller-side)\n - - Subscribe and Ingest premium datasets (buyer-side)\n - """ pass @marketplace.command() @click.pass_context def ls(ctx): - """List all available datasets. - """ click.echo('Listing of available data sources on the marketplace:', sys.stdout) marketplace = Marketplace() @@ -793,8 +948,6 @@ def ls(ctx): ) @click.pass_context def subscribe(ctx, dataset): - """Subscribe to an exisiting dataset. - """ if dataset is None: ctx.fail("must specify a dataset to subscribe to with '--dataset'\n" "List available dataset on the marketplace with " @@ -833,8 +986,6 @@ def subscribe(ctx, dataset): ) @click.pass_context def ingest(ctx, dataset, data_frequency, start, end): - """Ingest a dataset (requires subscription). - """ if dataset is None: ctx.fail("must specify a dataset to clean with '--dataset'\n" "List available dataset on the marketplace with " @@ -852,10 +1003,8 @@ def ingest(ctx, dataset, data_frequency, start, end): ) @click.pass_context def clean(ctx, dataset): - """Clean/Remove local data for a given dataset. - """ if dataset is None: - ctx.fail("must specify a dataset to clean up with '--dataset'\n" + ctx.fail("must specify a dataset to ingest with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) @@ -867,8 +1016,6 @@ def clean(ctx, dataset): @marketplace.command() @click.pass_context def register(ctx): - """Register a new dataset. - """ marketplace = Marketplace() marketplace.register() @@ -892,8 +1039,6 @@ def register(ctx): ) @click.pass_context def publish(ctx, dataset, datadir, watch): - """Publish data for a registered dataset. - """ marketplace = Marketplace() if dataset is None: ctx.fail("must specify a dataset to publish data for " diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py new file mode 100644 index 000000000..ff4d949d9 --- /dev/null +++ b/catalyst/utils/run_server.py @@ -0,0 +1,40 @@ +import requests +import base64 + + +def run_server( + algofile, + algotext, + define, + capital_base, + end, + output, + print_algo, + live, + exchange, + algo_namespace, + base_currency, + live_graph, + simulate_orders, + auth_aliases, + ): + + json_file = {'arguments': { + '--algofile': base64.b64encode(algofile.read()), + '--algotext': algotext, + '--define': define, + '--capital-base': capital_base, + '--end': end, + '--output': output, + 'print-algo': print_algo, + 'live': True, + '--exchange': exchange, + '--algo-namespace': algo_namespace, + '--base-currency': base_currency, + 'live-graph': live_graph, + 'simulate-orders': simulate_orders, + '--auth-aliases': auth_aliases, + }} + + url = 'http://127.0.0.1:5000/todo/api/v1.0/tasks' + response = requests.post(url, json=json_file) From b172f6b7b019ca12b891414bed2814f0f8addd72 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 6 Mar 2018 16:02:02 +0200 Subject: [PATCH 03/35] DEV: able to run locally through api- added backtesting as well (WIP) --- catalyst/__main__.py | 210 ++++++++++++++++++++++++++++++++++- catalyst/utils/run_server.py | 179 +++++++++++++++++++++++++---- 2 files changed, 367 insertions(+), 22 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 529206239..874cf4aa9 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -506,6 +506,197 @@ def live(ctx, return perf +@main.command(name='serve') +@click.option( + '-f', + '--algofile', + default=None, + type=click.File('r'), + help='The file that contains the algorithm to run.', +) +@click.option( + '-t', + '--algotext', + help='The algorithm script to run.', +) +@click.option( + '-D', + '--define', + multiple=True, + help="Define a name to be bound in the namespace before executing" + " the algotext. For example '-Dname=value'. The value may be" + " any python expression. These are evaluated in order so they" + " may refer to previously defined names.", +) +@click.option( + '--data-frequency', + type=click.Choice({'daily', 'minute'}), + default='daily', + show_default=True, + help='The data frequency of the simulation.', +) +@click.option( + '--capital-base', + type=float, + show_default=True, + help='The starting capital for the simulation.', +) +@click.option( + '-b', + '--bundle', + default='poloniex', + metavar='BUNDLE-NAME', + show_default=True, + help='The data bundle to use for the simulation.', +) +@click.option( + '--bundle-timestamp', + type=Timestamp(), + default=pd.Timestamp.utcnow(), + show_default=False, + help='The date to lookup data on or before.\n' + '[default: ]' +) +@click.option( + '-s', + '--start', + type=Date(tz='utc', as_timestamp=True), + help='The start date of the simulation.', +) +@click.option( + '-e', + '--end', + type=Date(tz='utc', as_timestamp=True), + help='The end date of the simulation.', +) +@click.option( + '-o', + '--output', + default='-', + metavar='FILENAME', + show_default=True, + help="The location to write the perf data. If this is '-' the perf" + " will be written to stdout.", +) +@click.option( + '--print-algo/--no-print-algo', + is_flag=True, + default=False, + help='Print the algorithm to stdout.', +) +@ipython_only(click.option( + '--local-namespace/--no-local-namespace', + is_flag=True, + default=None, + help='Should the algorithm methods be resolved in the local namespace.' +)) +@click.option( + '-x', + '--exchange-name', + help='The name of the targeted exchange.', +) +@click.option( + '-n', + '--algo-namespace', + help='A label assigned to the algorithm for data storage purposes.' +) +@click.option( + '-c', + '--base-currency', + help='The base currency used to calculate statistics ' + '(e.g. usd, btc, eth).', +) +@click.pass_context +def run(ctx, + algofile, + algotext, + define, + data_frequency, + capital_base, + bundle, + bundle_timestamp, + start, + end, + output, + print_algo, + local_namespace, + exchange_name, + algo_namespace, + base_currency): + """Run a backtest for the given algorithm on the server. + """ + + if (algotext is not None) == (algofile is not None): + ctx.fail( + "must specify exactly one of '-f' / '--algofile' or" + " '-t' / '--algotext'", + ) + + # check that the start and end dates are passed correctly + if start is None and end is None: + # check both at the same time to avoid the case where a user + # does not pass either of these and then passes the first only + # to be told they need to pass the second argument also + ctx.fail( + "must specify dates with '-s' / '--start' and '-e' / '--end'" + " in backtest mode", + ) + if start is None: + ctx.fail("must specify a start date with '-s' / '--start'" + " in backtest mode") + if end is None: + ctx.fail("must specify an end date with '-e' / '--end'" + " in backtest mode") + + if exchange_name is None: + ctx.fail("must specify an exchange name '-x'") + + if base_currency is None: + ctx.fail("must specify a base currency with '-c' in backtest mode") + + if capital_base is None: + ctx.fail("must specify a capital base with '--capital-base'") + + click.echo('Running in backtesting mode.', sys.stdout) + + perf = run_server( + initialize=None, + handle_data=None, + before_trading_start=None, + analyze=None, + algofile=algofile, + algotext=algotext, + defines=define, + data_frequency=data_frequency, + capital_base=capital_base, + data=None, + bundle=bundle, + bundle_timestamp=bundle_timestamp, + start=start, + end=end, + output=output, + print_algo=print_algo, + local_namespace=local_namespace, + environ=os.environ, + live=False, + exchange=exchange_name, + algo_namespace=algo_namespace, + base_currency=base_currency, + analyze_live=None, + live_graph=False, + simulate_orders=True, + auth_aliases=None, + stats_output=None, + ) + + if output == '-': + click.echo(str(perf), sys.stdout) + elif output != os.devnull: # make the catalyst magic not write any data + perf.to_pickle(output) + + return perf + + @main.command(name='serve-live') @click.option( '-f', @@ -642,20 +833,33 @@ def serve_live(ctx, click.echo('Running in live trading mode.', sys.stdout) perf = run_server( - algofile=algofile, + initialize=None, + handle_data=None, + before_trading_start=None, + analyze=None, + algofile=algofile, algotext=algotext, - define=define, + defines=define, + data_frequency=None, capital_base=capital_base, + data=None, + bundle=None, + bundle_timestamp=None, + start=None, end=end, output=output, print_algo=print_algo, - live=False, + local_namespace=local_namespace, + environ=os.environ, + live=True, exchange=exchange_name, algo_namespace=algo_namespace, base_currency=base_currency, live_graph=live_graph, + analyze_live=None, simulate_orders=simulate_orders, auth_aliases=auth_aliases, + stats_output=None, ) if output == '-': diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index ff4d949d9..413128e86 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -1,40 +1,181 @@ -import requests +#!flask/bin/python import base64 +import requests +import pandas as pd +import json +from flask import ( + Flask, jsonify, abort, + make_response, request, + url_for, + ) + +from catalyst.utils.run_algo import _run + + +def convert_date(date): + if isinstance(date, pd.Timestamp): + return date.__str__() + def run_server( + initialize, + handle_data, + before_trading_start, + analyze, algofile, algotext, - define, + defines, + data_frequency, capital_base, + data, + bundle, + bundle_timestamp, + start, end, output, print_algo, + local_namespace, + environ, live, exchange, algo_namespace, base_currency, live_graph, + analyze_live, simulate_orders, auth_aliases, - ): + stats_output, + ): + + if algotext: + algotext = base64.b64encode(algotext) + else: + algotext = base64.b64encode(algofile.read()) + algofile = None json_file = {'arguments': { - '--algofile': base64.b64encode(algofile.read()), - '--algotext': algotext, - '--define': define, - '--capital-base': capital_base, - '--end': end, - '--output': output, - 'print-algo': print_algo, - 'live': True, - '--exchange': exchange, - '--algo-namespace': algo_namespace, - '--base-currency': base_currency, - 'live-graph': live_graph, - 'simulate-orders': simulate_orders, - '--auth-aliases': auth_aliases, + 'initialize': initialize, + 'handle_data': handle_data, + 'before_trading_start': before_trading_start, + 'analyze': analyze, + 'algotext': algotext, + 'defines': defines, + 'data_frequency': data_frequency, + 'capital_base': capital_base, + 'data': data, + 'bundle': bundle, + 'bundle_timestamp': bundle_timestamp, + 'start': start, + 'end': end, + 'local_namespace': local_namespace, + 'environ': None, + 'analyze_live': analyze_live, + 'stats_output': stats_output, + 'algofile': algofile, + 'output': output, + 'print_algo': print_algo, + 'live': live, + 'exchange': exchange, + 'algo_namespace': algo_namespace, + 'base_currency': base_currency, + 'live_graph': live_graph, + 'simulate_orders': simulate_orders, + 'auth_aliases': auth_aliases, }} - url = 'http://127.0.0.1:5000/todo/api/v1.0/tasks' - response = requests.post(url, json=json_file) + url = 'http://127.0.0.1:5000/api/v1.0/tasks' + response = requests.post(url, json=json.dumps( + json_file, default=convert_date)) + + +app = Flask(__name__) + + +def exec_catalyst(arguments): + arguments['algotext'] = base64.b64decode(arguments['algotext']) + if arguments['start'] is not None: + arguments['start'] = pd.Timestamp(arguments['start']) + if arguments['end'] is not None: + arguments['end'] = pd.Timestamp(arguments['end']) + _run(**arguments) + + +def make_public_task(task): + new_task = {} + for field in task: + if field == 'id': + new_task['uri'] = url_for('get_task', task_id=task['id'], _external=True) + else: + new_task[field] = task[field] + return new_task + + +# @app.route('/') +# def index(): +# return "Hello, World!" + + +# @app.route('/api/v1.0/tasks/', methods=['GET']) +# def get_task(task_id): +# task = [task for task in tasks if task['id'] == task_id] +# if len(task) == 0: +# abort(404) +# return jsonify({'task': make_public_task(task[0])}) +# +# +# @app.route('/api/v1.0/tasks', methods=['GET']) +# def get_tasks(): +# return jsonify({'tasks': [make_public_task(task) for task in tasks]}) + + +@app.errorhandler(404) +def not_found(error): + return make_response(jsonify({'error': 'Not found'}), 400) + + +@app.route('/api/v1.0/tasks', methods=['POST']) +def create_task(): + if not request.json or 'arguments' not in request.json: + abort(400) + arguments = json.loads(request.json)['arguments'] + exec_catalyst(arguments) + # task = { + # 'id': tasks[-1]['id'] + 1, + # 'arguments': request.json['arguments'], + # 'done': False + # } + # tasks.append(task) + # + # return jsonify({'task': [make_public_task(task)]}), 201 + return jsonify({"success": "man"}), 201 + +# @app.route('/api/v1.0/tasks/', methods=['PUT']) +# def update_task(task_id): +# task = [task for task in tasks if task['id'] == task_id] +# if len(task) == 0: +# abort(404) +# if not request.json: +# abort(400) +# if 'arguments' in request.json and type(request.json['arguments']) != unicode: +# abort(400) +# if 'description' in request.json and type(request.json['description']) is not unicode: +# abort(400) +# if 'done' in request.json and type(request.json['done']) is not bool: +# abort(400) +# task[0]['arguments'] = request.json.get('arguments', task[0]['arguments']) +# task[0]['done'] = request.json.get('done', task[0]['done']) +# return jsonify({'task': [make_public_task(task[0])]}) +# +# +# @app.route('/api/v1.0/tasks/', methods=['DELETE']) +# def delete_task(task_id): +# task = [task for task in tasks if task['id'] == task_id] +# if len(task) == 0: +# abort(404) +# tasks.remove(task[0]) +# return jsonify({'result': True}) + + +if __name__ == '__main__': + app.run(debug=True, threaded=True) From e4933ee48ef69d9483c4cb9631c4b29650396549 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 7 Mar 2018 09:48:56 +0200 Subject: [PATCH 04/35] DEV: split client and server- added some docs (WIP) --- catalyst/utils/run_server.py | 118 +++++------------------------------ 1 file changed, 16 insertions(+), 102 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 413128e86..400bab736 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -4,16 +4,15 @@ import requests import pandas as pd import json -from flask import ( - Flask, jsonify, abort, - make_response, request, - url_for, - ) - -from catalyst.utils.run_algo import _run def convert_date(date): + """ + when transferring dates by json, + converts it to str + :param date: + :return: str(date) + """ if isinstance(date, pd.Timestamp): return date.__str__() @@ -48,6 +47,10 @@ def run_server( stats_output, ): + # address to send + url = 'http://127.0.0.1:5000/api/catalyst/serve' + + # argument preparation - encode the file for transfer if algotext: algotext = base64.b64encode(algotext) else: @@ -84,98 +87,9 @@ def run_server( 'auth_aliases': auth_aliases, }} - url = 'http://127.0.0.1:5000/api/v1.0/tasks' - response = requests.post(url, json=json.dumps( - json_file, default=convert_date)) - - -app = Flask(__name__) - - -def exec_catalyst(arguments): - arguments['algotext'] = base64.b64decode(arguments['algotext']) - if arguments['start'] is not None: - arguments['start'] = pd.Timestamp(arguments['start']) - if arguments['end'] is not None: - arguments['end'] = pd.Timestamp(arguments['end']) - _run(**arguments) - - -def make_public_task(task): - new_task = {} - for field in task: - if field == 'id': - new_task['uri'] = url_for('get_task', task_id=task['id'], _external=True) - else: - new_task[field] = task[field] - return new_task - - -# @app.route('/') -# def index(): -# return "Hello, World!" - - -# @app.route('/api/v1.0/tasks/', methods=['GET']) -# def get_task(task_id): -# task = [task for task in tasks if task['id'] == task_id] -# if len(task) == 0: -# abort(404) -# return jsonify({'task': make_public_task(task[0])}) -# -# -# @app.route('/api/v1.0/tasks', methods=['GET']) -# def get_tasks(): -# return jsonify({'tasks': [make_public_task(task) for task in tasks]}) - - -@app.errorhandler(404) -def not_found(error): - return make_response(jsonify({'error': 'Not found'}), 400) - - -@app.route('/api/v1.0/tasks', methods=['POST']) -def create_task(): - if not request.json or 'arguments' not in request.json: - abort(400) - arguments = json.loads(request.json)['arguments'] - exec_catalyst(arguments) - # task = { - # 'id': tasks[-1]['id'] + 1, - # 'arguments': request.json['arguments'], - # 'done': False - # } - # tasks.append(task) - # - # return jsonify({'task': [make_public_task(task)]}), 201 - return jsonify({"success": "man"}), 201 - -# @app.route('/api/v1.0/tasks/', methods=['PUT']) -# def update_task(task_id): -# task = [task for task in tasks if task['id'] == task_id] -# if len(task) == 0: -# abort(404) -# if not request.json: -# abort(400) -# if 'arguments' in request.json and type(request.json['arguments']) != unicode: -# abort(400) -# if 'description' in request.json and type(request.json['description']) is not unicode: -# abort(400) -# if 'done' in request.json and type(request.json['done']) is not bool: -# abort(400) -# task[0]['arguments'] = request.json.get('arguments', task[0]['arguments']) -# task[0]['done'] = request.json.get('done', task[0]['done']) -# return jsonify({'task': [make_public_task(task[0])]}) -# -# -# @app.route('/api/v1.0/tasks/', methods=['DELETE']) -# def delete_task(task_id): -# task = [task for task in tasks if task['id'] == task_id] -# if len(task) == 0: -# abort(404) -# tasks.remove(task[0]) -# return jsonify({'result': True}) - - -if __name__ == '__main__': - app.run(debug=True, threaded=True) + response = requests.post(url, + json=json.dumps( + json_file, + default=convert_date + ) + ) From 8a812ea455f2380d34b7b05b456bce9123ff4a19 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Thu, 8 Mar 2018 17:08:10 +0200 Subject: [PATCH 05/35] DEV: modified to the instance url (WIP) --- catalyst/utils/run_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 400bab736..571e77886 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -48,7 +48,7 @@ def run_server( ): # address to send - url = 'http://127.0.0.1:5000/api/catalyst/serve' + url = 'http://sandbox.enigma.co/api/catalyst/serve' # argument preparation - encode the file for transfer if algotext: From 4712db4f4fa367d6011f16e434d0c9d46e417bef Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 12 Mar 2018 10:53:53 +0200 Subject: [PATCH 06/35] DEV: receiving a log from the cloud (WIP) --- catalyst/utils/run_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 571e77886..7281a5af3 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -93,3 +93,9 @@ def run_server( default=convert_date ) ) + if response.status_code == 500: + raise Exception("issues with cloud connections, " + "unable to run catalyst on the cloud") + recieved_data = response.json() + cloud_log_tail = base64.b64decode(recieved_data["logs"]) + print(cloud_log_tail) From d0abc8eac08019e0d7efd2884a3765f7f47678a3 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 14 Mar 2018 09:40:45 +0200 Subject: [PATCH 07/35] DEV: minor change in order to get log tail from server (WIP) --- catalyst/utils/run_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 7281a5af3..43b782e7e 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -49,6 +49,7 @@ def run_server( # address to send url = 'http://sandbox.enigma.co/api/catalyst/serve' + # url = 'http://127.0.0.1:5000/api/catalyst/serve' # argument preparation - encode the file for transfer if algotext: @@ -93,9 +94,10 @@ def run_server( default=convert_date ) ) + if response.status_code == 500: raise Exception("issues with cloud connections, " "unable to run catalyst on the cloud") recieved_data = response.json() - cloud_log_tail = base64.b64decode(recieved_data["logs"]) + cloud_log_tail = base64.b64decode(recieved_data["log"]) print(cloud_log_tail) From ca76d7271f23311606a7ecffa5d048aa2292532a Mon Sep 17 00:00:00 2001 From: avishaiw Date: Wed, 21 Mar 2018 13:06:14 +0200 Subject: [PATCH 08/35] DEV: encoding compatible to unix as well --- catalyst/utils/run_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 43b782e7e..a94dd6355 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -55,7 +55,7 @@ def run_server( if algotext: algotext = base64.b64encode(algotext) else: - algotext = base64.b64encode(algofile.read()) + algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8') algofile = None json_file = {'arguments': { @@ -98,6 +98,6 @@ def run_server( if response.status_code == 500: raise Exception("issues with cloud connections, " "unable to run catalyst on the cloud") - recieved_data = response.json() - cloud_log_tail = base64.b64decode(recieved_data["log"]) + received_data = response.json() + cloud_log_tail = base64.b64decode(received_data["log"]) print(cloud_log_tail) From 9ed940549c5c2c64a2b93603281a87aaa44431f1 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Fri, 30 Mar 2018 16:59:19 +0300 Subject: [PATCH 09/35] DEV: added interactive logging and receiving results from server (WIP) --- catalyst/__main__.py | 4 ++-- catalyst/exchange/exchange_algorithm.py | 27 ++++++++++++++++++++++++- catalyst/utils/cli.py | 2 +- catalyst/utils/run_server.py | 19 +++++++++++++++-- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 874cf4aa9..aad85bdd8 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -401,7 +401,7 @@ def catalyst_magic(line, cell=None): @click.option( '-e', '--end', - type=Date(tz='utc', as_timestamp=True), + type=Date(tz='utc', as_timestamp=False), help='An optional end date at which to stop the execution.', ) @click.option( @@ -765,7 +765,7 @@ def run(ctx, @click.option( '-e', '--end', - type=Date(tz='utc', as_timestamp=True), + type=Date(tz='utc', as_timestamp=False), help='An optional end date at which to stop the execution.', ) @click.option( diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index b9c319f19..afda0ffcd 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -163,6 +163,25 @@ def _calculate_order(self, asset, amount, style) return amount, style + def _calculate_order_target_amount(self, asset, target): + """ + removes order amounts so we won't run into issues + when two orders are placed one after the other. + it then proceeds to removing positions amount at TradingAlgorithm + :param asset: + :param target: + :return: target + """ + if asset in self.blotter.open_orders: + for open_order in self.blotter.open_orders[asset]: + current_amount = open_order.amount + target -= current_amount + + target = super(ExchangeTradingAlgorithmBase, self). \ + _calculate_order_target_amount(asset, target) + + return target + def round_order(self, amount, asset): """ We need fractions with cryptocurrencies @@ -423,6 +442,12 @@ def interrupt_algorithm(self): ------- """ + save_algo_object( + algo_name=self.algo_namespace, + key=self.datetime.floor('1D').strftime('%Y-%m-%d'), + obj=self.frame_stats, + rel_path='frame_stats' + ) self.is_running = False if self._analyze is None: @@ -454,7 +479,7 @@ def interrupt_algorithm(self): stats = pd.DataFrame(period_stats_list) stats.set_index('period_close', drop=False, inplace=True) - stats = pd.concat([stats, current_stats]) + # stats = pd.concat([stats, current_stats]) else: stats = current_stats diff --git a/catalyst/utils/cli.py b/catalyst/utils/cli.py index 59a7300a3..781eb6558 100644 --- a/catalyst/utils/cli.py +++ b/catalyst/utils/cli.py @@ -116,7 +116,7 @@ def __init__(self, tz=None, as_timestamp=False): def parser(self, value): ts = super(Date, self).parser(value) - return ts.normalize() if self.as_timestamp else ts.date() + return ts.normalize() if self.as_timestamp else ts class Time(_DatetimeParam): diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index a94dd6355..9d140dfe8 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -4,6 +4,17 @@ import requests import pandas as pd import json +import zlib +import pickle +import zmq +from logbook.queues import ZeroMQSubscriber +from logbook import FileHandler, StreamHandler,StderrHandler + + +# my_handler = FileHandler('/Users/avishaiw/Documents/test.txt') +my_handler = StderrHandler() +subscriber = ZeroMQSubscriber('tcp://127.0.0.1:1056', multi=True) +controller = subscriber.dispatch_in_background(my_handler) def convert_date(date): @@ -98,6 +109,10 @@ def run_server( if response.status_code == 500: raise Exception("issues with cloud connections, " "unable to run catalyst on the cloud") + controller.stop() received_data = response.json() - cloud_log_tail = base64.b64decode(received_data["log"]) - print(cloud_log_tail) + data_perf_compressed = base64.b64decode(received_data["data"]) + data_perf_pickled = zlib.decompress(data_perf_compressed) + data_perf = pickle.loads(data_perf_pickled) + # cloud_log_tail = base64.b64decode(received_data["log"]) + # print(cloud_log_tail) From 9ae3c83a4e78d05417aae53135b8f18919494679 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 4 Apr 2018 15:53:24 +0300 Subject: [PATCH 10/35] DEV: interactive logging executing successfully. added docs to the code as well --- catalyst/utils/run_server.py | 60 +++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 9d140dfe8..9869b696e 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -6,24 +6,45 @@ import json import zlib import pickle -import zmq + from logbook.queues import ZeroMQSubscriber -from logbook import FileHandler, StreamHandler,StderrHandler +from logbook import StderrHandler -# my_handler = FileHandler('/Users/avishaiw/Documents/test.txt') +# adding the handlers which receive the records from the server my_handler = StderrHandler() -subscriber = ZeroMQSubscriber('tcp://127.0.0.1:1056', multi=True) +subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5050', multi=True) controller = subscriber.dispatch_in_background(my_handler) +def prepare_args(file, text): + """ + send the algo as a base64 decoded text object + + :param file: File + :param text: str + :return: None, text: str + """ + + if text: + text = base64.b64encode(text) + else: + text = base64.b64encode(bytes(file.read(), 'utf-8')).decode('utf-8') + file = None + return file, text + + def convert_date(date): """ when transferring dates by json, converts it to str + # any instances which need a conversion, + # must be done here + :param date: :return: str(date) """ + if isinstance(date, pd.Timestamp): return date.__str__() @@ -59,15 +80,11 @@ def run_server( ): # address to send - url = 'http://sandbox.enigma.co/api/catalyst/serve' - # url = 'http://127.0.0.1:5000/api/catalyst/serve' + # url = 'http://sandbox.enigma.co/api/catalyst/serve' + url = 'http://127.0.0.1:5000/api/catalyst/serve' # argument preparation - encode the file for transfer - if algotext: - algotext = base64.b64encode(algotext) - else: - algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8') - algofile = None + algofile, algotext = prepare_args(algofile, algotext) json_file = {'arguments': { 'initialize': initialize, @@ -99,20 +116,27 @@ def run_server( 'auth_aliases': auth_aliases, }} + # call the server with the following arguments + # if any issues raised related to the format of the dates, convert them response = requests.post(url, json=json.dumps( json_file, default=convert_date ) ) + # close the handlers, which are not needed anymore + controller.stop() + subscriber.close() if response.status_code == 500: raise Exception("issues with cloud connections, " "unable to run catalyst on the cloud") - controller.stop() - received_data = response.json() - data_perf_compressed = base64.b64decode(received_data["data"]) - data_perf_pickled = zlib.decompress(data_perf_compressed) - data_perf = pickle.loads(data_perf_pickled) - # cloud_log_tail = base64.b64decode(received_data["log"]) - # print(cloud_log_tail) + + elif response.status_code == 202: + print(response.json()['error']) + + else: # if the run was successful + received_data = response.json() + data_perf_compressed = base64.b64decode(received_data["data"]) + data_perf_pickled = zlib.decompress(data_perf_compressed) + data_perf = pickle.loads(data_perf_pickled) From cee9dd36641b325de2d4276f5fb81dc1d344cef1 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 17 Jun 2018 11:34:14 +0300 Subject: [PATCH 11/35] DEV: added changes of quote_currency in cloud code --- catalyst/__main__.py | 26 +++++++++++++------------- catalyst/utils/run_server.py | 11 ++++++----- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 449b11f27..8a0cf22dc 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -611,8 +611,8 @@ def live(ctx, ) @click.option( '-c', - '--base-currency', - help='The base currency used to calculate statistics ' + '--quote-currency', + help='The quote currency used to calculate statistics ' '(e.g. usd, btc, eth).', ) @click.pass_context @@ -631,7 +631,7 @@ def run(ctx, local_namespace, exchange_name, algo_namespace, - base_currency): + quote_currency): """Run a backtest for the given algorithm on the server. """ @@ -660,8 +660,8 @@ def run(ctx, if exchange_name is None: ctx.fail("must specify an exchange name '-x'") - if base_currency is None: - ctx.fail("must specify a base currency with '-c' in backtest mode") + if quote_currency is None: + ctx.fail("must specify a quote currency with '-c' in backtest mode") if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") @@ -690,7 +690,7 @@ def run(ctx, live=False, exchange=exchange_name, algo_namespace=algo_namespace, - base_currency=base_currency, + quote_currency=quote_currency, analyze_live=None, live_graph=False, simulate_orders=True, @@ -718,7 +718,7 @@ def run(ctx, '--capital-base', type=float, show_default=True, - help='The amount of capital (in base_currency) allocated to trading.', + help='The amount of capital (in quote_currency) allocated to trading.', ) @click.option( '-t', @@ -767,8 +767,8 @@ def run(ctx, ) @click.option( '-c', - '--base-currency', - help='The base currency used to calculate statistics ' + '--quote-currency', + help='The quote currency used to calculate statistics ' '(e.g. usd, btc, eth).', ) @click.option( @@ -810,7 +810,7 @@ def serve_live(ctx, local_namespace, exchange_name, algo_namespace, - base_currency, + quote_currency, end, live_graph, auth_aliases, @@ -829,8 +829,8 @@ def serve_live(ctx, if algo_namespace is None: ctx.fail("must specify an algorithm name '-n' in live execution mode") - if base_currency is None: - ctx.fail("must specify a base currency '-c' in live execution mode") + if quote_currency is None: + ctx.fail("must specify a quote currency '-c' in live execution mode") if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") @@ -863,7 +863,7 @@ def serve_live(ctx, live=True, exchange=exchange_name, algo_namespace=algo_namespace, - base_currency=base_currency, + quote_currency=quote_currency, live_graph=live_graph, analyze_live=None, simulate_orders=simulate_orders, diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 9869b696e..97dc534bf 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -13,7 +13,8 @@ # adding the handlers which receive the records from the server my_handler = StderrHandler() -subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5050', multi=True) +subscriber = ZeroMQSubscriber("tcp://35.170.19.246:5050") +# subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5050', multi=True) controller = subscriber.dispatch_in_background(my_handler) @@ -71,7 +72,7 @@ def run_server( live, exchange, algo_namespace, - base_currency, + quote_currency, live_graph, analyze_live, simulate_orders, @@ -80,8 +81,8 @@ def run_server( ): # address to send - # url = 'http://sandbox.enigma.co/api/catalyst/serve' - url = 'http://127.0.0.1:5000/api/catalyst/serve' + url = 'http://sandbox.enigma.co/api/catalyst/serve' + # url = 'http://127.0.0.1:5000/api/catalyst/serve' # argument preparation - encode the file for transfer algofile, algotext = prepare_args(algofile, algotext) @@ -110,7 +111,7 @@ def run_server( 'live': live, 'exchange': exchange, 'algo_namespace': algo_namespace, - 'base_currency': base_currency, + 'quote_currency': quote_currency, 'live_graph': live_graph, 'simulate_orders': simulate_orders, 'auth_aliases': auth_aliases, From 7b7b3784473621eb1a2f64702536545269860584 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 2 Jul 2018 12:06:00 +0300 Subject: [PATCH 12/35] DEV: added https in communicating to the server --- catalyst/utils/run_server.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 97dc534bf..1d2eaab83 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -13,10 +13,10 @@ # adding the handlers which receive the records from the server my_handler = StderrHandler() -subscriber = ZeroMQSubscriber("tcp://35.170.19.246:5050") +subscriber = ZeroMQSubscriber(uri="tcp://34.202.72.107:5050") # subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5050', multi=True) controller = subscriber.dispatch_in_background(my_handler) - +# subscriber.dispatch_forever() def prepare_args(file, text): """ @@ -81,7 +81,7 @@ def run_server( ): # address to send - url = 'http://sandbox.enigma.co/api/catalyst/serve' + url = 'https://sandbox2.enigma.co/api/catalyst/serve' # url = 'http://127.0.0.1:5000/api/catalyst/serve' # argument preparation - encode the file for transfer @@ -123,16 +123,20 @@ def run_server( json=json.dumps( json_file, default=convert_date - ) + ), + verify=False ) # close the handlers, which are not needed anymore - controller.stop() + # controller.stop() subscriber.close() if response.status_code == 500: raise Exception("issues with cloud connections, " "unable to run catalyst on the cloud") - + elif response.status_code == 502: + raise Exception("The server is down at the moment, please contact " + "Catalyst support to fix this issue at " + "https://github.com/enigmampc/catalyst/issues/") elif response.status_code == 202: print(response.json()['error']) From 0ddc6c4068ff725257986f38fd08308a79fea012 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 15 Jul 2018 12:24:22 +0300 Subject: [PATCH 13/35] BLD: logging from server to client works properly with zmqHandler --- catalyst/__main__.py | 5 +++-- catalyst/utils/run_server.py | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 8a0cf22dc..0d43ea9e1 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -581,7 +581,6 @@ def live(ctx, @click.option( '-o', '--output', - default='-', metavar='FILENAME', show_default=True, help="The location to write the perf data. If this is '-' the perf" @@ -683,7 +682,7 @@ def run(ctx, bundle_timestamp=bundle_timestamp, start=start, end=end, - output=output, + output='-', print_algo=print_algo, local_namespace=local_namespace, environ=os.environ, @@ -700,6 +699,8 @@ def run(ctx, if output == '-': click.echo(str(perf), sys.stdout) + elif output is None: + pass elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 1d2eaab83..d17f686d6 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -6,17 +6,12 @@ import json import zlib import pickle - from logbook.queues import ZeroMQSubscriber -from logbook import StderrHandler - -# adding the handlers which receive the records from the server -my_handler = StderrHandler() subscriber = ZeroMQSubscriber(uri="tcp://34.202.72.107:5050") -# subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5050', multi=True) -controller = subscriber.dispatch_in_background(my_handler) -# subscriber.dispatch_forever() +# subscriber = ZeroMQSubscriber(uri="tcp://127.0.0.1:5050") +controller = subscriber.dispatch_in_background() + def prepare_args(file, text): """ @@ -81,7 +76,7 @@ def run_server( ): # address to send - url = 'https://sandbox2.enigma.co/api/catalyst/serve' + url = 'http://34.202.72.107:5000/api/catalyst/serve' # url = 'http://127.0.0.1:5000/api/catalyst/serve' # argument preparation - encode the file for transfer @@ -124,10 +119,11 @@ def run_server( json_file, default=convert_date ), - verify=False + verify=False, ) + # close the handlers, which are not needed anymore - # controller.stop() + controller.stop() subscriber.close() if response.status_code == 500: @@ -137,11 +133,12 @@ def run_server( raise Exception("The server is down at the moment, please contact " "Catalyst support to fix this issue at " "https://github.com/enigmampc/catalyst/issues/") - elif response.status_code == 202: - print(response.json()['error']) + elif response.status_code == 202 or response.status_code == 400: + print(response.json()['error']) if response.json()['error'] else None else: # if the run was successful received_data = response.json() data_perf_compressed = base64.b64decode(received_data["data"]) data_perf_pickled = zlib.decompress(data_perf_compressed) data_perf = pickle.loads(data_perf_pickled) + print(data_perf) From 51c97ddd7928251a557ecab993d3a33f17ecf691 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 22 Jul 2018 19:14:57 +0300 Subject: [PATCH 14/35] BLD: Added perf handling --- catalyst/__main__.py | 3 +- catalyst/utils/run_server.py | 53 ++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 0d43ea9e1..be95aa8af 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -581,6 +581,7 @@ def live(ctx, @click.option( '-o', '--output', + default='-', metavar='FILENAME', show_default=True, help="The location to write the perf data. If this is '-' the perf" @@ -699,8 +700,6 @@ def run(ctx, if output == '-': click.echo(str(perf), sys.stdout) - elif output is None: - pass elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index d17f686d6..941e755b4 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -45,6 +45,40 @@ def convert_date(date): return date.__str__() +def handle_response(response): + """ + handles the response given by the server according to it's status code + + :param response: the format returned from a request + :return: DataFrame/ str + """ + if response.status_code == 500: + raise Exception("issues with cloud connections, " + "unable to run catalyst on the cloud") + elif response.status_code == 502: + raise Exception("The server is down at the moment, please contact " + "Catalyst support to fix this issue at " + "https://github.com/enigmampc/catalyst/issues/") + elif response.status_code == 202 or response.status_code == 400: + return response.json()['error'] if response.json()['error'] else None + + else: # if the run was successful + return json_to_df(response.json()) + + +def json_to_df(json): + """ + converts the data returned from the algorithm run from base64 to DF + + :param json: the response in a json format + :return: data_perf: the data in a DataFrame format + """ + data_perf_compressed = base64.b64decode(json["data"]) + data_perf_pickled = zlib.decompress(data_perf_compressed) + data_perf = pickle.loads(data_perf_pickled) + return data_perf + + def run_server( initialize, handle_data, @@ -76,7 +110,7 @@ def run_server( ): # address to send - url = 'http://34.202.72.107:5000/api/catalyst/serve' + url = 'https://34.202.72.107/api/catalyst/serve' # url = 'http://127.0.0.1:5000/api/catalyst/serve' # argument preparation - encode the file for transfer @@ -126,19 +160,4 @@ def run_server( controller.stop() subscriber.close() - if response.status_code == 500: - raise Exception("issues with cloud connections, " - "unable to run catalyst on the cloud") - elif response.status_code == 502: - raise Exception("The server is down at the moment, please contact " - "Catalyst support to fix this issue at " - "https://github.com/enigmampc/catalyst/issues/") - elif response.status_code == 202 or response.status_code == 400: - print(response.json()['error']) if response.json()['error'] else None - - else: # if the run was successful - received_data = response.json() - data_perf_compressed = base64.b64decode(received_data["data"]) - data_perf_pickled = zlib.decompress(data_perf_compressed) - data_perf = pickle.loads(data_perf_pickled) - print(data_perf) + return handle_response(response) From 027c606f086b7623dba546b72642f88a7219c79a Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 25 Jul 2018 17:55:23 +0300 Subject: [PATCH 15/35] BLD: remove the analyze method in case it exists --- catalyst/utils/run_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 941e755b4..62e877a58 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -107,14 +107,14 @@ def run_server( simulate_orders, auth_aliases, stats_output, - ): - +): # address to send url = 'https://34.202.72.107/api/catalyst/serve' # url = 'http://127.0.0.1:5000/api/catalyst/serve' - # argument preparation - encode the file for transfer - algofile, algotext = prepare_args(algofile, algotext) + if algotext or algofile: + # argument preparation - encode the file for transfer + algofile, algotext = prepare_args(algofile, algotext) json_file = {'arguments': { 'initialize': initialize, From 2a85d562a1b3ebda321eb53fb35019c3d37a5baf Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 29 Jul 2018 19:52:15 +0300 Subject: [PATCH 16/35] BLD: added DigestAuth and modified the logging handler (WIP) --- catalyst/utils/run_server.py | 80 ++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 62e877a58..ab38f817f 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -1,16 +1,16 @@ #!flask/bin/python import base64 - -import requests -import pandas as pd import json import zlib import pickle -from logbook.queues import ZeroMQSubscriber +import requests -subscriber = ZeroMQSubscriber(uri="tcp://34.202.72.107:5050") -# subscriber = ZeroMQSubscriber(uri="tcp://127.0.0.1:5050") -controller = subscriber.dispatch_in_background() +import pandas as pd +from hashlib import md5 + +AUTH_SERVER = 'http://localhost:5000' +key = '823fu8d4g' +secret = 'iu4f9f4iou3hf3498hf' def prepare_args(file, text): @@ -60,23 +60,23 @@ def handle_response(response): "Catalyst support to fix this issue at " "https://github.com/enigmampc/catalyst/issues/") elif response.status_code == 202 or response.status_code == 400: + lf = json_to_file(response.json()['logs']) + print(lf.decode('utf-8')) return response.json()['error'] if response.json()['error'] else None else: # if the run was successful - return json_to_df(response.json()) + return load_response(response.json()) -def json_to_df(json): - """ - converts the data returned from the algorithm run from base64 to DF +def load_response(json): + lf = json_to_file(json['logs']) + print(lf.decode('utf-8')) + data_df = pickle.loads(json_to_file(json['data'])) + return data_df - :param json: the response in a json format - :return: data_perf: the data in a DataFrame format - """ - data_perf_compressed = base64.b64decode(json["data"]) - data_perf_pickled = zlib.decompress(data_perf_compressed) - data_perf = pickle.loads(data_perf_pickled) - return data_perf +def json_to_file(encoded_data): + compressed_file = base64.b64decode(encoded_data) + return zlib.decompress(compressed_file) def run_server( @@ -108,10 +108,6 @@ def run_server( auth_aliases, stats_output, ): - # address to send - url = 'https://34.202.72.107/api/catalyst/serve' - # url = 'http://127.0.0.1:5000/api/catalyst/serve' - if algotext or algofile: # argument preparation - encode the file for transfer algofile, algotext = prepare_args(algofile, algotext) @@ -146,18 +142,32 @@ def run_server( 'auth_aliases': auth_aliases, }} - # call the server with the following arguments - # if any issues raised related to the format of the dates, convert them - response = requests.post(url, - json=json.dumps( - json_file, - default=convert_date - ), - verify=False, - ) - - # close the handlers, which are not needed anymore - controller.stop() - subscriber.close() + session = requests.Session() + response = session.post('{}/backtest/run'.format(AUTH_SERVER), headers={ + 'Authorization': 'Digest username="{0}",password="{1}"'. + format(key, secret)}) + + header = response.headers.get('WWW-Authenticate') + auth_type, auth_info = header.split(None, 1) + d = requests.utils.parse_dict_header(auth_info) + + a1 = key + ":" + d['realm'] + ":" + secret + ha1 = md5(a1.encode('utf-8')).hexdigest() + a2 = "POST:/backtest/run" + ha2 = md5(a2.encode('utf-8')).hexdigest() + a3 = ha1 + ":" + d['nonce'] + ":" + ha2 + result = md5(a3.encode('utf-8')).hexdigest() + + response = session.post('{}/backtest/run'.format(AUTH_SERVER), + json=json.dumps(json_file, default=convert_date), + verify=False, + headers={ + 'Authorization': 'Digest username="{0}",' + 'realm="{1}",nonce="{2}",' + 'uri="/backtest/run",' + 'response="{3}",' + 'opaque="{4}"'. + format(key, d['realm'], d['nonce'], + result, d['opaque'])}) return handle_response(response) From 126ba85b9b07dc72ba382576058c1599ea9274ea Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 30 Jul 2018 15:06:15 +0300 Subject: [PATCH 17/35] BLD: added location to store and retrieve key pair from. --- catalyst/exchange/exchange_errors.py | 7 +++ catalyst/exchange/utils/exchange_utils.py | 52 +++++++++++++++++++++++ catalyst/utils/run_server.py | 22 ++++++++-- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 7b18afcc2..8b0b5b6c9 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -73,6 +73,13 @@ class ExchangeAuthEmpty(ZiplineError): ).strip() +class RemoteAuthEmpty(ZiplineError): + msg = ( + 'Please enter your API token key and secret for the remote server ' + 'in the following file: {filename}' + ).strip() + + class ExchangeSymbolsNotFound(ZiplineError): msg = ( 'Unable to download or find a local copy of symbols.json for exchange ' diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 50e5124a6..3eb95aee2 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -38,6 +38,29 @@ def get_sid(symbol): return sid +def get_remote_folder(environ=None): + """ + The root path of the remote folder. + + Parameters + ---------- + environ: + + Returns + ------- + str + + """ + if not environ: + environ = os.environ + + root = data_root(environ) + remote_folder = os.path.join(root, 'remote') + ensure_directory(remote_folder) + + return remote_folder + + def get_exchange_folder(exchange_name, environ=None): """ The root path of an exchange folder. @@ -221,6 +244,35 @@ def get_exchange_auth(exchange_name, alias=None, environ=None): return data +def get_remote_auth(alias=None, environ=None): + """ + The de-serialized content of the remote auth.json file. + + Parameters + ---------- + environ: + + Returns + ------- + Object + + """ + remote_folder = get_remote_folder(environ) + name = 'remote_auth' if alias is None else alias + filename = os.path.join(remote_folder, '{}.json'.format(name)) + + if os.path.isfile(filename): + with open(filename) as data_file: + data = json.load(data_file) + return data + else: + data = dict(key='', secret='') + with open(filename, 'w') as f: + json.dump(data, f, sort_keys=False, indent=2, + separators=(',', ':')) + return data + + def delete_algo_folder(algo_name, environ=None): """ Delete the folder containing the algo state. diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index ab38f817f..052e75799 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -4,13 +4,16 @@ import zlib import pickle import requests +import os import pandas as pd from hashlib import md5 +from catalyst.exchange.utils.exchange_utils import get_remote_auth,\ + get_remote_folder +from catalyst.exchange.exchange_errors import RemoteAuthEmpty + AUTH_SERVER = 'http://localhost:5000' -key = '823fu8d4g' -secret = 'iu4f9f4iou3hf3498hf' def prepare_args(file, text): @@ -74,6 +77,7 @@ def load_response(json): data_df = pickle.loads(json_to_file(json['data'])) return data_df + def json_to_file(encoded_data): compressed_file = base64.b64decode(encoded_data) return zlib.decompress(compressed_file) @@ -141,7 +145,7 @@ def run_server( 'simulate_orders': simulate_orders, 'auth_aliases': auth_aliases, }} - + key, secret = retrieve_remote_auth() session = requests.Session() response = session.post('{}/backtest/run'.format(AUTH_SERVER), headers={ 'Authorization': 'Digest username="{0}",password="{1}"'. @@ -171,3 +175,15 @@ def run_server( result, d['opaque'])}) return handle_response(response) + + +def retrieve_remote_auth(): + remote_auth_dict = get_remote_auth() + has_auth = (remote_auth_dict['key'] != '' and + remote_auth_dict['secret'] != '') + if not has_auth: + raise RemoteAuthEmpty( + filename=os.path.join(get_remote_folder(), 'remote_auth.json') + ) + else: + return remote_auth_dict['key'], remote_auth_dict['secret'] From e1582faba84cad465da35ef032470de4fdeb638a Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 31 Jul 2018 16:27:42 +0300 Subject: [PATCH 18/35] MAINT: cleaning the code and pep8 fixes --- catalyst/utils/run_server.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/catalyst/utils/run_server.py b/catalyst/utils/run_server.py index 052e75799..f8dbee350 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/run_server.py @@ -13,7 +13,10 @@ get_remote_folder from catalyst.exchange.exchange_errors import RemoteAuthEmpty -AUTH_SERVER = 'http://localhost:5000' +# AUTH_SERVER = 'http://localhost:5000' +AUTH_SERVER = "https://sandbox2.enigma.co" +BACKTEST_PATH = '/backtest/run' +METHOD = 'POST' def prepare_args(file, text): @@ -71,10 +74,10 @@ def handle_response(response): return load_response(response.json()) -def load_response(json): - lf = json_to_file(json['logs']) +def load_response(json_file): + lf = json_to_file(json_file['logs']) print(lf.decode('utf-8')) - data_df = pickle.loads(json_to_file(json['data'])) + data_df = pickle.loads(json_to_file(json_file['data'])) return data_df @@ -147,9 +150,11 @@ def run_server( }} key, secret = retrieve_remote_auth() session = requests.Session() - response = session.post('{}/backtest/run'.format(AUTH_SERVER), headers={ - 'Authorization': 'Digest username="{0}",password="{1}"'. - format(key, secret)}) + response = session.post('{}{}'.format(AUTH_SERVER, BACKTEST_PATH), + headers={'Authorization': 'Digest username="{0}",' + 'password="{1}"'. + format(key, secret) + }) header = response.headers.get('WWW-Authenticate') auth_type, auth_info = header.split(None, 1) @@ -157,21 +162,21 @@ def run_server( a1 = key + ":" + d['realm'] + ":" + secret ha1 = md5(a1.encode('utf-8')).hexdigest() - a2 = "POST:/backtest/run" + a2 = "{}:{}".format(METHOD, BACKTEST_PATH) ha2 = md5(a2.encode('utf-8')).hexdigest() a3 = ha1 + ":" + d['nonce'] + ":" + ha2 result = md5(a3.encode('utf-8')).hexdigest() - response = session.post('{}/backtest/run'.format(AUTH_SERVER), + response = session.post('{}{}'.format(AUTH_SERVER, BACKTEST_PATH), json=json.dumps(json_file, default=convert_date), verify=False, headers={ 'Authorization': 'Digest username="{0}",' 'realm="{1}",nonce="{2}",' - 'uri="/backtest/run",' - 'response="{3}",' - 'opaque="{4}"'. - format(key, d['realm'], d['nonce'], + 'uri="{3}",' + 'response="{4}",' + 'opaque="{5}"'. + format(key, d['realm'], d['nonce'], BACKTEST_PATH, result, d['opaque'])}) return handle_response(response) From cb22ec6374a41c9a3b87852851214f1808787e9b Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 7 Aug 2018 20:30:33 +0300 Subject: [PATCH 19/35] BLD: modified names and added mail to arguments given by user (WIP) --- catalyst/__main__.py | 362 ++++++++++---------- catalyst/utils/{run_server.py => remote.py} | 17 +- 2 files changed, 199 insertions(+), 180 deletions(-) rename catalyst/utils/{run_server.py => remote.py} (93%) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index be95aa8af..1cc7ca64e 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -1,6 +1,7 @@ import errno import os from functools import wraps +import re import click import sys @@ -15,7 +16,7 @@ from catalyst.utils.cli import Date, Timestamp from catalyst.utils.run_algo import _run, load_extensions from catalyst.exchange.utils.bundle_utils import EXCHANGE_NAMES -from catalyst.utils.run_server import run_server +from catalyst.utils.remote import remote_backtest try: __IPYTHON__ @@ -515,7 +516,7 @@ def live(ctx, return perf -@main.command(name='serve') +@main.command(name='remote-run') @click.option( '-f', '--algofile', @@ -578,6 +579,12 @@ def live(ctx, type=Date(tz='utc', as_timestamp=True), help='The end date of the simulation.', ) +@click.option( + '-m', + '--mail', + show_default=True, + help='an E-mail address to send the results to', +) @click.option( '-o', '--output', @@ -616,7 +623,7 @@ def live(ctx, '(e.g. usd, btc, eth).', ) @click.pass_context -def run(ctx, +def remote_run(ctx, algofile, algotext, define, @@ -626,6 +633,7 @@ def run(ctx, bundle_timestamp, start, end, + mail, output, print_algo, local_namespace, @@ -666,9 +674,12 @@ def run(ctx, if capital_base is None: ctx.fail("must specify a capital base with '--capital-base'") + if mail is None or not re.match(r"[^@]+@[^@]+\.[^@]+", mail): + ctx.fail("must specify a valid email with '--mail'") + click.echo('Running in backtesting mode.', sys.stdout) - perf = run_server( + perf = remote_backtest( initialize=None, handle_data=None, before_trading_start=None, @@ -696,6 +707,7 @@ def run(ctx, simulate_orders=True, auth_aliases=None, stats_output=None, + mail=mail, ) if output == '-': @@ -706,177 +718,177 @@ def run(ctx, return perf -@main.command(name='serve-live') -@click.option( - '-f', - '--algofile', - default=None, - type=click.File('r'), - help='The file that contains the algorithm to run.', -) -@click.option( - '--capital-base', - type=float, - show_default=True, - help='The amount of capital (in quote_currency) allocated to trading.', -) -@click.option( - '-t', - '--algotext', - help='The algorithm script to run.', -) -@click.option( - '-D', - '--define', - multiple=True, - help="Define a name to be bound in the namespace before executing" - " the algotext. For example '-Dname=value'. The value may be" - " any python expression. These are evaluated in order so they" - " may refer to previously defined names.", -) -@click.option( - '-o', - '--output', - default='-', - metavar='FILENAME', - show_default=True, - help="The location to write the perf data. If this is '-' the perf will" - " be written to stdout.", -) -@click.option( - '--print-algo/--no-print-algo', - is_flag=True, - default=False, - help='Print the algorithm to stdout.', -) -@ipython_only(click.option( - '--local-namespace/--no-local-namespace', - is_flag=True, - default=None, - help='Should the algorithm methods be resolved in the local namespace.' -)) -@click.option( - '-x', - '--exchange-name', - help='The name of the targeted exchange.', -) -@click.option( - '-n', - '--algo-namespace', - help='A label assigned to the algorithm for data storage purposes.' -) -@click.option( - '-c', - '--quote-currency', - help='The quote currency used to calculate statistics ' - '(e.g. usd, btc, eth).', -) -@click.option( - '-e', - '--end', - type=Date(tz='utc', as_timestamp=False), - help='An optional end date at which to stop the execution.', -) -@click.option( - '--live-graph/--no-live-graph', - is_flag=True, - default=False, - help='Display live graph.', -) -@click.option( - '--simulate-orders/--no-simulate-orders', - is_flag=True, - default=True, - help='Simulating orders enable the paper trading mode. No orders will be ' - 'sent to the exchange unless set to false.', -) -@click.option( - '--auth-aliases', - default=None, - help='Authentication file aliases for the specified exchanges. By default,' - 'each exchange uses the "auth.json" file in the exchange folder. ' - 'Specifying an "auth2" alias would use "auth2.json". It should be ' - 'specified like this: "[exchange_name],[alias],..." For example, ' - '"binance,auth2" or "binance,auth2,bittrex,auth2".', -) -@click.pass_context -def serve_live(ctx, - algofile, - capital_base, - algotext, - define, - output, - print_algo, - local_namespace, - exchange_name, - algo_namespace, - quote_currency, - end, - live_graph, - auth_aliases, - simulate_orders): - """Trade live with the given algorithm on the server. - """ - if (algotext is not None) == (algofile is not None): - ctx.fail( - "must specify exactly one of '-f' / '--algofile' or" - " '-t' / '--algotext'", - ) - - if exchange_name is None: - ctx.fail("must specify an exchange name '-x'") - - if algo_namespace is None: - ctx.fail("must specify an algorithm name '-n' in live execution mode") - - if quote_currency is None: - ctx.fail("must specify a quote currency '-c' in live execution mode") - - if capital_base is None: - ctx.fail("must specify a capital base with '--capital-base'") - - if simulate_orders: - click.echo('Running in paper trading mode.', sys.stdout) - - else: - click.echo('Running in live trading mode.', sys.stdout) - - perf = run_server( - initialize=None, - handle_data=None, - before_trading_start=None, - analyze=None, - algofile=algofile, - algotext=algotext, - defines=define, - data_frequency=None, - capital_base=capital_base, - data=None, - bundle=None, - bundle_timestamp=None, - start=None, - end=end, - output=output, - print_algo=print_algo, - local_namespace=local_namespace, - environ=os.environ, - live=True, - exchange=exchange_name, - algo_namespace=algo_namespace, - quote_currency=quote_currency, - live_graph=live_graph, - analyze_live=None, - simulate_orders=simulate_orders, - auth_aliases=auth_aliases, - stats_output=None, - ) - - if output == '-': - click.echo(str(perf), sys.stdout) - elif output != os.devnull: # make the catalyst magic not write any data - perf.to_pickle(output) - - return perf +# @main.command(name='serve-live') +# @click.option( +# '-f', +# '--algofile', +# default=None, +# type=click.File('r'), +# help='The file that contains the algorithm to run.', +# ) +# @click.option( +# '--capital-base', +# type=float, +# show_default=True, +# help='The amount of capital (in quote_currency) allocated to trading.', +# ) +# @click.option( +# '-t', +# '--algotext', +# help='The algorithm script to run.', +# ) +# @click.option( +# '-D', +# '--define', +# multiple=True, +# help="Define a name to be bound in the namespace before executing" +# " the algotext. For example '-Dname=value'. The value may be" +# " any python expression. These are evaluated in order so they" +# " may refer to previously defined names.", +# ) +# @click.option( +# '-o', +# '--output', +# default='-', +# metavar='FILENAME', +# show_default=True, +# help="The location to write the perf data. If this is '-' the perf will" +# " be written to stdout.", +# ) +# @click.option( +# '--print-algo/--no-print-algo', +# is_flag=True, +# default=False, +# help='Print the algorithm to stdout.', +# ) +# @ipython_only(click.option( +# '--local-namespace/--no-local-namespace', +# is_flag=True, +# default=None, +# help='Should the algorithm methods be resolved in the local namespace.' +# )) +# @click.option( +# '-x', +# '--exchange-name', +# help='The name of the targeted exchange.', +# ) +# @click.option( +# '-n', +# '--algo-namespace', +# help='A label assigned to the algorithm for data storage purposes.' +# ) +# @click.option( +# '-c', +# '--quote-currency', +# help='The quote currency used to calculate statistics ' +# '(e.g. usd, btc, eth).', +# ) +# @click.option( +# '-e', +# '--end', +# type=Date(tz='utc', as_timestamp=False), +# help='An optional end date at which to stop the execution.', +# ) +# @click.option( +# '--live-graph/--no-live-graph', +# is_flag=True, +# default=False, +# help='Display live graph.', +# ) +# @click.option( +# '--simulate-orders/--no-simulate-orders', +# is_flag=True, +# default=True, +# help='Simulating orders enable the paper trading mode. No orders will be ' +# 'sent to the exchange unless set to false.', +# ) +# @click.option( +# '--auth-aliases', +# default=None, +# help='Authentication file aliases for the specified exchanges. By default,' +# 'each exchange uses the "auth.json" file in the exchange folder. ' +# 'Specifying an "auth2" alias would use "auth2.json". It should be ' +# 'specified like this: "[exchange_name],[alias],..." For example, ' +# '"binance,auth2" or "binance,auth2,bittrex,auth2".', +# ) +# @click.pass_context +# def serve_live(ctx, +# algofile, +# capital_base, +# algotext, +# define, +# output, +# print_algo, +# local_namespace, +# exchange_name, +# algo_namespace, +# quote_currency, +# end, +# live_graph, +# auth_aliases, +# simulate_orders): +# """Trade live with the given algorithm on the server. +# """ +# if (algotext is not None) == (algofile is not None): +# ctx.fail( +# "must specify exactly one of '-f' / '--algofile' or" +# " '-t' / '--algotext'", +# ) +# +# if exchange_name is None: +# ctx.fail("must specify an exchange name '-x'") +# +# if algo_namespace is None: +# ctx.fail("must specify an algorithm name '-n' in live execution mode") +# +# if quote_currency is None: +# ctx.fail("must specify a quote currency '-c' in live execution mode") +# +# if capital_base is None: +# ctx.fail("must specify a capital base with '--capital-base'") +# +# if simulate_orders: +# click.echo('Running in paper trading mode.', sys.stdout) +# +# else: +# click.echo('Running in live trading mode.', sys.stdout) +# +# perf = remote_backtest( +# initialize=None, +# handle_data=None, +# before_trading_start=None, +# analyze=None, +# algofile=algofile, +# algotext=algotext, +# defines=define, +# data_frequency=None, +# capital_base=capital_base, +# data=None, +# bundle=None, +# bundle_timestamp=None, +# start=None, +# end=end, +# output=output, +# print_algo=print_algo, +# local_namespace=local_namespace, +# environ=os.environ, +# live=True, +# exchange=exchange_name, +# algo_namespace=algo_namespace, +# quote_currency=quote_currency, +# live_graph=live_graph, +# analyze_live=None, +# simulate_orders=simulate_orders, +# auth_aliases=auth_aliases, +# stats_output=None, +# ) +# +# if output == '-': +# click.echo(str(perf), sys.stdout) +# elif output != os.devnull: # make the catalyst magic not write any data +# perf.to_pickle(output) +# +# return perf @main.command(name='ingest-exchange') diff --git a/catalyst/utils/run_server.py b/catalyst/utils/remote.py similarity index 93% rename from catalyst/utils/run_server.py rename to catalyst/utils/remote.py index f8dbee350..1eaceed58 100644 --- a/catalyst/utils/run_server.py +++ b/catalyst/utils/remote.py @@ -8,13 +8,16 @@ import pandas as pd from hashlib import md5 +from logbook import Logger from catalyst.exchange.utils.exchange_utils import get_remote_auth,\ get_remote_folder from catalyst.exchange.exchange_errors import RemoteAuthEmpty -# AUTH_SERVER = 'http://localhost:5000' -AUTH_SERVER = "https://sandbox2.enigma.co" +log = Logger('remote') + +AUTH_SERVER = 'http://localhost:5000' +# AUTH_SERVER = "https://sandbox2.enigma.co" BACKTEST_PATH = '/backtest/run' METHOD = 'POST' @@ -71,7 +74,8 @@ def handle_response(response): return response.json()['error'] if response.json()['error'] else None else: # if the run was successful - return load_response(response.json()) + log.info('In order to follow your algo run use the following id: ' + + response.json()['algo_id']) def load_response(json_file): @@ -86,7 +90,7 @@ def json_to_file(encoded_data): return zlib.decompress(compressed_file) -def run_server( +def remote_backtest( initialize, handle_data, before_trading_start, @@ -114,6 +118,7 @@ def run_server( simulate_orders, auth_aliases, stats_output, + mail, ): if algotext or algofile: # argument preparation - encode the file for transfer @@ -147,6 +152,7 @@ def run_server( 'live_graph': live_graph, 'simulate_orders': simulate_orders, 'auth_aliases': auth_aliases, + 'mail': mail }} key, secret = retrieve_remote_auth() session = requests.Session() @@ -154,7 +160,8 @@ def run_server( headers={'Authorization': 'Digest username="{0}",' 'password="{1}"'. format(key, secret) - }) + }, + verify=False) header = response.headers.get('WWW-Authenticate') auth_type, auth_info = header.split(None, 1) From e797811d5aaea3df31b0052c364011756a7eb4fd Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 20 Aug 2018 17:53:16 +0300 Subject: [PATCH 20/35] BLD: added status mode (WIP- tests needed to be added) --- catalyst/__main__.py | 56 ++++++++++++- catalyst/utils/remote.py | 143 ++++++++++++++++----------------- catalyst/utils/remote_utils.py | 68 ++++++++++++++++ 3 files changed, 192 insertions(+), 75 deletions(-) create mode 100644 catalyst/utils/remote_utils.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 1cc7ca64e..27af2306f 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -16,7 +16,7 @@ from catalyst.utils.cli import Date, Timestamp from catalyst.utils.run_algo import _run, load_extensions from catalyst.exchange.utils.bundle_utils import EXCHANGE_NAMES -from catalyst.utils.remote import remote_backtest +from catalyst.utils.remote import remote_backtest, get_remote_status try: __IPYTHON__ @@ -718,6 +718,60 @@ def remote_run(ctx, return perf +@main.command(name='remote-status') +@click.option( + '-i', + '--algo-id', + show_default=True, + help='The algo id of your running algorithm on the cloud', +) +@click.option( + '-d', + '--data-output', + default='-', + metavar='FILENAME', + show_default=True, + help="The location to write the perf data, if it exists. If this is '-' " + "the perf will be written to stdout.", +) +@click.option( + '-l', + '--log-output', + default='-', + metavar='FILENAME', + show_default=True, + help="The location to write the current logging. " + "If this is '-' the log will be written to stdout.", +) +@click.pass_context +def remote_status(ctx, algo_id, data_output, log_output): + """ + Get the status of the running algorithm on the cloud + """ + if algo_id is None: + ctx.fail("must specify an id of your running algorithm with '--id'") + + click.echo('Running in backtesting mode.', sys.stdout) + + perf, log = get_remote_status( + algo_id=algo_id + ) + + if log_output == '-': + click.echo(str(log), sys.stdout) + elif log_output != os.devnull: + with open(log_output, 'w') as file: + file.write(log) + + if data_output == '-' or perf is None: + click.echo('the performance data is:\n' + str(perf), sys.stdout) + elif data_output != os.devnull: + # make the catalyst magic not write any data + perf.to_pickle(data_output) + + return perf + + # @main.command(name='serve-live') # @click.option( # '-f', diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index 1eaceed58..a0a23249a 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -1,15 +1,13 @@ #!flask/bin/python -import base64 import json -import zlib -import pickle import requests import os -import pandas as pd from hashlib import md5 from logbook import Logger +from catalyst.utils.remote_utils import BACKTEST_PATH, STATUS_PATH, POST, \ + GET, EXCEPTION_LOG, convert_date, prepare_args, handle_status from catalyst.exchange.utils.exchange_utils import get_remote_auth,\ get_remote_folder from catalyst.exchange.exchange_errors import RemoteAuthEmpty @@ -18,43 +16,12 @@ AUTH_SERVER = 'http://localhost:5000' # AUTH_SERVER = "https://sandbox2.enigma.co" -BACKTEST_PATH = '/backtest/run' -METHOD = 'POST' +BACKTEST = 'backtest' +STATUS = 'status' -def prepare_args(file, text): - """ - send the algo as a base64 decoded text object - - :param file: File - :param text: str - :return: None, text: str - """ - - if text: - text = base64.b64encode(text) - else: - text = base64.b64encode(bytes(file.read(), 'utf-8')).decode('utf-8') - file = None - return file, text - - -def convert_date(date): - """ - when transferring dates by json, - converts it to str - # any instances which need a conversion, - # must be done here - :param date: - :return: str(date) - """ - - if isinstance(date, pd.Timestamp): - return date.__str__() - - -def handle_response(response): +def handle_response(response, mode): """ handles the response given by the server according to it's status code @@ -62,32 +29,23 @@ def handle_response(response): :return: DataFrame/ str """ if response.status_code == 500: - raise Exception("issues with cloud connections, " - "unable to run catalyst on the cloud") + raise Exception("issues with cloud connections,\n" + "unable to run catalyst on the cloud,\n" + "try running again and if you get the same response,\n" + + EXCEPTION_LOG + ) elif response.status_code == 502: - raise Exception("The server is down at the moment, please contact " - "Catalyst support to fix this issue at " - "https://github.com/enigmampc/catalyst/issues/") - elif response.status_code == 202 or response.status_code == 400: - lf = json_to_file(response.json()['logs']) - print(lf.decode('utf-8')) - return response.json()['error'] if response.json()['error'] else None - + raise Exception("The server is down at the moment,\n" + EXCEPTION_LOG) + elif response.status_code == 400: + raise Exception("There is a connection but it was aborted due " + "to wrong arguments given to the server.\n" + + EXCEPTION_LOG) else: # if the run was successful - log.info('In order to follow your algo run use the following id: ' + - response.json()['algo_id']) - - -def load_response(json_file): - lf = json_to_file(json_file['logs']) - print(lf.decode('utf-8')) - data_df = pickle.loads(json_to_file(json_file['data'])) - return data_df - - -def json_to_file(encoded_data): - compressed_file = base64.b64decode(encoded_data) - return zlib.decompress(compressed_file) + if mode == BACKTEST: + log.info('In order to follow your algo run use the following id: ' + + response.json()['algo_id']) + elif mode == STATUS: + return handle_status(response.json()) def remote_backtest( @@ -154,14 +112,41 @@ def remote_backtest( 'auth_aliases': auth_aliases, 'mail': mail }} + response = send_digest_request( + json_file=json_file, path=BACKTEST_PATH, method=POST + ) + return handle_response(response, BACKTEST) + + +def get_remote_status(algo_id): + json_file = {'algo_id': algo_id} + log.info("retrieving the status of {} algorithm".format(algo_id)) + response = send_digest_request( + json_file=json_file, path=STATUS_PATH, method=GET + ) + return handle_response(response, STATUS) + + +def send_digest_request(json_file, path, method): key, secret = retrieve_remote_auth() + json_file['key'] = key session = requests.Session() - response = session.post('{}{}'.format(AUTH_SERVER, BACKTEST_PATH), - headers={'Authorization': 'Digest username="{0}",' - 'password="{1}"'. - format(key, secret) - }, - verify=False) + if method == POST: + response = session.post('{}{}'.format(AUTH_SERVER, path), + headers={ + 'Authorization': + 'Digest username="{0}",' + 'password="{1}"'.format(key, secret) + }, + verify=False) + else: # method == GET: + response = session.get('{}{}'.format(AUTH_SERVER, path), + headers={ + 'Authorization': + 'Digest username="{0}",' + 'password="{1}"'.format(key, secret) + }, + verify=False) header = response.headers.get('WWW-Authenticate') auth_type, auth_info = header.split(None, 1) @@ -169,12 +154,13 @@ def remote_backtest( a1 = key + ":" + d['realm'] + ":" + secret ha1 = md5(a1.encode('utf-8')).hexdigest() - a2 = "{}:{}".format(METHOD, BACKTEST_PATH) + a2 = "{}:{}".format(method, path) ha2 = md5(a2.encode('utf-8')).hexdigest() a3 = ha1 + ":" + d['nonce'] + ":" + ha2 result = md5(a3.encode('utf-8')).hexdigest() - response = session.post('{}{}'.format(AUTH_SERVER, BACKTEST_PATH), + if method == POST: + return session.post('{}{}'.format(AUTH_SERVER, path), json=json.dumps(json_file, default=convert_date), verify=False, headers={ @@ -183,11 +169,20 @@ def remote_backtest( 'uri="{3}",' 'response="{4}",' 'opaque="{5}"'. - format(key, d['realm'], d['nonce'], BACKTEST_PATH, + format(key, d['realm'], d['nonce'], path, + result, d['opaque'])}) + else: # method == GET + return session.get('{}{}'.format(AUTH_SERVER, path), + json=json.dumps(json_file, default=convert_date), + verify=False, + headers={ + 'Authorization': 'Digest username="{0}",' + 'realm="{1}",nonce="{2}",' + 'uri="{3}",' + 'response="{4}",' + 'opaque="{5}"'. + format(key, d['realm'], d['nonce'], path, result, d['opaque'])}) - - return handle_response(response) - def retrieve_remote_auth(): remote_auth_dict = get_remote_auth() diff --git a/catalyst/utils/remote_utils.py b/catalyst/utils/remote_utils.py new file mode 100644 index 000000000..45eace389 --- /dev/null +++ b/catalyst/utils/remote_utils.py @@ -0,0 +1,68 @@ +import base64 +import pickle +import zlib + +import pandas as pd +from logbook import Logger + + +log = Logger('remote') + +BACKTEST_PATH = '/backtest/run' +STATUS_PATH = '/status' + +POST = 'POST' +GET = 'GET' + +EXCEPTION_LOG = "please contact Catalyst support to fix this issue at\n" \ + "https://github.com/enigmampc/catalyst/issues/" + + +def prepare_args(file, text): + """ + send the algo as a base64 decoded text object + + :param file: File + :param text: str + :return: None, text: str + """ + + if text: + text = base64.b64encode(text) + else: + text = base64.b64encode(bytes(file.read(), 'utf-8')).decode('utf-8') + file = None + return file, text + + +def convert_date(date): + """ + when transferring dates by json, + converts it to str + # any instances which need a conversion, + # must be done here + + :param date: + :return: str(date) + """ + + if isinstance(date, pd.Timestamp): + return date.__str__() + + +def load_response(json_file): + log_file = decompress_data(json_file['log']).decode('utf-8') + data_df = None if json_file['data'] is None \ + else pickle.loads(decompress_data(json_file['data'])) + return data_df, log_file + + +def decompress_data(encoded_data): + compressed_file = base64.b64decode(encoded_data) + return zlib.decompress(compressed_file) + + +def handle_status(received_content): + log.info("The status of the algorithm is: '{}'". + format(received_content['status'])) + return load_response(received_content) From c0c9981897582a4155da13a656c679c38b3be1b8 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 21 Aug 2018 16:42:37 +0300 Subject: [PATCH 21/35] BLD: return the received algo id to the user --- catalyst/__main__.py | 13 +++---------- catalyst/utils/remote.py | 6 ++++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 27af2306f..752169903 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -677,9 +677,7 @@ def remote_run(ctx, if mail is None or not re.match(r"[^@]+@[^@]+\.[^@]+", mail): ctx.fail("must specify a valid email with '--mail'") - click.echo('Running in backtesting mode.', sys.stdout) - - perf = remote_backtest( + algo_id = remote_backtest( initialize=None, handle_data=None, before_trading_start=None, @@ -709,13 +707,8 @@ def remote_run(ctx, stats_output=None, mail=mail, ) - - if output == '-': - click.echo(str(perf), sys.stdout) - elif output != os.devnull: # make the catalyst magic not write any data - perf.to_pickle(output) - - return perf + print(algo_id) + return algo_id @main.command(name='remote-status') diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index a0a23249a..7f7c4f500 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -42,8 +42,10 @@ def handle_response(response, mode): EXCEPTION_LOG) else: # if the run was successful if mode == BACKTEST: + algo_id = response.json()['algo_id'] log.info('In order to follow your algo run use the following id: ' - + response.json()['algo_id']) + + algo_id) + return algo_id elif mode == STATUS: return handle_status(response.json()) @@ -171,7 +173,7 @@ def send_digest_request(json_file, path, method): 'opaque="{5}"'. format(key, d['realm'], d['nonce'], path, result, d['opaque'])}) - else: # method == GET + else: # method == GET return session.get('{}{}'.format(AUTH_SERVER, path), json=json.dumps(json_file, default=convert_date), verify=False, From 09fc44c76a8bf4ea4d2479c680a119138978d5fc Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 21 Aug 2018 21:10:09 +0300 Subject: [PATCH 22/35] BLD: non existing id handling on status request --- catalyst/__main__.py | 36 ++++++++++++++++++---------------- catalyst/utils/remote_utils.py | 11 +++++++++-- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 752169903..e3f5ed0eb 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -744,25 +744,27 @@ def remote_status(ctx, algo_id, data_output, log_output): if algo_id is None: ctx.fail("must specify an id of your running algorithm with '--id'") - click.echo('Running in backtesting mode.', sys.stdout) - - perf, log = get_remote_status( + status_response = get_remote_status( algo_id=algo_id ) - - if log_output == '-': - click.echo(str(log), sys.stdout) - elif log_output != os.devnull: - with open(log_output, 'w') as file: - file.write(log) - - if data_output == '-' or perf is None: - click.echo('the performance data is:\n' + str(perf), sys.stdout) - elif data_output != os.devnull: - # make the catalyst magic not write any data - perf.to_pickle(data_output) - - return perf + if isinstance(status_response, tuple): + status, perf, log= status_response + if log_output == '-': + click.echo(str(log), sys.stderr) + elif log_output != os.devnull: + with open(log_output, 'w') as file: + file.write(log) + + if data_output == '-' or perf is None: + click.echo('the performance data is:\n' + str(perf), sys.stdout) + elif data_output != os.devnull: + # make the catalyst magic not write any data + perf.to_pickle(data_output) + print(status) + return status, perf + else: + print(status_response) + return status_response # @main.command(name='serve-live') diff --git a/catalyst/utils/remote_utils.py b/catalyst/utils/remote_utils.py index 45eace389..a9f39be70 100644 --- a/catalyst/utils/remote_utils.py +++ b/catalyst/utils/remote_utils.py @@ -13,6 +13,7 @@ POST = 'POST' GET = 'GET' +NONEXISTENT = 'nonexistent' EXCEPTION_LOG = "please contact Catalyst support to fix this issue at\n" \ "https://github.com/enigmampc/catalyst/issues/" @@ -63,6 +64,12 @@ def decompress_data(encoded_data): def handle_status(received_content): + status = received_content['status'] + if status == NONEXISTENT: + log.error(received_content['message']) + return status + log.info("The status of the algorithm is: '{}'". - format(received_content['status'])) - return load_response(received_content) + format(status)) + perf, log_file = load_response(received_content) + return status, perf, log_file From bce929374bbd9d4836db9d549f4428a0f9920544 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 22 Aug 2018 13:08:04 +0300 Subject: [PATCH 23/35] BLD: validate the uuid when receiving the status --- catalyst/utils/remote.py | 6 +++++- catalyst/utils/remote_utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index 7f7c4f500..9297a470f 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -7,7 +7,8 @@ from logbook import Logger from catalyst.utils.remote_utils import BACKTEST_PATH, STATUS_PATH, POST, \ - GET, EXCEPTION_LOG, convert_date, prepare_args, handle_status + GET, EXCEPTION_LOG, convert_date, prepare_args, handle_status, \ + is_valid_uuid from catalyst.exchange.utils.exchange_utils import get_remote_auth,\ get_remote_folder from catalyst.exchange.exchange_errors import RemoteAuthEmpty @@ -121,6 +122,9 @@ def remote_backtest( def get_remote_status(algo_id): + if not is_valid_uuid(algo_id): + raise Exception("the id you entered is invalid! " + "please enter a valid id.") json_file = {'algo_id': algo_id} log.info("retrieving the status of {} algorithm".format(algo_id)) response = send_digest_request( diff --git a/catalyst/utils/remote_utils.py b/catalyst/utils/remote_utils.py index a9f39be70..8037f255b 100644 --- a/catalyst/utils/remote_utils.py +++ b/catalyst/utils/remote_utils.py @@ -1,6 +1,7 @@ import base64 import pickle import zlib +from uuid import UUID import pandas as pd from logbook import Logger @@ -73,3 +74,31 @@ def handle_status(received_content): format(status)) perf, log_file = load_response(received_content) return status, perf, log_file + + +def is_valid_uuid(uuid_to_test, version=4): + """ + Check if uuid_to_test is a valid UUID. + + Parameters + ---------- + uuid_to_test : str + version : {1, 2, 3, 4} + + Returns + ------- + `True` if uuid_to_test is a valid UUID, otherwise `False`. + + Examples + -------- + >>> is_valid_uuid('c9bf9e57-1685-4c89-bafb-ff5af830be8a') + True + >>> is_valid_uuid('c9bf9e58') + False + """ + try: + uuid_obj = UUID(uuid_to_test, version=version).hex + except: + return False + + return str(uuid_obj) == uuid_to_test From 279b08f9a8e86c7483f674f4aa9b4ebc94a88b88 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 27 Aug 2018 11:30:26 +0300 Subject: [PATCH 24/35] MAINT: pep8 fixes --- catalyst/utils/remote.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index 9297a470f..f6ae1983f 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -27,6 +27,7 @@ def handle_response(response, mode): handles the response given by the server according to it's status code :param response: the format returned from a request + :param mode: Backtest/ status :return: DataFrame/ str """ if response.status_code == 500: @@ -145,14 +146,13 @@ def send_digest_request(json_file, path, method): 'password="{1}"'.format(key, secret) }, verify=False) - else: # method == GET: + else: # method == GET: response = session.get('{}{}'.format(AUTH_SERVER, path), - headers={ - 'Authorization': - 'Digest username="{0}",' - 'password="{1}"'.format(key, secret) - }, - verify=False) + headers={'Authorization': + 'Digest username="{0}", ' + 'password="{1}"'. + format(key, secret)}, + verify=False) header = response.headers.get('WWW-Authenticate') auth_type, auth_info = header.split(None, 1) @@ -179,16 +179,15 @@ def send_digest_request(json_file, path, method): result, d['opaque'])}) else: # method == GET return session.get('{}{}'.format(AUTH_SERVER, path), - json=json.dumps(json_file, default=convert_date), - verify=False, - headers={ - 'Authorization': 'Digest username="{0}",' - 'realm="{1}",nonce="{2}",' - 'uri="{3}",' - 'response="{4}",' - 'opaque="{5}"'. - format(key, d['realm'], d['nonce'], path, - result, d['opaque'])}) + json=json.dumps(json_file, default=convert_date), + verify=False, + headers={'Authorization': + 'Digest username="{0}", realm="{1}",' + 'nonce="{2}",uri="{3}", ' + 'response="{4}",opaque="{5}"'. + format(key, d['realm'], d['nonce'], + path, result, d['opaque'])}) + def retrieve_remote_auth(): remote_auth_dict = get_remote_auth() From b0c3cbc4e5dcff36c4965ec7d1dbde89472d9763 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 27 Aug 2018 16:41:30 +0300 Subject: [PATCH 25/35] DOC: modified the cloud commands --help docs --- catalyst/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index e3f5ed0eb..ce229884f 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -640,7 +640,7 @@ def remote_run(ctx, exchange_name, algo_namespace, quote_currency): - """Run a backtest for the given algorithm on the server. + """Run a backtest for the given algorithm on the cloud. """ if (algotext is not None) == (algofile is not None): @@ -739,7 +739,7 @@ def remote_run(ctx, @click.pass_context def remote_status(ctx, algo_id, data_output, log_output): """ - Get the status of the running algorithm on the cloud + Get the status of a running algorithm on the cloud """ if algo_id is None: ctx.fail("must specify an id of your running algorithm with '--id'") From cc7bfcc76f48458e758a1d06d7a30ea9ed6a34b8 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 3 Sep 2018 10:59:03 +0300 Subject: [PATCH 26/35] MAINT: support htttps --- catalyst/utils/remote.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index f6ae1983f..3180a270b 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -1,8 +1,8 @@ #!flask/bin/python import json -import requests import os +import requests from hashlib import md5 from logbook import Logger @@ -145,14 +145,14 @@ def send_digest_request(json_file, path, method): 'Digest username="{0}",' 'password="{1}"'.format(key, secret) }, - verify=False) + ) else: # method == GET: response = session.get('{}{}'.format(AUTH_SERVER, path), headers={'Authorization': 'Digest username="{0}", ' 'password="{1}"'. format(key, secret)}, - verify=False) + ) header = response.headers.get('WWW-Authenticate') auth_type, auth_info = header.split(None, 1) @@ -168,7 +168,6 @@ def send_digest_request(json_file, path, method): if method == POST: return session.post('{}{}'.format(AUTH_SERVER, path), json=json.dumps(json_file, default=convert_date), - verify=False, headers={ 'Authorization': 'Digest username="{0}",' 'realm="{1}",nonce="{2}",' @@ -180,7 +179,6 @@ def send_digest_request(json_file, path, method): else: # method == GET return session.get('{}{}'.format(AUTH_SERVER, path), json=json.dumps(json_file, default=convert_date), - verify=False, headers={'Authorization': 'Digest username="{0}", realm="{1}",' 'nonce="{2}",uri="{3}", ' From 12dd867dcb094fa3958153b3a20190fc439adb39 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 3 Sep 2018 11:18:52 +0300 Subject: [PATCH 27/35] MAINT: pep8 --- catalyst/__main__.py | 207 ++++--------------------------------------- 1 file changed, 17 insertions(+), 190 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index e0540bd5d..11d5358af 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -624,22 +624,22 @@ def live(ctx, ) @click.pass_context def remote_run(ctx, - algofile, - algotext, - define, - data_frequency, - capital_base, - bundle, - bundle_timestamp, - start, - end, - mail, - output, - print_algo, - local_namespace, - exchange_name, - algo_namespace, - quote_currency): + algofile, + algotext, + define, + data_frequency, + capital_base, + bundle, + bundle_timestamp, + start, + end, + mail, + output, + print_algo, + local_namespace, + exchange_name, + algo_namespace, + quote_currency): """Run a backtest for the given algorithm on the cloud. """ @@ -748,7 +748,7 @@ def remote_status(ctx, algo_id, data_output, log_output): algo_id=algo_id ) if isinstance(status_response, tuple): - status, perf, log= status_response + status, perf, log = status_response if log_output == '-': click.echo(str(log), sys.stderr) elif log_output != os.devnull: @@ -767,179 +767,6 @@ def remote_status(ctx, algo_id, data_output, log_output): return status_response -# @main.command(name='serve-live') -# @click.option( -# '-f', -# '--algofile', -# default=None, -# type=click.File('r'), -# help='The file that contains the algorithm to run.', -# ) -# @click.option( -# '--capital-base', -# type=float, -# show_default=True, -# help='The amount of capital (in quote_currency) allocated to trading.', -# ) -# @click.option( -# '-t', -# '--algotext', -# help='The algorithm script to run.', -# ) -# @click.option( -# '-D', -# '--define', -# multiple=True, -# help="Define a name to be bound in the namespace before executing" -# " the algotext. For example '-Dname=value'. The value may be" -# " any python expression. These are evaluated in order so they" -# " may refer to previously defined names.", -# ) -# @click.option( -# '-o', -# '--output', -# default='-', -# metavar='FILENAME', -# show_default=True, -# help="The location to write the perf data. If this is '-' the perf will" -# " be written to stdout.", -# ) -# @click.option( -# '--print-algo/--no-print-algo', -# is_flag=True, -# default=False, -# help='Print the algorithm to stdout.', -# ) -# @ipython_only(click.option( -# '--local-namespace/--no-local-namespace', -# is_flag=True, -# default=None, -# help='Should the algorithm methods be resolved in the local namespace.' -# )) -# @click.option( -# '-x', -# '--exchange-name', -# help='The name of the targeted exchange.', -# ) -# @click.option( -# '-n', -# '--algo-namespace', -# help='A label assigned to the algorithm for data storage purposes.' -# ) -# @click.option( -# '-c', -# '--quote-currency', -# help='The quote currency used to calculate statistics ' -# '(e.g. usd, btc, eth).', -# ) -# @click.option( -# '-e', -# '--end', -# type=Date(tz='utc', as_timestamp=False), -# help='An optional end date at which to stop the execution.', -# ) -# @click.option( -# '--live-graph/--no-live-graph', -# is_flag=True, -# default=False, -# help='Display live graph.', -# ) -# @click.option( -# '--simulate-orders/--no-simulate-orders', -# is_flag=True, -# default=True, -# help='Simulating orders enable the paper trading mode. No orders will be ' -# 'sent to the exchange unless set to false.', -# ) -# @click.option( -# '--auth-aliases', -# default=None, -# help='Authentication file aliases for the specified exchanges. By default,' -# 'each exchange uses the "auth.json" file in the exchange folder. ' -# 'Specifying an "auth2" alias would use "auth2.json". It should be ' -# 'specified like this: "[exchange_name],[alias],..." For example, ' -# '"binance,auth2" or "binance,auth2,bittrex,auth2".', -# ) -# @click.pass_context -# def serve_live(ctx, -# algofile, -# capital_base, -# algotext, -# define, -# output, -# print_algo, -# local_namespace, -# exchange_name, -# algo_namespace, -# quote_currency, -# end, -# live_graph, -# auth_aliases, -# simulate_orders): -# """Trade live with the given algorithm on the server. -# """ -# if (algotext is not None) == (algofile is not None): -# ctx.fail( -# "must specify exactly one of '-f' / '--algofile' or" -# " '-t' / '--algotext'", -# ) -# -# if exchange_name is None: -# ctx.fail("must specify an exchange name '-x'") -# -# if algo_namespace is None: -# ctx.fail("must specify an algorithm name '-n' in live execution mode") -# -# if quote_currency is None: -# ctx.fail("must specify a quote currency '-c' in live execution mode") -# -# if capital_base is None: -# ctx.fail("must specify a capital base with '--capital-base'") -# -# if simulate_orders: -# click.echo('Running in paper trading mode.', sys.stdout) -# -# else: -# click.echo('Running in live trading mode.', sys.stdout) -# -# perf = remote_backtest( -# initialize=None, -# handle_data=None, -# before_trading_start=None, -# analyze=None, -# algofile=algofile, -# algotext=algotext, -# defines=define, -# data_frequency=None, -# capital_base=capital_base, -# data=None, -# bundle=None, -# bundle_timestamp=None, -# start=None, -# end=end, -# output=output, -# print_algo=print_algo, -# local_namespace=local_namespace, -# environ=os.environ, -# live=True, -# exchange=exchange_name, -# algo_namespace=algo_namespace, -# quote_currency=quote_currency, -# live_graph=live_graph, -# analyze_live=None, -# simulate_orders=simulate_orders, -# auth_aliases=auth_aliases, -# stats_output=None, -# ) -# -# if output == '-': -# click.echo(str(perf), sys.stdout) -# elif output != os.devnull: # make the catalyst magic not write any data -# perf.to_pickle(output) -# -# return perf - - @main.command(name='ingest-exchange') @click.option( '-x', From 3b3036bda799ca014e8647c9c1f844f4ed990fd6 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 3 Sep 2018 16:12:46 +0300 Subject: [PATCH 28/35] MAINT: pep8 --- catalyst/exchange/exchange_bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 786179e76..462867c75 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -844,7 +844,7 @@ def get_history_window_series_and_load(self, field: str data_frequency: str algo_end_dt: pd.Timestamp - force_auto_ingest: + force_auto_ingest: Returns ------- From 0f8bd40e80b9fa7d88699c0f5468338239be5514 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Tue, 4 Sep 2018 19:02:28 +0300 Subject: [PATCH 29/35] BLD: remove unneccessary flask requirements --- etc/requirements_blaze.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/etc/requirements_blaze.txt b/etc/requirements_blaze.txt index 8ffaa2cbc..702c127b8 100644 --- a/etc/requirements_blaze.txt +++ b/etc/requirements_blaze.txt @@ -6,8 +6,6 @@ partd==0.3.7 locket==0.2.0 cloudpickle==0.2.1 itsdangerous==0.24 -flask==0.10.1 -flask-cors==2.1.2 Jinja2==2.7.3 MarkupSafe==1.0 Werkzeug==0.10.4 From 6882cecb8700753f0394717fdfc27851327b26b8 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 4 Sep 2018 19:36:26 +0300 Subject: [PATCH 30/35] DEV: added the python version of the client's side to the request --- catalyst/utils/remote.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index 3180a270b..a3ce823e6 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -1,6 +1,7 @@ #!flask/bin/python import json import os +import sys import requests from hashlib import md5 @@ -38,10 +39,10 @@ def handle_response(response, mode): ) elif response.status_code == 502: raise Exception("The server is down at the moment,\n" + EXCEPTION_LOG) - elif response.status_code == 400: + elif response.status_code == 400 or response.status_code == 401: raise Exception("There is a connection but it was aborted due " "to wrong arguments given to the server.\n" + - EXCEPTION_LOG) + response.content + EXCEPTION_LOG) else: # if the run was successful if mode == BACKTEST: algo_id = response.json()['algo_id'] @@ -114,7 +115,9 @@ def remote_backtest( 'live_graph': live_graph, 'simulate_orders': simulate_orders, 'auth_aliases': auth_aliases, - 'mail': mail + 'mail': mail, + 'py_version': sys.version_info[0], # the python version running on + # the client's side. 2 or 3 }} response = send_digest_request( json_file=json_file, path=BACKTEST_PATH, method=POST From 76b81169678e1acfe29c03abe4987fec94aab6c1 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Thu, 6 Sep 2018 19:10:51 +0300 Subject: [PATCH 31/35] MAINT: modified server name and fixed exception --- catalyst/utils/remote.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index a3ce823e6..60e3c187a 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -16,8 +16,8 @@ log = Logger('remote') -AUTH_SERVER = 'http://localhost:5000' -# AUTH_SERVER = "https://sandbox2.enigma.co" +# AUTH_SERVER = 'http://localhost:5000' +AUTH_SERVER = "https://sandbox2.enigma.co" BACKTEST = 'backtest' STATUS = 'status' @@ -42,7 +42,8 @@ def handle_response(response, mode): elif response.status_code == 400 or response.status_code == 401: raise Exception("There is a connection but it was aborted due " "to wrong arguments given to the server.\n" + - response.content + EXCEPTION_LOG) + response.content.decode('utf-8') + '\n' + + EXCEPTION_LOG) else: # if the run was successful if mode == BACKTEST: algo_id = response.json()['algo_id'] From 902bb816665513e576ea7bc2ffffb4c332f51814 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 12 Sep 2018 16:34:03 +0300 Subject: [PATCH 32/35] BUG: modified the status preview in case there is no data yet --- catalyst/__main__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 11d5358af..98db70ee5 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -749,17 +749,20 @@ def remote_status(ctx, algo_id, data_output, log_output): ) if isinstance(status_response, tuple): status, perf, log = status_response + if log_output == '-': click.echo(str(log), sys.stderr) elif log_output != os.devnull: with open(log_output, 'w') as file: file.write(log) - if data_output == '-' or perf is None: - click.echo('the performance data is:\n' + str(perf), sys.stdout) - elif data_output != os.devnull: - # make the catalyst magic not write any data - perf.to_pickle(data_output) + if perf is not None: + if data_output == '-': + click.echo('the performance data is:\n' + + str(perf), sys.stdout) + elif data_output != os.devnull: + # make the catalyst magic not write any data + perf.to_pickle(data_output) print(status) return status, perf else: From be6480f1782e72cf44f438fc1f227c2f79d6d9b1 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Wed, 12 Sep 2018 17:58:51 +0300 Subject: [PATCH 33/35] BLD: propogate the ccxt market active parameter to the trading_state attribute of the TradingPair #460 --- catalyst/exchange/utils/exchange_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 50e5124a6..87f344bab 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -661,6 +661,9 @@ def mixin_market_params(exchange_name, params, market): if 'lot' not in params: params['lot'] = params['min_trade_size'] + if 'active' in market: + params['trading_state'] = market['active'] + def group_assets_by_exchange(assets): exchange_assets = dict() From 3e0cd93cc7049201d64b144738d1c672e58370c5 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Thu, 13 Sep 2018 17:28:32 +0300 Subject: [PATCH 34/35] BLD: handle server down for maintenance --- catalyst/utils/remote.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/catalyst/utils/remote.py b/catalyst/utils/remote.py index 60e3c187a..6e76cc7c7 100644 --- a/catalyst/utils/remote.py +++ b/catalyst/utils/remote.py @@ -44,6 +44,9 @@ def handle_response(response, mode): "to wrong arguments given to the server.\n" + response.content.decode('utf-8') + '\n' + EXCEPTION_LOG) + elif response.status_code == 202: + raise Exception("The server is under maintenance. " + "please try again in a few minutes") else: # if the run was successful if mode == BACKTEST: algo_id = response.json()['algo_id'] From e737abfc50b2842c427bfc06e867f1044b3b4a49 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Thu, 13 Sep 2018 17:33:26 +0300 Subject: [PATCH 35/35] BLD: ignore perf_data print out to stdout if running on the cloud --- catalyst/__main__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 98db70ee5..037b70264 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -294,7 +294,9 @@ def run(ctx, stats_output=None, ) - if output == '-': + if output == '--': + pass + elif output == '-': click.echo(str(perf), sys.stdout) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) @@ -692,7 +694,7 @@ def remote_run(ctx, bundle_timestamp=bundle_timestamp, start=start, end=end, - output='-', + output='--', print_algo=print_algo, local_namespace=local_namespace, environ=os.environ,