download data.

README.md

@@ -22,4 +22,23 @@ script the following command
 > python setup.py install 
 
 or you can use pip to install the howtrader.
-> pip install git+https://github.com/ramoslin02/howtrader.git
+> pip install git+https://github.com/ramoslin02/howtrader.git
+
+
+# 使用 Usage
+你需要在项目下面创建一个文件夹加，howtrader, 这个主要是存放一些日记和配置文件的信息。
+如果不不知道配置文件如何配置，你可以启动examples文件目录下面的main_window.py文件，就可以看到其下面的一些日志和配置文件信息了。
+
+1. firstly you need to create a folder(howtrader) at your project, at
+   this folder, there are log file or configuration file. If you're not
+   sure how to config, you can simply run the main_window.py at examples
+   folder, you can play with UI.
+# 数据爬取
+howtrader可以通过data_manager的app直接下载数据，但是这个过程比较慢，适合少量数据的更新。
+如果你想批量获取数据，可以参考examples下面的download_data_demo2.py文件.
+
+you can download the data through data_manage app, but it's pretty slow,
+it just designs for small piece of data updating and strategy data
+warming. If you want to download the data as soon as possible, you can
+try the download_data_demo2.py or download_data_demo1.py at examples
+folder by using the multi-threads for speeding.

example/download_data.py → examples/download_data_demo1.py

@@ -8,33 +8,40 @@
 from howtrader.trader.engine import MainEngine
 
 from howtrader.gateway.binances import BinancesGateway
+from howtrader.gateway.binance import BinanceGateway
 from howtrader.gateway.binances.binances_gateway import BinancesRestApi
 from howtrader.trader.object import Exchange, Interval
 from tzlocal import get_localzone
 from howtrader.trader.object import HistoryRequest
 from howtrader.trader.database import database_manager
 from threading import Thread
 
-
 SETTINGS["log.active"] = True
 SETTINGS["log.level"] = INFO
 SETTINGS["log.console"] = True
 
-
 binances_setting = {
-        "key": "",
-        "secret": "",
-        "会话数": 3,
-        "服务器": "REAL",
-        "合约模式": "正向",
-        "代理地址": "",
-        "代理端口": 0,
-    }
+    "key": "",
+    "secret": "",
+    "会话数": 3,
+    "服务器": "REAL",
+    "合约模式": "正向",
+    "代理地址": "",
+    "代理端口": 0,
+}
+
+binance_setting = {
+    "key": "",
+    "secret": "",
+    "session_number": 3,
+    "proxy_host": "",
+    "proxy_port": 0
+}
 
 
 def request1():
-    start = datetime(2020, 11, 13, tzinfo=get_localzone())
-    end = datetime(2020, 11, 14, tzinfo=get_localzone())
+    start = datetime(2020, 10, 1, tzinfo=get_localzone())
+    end = datetime(2020, 10, 2, tzinfo=get_localzone())
     req = HistoryRequest(
         symbol=symbol,
         exchange=exchange,
@@ -45,10 +52,14 @@ def request1():
 
     data = gate_way.query_history(req)
 
+    print(data)
+
     if data:
         database_manager.save_bar_data(data)
 
+
 def request2():
+    print("start2")
     start = datetime(2020, 11, 12, tzinfo=get_localzone())
     end = datetime(2020, 11, 13, tzinfo=get_localzone())
     req = HistoryRequest(
@@ -60,39 +71,40 @@ def request2():
     )
 
     data = gate_way.query_history(req)
-    if data:
-        database_manager.save_bar_data(data)
+    print("start12_end13", data)
 
 
 if __name__ == "__main__":
-
+    """
+        for crawling data from Binance exchange.
+    """
     SETTINGS["log.file"] = True
 
     event_engine = EventEngine()
     main_engine = MainEngine(event_engine)
-    main_engine.add_gateway(BinancesGateway)
-
-    main_engine.connect(binances_setting, "BINANCES")
+    main_engine.add_gateway(BinanceGateway)  # spot
+    # main_engine.add_gateway(BinancesGateway)  # future
+    main_engine.connect(binance_setting, "BINANCE")  # spot
+    # main_engine.connect(binances_setting, "BINANCES")  # future
 
     sleep(3)
 
-    main_engine.write_log("连接BINANCES接口")
-
-    gate_way = main_engine.get_gateway("BINANCES")
-
+    main_engine.write_log("连接BINANCE接口")  # spot
+    # main_engine.write_log("连接BINANCES接口") # future
+    gate_way = main_engine.get_gateway("BINANCE")  # spot
+    # gate_way = main_engine.get_gateway("BINANCES")  # future
     print(gate_way)
 
-    symbol = "BTCUSDT"
+    symbol = "btcusdt"  # spot for lower case while the future will be upper case.
 
     exchange = Exchange.BINANCE  # binance.
-
     interval = Interval.MINUTE  # minute
 
     t1 = Thread(target=request1)
-    t2 = Thread(target=request2)
+    # t2 = Thread(target=request2)
 
     t1.start()
-    t2.start()
+    # t2.start()
 
 
 

examples/download_data_demo2.py

@@ -0,0 +1,169 @@
+"""
+we use the ccxt to crawl data then save it to csv file.
+you need to install ccxt by running firstly:
+
+"""
+
+import pandas as pd
+import time
+from datetime import datetime
+import requests
+import pytz
+from howtrader.trader.database import database_manager
+
+pd.set_option('expand_frame_repr', False)  #
+from howtrader.trader.object import BarData, Interval, Exchange
+
+BINANCE_SPOT_LIMIT = 1000
+BINANCE_FUTURE_LIMIT = 1500
+
+CHINA_TZ = pytz.timezone("Asia/Shanghai")
+from threading import Thread
+
+
+def generate_datetime(timestamp: float) -> datetime:
+    """
+    :param timestamp:
+    :return:
+    """
+    dt = datetime.fromtimestamp(timestamp / 1000)
+    dt = CHINA_TZ.localize(dt)
+    return dt
+
+
+def get_binance_data(symbol: str, exchanges: str, start_time: str, end_time: str):
+    """
+    爬取币安交易所的数据
+    :param symbol: BTCUSDT.
+    :param exchanges: 现货、USDT合约, 或者币币合约.
+    :param start_time: 格式如下:2020-1-1 或者2020-01-01
+    :param end_time: 格式如下:2020-1-1 或者2020-01-01
+    :return:
+    """
+
+    api_url = ''
+    save_symbol = symbol
+    gate_way = 'BINANCES'
+
+    if exchanges == 'spot':
+        print("spot")
+        limit = BINANCE_SPOT_LIMIT
+        save_symbol = symbol.lower()
+        gate_way = 'BINANCE'
+        api_url = f'https://api.binance.com/api/v3/klines?symbol={symbol}&interval=1m&limit={limit}'
+
+    elif exchanges == 'future':
+        print('future')
+        limit = BINANCE_FUTURE_LIMIT
+        api_url = f'https://fapi.binance.com/fapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
+
+    elif exchanges == 'coin_future':
+        print("coin_future")
+        limit = BINANCE_FUTURE_LIMIT
+        f'https://dapi.binance.com/dapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
+        pass
+
+    else:
+        raise Exception('交易所名称请输入以下其中一个：spot, future, coin_future')
+
+    start_time = int(datetime.strptime(start_time, '%Y-%m-%d').timestamp() * 1000)
+    end_time = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp() * 1000)
+
+    while True:
+        try:
+            print(start_time)
+            url = f'{api_url}&startTime={start_time}'
+            print(url)
+
+            data = requests.get(url=url).json()
+
+            """
+            [
+                [
+                    1591258320000,      // 开盘时间
+                    "9640.7",           // 开盘价
+                    "9642.4",           // 最高价
+                    "9640.6",           // 最低价
+                    "9642.0",           // 收盘价(当前K线未结束的即为最新价)
+                    "206",              // 成交量
+                    1591258379999,      // 收盘时间
+                    "2.13660389",       // 成交额(标的数量)
+                    48,                 // 成交笔数
+                    "119",              // 主动买入成交量
+                    "1.23424865",      // 主动买入成交额(标的数量)
+                    "0"                 // 请忽略该参数
+                ]
+
+            """
+
+            buf = []
+
+            for l in data:
+                bar = BarData(
+                    symbol=save_symbol,
+                    exchange=Exchange.BINANCE,
+                    datetime=generate_datetime(l[0]),
+                    interval=Interval.MINUTE,
+                    volume=float(l[5]),
+                    open_price=float(l[1]),
+                    high_price=float(l[2]),
+                    low_price=float(l[3]),
+                    close_price=float(l[4]),
+                    gateway_name=gate_way
+                )
+                buf.append(bar)
+
+            database_manager.save_bar_data(buf)
+
+            # 到结束时间就退出, 后者收盘价大于当前的时间.
+            if (data[-1][0] > end_time) or data[-1][6] >= (int(time.time() * 1000) - 60 * 1000):
+                break
+
+            start_time = data[-1][0]
+
+        except Exception as error:
+            print(error)
+            time.sleep(10)
+
+
+def download_spot():
+    """
+    下载现货数据的方法.
+    :return:
+    """
+    t1 = Thread(target=get_binance_data, args=('BTCUSDT', 'spot', "2018-1-1", "2019-1-1"))
+
+    t2 = Thread(target=get_binance_data, args=('BTCUSDT', 'spot', "2019-1-1", "2020-1-1"))
+
+    t3 = Thread(target=get_binance_data, args=('BTCUSDT', 'spot', "2020-1-1", "2020-11-16"))
+
+    t1.start()
+    t2.start()
+    t3.start()
+
+    t1.join()
+    t2.join()
+    t3.join()
+
+
+def download_future():
+    """
+    下载合约数据的方法。
+    :return:
+    """
+    t1 = Thread(target=get_binance_data, args=('BTCUSDT', 'future', "2019-9-10", "2020-3-1"))
+    t2 = Thread(target=get_binance_data, args=('BTCUSDT', 'future', "2019-3-1", "2020-11-16"))
+
+    t1.start()
+    t2.start()
+
+    t1.join()
+    t2.join()
+
+
+if __name__ == '__main__':
+    # download_spot() # 下载现货的数据.
+
+    download_future()  # 下载合约的数据
+
+