refactor

Former-commit-id: 03d0d8d
fyenne · Jan 13, 2020 · 0b83f56 · 0b83f56
1 parent 1c168c8
commit 0b83f56
Show file tree

Hide file tree

Showing 14 changed files with 131 additions and 138 deletions.
diff --git a/tests/recorders/common/test_china_stock_list_recorder.py b/tests/recorders/common/test_china_stock_list_recorder.py
@@ -3,7 +3,7 @@
 
 init_test_context()
 
-from zvt.recorders.common.china_stock_list_spider import ChinaStockListRecorder
+from zvt.recorders.exchange.china_stock_list_spider import ChinaStockListRecorder
 
 
 def test_china_stock_recorder():

diff --git a/zvt/recorders/__init__.py b/zvt/recorders/__init__.py
@@ -1,10 +1,6 @@
 # -*- coding: utf-8 -*-
-from zvt.recorders.common.init_data import init_main_index
-
-init_main_index()
-
-from zvt.recorders.common import *
 from zvt.recorders.ccxt import *
 from zvt.recorders.eastmoney import *
+from zvt.recorders.exchange import *
 from zvt.recorders.joinquant import *
 from zvt.recorders.sina import *
diff --git a/zvt/recorders/common/__init__.py b/zvt/recorders/common/__init__.py
diff --git a/zvt/recorders/common/init_data.py b/zvt/recorders/common/init_data.py
diff --git a/zvt/recorders/eastmoney/meta_runner.py b/zvt/recorders/eastmoney/meta_runner.py
@@ -6,7 +6,7 @@
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from zvt import init_log
-from zvt.recorders.common.china_stock_list_spider import ChinaStockListRecorder
+from zvt.recorders.exchange.china_stock_list_spider import ChinaStockListRecorder
 from zvt.recorders.eastmoney.meta.china_stock_category_recorder import ChinaStockCategoryRecorder
 from zvt.recorders.eastmoney.meta.china_stock_meta_recorder import ChinaStockMetaRecorder
 

diff --git a/zvt/recorders/exchange/__init__.py b/zvt/recorders/exchange/__init__.py
@@ -0,0 +1,122 @@
+import pandas as pd
+
+from zvdata.api import persist_entities
+from zvdata.utils.time_utils import to_pd_timestamp
+from zvt.recorders.exchange.china_etf_list_spider import *
+from zvt.recorders.exchange.china_index_list_spider import *
+from zvt.recorders.exchange.china_stock_list_spider import *
+
+CHINA_STOCK_MAIN_INDEX = [{'id': 'index_cn_000001',
+                           'entity_id': 'index_cn_000001',
+                           'code': '000001',
+                           'name': '上证指数',
+                           'timestamp': '1990-12-19',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_000016',
+                           'entity_id': 'index_cn_000016',
+                           'code': '000016',
+                           'name': '上证50',
+                           'timestamp': '2004-01-02',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_000905',
+                           'entity_id': 'index_cn_000905',
+                           'code': '000905',
+                           'name': '中证500',
+                           'timestamp': '2005-01-04',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_399001',
+                           'entity_id': 'index_cn_399001',
+                           'code': '399001',
+                           'name': '深证成指',
+                           'timestamp': '1991-04-03',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_399106',
+                           'entity_id': 'index_cn_399106',
+                           'code': '399106',
+                           'name': '深证综指',
+                           'timestamp': '1991-04-03',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_399300',
+                           'entity_id': 'index_cn_399300',
+                           'code': '399300',
+                           'name': '沪深300',
+                           'timestamp': '2002-01-04',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_399005',
+                           'entity_id': 'index_cn_399005',
+                           'code': '399005',
+                           'name': '中小板指',
+                           'timestamp': '2006-01-24',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_399006',
+                           'entity_id': 'index_cn_399006',
+                           'code': '399006',
+                           'name': '创业板指',
+                           'timestamp': '2010-06-01',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          # # 聚宽编码
+                          # # 市场通编码	市场通名称
+                          # # 310001	沪股通
+                          # # 310002	深股通
+                          # # 310003	港股通（沪）
+                          # # 310004	港股通（深）
+                          {'id': 'index_cn_310001',
+                           'entity_id': 'index_cn_310001',
+                           'code': '310001',
+                           'name': '沪股通',
+                           'timestamp': '2014-11-17',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_310002',
+                           'entity_id': 'index_cn_310002',
+                           'code': '310002',
+                           'name': '深股通',
+                           'timestamp': '2014-11-17',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_310003',
+                           'entity_id': 'index_cn_310003',
+                           'code': '310003',
+                           'name': '港股通（沪）',
+                           'timestamp': '2014-11-17',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'},
+                          {'id': 'index_cn_310004',
+                           'entity_id': 'index_cn_310004',
+                           'code': '310004',
+                           'name': '港股通（深）',
+                           'timestamp': '2014-11-17',
+                           'exchange': 'cn',
+                           'entity_type': 'index',
+                           'category': 'main'}
+                          ]
+
+
+def init_main_index(provider='exchange'):
+    for item in CHINA_STOCK_MAIN_INDEX:
+        item['timestamp'] = to_pd_timestamp(item['timestamp'])
+    df = pd.DataFrame(CHINA_STOCK_MAIN_INDEX)
+    # print(df)
+    persist_entities(df, entity_type='index', provider=provider)
+
+
+init_main_index()
diff --git a/...recorders/common/china_etf_list_spider.py → ...corders/exchange/china_etf_list_spider.py b/...recorders/common/china_etf_list_spider.py → ...corders/exchange/china_etf_list_spider.py
diff --git a/...corders/common/china_index_list_spider.py → ...rders/exchange/china_index_list_spider.py b/...corders/common/china_index_list_spider.py → ...rders/exchange/china_index_list_spider.py
diff --git a/...corders/common/china_stock_list_spider.py → ...rders/exchange/china_stock_list_spider.py b/...corders/common/china_stock_list_spider.py → ...rders/exchange/china_stock_list_spider.py
diff --git a/zvt/recorders/joinquant/meta/china_stock_etf_recorder.py b/zvt/recorders/joinquant/meta/china_stock_etf_recorder.py
@@ -116,5 +116,5 @@ def record(self, entity, start, end, size, timestamps):
 __all__ = ['JqChinaStockMetaRecorder', 'JqChinaStockEtfPortfolioRecorder']
 
 if __name__ == '__main__':
-    # JqChinaStockMetaRecorder().run()
-    JqChinaStockEtfPortfolioRecorder(codes=SAMPLE_ETF_CODES).run()
+    JqChinaStockMetaRecorder().run()
+    # JqChinaStockEtfPortfolioRecorder(codes=SAMPLE_ETF_CODES).run()
diff --git a/zvt/recorders/sina/meta/sina_china_stock_category_recorder.py b/zvt/recorders/sina/meta/sina_china_stock_category_recorder.py
@@ -86,7 +86,7 @@ def run(self):
                         df = pd.DataFrame.from_records(the_list)
                         df_to_db(data_schema=self.data_schema, df=df, provider=self.provider)
 
-                    self.logger.info('finish recording index:{},{}'.format(block.category, block.name))
+                    self.logger.info('finish recording BlockStock:{},{}'.format(block.category, block.name))
 
                 except Exception as e:
                     self.logger.error("error:,resp.text:", e, resp.text)

diff --git a/zvt/recorders/sina/money_flow/__init__.py b/zvt/recorders/sina/money_flow/__init__.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from .sina_index_money_flow_recorder import SinaBlockMoneyFlowRecorder
+from .sina_block_money_flow_recorder import SinaBlockMoneyFlowRecorder
 from .sina_stock_money_flow_recorder import SinaStockMoneyFlowRecorder
 
 __all__ = ['SinaBlockMoneyFlowRecorder', 'SinaStockMoneyFlowRecorder']
diff --git a/...ey_flow/sina_index_money_flow_recorder.py → ...ey_flow/sina_block_money_flow_recorder.py b/...ey_flow/sina_index_money_flow_recorder.py → ...ey_flow/sina_block_money_flow_recorder.py
@@ -105,5 +105,5 @@ def record(self, entity, start, end, size, timestamps):
 
 
 if __name__ == '__main__':
-    SinaBlockMoneyFlowRecorder(codes=['new_blhy']).run()
+    SinaBlockMoneyFlowRecorder(codes=['gn_5Ggn']).run()
     # SinaIndexMoneyFlowRecorder().run()
diff --git a/zvt/recorders/sina/money_flow/sina_stock_money_flow_recorder.py b/zvt/recorders/sina/money_flow/sina_stock_money_flow_recorder.py
@@ -11,7 +11,7 @@
 
 
 class SinaStockMoneyFlowRecorder(FixedCycleDataRecorder):
-    entity_provider = 'sina'
+    entity_provider = 'joinquant'
     entity_schema = Stock
 
     provider = 'sina'