From 0b83f569f3b98623ebe6f0a1e8da8f979d198b4f Mon Sep 17 00:00:00 2001 From: foolcage <5533061@qq.com> Date: Mon, 13 Jan 2020 22:53:34 +0800 Subject: [PATCH] refactor Former-commit-id: 03d0d8d64570e2dfd887ca41c3b75a196b4cb490 --- .../common/test_china_stock_list_recorder.py | 2 +- zvt/recorders/__init__.py | 6 +- zvt/recorders/common/__init__.py | 5 - zvt/recorders/common/init_data.py | 120 ----------------- zvt/recorders/eastmoney/meta_runner.py | 2 +- zvt/recorders/exchange/__init__.py | 122 ++++++++++++++++++ .../china_etf_list_spider.py | 0 .../china_index_list_spider.py | 0 .../china_stock_list_spider.py | 0 .../meta/china_stock_etf_recorder.py | 4 +- .../sina_china_stock_category_recorder.py | 2 +- zvt/recorders/sina/money_flow/__init__.py | 2 +- ...r.py => sina_block_money_flow_recorder.py} | 2 +- .../sina_stock_money_flow_recorder.py | 2 +- 14 files changed, 131 insertions(+), 138 deletions(-) delete mode 100644 zvt/recorders/common/__init__.py delete mode 100644 zvt/recorders/common/init_data.py rename zvt/recorders/{common => exchange}/china_etf_list_spider.py (100%) rename zvt/recorders/{common => exchange}/china_index_list_spider.py (100%) rename zvt/recorders/{common => exchange}/china_stock_list_spider.py (100%) rename zvt/recorders/sina/money_flow/{sina_index_money_flow_recorder.py => sina_block_money_flow_recorder.py} (98%) diff --git a/tests/recorders/common/test_china_stock_list_recorder.py b/tests/recorders/common/test_china_stock_list_recorder.py index ecc54cb4..293c03a0 100644 --- a/tests/recorders/common/test_china_stock_list_recorder.py +++ b/tests/recorders/common/test_china_stock_list_recorder.py @@ -3,7 +3,7 @@ init_test_context() -from zvt.recorders.common.china_stock_list_spider import ChinaStockListRecorder +from zvt.recorders.exchange.china_stock_list_spider import ChinaStockListRecorder def test_china_stock_recorder(): diff --git a/zvt/recorders/__init__.py b/zvt/recorders/__init__.py index 3bc14503..d1f19cd1 100644 --- a/zvt/recorders/__init__.py +++ b/zvt/recorders/__init__.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- -from zvt.recorders.common.init_data import init_main_index - -init_main_index() - -from zvt.recorders.common import * from zvt.recorders.ccxt import * from zvt.recorders.eastmoney import * +from zvt.recorders.exchange import * from zvt.recorders.joinquant import * from zvt.recorders.sina import * diff --git a/zvt/recorders/common/__init__.py b/zvt/recorders/common/__init__.py deleted file mode 100644 index 4f8ca4cf..00000000 --- a/zvt/recorders/common/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# -*- coding: utf-8 -*- - -from zvt.recorders.common.china_etf_list_spider import * -from zvt.recorders.common.china_index_list_spider import * -from zvt.recorders.common.china_stock_list_spider import * diff --git a/zvt/recorders/common/init_data.py b/zvt/recorders/common/init_data.py deleted file mode 100644 index 06cff3c1..00000000 --- a/zvt/recorders/common/init_data.py +++ /dev/null @@ -1,120 +0,0 @@ -import pandas as pd - -from zvdata.api import persist_entities -from zvdata.utils.time_utils import to_pd_timestamp - -CHINA_STOCK_MAIN_INDEX = [{'id': 'index_cn_000001', - 'entity_id': 'index_cn_000001', - 'code': '000001', - 'name': '上证指数', - 'timestamp': '1990-12-19', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_000016', - 'entity_id': 'index_cn_000016', - 'code': '000016', - 'name': '上证50', - 'timestamp': '2004-01-02', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_000905', - 'entity_id': 'index_cn_000905', - 'code': '000905', - 'name': '中证500', - 'timestamp': '2005-01-04', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_399001', - 'entity_id': 'index_cn_399001', - 'code': '399001', - 'name': '深证成指', - 'timestamp': '1991-04-03', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_399106', - 'entity_id': 'index_cn_399106', - 'code': '399106', - 'name': '深证综指', - 'timestamp': '1991-04-03', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_399300', - 'entity_id': 'index_cn_399300', - 'code': '399300', - 'name': '沪深300', - 'timestamp': '2002-01-04', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_399005', - 'entity_id': 'index_cn_399005', - 'code': '399005', - 'name': '中小板指', - 'timestamp': '2006-01-24', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_399006', - 'entity_id': 'index_cn_399006', - 'code': '399006', - 'name': '创业板指', - 'timestamp': '2010-06-01', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - # # 聚宽编码 - # # 市场通编码 市场通名称 - # # 310001 沪股通 - # # 310002 深股通 - # # 310003 港股通（沪） - # # 310004 港股通（深） - {'id': 'index_cn_310001', - 'entity_id': 'index_cn_310001', - 'code': '310001', - 'name': '沪股通', - 'timestamp': '2014-11-17', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_310002', - 'entity_id': 'index_cn_310002', - 'code': '310002', - 'name': '深股通', - 'timestamp': '2014-11-17', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_310003', - 'entity_id': 'index_cn_310003', - 'code': '310003', - 'name': '港股通（沪）', - 'timestamp': '2014-11-17', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'}, - {'id': 'index_cn_310004', - 'entity_id': 'index_cn_310004', - 'code': '310004', - 'name': '港股通（深）', - 'timestamp': '2014-11-17', - 'exchange': 'cn', - 'entity_type': 'index', - 'category': 'main'} - ] - - -def init_main_index(provider='exchange'): - for item in CHINA_STOCK_MAIN_INDEX: - item['timestamp'] = to_pd_timestamp(item['timestamp']) - df = pd.DataFrame(CHINA_STOCK_MAIN_INDEX) - # print(df) - persist_entities(df, entity_type='index', provider=provider) - - -if __name__ == '__main__': - init_main_index() diff --git a/zvt/recorders/eastmoney/meta_runner.py b/zvt/recorders/eastmoney/meta_runner.py index ff505a22..bd7cc3ec 100644 --- a/zvt/recorders/eastmoney/meta_runner.py +++ b/zvt/recorders/eastmoney/meta_runner.py @@ -6,7 +6,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from zvt import init_log -from zvt.recorders.common.china_stock_list_spider import ChinaStockListRecorder +from zvt.recorders.exchange.china_stock_list_spider import ChinaStockListRecorder from zvt.recorders.eastmoney.meta.china_stock_category_recorder import ChinaStockCategoryRecorder from zvt.recorders.eastmoney.meta.china_stock_meta_recorder import ChinaStockMetaRecorder diff --git a/zvt/recorders/exchange/__init__.py b/zvt/recorders/exchange/__init__.py index e69de29b..91e736b4 100644 --- a/zvt/recorders/exchange/__init__.py +++ b/zvt/recorders/exchange/__init__.py @@ -0,0 +1,122 @@ +import pandas as pd + +from zvdata.api import persist_entities +from zvdata.utils.time_utils import to_pd_timestamp +from zvt.recorders.exchange.china_etf_list_spider import * +from zvt.recorders.exchange.china_index_list_spider import * +from zvt.recorders.exchange.china_stock_list_spider import * + +CHINA_STOCK_MAIN_INDEX = [{'id': 'index_cn_000001', + 'entity_id': 'index_cn_000001', + 'code': '000001', + 'name': '上证指数', + 'timestamp': '1990-12-19', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_000016', + 'entity_id': 'index_cn_000016', + 'code': '000016', + 'name': '上证50', + 'timestamp': '2004-01-02', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_000905', + 'entity_id': 'index_cn_000905', + 'code': '000905', + 'name': '中证500', + 'timestamp': '2005-01-04', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_399001', + 'entity_id': 'index_cn_399001', + 'code': '399001', + 'name': '深证成指', + 'timestamp': '1991-04-03', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_399106', + 'entity_id': 'index_cn_399106', + 'code': '399106', + 'name': '深证综指', + 'timestamp': '1991-04-03', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_399300', + 'entity_id': 'index_cn_399300', + 'code': '399300', + 'name': '沪深300', + 'timestamp': '2002-01-04', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_399005', + 'entity_id': 'index_cn_399005', + 'code': '399005', + 'name': '中小板指', + 'timestamp': '2006-01-24', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_399006', + 'entity_id': 'index_cn_399006', + 'code': '399006', + 'name': '创业板指', + 'timestamp': '2010-06-01', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + # # 聚宽编码 + # # 市场通编码 市场通名称 + # # 310001 沪股通 + # # 310002 深股通 + # # 310003 港股通（沪） + # # 310004 港股通（深） + {'id': 'index_cn_310001', + 'entity_id': 'index_cn_310001', + 'code': '310001', + 'name': '沪股通', + 'timestamp': '2014-11-17', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_310002', + 'entity_id': 'index_cn_310002', + 'code': '310002', + 'name': '深股通', + 'timestamp': '2014-11-17', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_310003', + 'entity_id': 'index_cn_310003', + 'code': '310003', + 'name': '港股通（沪）', + 'timestamp': '2014-11-17', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'}, + {'id': 'index_cn_310004', + 'entity_id': 'index_cn_310004', + 'code': '310004', + 'name': '港股通（深）', + 'timestamp': '2014-11-17', + 'exchange': 'cn', + 'entity_type': 'index', + 'category': 'main'} + ] + + +def init_main_index(provider='exchange'): + for item in CHINA_STOCK_MAIN_INDEX: + item['timestamp'] = to_pd_timestamp(item['timestamp']) + df = pd.DataFrame(CHINA_STOCK_MAIN_INDEX) + # print(df) + persist_entities(df, entity_type='index', provider=provider) + + +init_main_index() diff --git a/zvt/recorders/common/china_etf_list_spider.py b/zvt/recorders/exchange/china_etf_list_spider.py similarity index 100% rename from zvt/recorders/common/china_etf_list_spider.py rename to zvt/recorders/exchange/china_etf_list_spider.py diff --git a/zvt/recorders/common/china_index_list_spider.py b/zvt/recorders/exchange/china_index_list_spider.py similarity index 100% rename from zvt/recorders/common/china_index_list_spider.py rename to zvt/recorders/exchange/china_index_list_spider.py diff --git a/zvt/recorders/common/china_stock_list_spider.py b/zvt/recorders/exchange/china_stock_list_spider.py similarity index 100% rename from zvt/recorders/common/china_stock_list_spider.py rename to zvt/recorders/exchange/china_stock_list_spider.py diff --git a/zvt/recorders/joinquant/meta/china_stock_etf_recorder.py b/zvt/recorders/joinquant/meta/china_stock_etf_recorder.py index 25f3021d..02118c5e 100644 --- a/zvt/recorders/joinquant/meta/china_stock_etf_recorder.py +++ b/zvt/recorders/joinquant/meta/china_stock_etf_recorder.py @@ -116,5 +116,5 @@ def record(self, entity, start, end, size, timestamps): __all__ = ['JqChinaStockMetaRecorder', 'JqChinaStockEtfPortfolioRecorder'] if __name__ == '__main__': - # JqChinaStockMetaRecorder().run() - JqChinaStockEtfPortfolioRecorder(codes=SAMPLE_ETF_CODES).run() + JqChinaStockMetaRecorder().run() + # JqChinaStockEtfPortfolioRecorder(codes=SAMPLE_ETF_CODES).run() diff --git a/zvt/recorders/sina/meta/sina_china_stock_category_recorder.py b/zvt/recorders/sina/meta/sina_china_stock_category_recorder.py index d63c0a84..215268a9 100644 --- a/zvt/recorders/sina/meta/sina_china_stock_category_recorder.py +++ b/zvt/recorders/sina/meta/sina_china_stock_category_recorder.py @@ -86,7 +86,7 @@ def run(self): df = pd.DataFrame.from_records(the_list) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider) - self.logger.info('finish recording index:{},{}'.format(block.category, block.name)) + self.logger.info('finish recording BlockStock:{},{}'.format(block.category, block.name)) except Exception as e: self.logger.error("error:,resp.text:", e, resp.text) diff --git a/zvt/recorders/sina/money_flow/__init__.py b/zvt/recorders/sina/money_flow/__init__.py index e7320dfd..893bcdfa 100644 --- a/zvt/recorders/sina/money_flow/__init__.py +++ b/zvt/recorders/sina/money_flow/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from .sina_index_money_flow_recorder import SinaBlockMoneyFlowRecorder +from .sina_block_money_flow_recorder import SinaBlockMoneyFlowRecorder from .sina_stock_money_flow_recorder import SinaStockMoneyFlowRecorder __all__ = ['SinaBlockMoneyFlowRecorder', 'SinaStockMoneyFlowRecorder'] diff --git a/zvt/recorders/sina/money_flow/sina_index_money_flow_recorder.py b/zvt/recorders/sina/money_flow/sina_block_money_flow_recorder.py similarity index 98% rename from zvt/recorders/sina/money_flow/sina_index_money_flow_recorder.py rename to zvt/recorders/sina/money_flow/sina_block_money_flow_recorder.py index c7a381c7..a0394fa2 100644 --- a/zvt/recorders/sina/money_flow/sina_index_money_flow_recorder.py +++ b/zvt/recorders/sina/money_flow/sina_block_money_flow_recorder.py @@ -105,5 +105,5 @@ def record(self, entity, start, end, size, timestamps): if __name__ == '__main__': - SinaBlockMoneyFlowRecorder(codes=['new_blhy']).run() + SinaBlockMoneyFlowRecorder(codes=['gn_5Ggn']).run() # SinaIndexMoneyFlowRecorder().run() diff --git a/zvt/recorders/sina/money_flow/sina_stock_money_flow_recorder.py b/zvt/recorders/sina/money_flow/sina_stock_money_flow_recorder.py index 9302b3fc..c7164afc 100644 --- a/zvt/recorders/sina/money_flow/sina_stock_money_flow_recorder.py +++ b/zvt/recorders/sina/money_flow/sina_stock_money_flow_recorder.py @@ -11,7 +11,7 @@ class SinaStockMoneyFlowRecorder(FixedCycleDataRecorder): - entity_provider = 'sina' + entity_provider = 'joinquant' entity_schema = Stock provider = 'sina'