config for plugin

docs/data_extending.md

@@ -190,7 +190,7 @@ from jqdatasdk import auth, logout, query, valuation, get_fundamentals_continuou
 from zvdata.api import df_to_db
 from zvdata.recorder import TimeSeriesDataRecorder
 from zvdata.utils.time_utils import now_pd_timestamp, now_time_str, to_time_str
-from zvt import zvt_env
+from zvt import zvt_config
 from zvt.domain import Stock, StockValuation, EtfStock
 from zvt.recorders.joinquant.common import to_jq_entity_id
 
@@ -210,7 +210,7 @@ class JqChinaStockValuationRecorder(TimeSeriesDataRecorder):
         super().__init__(entity_type, exchanges, entity_ids, codes, batch_size, force_update, sleeping_time,
                          default_size, real_time, fix_duplicate_way, start_timestamp, end_timestamp, close_hour,
                          close_minute)
-        auth(zvt_env['jq_username'], zvt_env['jq_password'])
+        auth(zvt_config['jq_username'], zvt_config['jq_password'])
 
     def on_finish(self):
         super().on_finish()

zvt/__init__.py

@@ -1,11 +1,15 @@
 # -*- coding: utf-8 -*-
 import enum
+import importlib
 import json
 import logging
 import os
+import pkgutil
+import pprint
 from logging.handlers import RotatingFileHandler
 
 import pandas as pd
+import pkg_resources
 from pkg_resources import get_distribution, DistributionNotFound
 
 from zvt.settings import DATA_SAMPLE_ZIP_PATH, ZVT_TEST_HOME, ZVT_HOME, ZVT_TEST_DATA_PATH, ZVT_TEST_ZIP_DATA_PATH
@@ -18,6 +22,8 @@
 finally:
     del get_distribution, DistributionNotFound
 
+logger = logging.getLogger(__name__)
+
 
 # common class
 class IntervalLevel(enum.Enum):
@@ -138,12 +144,12 @@ def init_log(file_name='zvt.log', log_dir=None, simple_formatter=True):
 
     file_name = os.path.join(log_dir, file_name)
 
-    fh = RotatingFileHandler(file_name, maxBytes=524288000, backupCount=10)
+    file_log_handler = RotatingFileHandler(file_name, maxBytes=524288000, backupCount=10)
 
-    fh.setLevel(logging.INFO)
+    file_log_handler.setLevel(logging.INFO)
 
-    ch = logging.StreamHandler()
-    ch.setLevel(logging.INFO)
+    console_log_handler = logging.StreamHandler()
+    console_log_handler.setLevel(logging.INFO)
 
     # create formatter and add it to the handlers
     if simple_formatter:
@@ -152,22 +158,27 @@ def init_log(file_name='zvt.log', log_dir=None, simple_formatter=True):
     else:
         formatter = logging.Formatter(
             "%(asctime)s  %(levelname)s  %(threadName)s  %(name)s:%(filename)s:%(lineno)s  %(funcName)s  %(message)s")
-    fh.setFormatter(formatter)
-    ch.setFormatter(formatter)
+    file_log_handler.setFormatter(formatter)
+    console_log_handler.setFormatter(formatter)
 
     # add the handlers to the logger
-    root_logger.addHandler(fh)
-    root_logger.addHandler(ch)
+    root_logger.addHandler(file_log_handler)
+    root_logger.addHandler(console_log_handler)
 
 
 pd.set_option('expand_frame_repr', False)
 pd.set_option('mode.chained_assignment', 'raise')
 
 zvt_env = {}
 
+zvt_config = {}
+
+_plugins = {}
+
 
-def init_env(zvt_home: str) -> None:
+def init_env(zvt_home: str, **kwargs) -> dict:
     """
+    init env
 
     :param zvt_home: home path for zvt
     """
@@ -193,21 +204,68 @@ def init_env(zvt_home: str) -> None:
     if not os.path.exists(zvt_env['log_path']):
         os.makedirs(zvt_env['log_path'])
 
+    init_log()
+
+    pprint.pprint(zvt_env)
+
+    # init config
+    init_config(current_config=zvt_config, **kwargs)
+    # init plugin
+    # init_plugins()
+
+    return zvt_env
+
+
+def init_config(pkg_name: str = None, current_config: dict = None, **kwargs) -> dict:
+    """
+    init config
+    """
+
     # create default config.json if not exist
-    config_path = os.path.join(zvt_home, 'config.json')
+    if pkg_name:
+        config_file = f'{pkg_name}_config.json'
+    else:
+        pkg_name = 'zvt'
+        config_file = 'config.json'
+
+    logger.info(f'init config for {pkg_name}, current_config:{current_config}')
+
+    config_path = os.path.join(zvt_env['zvt_home'], config_file)
     if not os.path.exists(config_path):
         from shutil import copyfile
-        copyfile(os.path.abspath(os.path.join(os.path.dirname(__file__), 'samples', 'config.json')), config_path)
+        try:
+            sample_config = pkg_resources.resource_filename(pkg_name, 'config.json')
+            if os.path.exists(sample_config):
+                copyfile(sample_config, config_path)
+        except Exception as e:
+            logger.warning(f'could not load config.json from package {pkg_name}')
 
-    with open(config_path) as f:
-        config_json = json.load(f)
-        for k in config_json:
-            zvt_env[k] = config_json[k]
+    if os.path.exists(config_path):
+        with open(config_path) as f:
+            config_json = json.load(f)
+            for k in config_json:
+                current_config[k] = config_json[k]
 
-    init_log()
+    # set and save the config
+    for k in kwargs:
+        current_config[k] = kwargs[k]
+        with open(config_path, 'w+') as outfile:
+            json.dump(current_config, outfile)
 
-    import pprint
-    pprint.pprint(zvt_env)
+    pprint.pprint(current_config)
+    logger.info(f'current_config:{current_config}')
+
+    return current_config
+
+
+def init_plugins():
+    for finder, name, ispkg in pkgutil.iter_modules():
+        if name.startswith('zvt_'):
+            try:
+                _plugins[name] = importlib.import_module(name)
+            except Exception as e:
+                logger.warning(f'failed to load plugin {name}', e)
+    logger.info(f'loaded plugins:{_plugins}')
 
 
 if os.getenv('TESTING_ZVT'):
@@ -233,4 +291,4 @@ def init_env(zvt_home: str) -> None:
 # import the recorders for register them to the domain
 import zvt.recorders as zvt_recorders
 
-__all__ = ['zvt_env', 'init_log', 'init_env', 'IntervalLevel', '__version__', 'AdjustType']
+__all__ = ['zvt_env', 'zvt_config', 'init_log', 'init_env', 'init_config', 'IntervalLevel', '__version__', 'AdjustType']

zvt/autocode/generator.py

@@ -76,7 +76,7 @@ def fill_package(dir_path: str):
         if os.path.isdir(dir_path):
             pkg_file = os.path.join(dir_path, '__init__.py')
             if not os.path.exists(pkg_file):
-                package_template = '# -*- coding: utf-8 -*-'
+                package_template = '# -*- coding: utf-8 -*-\n'
                 with open(pkg_file, 'w') as outfile:
                     outfile.write(package_template)
 
@@ -238,4 +238,5 @@ def gen_plugin_project(entity_type, prefix: str = 'zvt', dir_path: str = '.', pr
 
 
 # the __all__ is generated
-__all__ = ['all_sub_modules', 'all_sub_all', 'fill_package_if_not_exist', 'fill_package', 'gen_exports', 'gen_kdata_schema', 'gen_plugin_project']
+__all__ = ['all_sub_modules', 'all_sub_all', 'fill_package_if_not_exist', 'fill_package', 'gen_exports',
+           'gen_kdata_schema', 'gen_plugin_project']

zvt/autocode/templates/__init__.py

@@ -40,10 +40,12 @@ def all_tpls(project: str, entity_type: str):
         # tests
         elif file_location == 'test_pass.py':
             file_location = f'tests/test_pass.py'
+        elif file_location == 'pkg_init.py':
+            file_location = f'{project}/__init__.py'
 
         tpls.append((file_location, string.Template(data)))
     return tpls
 
 
 # the __all__ is generated
-__all__ = ['all_tpls']
+__all__ = ['all_tpls']

zvt/autocode/templates/pkg_init.py.template

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+import functools
+
+from zvt import init_config
+
+${project}_config = {}
+
+int_${project}_config = functools.partial(init_config, pkg_name='${project}', current_config=${project}_config)
+
+int_${project}_config()
+
+__all__ = ['int_${project}_config']

zvt/informer/informer.py

@@ -9,7 +9,7 @@
 
 import requests
 
-from zvt import zvt_env
+from zvt import zvt_config
 
 
 class Informer(object):
@@ -25,8 +25,8 @@ def __init__(self, ssl=True) -> None:
         self.ssl = ssl
 
     def send_message_(self, to_user, title, body, **kwargs):
-        host = zvt_env['smtp_host']
-        port = zvt_env['smtp_port']
+        host = zvt_config['smtp_host']
+        port = zvt_config['smtp_port']
         if self.ssl:
             try:
                 smtp_client = smtplib.SMTP_SSL(host=host, port=port)
@@ -39,10 +39,10 @@ def send_message_(self, to_user, title, body, **kwargs):
                 smtp_client = smtplib.SMTP()
 
         smtp_client.connect(host=host, port=port)
-        smtp_client.login(zvt_env['email_username'], zvt_env['email_password'])
+        smtp_client.login(zvt_config['email_username'], zvt_config['email_password'])
         msg = MIMEMultipart('alternative')
         msg['Subject'] = Header(title).encode()
-        msg['From'] = "{} <{}>".format(Header('zvt').encode(), zvt_env['email_username'])
+        msg['From'] = "{} <{}>".format(Header('zvt').encode(), zvt_config['email_username'])
         if type(to_user) is list:
             msg['To'] = ", ".join(to_user)
         else:
@@ -54,7 +54,7 @@ def send_message_(self, to_user, title, body, **kwargs):
         msg.attach(plain_text)
 
         try:
-            smtp_client.sendmail(zvt_env['email_username'], to_user, msg.as_string())
+            smtp_client.sendmail(zvt_config['email_username'], to_user, msg.as_string())
         except Exception as e:
             self.logger.exception('send email failed', e)
 
@@ -71,15 +71,15 @@ def send_message(self, to_user, title, body, sub_size=20, with_sender=True, **kw
             for step in range(step_size):
                 sub_to_user = to_user[sub_size * step:sub_size * (step + 1)]
                 if with_sender:
-                    sub_to_user.append(zvt_env['email_username'])
+                    sub_to_user.append(zvt_config['email_username'])
                 self.send_message_(sub_to_user, title, body, **kwargs)
         else:
             self.send_message_(to_user, title, body, **kwargs)
 
 
 class WechatInformer(Informer):
     GET_TOKEN_URL = "https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid={}&secret={}".format(
-        zvt_env['wechat_app_id'], zvt_env['wechat_app_secrect'])
+        zvt_config['wechat_app_id'], zvt_config['wechat_app_secrect'])
 
     GET_TEMPLATE_URL = "https://api.weixin.qq.com/cgi-bin/template/get_all_private_template?access_token={}"
     SEND_MSG_URL = "https://api.weixin.qq.com/cgi-bin/message/template/send?access_token={}"

zvt/main.py

@@ -1,10 +1,12 @@
+from zvt import init_plugins
 from zvt.ui import zvt_app
 from zvt.ui.apps.trader_app import serve_layout
 
 zvt_app.layout = serve_layout
 
 
 def main():
+    init_plugins()
     zvt_app.run_server()
 
 

zvt/recorders/joinquant/quotes/jq_stock_kdata_recorder.py

@@ -4,7 +4,7 @@
 import pandas as pd
 from jqdatapy.api import get_token, get_bars
 
-from zvt import init_log, zvt_env
+from zvt import init_log, zvt_config
 from zvt.api.quote import generate_kdata_id, get_kdata_schema, get_kdata
 from zvt.contract import IntervalLevel, AdjustType
 from zvt.contract.api import df_to_db
@@ -53,7 +53,7 @@ def __init__(self,
                          close_minute, level, kdata_use_begin_time, one_day_trading_minutes)
         self.adjust_type = adjust_type
 
-        get_token(zvt_env['jq_username'], zvt_env['jq_password'], force=True)
+        get_token(zvt_config['jq_username'], zvt_config['jq_password'], force=True)
 
     def generate_domain_id(self, entity, original_data):
         return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)