1)Close WAL mode for most of db 2)add em api get_top_tradable_list 3)add today shoot/top runner 4)Upgrade some requirements to be compatible with Python 3.9 latter

Author: foolcage

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
   autoupdate_schedule: monthly
 repos:
   - repo: https://github.com/psf/black
-    rev: 21.11b1
+    rev: 24.8.0
     hooks:
       - id: black
 #  - repo: https://github.com/PyCQA/flake8

README-cn.md

@@ -12,6 +12,13 @@
 
 [炒股的三大原理](https://mp.weixin.qq.com/s/FoFR63wFSQIE_AyFubkZ6Q)
 
+**声明**
+
+本项目目前不保证任何向后兼容性，请谨慎升级。  
+随着作者思想的变化，一些以前觉得重要的东西可能也变得不重要，从而可能不会进行维护。  
+而一些新的东西的加入对你是否有用，需要自己去评估。
+
+
 **Read this in other languages: [English](README-cn.md).**  
 
 **详细文档:[https://zvt.readthedocs.io/en/latest/](https://zvt.readthedocs.io/en/latest/)**

README.md

@@ -12,6 +12,12 @@
 
 [The Three Major Principles of Stock Trading](https://mp.weixin.qq.com/s/FoFR63wFSQIE_AyFubkZ6Q)
 
+**Declaration**
+
+This project does not currently guarantee any backward compatibility, so please upgrade with caution.    
+As the author's thoughts evolve, some things that were once considered important may become less so, and thus may not be maintained.    
+Whether the addition of some new elements will be useful to you needs to be assessed by yourself.
+
 **Read this in other languages: [中文](README-cn.md).**  
 
 **Read the docs:[https://zvt.readthedocs.io/en/latest/](https://zvt.readthedocs.io/en/latest/)**

requirements.txt

@@ -1,16 +1,16 @@
 requests==2.31.0
-SQLAlchemy==2.0.28
+SQLAlchemy==2.0.36
 pandas==2.0.3
 pydantic==2.6.4
 arrow==1.2.3
 openpyxl==3.1.1
 demjson3==3.0.6
 plotly==5.13.0
-dash==2.8.1
+dash==2.18.2
 jqdatapy==0.1.8
 dash-bootstrap-components==1.3.1
 dash_daq==0.5.0
-scikit-learn==1.2.1
+scikit-learn==1.5.2
 fastapi==0.110.0
 fastapi-pagination==0.12.23
 apscheduler==3.10.4

src/zvt/broker/qmt/qmt_quote.py

@@ -282,7 +282,7 @@ def download_capital_data():
 def clear_history_quote():
     session = get_db_session("qmt", data_schema=StockQuote)
     session.query(StockQuote).filter(StockQuote.timestamp < current_date()).delete()
-    start_date = date_time_by_interval(current_date(), -20)
+    start_date = date_time_by_interval(current_date(), -10)
     session.query(Stock1mQuote).filter(Stock1mQuote.timestamp < start_date).delete()
     session.query(StockQuoteLog).filter(StockQuoteLog.timestamp < start_date).delete()
     session.commit()

src/zvt/contract/register.py

@@ -108,7 +108,10 @@ def register_schema(
             added_columns = [c for c in table.columns if c.name not in existing_columns]
             index_list = []
             with engine.connect() as con:
-                con.execute(text("PRAGMA journal_mode=WAL;"))
+                # FIXME: close WAL mode for saving space, most of time no need to write in multiple process
+                if db_name in ("zvt_info","stock_news","stock_tags"):
+                    con.execute(text("PRAGMA journal_mode=WAL;"))
+
                 rs = con.execute(text("PRAGMA INDEX_LIST('{}')".format(table_name)))
                 for row in rs:
                     index_list.append(row[1])

src/zvt/recorders/em/em_api.py

@@ -8,7 +8,6 @@
 import requests
 import sqlalchemy
 from requests import Session
-from sqlalchemy import func
 
 from zvt.api.kdata import generate_kdata_id
 from zvt.api.utils import value_to_pct, china_stock_code_to_id
@@ -589,6 +588,61 @@ def get_future_list():
     return df
 
 
+def get_top_tradable_list(entity_type, fields, limit, entity_flag, exchange=None, return_quote=False):
+    url = f"https://push2.eastmoney.com/api/qt/clist/get?np=1&fltt=2&invt=2&fields={fields}&pn=1&pz={limit}&fid=f3&po=1&{entity_flag}&ut=f057cbcbce2a86e2866ab8877db1d059&forcect=1&cb=cbCallbackMore&&callback=jQuery34109676853980006124_{now_timestamp() - 1}&_={now_timestamp()}"
+    resp = requests.get(url, headers=DEFAULT_HEADER)
+
+    resp.raise_for_status()
+
+    result = json_callback_param(resp.text)
+    resp.close()
+    data = result["data"]["diff"]
+    df = pd.DataFrame.from_records(data=data)
+
+    if return_quote:
+        df = df[
+            [
+                "f12",
+                "f14",
+                "f2",
+                "f3",
+            ]
+        ]
+        df.columns = ["code", "name", "price", "change_pct"]
+    else:
+        if entity_type == TradableType.stock:
+            df = df[["f12", "f13", "f14", "f20", "f21", "f9", "f23"]]
+            df.columns = ["code", "exchange", "name", "cap", "cap1", "pe", "pb"]
+            df[["cap", "cap1", "pe", "pb"]] = df[["cap", "cap1", "pe", "pb"]].apply(pd.to_numeric, errors="coerce")
+        else:
+            df = df[["f12", "f13", "f14"]]
+            df.columns = ["code", "exchange", "name"]
+        if exchange:
+            df["exchange"] = exchange.value
+        df["entity_type"] = entity_type.value
+        df["id"] = df[["entity_type", "exchange", "code"]].apply(lambda x: "_".join(x.astype(str)), axis=1)
+        df["entity_id"] = df["id"]
+
+    return df
+
+
+def get_top_stocks(limit=100):
+    # 沪深和北交所
+    entity_flag = "fs=m:0+t:6+f:!2,m:0+t:13+f:!2,m:0+t:80+f:!2,m:1+t:2+f:!2,m:1+t:23+f:!2,m:0+t:81+s:2048"
+    fields = "f2,f3,f12,f14"
+    return get_top_tradable_list(
+        entity_type=TradableType.stock, fields=fields, limit=limit, entity_flag=entity_flag, return_quote=True
+    )
+
+
+def get_top_stockhks(limit=20):
+    entity_flag = "fs=b:DLMK0144,b:DLMK0146"
+    fields = "f2,f3,f12,f14"
+    return get_top_tradable_list(
+        entity_type=TradableType.stockhk, fields=fields, limit=limit, entity_flag=entity_flag, return_quote=True
+    )
+
+
 def get_tradable_list(
     entity_type: Union[TradableType, str] = "stock",
     exchange: Union[Exchange, str] = None,
@@ -665,31 +719,15 @@ def get_tradable_list(
                 else:
                     assert False
 
+        # f2, f3, f4, f12, f13, f14, f19, f111, f148
         fields = "f1,f2,f3,f4,f12,f13,f14"
         if entity_type == TradableType.stock:
             # 市值,流通市值,pe,pb
             fields = fields + ",f20,f21,f9,f23"
-        url = f"https://push2.eastmoney.com/api/qt/clist/get?np=1&fltt=2&invt=2&fields={fields}&pn=1&pz={limit}&fid=f3&po=1&{entity_flag}&ut=f057cbcbce2a86e2866ab8877db1d059&forcect=1&cb=cbCallbackMore&&callback=jQuery34109676853980006124_{now_timestamp() - 1}&_={now_timestamp()}"
-        resp = requests.get(url, headers=DEFAULT_HEADER)
-
-        resp.raise_for_status()
-
-        result = json_callback_param(resp.text)
-        resp.close()
-        data = result["data"]["diff"]
-        df = pd.DataFrame.from_records(data=data)
-        if entity_type == TradableType.stock:
-            df = df[["f12", "f13", "f14", "f20", "f21", "f9", "f23"]]
-            df.columns = ["code", "exchange", "name", "cap", "cap1", "pe", "pb"]
-            df[["cap", "cap1", "pe", "pb"]] = df[["cap", "cap1", "pe", "pb"]].apply(pd.to_numeric, errors="coerce")
-        else:
-            df = df[["f12", "f13", "f14"]]
-            df.columns = ["code", "exchange", "name"]
 
-        df["exchange"] = exchange.value
-        df["entity_type"] = entity_type.value
-        df["id"] = df[["entity_type", "exchange", "code"]].apply(lambda x: "_".join(x.astype(str)), axis=1)
-        df["entity_id"] = df["id"]
+        df = get_top_tradable_list(
+            entity_type=entity_type, fields=fields, limit=limit, entity_flag=entity_flag, exchange=exchange
+        )
         if entity_type == TradableType.block:
             df["category"] = block_category.value
 
@@ -996,11 +1034,14 @@ def to_zvt_code(code):
     # print(df)
     # print(len(df))
     # df = get_tradable_list(entity_type="block")
+    # print(df)
     # df = get_tradable_list(entity_type="indexus")
+    # print(df)
     # df = get_tradable_list(entity_type="currency")
+    # print(df)
     # df = get_tradable_list(entity_type="index")
-    # df = get_kdata(entity_id="index_us_SPX", level="1d")
     # print(df)
+    # df = get_kdata(entity_id="index_us_SPX", level="1d")
     # df = get_treasury_yield(pn=1, ps=50, fetch_all=False)
     # print(df)
     # df = get_future_list()
@@ -1022,10 +1063,12 @@ def to_zvt_code(code):
     # print(events)
     # print(get_hot_topic())
     # record_hot_topic()
-    df = StockHotTopic.query_data(
-        filters=[func.json_extract(StockHotTopic.entity_ids, "$").contains("stock_sh_600809")],
-    )
-    print(df)
+    # df = StockHotTopic.query_data(
+    #     filters=[func.json_extract(StockHotTopic.entity_ids, "$").contains("stock_sh_600809")],
+    # )
+    # print(df)
+    print(get_top_stocks())
+    print(get_top_stockhks())
 
 
 # the __all__ is generated

src/zvt/tag/tag_service.py

@@ -651,6 +651,7 @@ def activate_sub_tags(activate_sub_tags_model: ActivateSubTagsModel):
             # entity_ids = df["entity_id"].tolist()
             entity_ids = None
 
+            # stock_tag with sub_tag but not set to related main_tag yet
             stock_tags = StockTags.query_data(
                 session=session,
                 entity_ids=entity_ids,

src/zvt/tasks/qmt_tick_runner.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
+from zvt import init_log
 from zvt.broker.qmt.qmt_quote import record_tick
 
 if __name__ == "__main__":
+    init_log("qmt_tick_runner.log")
     from apscheduler.schedulers.background import BackgroundScheduler
 
     sched = BackgroundScheduler()

src/zvt/tasks/stock_pool_runner.py

@@ -3,7 +3,7 @@
 
 from apscheduler.schedulers.background import BackgroundScheduler
 
-from zvt import zvt_config
+from zvt import zvt_config, init_log
 from zvt.api.selector import get_entity_ids_by_filter
 from zvt.domain import (
     Stock,
@@ -125,6 +125,7 @@ def record_data_and_build_stock_pools():
 
 
 if __name__ == "__main__":
+    init_log("sotck_pool_runner.log")
     record_data_and_build_stock_pools()
     sched.add_job(func=record_data_and_build_stock_pools, trigger="cron", hour=16, minute=00, day_of_week="mon-fri")
     sched.start()