feat(dl): support new safetensors model & drop gitcode 👎

Author: fumiama

.github/workflows/checksum.yml

@@ -13,7 +13,7 @@ jobs:
 
       - name: Run RVC-Models-Downloader
         run: |
-          wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.7/rvcmd_linux_amd64.deb
+          wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.8/rvcmd_linux_amd64.deb
           sudo apt -y install ./rvcmd_linux_amd64.deb
           rm -f ./rvcmd_linux_amd64.deb
           rvcmd -notrs -w 1 -notui assets/chtts

ChatTTS/res/sha256_map.json

@@ -1,8 +1,11 @@
 {
-	"sha256_asset_Decoder_pt"   : "9964e36e840f0e3a748c5f716fe6de6490d2135a5f5155f4a642d51860e2ec38",
+	"sha256_asset_Decoder_pt"        : "9964e36e840f0e3a748c5f716fe6de6490d2135a5f5155f4a642d51860e2ec38",
 	"sha256_asset_DVAE_full_pt"      : "553eb75763511e23f3e5f86303e2163c5ca775489d637fb635d979c8ae58bbe5",
-	"sha256_asset_GPT_pt"       : "d7d4ee6461ea097a2be23eb40d73fb94ad3b3d39cb64fbb50cb3357fd466cadb",
-	"sha256_asset_Vocos_pt"     : "09a670eda1c08b740013679c7a90ebb7f1a97646ea7673069a6838e6b51d6c58",
+	"sha256_asset_Embed_safetensors" : "2ff0be7134934155741b643b74e32fb6bf3eec41257984459b2ed60cdb4c48b0",
+	"sha256_asset_Vocos_pt"          : "09a670eda1c08b740013679c7a90ebb7f1a97646ea7673069a6838e6b51d6c58",
+
+	"sha256_asset_gpt_config_json"         : "0aaa1ecd96c49ad4f473459eb1982fa7ad79fa5de08cde2781bf6ad1f9a0c236",
+	"sha256_asset_gpt_model_safetensors"   : "cd0806fd971f52f6a22c923ec64982b305e817bcc41ca83417fcf9141b984a0f",
 
 	"sha256_asset_tokenizer_special_tokens_map_json": "bd0ac9d9bb1657996b5c5fbcaa7d80f8de530d01a283da97f89deae5b1b8d011",
 	"sha256_asset_tokenizer_tokenizer_config_json"  : "43e9d658b554fa5ee8d8e1d763349323bfef1ed7a89c0794220ab8861387d421",

ChatTTS/utils/dl.py

@@ -3,7 +3,7 @@
 import hashlib
 import requests
 from io import BytesIO
-from typing import Dict
+from typing import Dict, Tuple, Optional
 from mmap import mmap, ACCESS_READ
 
 from .log import logger
@@ -42,46 +42,54 @@ def check_model(
             os.remove(bakfile)
     return True
 
+def check_folder(base_dir: Path, *innder_dirs: str, names: Tuple[str], sha256_map: Dict[str, str], update=False) -> bool:
+    key = "sha256_"
+    current_dir = base_dir
+    for d in innder_dirs:
+        current_dir /= d
+        key += f"{d}_"
+
+    for model in names:
+        menv = model.replace(".", "_")
+        if not check_model(
+            current_dir, model, sha256_map[f"{key}{menv}"], update
+        ):
+            return False
+    return True
 
 def check_all_assets(base_dir: Path, sha256_map: Dict[str, str], update=False) -> bool:
     logger.get_logger().info("checking assets...")
 
-    current_dir = base_dir / "asset"
-    names = [
+    if not check_folder(base_dir, "asset", names=(
         "Decoder.pt",
         "DVAE_full.pt",
-        "GPT.pt",
+        "Embed.safetensors",
         "Vocos.pt",
-    ]
-    for model in names:
-        menv = model.replace(".", "_")
-        if not check_model(
-            current_dir, model, sha256_map[f"sha256_asset_{menv}"], update
-        ):
-            return False
+    ), sha256_map=sha256_map, update=update):
+        return False
+
+    if not check_folder(base_dir, "asset", "gpt", names=(
+        "config.json",
+        "model.safetensors",
+    ), sha256_map=sha256_map, update=update):
+        return False
 
-    current_dir = base_dir / "asset" / "tokenizer"
-    names = [
+    if not check_folder(base_dir, "asset", "tokenizer", names=(
         "special_tokens_map.json",
         "tokenizer_config.json",
         "tokenizer.json",
-    ]
-    for model in names:
-        menv = model.replace(".", "_")
-        if not check_model(
-            current_dir, model, sha256_map[f"sha256_asset_tokenizer_{menv}"], update
-        ):
-            return False
+    ), sha256_map=sha256_map, update=update):
+        return False
 
     logger.get_logger().info("all assets are already latest.")
     return True
 
 
-def download_and_extract_tar_gz(url: str, folder: str):
+def download_and_extract_tar_gz(url: str, folder: str, headers: Optional[Dict[str, str]] = None):
     import tarfile
 
     logger.get_logger().info(f"downloading {url}")
-    response = requests.get(url, stream=True, timeout=(5, 10))
+    response = requests.get(url, headers=headers, stream=True, timeout=(10, 3))
     with BytesIO() as out_file:
         out_file.write(response.content)
         out_file.seek(0)
@@ -91,11 +99,11 @@ def download_and_extract_tar_gz(url: str, folder: str):
         logger.get_logger().info(f"extracted into {folder}")
 
 
-def download_and_extract_zip(url: str, folder: str):
+def download_and_extract_zip(url: str, folder: str, headers: Optional[Dict[str, str]] = None):
     import zipfile
 
     logger.get_logger().info(f"downloading {url}")
-    response = requests.get(url, stream=True, timeout=(5, 10))
+    response = requests.get(url, headers=headers, stream=True, timeout=(10, 3))
     with BytesIO() as out_file:
         out_file.write(response.content)
         out_file.seek(0)
@@ -105,15 +113,15 @@ def download_and_extract_zip(url: str, folder: str):
         logger.get_logger().info(f"extracted into {folder}")
 
 
-def download_dns_yaml(url: str, folder: str):
+def download_dns_yaml(url: str, folder: str, headers: Dict[str, str]):
     logger.get_logger().info(f"downloading {url}")
-    response = requests.get(url, stream=True, timeout=(5, 10))
+    response = requests.get(url, headers=headers, stream=True, timeout=(100, 3))
     with open(os.path.join(folder, "dns.yaml"), "wb") as out_file:
         out_file.write(response.content)
         logger.get_logger().info(f"downloaded into {folder}")
 
 
-def download_all_assets(tmpdir: str, version="0.2.7"):
+def download_all_assets(tmpdir: str, version="0.2.8"):
     import subprocess
     import platform
 
@@ -150,20 +158,13 @@ def download_all_assets(tmpdir: str, version="0.2.7"):
             os.chmod(cmdfile, 0o755)
         subprocess.run([cmdfile, "-notui", "-w", "0", "assets/chtts"])
     except Exception:
-        BASE_URL = "https://raw.gitcode.com/u011570312/RVC-Models-Downloader/assets/"
-        suffix = {
-            "darwin_amd64": "987",
-            "darwin_arm64": "988",
-            "linux_386": "989",
-            "linux_amd64": "990",
-            "linux_arm64": "991",
-            "windows_386": "992",
-            "windows_amd64": "993",
-        }[f"{system_type}_{architecture}"]
-        RVCMD_URL = BASE_URL + suffix
+        BASE_URL = "https://gitea.seku.su/fumiama/RVC-Models-Downloader/releases/download/"
+        suffix = "zip" if is_win else "tar.gz"
+        RVCMD_URL = BASE_URL + f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
         download_dns_yaml(
-            "https://raw.gitcode.com/u011570312/RVC-Models-Downloader/raw/main/dns.yaml",
+            "https://gitea.seku.su/fumiama/RVC-Models-Downloader/raw/branch/main/dns.yaml",
             tmpdir,
+            headers={"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0"},
         )
         if is_win:
             download_and_extract_zip(RVCMD_URL, tmpdir)

tools/checksum/tmpl.go

@@ -3,19 +3,25 @@ package main
 var files = [...]string{
 	"asset/Decoder.pt",
 	"asset/DVAE_full.pt",
-	"asset/GPT.pt",
+	"asset/Embed.safetensors",
 	"asset/Vocos.pt",
 
+	"asset/gpt/config.json",
+	"asset/gpt/model.safetensors",
+
 	"asset/tokenizer/special_tokens_map.json",
 	"asset/tokenizer/tokenizer_config.json",
 	"asset/tokenizer/tokenizer.json",
 }
 
 const jsontmpl = `{
-	"sha256_asset_Decoder_pt"   : "%s",
+	"sha256_asset_Decoder_pt"        : "%s",
 	"sha256_asset_DVAE_full_pt"      : "%s",
-	"sha256_asset_GPT_pt"       : "%s",
-	"sha256_asset_Vocos_pt"     : "%s",
+	"sha256_asset_Embed_safetensors" : "%s",
+	"sha256_asset_Vocos_pt"          : "%s",
+
+	"sha256_asset_gpt_config_json"         : "%s",
+	"sha256_asset_gpt_model_safetensors"   : "%s",
 
 	"sha256_asset_tokenizer_special_tokens_map_json": "%s",
 	"sha256_asset_tokenizer_tokenizer_config_json"  : "%s",