From a0b21475208e9a7d5fcde1666e589ecc8a4b8009 Mon Sep 17 00:00:00 2001 From: ParzHe Date: Thu, 15 Aug 2024 15:36:28 +0800 Subject: [PATCH 1/3] Create README_zh.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加中文文档 --- README_zh.md | 238 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 README_zh.md diff --git a/README_zh.md b/README_zh.md new file mode 100644 index 0000000..73f7b93 --- /dev/null +++ b/README_zh.md @@ -0,0 +1,238 @@ +![FLUX Finetuning scripts](./assets/readme/dark/header-rev1.png) + + + +[English](/README.md) / 中文 + +此仓库提供了用于 Black Forest Labs 提出的 [Flux](https://github.com/black-forest-labs/flux) 模型。
+[XLabs AI](https://github.com/XLabs-AI) 团队非常高兴能发布 Flux 的微调脚本，支持的功能包括: + +- **LoRA** 🔥 +- **ControlNet** 🔥 + +# 训练 + +我们训练 LoRA 和 ControlNet 模型的脚本用到了 [DeepSpeed](https://github.com/microsoft/DeepSpeed)!
+这可以使得 1024x1024 分辨率可用! + +## 模型 + +我们训练了适用于 [`FLUX.1 [dev]`](https://github.com/black-forest-labs/flux) 的 **Canny ControlNet**，**Depth ControlNet**，**HED ControlNet** 和**LoRA** 模型文件。
+你可以通过 Hugging Face下载它们。 + +- [flux-controlnet-collections](https://huggingface.co/XLabs-AI/flux-controlnet-collections) +- [flux-controlnet-canny](https://huggingface.co/XLabs-AI/flux-controlnet-canny) +- [flux-RealismLora](https://huggingface.co/XLabs-AI/flux-RealismLora) +- [flux-lora-collections](https://huggingface.co/XLabs-AI/flux-lora-collection) +- [flux-furry-lora](https://huggingface.co/XLabs-AI/flux-furry-lora) + +### LoRA + +```bash +accelerate launch train_flux_lora_deepspeed.py --config "train_configs/test_lora.yaml" +``` + +### ControlNet + +```bash +accelerate launch train_flux_deepspeed_controlnet.py --config "train_configs/test_canny_controlnet.yaml" +``` + +## 训练数据集 + +训练数据集应当具有以下格式： + +```text +├── images/ +│ ├── 1.png +│ ├── 1.json +│ ├── 2.png +│ ├── 2.json +│ ├── ... +``` + +### `images/*.json` 文件的示例 + +一个 `.json` 文件包括 "caption" 信息 with a text prompt. + +```json +{ + "caption": "A figure stands in a misty landscape, wearing a mask with antlers and dark, embellished attire, exuding mystery and otherworldlines" +} +``` + +## 推理 + +为了测试保存的模型文件，应当使用下列的指令： + +### LoRA +![示例图片 1](./assets/readme/examples/picture-5-rev1.png) +提示词: "A girl in a suit covered with bold tattoos and holding a vest pistol, beautiful woman, 25 years old, cool, future fantasy, turquoise & light orange ping curl hair" +![示例图片 2](./assets/readme/examples/picture-6-rev1.png) +提示词: "A handsome man in a suit, 25 years old, cool, futuristic" + +```bash +python3 main.py \ + --prompt "Female furry Pixie with text 'hello world'" \ + --lora_repo_id XLabs-AI/flux-furry-lora --lora_name furry_lora.safetensors --device cuda --offload --use_lora \ + --model_type flux-dev-fp8 --width 1024 --height 1024 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` + +![示例图片 1](./assets/readme/examples/furry4.png) + +```bash +python3 main.py \ +--prompt "A cute corgi lives in a house made out of sushi, anime" \ +--lora_repo_id XLabs-AI/flux-lora-collection --lora_name anime_lora.safetensors \ +--device cuda --offload --use_lora --model_type flux-dev-fp8 --width 1024 --height 1024 + +``` +![示例图片 3](./assets/readme/examples/result_14.png) + + +```bash +python3 main.py \ + --use_lora --lora_weight 0.7 \ + --width 1024 --height 768 \ + --lora_repo_id XLabs-AI/flux-lora-collection --lora_name realism_lora.safetensors \ + --guidance 4 \ + --prompt "contrast play photography of a black female wearing white suit and albino asian geisha female wearing black suit, solid background, avant garde, high fashion" +``` +![Example Picture 3](./assets/readme/examples/picture-7-rev1.png) + +## Canny ControlNet +```bash +python3 main.py \ + --prompt "a viking man with white hair looking, cinematic, MM full HD" \ + --image input_image_canny.jpg \ + --control_type canny \ + --repo_id XLabs-AI/flux-controlnet-collections --name flux-canny-controlnet.safetensors --device cuda --use_controlnet \ + --model_type flux-dev --width 768 --height 768 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` +![示例图片 1](./assets/readme/examples/canny_example_1.png?raw=true) + +## Depth ControlNet +```bash +python3 main.py \ + --prompt "Photo of the bold man with beard and laptop, full hd, cinematic photo" \ + --image input_image_depth1.jpg \ + --control_type depth \ + --repo_id XLabs-AI/flux-controlnet-collections --name flux-depth-controlnet.safetensors --device cuda --use_controlnet \ + --model_type flux-dev --width 1024 --height 1024 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` +![示例图片 2](./assets/readme/examples/depth_example_1.png?raw=true) + +```bash +python3 main.py \ + --prompt "photo of handsome fluffy black dog standing on a forest path, full hd, cinematic photo" \ + --image input_image_depth2.jpg \ + --control_type depth \ + --repo_id XLabs-AI/flux-controlnet-collections --name flux-depth-controlnet.safetensors --device cuda --use_controlnet \ + --model_type flux-dev --width 1024 --height 1024 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` +![示例图片 2](./assets/readme/examples/depth_example_2.png?raw=true) + +```bash +python3 main.py \ + --prompt "Photo of japanese village with houses and sakura, full hd, cinematic photo" \ + --image input_image_depth3.webp \ + --control_type depth \ + --repo_id XLabs-AI/flux-controlnet-collections --name flux-depth-controlnet.safetensors --device cuda --use_controlnet \ + --model_type flux-dev --width 1024 --height 1024 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` +![示例图片 2](./assets/readme/examples/depth_example_3.png?raw=true) + + +## HED ControlNet +```bash +python3 main.py \ + --prompt "2d art of a sitting african rich woman, full hd, cinematic photo" \ + --image input_image_hed1.jpg \ + --control_type hed \ + --repo_id XLabs-AI/flux-controlnet-collections --name flux-hed-controlnet.safetensors --device cuda --use_controlnet \ + --model_type flux-dev --width 768 --height 768 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` +![Example Picture 2](./assets/readme/examples/hed_example_1.png?raw=true) + +```bash +python3 main.py \ + --prompt "anime ghibli style art of a running happy white dog, full hd" \ + --image input_image_hed2.jpg \ + --control_type hed \ + --repo_id XLabs-AI/flux-controlnet-collections --name flux-hed-controlnet.safetensors --device cuda --use_controlnet \ + --model_type flux-dev --width 768 --height 768 \ + --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 + +``` +![Example Picture 2](./assets/readme/examples/hed_example_2.png?raw=true) + +## 低显存模式 + +基于[Flux-dev-F8](https://huggingface.co/XLabs-AI/flux-dev-fp8) 模型，训练 LoRA 和 Controlnet FP8 版本，可以通过 `--offload` 和 `--name flux-dev-fp8` 设置来实现低显存的使用 (22 GB): +```bash +python3 main.py \ + --offload --name flux-dev-fp8 \ + --lora_repo_id XLabs-AI/flux-lora-collection --lora_name realism_lora.safetensors \ + --guidance 4 \ + --prompt "A handsome girl in a suit covered with bold tattoos and holding a pistol. Animatrix illustration style, fantasy style, natural photo cinematic" +``` +![Example Picture 0](./assets/readme/examples/picture-0-rev1.png) + +## 要求 + +通过以下指令安装依赖： + +```bash +pip3 install -r requirements.txt +``` + +## 加速设置的示例 + +```yaml +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + gradient_accumulation_steps: 2 + gradient_clipping: 1.0 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: false + zero_stage: 2 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +enable_cpu_affinity: false +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false + +``` +## 模型许可证书 + +我们的模型在 [FLUX.1 [dev] Non-Commercial License](https://github.com/black-forest-labs/flux/blob/main/model_licenses/LICENSE-FLUX1-dev) 之下
但是，我们的训练和推理脚本在 Apache 2 协议下。 + +## 最近的更新 + +我们发布了新的 适合于Flux的 ControlNet 模型权重: **OpenPose**, **Depth** 和其他!
+继续训练和调整可以参考 [XLabs AI](https://github.com/XLabs-AI) 的 **IP-Adapters** 应对Flux的部分. + +![关注我们的持续更新](./assets/readme/dark/follow-cta-rev2.png) From 72ddd146725430a2296ee9d1ae82976156db021a Mon Sep 17 00:00:00 2001 From: ParzHe Date: Thu, 15 Aug 2024 15:43:18 +0800 Subject: [PATCH 2/3] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加中文 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 336dd5c..2b63a85 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ +English / [中文](/README_zh.md) + This repository provides training scripts for [Flux model](https://github.com/black-forest-labs/flux) by Black Forest Labs.
[XLabs AI](https://github.com/XLabs-AI) team is happy to publish fune-tuning Flux scripts, including: From 844b42805c3de23a75ea8aa38128f72ee02abd59 Mon Sep 17 00:00:00 2001 From: ParzHe Date: Tue, 3 Sep 2024 10:18:14 +0800 Subject: [PATCH 3/3] Update README_zh.md --- README_zh.md | 200 ++++++++++++++++++++++++++++----------------------- 1 file changed, 110 insertions(+), 90 deletions(-) diff --git a/README_zh.md b/README_zh.md index 73f7b93..0ca318c 100644 --- a/README_zh.md +++ b/README_zh.md @@ -9,23 +9,53 @@ - **LoRA** 🔥 - **ControlNet** 🔥 +[](https://discord.gg/FHY2guThfy) + +# ComfyUI + +查看我们的 [ComfyUI 工作流GitHub](https://github.com/XLabs-AI/x-flux-comfyui)。 +![Example Picture 1](https://github.com/XLabs-AI/x-flux-comfyui/blob/main/assets/image1.png?raw=true) + +## 环境要求 +1. Python >= 3.10 +2. PyTorch >= 2.1 +3. 需要 HuggingFace CLI 用于下载我们的模型: ```huggingface-cli login``` + +# 安装指南 +1. 克隆我们的仓库: +```bash +git clone https://github.com/XLabs-AI/x-flux.git +``` +2. 创建新的虚拟环境: +```bash +python3 -m venv xflux_env +source xflux_env/bin/activate +``` +3. 通过执行以下指令安装我们需要的依赖: +```bash +cd x-flux +pip install -r requirements.txt +``` # 训练 我们训练 LoRA 和 ControlNet 模型的脚本用到了 [DeepSpeed](https://github.com/microsoft/DeepSpeed)!
-这可以使得 1024x1024 分辨率可用! +这可以支持 1024x1024 分辨率的数据! ## 模型 -我们训练了适用于 [`FLUX.1 [dev]`](https://github.com/black-forest-labs/flux) 的 **Canny ControlNet**，**Depth ControlNet**，**HED ControlNet** 和**LoRA** 模型文件。
+我们训练了适用于 [`FLUX.1 [dev]`](https://github.com/black-forest-labs/flux) 的 **IP-Adapter**，**Canny ControlNet**，**Depth ControlNet**，**HED ControlNet** 和**LoRA** 模型文件。
你可以通过 Hugging Face下载它们。 +- [flux-ip-adapter](https://huggingface.co/XLabs-AI/flux-ip-adapter) - [flux-controlnet-collections](https://huggingface.co/XLabs-AI/flux-controlnet-collections) - [flux-controlnet-canny](https://huggingface.co/XLabs-AI/flux-controlnet-canny) - [flux-RealismLora](https://huggingface.co/XLabs-AI/flux-RealismLora) - [flux-lora-collections](https://huggingface.co/XLabs-AI/flux-lora-collection) - [flux-furry-lora](https://huggingface.co/XLabs-AI/flux-furry-lora) +同时, 我们的模型也可以在 [civit.ai](https://civitai.com/user/xlabs_ai) 访问。 + ### LoRA ```bash @@ -63,120 +93,118 @@ accelerate launch train_flux_deepspeed_controlnet.py --config "train_configs/tes ## 推理 -为了测试保存的模型文件，应当使用下列的指令： - -### LoRA -![示例图片 1](./assets/readme/examples/picture-5-rev1.png) -提示词: "A girl in a suit covered with bold tattoos and holding a vest pistol, beautiful woman, 25 years old, cool, future fantasy, turquoise & light orange ping curl hair" -![示例图片 2](./assets/readme/examples/picture-6-rev1.png) -提示词: "A handsome man in a suit, 25 years old, cool, futuristic" +为了测试我们的模型，你有以下选项: +1. 在我们的ComfyUI 工作流启动 adapters, [查看我们的仓库](https://github.com/XLabs-AI/x-flux-comfyui) for more details +2. 使用 `main.py` 在 CLI 命令行 +3. 使用 Gradio demo UI +### Gradio +可以按照以下指令启动 gradio: +``` +python3 gradio_demo.py --ckpt_dir model_weights +``` + `--ckpt_dir` 是下载 XLabs AI adapter 模型权重 (LoRAs, IP-adapter, ControlNets) 的位置。 + +### IP-Adapter ```bash python3 main.py \ - --prompt "Female furry Pixie with text 'hello world'" \ - --lora_repo_id XLabs-AI/flux-furry-lora --lora_name furry_lora.safetensors --device cuda --offload --use_lora \ - --model_type flux-dev-fp8 --width 1024 --height 1024 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 - + --prompt "wearing glasses" \ + --ip_repo_id XLabs-AI/flux-ip-adapter --ip_name flux-ip-adapter.safetensors --device cuda --use_ip \ + --width 1024 --height 1024 \ + --timestep_to_start_cfg 1 --num_steps 25 \ + --true_gs 3.5 --guidance 4 \ + --img_prompt assets/example_images/statue.jpg ``` -![示例图片 1](./assets/readme/examples/furry4.png) +### LoRA +![Example Picture 1](./assets/readme/examples/picture-5-rev1.png) +prompt: "A girl in a suit covered with bold tattoos and holding a vest pistol, beautiful woman, 25 years old, cool, future fantasy, turquoise & light orange ping curl hair" +![Example Picture 2](./assets/readme/examples/picture-6-rev1.png) +prompt: "A handsome man in a suit, 25 years old, cool, futuristic" ```bash python3 main.py \ ---prompt "A cute corgi lives in a house made out of sushi, anime" \ ---lora_repo_id XLabs-AI/flux-lora-collection --lora_name anime_lora.safetensors \ ---device cuda --offload --use_lora --model_type flux-dev-fp8 --width 1024 --height 1024 - + --prompt "A cute corgi lives in a house made out of sushi, anime" \ + --lora_repo_id XLabs-AI/flux-lora-collection \ + --lora_name anime_lora.safetensors \ + --use_lora --width 1024 --height 1024 ``` -![示例图片 3](./assets/readme/examples/result_14.png) +![Example Picture 3](./assets/readme/examples/result_14.png) ```bash python3 main.py \ - --use_lora --lora_weight 0.7 \ - --width 1024 --height 768 \ - --lora_repo_id XLabs-AI/flux-lora-collection --lora_name realism_lora.safetensors \ - --guidance 4 \ - --prompt "contrast play photography of a black female wearing white suit and albino asian geisha female wearing black suit, solid background, avant garde, high fashion" + --use_lora --lora_weight 0.7 \ + --width 1024 --height 768 \ + --lora_repo_id XLabs-AI/flux-lora-collection \ + --lora_name realism_lora.safetensors \ + --guidance 4 \ + --prompt "contrast play photography of a black female wearing white suit and albino asian geisha female wearing black suit, solid background, avant garde, high fashion" ``` ![Example Picture 3](./assets/readme/examples/picture-7-rev1.png) -## Canny ControlNet +## Canny ControlNet V3 ```bash python3 main.py \ - --prompt "a viking man with white hair looking, cinematic, MM full HD" \ - --image input_image_canny.jpg \ - --control_type canny \ - --repo_id XLabs-AI/flux-controlnet-collections --name flux-canny-controlnet.safetensors --device cuda --use_controlnet \ - --model_type flux-dev --width 768 --height 768 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 - + --prompt "cyberpank dining room, full hd, cinematic" \ + --image input_canny1.png --control_type canny \ + --repo_id XLabs-AI/flux-controlnet-canny-v3 \ + --name flux-canny-controlnet-v3.safetensors \ + --use_controlnet --model_type flux-dev \ + --width 1024 --height 1024 --timestep_to_start_cfg 1 \ + --num_steps 25 --true_gs 4 --guidance 4 ``` -![示例图片 1](./assets/readme/examples/canny_example_1.png?raw=true) - -## Depth ControlNet +![Example Picture 1](./assets/readme/examples/canny_result1.png?raw=true) ```bash python3 main.py \ - --prompt "Photo of the bold man with beard and laptop, full hd, cinematic photo" \ - --image input_image_depth1.jpg \ - --control_type depth \ - --repo_id XLabs-AI/flux-controlnet-collections --name flux-depth-controlnet.safetensors --device cuda --use_controlnet \ - --model_type flux-dev --width 1024 --height 1024 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 - + --prompt "handsome korean woman, full hd, cinematic" \ + --image input_canny2.png --control_type canny \ + --repo_id XLabs-AI/flux-controlnet-canny-v3 \ + --name flux-canny-controlnet-v3.safetensors \ + --use_controlnet --model_type flux-dev \ + --width 1024 --height 1024 --timestep_to_start_cfg 1 \ + --num_steps 25 --true_gs 4 --guidance 4 ``` -![示例图片 2](./assets/readme/examples/depth_example_1.png?raw=true) +![Example Picture 1](./assets/readme/examples/canny_result2.png?raw=true) +## Depth ControlNet V3 ```bash python3 main.py \ - --prompt "photo of handsome fluffy black dog standing on a forest path, full hd, cinematic photo" \ - --image input_image_depth2.jpg \ - --control_type depth \ - --repo_id XLabs-AI/flux-controlnet-collections --name flux-depth-controlnet.safetensors --device cuda --use_controlnet \ - --model_type flux-dev --width 1024 --height 1024 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 - + --prompt "handsome man in balenciaga style, fashion" \ + --image input_depth1.png --control_type depth \ + --repo_id XLabs-AI/flux-controlnet-depth-v3 \ + --name flux-depth-controlnet-v3.safetensors \ + --use_controlnet --model_type flux-dev \ + --width 1024 --height 1024 --timestep_to_start_cfg 1 \ + --num_steps 25 --true_gs 3.5 --guidance 3 ``` -![示例图片 2](./assets/readme/examples/depth_example_2.png?raw=true) +![Example Picture 2](./assets/readme/examples/depth_result1.png?raw=true) ```bash python3 main.py \ - --prompt "Photo of japanese village with houses and sakura, full hd, cinematic photo" \ - --image input_image_depth3.webp \ - --control_type depth \ - --repo_id XLabs-AI/flux-controlnet-collections --name flux-depth-controlnet.safetensors --device cuda --use_controlnet \ - --model_type flux-dev --width 1024 --height 1024 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 - + --prompt "a village in minecraft style, 3d, full hd" \ + --image input_depth2.png --control_type depth \ + --repo_id XLabs-AI/flux-controlnet-depth-v3 \ + --name flux-depth-controlnet-v3.safetensors \ + --use_controlnet --model_type flux-dev \ + --width 1024 --height 1024 --timestep_to_start_cfg 1 \ + --num_steps 25 --true_gs 3.5 --guidance 3 ``` -![示例图片 2](./assets/readme/examples/depth_example_3.png?raw=true) - - -## HED ControlNet -```bash -python3 main.py \ - --prompt "2d art of a sitting african rich woman, full hd, cinematic photo" \ - --image input_image_hed1.jpg \ - --control_type hed \ - --repo_id XLabs-AI/flux-controlnet-collections --name flux-hed-controlnet.safetensors --device cuda --use_controlnet \ - --model_type flux-dev --width 768 --height 768 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 +![Example Picture 2](./assets/readme/examples/depth_result2.png?raw=true) -``` -![Example Picture 2](./assets/readme/examples/hed_example_1.png?raw=true) +## HED ControlNet V3 ```bash -python3 main.py \ - --prompt "anime ghibli style art of a running happy white dog, full hd" \ - --image input_image_hed2.jpg \ - --control_type hed \ - --repo_id XLabs-AI/flux-controlnet-collections --name flux-hed-controlnet.safetensors --device cuda --use_controlnet \ - --model_type flux-dev --width 768 --height 768 \ - --timestep_to_start_cfg 1 --num_steps 25 --true_gs 3.5 --guidance 4 - + python3 main.py \ + --prompt "A beautiful woman with white hair and light freckles, her neck area bare and visible" \ + --image input_hed1.png --control_type hed \ + --repo_id XLabs-AI/flux-controlnet-hed-v3 \ + --name flux-hed-controlnet-v3.safetensors \ + --use_controlnet --model_type flux-dev \ + --width 1024 --height 1024 --timestep_to_start_cfg 1 \ + --num_steps 25 --true_gs 3.5 --guidance 4 ``` -![Example Picture 2](./assets/readme/examples/hed_example_2.png?raw=true) +![Example Picture 2](./assets/readme/examples/hed_result1.png?raw=true) ## 低显存模式 @@ -190,14 +218,6 @@ python3 main.py \ ``` ![Example Picture 0](./assets/readme/examples/picture-0-rev1.png) -## 要求 - -通过以下指令安装依赖： - -```bash -pip3 install -r requirements.txt -``` - ## 加速设置的示例 ```yaml