Skip to content

Commit f6548bf

Browse files
x-CK-xx-CK-x
authored
lots of updates
1 parent 604f593 commit f6548bf

File tree

11 files changed

+803
-298
lines changed

11 files changed

+803
-298
lines changed

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ A tool for downloading from public image boards (which allow scraping) / preview
66

77
## Important Information
88

9-
- the most current **STABLE** build --> [v4.2.6](https://github.com/x-CK-x/Dataset-Curation-Tool/releases/tag/v4.2.6)
9+
- the most current **STABLE** build --> [v4.2.7](https://github.com/x-CK-x/Dataset-Curation-Tool/releases/tag/v4.2.7)
1010
- for new users, it's highly recommended to use releases instead of pulling from the main branch
1111
- in addition it is important to avoid using the **alpha** builds in the [releases](https://github.com/x-CK-x/Dataset-Curation-Tool/releases)
1212
- if an alpha build is present it will be labeled as a **pre-release** and the **main-branch** of the repo is also likely to contain those changes; as such please use the most recent stable build as denoted above
@@ -23,7 +23,7 @@ A tool for downloading from public image boards (which allow scraping) / preview
2323
- [X] CSV load time optimization with the pandas framework
2424
- [X] .sh & .bat installer scripts for conda
2525
- [X] Image Board manager class object
26-
- [ ] PNG Info & tag combination options **(Currently In Progress)**
26+
- [X] PNG Info & tag combination options
2727
- [ ] Add Aliases for tags suggestions in the textboxes **(Currently In Progress)**
2828
- [ ] Add Support for *brand new* tag & captioning models & tag combining options
2929
- deepdanbooru

‎UI_tabs/custom_dataset_tab.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ def refresh_model_list(self):
6666
and os.path.exists(os.path.join(os.getcwd(), 'Fluffusion-AutoTag')) \
6767
and os.path.exists(os.path.join(os.path.join(os.getcwd(), 'Fluffusion-AutoTag'), 'Fluffusion-AutoTag.pb')):
6868
self.auto_tag_models.append('Fluffusion-AutoTag')
69+
70+
self.auto_tag_models.append("PNG Info")
71+
6972
model_choice_dropdown = gr.update(choices=self.auto_tag_models)
7073
return model_choice_dropdown
7174

@@ -177,12 +180,17 @@ def load_model(self, model_name, use_cpu, event_data: gr.SelectData):
177180
if "Z3D" in event_data.value:
178181
model_path = os.path.join(os.getcwd(), 'Z3D-E621-Convnext')
179182
model_name = "Z3D-E621-Convnext.onnx"
183+
self.autotagmodel.set_run_option("model")
184+
self.autotagmodel.load_model(model_dir=model_path, model_name=model_name, use_cpu=use_cpu)
180185
elif "fluff" in (event_data.value).lower():
181186
model_path = os.path.join(os.getcwd(), 'Fluffusion-AutoTag')
182187
model_name = "Fluffusion-AutoTag.pb"
183-
184-
self.autotagmodel.load_model(model_dir=model_path, model_name=model_name, use_cpu=use_cpu)
185-
help.verbose_print(f"model loaded using cpu={use_cpu}")
188+
self.autotagmodel.set_run_option("model")
189+
self.autotagmodel.load_model(model_dir=model_path, model_name=model_name, use_cpu=use_cpu)
190+
elif "info" in (event_data.value).lower():
191+
self.autotagmodel.set_run_option("info")
192+
help.verbose_print(f"selected option is:\t{event_data.value}")
193+
help.verbose_print(f"selected loaded using cpu={use_cpu}")
186194

187195
# def re_load_model(model_name, use_cpu):
188196
# global self.autotagmodel
@@ -616,11 +624,13 @@ def render_tab(self):
616624
and os.path.exists(
617625
os.path.join(os.path.join(os.getcwd(), 'Z3D-E621-Convnext'), 'Z3D-E621-Convnext.onnx')):
618626
self.auto_tag_models.append('Z3D-E621-Convnext')
627+
self.auto_tag_models.append("PNG Info")
619628
if not "Fluffusion-AutoTag" in self.auto_tag_models and os.path.exists(
620629
os.path.join(os.getcwd(), 'Fluffusion-AutoTag')) \
621630
and os.path.exists(
622631
os.path.join(os.path.join(os.getcwd(), 'Fluffusion-AutoTag'), 'Fluffusion-AutoTag.pb')):
623632
self.auto_tag_models.append('Fluffusion-AutoTag')
633+
self.auto_tag_models.append("PNG Info")
624634

625635
write_tag_opts = ['Overwrite', 'Merge', 'Pre-pend', 'Append']
626636
merge_tag_opts = ['Union', 'Intersection', 'New-Original', 'Original-New']

‎UI_tabs/download_tab.py

Lines changed: 135 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1721,22 +1721,102 @@ def get_event_listeners(self):
17211721
)
17221722
self.config_save_var.click(
17231723
fn=self.config_save_button,
1724-
inputs=[self.batch_folder,self.resized_img_folder,self.tag_sep,self.tag_order_format,self.prepend_tags,self.append_tags,self.img_ext,
1725-
self.method_tag_files,self.min_score,self.min_fav_count,self.min_area,self.top_n,self.min_short_side,self.skip_posts_file,
1726-
self.skip_posts_type,self.collect_from_listed_posts_file,self.collect_from_listed_posts_type,
1727-
self.apply_filter_to_listed_posts,self.save_searched_list_type,self.save_searched_list_path,
1728-
self.downloaded_posts_folder,self.png_folder,self.jpg_folder,self.webm_folder,self.gif_folder,self.swf_folder,self.save_filename_type,
1729-
self.remove_tags_list,self.replace_tags_list,self.tag_count_list_folder,self.min_month,self.min_day,self.min_year,
1730-
self.collect_checkbox_group_var,self.download_checkbox_group_var,self.resize_checkbox_group_var,
1731-
self.create_new_config_checkbox,self.settings_path,self.proxy_url_textbox,
1732-
self.custom_csv_path_textbox, self.use_csv_custom_checkbox],
1724+
inputs=[
1725+
self.batch_folder,
1726+
self.resized_img_folder,
1727+
self.tag_sep,
1728+
self.tag_order_format,
1729+
self.prepend_tags,
1730+
self.append_tags,
1731+
self.img_ext,
1732+
self.method_tag_files,
1733+
self.min_score,
1734+
self.min_fav_count,
1735+
self.min_area,
1736+
self.top_n,
1737+
self.min_short_side,
1738+
self.skip_posts_file,
1739+
self.skip_posts_type,
1740+
self.collect_from_listed_posts_file,
1741+
self.collect_from_listed_posts_type,
1742+
self.apply_filter_to_listed_posts,
1743+
self.save_searched_list_type,
1744+
self.save_searched_list_path,
1745+
self.downloaded_posts_folder,
1746+
self.png_folder,
1747+
self.jpg_folder,
1748+
self.webm_folder,
1749+
self.gif_folder,
1750+
self.swf_folder,
1751+
self.save_filename_type,
1752+
self.remove_tags_list,
1753+
self.replace_tags_list,
1754+
self.tag_count_list_folder,
1755+
self.min_month,
1756+
self.min_day,
1757+
self.min_year,
1758+
self.collect_checkbox_group_var,
1759+
self.download_checkbox_group_var,
1760+
self.resize_checkbox_group_var,
1761+
self.create_new_config_checkbox,
1762+
self.settings_path,
1763+
self.proxy_url_textbox,
1764+
self.custom_csv_path_textbox,
1765+
self.use_csv_custom_checkbox
1766+
],
17331767
outputs=[self.all_json_files_checkboxgroup, self.quick_json_select]
17341768
).then(
17351769
fn=self.check_to_reload_auto_complete_config,
17361770
inputs=[],
17371771
outputs=[]
17381772
)
17391773
self.run_button.click(
1774+
fn=self.config_save_button,
1775+
inputs=[
1776+
self.batch_folder,
1777+
self.resized_img_folder,
1778+
self.tag_sep,
1779+
self.tag_order_format,
1780+
self.prepend_tags,
1781+
self.append_tags,
1782+
self.img_ext,
1783+
self.method_tag_files,
1784+
self.min_score,
1785+
self.min_fav_count,
1786+
self.min_area,
1787+
self.top_n,
1788+
self.min_short_side,
1789+
self.skip_posts_file,
1790+
self.skip_posts_type,
1791+
self.collect_from_listed_posts_file,
1792+
self.collect_from_listed_posts_type,
1793+
self.apply_filter_to_listed_posts,
1794+
self.save_searched_list_type,
1795+
self.save_searched_list_path,
1796+
self.downloaded_posts_folder,
1797+
self.png_folder,
1798+
self.jpg_folder,
1799+
self.webm_folder,
1800+
self.gif_folder,
1801+
self.swf_folder,
1802+
self.save_filename_type,
1803+
self.remove_tags_list,
1804+
self.replace_tags_list,
1805+
self.tag_count_list_folder,
1806+
self.min_month,
1807+
self.min_day,
1808+
self.min_year,
1809+
self.collect_checkbox_group_var,
1810+
self.download_checkbox_group_var,
1811+
self.resize_checkbox_group_var,
1812+
self.create_new_config_checkbox,
1813+
self.settings_path,
1814+
self.proxy_url_textbox,
1815+
self.custom_csv_path_textbox,
1816+
self.use_csv_custom_checkbox
1817+
],
1818+
outputs=[self.all_json_files_checkboxgroup, self.quick_json_select]
1819+
).then(
17401820
fn=self.run_script,
17411821
inputs=[self.basefolder,self.settings_path,self.numcpu,self.phaseperbatch,self.keepdb,self.cachepostsdb,self.postscsv,self.tagscsv,self.postsparquet,
17421822
self.tagsparquet],
@@ -1790,6 +1870,52 @@ def get_event_listeners(self):
17901870
]
17911871
)
17921872
self.run_button_batch.click(
1873+
fn=self.config_save_button,
1874+
inputs=[
1875+
self.batch_folder,
1876+
self.resized_img_folder,
1877+
self.tag_sep,
1878+
self.tag_order_format,
1879+
self.prepend_tags,
1880+
self.append_tags,
1881+
self.img_ext,
1882+
self.method_tag_files,
1883+
self.min_score,
1884+
self.min_fav_count,
1885+
self.min_area,
1886+
self.top_n,
1887+
self.min_short_side,
1888+
self.skip_posts_file,
1889+
self.skip_posts_type,
1890+
self.collect_from_listed_posts_file,
1891+
self.collect_from_listed_posts_type,
1892+
self.apply_filter_to_listed_posts,
1893+
self.save_searched_list_type,
1894+
self.save_searched_list_path,
1895+
self.downloaded_posts_folder,
1896+
self.png_folder,
1897+
self.jpg_folder,
1898+
self.webm_folder,
1899+
self.gif_folder,
1900+
self.swf_folder,
1901+
self.save_filename_type,
1902+
self.remove_tags_list,
1903+
self.replace_tags_list,
1904+
self.tag_count_list_folder,
1905+
self.min_month,
1906+
self.min_day,
1907+
self.min_year,
1908+
self.collect_checkbox_group_var,
1909+
self.download_checkbox_group_var,
1910+
self.resize_checkbox_group_var,
1911+
self.create_new_config_checkbox,
1912+
self.settings_path,
1913+
self.proxy_url_textbox,
1914+
self.custom_csv_path_textbox,
1915+
self.use_csv_custom_checkbox
1916+
],
1917+
outputs=[self.all_json_files_checkboxgroup, self.quick_json_select]
1918+
).then(
17931919
fn=self.make_run_visible,
17941920
inputs=[],
17951921
outputs=[self.progress_run_batch]).then(

‎environment.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ dependencies:
2222
- onnxruntime
2323
- torch
2424
- onnxruntime-gpu
25+
- piexif

‎linux_run.sh

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,33 +25,37 @@ if [ -f "$PATHFILE" ]; then
2525
cd "$STORED_PATH"
2626
else
2727
# Check if the current directory is "Dataset-Curation-Tool"
28-
if [ "$(basename "$PWD")" != "Dataset-Curation-Tool" ]; then
28+
if [ "$(basename "$PWD")" == "Dataset-Curation-Tool" ]; then
29+
echo "Already in 'Dataset-Curation-Tool' directory."
30+
else
2931
# Check and clone the GitHub repository if not already cloned
3032
if [ ! -d "Dataset-Curation-Tool" ]; then
3133
git clone https://github.com/x-CK-x/Dataset-Curation-Tool.git
34+
cd Dataset-Curation-Tool
3235
else
33-
echo "Repository already exists. Skipping clone."
36+
echo "Repository already exists. Please move to a different directory to clone again."
37+
exit 1
3438
fi
35-
cd Dataset-Curation-Tool
36-
else
37-
echo "Already in 'Dataset-Curation-Tool' directory."
3839
fi
3940
# Store the current path for future use
4041
echo "$PWD" > "$PATHFILE"
4142
fi
4243

43-
# Fetch latest changes and tags from remote
44+
# Delete the specified files
45+
rm -f linux_run.sh mac_run.sh run.bat
46+
47+
# Fetch the latest changes and tags
4448
git fetch
4549

4650
# Stash any user changes
4751
git stash
4852

4953
# Check the current tag
5054
CURRENT_TAG=$(git describe --tags --exact-match 2> /dev/null)
51-
if [ "$CURRENT_TAG" != "v4.2.6" ]; then
52-
git checkout tags/v4.2.6
55+
if [ "$CURRENT_TAG" != "v4.2.7" ]; then
56+
git checkout tags/v4.2.7
5357
else
54-
echo "Already on tag v4.2.6."
58+
echo "Already on tag v4.2.7."
5559
fi
5660

5761
# Apply stashed user changes
@@ -62,7 +66,8 @@ conda info --envs | grep data-curation > /dev/null
6266
if [ $? -ne 0 ]; then
6367
conda env create -f environment.yml
6468
else
65-
echo "Conda environment 'data-curation' already exists. Skipping environment creation."
69+
echo "Conda environment 'data-curation' already exists. Checking for updates..."
70+
conda env update -n data-curation -f environment.yml
6671
fi
6772

6873
# Activate the conda environment

‎mac_run.sh

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,33 +25,37 @@ if [ -f "$PATHFILE" ]; then
2525
cd "$STORED_PATH"
2626
else
2727
# Check if the current directory is "Dataset-Curation-Tool"
28-
if [ "$(basename "$PWD")" != "Dataset-Curation-Tool" ]; then
28+
if [ "$(basename "$PWD")" == "Dataset-Curation-Tool" ]; then
29+
echo "Already in 'Dataset-Curation-Tool' directory."
30+
else
2931
# Check and clone the GitHub repository if not already cloned
3032
if [ ! -d "Dataset-Curation-Tool" ]; then
3133
git clone https://github.com/x-CK-x/Dataset-Curation-Tool.git
34+
cd Dataset-Curation-Tool
3235
else
33-
echo "Repository already exists. Skipping clone."
36+
echo "Repository already exists. Please move to a different directory to clone again."
37+
exit 1
3438
fi
35-
cd Dataset-Curation-Tool
36-
else
37-
echo "Already in 'Dataset-Curation-Tool' directory."
3839
fi
3940
# Store the current path for future use
4041
echo "$PWD" > "$PATHFILE"
4142
fi
4243

43-
# Fetch latest changes and tags from remote
44+
# Delete the specified files
45+
rm -f linux_run.sh mac_run.sh run.bat
46+
47+
# Fetch the latest changes and tags
4448
git fetch
4549

4650
# Stash any user changes
4751
git stash
4852

4953
# Check the current tag
5054
CURRENT_TAG=$(git describe --tags --exact-match 2> /dev/null)
51-
if [ "$CURRENT_TAG" != "v4.2.6" ]; then
52-
git checkout tags/v4.2.6
55+
if [ "$CURRENT_TAG" != "v4.2.7" ]; then
56+
git checkout tags/v4.2.7
5357
else
54-
echo "Already on tag v4.2.6."
58+
echo "Already on tag v4.2.7."
5559
fi
5660

5761
# Apply stashed user changes
@@ -62,7 +66,8 @@ conda info --envs | grep data-curation > /dev/null
6266
if [ $? -ne 0 ]; then
6367
conda env create -f environment.yml
6468
else
65-
echo "Conda environment 'data-curation' already exists. Skipping environment creation."
69+
echo "Conda environment 'data-curation' already exists. Checking for updates..."
70+
conda env update -n data-curation -f environment.yml
6671
fi
6772

6873
# Activate the conda environment

‎requirements.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ Pillow
99
pandas
1010
onnxruntime
1111
torch
12-
onnxruntime-gpu
12+
onnxruntime-gpu
13+
piexif

0 commit comments

Comments
 (0)