Skip to content

Commit 4bde31b

Browse files
dylanbouchardmehtajineshzeya30
authored
Enable rich progress bars (#216)
* completed changes for progress bar implementation * added try catch to stop progress for execption * added progress bar description * added progressbar parameters for all notebooks * fixed linting and formatting errors * completed progress bar - whitespace issue and gender attrib issue * fixed linting * update attribute info in notebook * reverting old notebook metadata * reverting metadata for counterfactual metrics * refactored message based on comments * added attributes for response generator * reverting changes * added attribute info and removed print statement * added attribute and removed print statement * added attribute info * completed progress bar for toxicity * completed changes for stereotype metrics progress bar * fixed bugs and linting issues * fixed output indentation issues * updated poetry lock file * updated attributions and info on example notebooks * bug fixes * updated poetry file and add rich * fixed linting issues * deprecate estimate_token_cost * fix toxicity progress bars * update progress bars for COBS * update progress bars for SA * create progress bar utilities * use progress utilities with toxicity * polish progress bars for counterfactual * polish progress bars for stereotype * polish progress bars for toxicity * polish progress bars for generator classes * update notebooks * fix logic when progress bar not displayed * add rich dependency * ruff changes * progress bar updates * polish autoeval progress bars * ruff changes * update progress bar for adversarial toxicity and sentiment bug * update uni tests * add disk space cleanup to github workflow * add unit tests for progress bar * update unit test * disable live rendering in CI * update unit test * update unit test * update ci.yaml * add conftest.py * mock progress bar to fix LiveError * mock progress bar to fix Rich Liv eError * mock progress bar * update unit tests * update unit tests * update ci.yaml * update progress bar completion * fix progress bars for adversarial --------- Co-authored-by: Jinesh Mehta <mehtajineshs@gmail.com> Co-authored-by: Zeya Ahmad <za2291@columbia.edu>
1 parent 093369c commit 4bde31b

30 files changed

+2789
-2434
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,4 @@ jobs:
129129
uses: github/codeql-action/upload-sarif@v3
130130
with:
131131
sarif_file: semgrep.sarif
132-
if: always()
132+
if: always()

‎examples/adversarial/adversarial_counterfactual.ipynb‎

Lines changed: 76 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,12 @@
1818
},
1919
{
2020
"cell_type": "code",
21-
"execution_count": 2,
21+
"execution_count": 1,
2222
"id": "dc3d7dcf-7443-4f86-ab76-b8a00423784c",
2323
"metadata": {
2424
"tags": []
2525
},
26-
"outputs": [
27-
{
28-
"name": "stderr",
29-
"output_type": "stream",
30-
"text": [
31-
"/Users/c594266/langfair-env/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
32-
" from .autonotebook import tqdm as notebook_tqdm\n"
33-
]
34-
}
35-
],
26+
"outputs": [],
3627
"source": [
3728
"from itertools import combinations\n",
3829
"\n",
@@ -73,7 +64,7 @@
7364
},
7465
{
7566
"cell_type": "code",
76-
"execution_count": 3,
67+
"execution_count": 2,
7768
"id": "349bd78f-9907-426c-a0c4-5a1c6b0706b4",
7869
"metadata": {
7970
"tags": []
@@ -90,7 +81,7 @@
9081
},
9182
{
9283
"cell_type": "code",
93-
"execution_count": 4,
84+
"execution_count": 3,
9485
"id": "70a79318-9fd2-423b-af34-c343425d104e",
9586
"metadata": {
9687
"tags": []
@@ -107,17 +98,17 @@
10798
"\n",
10899
"# Setup langchain llm object\n",
109100
"llm = AzureChatOpenAI(\n",
110-
" deployment_name=\"gpt-4o\",\n",
101+
" deployment_name=\"gpt-4o-mini\",\n",
111102
" openai_api_type=\"azure\",\n",
112103
" openai_api_version=\"2024-02-15-preview\",\n",
113104
" temperature=1, # User to set temperature\n",
114-
" rate_limiter=rate_limiter,\n",
105+
" # rate_limiter=rate_limiter,\n",
115106
")"
116107
]
117108
},
118109
{
119110
"cell_type": "code",
120-
"execution_count": 5,
111+
"execution_count": 4,
121112
"id": "c0275378-bfa1-4827-8680-64e097fc4bfe",
122113
"metadata": {
123114
"tags": []
@@ -155,18 +146,78 @@
155146
},
156147
{
157148
"cell_type": "code",
158-
"execution_count": 8,
149+
"execution_count": 5,
159150
"id": "1bed3bd5-db62-4a4a-bb64-a7370ce60fcd",
160151
"metadata": {
161152
"tags": []
162153
},
163154
"outputs": [
155+
{
156+
"data": {
157+
"application/vnd.jupyter.widget-view+json": {
158+
"model_id": "4083c4ca68d14be9881697fb667be886",
159+
"version_major": 2,
160+
"version_minor": 0
161+
},
162+
"text/plain": [
163+
"Output()"
164+
]
165+
},
166+
"metadata": {},
167+
"output_type": "display_data"
168+
},
169+
{
170+
"name": "stdout",
171+
"output_type": "stream",
172+
"text": [
173+
"Generating responses with benign system prompts...\n"
174+
]
175+
},
176+
{
177+
"data": {
178+
"text/html": [
179+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
180+
],
181+
"text/plain": []
182+
},
183+
"metadata": {},
184+
"output_type": "display_data"
185+
},
186+
{
187+
"data": {
188+
"application/vnd.jupyter.widget-view+json": {
189+
"model_id": "0bbe98a5348342a6ad8277bcd90fb510",
190+
"version_major": 2,
191+
"version_minor": 0
192+
},
193+
"text/plain": [
194+
"Output()"
195+
]
196+
},
197+
"metadata": {},
198+
"output_type": "display_data"
199+
},
200+
{
201+
"name": "stdout",
202+
"output_type": "stream",
203+
"text": [
204+
"Generating responses with adversarial system prompts...\n"
205+
]
206+
},
207+
{
208+
"data": {
209+
"text/html": [
210+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
211+
],
212+
"text/plain": []
213+
},
214+
"metadata": {},
215+
"output_type": "display_data"
216+
},
164217
{
165218
"name": "stdout",
166219
"output_type": "stream",
167220
"text": [
168-
"Generating responses with benign system prompts...\n",
169-
"Generating responses with adversarial system prompts...\n",
170221
"Responses successfully generated!\n"
171222
]
172223
}
@@ -759,15 +810,15 @@
759810
],
760811
"metadata": {
761812
"environment": {
762-
"kernel": "atlas",
763-
"name": "workbench-notebooks.m125",
813+
"kernel": "langfair",
814+
"name": "workbench-notebooks.m126",
764815
"type": "gcloud",
765-
"uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m125"
816+
"uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m126"
766817
},
767818
"kernelspec": {
768-
"display_name": "langfair-env",
819+
"display_name": "langfair",
769820
"language": "python",
770-
"name": "python3"
821+
"name": "langfair"
771822
},
772823
"language_info": {
773824
"codemirror_mode": {
@@ -779,7 +830,7 @@
779830
"name": "python",
780831
"nbconvert_exporter": "python",
781832
"pygments_lexer": "ipython3",
782-
"version": "3.12.7"
833+
"version": "3.9.21"
783834
}
784835
},
785836
"nbformat": 4,

‎examples/adversarial/adversarial_toxicity.ipynb‎

Lines changed: 41 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
},
1919
{
2020
"cell_type": "code",
21-
"execution_count": 2,
21+
"execution_count": 1,
2222
"id": "dc3d7dcf-7443-4f86-ab76-b8a00423784c",
2323
"metadata": {
2424
"tags": []
@@ -71,6 +71,7 @@
7171
"- `custom_system_prompt` - (**str or None, default=None**) Optional argument for user to provide custom system prompt for toxicity generation if not using RealToxicityPrompts\n",
7272
"- `sampling_seed` - (**int, default=123**) For setting the random state when sampling from RealToxicityPrompts\n",
7373
"- `count` - (**int, default=25**) Specifies number of responses to generate for each prompt. The convention is to use 25 generations per prompt in evaluating toxicity. See, for example DecodingTrust (https://arxiv.org/abs//2306.11698) or HELM (https://arxiv.org/abs/2211.09110)\n",
74+
"- `show_progress_bars` - (**bool, default=True**) Whether to show progress bars for the generation process.\n",
7475
"\n",
7576
"###### Returns:\n",
7677
"A dictionary with two keys: `data` and `metadata`.\n",
@@ -88,7 +89,7 @@
8889
},
8990
{
9091
"cell_type": "code",
91-
"execution_count": 3,
92+
"execution_count": 2,
9293
"id": "f716e0dc-f807-4959-ad34-cee45c630374",
9394
"metadata": {
9495
"tags": []
@@ -105,7 +106,7 @@
105106
},
106107
{
107108
"cell_type": "code",
108-
"execution_count": 4,
109+
"execution_count": 3,
109110
"id": "ecae0eb4-5001-45c6-9f90-2d9ab568b024",
110111
"metadata": {
111112
"tags": []
@@ -122,17 +123,17 @@
122123
"\n",
123124
"# Setup langchain llm object\n",
124125
"llm = AzureChatOpenAI(\n",
125-
" deployment_name=\"gpt-4o\",\n",
126+
" deployment_name=\"gpt-4o-mini\",\n",
126127
" openai_api_type=\"azure\",\n",
127128
" openai_api_version=\"2024-02-15-preview\",\n",
128129
" temperature=1, # User to set temperature\n",
129-
" rate_limiter=rate_limiter,\n",
130+
" # rate_limiter=rate_limiter,\n",
130131
")"
131132
]
132133
},
133134
{
134135
"cell_type": "code",
135-
"execution_count": 5,
136+
"execution_count": 4,
136137
"id": "85f56e22-9bbf-4435-8f38-8bfde82b8aa8",
137138
"metadata": {
138139
"tags": []
@@ -160,53 +161,61 @@
160161
},
161162
{
162163
"cell_type": "code",
163-
"execution_count": 6,
164+
"execution_count": 5,
164165
"id": "6c5974cf-a4ae-40c8-9f0e-f126695b85fb",
165166
"metadata": {
166167
"tags": []
167168
},
168169
"outputs": [
169170
{
170-
"name": "stdout",
171-
"output_type": "stream",
172-
"text": [
173-
"Generating 25 responses per prompt...\n",
174-
"Responses successfully generated!\n"
175-
]
171+
"data": {
172+
"application/vnd.jupyter.widget-view+json": {
173+
"model_id": "bc4cd04aa89240178d399549fd133655",
174+
"version_major": 2,
175+
"version_minor": 0
176+
},
177+
"text/plain": [
178+
"Output()"
179+
]
180+
},
181+
"metadata": {},
182+
"output_type": "display_data"
183+
},
184+
{
185+
"data": {
186+
"text/html": [
187+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
188+
],
189+
"text/plain": []
190+
},
191+
"metadata": {},
192+
"output_type": "display_data"
176193
}
177194
],
178195
"source": [
179196
"# With toxic task prompts\n",
180197
"benign_toxic = await ag.toxicity(\n",
181198
" system_style=\"benign\",\n",
182199
" prompt_style=\"toxic\",\n",
183-
" sample_size=1000, # 1000 is the recommended sample_size\n",
200+
" sample_size=10, # 1000 is the recommended sample_size\n",
184201
")"
185202
]
186203
},
187204
{
188205
"cell_type": "code",
189-
"execution_count": 14,
206+
"execution_count": 6,
190207
"id": "436d911a-388f-4e31-a7ea-70eb1cc42007",
191208
"metadata": {
192209
"tags": []
193210
},
194-
"outputs": [
195-
{
196-
"name": "stdout",
197-
"output_type": "stream",
198-
"text": [
199-
"Generating 25 responses per prompt...\n",
200-
"Responses successfully generated!\n"
201-
]
202-
}
203-
],
211+
"outputs": [],
204212
"source": [
205213
"# With nontoxic task prompts\n",
206214
"benign_nontoxic = await ag.toxicity(\n",
207215
" system_style=\"benign\",\n",
208216
" prompt_style=\"nontoxic\",\n",
209-
" sample_size=1000, # 1000 is the recommended sample_size\n",
217+
" sample_size=10, # 1000 is the recommended sample_size\n",
218+
" show_progress_bars=False,\n",
210219
")"
211220
]
212221
},
@@ -517,15 +526,15 @@
517526
],
518527
"metadata": {
519528
"environment": {
520-
"kernel": "atlas",
521-
"name": "workbench-notebooks.m125",
529+
"kernel": "langfair",
530+
"name": "workbench-notebooks.m126",
522531
"type": "gcloud",
523-
"uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m125"
532+
"uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m126"
524533
},
525534
"kernelspec": {
526-
"display_name": "langfair-env",
535+
"display_name": "langfair",
527536
"language": "python",
528-
"name": "python3"
537+
"name": "langfair"
529538
},
530539
"language_info": {
531540
"codemirror_mode": {
@@ -537,7 +546,7 @@
537546
"name": "python",
538547
"nbconvert_exporter": "python",
539548
"pygments_lexer": "ipython3",
540-
"version": "3.12.7"
549+
"version": "3.9.21"
541550
}
542551
},
543552
"nbformat": 4,

0 commit comments

Comments
 (0)