Update data.py

anas-awadalla · web-flow · commit 3945c871f0f5 · 2024-03-17T11:21:19.000-07:00
diff --git a/open_flamingo/train/data.py b/open_flamingo/train/data.py
@@ -84,8 +84,6 @@ def preprocess_laion_text(sample, tokenizer, max_tokens=128):
         (f"<image>{re.sub(CAPTION_BAN_PATTERN, '', s.split('<|synthetic caption|>')[-1].strip())}{tokenizer.eos_token}") for s in sample
     ]
 
-    print(sample[0])
-
     text = tokenizer(
         sample,
         max_length=max_tokens,
@@ -528,4 +526,4 @@ def get_data(args, image_processor, tokenizer, dataset_type, epoch=0):
             args, image_processor=image_processor, epoch=epoch, tokenizer=tokenizer
         )
     else:
-        raise ValueError(f"Unsupported dataset: {dataset_type}")
+        raise ValueError(f"Unsupported dataset: {dataset_type}")

Original file line number	Diff line number	Diff line change
`@@ -84,8 +84,6 @@ def preprocess_laion_text(sample, tokenizer, max_tokens=128):`
`84`	`84`	`(f"<image>{re.sub(CAPTION_BAN_PATTERN, '', s.split('<\|synthetic caption\|>')[-1].strip())}{tokenizer.eos_token}") for s in sample`
`85`	`85`	`]`
`86`	`86`
`87`		`- print(sample[0])`
`88`		`-`
`89`	`87`	`text = tokenizer(`
`90`	`88`	`sample,`
`91`	`89`	`max_length=max_tokens,`
`@@ -528,4 +526,4 @@ def get_data(args, image_processor, tokenizer, dataset_type, epoch=0):`
`528`	`526`	`args, image_processor=image_processor, epoch=epoch, tokenizer=tokenizer`
`529`	`527`	`)`
`530`	`528`	`else:`
`531`		`- raise ValueError(f"Unsupported dataset: {dataset_type}")`
	`529`	`+ raise ValueError(f"Unsupported dataset: {dataset_type}")`