[Share] Send annotation image to ChatGPT-4-vision

[volatile-static]

Description

How to setup

How to use

Results

Event

None

Operation

Script

Data

/**
 * Ask Annotation
 * @author volatile-static
 * @link https://github.com/windingwind/zotero-actions-tags/discussions/225
 */
const BASE_URL = 'https://YOURNAME.openai.azure.com/openai/deployments/gpt-4/chat/completions?api-version=2023-07-01-preview';
const API_KEY = 'your0api1key';

if (!item) return;

const { Zotero, Zotero_Tabs, console, alert } = require('window'),
	popMsg = new Zotero.ProgressWindow(),
	progMeter = new popMsg.ItemProgress('chrome://zotero/skin/tick.png', 'AI is thinking.'),
	reader = Zotero.Reader.getByTabID(Zotero_Tabs.selectedID),
	ann = reader?._state.annotations.find(a => a.id == item.key);
if (!ann?.image) return 'No image found.';

const payload = {
	model: 'gpt-4-vision-preview',
	max_tokens: 3000,
	messages: [{
		role: 'user',
		content: [
			{ type: 'text', text: `这是文档\`${reader._title}\`中的一幅图，请根据标题简要描述图中内容` },
			{ type: 'image_url', image_url: { url: ann.image } }
		]
	}]
};
console.debug('Action chat payload', payload);

Zotero.HTTP.doPost(
	BASE_URL,
	JSON.stringify(payload),
	resp => {
		console.debug('Action response', resp);
		const res = JSON.parse(resp.response);
		if (resp.status != 200) {
			progMeter.setIcon('chrome://zotero/skin/cross.png');
			progMeter.setText(res.statusCode);
			new popMsg.ItemProgress('', res.message);
		} else {
			const dat = res.choices[0];
			progMeter.setProgress(100);
			progMeter.setText(`Finished [${dat.finish_details.type}]`);
			new popMsg.ItemProgress('', dat.message.content);
			alert(dat.message.content);
		}
	},
	{ 
		'Content-Type': 'application/json', 
		'api-key': API_KEY,  // for Azure
		Authorization: 'Bearer ' + API_KEY  // for OpenAI 
	}
);

popMsg.changeHeadline('Action:', '', 'Ask Annotation');
popMsg.show();

Anything else

You must have access to model gpt-4-vision-preview.

[tricktreat]

如何把生成的解释，设置为annotation的comment呢？尝试了item.annotationComment = ${dat.message.content}和ann.comment = ${dat.message.content}都不能刷新comment。 @volatile-static @windingwind

[volatile-static]

item.annotationComment = `${dat.message.content}`

你写的这个就可以，需要注意的是，对条目修改后需要调用item.saveTx()进行保存
欢迎分享你修改后的版本

[tricktreat]

保存GPT-4v对图片的解释到annotation的comment中：

更新后的代码：

/**
 * Ask Annotation
 * @author volatile-static
 * @link https://github.com/windingwind/zotero-actions-tags/discussions/225
 */
const BASE_URL = 'https://YOURNAME.openai.azure.com/openai/deployments/gpt-4/chat/completions?api-version=2023-07-01-preview';
const API_KEY = 'your0api1key';

if (!item) return;

const { Zotero, Zotero_Tabs, console, alert } = require('window'),
	popMsg = new Zotero.ProgressWindow(),
	progMeter = new popMsg.ItemProgress('chrome://zotero/skin/tick.png', 'AI is thinking.'),
	reader = Zotero.Reader.getByTabID(Zotero_Tabs.selectedID),
	ann = reader?._state.annotations.find(a => a.id == item.key);
if (!ann?.image) return 'No image found.';

const payload = {
	model: 'gpt-4-vision-preview',
	max_tokens: 3000,
	messages: [{
		role: 'user',
		content: [
			{ type: 'text', text: `这是文档\`${reader._title}\`中的一幅图，请根据标题简要描述图中内容` },
			{ type: 'image_url', image_url: { url: ann.image } }
		]
	}]
};
console.debug('Action chat payload', payload);

Zotero.HTTP.doPost(
	BASE_URL,
	JSON.stringify(payload),
	resp => {
		console.debug('Action response', resp);
		const res = JSON.parse(resp.response);
		if (resp.status != 200) {
			progMeter.setIcon('chrome://zotero/skin/cross.png');
			progMeter.setText(res.statusCode);
			new popMsg.ItemProgress('', res.message);
		} else {
			const dat = res.choices[0];
			progMeter.setProgress(100);
			progMeter.setText(`Finished [${dat.finish_details.type}]`);
			new popMsg.ItemProgress('', dat.message.content);
			item.annotationComment = `${dat.message.content}`;
			item.saveTx();
		}
	},
	{ 
		'Content-Type': 'application/json', 
		'api-key': API_KEY,  // for Azure
		Authorization: 'Bearer ' + API_KEY  // for OpenAI 
	}
);

popMsg.changeHeadline('Action:', '', 'Ask Annotation');
popMsg.show();

[volatile-static]

update：

- progMeter.setText(`Finished [${dat.finish_details.type}]`);
+ progMeter.setText(`Finished [${dat.finish_details?.type}]`);