3d + improve iterations + allow refreshing

Author: firtoz

.gitignore

@@ -34,3 +34,4 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+.idea

app/PreviewShape/PreviewShape.tsx

@@ -10,12 +10,14 @@ import {
 	stopEventPropagation,
 	DefaultSpinner,
 } from '@tldraw/tldraw'
+import { CompletionMessageItem } from '../lib/getHtmlFromOpenAI'
+import React from 'react'
 
 export type PreviewShape = TLBaseShape<
 	'preview',
 	{
 		html: string
-		source: string
+		history: CompletionMessageItem[],
 		w: number
 		h: number
 	}
@@ -26,7 +28,7 @@ export class PreviewShapeUtil extends BaseBoxShapeUtil<PreviewShape> {
 	getDefaultProps(): PreviewShape['props'] {
 		return {
 			html: '',
-			source: '',
+			history: [],
 			w: (960 * 2) / 3,
 			h: (540 * 2) / 3,
 		}
@@ -39,9 +41,10 @@ export class PreviewShapeUtil extends BaseBoxShapeUtil<PreviewShape> {
 
 	override component(shape: PreviewShape) {
 		const isEditing = useIsEditing(shape.id)
+		const [reloadKey, setReloadKey] = React.useState(0)
 		const toast = useToasts()
 		return (
-			<HTMLContainer className="tl-embed-container" id={shape.id}>
+			<HTMLContainer className="tl-embed-container" id={shape.id} key={reloadKey}>
 				{shape.props.html ? (
 					<iframe
 						className="tl-embed"
@@ -95,6 +98,26 @@ export class PreviewShapeUtil extends BaseBoxShapeUtil<PreviewShape> {
 				>
 					<Icon icon="duplicate" />
 				</div>
+				<div
+					style={{
+						position: 'absolute',
+						top: 0,
+						right: -80,
+						height: 40,
+						width: 40,
+						display: 'flex',
+						alignItems: 'center',
+						justifyContent: 'center',
+						cursor: 'pointer',
+						pointerEvents: 'all',
+					}}
+					onClick={() => {
+						setReloadKey(reloadKey + 1)
+					}}
+					onPointerDown={stopEventPropagation}
+				>
+					<Icon icon="undo" />
+				</div>
 			</HTMLContainer>
 		)
 	}

app/components/ExportButton.tsx

@@ -1,20 +1,20 @@
-import { useEditor, getSvgAsImage, useToasts, createShapeId } from '@tldraw/tldraw'
-import { useState } from 'react'
-import { PreviewShape } from '../PreviewShape/PreviewShape'
-import { getHtmlFromOpenAI } from '../lib/getHtmlFromOpenAI'
 import { useMakeReal } from '../hooks/useMakeReal'
 
-export function ExportButton() {
-	const makeReal = useMakeReal()
+export function ExportButton({
+															 mode,
+														 }: {
+	mode: 'tailwind' | 'threejs',
+}) {
+	const makeReal = useMakeReal(mode)
 
 	// A tailwind styled button that is pinned to the bottom right of the screen
 	return (
 		<button
 			onClick={makeReal}
-			className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded m-2"
+			className='bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded m-2'
 			style={{ cursor: 'pointer', zIndex: 100000, pointerEvents: 'all' }}
 		>
-			Make Real
+			{`Make Real${mode === 'threejs' ? ' (3D!)' : ''}`}
 		</button>
 	)
 }

app/hooks/useMakeReal.ts

@@ -1,16 +1,19 @@
 import { useEditor, useToasts } from '@tldraw/tldraw'
 import { useCallback } from 'react'
 import { makeReal } from '../lib/makeReal'
+import { CompletionMessageItem } from '../lib/getHtmlFromOpenAI'
 
-export function useMakeReal() {
+export function useMakeReal(
+	mode: 'tailwind' | 'threejs',
+) {
 	const editor = useEditor()
 	const toast = useToasts()
 
 	return useCallback(async () => {
 		const input = document.getElementById('openai_key_risky_but_cool') as HTMLInputElement
 		const apiKey = input?.value ?? null
 		try {
-			await makeReal(editor, apiKey)
+			await makeReal(editor, apiKey, mode)
 		} catch (e: any) {
 			console.error(e)
 			toast.addToast({
@@ -19,5 +22,5 @@ export function useMakeReal() {
 				description: `${e.message.slice(0, 100)}`,
 			})
 		}
-	}, [editor, toast])
+	}, [editor, mode, toast])
 }

app/lib/getHtmlFromOpenAI.ts

@@ -1,4 +1,4 @@
-const systemPrompt = `You are an expert web developer who specializes in tailwind css.
+const tailwindSystemPrompt = `You are an expert web developer who specializes in tailwind css.
 A user will provide you with a low-fidelity wireframe of an application. 
 You will return a single html file that uses HTML, tailwind css, and JavaScript to create a high fidelity website.
 Include any extra CSS and JavaScript in the html file.
@@ -9,49 +9,87 @@ They may also provide you with the html of a previous design that they want you
 Carry out any changes they request from you.
 In the wireframe, the previous design's html will appear as a white rectangle.
 Use creative license to make the application more fleshed out.
-Use JavaScript modules and unkpkg to import any necessary dependencies.
+Use JavaScript modules and unpkg to import any necessary dependencies.
 
 Respond ONLY with the contents of the html file.`
 
+const threeJsSystemPrompt = `You are an expert web developer who specializes in three.js.
+A user will provide you with a low-fidelity wireframe of an application.
+You will return a single html file that uses HTML, three.js, and JavaScript to create a high fidelity website.
+Add a hidden div into the start of the body that includes the detailed instructions from the user, the contents adjusted to take the requested changes into account. If for example the user requests a change to the color of a button, change the color of the button in the instructions also, instead of saying "change button color".
+Include any extra CSS and JavaScript in the html file.
+The user will provide you with notes in text, arrows, or drawings.
+The user may also include images of other websites as style references. Transfer the styles as best as you can, matching colors.
+Prefer to use standard materials instead of basic and avoid custom shaders.
+Unless otherwise specified, set up orbit controls and a directional light attached to the camera.
+Use an import map e.g. <script type="importmap">{"imports": {"three": "https://unpkg.com/[email protected]/build/three.module.js","OrbitControls": "https://unpkg.com/[email protected]/examples/jsm/controls/OrbitControls.js"}}</script>
+They may also provide you with the html of a previous design that they want you to iterate from.
+Carry out any changes they request from you, keep everything the same except for the requested changes.
+In the wireframe, the previous design's html will appear as a white rectangle.
+Use creative license to make the application more fleshed out.
+Use JavaScript modules and unpkg to import any necessary dependencies.
+Respond ONLY with the contents of the html file.`
+
+const threeJsText = 'Turn this into a single html file using three.js.'
+const tailwindText = 'Turn this into a single html file using tailwind.'
+const changesText = 'Please make the changes as specified in the image.'
+
 export async function getHtmlFromOpenAI({
-	image,
-	html,
-	apiKey,
-}: {
-	image: string
-	html: string
-	apiKey: string
+																					image,
+																					apiKey,
+																					mode,
+																					history,
+																				}: {
+	image: string;
+	apiKey: string;
+	mode: 'tailwind' | 'threejs',
+	history: CompletionMessageItem[],
 }) {
+	const userMessage: CompletionMessageItem = {
+		role: 'user',
+		content: [
+			{
+				type: 'image_url',
+				image_url: {
+					url: image,
+					detail: 'high',
+				},
+			},
+		],
+	}
+
+	if (history.length === 0) {
+		(userMessage.content as any[]).push({
+			type: 'text',
+			text: mode === 'tailwind' ? tailwindText : threeJsText,
+		})
+	} else {
+		(userMessage.content as any[]).push({
+			type: 'text',
+			text: changesText,
+		})
+	}
+
+	const systemMessage: CompletionMessageItem = {
+		role: 'system',
+		content: mode === 'tailwind' ? tailwindSystemPrompt : threeJsSystemPrompt,
+	}
+
+	const last4NonSystemMessages = history
+		.filter((item) => item.role !== 'system')
+		.slice(-4)
+
+	const messages: CompletionMessageItem[] = [
+		systemMessage,
+		...last4NonSystemMessages,
+		userMessage,
+	]
+
 	const body: GPT4VCompletionRequest = {
 		model: 'gpt-4-vision-preview',
 		max_tokens: 4096,
 		temperature: 0,
-		messages: [
-			{
-				role: 'system',
-				content: systemPrompt,
-			},
-			{
-				role: 'user',
-				content: [
-					{
-						type: 'image_url',
-						image_url: {
-							url: image,
-							detail: 'high',
-						},
-					},
-					{
-						type: 'text',
-						text: 'Turn this into a single html file using tailwind.',
-					},
-					{
-						type: 'text',
-						text: html,
-					},
-				],
-			},
-		],
+		messages,
 	}
 
 	let json = null
@@ -67,41 +105,51 @@ export async function getHtmlFromOpenAI({
 			},
 			body: JSON.stringify(body),
 		})
-		console.log(resp)
 		json = await resp.json()
 	} catch (e) {
-		console.log(e)
+		console.error(e)
+	}
+
+	const newHistory = [...messages]
+
+	if (json) {
+		newHistory.push(json.choices[0].message)
 	}
 
-	return json
+	return {
+		response: json,
+		history: newHistory,
+	}
 }
 
 type MessageContent =
 	| string
 	| (
-			| string
-			| {
-					type: 'image_url'
-					image_url:
-						| string
-						| {
-								url: string
-								detail: 'low' | 'high' | 'auto'
-						  }
-			  }
-			| {
-					type: 'text'
-					text: string
-			  }
-	  )[]
+	| string
+	| {
+	type: 'image_url'
+	image_url:
+		| string
+		| {
+		url: string
+		detail: 'low' | 'high' | 'auto'
+	}
+}
+	| {
+	type: 'text'
+	text: string
+}
+	)[]
+
+export type CompletionMessageItem = {
+	role: 'system' | 'user' | 'assistant' | 'function'
+	content: MessageContent
+	name?: string | undefined
+};
 
 export type GPT4VCompletionRequest = {
 	model: 'gpt-4-vision-preview'
-	messages: {
-		role: 'system' | 'user' | 'assistant' | 'function'
-		content: MessageContent
-		name?: string | undefined
-	}[]
+	messages: CompletionMessageItem[]
 	functions?: any[] | undefined
 	function_call?: any | undefined
 	stream?: boolean | undefined
@@ -114,8 +162,8 @@ export type GPT4VCompletionRequest = {
 	presence_penalty?: number | undefined
 	logit_bias?:
 		| {
-				[x: string]: number
-		  }
+		[x: string]: number
+	}
 		| undefined
 	stop?: (string[] | string) | undefined
 }