5003 equation-from-image includes a latex equation style guide (#5027)

packages/gollm/gollm_openai/prompts/latex_style_guide.py

@@ -1,5 +1,5 @@
 LATEXT_STYLE_GUIDE = """
-1) Derivatives must be written in Leibniz notation
+1) Derivatives must be written in Leibniz notation (for example, \\frac{d X}{d t}). Equations that are written in other notations, like Newton or Lagrange, should be converted.
 2) First-order derivative must be on the left of the equal sign
 3) Use whitespace to indicate multiplication
  a) "*" is optional but probably should be avoided
@@ -15,4 +15,5 @@
 11) Avoid using homoglyphs
 12) Avoid words or multi-character names for variables and names
  a) Use camel case to express multi-word or multi-character names
+13) Do not use \\cdot for multiplication. Use whitespace instead.
 """

packages/gollm/gollm_openai/tool_utils.py

@@ -16,6 +16,7 @@
 from gollm_openai.prompts.equations_from_image import EQUATIONS_FROM_IMAGE_PROMPT
 from gollm_openai.prompts.general_instruction import GENERAL_INSTRUCTION_PROMPT
 from gollm_openai.prompts.interventions_from_document import INTERVENTIONS_FROM_DOCUMENT_PROMPT
+from gollm_openai.prompts.latex_style_guide import LATEXT_STYLE_GUIDE
 from gollm_openai.prompts.model_card import INSTRUCTIONS
 from gollm_openai.prompts.model_meta_compare import MODEL_METADATA_COMPARE_PROMPT
 from openai import OpenAI
@@ -74,6 +75,11 @@ def equations_from_image(image: str) -> dict:
  response_schema = json.load(config_file)
  validate_schema(response_schema)
 
+ print("Building prompt to extract equations an image...")
+ prompt = EQUATIONS_FROM_IMAGE_PROMPT.format(
+ style_guide=LATEXT_STYLE_GUIDE
+ )
+
  client = OpenAI()
  output = client.chat.completions.create(
  model="gpt-4o-mini",
@@ -94,7 +100,7 @@ def equations_from_image(image: str) -> dict:
  {
  "role": "user",
  "content": [
- {"type": "text", "text": EQUATIONS_FROM_IMAGE_PROMPT},
+ {"type": "text", "text": prompt},
  {"type": "image_url", "image_url": {"url": f"{base64_image_str}{image}"}}
  ]
  },
@@ -245,7 +251,7 @@ def model_card_chain(amr: str, research_paper: str = None) -> dict:
  model="gpt-4o-2024-08-06",
  temperature=0,
  frequency_penalty=0,
- max_tokens=4000,
+ max_tokens=16000,
  presence_penalty=0,
  seed=123,
  top_p=1,
@@ -391,7 +397,7 @@ def compare_models(amrs: List[str]) -> str:
  presence_penalty=0,
  seed=123,
  temperature=0,
- max_tokens=2048,
+ max_tokens=16000,
  response_format={
  "type": "json_schema",
  "json_schema": {