Merge branch 'ml4ai:main' into main

.github/workflows/tests-and-docs.yml

@@ -86,7 +86,7 @@ jobs:
  # Install tree-sitter parser (for Python component unit tests)
  - name: Install tree-sitter parsers
  working-directory: .
- run: python skema/program_analysis/tree_sitter_parsers/build_parsers.py --all
+ run: python skema/program_analysis/tree_sitter_parsers/build_parsers.py --ci --all
 
 
  # docs (API)

docs/dev/cast_frontend.md

@@ -0,0 +1,9 @@
+## CAST FrontEnd Generation Notes
+### Using Var vs Name nodes
+Currently in the CAST generation we have a convention on when to use Var and Name nodes.
+The GroMEt generation depends on these being conistent, otherwise there will be errors in the generation.
+In the future this convention might change, or be eliminated altogether, but for now this is the current set of rules.
+
+- If the variable in question is being stored into (i.e. as the result of an assignment), then we use Var. Even if it's a variable that has already been defined.
+- If the variable in question is being read from (i.e. being used in an expression), then we use Name.
+- Whenever we're creating a function call Call() node, the name of the function is specified using the Name node.

mkdocs.yml

@@ -45,6 +45,7 @@ nav:
  - Generating code2fn model coverage reports: "dev/generating_code2fn_model_coverage.md"
  - Using code ingestion frontends: "dev/using_code_ingestion_frontends.md"
  - Using tree-sitter preprocessor: "dev/using_tree_sitter_preprocessor.md"
+ - CAST Front-end generation: "dev/cast_frontend.md"
  - Coverage:
  - Code2fn coverage reports: "coverage/code2fn_coverage/report.html"
  - TA1 Integration Dashboard: "https://integration-dashboard.terarium.ai/TA1"

pyproject.toml

@@ -23,6 +23,7 @@ dependencies=[
  "typing_extensions", # see https://github.com/pydantic/pydantic/issues/5821#issuecomment-1559196859
  "fastapi~=0.100.0",
  "starlette",
+ "httpx",
  "pydantic>=2.0.0",
  "uvicorn",
  "python-multipart",
@@ -42,7 +43,7 @@ dynamic = ["readme"]
 # Pygraphviz is often tricky to install, so we reserve it for the dev extras
 # list.
 # - six: Required by auto-generated Swagger models
-dev = ["pytest", "pytest-cov", "pytest-xdist", "httpx", "black", "mypy", "coverage", "pygraphviz", "six"]
+dev = ["pytest", "pytest-cov", "pytest-xdist", "pytest-asyncio", "black", "mypy", "coverage", "pygraphviz", "six"]
 
 demo = ["notebook"]
 

skema/gromet/fn/gromet_fn_module.py

@@ -191,7 +191,7 @@ def fn_array(self, fn_array):
  def metadata_collection(self):
  """Gets the metadata_collection of this GrometFNModule. # noqa: E501
 
- Table (array) of lists (arrays) of metadata, where each list in the Table-array represents the collection of metadata associated with a GroMEt object. # noqa: E501
+ Table (array) of lists (arrays) of metadata, where each list in the Table-array represents the collection of metadata associated with a GrometFNModule object. # noqa: E501
 
  :return: The metadata_collection of this GrometFNModule. # noqa: E501
  :rtype: list[list[Metadata]]
@@ -202,7 +202,7 @@ def metadata_collection(self):
  def metadata_collection(self, metadata_collection):
  """Sets the metadata_collection of this GrometFNModule.
 
- Table (array) of lists (arrays) of metadata, where each list in the Table-array represents the collection of metadata associated with a GroMEt object. # noqa: E501
+ Table (array) of lists (arrays) of metadata, where each list in the Table-array represents the collection of metadata associated with a GrometFNModule object. # noqa: E501
 
  :param metadata_collection: The metadata_collection of this GrometFNModule. # noqa: E501
  :type: list[list[Metadata]]

skema/img2mml/eqn2mml.py

@@ -7,7 +7,7 @@
 
 from typing import Text
 from typing_extensions import Annotated
-from fastapi import APIRouter, FastAPI, Response, Request, Query, UploadFile
+from fastapi import APIRouter, FastAPI, status, Response, Request, Query, UploadFile
 from skema.rest.proxies import SKEMA_MATHJAX_ADDRESS
 from skema.img2mml.api import (
  get_mathml_from_bytes,
@@ -86,23 +86,23 @@ def process_latex_equation(eqn: Text) -> Response:
  "/img2mml/healthcheck",
  summary="Check health of eqn2mml service",
  response_model=int,
- status_code=200,
+ status_code=status.HTTP_200_OK,
 )
 def img2mml_healthcheck() -> int:
- return 200
+ return status.HTTP_200_OK
 
 
 @router.get(
  "/latex2mml/healthcheck",
  summary="Check health of mathjax service",
  response_model=int,
- status_code=200,
+ status_code=status.HTTP_200_OK,
 )
 def latex2mml_healthcheck() -> int:
  try:
  return int(requests.get(f"{SKEMA_MATHJAX_ADDRESS}/healthcheck").status_code)
  except:
- return 500
+ return status.HTTP_500_INTERNAL_SERVER_ERROR
 
 
 @router.post("/image/mml", summary="Get MathML representation of an equation image")

skema/isa/data.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+mml = """<math>
+ <mfrac>
+ <mrow>
+ <mi>&#x2202;</mi>
+ <mi>H</mi>
+ </mrow>
+ <mrow>
+ <mi>&#x2202;</mi>
+ <mi>t</mi>
+ </mrow>
+ </mfrac>
+ <mo>=</mo>
+ <mi>&#x2207;</mi>
+ <mo>&#x22C5;</mo>
+ <mo>(</mo>
+ <mi>&#x0393;</mi>
+ <msup>
+ <mi>H</mi>
+ <mrow>
+ <mi>n</mi>
+ <mo>+</mo>
+ <mn>2</mn>
+ </mrow>
+ </msup>
+ <mo>|</mo>
+ <mi>&#x2207;</mi>
+ <mi>H</mi>
+ <msup>
+ <mo>|</mo>
+ <mrow>
+ <mi>n</mi>
+ <mo>&#x2212;</mo>
+ <mn>1</mn>
+ </mrow>
+ </msup>
+ <mi>&#x2207;</mi>
+ <mi>H</mi>
+ <mo>)</mo>
+ </math>
+ """
+
+expected = 'digraph G {\n0 [color=blue, label="Div(Γ*(H^(n+2))*(Abs(Grad(H))^(n-1))*Grad(H))"];\n1 [color=blue, label="D(1, t)(H)"];\n2 [color=blue, label="Γ*(H^(n+2))*(Abs(Grad(H))^(n-1))*Grad(H)"];\n3 [color=blue, label="Γ"];\n4 [color=blue, label="H^(n+2)"];\n5 [color=blue, label="H"];\n6 [color=blue, label="n+2"];\n7 [color=blue, label="n"];\n8 [color=blue, label="2"];\n9 [color=blue, label="Abs(Grad(H))^(n-1)"];\n10 [color=blue, label="Abs(Grad(H))"];\n11 [color=blue, label="Grad(H)"];\n12 [color=blue, label="n-1"];\n13 [color=blue, label="1"];\n1 -> 0 [color=blue, label="="];\n2 -> 0 [color=blue, label="Div"];\n3 -> 2 [color=blue, label="*"];\n4 -> 2 [color=blue, label="*"];\n5 -> 4 [color=blue, label="^"];\n6 -> 4 [color=blue, label="^"];\n7 -> 6 [color=blue, label="+"];\n8 -> 6 [color=blue, label="+"];\n9 -> 2 [color=blue, label="*"];\n10 -> 9 [color=blue, label="^"];\n11 -> 10 [color=blue, label="Abs"];\n5 -> 11 [color=blue, label="Grad"];\n12 -> 9 [color=blue, label="^"];\n7 -> 12 [color=blue, label="+"];\n13 -> 12 [color=blue, label="-"];\n11 -> 2 [color=blue, label="*"];\n}\n'

skema/isa/isa_service.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
 
-from fastapi import FastAPI, File
+from fastapi import Depends, FastAPI, APIRouter, status
 from skema.isa.lib import align_mathml_eqs
+import skema.isa.data as isa_data
+from skema.rest import utils
 from pydantic import BaseModel
+import httpx
 
-# Create a web app using FastAPI
+from skema.rest.proxies import SKEMA_RS_ADDESS
 
-app = FastAPI()
+router = APIRouter()
 
 
 # Model for ISA_Result
@@ -15,17 +18,39 @@ class ISA_Result(BaseModel):
  union_graph: str = None
 
 
-@app.get("/ping", summary="Ping endpoint to test health of service")
-def ping():
- return "The ISA service is running."
+@router.get(
+ "/healthcheck", 
+ summary="Status of ISA service",
+ response_model=int,
+ status_code=status.HTTP_200_OK
+)
+async def healthcheck(client: httpx.AsyncClient = Depends(utils.get_client)) -> int:
+ res = await client.get(f"{SKEMA_RS_ADDESS}/ping")
+ return res.status_code
 
 
-@app.put("/align-eqns", summary="Align two MathML equations")
+@router.post(
+ "/align-eqns", 
+ summary="Align two MathML equations"
+)
 async def align_eqns(
- file1: str, file2: str, mention_json1: str = "", mention_json2: str = ""
+ mml1: str, mml2: str, mention_json1: str = "", mention_json2: str = ""
 ) -> ISA_Result:
- """
+ f"""
  Endpoint for align two MathML equations.
+
+ ### Python example
+
+ ```
+ import requests
+
+ request = {{
+ "mml1": {isa_data.mml},
+ "mml2": {isa_data.mml}
+ }}
+
+ response=requests.post("/isa/align-eqns", json=request)
+ res = response.json()
  """
  (
  matching_ratio,
@@ -36,8 +61,16 @@ async def align_eqns(
  aligned_indices2,
  union_graph,
  perfectly_matched_indices1,
- ) = align_mathml_eqs(file1, file2, mention_json1, mention_json2)
- ir = ISA_Result()
- ir.matching_ratio = matching_ratio
- ir.union_graph = union_graph.to_string()
- return ir
+ ) = align_mathml_eqs(mml1, mml2, mention_json1, mention_json2)
+ return ISA_Result(
+ matching_ratio = matching_ratio,
+ union_graph = union_graph.to_string()
+ )
+
+
+app = FastAPI()
+app.include_router(
+ router,
+ prefix="/isa",
+ tags=["isa"],
+)

skema/isa/lib.py

@@ -2,13 +2,14 @@
 """
 All the functions required by performing incremental structure alignment (ISA)
 Author: Liang Zhang ([email protected])
-Updated date: August 24, 2023
+Updated date: December 18, 2023
 """
 import json
 import warnings
 from typing import List, Any, Union, Dict
 from numpy import ndarray
 from pydot import Dot
+from skema.rest.proxies import SKEMA_RS_ADDESS
 
 warnings.filterwarnings("ignore")
 import requests
@@ -173,8 +174,9 @@ def generate_graph(file: str = "", render: bool = False) -> pydot.Dot:
  content = f.read()
 
  digraph = requests.put(
- "http://localhost:8080/mathml/math-exp-graph", data=content.encode("utf-8")
+ f"{SKEMA_RS_ADDESS}/mathml/math-exp-graph", data=content.encode("utf-8")
  )
+
  if render:
  src = Source(digraph.text)
  src.render("doctest-output/mathml_exp_tree", view=True)
@@ -671,8 +673,8 @@ def check_square_array(arr: np.ndarray) -> List[int]:
 
 
 def align_mathml_eqs(
- file1: str = "",
- file2: str = "",
+ mml1: str = "",
+ mml2: str = "",
  mention_json1: str = "",
  mention_json2: str = "",
  mode: int = 2,
@@ -685,7 +687,7 @@ def align_mathml_eqs(
  [1] Fishkind, D. E., Adali, S., Patsolic, H. G., Meng, L., Singh, D., Lyzinski, V., & Priebe, C. E. (2019).
  Seeded graph matching. Pattern recognition, 87, 203-215.
 
- Input: the paths of the two equation MathMLs; mention_json1: the mention file of paper 1; mention_json1: the mention file of paper 2;
+ Input: mml1 & mml2: the file path or contents of the two equation MathMLs; mention_json1: the mention file of paper 1; mention_json1: the mention file of paper 2;
  mode 0: without considering any priors; mode 1: having a heuristic prior
  with the similarity of node labels; mode 2: using the variable definitions
  Output:
@@ -698,8 +700,8 @@ def align_mathml_eqs(
  union_graph: the visualization of the alignment result
  perfectly_matched_indices1: strictly matched node indices in Graph 1
  """
- graph1 = generate_graph(file1)
- graph2 = generate_graph(file2)
+ graph1 = generate_graph(mml1)
+ graph2 = generate_graph(mml2)
 
  amatrix1, node_labels1 = generate_amatrix(graph1)
  amatrix2, node_labels2 = generate_amatrix(graph2)

skema/program_analysis/CAST/fortran/node_helper.py

@@ -1,8 +1,10 @@
+import itertools
 from typing import List, Dict
-from skema.program_analysis.CAST2FN.model.cast import SourceRef
 
 from tree_sitter import Node
 
+from skema.program_analysis.CAST2FN.model.cast import SourceRef
+
 CONTROL_CHARACTERS = [
  ",",
  "=",
@@ -41,7 +43,7 @@ def __init__(self, source: str, source_file_name: str):
  # get_identifier optimization variables
  self.source_lines = source.splitlines(keepends=True)
  self.line_lengths = [len(line) for line in self.source_lines]
- self.line_length_sums = [sum(self.line_lengths[:i+1]) for i in range(len(self.source_lines))]
+ self.line_length_sums = list(itertools.accumulate(self.line_lengths))#[sum(self.line_lengths[:i+1]) for i in range(len(self.source_lines))]
 
  def get_source_ref(self, node: Node) -> SourceRef:
  """Given a node and file name, return a CAST SourceRef object."""
@@ -96,6 +98,9 @@ def get_children_by_types(node: Node, types: List):
  """Takes in a node and a list of types as inputs and returns all children matching those types. Otherwise, return an empty list"""
  return [child for child in node.children if child.type in types]
 
+def get_children_except_types(node: Node, types: List):
+ """Takes in a node and a list of types as inputs and returns all children not matching those types. Otherwise, return an empty list"""
+ return [child for child in node.children if child.type not in types]
 
 def get_first_child_index(node, type: str):
  """Get the index of the first child of node with type type."""

skema/program_analysis/CAST/fortran/preprocessor/preprocess.py

@@ -34,7 +34,7 @@ def preprocess(
  """
  # NOTE: The order of preprocessing steps does matter. We have to run the GCC preprocessor before correcting the continuation lines or there could be issues
 
- # TODO: Create single location for generating include base path 
+ # TODO: Create single location for generating include base path
  source = source_path.read_text()
 
  # Get paths for intermediate products
@@ -67,21 +67,21 @@ def preprocess(
 
  # Step 2: Correct include directives to remove system references
  source = fix_include_directives(source)
- 
+
  # Step 3: Process with gcc c-preprocessor
  include_base_directory = Path(source_path.parent, f"include_{source_path.stem}")
  if not include_base_directory.exists():
  include_base_directory = include_base_directory.parent
  source = run_c_preprocessor(source, include_base_directory)
  if out_gcc:
  gcc_path.write_text(source)
- 
+
  # Step 4: Prepare for tree-sitter
  # This step removes any additional preprocessor directives added or not removed by GCC
  source = "\n".join(
  ["!" + line if line.startswith("#") else line for line in source.splitlines()]
  )
- 
+
  # Step 5: Check for unsupported idioms
  if out_unsupported:
  unsupported_path.write_text(
@@ -173,7 +173,7 @@ def fix_include_directives(source: str) -> str:
 def run_c_preprocessor(source: str, include_base_path: Path) -> str:
  """Run the gcc c-preprocessor. Its run from the context of the include_base_path, so that it can find all included files"""
  result = run(
- ["gcc", "-cpp", "-E", "-"],
+ ["gcc", "-cpp", "-E", "-x", "f95", "-"],
  input=source,
  text=True,
  capture_output=True,
@@ -183,8 +183,14 @@ def run_c_preprocessor(source: str, include_base_path: Path) -> str:
  return result.stdout
 
 
+def convert_assigned(source: str) -> str:
+ """Convered ASSIGNED GO TO to COMPUTED GO TO"""
+ pass
+
+
 def convert_to_free_form(source: str) -> str:
  """If fixed-form Fortran source, convert to free-form"""
+
  def validate_parse_tree(source: str) -> bool:
  """Parse source with tree-sitter and check if an error is returned."""
  language = Language(INSTALLED_LANGUAGES_FILEPATH, "fortran")
@@ -204,7 +210,7 @@ def validate_parse_tree(source: str) -> bool:
  )
  if validate_parse_tree(free_source):
  return free_source
- 
+
  return source