Replace ANTLR with hand-rolled parser (#917)

Co-authored-by: Kushal Pisavadia <[email protected]>
Co-authored-by: Daniel Chao <[email protected]>

Author: 3 people

.gitignore

@@ -5,6 +5,7 @@
 .gradle/
 build/
 generated/
+testgenerated/
 
 # IntelliJ
 .idea/

DEVELOPMENT.adoc

@@ -83,8 +83,8 @@ based on version information from https://search.maven.org, https://plugins.grad
 
 * Truffle code generation is performed by Truffle's annotation processor, which runs as part of task `:pkl-core:compileJava`
 ** Output dir is `generated/truffle/`
-* ANTLR code generation is performed by task `:pkl-core:generateGrammarSource`
-** Output dir is `generated/antlr/`
+* ANTLR code generation is performed by task `:pkl-core:generateTestGrammarSource`
+** Output dir is `testgenerated/antlr/`
 
 == Remote JVM Debugging
 
@@ -96,12 +96,6 @@ Example: `./gradlew test -Djvmdebug=true`
 == Resources
 For automated build setup examples see our https://github.com/apple/pkl/blob/main/.circleci/[CircleCI] jobs like our https://github.com/apple/pkl/blob/main/.circleci/jobs/BuildNativeJob.pkl[BuildNativeJob.pkl], where we build Pkl automatically.
 
-=== ANTLR
-
-* https://github.com/antlr/antlr4/blob/master/doc/index.md[Documentation]
-* https://groups.google.com/forum/#!forum/antlr-discussion[Forums]
-* https://github.com/mobileink/lab.clj.antlr/tree/main/doc[Some third-party docs]
-
 === Truffle
 
 * http://ssw.jku.at/Research/Projects/JVM/Truffle.html[Homepage]

THIRD-PARTY-NOTICES.txt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,8 +25,6 @@ val graal: Configuration by configurations.creating
 @Suppress("UnstableApiUsage")
 dependencies {
   jmh(projects.pklCore)
-  // necessary because antlr4-runtime is declared as implementation dependency in pkl-core.gradle
-  jmh(libs.antlrRuntime)
   truffle(libs.truffleApi)
   graal(libs.graalCompiler)
 }

bench/bench.gradle.kts

@@ -1,7 +1,6 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.sf.jopt-simple:jopt-simple:5.0.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.apache.commons:commons-math3:3.6.1=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath

bench/gradle.lockfile

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

bench/src/jmh/java/org/pkl/core/parser/ParserBenchmark.java

@@ -39,7 +39,6 @@ val firstPartySourcesJarsConfiguration: Configuration =
 val relocations =
   mapOf(
     // pkl-core dependencies
-    "org.antlr.v4." to "org.pkl.thirdparty.antlr.v4.",
     "org.organicdesign.fp." to "org.pkl.thirdparty.paguro.",
     "org.snakeyaml.engine." to "org.pkl.thirdparty.snakeyaml.engine.",
     "org.msgpack." to "org.pkl.thirdparty.msgpack.",

buildSrc/src/main/kotlin/pklFatJar.gradle.kts

@@ -3196,7 +3196,6 @@ This section discusses language features that are generally more relevant to tem
 <<glob-patterns,Glob Patterns>> +
 <<doc-comments,Doc Comments>> +
 <<name-resolution,Name Resolution>> +
-<<grammar-definition,Grammar Definition>> +
 <<reserved-keywords,Reserved Keywords>> +
 <<blank-identifiers,Blank Identifiers>> +
 <<projects,Projects>> +
@@ -5274,11 +5273,6 @@ For example, the prototype chain of value `42` contains, now listed from top to
 
 A prototype chain never contains a non-object value, such as `42`.
 
-[[grammar-definition]]
-=== Grammar Definition
-
-Pkl's link:{uri-antlr4}[ANTLR 4] grammar is defined in link:{uri-github-PklLexer}[PklLexer.g4] and link:{uri-github-PklParser}[PklParser.g4].
-
 [[reserved-keywords]]
 === Reserved keywords
 

docs/modules/language-reference/pages/index.adoc

@@ -16,15 +16,14 @@ import org.pkl.core.Loggers
 import org.pkl.core.SecurityManagers
 import org.pkl.core.StackFrameTransformers
 import org.pkl.core.module.ModuleKeyFactories
-import org.pkl.core.parser.LexParseException
-import org.pkl.core.parser.Parser
-import org.pkl.core.parser.antlr.PklParser
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
 import org.pkl.core.repl.ReplServer
 import org.pkl.core.util.IoUtils
-import org.antlr.v4.runtime.ParserRuleContext
 import org.pkl.core.http.HttpClient
+import org.pkl.core.parser.Parser
+import org.pkl.core.parser.ParserError
+import org.pkl.core.parser.ast.ClassProperty
 import org.pkl.core.resource.ResourceReaders
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
@@ -303,7 +302,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
 
       override fun getType() = Type.TEST
 
-      private val parsed: ParserRuleContext by lazy {
+      private val parsed: org.pkl.core.parser.ast.Node by lazy {
         when (language) {
           "pkl" -> Parser().parseModule(code)
           "pkl-expr" -> Parser().parseExpressionInput(code)
@@ -318,7 +317,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
             if (expectError) {
               throw AssertionError("Expected a parse error, but got none.")
             }
-          } catch (e: LexParseException) {
+          } catch (e: ParserError) {
             if (!expectError) {
               throw AssertionError(e.message)
             }
@@ -335,7 +334,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
           )
         )
 
-        val properties = parsed.children.filterIsInstance<PklParser.ClassPropertyContext>()
+        val properties = parsed.children()?.filterIsInstance<ClassProperty>() ?: emptyList()
 
         val responses = mutableListOf<ReplResponse>()
 
@@ -344,7 +343,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
           responses.addAll(context.replServer.handleRequest(
             ReplRequest.Eval(
               "snippet",
-              prop.Identifier().text,
+              prop.name.value,
               false,
               true
             )

docs/src/test/kotlin/DocSnippetTests.kt

@@ -19,7 +19,6 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=test
 com.google.j2objc:j2objc-annotations:3.0.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.jayway.jsonpath:json-path:2.9.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.networknt:json-schema-validator:1.5.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=runtimeClasspath,testRuntimeClasspath
 commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 commons-io:commons-io:2.11.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath