Implement transpiler support

com.dynamo.cr/com.dynamo.cr.bob/src/com/defold/extension/pipeline/ILuaTranspiler.java

@@ -0,0 +1,114 @@
+// Copyright 2020-2024 The Defold Foundation
+// Copyright 2014-2020 King
+// Copyright 2009-2014 Ragnar Svensson, Christian Murray
+// Licensed under the Defold License version 1.0 (the "License"); you may not use
+// this file except in compliance with the License.
+//
+// You may obtain a copy of the License, together with FAQs at
+// https://www.defold.com/license
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+package com.defold.extension.pipeline;
+
+import java.io.File;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Implement this interface in com.defold.extension.pipeline package with no-argument constructor
+ * to add support for transpiling a language to Lua that is then will be used during the build
+ */
+public interface ILuaTranspiler {
+
+ /**
+ * @return A resource path to a build file, a project-relative path starting with slash,
+ * e.g. {@code "/tlconfig.lua"}
+ */
+ String getBuildFileResourcePath();
+
+ /**
+ * @return A file extension for the source code files of the transpiled language,
+ * e.g. {@code "ext"}
+ */
+ String getSourceExt();
+
+ /**
+ * Build the project from the source dir to output dir
+ *
+ * @param pluginDir a dir inside the project root that contains unpacked plugins that may be
+ * used for compilation (i.e. transpiler binaries)
+ * @param sourceDir a dir that is guaranteed to have all the source code files as reported
+ * by {@link #getSourceExt()}, and a build file, as reported by
+ * {@link #getBuildFileResourcePath()}. This might be a real project dir,
+ * or it could be a temporary directory if some sources are coming from
+ * the dependency zip files.
+ * @param outputDir a dir to put the transpiled lua files to. All lua files from the
+ * directory will be compiled as a part of the project compilation. The
+ * directory is preserved over editor/bob transpiler runs and editor restarts,
+ * so it's a responsibility of the transpiler to clear it from the old files
+ * @return a possibly empty or nullable list of build issues
+ */
+ List<Issue> transpile(File pluginDir, File sourceDir, File outputDir) throws Exception;
+
+ final class Issue {
+ public final Severity severity;
+ public final String resourcePath;
+ public final int lineNumber;
+ public final String message;
+
+ /**
+ * @param severity issue severity
+ * @param resourcePath compiled resource path of the build file or some of the source
+ * files relative to project root, must start with slash
+ * @param lineNumber 1-indexed line number
+ * @param message issue message to present to the user
+ */
+ public Issue(Severity severity, String resourcePath, int lineNumber, String message) {
+ if (severity == null) {
+ throw new IllegalArgumentException("Issue severity must not be null");
+ }
+ if (resourcePath == null || resourcePath.isEmpty() || resourcePath.charAt(0) != '/') {
+ throw new IllegalArgumentException("Invalid issue resource path: " + resourcePath);
+ }
+ if (lineNumber <= 0) {
+ throw new IllegalArgumentException("Invalid issue line number: " + lineNumber);
+ }
+ if (message == null || message.isEmpty()) {
+ throw new IllegalArgumentException("Invalid issue message: " + message);
+ }
+ this.severity = severity;
+ this.resourcePath = resourcePath;
+ this.lineNumber = lineNumber;
+ this.message = message;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Issue issue = (Issue) o;
+ return lineNumber == issue.lineNumber && severity == issue.severity && Objects.equals(resourcePath, issue.resourcePath) && Objects.equals(message, issue.message);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(severity, resourcePath, lineNumber, message);
+ }
+
+ @Override
+ public String toString() {
+ return "Issue{" +
+ "severity=" + severity +
+ ", resourcePath='" + resourcePath + '\'' +
+ ", lineNumber=" + lineNumber +
+ ", message='" + message + '\'' +
+ '}';
+ }
+ }
+
+ enum Severity {INFO, WARNING, ERROR}
+}

com.dynamo.cr/com.dynamo.cr.bob/src/com/dynamo/bob/Project.java

@@ -33,6 +33,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URI;
+import java.nio.file.attribute.FileTime;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -54,11 +55,22 @@
 import java.util.jar.Attributes;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
+import java.util.logging.Level;
 import java.util.stream.Collectors;
 import java.util.zip.ZipException;
 import java.util.zip.ZipFile;
 import java.util.zip.ZipOutputStream;
 
+import com.defold.extension.pipeline.ILuaTranspiler;
+import com.dynamo.bob.fs.ClassLoaderMountPoint;
+import com.dynamo.bob.fs.DefaultFileSystem;
+import com.dynamo.bob.fs.DefaultResource;
+import com.dynamo.bob.fs.FileSystemMountPoint;
+import com.dynamo.bob.fs.FileSystemWalker;
+import com.dynamo.bob.fs.IFileSystem;
+import com.dynamo.bob.fs.IResource;
+import com.dynamo.bob.fs.ZipMountPoint;
+import com.dynamo.bob.plugin.PluginScanner;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
@@ -79,11 +91,6 @@
 import com.dynamo.bob.bundle.BundleHelper;
 import com.dynamo.bob.bundle.IBundler;
 import com.dynamo.bob.bundle.BundlerParams;
-import com.dynamo.bob.fs.ClassLoaderMountPoint;
-import com.dynamo.bob.fs.FileSystemWalker;
-import com.dynamo.bob.fs.IFileSystem;
-import com.dynamo.bob.fs.IResource;
-import com.dynamo.bob.fs.ZipMountPoint;
 import com.dynamo.bob.pipeline.ExtenderUtil;
 import com.dynamo.bob.pipeline.IShaderCompiler;
 import com.dynamo.bob.pipeline.ShaderCompilers;
@@ -450,7 +457,7 @@ public Task<?> createTask(String inputPath, Class<? extends Builder<?>> builderC
  /**
  * Create task from resource. Typically called from builder
  * that create intermediate output/input-files
- * @param input input resource
+ * @param inputResource input resource
  * @return task
  * @throws CompileExceptionError
  */
@@ -467,7 +474,7 @@ public Task<?> createTask(IResource inputResource) throws CompileExceptionError
  /**
  * Create task from resource with explicit builder.
  * Make sure that task is unique.
- * @param input input resource
+ * @param inputResource input resource
  * @param builderClass class to build resource with
  * @return task
  * @throws CompileExceptionError
@@ -1327,7 +1334,7 @@ public void scanJavaClasses() throws IOException, CompileExceptionError {
  }
 
  private Future buildRemoteEngine(IProgress monitor, ExecutorService executor) {
- Callable<Void> callable = new Callable<>() {
+ Callable<Void> callable = new Callable<Void>() {
  public Void call() throws Exception {
  logInfo("Build Remote Engine...");
  TimeProfiler.addMark("StartBuildRemoteEngine", "Build Remote Engine");
@@ -1425,6 +1432,89 @@ private void configurePreBuildProjectOptions() throws IOException, CompileExcept
  }
  }
 
+ private void transpileLua(IProgress monitor) throws CompileExceptionError, IOException {
+ List<ILuaTranspiler> transpilers = PluginScanner.getOrCreatePlugins("com.defold.extension.pipeline", ILuaTranspiler.class);
+ if (transpilers != null) {
+ IProgress transpilerProgress = monitor.subProgress(1);
+ transpilerProgress.beginTask("Transpiling to Lua", 1);
+ for (ILuaTranspiler transpiler : transpilers) {
+ IResource buildFileResource = getResource(transpiler.getBuildFileResourcePath());
+ if (buildFileResource.exists()) {
+ String ext = "." + transpiler.getSourceExt();
+ List<IResource> sources = inputs.stream()
+ .filter(s -> s.endsWith(ext))
+ .map(this::getResource)
+ .collect(Collectors.toList());
+ if (!sources.isEmpty()) {
+ boolean useProjectDir = buildFileResource instanceof DefaultResource && sources.stream().allMatch(s -> s instanceof DefaultResource);
+ File sourceDir;
+ if (useProjectDir) {
+ sourceDir = new File(rootDirectory);
+ } else {
+ sourceDir = Files.createTempDirectory("tr-" + transpiler.getClass().getSimpleName()).toFile();
+ buildFileResource.getContent();
+ File buildFile = new File(sourceDir, buildFileResource.getPath());
+ Path buildFilePath = buildFile.toPath();
+ Files.write(buildFilePath, buildFileResource.getContent());
+ Files.setLastModifiedTime(buildFilePath, FileTime.fromMillis(buildFileResource.getLastModified()));
+ for (IResource source : sources) {
+ Path sourcePath = new File(sourceDir, source.getPath()).toPath();
+ Files.write(sourcePath, source.getContent());
+ Files.setLastModifiedTime(sourcePath, FileTime.fromMillis(source.getLastModified()));
+ }
+ }
+ File outputDir = new File(rootDirectory, "build/tr/" + transpiler.getClass().getSimpleName());
+ Files.createDirectories(outputDir.toPath());
+ try {
+ List<ILuaTranspiler.Issue> issues = transpiler.transpile(new File(getPluginsDirectory()), sourceDir, outputDir);
+ List<ILuaTranspiler.Issue> errors = issues.stream().filter(issue -> issue.severity == ILuaTranspiler.Severity.ERROR).collect(Collectors.toList());
+ if (!errors.isEmpty()) {
+ MultipleCompileException exception = new MultipleCompileException("Transpilation failed", null);
+ errors.forEach(issue -> exception.addIssue(issue.severity.ordinal(), getResource(issue.resourcePath), issue.message, issue.lineNumber));
+ throw exception;
+ } else {
+ issues.forEach(issue -> {
+ Level level;
+ switch (issue.severity) {
+ case INFO:
+ level = Level.INFO;
+ break;
+ case WARNING:
+ level = Level.WARNING;
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ logger.log(level, issue.resourcePath + ":" + issue.lineNumber + ": " + issue.message);
+ });
+ }
+ DefaultFileSystem fs = new DefaultFileSystem();
+ fs.setRootDirectory(outputDir.toString());
+ ArrayList<String> results = new ArrayList<>();
+ fs.walk("", new FileSystemWalker() {
+ @Override
+ public void handleFile(String path, Collection<String> results) {
+ if (path.endsWith(".lua")) {
+ results.add(path);
+ }
+ }
+ }, results);
+ inputs.addAll(results);
+ fileSystem.addMountPoint(new FileSystemMountPoint(fileSystem, fs));
+ } catch (Exception e) {
+ throw new CompileExceptionError(buildFileResource, 1, "Transpilation failed", e);
+ } finally {
+ if (!useProjectDir) {
+ FileUtils.deleteDirectory(sourceDir);
+ }
+ }
+ }
+ }
+ }
+ transpilerProgress.done();
+ }
+ }
+
  private List<TaskResult> createAndRunTasks(IProgress monitor) throws IOException, CompileExceptionError {
  // Do early test if report files are writable before we start building
  boolean generateReport = this.hasOption("build-report") || this.hasOption("build-report-html");
@@ -1580,6 +1670,15 @@ private List<TaskResult> doBuild(IProgress monitor, String... commands) throws T
  Future<Void> remoteBuildFuture = null;
  // Get or build engine binary
  boolean shouldBuildRemoteEngine = ExtenderUtil.hasNativeExtensions(this);
+ boolean shouldBuildProject = shouldBuildEngine() && BundleHelper.isArchiveIncluded(this);
+
+ if (shouldBuildProject) {
+ // do this before buildRemoteEngine to prevent concurrent modification exception, since
+ // lua transpilation adds new mounts with compiled Lua that buildRemoteEngine iterates over
+ // when sending to extender
+ transpileLua(monitor);
+ }
+
  if (shouldBuildRemoteEngine) {
  remoteBuildFuture = buildRemoteEngine(monitor, executor);
  }
@@ -1593,7 +1692,7 @@ private List<TaskResult> doBuild(IProgress monitor, String... commands) throws T
  }
  }
 
- if (shouldBuildEngine() && BundleHelper.isArchiveIncluded(this)) {
+ if (shouldBuildProject) {
  result = createAndRunTasks(monitor);
  }