Skip to content

Commit

Permalink
Implement transpiler support
Browse files Browse the repository at this point in the history
This changeset adds initial Bob and Editor support for plugins that transpile 3rd-party languages to Lua. Transpiler plugins need to implement an `ILuaTranspiler` interface. See [the Teal extension](https://github.com/defold/extension-teal) for reference implementation.

Fixes #8291
  • Loading branch information
vlaaad committed Apr 23, 2024
1 parent bcc6810 commit 68d639d
Show file tree
Hide file tree
Showing 41 changed files with 1,463 additions and 420 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package com.defold.extension.pipeline;

import java.io.File;
import java.util.List;
import java.util.Objects;

public interface ILuaTranspiler {

/**
* @return A resource path to a build file, a project-relative path starting with slash,
* e.g. {@code "/tlconfig.lua"}
*/
String getBuildFileResourcePath();

/**
* @return A file extension for the source code files of the transpiled language,
* e.g. {@code "ext"}
*/
String getSourceExt();

/**
* Build the project from the source dir to output dir
*
* @param pluginDir a dir inside the project root that contains unpacked plugins that may be
* used for compilation (i.e. transpiler binaries)
* @param sourceDir a dir that is guaranteed to have all the source code files as reported
* by {@link #getSourceExt()}, and a build file, as reported by
* {@link #getBuildFileResourcePath()}. This might be a real project dir,
* or it could be a temporary directory if some sources are coming from
* the dependency zip files.
* @param outputDir a dir to put the transpiled lua files to. All lua files from the
* directory will be compiled as a part of the project compilation. The
* directory is preserved over editor transpiler runs and editor restarts,
* so it's a responsibility of the transpiler to clear it from the old files
* @return a possibly empty or nullable list of build issues
*/
List<Issue> transpile(File pluginDir, File sourceDir, File outputDir) throws Exception;

final class Issue {
public final Severity severity;
public final String resourcePath;
public final int lineNumber;
public final String message;

/**
* @param severity issue severity
* @param resourcePath compiled resource path of the build file or some of the source
* files relative to project root, must start with slash
* @param lineNumber 1-indexed line number
* @param message issue message to present to the user
*/
public Issue(Severity severity, String resourcePath, int lineNumber, String message) {
if (severity == null) {
throw new IllegalArgumentException("Issue severity must not be null");
}
if (resourcePath == null || resourcePath.isEmpty() || resourcePath.charAt(0) != '/') {
throw new IllegalArgumentException("Invalid issue resource path: " + resourcePath);
}
if (lineNumber <= 0) {
throw new IllegalArgumentException("Invalid issue line number: " + lineNumber);
}
if (message == null || message.isEmpty()) {
throw new IllegalArgumentException("Invalid issue message: " + message);
}
this.severity = severity;
this.resourcePath = resourcePath;
this.lineNumber = lineNumber;
this.message = message;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Issue issue = (Issue) o;
return lineNumber == issue.lineNumber && severity == issue.severity && Objects.equals(resourcePath, issue.resourcePath) && Objects.equals(message, issue.message);
}

@Override
public int hashCode() {
return Objects.hash(severity, resourcePath, lineNumber, message);
}

@Override
public String toString() {
return "Issue{" +
"severity=" + severity +
", resourcePath='" + resourcePath + '\'' +
", lineNumber=" + lineNumber +
", message='" + message + '\'' +
'}';
}
}

enum Severity {INFO, WARNING, ERROR}
}
117 changes: 108 additions & 9 deletions com.dynamo.cr/com.dynamo.cr.bob/src/com/dynamo/bob/Project.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URI;
import java.nio.file.attribute.FileTime;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -54,11 +55,22 @@
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

import com.defold.extension.pipeline.ILuaTranspiler;
import com.dynamo.bob.fs.ClassLoaderMountPoint;
import com.dynamo.bob.fs.DefaultFileSystem;
import com.dynamo.bob.fs.DefaultResource;
import com.dynamo.bob.fs.FileSystemMountPoint;
import com.dynamo.bob.fs.FileSystemWalker;
import com.dynamo.bob.fs.IFileSystem;
import com.dynamo.bob.fs.IResource;
import com.dynamo.bob.fs.ZipMountPoint;
import com.dynamo.bob.plugin.PluginScanner;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
Expand All @@ -79,11 +91,6 @@
import com.dynamo.bob.bundle.BundleHelper;
import com.dynamo.bob.bundle.IBundler;
import com.dynamo.bob.bundle.BundlerParams;
import com.dynamo.bob.fs.ClassLoaderMountPoint;
import com.dynamo.bob.fs.FileSystemWalker;
import com.dynamo.bob.fs.IFileSystem;
import com.dynamo.bob.fs.IResource;
import com.dynamo.bob.fs.ZipMountPoint;
import com.dynamo.bob.pipeline.ExtenderUtil;
import com.dynamo.bob.pipeline.IShaderCompiler;
import com.dynamo.bob.pipeline.ShaderCompilers;
Expand Down Expand Up @@ -450,7 +457,7 @@ public Task<?> createTask(String inputPath, Class<? extends Builder<?>> builderC
/**
* Create task from resource. Typically called from builder
* that create intermediate output/input-files
* @param input input resource
* @param inputResource input resource
* @return task
* @throws CompileExceptionError
*/
Expand All @@ -467,7 +474,7 @@ public Task<?> createTask(IResource inputResource) throws CompileExceptionError
/**
* Create task from resource with explicit builder.
* Make sure that task is unique.
* @param input input resource
* @param inputResource input resource
* @param builderClass class to build resource with
* @return task
* @throws CompileExceptionError
Expand Down Expand Up @@ -1327,7 +1334,7 @@ public void scanJavaClasses() throws IOException, CompileExceptionError {
}

private Future buildRemoteEngine(IProgress monitor, ExecutorService executor) {
Callable<Void> callable = new Callable<>() {
Callable<Void> callable = new Callable<Void>() {
public Void call() throws Exception {
logInfo("Build Remote Engine...");
TimeProfiler.addMark("StartBuildRemoteEngine", "Build Remote Engine");
Expand Down Expand Up @@ -1425,6 +1432,89 @@ private void configurePreBuildProjectOptions() throws IOException, CompileExcept
}
}

private void transpileLua(IProgress monitor) throws CompileExceptionError, IOException {
List<ILuaTranspiler> transpilers = PluginScanner.getOrCreatePlugins("com.defold.extension.pipeline", ILuaTranspiler.class);
if (transpilers != null) {
IProgress transpilerProgress = monitor.subProgress(1);
transpilerProgress.beginTask("Transpiling to Lua", 1);
for (ILuaTranspiler transpiler : transpilers) {
IResource buildFileResource = getResource(transpiler.getBuildFileResourcePath());
if (buildFileResource.exists()) {
String ext = "." + transpiler.getSourceExt();
List<IResource> sources = inputs.stream()
.filter(s -> s.endsWith(ext))
.map(this::getResource)
.collect(Collectors.toList());
if (!sources.isEmpty()) {
boolean useProjectDir = buildFileResource instanceof DefaultResource && sources.stream().allMatch(s -> s instanceof DefaultResource);
File sourceDir;
if (useProjectDir) {
sourceDir = new File(rootDirectory);
} else {
sourceDir = Files.createTempDirectory("tr-" + transpiler.getClass().getSimpleName()).toFile();
buildFileResource.getContent();
File buildFile = new File(sourceDir, buildFileResource.getPath());
Path buildFilePath = buildFile.toPath();
Files.write(buildFilePath, buildFileResource.getContent());
Files.setLastModifiedTime(buildFilePath, FileTime.fromMillis(buildFileResource.getLastModified()));
for (IResource source : sources) {
Path sourcePath = new File(sourceDir, source.getPath()).toPath();
Files.write(sourcePath, source.getContent());
Files.setLastModifiedTime(sourcePath, FileTime.fromMillis(source.getLastModified()));
}
}
File outputDir = new File(rootDirectory, "build/tr/" + transpiler.getClass().getSimpleName());
Files.createDirectories(outputDir.toPath());
try {
List<ILuaTranspiler.Issue> issues = transpiler.transpile(new File(getPluginsDirectory()), sourceDir, outputDir);
List<ILuaTranspiler.Issue> errors = issues.stream().filter(issue -> issue.severity == ILuaTranspiler.Severity.ERROR).collect(Collectors.toList());
if (!errors.isEmpty()) {
MultipleCompileException exception = new MultipleCompileException("Transpilation failed", null);
errors.forEach(issue -> exception.addIssue(issue.severity.ordinal(), getResource(issue.resourcePath), issue.message, issue.lineNumber));
throw exception;
} else {
issues.forEach(issue -> {
Level level;
switch (issue.severity) {
case INFO:
level = Level.INFO;
break;
case WARNING:
level = Level.WARNING;
break;
default:
throw new IllegalStateException();
}
logger.log(level, issue.resourcePath + ":" + issue.lineNumber + ": " + issue.message);
});
}
DefaultFileSystem fs = new DefaultFileSystem();
fs.setRootDirectory(outputDir.toString());
ArrayList<String> results = new ArrayList<>();
fs.walk("", new FileSystemWalker() {
@Override
public void handleFile(String path, Collection<String> results) {
if (path.endsWith(".lua")) {
results.add(path);
}
}
}, results);
inputs.addAll(results);
fileSystem.addMountPoint(new FileSystemMountPoint(fileSystem, fs));
} catch (Exception e) {
throw new CompileExceptionError(buildFileResource, 1, "Transpilation failed", e);
} finally {
if (!useProjectDir) {
FileUtils.deleteDirectory(sourceDir);
}
}
}
}
}
transpilerProgress.done();
}
}

private List<TaskResult> createAndRunTasks(IProgress monitor) throws IOException, CompileExceptionError {
// Do early test if report files are writable before we start building
boolean generateReport = this.hasOption("build-report") || this.hasOption("build-report-html");
Expand Down Expand Up @@ -1580,6 +1670,15 @@ private List<TaskResult> doBuild(IProgress monitor, String... commands) throws T
Future<Void> remoteBuildFuture = null;
// Get or build engine binary
boolean shouldBuildRemoteEngine = ExtenderUtil.hasNativeExtensions(this);
boolean shouldBuildProject = shouldBuildEngine() && BundleHelper.isArchiveIncluded(this);

if (shouldBuildProject) {
// do this before buildRemoteEngine to prevent concurrent modification exception, since
// lua transpilation adds new mounts with compiled Lua that buildRemoteEngine iterates over
// when sending to extender
transpileLua(monitor);
}

if (shouldBuildRemoteEngine) {
remoteBuildFuture = buildRemoteEngine(monitor, executor);
}
Expand All @@ -1593,7 +1692,7 @@ private List<TaskResult> doBuild(IProgress monitor, String... commands) throws T
}
}

if (shouldBuildEngine() && BundleHelper.isArchiveIncluded(this)) {
if (shouldBuildProject) {
result = createAndRunTasks(monitor);
}

Expand Down

0 comments on commit 68d639d

Please sign in to comment.