diff --git a/data/GhidrathonConfig.xml b/data/GhidrathonConfig.xml new file mode 100644 index 0000000..401022b --- /dev/null +++ b/data/GhidrathonConfig.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/data/python/tests/test_cpython.py b/data/python/tests/test_cpython.py new file mode 100644 index 0000000..c4cf4c8 --- /dev/null +++ b/data/python/tests/test_cpython.py @@ -0,0 +1,26 @@ +# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: [package root]/LICENSE.txt +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +"""Unit tests to verify CPython modules + +Note: you must run these tests from the Ghidra script manager or headless mode +""" + +import unittest +import warnings + + +class TestCPython(unittest.TestCase): + def test_numpy(self): + try: + import numpy + + a = numpy.array(["cat", "dog"]) + except ImportError: + warnings.warn("numpy module is not installed - ignoring test") + pass diff --git a/data/python/tests/test_jepbridge.py b/data/python/tests/test_jepbridge.py index 30d7f40..ff31581 100644 --- a/data/python/tests/test_jepbridge.py +++ b/data/python/tests/test_jepbridge.py @@ -21,6 +21,12 @@ def assertIsJavaObject(self, o): if not (o is None or isinstance(o, Object)): raise AssertionError("Object %s is not valid" % str(o)) + def assertIsNotJavaObject(self, o): + from java.lang import Object + + if isinstance(o, Object): + raise AssertionError("Object %s is not valid" % str(o)) + def test_type_instance(self): # see Jep: https://github.com/ninia/jep/blob/15e36a7ba54eb7d8f7ffd85f16675fa4fd54eb1d/src/test/python/test_import.py#L54-L65 from java.lang import Object @@ -46,3 +52,8 @@ def test_ghidra_script_variables(self): def test_ghidra_script_methods(self): self.assertIsInstance(getGhidraVersion(), str) + + def test_java_excluded_packages(self): + import pdb + + self.assertIsNotJavaObject(pdb) diff --git a/src/main/java/ghidrathon/GhidrathonClassEnquirer.java b/src/main/java/ghidrathon/GhidrathonClassEnquirer.java new file mode 100644 index 0000000..09fa0f8 --- /dev/null +++ b/src/main/java/ghidrathon/GhidrathonClassEnquirer.java @@ -0,0 +1,50 @@ +// Copyright (C) 2022 Mandiant, Inc. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: [package root]/LICENSE.txt +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and limitations under the License. + +package ghidrathon; + +import java.util.List; +import java.util.ArrayList; + +import jep.ClassList; +import jep.ClassEnquirer; + +/** + * Implements Jep ClassEnquirer used to handle Java imports from Python - specifically we + * use this class to handle naming conflicts, e.g. pdb + */ +public class GhidrathonClassEnquirer implements ClassEnquirer { + + private final List javaExcludeLibs = new ArrayList(); + private final ClassEnquirer classList = ClassList.getInstance(); + + public void addJavaExcludeLib(String name) { + javaExcludeLibs.add(name); + } + + public void addJavaExcludeLibs(List names) { + javaExcludeLibs.addAll(names); + } + + public boolean isJavaPackage(String name) { + if (javaExcludeLibs.contains(name)) { + return false; + } + + return classList.isJavaPackage(name); + } + + public String[] getClassNames(String name) { + return classList.getClassNames(name); + } + + public String[] getSubPackages(String name) { + return classList.getSubPackages(name); + } + +} diff --git a/src/main/java/ghidrathon/GhidrathonConfig.java b/src/main/java/ghidrathon/GhidrathonConfig.java new file mode 100644 index 0000000..d3ae56a --- /dev/null +++ b/src/main/java/ghidrathon/GhidrathonConfig.java @@ -0,0 +1,85 @@ +// Copyright (C) 2022 Mandiant, Inc. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: [package root]/LICENSE.txt +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and limitations under the License. + +package ghidrathon; + +import java.util.List; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collections; + +/** + * Ghidrathon's configuration class + * + * Stores + * - stdout and stderr + * - Python modules to handle as shared modules - relevant to CPython modules + * - Java package names to exclude from Python imports + * - Python include paths to add to Python interpreter environment + */ +public class GhidrathonConfig { + + private final List javaExcludeLibs = new ArrayList(); + private final List pyIncludePaths = new ArrayList(); + private final List pySharedModules = new ArrayList(); + + private PrintWriter out = null; + private PrintWriter err = null; + + public void addStdOut(PrintWriter out) { + this.out = out; + } + + public void addStdErr(PrintWriter err) { + this.err = err; + } + + public PrintWriter getStdOut() { + return out; + } + + public PrintWriter getStdErr() { + return err; + } + + public void addPythonSharedModule(String name) { + pySharedModules.add(name); + } + + public void addPythonSharedModules(List names) { + pySharedModules.addAll(names); + } + + public Iterable getPythonSharedModules() { + return Collections.unmodifiableList(pySharedModules); + } + + public void addJavaExcludeLib(String name) { + javaExcludeLibs.add(name); + } + + public void addJavaExcludeLibs(List names) { + javaExcludeLibs.addAll(names); + } + + public Iterable getJavaExcludeLibs() { + return Collections.unmodifiableList(javaExcludeLibs); + } + + public void addPythonIncludePath(String path) { + pyIncludePaths.add(path); + } + + public void addPythonIncludePaths(List paths) { + pyIncludePaths.addAll(paths); + } + + public Iterable getPythonIncludePaths() { + return Collections.unmodifiableList(pyIncludePaths); + } +} diff --git a/src/main/java/ghidrathon/GhidrathonConsoleInputThread.java b/src/main/java/ghidrathon/GhidrathonConsoleInputThread.java index ddc30fb..7217852 100644 --- a/src/main/java/ghidrathon/GhidrathonConsoleInputThread.java +++ b/src/main/java/ghidrathon/GhidrathonConsoleInputThread.java @@ -8,28 +8,33 @@ package ghidrathon; -import java.io.BufferedReader; import java.io.File; +import java.io.PrintWriter; import java.io.IOException; +import java.io.BufferedReader; import java.io.InputStreamReader; -import java.io.PrintWriter; import java.util.concurrent.atomic.AtomicBoolean; import generic.jar.ResourceFile; -import ghidra.app.plugin.core.interpreter.InterpreterConsole; -import ghidra.app.script.GhidraState; + import ghidra.util.Msg; +import ghidra.app.script.GhidraState; +import ghidra.app.plugin.core.interpreter.InterpreterConsole; + +import ghidrathon.GhidrathonUtils; +import ghidrathon.GhidrathonConfig; import ghidrathon.interpreter.GhidrathonInterpreter; public class GhidrathonConsoleInputThread extends Thread { private static int generationCount = 0; + private GhidrathonPlugin plugin = null; private InterpreterConsole console = null; - private AtomicBoolean shouldContinue = new AtomicBoolean(true); private GhidrathonInterpreter python = null; - private PrintWriter err = null; - private PrintWriter out = null; + + private AtomicBoolean shouldContinue = new AtomicBoolean(true); + private GhidrathonConfig config = GhidrathonUtils.getDefaultGhidrathonConfig(); GhidrathonConsoleInputThread(GhidrathonPlugin plugin) { @@ -37,8 +42,10 @@ public class GhidrathonConsoleInputThread extends Thread { this.plugin = plugin; this.console = plugin.getConsole(); - this.err = console.getErrWriter(); - this.out = console.getOutWriter(); + + // init Ghidrathon configuration + config.addStdErr(console.getErrWriter()); + config.addStdOut(console.getOutWriter()); } @@ -56,7 +63,7 @@ public void run() { try { - python = GhidrathonInterpreter.get(out, err); + python = GhidrathonInterpreter.get(config); python.printWelcome(); @@ -66,7 +73,7 @@ public void run() { python.close(); } - e.printStackTrace(err); + e.printStackTrace(config.getStdErr()); return; } @@ -95,7 +102,6 @@ public void run() { continue; } - boolean moreInputWanted = evalPython(line); this.plugin.flushConsole(); diff --git a/src/main/java/ghidrathon/GhidrathonScript.java b/src/main/java/ghidrathon/GhidrathonScript.java index 0fdf576..b64a02c 100644 --- a/src/main/java/ghidrathon/GhidrathonScript.java +++ b/src/main/java/ghidrathon/GhidrathonScript.java @@ -20,6 +20,8 @@ import ghidra.framework.plugintool.PluginTool; import ghidra.app.script.GhidraScriptProvider; +import ghidrathon.GhidrathonUtils; +import ghidrathon.GhidrathonConfig; import ghidrathon.interpreter.GhidrathonInterpreter; public class GhidrathonScript extends GhidraScript { @@ -28,24 +30,26 @@ public class GhidrathonScript extends GhidraScript { protected void run() { GhidrathonInterpreter python = null; - - final PrintWriter out = getStdOut(); - final PrintWriter err = getStdErr(); + GhidrathonConfig config = GhidrathonUtils.getDefaultGhidrathonConfig(); + + // init Ghidrathon configuration + config.addStdOut(getStdOut()); + config.addStdErr(getStdErr()); try { - python = GhidrathonInterpreter.get(out, err); + python = GhidrathonInterpreter.get(config); // run Python script from Python interpreter python.runScript(getSourceFile(), this); // flush stdout and stderr to ensure all is printed to console window - err.flush(); - out.flush(); + config.getStdErr().flush(); + config.getStdOut().flush(); } catch (RuntimeException e) { - e.printStackTrace(err); + e.printStackTrace(config.getStdErr()); } finally { @@ -68,13 +72,14 @@ protected void run() { public void runScript(String name, GhidraState scriptState) { GhidrathonInterpreter python = null; - - final PrintWriter out = getStdOut(); - final PrintWriter err = getStdErr(); + GhidrathonConfig config = GhidrathonUtils.getDefaultGhidrathonConfig(); + + config.addStdOut(getStdOut()); + config.addStdErr(getStdErr()); try { - python = GhidrathonInterpreter.get(out, err); + python = GhidrathonInterpreter.get(config); ResourceFile source = GhidraScriptUtil.findScriptByName(name); if (source == null) { @@ -109,7 +114,7 @@ public void runScript(String name, GhidraState scriptState) { } catch (Exception e) { - e.printStackTrace(err); + e.printStackTrace(config.getStdErr()); } finally { diff --git a/src/main/java/ghidrathon/GhidrathonUtils.java b/src/main/java/ghidrathon/GhidrathonUtils.java new file mode 100644 index 0000000..f68fcad --- /dev/null +++ b/src/main/java/ghidrathon/GhidrathonUtils.java @@ -0,0 +1,102 @@ +// Copyright (C) 2022 Mandiant, Inc. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: [package root]/LICENSE.txt +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and limitations under the License. + +package ghidrathon; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +import ghidra.util.Msg; +import ghidra.framework.Application; +import ghidra.framework.options.SaveState; + +import ghidrathon.GhidrathonConfig; + +/** + * Utility functions + */ +public class GhidrathonUtils { + + // name of this extension e.g. "Ghidrathon" + public static final String THIS_EXTENSION_NAME = Application.getMyModuleRootDirectory().getName(); + + private static final String DEFAULT_CONFIG_FILENAME = "GhidrathonConfig.xml"; + private static final String JAVA_EXCLUDE_LIBS_KEY = "JAVA_EXCLUDE_LIBS"; + private static final String PY_SHARED_MODULES_KEY = "PYTHON_SHARED_MODULES"; + private static final String PY_INCLUDE_PATHS_KEY = "PYTHON_INCLUDE_PATHS"; + + /** + * Get Ghidrathon's default configuration - default configuration is stored in data/ and copied to Ghidra user + * settings directory when first accessed + */ + public static GhidrathonConfig getDefaultGhidrathonConfig() { + + GhidrathonConfig config = new GhidrathonConfig(); + File userSettingsPath = new File(Application.getUserSettingsDirectory(), DEFAULT_CONFIG_FILENAME); + + // copy configuration from /data to Ghidra user settings if file does not already exist + if (!userSettingsPath.isFile()) { + + Msg.info(GhidrathonUtils.class, "Addings configuration to user settings at " + userSettingsPath); + + try { + + File dataPath = Application.getModuleDataFile(THIS_EXTENSION_NAME, DEFAULT_CONFIG_FILENAME).getFile(false); + Files.copy(dataPath.toPath(), userSettingsPath.toPath(), StandardCopyOption.REPLACE_EXISTING); + + } catch (IOException e) { + + Msg.error(GhidrathonUtils.class, "Failed to write user configuration [" + e + "]"); + return config; + + } + } + + SaveState state = null; + + // attempt to read configuration from Ghidra user settings + try { + + state = new SaveState(userSettingsPath); + + } catch (IOException e) { + + Msg.error(GhidrathonUtils.class, "Failed to read configuration state [" + e + "]"); + return config; + + } + + // add Java exclude libs that will be ignored when importing from Python - this is used to avoid + // naming conflicts, e.g. "pdb" + for (String name: state.getStrings(JAVA_EXCLUDE_LIBS_KEY, new String[0])) { + + config.addJavaExcludeLib(name); + + } + + // add Python include paths + for (String name: state.getStrings(PY_INCLUDE_PATHS_KEY, new String[0])) { + + config.addPythonIncludePath(name); + + } + + // add Python shared modules - these modules are handled specially by Jep to avoid crashes caused + // by CPython extensions, e.g. numpy + for (String name: state.getStrings(PY_SHARED_MODULES_KEY, new String[0])) { + + config.addPythonSharedModule(name); + + } + + return config; + } + +} diff --git a/src/main/java/ghidrathon/interpreter/GhidrathonInterpreter.java b/src/main/java/ghidrathon/interpreter/GhidrathonInterpreter.java index c7321de..eed15ab 100644 --- a/src/main/java/ghidrathon/interpreter/GhidrathonInterpreter.java +++ b/src/main/java/ghidrathon/interpreter/GhidrathonInterpreter.java @@ -27,22 +27,23 @@ import jep.JepException; import jep.MainInterpreter; +import ghidrathon.GhidrathonUtils; import ghidrathon.GhidrathonScript; +import ghidrathon.GhidrathonConfig; +import ghidrathon.GhidrathonClassEnquirer; /** * Utility class used to configure a Jep instance to access Ghidra */ public class GhidrathonInterpreter { - private Jep jep; - private JepConfig config; + private Jep jep = null; + private GhidrathonConfig ghidrathonConfig = null; - private boolean scriptMethodsInjected = false; + private final JepConfig jepConfig = new JepConfig(); + private final GhidrathonClassEnquirer ghidrathonClassEnquirer = new GhidrathonClassEnquirer(); - private PrintWriter err = null; - private PrintWriter out = null; - - private static String extname = Application.getMyModuleRootDirectory().getName(); + private boolean scriptMethodsInjected = false; /** * Create and configure a new GhidrathonInterpreter instance. @@ -50,46 +51,86 @@ public class GhidrathonInterpreter { * @throws JepException * @throws IOException */ - private GhidrathonInterpreter(PrintWriter out, PrintWriter err) throws JepException, IOException{ + private GhidrathonInterpreter(GhidrathonConfig config) throws JepException, IOException{ + ghidrathonConfig = config; + // configure the Python includes path with the user's Ghdira script directory String paths = ""; for (ResourceFile resourceFile : GhidraScriptUtil.getScriptSourceDirectories()) { + paths += resourceFile.getFile(false).getAbsolutePath() + File.pathSeparator; + } // add data/python/ to Python includes directory - paths += Application.getModuleDataSubDirectory(extname, "python") + File.pathSeparator; + paths += Application.getModuleDataSubDirectory(GhidrathonUtils.THIS_EXTENSION_NAME, "python") + File.pathSeparator; - config = new JepConfig(); + // add paths specified in Ghidrathon config + for (String path: ghidrathonConfig.getPythonIncludePaths()) { + + paths += path + File.pathSeparator; + + } + + // configure Java names that will be ignored when importing from Python + for (String name: ghidrathonConfig.getJavaExcludeLibs()) { + + ghidrathonClassEnquirer.addJavaExcludeLib(name); + + } // set the class loader with access to Ghidra scripting API - config.setClassLoader(ClassLoader.getSystemClassLoader()); + jepConfig.setClassLoader(ClassLoader.getSystemClassLoader()); + + // set class enquirer used to handle Java imports from Python + jepConfig.setClassEnquirer(ghidrathonClassEnquirer); // configure Python includes Path - config.addIncludePaths(paths); - - // configure Jep stdout and stderr - config.redirectStdout(new WriterOutputStream(out, System.getProperty("file.encoding")) { - @Override - public void write(byte[] b, int off, int len) throws IOException { - super.write(b, off, len); - flush(); // flush the output to ensure it is displayed in real-time - } - }); - config.redirectStdErr(new WriterOutputStream(err, System.getProperty("file.encoding")) { - @Override - public void write(byte[] b, int off, int len) throws IOException { - super.write(b, off, len); - flush(); // flush the error to ensure it is displayed in real-time - } - }); + jepConfig.addIncludePaths(paths); + + // add Python shared modules - these should be CPython modules for Jep to handle specially + for (String name: ghidrathonConfig.getPythonSharedModules()) { + + jepConfig.addSharedModules(name); + + } + + // configure Jep stdout + if (ghidrathonConfig.getStdOut() != null) { + + jepConfig.redirectStdout(new WriterOutputStream(ghidrathonConfig.getStdOut(), System.getProperty("file.encoding")) { + + @Override + public void write(byte[] b, int off, int len) throws IOException { + super.write(b, off, len); + flush(); // flush the output to ensure it is displayed in real-time + } + + }); + } + + // configure Jep stderr + if (ghidrathonConfig.getStdErr() != null ) { + jepConfig.redirectStdErr(new WriterOutputStream(ghidrathonConfig.getStdErr(), System.getProperty("file.encoding")) { + + @Override + public void write(byte[] b, int off, int len) throws IOException { + super.write(b, off, len); + flush(); // flush the error to ensure it is displayed in real-time + } + + }); + + } + + // we must set the native Jep library before creating a Jep instance setJepNativeBinaryPath(); // create a new Jep interpreter instance - jep = new jep.SubInterpreter(config); + jep = new jep.SubInterpreter(jepConfig); // now that everything is configured, we should be able to run some utility scripts // to help us further configure the Python environment @@ -115,12 +156,12 @@ private void setJepNativeBinaryPath() throws JepException, FileNotFoundException try { - nativeJep = Application.getOSFile(extname, "libjep.so"); + nativeJep = Application.getOSFile(GhidrathonUtils.THIS_EXTENSION_NAME, "libjep.so"); } catch (FileNotFoundException e) { // whoops try Windows - nativeJep = Application.getOSFile(extname, "jep.dll"); + nativeJep = Application.getOSFile(GhidrathonUtils.THIS_EXTENSION_NAME, "jep.dll"); } @@ -148,7 +189,7 @@ private void setJepNativeBinaryPath() throws JepException, FileNotFoundException */ private void setJepEval() throws JepException, FileNotFoundException { - ResourceFile file = Application.getModuleDataFile(extname, "python/jepeval.py"); + ResourceFile file = Application.getModuleDataFile(GhidrathonUtils.THIS_EXTENSION_NAME, "python/jepeval.py"); jep.runScript(file.getAbsolutePath()); @@ -165,7 +206,7 @@ private void setJepEval() throws JepException, FileNotFoundException { */ private void setJepRunScript() throws JepException, FileNotFoundException { - ResourceFile file = Application.getModuleDataFile(extname, "python/jeprunscript.py"); + ResourceFile file = Application.getModuleDataFile(GhidrathonUtils.THIS_EXTENSION_NAME, "python/jeprunscript.py"); jep.runScript(file.getAbsolutePath()); @@ -187,7 +228,7 @@ private void injectScriptHierarchy(GhidraScript script) throws JepException, Fil return; } - ResourceFile file = Application.getModuleDataFile(extname, "python/jepbuiltins.py"); + ResourceFile file = Application.getModuleDataFile(GhidrathonUtils.THIS_EXTENSION_NAME, "python/jepbuiltins.py"); jep.runScript(file.getAbsolutePath()); // inject GhidraScript public/private fields e.g. currentAddress into Python @@ -210,7 +251,7 @@ private void injectScriptHierarchy(GhidraScript script) throws JepException, Fil if (!scriptMethodsInjected) { // inject GhidraScript methods into Python - file = Application.getModuleDataFile(extname, "python/jepinject.py"); + file = Application.getModuleDataFile(GhidrathonUtils.THIS_EXTENSION_NAME, "python/jepinject.py"); jep.set("__ghidra_script__", script); jep.runScript(file.getAbsolutePath()); } @@ -224,11 +265,11 @@ private void injectScriptHierarchy(GhidraScript script) throws JepException, Fil * @return GhidrathonInterpreter * @throws RuntimeException */ - public static GhidrathonInterpreter get(PrintWriter out, PrintWriter err) throws RuntimeException { + public static GhidrathonInterpreter get(GhidrathonConfig ghidrathonConfig) throws RuntimeException { try { - return new GhidrathonInterpreter(out, err); + return new GhidrathonInterpreter(ghidrathonConfig); } catch (Exception e) { @@ -402,7 +443,7 @@ public void printWelcome() { try { - ResourceFile file = Application.getModuleDataFile(extname, "python/jepwelcome.py"); + ResourceFile file = Application.getModuleDataFile(GhidrathonUtils.THIS_EXTENSION_NAME, "python/jepwelcome.py"); jep.set("GhidraVersion", Application.getApplicationVersion());