diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..945c9cb --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,13 @@ +{ + "name": "Java 17 Dev Container", + "image": "mcr.microsoft.com/devcontainers/java:17", + "features": { + "ghcr.io/devcontainers/features/java:1": { + "version": "none", + "installGradle": "false", + "installMaven": "true" + } + }, + "settings": {}, + "extensions": [] +} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e935a91..ab20f69 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,6 +37,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 5de62b4..37e2eb5 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -14,10 +14,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Set up JDK 17 + uses: actions/setup-java@v3 with: - java-version: 1.8 + distribution: 'temurin' + java-version: '17' server-id: github # Value of the distributionManagement/repository/id field of the pom.xml settings-path: ${{ github.workspace }} # location for the settings.xml file diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c8aeb3a..ae9ed8c 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -16,9 +16,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Set up JDK 17 + uses: actions/setup-java@v3 with: - java-version: 1.8 + distribution: 'temurin' + java-version: '17' - name: Build with Maven run: mvn -B package --file pom.xml diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml index 12fc0d0..bc81d14 100644 --- a/dependency-reduced-pom.xml +++ b/dependency-reduced-pom.xml @@ -6,30 +6,9 @@ 4.0.0 - - maven-install-plugin - - - hack-binary - clean - - install-file - - - ${JAVA_HOME}/lib/tools.jar - default - com.sun - tools - jdk1.8.0 - jar - true - - - - - - maven-shade-plugin - 3.0.0 + + maven-shade-plugin + 3.0.0 package @@ -44,20 +23,7 @@ - - - windows-profile - - ${JAVA_HOME}/lib/tools.jar - - - - mac-profile - - ${java.home}/../lib/tools.jar - - - + junit @@ -73,8 +39,8 @@ - 1.8 - 1.8 - - + 17 + 17 + + diff --git a/nb-configuration.xml b/nb-configuration.xml index 15dea5e..469d086 100644 --- a/nb-configuration.xml +++ b/nb-configuration.xml @@ -13,6 +13,6 @@ You can copy and paste the single properties, into the pom.xml file and the IDE That way multiple projects can share the same settings (useful for formatting rules for example). Any value defined here will override the pom.xml file value but is only applicable to the current project. --> - JDK_1.8 - - + JDK_17 + + diff --git a/pom.xml b/pom.xml index aef3cf6..448acde 100644 --- a/pom.xml +++ b/pom.xml @@ -4,54 +4,30 @@ hellofunction 6.0.1 - 1.8 - 1.8 + 17 + 17 + 17 UTF-8 + - - - windows-profile - - true - - ${JAVA_HOME}/lib/tools.jar - - - - ${JAVA_HOME}/lib/tools.jar - - - - mac-profile - - false - - ${java.home}/../lib/tools.jar - - - - ${java.home}/../lib/tools.jar - - - com.amazonaws aws-java-sdk-bom - 1.11.911 + 1.12.789 pom import @@ -62,7 +38,7 @@ com.amazonaws aws-java-sdk-lambda - 1.11.893 + 1.12.789 @@ -75,13 +51,13 @@ com.fasterxml.jackson.core jackson-core - 2.12.0 + 2.15.4 com.fasterxml.jackson.core jackson-databind - 2.12.0 + 2.15.4 @@ -93,13 +69,14 @@ --> + - + 3.42.0 - + --> + junit junit @@ -140,7 +117,18 @@ - + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + true + + + + org.apache.maven.plugins maven-shade-plugin diff --git a/src/main/java/run/myCode/FileManager.java b/src/main/java/run/myCode/FileManager.java index 1376a97..aa29611 100644 --- a/src/main/java/run/myCode/FileManager.java +++ b/src/main/java/run/myCode/FileManager.java @@ -5,9 +5,11 @@ import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import run.myCode.compiler.InMemoryJavaFileObject; +import java.util.Base64; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import run.myCode.compiler.InMemoryJavaFileObject; /** * @@ -24,19 +26,31 @@ public static List createSourceFileObjects(CompileReques List objects = new ArrayList<>(req.getSourceFiles().size()); - // Pull all of the source code files in the request - req.getSourceFiles().forEach((f) -> { - String contents = ""; - for (String s : f.getContents()) { - // Save the file's contents - contents += s + '\n'; - } - - // Add them to a list of files - objects.add(new InMemoryJavaFileObject(f.getName(), contents)); - }); - return objects; - } + // Pull all of the source code files in the request + req.getSourceFiles().forEach((f) -> { + String contents = ""; + for (String s : f.getContents()) { + // Save the file's contents + contents += s + '\n'; + } + + String fileName = f.getName(); + // If the source declares a package, prefix the file name with its + // package path so the compiler's file manager can resolve the + // class correctly. ECJ expects the source path to mirror the + // package declaration, otherwise it will try to locate the file on + // disk and fail with "File ... is missing". + Pattern pkgPattern = Pattern.compile("(?m)^\\s*package\\s+([\\w\\.]+)\\s*;"); + Matcher m = pkgPattern.matcher(contents); + if (m.find()) { + fileName = m.group(1).replace('.', '/') + "/" + fileName; + } + + // Add them to a list of files + objects.add(new InMemoryJavaFileObject(fileName, contents)); + }); + return objects; + } public static void saveData(DataRequest data) { if (data == null) { diff --git a/src/main/java/run/myCode/compiler/FromMemoryClassLoader.java b/src/main/java/run/myCode/compiler/FromMemoryClassLoader.java index a8a8702..96ff919 100644 --- a/src/main/java/run/myCode/compiler/FromMemoryClassLoader.java +++ b/src/main/java/run/myCode/compiler/FromMemoryClassLoader.java @@ -5,10 +5,12 @@ import java.util.List; import java.util.Map; -public class FromMemoryClassLoader extends ClassLoader { - - private final Map m = new HashMap<>(); - private final ClassLoader parent; +public class FromMemoryClassLoader extends ClassLoader { + + private static final boolean DEBUG = Boolean.getBoolean("run.mycode.debug"); + + private final Map m = new HashMap<>(); + private final ClassLoader parent; public FromMemoryClassLoader(ClassLoader parent) { this.parent = parent; @@ -20,13 +22,16 @@ public Class findClass(String name) throws ClassNotFoundException { try { // System.out.println("Class search:" + name); MemoryByteCode mbc = m.get(name); - if (mbc == null) { - mbc = m.get(name.replace(".", "/")); - if (mbc == null) { - return super.findClass(name); - } - } - return defineClass(name, mbc.getBytes(), 0, mbc.getBytes().length); + if (mbc == null) { + mbc = m.get(name.replace(".", "/")); + if (mbc == null) { + return super.findClass(name); + } + } + if (DEBUG) { + System.out.println("Loading in-memory class " + name); + } + return defineClass(name, mbc.getBytes(), 0, mbc.getBytes().length); } catch (ClassNotFoundException e) { // System.err.println("Could not find: " + name); if (this.parent != null) { @@ -46,8 +51,11 @@ public Class findClass(String name) throws ClassNotFoundException { public void addClass(String name, MemoryByteCode mbc) { // System.err.println("Added class: " + name); // System.out.println("Added class:" + name); - m.put(name, mbc); - } + m.put(name, mbc); + if (DEBUG) { + System.out.println("Added in-memory class " + name); + } + } public List getClassNames() { return new ArrayList(m.keySet()); diff --git a/src/main/java/run/myCode/compiler/InMemoryJavaFileManager.java b/src/main/java/run/myCode/compiler/InMemoryJavaFileManager.java index fc32b10..2527b4e 100644 --- a/src/main/java/run/myCode/compiler/InMemoryJavaFileManager.java +++ b/src/main/java/run/myCode/compiler/InMemoryJavaFileManager.java @@ -1,17 +1,23 @@ package run.myCode.compiler; -import java.io.IOException; - -import javax.tools.FileObject; -import javax.tools.ForwardingJavaFileManager; -import javax.tools.JavaFileObject; -import javax.tools.StandardJavaFileManager; -import javax.tools.JavaFileManager.Location; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import javax.tools.FileObject; +import javax.tools.ForwardingJavaFileManager; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.JavaFileManager.Location; @SuppressWarnings({"unused", "rawtypes"}) -public class InMemoryJavaFileManager extends ForwardingJavaFileManager { - - private final FromMemoryClassLoader xcl; +public class InMemoryJavaFileManager extends ForwardingJavaFileManager { + + private static final boolean DEBUG = Boolean.getBoolean("run.mycode.debug"); + + private final FromMemoryClassLoader xcl; + private final Map sources = new HashMap<>(); + private final Map sourcePaths = new HashMap<>(); @SuppressWarnings("unchecked") public InMemoryJavaFileManager(StandardJavaFileManager sjfm, FromMemoryClassLoader xcl) { @@ -19,12 +25,98 @@ public InMemoryJavaFileManager(StandardJavaFileManager sjfm, FromMemoryClassLoad this.xcl = xcl; } - @Override - public JavaFileObject getJavaFileForOutput(Location location, String name, JavaFileObject.Kind kind, FileObject sibling) throws IOException { - MemoryByteCode mbc = new MemoryByteCode(name); - xcl.addClass(name, mbc); - return mbc; - } + public void addSource(JavaFileObject file) { + // The JavaFileObject#getName returns the URI path. Normalise it so it + // can be looked up with or without a leading slash and use the path to + // derive the binary name for the class. + String path = file.getName(); + if (path.startsWith("/")) { + path = path.substring(1); + } + // Track both the plain path and the full URI so lookups using either + // form can be resolved (e.g. "Calculator.java" vs "mem:///Calculator.java"). + sourcePaths.put(path, file); + sourcePaths.put(file.toUri().toString(), file); + + if (path.endsWith(".java")) { + String className = path.substring(0, path.length() - 5) + .replace('/', '.'); + sources.put(className, file); + + // Also record the expected package path (e.g. foo/Bar.java) so + // lookups using package-qualified names can be resolved. + String pkgPath = className.replace('.', '/') + ".java"; + sourcePaths.put(pkgPath, file); + + if (DEBUG) { + System.out.println("Registered source " + file.getName()); + } + } + } + + @Override + public String inferBinaryName(Location location, JavaFileObject file) { + String name = file.getName(); + if (name.startsWith("/")) { + name = name.substring(1); + } + if (name.endsWith(".java")) { + name = name.substring(0, name.length() - 5); + } + return name.replace('/', '.'); + } + + @Override + public JavaFileObject getJavaFileForInput(Location location, String className, JavaFileObject.Kind kind) throws IOException { + String name = className; + if (name.startsWith("/")) { + name = name.substring(1); + } + if (name.startsWith("mem:///")) { + name = name.substring("mem:///".length()); + } + if (name.endsWith(".java")) { + name = name.substring(0, name.length() - 5); + } + name = name.replace('/', '.'); + JavaFileObject file = sources.get(name); + if (DEBUG) { + System.out.println("getJavaFileForInput(" + className + ") -> " + (file != null ? "memory" : "disk")); + } + if (file != null) { + return file; + } + return super.getJavaFileForInput(location, className, kind); + } + + @Override + public FileObject getFileForInput(Location location, String packageName, String relativeName) throws IOException { + String path = packageName == null || packageName.isEmpty() ? relativeName : packageName + "/" + relativeName; + if (path.startsWith("/")) { + path = path.substring(1); + } + if (path.startsWith("mem:///")) { + path = path.substring("mem:///".length()); + } + JavaFileObject file = sourcePaths.get(path); + if (DEBUG) { + System.out.println("getFileForInput(" + path + ") -> " + (file != null ? "memory" : "disk")); + } + if (file != null) { + return file; + } + return super.getFileForInput(location, packageName, relativeName); + } + + @Override + public JavaFileObject getJavaFileForOutput(Location location, String name, JavaFileObject.Kind kind, FileObject sibling) throws IOException { + MemoryByteCode mbc = new MemoryByteCode(name); + xcl.addClass(name, mbc); + if (DEBUG) { + System.out.println("Storing compiled class " + name); + } + return mbc; + } @Override public ClassLoader getClassLoader(Location location) { diff --git a/src/main/java/run/myCode/compiler/InMemoryJavaFileObject.java b/src/main/java/run/myCode/compiler/InMemoryJavaFileObject.java index 674fc50..08a4af5 100644 --- a/src/main/java/run/myCode/compiler/InMemoryJavaFileObject.java +++ b/src/main/java/run/myCode/compiler/InMemoryJavaFileObject.java @@ -1,19 +1,24 @@ package run.myCode.compiler; -import java.io.IOException; -import java.net.URI; - -import javax.tools.SimpleJavaFileObject; - -import javax.tools.JavaFileObject.Kind; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; + +import javax.tools.SimpleJavaFileObject; + +import javax.tools.JavaFileObject.Kind; /** * java File Object represents an in-memory java source file so there is no need * to put the source file on hard disk */ -public class InMemoryJavaFileObject extends SimpleJavaFileObject { - - private String contents = null; +public class InMemoryJavaFileObject extends SimpleJavaFileObject { + + private static final boolean DEBUG = Boolean.getBoolean("run.mycode.debug"); + + private String contents = null; /** * Create a Java file object in memory with a name and text contents. @@ -21,20 +26,41 @@ public class InMemoryJavaFileObject extends SimpleJavaFileObject { * @param fileName the name of the file, with extension * @param contents the contents of the file as a single string object */ - public InMemoryJavaFileObject(String fileName, String contents) { - // Create a file object with a classname instead of a filename by - // removing the file's extension and convert the . separators into slashes - super(URI.create("file:///" + fileName), Kind.SOURCE); - - // Save the file's contents - this.contents = contents; - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { - return contents; - } - + public InMemoryJavaFileObject(String fileName, String contents) { + // Use a custom URI scheme so the Eclipse compiler does not resolve the + // file on disk. The URI's path is the provided filename which keeps + // the compiler from looking for a physical file like "/MyClass.java". + // + // Using a non-file scheme is important because ECJ will otherwise try + // to open the path returned by getName() from the filesystem which + // causes "File ... is missing" errors when compiling in memory. + // Use the "string" URI scheme which the Eclipse compiler treats as an + // in-memory source and therefore does not attempt to resolve on the + // filesystem. Any other scheme will cause ECJ to verify the file on + // disk and emit a "File ... is missing" error. + super(URI.create("string:///" + fileName), Kind.SOURCE); + + // Save the file's contents + this.contents = contents; + + if (DEBUG) { + System.out.println("Created in-memory source " + toUri()); + } + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { + return contents; + } + + @Override + public InputStream openInputStream() throws IOException { + if (DEBUG) { + System.out.println("Opening in-memory source " + getName()); + } + return new ByteArrayInputStream(contents.getBytes(StandardCharsets.UTF_8)); + } + @Override public String toString() { String s = this.getName() + ":\n"; diff --git a/src/main/java/run/myCode/compiler/JavaCodeCompiler.java b/src/main/java/run/myCode/compiler/JavaCodeCompiler.java index 861d0d1..09cdbb1 100644 --- a/src/main/java/run/myCode/compiler/JavaCodeCompiler.java +++ b/src/main/java/run/myCode/compiler/JavaCodeCompiler.java @@ -17,10 +17,12 @@ import javax.tools.JavaFileObject; import javax.tools.StandardJavaFileManager; -import org.eclipse.jdt.internal.compiler.tool.EclipseCompiler; +import javax.tools.ToolProvider; public class JavaCodeCompiler { + private static final boolean DEBUG = Boolean.getBoolean("run.mycode.debug"); + /** * Compile Java source files into memory * @@ -54,7 +56,10 @@ public static FromMemoryClassLoader compile(Iterable f final FromMemoryClassLoader classLoader = new FromMemoryClassLoader(urlcl); // get system compiler: - final JavaCompiler compiler = new EclipseCompiler(); + // Use the standard Java compiler provided by the JDK. This avoids ECJ + // attempting to resolve in-memory sources on disk which resulted in + // "File ... is missing" errors during tests. + final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); // create a diagnostic listener for compilation diagnostic message processing on // compilation WARNING/ERROR @@ -62,6 +67,12 @@ public static FromMemoryClassLoader compile(Iterable f final StandardJavaFileManager stdfileManager = compiler.getStandardFileManager(diag, Locale.ENGLISH, null); InMemoryJavaFileManager fileManager = new InMemoryJavaFileManager(stdfileManager, classLoader); + for (JavaFileObject file : files) { + fileManager.addSource(file); + if (DEBUG) { + System.out.println("Compiling source " + file.getName()); + } + } // specify options for compiler if (options == null) { @@ -69,10 +80,10 @@ public static FromMemoryClassLoader compile(Iterable f } // Build the classpath from the current folder and the system classpath - StringBuilder classpathBuilder = - new StringBuilder("." - + System.getProperty("path.separator") - + System.getProperty("java.class.path")); + StringBuilder classpathBuilder = + new StringBuilder("." + + System.getProperty("path.separator") + + System.getProperty("java.class.path")); // Add any included jar files from the lib folder to the classpath (wildcard isn't working) try { @@ -97,12 +108,15 @@ public static FromMemoryClassLoader compile(Iterable f // Set the classpath and java version for the compiler options.addAll(Arrays.asList("-classpath", classpath)); - options.addAll(Arrays.asList("-1.8")); + options.addAll(Arrays.asList("--release", "17")); Writer out = new PrintWriter(System.out); // Compile the code JavaCompiler.CompilationTask task = compiler.getTask(out, fileManager, diag, options, null, files); + if (DEBUG) { + System.out.println("Starting compilation with mem URIs"); + } boolean result = task.call(); // Return the classloader containing the compiled classes diff --git a/src/main/java/zss/compiler/InMemoryJavaFileObject.java b/src/main/java/zss/compiler/InMemoryJavaFileObject.java index 4e21c15..838f98f 100644 --- a/src/main/java/zss/compiler/InMemoryJavaFileObject.java +++ b/src/main/java/zss/compiler/InMemoryJavaFileObject.java @@ -1,46 +1,56 @@ -package zss.compiler; - -import java.io.IOException; -import java.net.URI; - -import javax.tools.SimpleJavaFileObject; - -import javax.tools.JavaFileObject.Kind; - -/** - * java File Object represents an in-memory java source file so there is no need - * to put the source file on hard disk - */ -@SuppressWarnings("unused") -public class InMemoryJavaFileObject extends SimpleJavaFileObject { - - private String contents = null; - - /** - * Create a Java file object in memory with a name and text contents. - * - * @param fileName the name of the file, with extension - * @param contents the contents of the file as a single string object - * @throws Exception - */ - public InMemoryJavaFileObject(String fileName, String contents) { - // Create a file object with a classname instead of a filename by - // removing the file's extension and convert the . separators into slashes - super(URI.create("file:///" + fileName), Kind.SOURCE); - - // Save the file's contents - this.contents = contents; - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { - return contents; - } - - public String toString() { - String s = this.getName() + ":\n"; - s += this.contents; - - return s; - } -} +package zss.compiler; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; + +import javax.tools.SimpleJavaFileObject; + +import javax.tools.JavaFileObject.Kind; + +/** + * java File Object represents an in-memory java source file so there is no need + * to put the source file on hard disk + */ +@SuppressWarnings("unused") +public class InMemoryJavaFileObject extends SimpleJavaFileObject { + + private String contents = null; + + /** + * Create a Java file object in memory with a name and text contents. + * + * @param fileName the name of the file, with extension + * @param contents the contents of the file as a single string object + */ + public InMemoryJavaFileObject(String fileName, String contents) { + // Use the "string" URI scheme so the compiler treats this source as an + // in-memory file and does not attempt to resolve it on disk. Using the + // "file" scheme caused ECJ to look for a physical file and fail with + // "File ... is missing" errors. + super(URI.create("string:///" + fileName), Kind.SOURCE); + + // Save the file's contents + this.contents = contents; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { + return contents; + } + + @Override + public InputStream openInputStream() throws IOException { + return new ByteArrayInputStream(contents.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String toString() { + String s = this.getName() + ":\n"; + s += this.contents; + + return s; + } +} diff --git a/src/main/java/zss/compiler/MemoryCompiler.java b/src/main/java/zss/compiler/MemoryCompiler.java index 45ab749..6787a0f 100644 --- a/src/main/java/zss/compiler/MemoryCompiler.java +++ b/src/main/java/zss/compiler/MemoryCompiler.java @@ -12,11 +12,12 @@ import javax.tools.JavaFileObject; import javax.tools.StandardJavaFileManager; -import org.eclipse.jdt.internal.compiler.tool.EclipseCompiler; +import javax.tools.ToolProvider; /** - * An instance of the Eclipse Compiler for Java that compiles memory files into - * memory byte code + * Compiles in-memory source files into byte code using the standard JDK + * compiler. This avoids relying on the Eclipse compiler which attempted to + * resolve sources on disk. */ public class MemoryCompiler { /** @@ -36,8 +37,10 @@ public static FromMemoryClassLoader compile(Iterable f // classloader final FromMemoryClassLoader classLoader = new FromMemoryClassLoader(urlcl); - // get system compiler: - final JavaCompiler compiler = new EclipseCompiler(); + // Use the standard JDK compiler to avoid ECJ attempting to read + // temporary files from disk which caused "File ... is missing" + // errors when compiling in memory. + final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); // create a diagnostic listener for compilation diagnostic message processing on // compilation WARNING/ERROR @@ -51,7 +54,8 @@ public static FromMemoryClassLoader compile(Iterable f options.addAll( Arrays.asList("-classpath", MemoryCompiler.class.getProtectionDomain().getCodeSource().getLocation() + ":" + System.getProperty("java.class.path"))); - options.addAll(Arrays.asList("-1.8", "-nowarn")); + options.addAll(Arrays.asList("--release", "17", "-nowarn")); + Writer out = new PrintWriter(System.out); JavaCompiler.CompilationTask task = compiler.getTask(out, fileManager, diag, options, null, files);