001package daikon.dcomp;
002
003import static java.nio.charset.StandardCharsets.UTF_8;
004
005import daikon.DynComp;
006import daikon.plumelib.bcelutil.BcelUtil;
007import daikon.plumelib.options.Options;
008import daikon.plumelib.reflection.Signatures;
009import java.io.File;
010import java.io.FileInputStream;
011import java.io.FileNotFoundException;
012import java.io.IOException;
013import java.io.InputStream;
014import java.io.PrintWriter;
015import java.io.UncheckedIOException;
016import java.net.URI;
017import java.nio.file.DirectoryStream;
018import java.nio.file.FileSystem;
019import java.nio.file.FileSystems;
020import java.nio.file.Files;
021import java.nio.file.Path;
022import java.time.LocalDateTime;
023import java.time.ZoneId;
024import java.time.format.DateTimeFormatter;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Enumeration;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.jar.JarEntry;
032import java.util.jar.JarFile;
033import org.apache.bcel.Const;
034import org.apache.bcel.classfile.ClassParser;
035import org.apache.bcel.classfile.JavaClass;
036import org.apache.bcel.generic.ClassGen;
037import org.apache.bcel.generic.MethodGen;
038import org.apache.bcel.generic.Type;
039import org.checkerframework.checker.nullness.qual.NonNull;
040import org.checkerframework.checker.signature.qual.BinaryName;
041
042/**
043 * BuildJDK uses {@link DCInstrument} to add comparability instrumentation to Java class files, then
044 * stores the modified files into a directory identified by a (required) command line argument.
045 *
046 * <p>DCInstrument duplicates each method of a class file. The new methods are distinguished by the
047 * addition of a final parameter of type DCompMarker and are instrumented to track comparability.
048 * Based on its invocation arguments, DynComp will decide whether to call the instrumented or
049 * uninstrumented version of a method.
050 */
051@SuppressWarnings({
052  "mustcall:type.argument",
053  "mustcall:type.arguments.not.inferred"
054}) // assignments into owning collection
055public class BuildJDK {
056
057  /** Creates a new BuildJDK. */
058  private BuildJDK() {}
059
060  /**
061   * The "java.home" system property. Note that there is also a JAVA_HOME variable that contains
062   * {@code System.getenv("JAVA_HOME")}.
063   */
064  public static final String java_home = System.getProperty("java.home");
065
066  /** Whether to print information about the classes being instrumented. */
067  private static boolean verbose = false;
068
069  /** Number of class files processed; used for progress display. */
070  private int _numFilesProcessed = 0;
071
072  /** Name of file in output jar containing the static-fields map. */
073  private static String static_field_id_filename = "dcomp_jdk_static_field_id";
074
075  /**
076   * Collects names of all methods that DCInstrument could not process. Should be empty. Format is
077   * &lt;fully-qualified class name&gt;.&lt;method name&gt;
078   */
079  private static List<String> skipped_methods = new ArrayList<>();
080
081  /**
082   * A list of methods known to cause DCInstrument to fail. This is used to remove known problems
083   * from the list of failures displayed at the end of BuildJDK's execution. Format is
084   * &lt;fully-qualified class name&gt;.&lt;method name&gt;
085   */
086  public static List<String> known_uninstrumentable_methods =
087      Arrays.asList(
088          // None at present
089          );
090
091  /**
092   * Instruments each class file in the Java runtime and puts the result in the first non-option
093   * command-line argument.
094   *
095   * <p>By default, BuildJDK will locate the appropriate Java runtime library and instrument each of
096   * its member class files. However, if there are additional arguments on the command line after
097   * the destination directory, then we assume these are class files to be instrumented and the Java
098   * runtime library is not used. This usage is primarily for testing purposes.
099   *
100   * @param args arguments being passed to BuildJDK
101   * @throws IOException if unable to read or write file {@code dcomp_jdk_static_field_id} or if
102   *     unable to write {@code jdk_classes.txt}
103   */
104  @SuppressWarnings("builder:required.method.not.called") // assignment into collection of @Owning
105  public static void main(String[] args) throws IOException {
106
107    System.out.println("BuildJDK starting at " + LocalDateTime.now(ZoneId.systemDefault()));
108
109    BuildJDK build = new BuildJDK();
110
111    Options options =
112        new Options(
113            "daikon.BuildJDK [options] dest_dir [classfiles...]",
114            DynComp.class,
115            DCInstrument.class);
116    String[] cl_args = options.parse(true, args);
117    if (cl_args.length < 1) {
118      System.err.println("must specify destination dir");
119      options.printUsage();
120      System.exit(1);
121    }
122    verbose = DynComp.verbose;
123
124    File dest_dir = new File(cl_args[0]);
125
126    // Key is a class file name, value is a stream that opens that file name.
127    //
128    // <p>We want to share code to read and instrument the Java class file members of a jar file
129    // (JDK 8) or a module file (JDK 9+). However, jar files and module files are located in two
130    // completely different file systems. So we open an InputStream for each class file we wish to
131    // instrument and save it in the class_stream_map with the file name as the key. From that point
132    // the code to instrument a class file can be shared.
133    Map<String, InputStream> class_stream_map;
134
135    if (cl_args.length > 1) {
136
137      // Arguments are <destdir> [<classfiles>...]
138      @SuppressWarnings("nullness:assignment") // https://tinyurl.com/cfissue/3224
139      @NonNull String[] class_files = Arrays.copyOfRange(cl_args, 1, cl_args.length);
140
141      // Instrumenting a specific list of class files is usually used for testing.
142      // But if we're using it to fix a broken classfile, then we need
143      // to restore the static-fields map from when our runtime jar was originally
144      // built.  We assume it is in the destination directory.
145      DCInstrument.restore_static_field_id(new File(dest_dir, static_field_id_filename));
146      System.out.printf(
147          "Restored %d entries in static map.%n", DCInstrument.static_field_id.size());
148
149      class_stream_map = new HashMap<>();
150      for (String classFileName : class_files) {
151        try {
152          class_stream_map.put(classFileName, new FileInputStream(classFileName));
153        } catch (FileNotFoundException e) {
154          throw new Error("File not found: " + classFileName, e);
155        }
156      }
157
158      // Instrument the classes identified in class_stream_map.
159      build.instrument_classes(dest_dir, class_stream_map);
160
161    } else {
162
163      check_java_home();
164
165      if (BcelUtil.javaVersion > 8) {
166        class_stream_map = build.gather_runtime_from_modules();
167      } else {
168        class_stream_map = build.gather_runtime_from_jar();
169      }
170
171      // Instrument the Java runtime classes identified in class_stream_map.
172      build.instrument_classes(dest_dir, class_stream_map);
173
174      // We've finished instrumenting all the class files. Now we create some
175      // abstract interface classes for use by the DynComp runtime.
176      build.addInterfaceClasses(dest_dir.getName());
177
178      // Write out the file containing the static-fields map.
179      System.out.printf("Found %d static fields.%n", DCInstrument.static_field_id.size());
180      DCInstrument.save_static_field_id(new File(dest_dir, static_field_id_filename));
181
182      // Write out the list of all classes in the jar file
183      File jdk_classes_file = new File(dest_dir, "java/lang/jdk_classes.txt");
184      System.out.printf("Writing a list of class names to %s%n", jdk_classes_file);
185      // Class names are written in internal form.
186      try (PrintWriter pw = new PrintWriter(jdk_classes_file, UTF_8.name())) {
187        for (String classFileName : class_stream_map.keySet()) {
188          pw.println(classFileName.replace(".class", ""));
189        }
190      }
191    }
192
193    // Print out any methods that could not be instrumented
194    print_skipped_methods();
195
196    System.out.println("BuildJDK done at " + LocalDateTime.now(ZoneId.systemDefault()));
197  }
198
199  /** Verify that java.home and JAVA_HOME match. Exits the JVM if there is an error. */
200  public static void check_java_home() {
201
202    // We are going to instrument the default Java runtime library.
203    // We need to verify where we should look for it.
204
205    String JAVA_HOME = System.getenv("JAVA_HOME");
206    if (JAVA_HOME == null) {
207      if (verbose) {
208        System.out.println("JAVA_HOME not defined; using java.home: " + java_home);
209      }
210      JAVA_HOME = java_home;
211    }
212
213    File jrt = new File(JAVA_HOME);
214    if (!jrt.exists()) {
215      System.err.printf("Java home directory %s does not exist.%n", jrt);
216      System.exit(1);
217    }
218
219    try {
220      jrt = jrt.getCanonicalFile();
221    } catch (Exception e) {
222      System.err.printf("Error geting canonical file for %s: %s", jrt, e.getMessage());
223      System.exit(1);
224    }
225
226    JAVA_HOME = jrt.getAbsolutePath();
227    if (!java_home.startsWith(JAVA_HOME)) {
228      System.err.printf(
229          "JAVA_HOME (%s) does not agree with java.home (%s).%n", JAVA_HOME, java_home);
230      System.err.printf("Please correct your Java environment.%n");
231      System.exit(1);
232    }
233  }
234
235  /**
236   * For Java 8 the Java runtime is located in rt.jar. This method creates an InputStream for each
237   * class in rt.jar and returns this information in a map from class file name to InputStream.
238   *
239   * @return a map from class file name to the associated InputStream
240   */
241  @SuppressWarnings({
242    "JdkObsolete", // JarEntry.entries() returns Enumeration
243    "builder:required.method.not.called" // assignment into collection of @Owning
244  })
245  Map<String, InputStream> gather_runtime_from_jar() {
246
247    Map<String, InputStream> class_stream_map = new HashMap<>();
248    String jar_name = java_home + "/lib/rt.jar";
249    System.out.printf("using jar file %s%n", jar_name);
250    try {
251      JarFile jfile = new JarFile(jar_name);
252      // Get each class to be instrumented and store it away
253      Enumeration<JarEntry> entries = jfile.entries();
254      while (entries.hasMoreElements()) {
255        JarEntry entry = entries.nextElement();
256        // System.out.printf("processing entry %s%n", entry);
257        final String entryName = entry.getName();
258        if (entryName.endsWith("/") || entryName.endsWith("~")) {
259          continue;
260        }
261
262        // Get the InputStream for this file
263        InputStream is = jfile.getInputStream(entry);
264        class_stream_map.put(entryName, is);
265      }
266    } catch (Exception e) {
267      throw new Error("Problem while reading " + jar_name, e);
268    }
269    return class_stream_map;
270  }
271
272  /**
273   * For Java 9+ the Java runtime is located in a series of modules. At this time, we are only
274   * pre-instrumenting the java.base module. This method initializes the DirectoryStream used to
275   * explore java.base. It calls gather_runtime_from_modules_directory to process the directory
276   * structure.
277   *
278   * @return a map from class file name to the associated InputStream
279   */
280  Map<String, InputStream> gather_runtime_from_modules() {
281
282    Map<String, InputStream> class_stream_map = new HashMap<>();
283    FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/"));
284    Path modules = fs.getPath("/modules");
285    // The path java_home+/lib/modules is the file in the host file system that
286    // corresponds to the modules file in the jrt: file system.
287    System.out.printf("using modules directory %s%n", java_home + "/lib/modules");
288    try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(modules, "java.base*")) {
289      for (Path moduleDir : directoryStream) {
290        gather_runtime_from_modules_directory(
291            moduleDir, moduleDir.toString().length(), class_stream_map);
292      }
293    } catch (IOException e) {
294      throw new UncheckedIOException(e);
295    }
296    return class_stream_map;
297  }
298
299  /**
300   * This is a helper method for {@link #gather_runtime_from_modules}. It recurses down the module
301   * directory tree, selects the classes we want to instrument, creates an InputStream for each of
302   * these classes, and adds this information to the {@code class_stream_map} argument.
303   *
304   * @param path module file, which might be subdirectory
305   * @param modulePrefixLength length of "/module/..." path prefix before start of actual member
306   *     path
307   * @param class_stream_map a map from class file name to InputStream that collects the results
308   */
309  @SuppressWarnings("builder:required.method.not.called") // assignment into collection of @Owning
310  void gather_runtime_from_modules_directory(
311      Path path, int modulePrefixLength, Map<String, InputStream> class_stream_map) {
312
313    if (Files.isDirectory(path)) {
314      try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(path)) {
315        for (Path subpath : directoryStream) {
316          gather_runtime_from_modules_directory(subpath, modulePrefixLength, class_stream_map);
317        }
318      } catch (IOException e) {
319        throw new UncheckedIOException(e);
320      }
321    } else {
322      String entryName = path.toString().substring(modulePrefixLength + 1);
323      try {
324        // Get the InputStream for this file
325        InputStream is = Files.newInputStream(path);
326        class_stream_map.put(entryName, is);
327      } catch (Exception e) {
328        throw new Error(e);
329      }
330    }
331  }
332
333  /**
334   * Instrument each of the classes indentified by the class_stream_map argument.
335   *
336   * @param dest_dir where to store the instrumented classes
337   * @param class_stream_map maps from class file name to an input stream on that file
338   */
339  void instrument_classes(File dest_dir, Map<String, InputStream> class_stream_map) {
340
341    try {
342      // Create the destination directory
343      dest_dir.mkdirs();
344
345      // Process each file.
346      for (String classFileName : class_stream_map.keySet()) {
347        if (verbose) {
348          System.out.println("instrument_classes: " + classFileName);
349        }
350
351        if (classFileName.equals("module-info.class")) {
352          System.out.printf("Skipping file %s%n", classFileName);
353          continue;
354        }
355
356        // Handle non-.class files and Object.class.  In JDK 8, copy them unchanged.
357        // For JDK 9+ we do not copy as these items will be loaded from the original module file.
358        if (!classFileName.endsWith(".class") || classFileName.equals("java/lang/Object.class")) {
359          if (BcelUtil.javaVersion > 8) {
360            if (verbose) {
361              System.out.printf("Skipping file %s%n", classFileName);
362            }
363            continue;
364          }
365          // This File constructor ignores dest_dir if classFileName is absolute.
366          File classFile = new File(dest_dir, classFileName);
367          if (classFile.getParentFile() == null) {
368            throw new Error("This can't happen: " + classFile);
369          }
370          classFile.getParentFile().mkdirs();
371          if (verbose) {
372            System.out.println("Copying Object.class or non-classfile: " + classFile);
373          }
374          try (InputStream in = class_stream_map.get(classFileName)) {
375            Files.copy(in, classFile.toPath());
376          }
377          continue;
378        }
379
380        // Get the binary for this class
381        JavaClass jc;
382        try (InputStream is = class_stream_map.get(classFileName)) {
383          ClassParser parser = new ClassParser(is, classFileName);
384          jc = parser.parse();
385        } catch (Throwable e) {
386          throw new Error("Failed to parse classfile " + classFileName, e);
387        }
388
389        // Instrument the class file.
390        try {
391          instrumentClassFile(jc, dest_dir, classFileName, class_stream_map.size());
392        } catch (Throwable e) {
393          throw new Error("Couldn't instrument " + classFileName, e);
394        }
395      }
396    } catch (Exception e) {
397      throw new Error(e);
398    }
399  }
400
401  /**
402   * Add abstract interface classes needed by the DynComp runtime.
403   *
404   * @param destDir where to store the interface classes
405   */
406  private void addInterfaceClasses(String destDir) {
407    // Create the DcompMarker class which is used to identify instrumented calls.
408    createDCompClass(destDir, "DCompMarker", false);
409
410    // The remainer of the generated classes are needed for JDK 9+ only.
411    if (BcelUtil.javaVersion > 8) {
412      createDCompClass(destDir, "DCompInstrumented", true);
413      createDCompClass(destDir, "DCompClone", false);
414      createDCompClass(destDir, "DCompToString", false);
415    }
416  }
417
418  /**
419   * Create an abstract interface class for use by the DynComp runtime.
420   *
421   * @param destDir where to store the new class
422   * @param className name of class
423   * @param dcompInstrumented if true, add equals_dcomp_instrumented method to class
424   */
425  private void createDCompClass(
426      String destDir, @BinaryName String className, boolean dcompInstrumented) {
427    try {
428      ClassGen dcomp_class =
429          new ClassGen(
430              Signatures.addPackage("java.lang", className),
431              "java.lang.Object",
432              "daikon.dcomp.BuildJDK tool",
433              Const.ACC_INTERFACE | Const.ACC_PUBLIC | Const.ACC_ABSTRACT,
434              new @BinaryName String[0]);
435      dcomp_class.setMinor(0);
436      // Convert from JDK version number to ClassFile major_version.
437      // A bit of a hack, but seems OK.
438      dcomp_class.setMajor(BcelUtil.javaVersion + 44);
439
440      if (dcompInstrumented) {
441        @SuppressWarnings("nullness:argument") // null instruction list is ok for abstract
442        MethodGen mg =
443            new MethodGen(
444                Const.ACC_PUBLIC | Const.ACC_ABSTRACT,
445                Type.BOOLEAN,
446                new Type[] {Type.OBJECT},
447                new String[] {"o"},
448                "equals_dcomp_instrumented",
449                dcomp_class.getClassName(),
450                null,
451                dcomp_class.getConstantPool());
452        dcomp_class.addMethod(mg.getMethod());
453      }
454
455      dcomp_class
456          .getJavaClass()
457          .dump(
458              // Path.of exists in Java 11 and later.
459              new File(new File(new File(destDir, "java"), "lang"), className + ".class"));
460    } catch (Exception e) {
461      throw new Error(e);
462    }
463  }
464
465  /** Formats just the time part of a DateTime. */
466  private DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
467
468  /**
469   * Instruments the JavaClass {@code jc} (whose name is {@code classFileName}). Writes the
470   * resulting class to its corresponding location in the directory outputDir.
471   *
472   * @param jc JavaClass to be instrumented
473   * @param outputDir output directory for instrumented class
474   * @param classFileName name of class to be instrumented
475   * @param classTotal total number of classes to be processed; used for progress display
476   * @throws IOException if unable to write out instrumented class
477   */
478  @SuppressWarnings("SystemConsoleNull") // https://errorprone.info/bugpattern/SystemConsoleNull
479  private void instrumentClassFile(
480      JavaClass jc, File outputDir, String classFileName, int classTotal)
481      throws java.io.IOException {
482    if (verbose) {
483      System.out.printf("processing target %s%n", classFileName);
484    }
485    DCInstrument dci = new DCInstrument(jc, true, null);
486    JavaClass inst_jc;
487    inst_jc = dci.instrument_jdk();
488    skipped_methods.addAll(dci.get_skipped_methods());
489    File classfile = new File(classFileName);
490    File dir;
491    if (classfile.getParent() == null) {
492      dir = outputDir;
493    } else {
494      dir = new File(outputDir, classfile.getParent());
495    }
496    dir.mkdirs();
497    File classpath = new File(dir, classfile.getName());
498    if (verbose) {
499      System.out.printf("writing to file %s%n", classpath);
500    }
501    inst_jc.dump(classpath);
502    _numFilesProcessed++;
503    if (((_numFilesProcessed % 100) == 0) && (System.console() != null)) {
504      System.out.printf(
505          "Processed %d/%d classes at %s%n",
506          _numFilesProcessed,
507          classTotal,
508          LocalDateTime.now(ZoneId.systemDefault()).format(timeFormatter));
509    }
510  }
511
512  /**
513   * Print information about methods that were not instrumented. This happens when a method fails
514   * BCEL's verifier (which is more strict than Java's).
515   */
516  private static void print_skipped_methods() {
517
518    if (skipped_methods.isEmpty()) {
519      // System.out.printf("No methods were skipped.%n");
520      return;
521    }
522
523    System.err.println(
524        "Warning: The following JDK methods could not be instrumented. DynComp will");
525    System.err.println("still work as long as these methods are not called by your application.");
526    System.err.println("If your application calls one, it will throw a NoSuchMethodException.");
527
528    List<String> unknown = new ArrayList<>(skipped_methods);
529    unknown.removeAll(known_uninstrumentable_methods);
530    List<String> known = new ArrayList<>(skipped_methods);
531    known.retainAll(known_uninstrumentable_methods);
532
533    if (!unknown.isEmpty()) {
534      System.err.println("Please report the following problems to the Daikon maintainers.");
535      System.err.println(
536          "Please give sufficient details; see \"Reporting problems\" in the Daikon manual.");
537      for (String method : unknown) {
538        System.err.printf("  %s%n", method);
539      }
540    }
541    if (!known.isEmpty()) {
542      System.err.printf("The following are known problems; you do not need to report them.");
543      for (String method : known) {
544        System.err.printf("  %s%n", method);
545      }
546    }
547  }
548}