001package daikon.dcomp; 002 003import static java.nio.charset.StandardCharsets.UTF_8; 004 005import daikon.DynComp; 006import daikon.plumelib.bcelutil.BcelUtil; 007import daikon.plumelib.options.Options; 008import daikon.plumelib.reflection.Signatures; 009import java.io.File; 010import java.io.FileInputStream; 011import java.io.FileNotFoundException; 012import java.io.IOException; 013import java.io.InputStream; 014import java.io.PrintWriter; 015import java.io.UncheckedIOException; 016import java.net.URI; 017import java.nio.file.DirectoryStream; 018import java.nio.file.FileSystem; 019import java.nio.file.FileSystems; 020import java.nio.file.Files; 021import java.nio.file.Path; 022import java.time.LocalDateTime; 023import java.time.ZoneId; 024import java.time.format.DateTimeFormatter; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Enumeration; 028import java.util.HashMap; 029import java.util.List; 030import java.util.Map; 031import java.util.jar.JarEntry; 032import java.util.jar.JarFile; 033import org.apache.bcel.Const; 034import org.apache.bcel.classfile.ClassParser; 035import org.apache.bcel.classfile.JavaClass; 036import org.apache.bcel.generic.ClassGen; 037import org.apache.bcel.generic.MethodGen; 038import org.apache.bcel.generic.Type; 039import org.checkerframework.checker.nullness.qual.NonNull; 040import org.checkerframework.checker.signature.qual.BinaryName; 041 042/** 043 * BuildJDK uses {@link DCInstrument} to add comparability instrumentation to Java class files, then 044 * stores the modified files into a directory identified by a (required) command line argument. 045 * 046 * <p>DCInstrument duplicates each method of a class file. The new methods are distinguished by the 047 * addition of a final parameter of type DCompMarker and are instrumented to track comparability. 048 * Based on its invocation arguments, DynComp will decide whether to call the instrumented or 049 * uninstrumented version of a method. 050 */ 051@SuppressWarnings({ 052 "mustcall:type.argument", 053 "mustcall:type.arguments.not.inferred" 054}) // assignments into owning collection 055public class BuildJDK { 056 057 /** Creates a new BuildJDK. */ 058 private BuildJDK() {} 059 060 /** 061 * The "java.home" system property. Note that there is also a JAVA_HOME variable that contains 062 * {@code System.getenv("JAVA_HOME")}. 063 */ 064 public static final String java_home = System.getProperty("java.home"); 065 066 /** Whether to print information about the classes being instrumented. */ 067 private static boolean verbose = false; 068 069 /** Number of class files processed; used for progress display. */ 070 private int _numFilesProcessed = 0; 071 072 /** Name of file in output jar containing the static-fields map. */ 073 private static String static_field_id_filename = "dcomp_jdk_static_field_id"; 074 075 /** 076 * Collects names of all methods that DCInstrument could not process. Should be empty. Format is 077 * <fully-qualified class name>.<method name> 078 */ 079 private static List<String> skipped_methods = new ArrayList<>(); 080 081 /** 082 * A list of methods known to cause DCInstrument to fail. This is used to remove known problems 083 * from the list of failures displayed at the end of BuildJDK's execution. Format is 084 * <fully-qualified class name>.<method name> 085 */ 086 public static List<String> known_uninstrumentable_methods = 087 Arrays.asList( 088 // None at present 089 ); 090 091 /** 092 * Instruments each class file in the Java runtime and puts the result in the first non-option 093 * command-line argument. 094 * 095 * <p>By default, BuildJDK will locate the appropriate Java runtime library and instrument each of 096 * its member class files. However, if there are additional arguments on the command line after 097 * the destination directory, then we assume these are class files to be instrumented and the Java 098 * runtime library is not used. This usage is primarily for testing purposes. 099 * 100 * @param args arguments being passed to BuildJDK 101 * @throws IOException if unable to read or write file {@code dcomp_jdk_static_field_id} or if 102 * unable to write {@code jdk_classes.txt} 103 */ 104 @SuppressWarnings("builder:required.method.not.called") // assignment into collection of @Owning 105 public static void main(String[] args) throws IOException { 106 107 System.out.println("BuildJDK starting at " + LocalDateTime.now(ZoneId.systemDefault())); 108 109 BuildJDK build = new BuildJDK(); 110 111 Options options = 112 new Options( 113 "daikon.BuildJDK [options] dest_dir [classfiles...]", 114 DynComp.class, 115 DCInstrument.class); 116 String[] cl_args = options.parse(true, args); 117 if (cl_args.length < 1) { 118 System.err.println("must specify destination dir"); 119 options.printUsage(); 120 System.exit(1); 121 } 122 verbose = DynComp.verbose; 123 124 File dest_dir = new File(cl_args[0]); 125 126 // Key is a class file name, value is a stream that opens that file name. 127 // 128 // <p>We want to share code to read and instrument the Java class file members of a jar file 129 // (JDK 8) or a module file (JDK 9+). However, jar files and module files are located in two 130 // completely different file systems. So we open an InputStream for each class file we wish to 131 // instrument and save it in the class_stream_map with the file name as the key. From that point 132 // the code to instrument a class file can be shared. 133 Map<String, InputStream> class_stream_map; 134 135 if (cl_args.length > 1) { 136 137 // Arguments are <destdir> [<classfiles>...] 138 @SuppressWarnings("nullness:assignment") // https://tinyurl.com/cfissue/3224 139 @NonNull String[] class_files = Arrays.copyOfRange(cl_args, 1, cl_args.length); 140 141 // Instrumenting a specific list of class files is usually used for testing. 142 // But if we're using it to fix a broken classfile, then we need 143 // to restore the static-fields map from when our runtime jar was originally 144 // built. We assume it is in the destination directory. 145 DCInstrument.restore_static_field_id(new File(dest_dir, static_field_id_filename)); 146 System.out.printf( 147 "Restored %d entries in static map.%n", DCInstrument.static_field_id.size()); 148 149 class_stream_map = new HashMap<>(); 150 for (String classFileName : class_files) { 151 try { 152 class_stream_map.put(classFileName, new FileInputStream(classFileName)); 153 } catch (FileNotFoundException e) { 154 throw new Error("File not found: " + classFileName, e); 155 } 156 } 157 158 // Instrument the classes identified in class_stream_map. 159 build.instrument_classes(dest_dir, class_stream_map); 160 161 } else { 162 163 check_java_home(); 164 165 if (BcelUtil.javaVersion > 8) { 166 class_stream_map = build.gather_runtime_from_modules(); 167 } else { 168 class_stream_map = build.gather_runtime_from_jar(); 169 } 170 171 // Instrument the Java runtime classes identified in class_stream_map. 172 build.instrument_classes(dest_dir, class_stream_map); 173 174 // We've finished instrumenting all the class files. Now we create some 175 // abstract interface classes for use by the DynComp runtime. 176 build.addInterfaceClasses(dest_dir.getName()); 177 178 // Write out the file containing the static-fields map. 179 System.out.printf("Found %d static fields.%n", DCInstrument.static_field_id.size()); 180 DCInstrument.save_static_field_id(new File(dest_dir, static_field_id_filename)); 181 182 // Write out the list of all classes in the jar file 183 File jdk_classes_file = new File(dest_dir, "java/lang/jdk_classes.txt"); 184 System.out.printf("Writing a list of class names to %s%n", jdk_classes_file); 185 // Class names are written in internal form. 186 try (PrintWriter pw = new PrintWriter(jdk_classes_file, UTF_8.name())) { 187 for (String classFileName : class_stream_map.keySet()) { 188 pw.println(classFileName.replace(".class", "")); 189 } 190 } 191 } 192 193 // Print out any methods that could not be instrumented 194 print_skipped_methods(); 195 196 System.out.println("BuildJDK done at " + LocalDateTime.now(ZoneId.systemDefault())); 197 } 198 199 /** Verify that java.home and JAVA_HOME match. Exits the JVM if there is an error. */ 200 public static void check_java_home() { 201 202 // We are going to instrument the default Java runtime library. 203 // We need to verify where we should look for it. 204 205 String JAVA_HOME = System.getenv("JAVA_HOME"); 206 if (JAVA_HOME == null) { 207 if (verbose) { 208 System.out.println("JAVA_HOME not defined; using java.home: " + java_home); 209 } 210 JAVA_HOME = java_home; 211 } 212 213 File jrt = new File(JAVA_HOME); 214 if (!jrt.exists()) { 215 System.err.printf("Java home directory %s does not exist.%n", jrt); 216 System.exit(1); 217 } 218 219 try { 220 jrt = jrt.getCanonicalFile(); 221 } catch (Exception e) { 222 System.err.printf("Error geting canonical file for %s: %s", jrt, e.getMessage()); 223 System.exit(1); 224 } 225 226 JAVA_HOME = jrt.getAbsolutePath(); 227 if (!java_home.startsWith(JAVA_HOME)) { 228 System.err.printf( 229 "JAVA_HOME (%s) does not agree with java.home (%s).%n", JAVA_HOME, java_home); 230 System.err.printf("Please correct your Java environment.%n"); 231 System.exit(1); 232 } 233 } 234 235 /** 236 * For Java 8 the Java runtime is located in rt.jar. This method creates an InputStream for each 237 * class in rt.jar and returns this information in a map from class file name to InputStream. 238 * 239 * @return a map from class file name to the associated InputStream 240 */ 241 @SuppressWarnings({ 242 "JdkObsolete", // JarEntry.entries() returns Enumeration 243 "builder:required.method.not.called" // assignment into collection of @Owning 244 }) 245 Map<String, InputStream> gather_runtime_from_jar() { 246 247 Map<String, InputStream> class_stream_map = new HashMap<>(); 248 String jar_name = java_home + "/lib/rt.jar"; 249 System.out.printf("using jar file %s%n", jar_name); 250 try { 251 JarFile jfile = new JarFile(jar_name); 252 // Get each class to be instrumented and store it away 253 Enumeration<JarEntry> entries = jfile.entries(); 254 while (entries.hasMoreElements()) { 255 JarEntry entry = entries.nextElement(); 256 // System.out.printf("processing entry %s%n", entry); 257 final String entryName = entry.getName(); 258 if (entryName.endsWith("/") || entryName.endsWith("~")) { 259 continue; 260 } 261 262 // Get the InputStream for this file 263 InputStream is = jfile.getInputStream(entry); 264 class_stream_map.put(entryName, is); 265 } 266 } catch (Exception e) { 267 throw new Error("Problem while reading " + jar_name, e); 268 } 269 return class_stream_map; 270 } 271 272 /** 273 * For Java 9+ the Java runtime is located in a series of modules. At this time, we are only 274 * pre-instrumenting the java.base module. This method initializes the DirectoryStream used to 275 * explore java.base. It calls gather_runtime_from_modules_directory to process the directory 276 * structure. 277 * 278 * @return a map from class file name to the associated InputStream 279 */ 280 Map<String, InputStream> gather_runtime_from_modules() { 281 282 Map<String, InputStream> class_stream_map = new HashMap<>(); 283 FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/")); 284 Path modules = fs.getPath("/modules"); 285 // The path java_home+/lib/modules is the file in the host file system that 286 // corresponds to the modules file in the jrt: file system. 287 System.out.printf("using modules directory %s%n", java_home + "/lib/modules"); 288 try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(modules, "java.base*")) { 289 for (Path moduleDir : directoryStream) { 290 gather_runtime_from_modules_directory( 291 moduleDir, moduleDir.toString().length(), class_stream_map); 292 } 293 } catch (IOException e) { 294 throw new UncheckedIOException(e); 295 } 296 return class_stream_map; 297 } 298 299 /** 300 * This is a helper method for {@link #gather_runtime_from_modules}. It recurses down the module 301 * directory tree, selects the classes we want to instrument, creates an InputStream for each of 302 * these classes, and adds this information to the {@code class_stream_map} argument. 303 * 304 * @param path module file, which might be subdirectory 305 * @param modulePrefixLength length of "/module/..." path prefix before start of actual member 306 * path 307 * @param class_stream_map a map from class file name to InputStream that collects the results 308 */ 309 @SuppressWarnings("builder:required.method.not.called") // assignment into collection of @Owning 310 void gather_runtime_from_modules_directory( 311 Path path, int modulePrefixLength, Map<String, InputStream> class_stream_map) { 312 313 if (Files.isDirectory(path)) { 314 try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(path)) { 315 for (Path subpath : directoryStream) { 316 gather_runtime_from_modules_directory(subpath, modulePrefixLength, class_stream_map); 317 } 318 } catch (IOException e) { 319 throw new UncheckedIOException(e); 320 } 321 } else { 322 String entryName = path.toString().substring(modulePrefixLength + 1); 323 try { 324 // Get the InputStream for this file 325 InputStream is = Files.newInputStream(path); 326 class_stream_map.put(entryName, is); 327 } catch (Exception e) { 328 throw new Error(e); 329 } 330 } 331 } 332 333 /** 334 * Instrument each of the classes indentified by the class_stream_map argument. 335 * 336 * @param dest_dir where to store the instrumented classes 337 * @param class_stream_map maps from class file name to an input stream on that file 338 */ 339 void instrument_classes(File dest_dir, Map<String, InputStream> class_stream_map) { 340 341 try { 342 // Create the destination directory 343 dest_dir.mkdirs(); 344 345 // Process each file. 346 for (String classFileName : class_stream_map.keySet()) { 347 if (verbose) { 348 System.out.println("instrument_classes: " + classFileName); 349 } 350 351 if (classFileName.equals("module-info.class")) { 352 System.out.printf("Skipping file %s%n", classFileName); 353 continue; 354 } 355 356 // Handle non-.class files and Object.class. In JDK 8, copy them unchanged. 357 // For JDK 9+ we do not copy as these items will be loaded from the original module file. 358 if (!classFileName.endsWith(".class") || classFileName.equals("java/lang/Object.class")) { 359 if (BcelUtil.javaVersion > 8) { 360 if (verbose) { 361 System.out.printf("Skipping file %s%n", classFileName); 362 } 363 continue; 364 } 365 // This File constructor ignores dest_dir if classFileName is absolute. 366 File classFile = new File(dest_dir, classFileName); 367 if (classFile.getParentFile() == null) { 368 throw new Error("This can't happen: " + classFile); 369 } 370 classFile.getParentFile().mkdirs(); 371 if (verbose) { 372 System.out.println("Copying Object.class or non-classfile: " + classFile); 373 } 374 try (InputStream in = class_stream_map.get(classFileName)) { 375 Files.copy(in, classFile.toPath()); 376 } 377 continue; 378 } 379 380 // Get the binary for this class 381 JavaClass jc; 382 try (InputStream is = class_stream_map.get(classFileName)) { 383 ClassParser parser = new ClassParser(is, classFileName); 384 jc = parser.parse(); 385 } catch (Throwable e) { 386 throw new Error("Failed to parse classfile " + classFileName, e); 387 } 388 389 // Instrument the class file. 390 try { 391 instrumentClassFile(jc, dest_dir, classFileName, class_stream_map.size()); 392 } catch (Throwable e) { 393 throw new Error("Couldn't instrument " + classFileName, e); 394 } 395 } 396 } catch (Exception e) { 397 throw new Error(e); 398 } 399 } 400 401 /** 402 * Add abstract interface classes needed by the DynComp runtime. 403 * 404 * @param destDir where to store the interface classes 405 */ 406 private void addInterfaceClasses(String destDir) { 407 // Create the DcompMarker class which is used to identify instrumented calls. 408 createDCompClass(destDir, "DCompMarker", false); 409 410 // The remainer of the generated classes are needed for JDK 9+ only. 411 if (BcelUtil.javaVersion > 8) { 412 createDCompClass(destDir, "DCompInstrumented", true); 413 createDCompClass(destDir, "DCompClone", false); 414 createDCompClass(destDir, "DCompToString", false); 415 } 416 } 417 418 /** 419 * Create an abstract interface class for use by the DynComp runtime. 420 * 421 * @param destDir where to store the new class 422 * @param className name of class 423 * @param dcompInstrumented if true, add equals_dcomp_instrumented method to class 424 */ 425 private void createDCompClass( 426 String destDir, @BinaryName String className, boolean dcompInstrumented) { 427 try { 428 ClassGen dcomp_class = 429 new ClassGen( 430 Signatures.addPackage("java.lang", className), 431 "java.lang.Object", 432 "daikon.dcomp.BuildJDK tool", 433 Const.ACC_INTERFACE | Const.ACC_PUBLIC | Const.ACC_ABSTRACT, 434 new @BinaryName String[0]); 435 dcomp_class.setMinor(0); 436 // Convert from JDK version number to ClassFile major_version. 437 // A bit of a hack, but seems OK. 438 dcomp_class.setMajor(BcelUtil.javaVersion + 44); 439 440 if (dcompInstrumented) { 441 @SuppressWarnings("nullness:argument") // null instruction list is ok for abstract 442 MethodGen mg = 443 new MethodGen( 444 Const.ACC_PUBLIC | Const.ACC_ABSTRACT, 445 Type.BOOLEAN, 446 new Type[] {Type.OBJECT}, 447 new String[] {"o"}, 448 "equals_dcomp_instrumented", 449 dcomp_class.getClassName(), 450 null, 451 dcomp_class.getConstantPool()); 452 dcomp_class.addMethod(mg.getMethod()); 453 } 454 455 dcomp_class 456 .getJavaClass() 457 .dump( 458 // Path.of exists in Java 11 and later. 459 new File(new File(new File(destDir, "java"), "lang"), className + ".class")); 460 } catch (Exception e) { 461 throw new Error(e); 462 } 463 } 464 465 /** Formats just the time part of a DateTime. */ 466 private DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss"); 467 468 /** 469 * Instruments the JavaClass {@code jc} (whose name is {@code classFileName}). Writes the 470 * resulting class to its corresponding location in the directory outputDir. 471 * 472 * @param jc JavaClass to be instrumented 473 * @param outputDir output directory for instrumented class 474 * @param classFileName name of class to be instrumented 475 * @param classTotal total number of classes to be processed; used for progress display 476 * @throws IOException if unable to write out instrumented class 477 */ 478 @SuppressWarnings("SystemConsoleNull") // https://errorprone.info/bugpattern/SystemConsoleNull 479 private void instrumentClassFile( 480 JavaClass jc, File outputDir, String classFileName, int classTotal) 481 throws java.io.IOException { 482 if (verbose) { 483 System.out.printf("processing target %s%n", classFileName); 484 } 485 DCInstrument dci = new DCInstrument(jc, true, null); 486 JavaClass inst_jc; 487 inst_jc = dci.instrument_jdk(); 488 skipped_methods.addAll(dci.get_skipped_methods()); 489 File classfile = new File(classFileName); 490 File dir; 491 if (classfile.getParent() == null) { 492 dir = outputDir; 493 } else { 494 dir = new File(outputDir, classfile.getParent()); 495 } 496 dir.mkdirs(); 497 File classpath = new File(dir, classfile.getName()); 498 if (verbose) { 499 System.out.printf("writing to file %s%n", classpath); 500 } 501 inst_jc.dump(classpath); 502 _numFilesProcessed++; 503 if (((_numFilesProcessed % 100) == 0) && (System.console() != null)) { 504 System.out.printf( 505 "Processed %d/%d classes at %s%n", 506 _numFilesProcessed, 507 classTotal, 508 LocalDateTime.now(ZoneId.systemDefault()).format(timeFormatter)); 509 } 510 } 511 512 /** 513 * Print information about methods that were not instrumented. This happens when a method fails 514 * BCEL's verifier (which is more strict than Java's). 515 */ 516 private static void print_skipped_methods() { 517 518 if (skipped_methods.isEmpty()) { 519 // System.out.printf("No methods were skipped.%n"); 520 return; 521 } 522 523 System.err.println( 524 "Warning: The following JDK methods could not be instrumented. DynComp will"); 525 System.err.println("still work as long as these methods are not called by your application."); 526 System.err.println("If your application calls one, it will throw a NoSuchMethodException."); 527 528 List<String> unknown = new ArrayList<>(skipped_methods); 529 unknown.removeAll(known_uninstrumentable_methods); 530 List<String> known = new ArrayList<>(skipped_methods); 531 known.retainAll(known_uninstrumentable_methods); 532 533 if (!unknown.isEmpty()) { 534 System.err.println("Please report the following problems to the Daikon maintainers."); 535 System.err.println( 536 "Please give sufficient details; see \"Reporting problems\" in the Daikon manual."); 537 for (String method : unknown) { 538 System.err.printf(" %s%n", method); 539 } 540 } 541 if (!known.isEmpty()) { 542 System.err.printf("The following are known problems; you do not need to report them."); 543 for (String method : known) { 544 System.err.printf(" %s%n", method); 545 } 546 } 547 } 548}