001package daikon; 002 003import static daikon.PptRelation.PptRelationType; 004import static daikon.PptTopLevel.PptFlags; 005import static daikon.PptTopLevel.PptType; 006import static daikon.VarInfo.LangFlags; 007import static daikon.VarInfo.RefType; 008import static daikon.VarInfo.VarFlags; 009import static daikon.VarInfo.VarKind; 010import static daikon.tools.nullness.NullnessUtil.castNonNullDeep; 011import static java.nio.charset.StandardCharsets.UTF_8; 012 013import daikon.Daikon.BugInDaikon; 014import daikon.config.Configuration; 015import daikon.derive.ValueAndModified; 016import daikon.diff.InvMap; 017import daikon.inv.Invariant; 018import java.io.BufferedReader; 019import java.io.BufferedWriter; 020import java.io.Closeable; 021import java.io.EOFException; 022import java.io.File; 023import java.io.IOException; 024import java.io.InputStream; 025import java.io.InputStreamReader; 026import java.io.InvalidClassException; 027import java.io.LineNumberReader; 028import java.io.ObjectInputStream; 029import java.io.PrintWriter; 030import java.io.Reader; 031import java.io.Serializable; 032import java.io.StringWriter; 033import java.io.UncheckedIOException; 034import java.net.ServerSocket; 035import java.net.Socket; 036import java.net.URI; 037import java.net.URL; 038import java.nio.file.Files; 039import java.nio.file.Path; 040import java.text.NumberFormat; 041import java.util.ArrayDeque; 042import java.util.ArrayList; 043import java.util.Arrays; 044import java.util.Collection; 045import java.util.Deque; 046import java.util.EnumSet; 047import java.util.HashMap; 048import java.util.LinkedHashMap; 049import java.util.List; 050import java.util.Map; 051import java.util.Scanner; 052import java.util.StringJoiner; 053import java.util.logging.Level; 054import java.util.logging.Logger; 055import java.util.zip.GZIPInputStream; 056import org.checkerframework.checker.calledmethods.qual.EnsuresCalledMethods; 057import org.checkerframework.checker.interning.qual.Interned; 058import org.checkerframework.checker.interning.qual.UsesObjectEquals; 059import org.checkerframework.checker.lock.qual.GuardSatisfied; 060import org.checkerframework.checker.mustcall.qual.MustCall; 061import org.checkerframework.checker.mustcall.qual.Owning; 062import org.checkerframework.checker.nullness.qual.EnsuresNonNull; 063import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; 064import org.checkerframework.checker.nullness.qual.KeyFor; 065import org.checkerframework.checker.nullness.qual.MonotonicNonNull; 066import org.checkerframework.checker.nullness.qual.NonNull; 067import org.checkerframework.checker.nullness.qual.Nullable; 068import org.checkerframework.checker.nullness.qual.RequiresNonNull; 069import org.checkerframework.checker.signedness.qual.UnknownSignedness; 070import org.checkerframework.dataflow.qual.Pure; 071import org.checkerframework.dataflow.qual.SideEffectFree; 072import org.plumelib.util.CollectionsPlume; 073import org.plumelib.util.FilesPlume; 074import org.plumelib.util.StringsPlume; 075 076/** File I/O utilities. */ 077public final class FileIO { 078 079 /** Nobody should ever instantiate a FileIO. */ 080 private FileIO() { 081 throw new Error(); 082 } 083 084 /// Constants 085 086 static final String declaration_header = "DECLARE"; 087 088 // Program point name tags 089 /** String used to append a ppt type to a ppt name. */ 090 public static final String ppt_tag_separator = ":::"; 091 092 /** String used to identify entry ppt names. */ 093 public static final String enter_suffix = "ENTER"; 094 095 /** String used to mark entry ppt names. */ 096 public static final String enter_tag = ppt_tag_separator + enter_suffix; 097 098 // EXIT does not necessarily appear at the end of the program point name; 099 // a number may follow it. 100 /** String used to identify exit ppt names. */ 101 public static final String exit_suffix = "EXIT"; 102 103 /** String used to mark exit ppt names. */ 104 public static final String exit_tag = ppt_tag_separator + exit_suffix; 105 106 /** To be deleted. */ 107 public static final String throws_suffix = "THROWS"; 108 109 /** To be deleted. */ 110 public static final String throws_tag = ppt_tag_separator + throws_suffix; 111 112 public static final String object_suffix = "OBJECT"; 113 114 /** String used to mark object ppt names. */ 115 public static final String object_tag = ppt_tag_separator + object_suffix; 116 117 /** String used to identify class ppt names. */ 118 public static final String class_static_suffix = "CLASS"; 119 120 /** String used to mark class ppt names. */ 121 public static final String class_static_tag = ppt_tag_separator + class_static_suffix; 122 123 /** String used to identify global ppt names. */ 124 public static final String global_suffix = "GLOBAL"; 125 126 private static final String lineSep = Global.lineSep; 127 128 /// Settings 129 130 // Variables starting with dkconfig_ should only be set via the 131 // daikon.config.Configuration interface. 132 133 /** 134 * When true, just ignore exit ppts that don't have a matching enter ppt rather than exiting with 135 * an error. Unmatched exits can occur if only a portion of a dtrace file is processed. 136 */ 137 public static boolean dkconfig_ignore_missing_enter = false; 138 139 /** 140 * Boolean. When false, set modbits to 1 iff the printed representation has changed. When true, 141 * set modbits to 1 if the printed representation has changed; leave other modbits as is. 142 */ 143 public static boolean dkconfig_add_changed = true; 144 145 /** Integer. Maximum number of lines to read from the dtrace file. If 0, reads the entire file. */ 146 public static int dkconfig_max_line_number = 0; 147 148 /** 149 * Boolean. When false, don't count the number of lines in the dtrace file before reading. This 150 * will disable the percentage progress printout. 151 */ 152 public static boolean dkconfig_count_lines = true; 153 154 /** 155 * Boolean. When true, only read the samples, but don't process them. Used to gather timing 156 * information. 157 */ 158 public static boolean dkconfig_read_samples_only = false; 159 160 /** 161 * Boolean. When true, don't print a warning about unmatched procedure entries, which are ignored 162 * by Daikon (unless the {@code --nohierarchy} command-line argument is provided). 163 */ 164 public static boolean dkconfig_unmatched_procedure_entries_quiet = false; 165 166 /** Boolean. If true, prints the unmatched procedure entries verbosely. */ 167 public static boolean dkconfig_verbose_unmatched_procedure_entries = false; 168 169 /** 170 * Boolean. When true, suppress exceptions related to file reading. This permits Daikon to 171 * continue even if there is a malformed trace file. Use this with care: in general, it is better 172 * to fix the problem that caused a bad trace file, rather than to suppress the exception. 173 */ 174 public static boolean dkconfig_continue_after_file_exception = false; 175 176 /** 177 * Long integer. If non-zero, this value will be used as the number of lines in (each) dtrace file 178 * input for the purposes of the progress display, and the counting of the lines in the file will 179 * be suppressed. 180 */ 181 public static long dkconfig_dtrace_line_count = 0; 182 183 /** True if declaration records are in the new format -- that is, decl-version 2.0. */ 184 // Set by read_decl_version; by read_data_trace_record if the file is non-empty; 185 // by read_serialized_pptmap; and by InvMap.readObject. 186 public static @MonotonicNonNull Boolean new_decl_format = null; 187 188 /** 189 * Do not use this routine unless you know what you are doing. This routine breaks the 190 * representation invariant that new_decl_format, once set, is never reset to null. This routine 191 * should be used only if you can guarantee that new_decl_format will be once again set to a 192 * non-null value before any code runs that depends on the fact that new_decl_format is non-null. 193 */ 194 @SuppressWarnings("nullness") // reinitialization 195 public static void resetNewDeclFormat() { 196 FileIO.new_decl_format = null; 197 } 198 199 /** 200 * If true, modified all ppt names to remove duplicate routine names within the ppt name. This is 201 * used when a stack trace (of active methods) is used as the ppt name. The routine names must be 202 * separated by vertical bars (|). 203 */ 204 public static boolean dkconfig_rm_stack_dups = false; 205 206 /// Variables 207 208 // This hashmap maps every program point to an array, which contains the 209 // old values of all variables in scope the last time the program point 210 // was executed. This enables us to determine whether the values have been 211 // modified since this program point was last executed. 212 static HashMap<PptTopLevel, String[]> ppt_to_value_reps = new HashMap<>(); 213 214 // For debugging purposes: printing out a modified trace file with 215 // changed modbits. 216 private static boolean to_write_nonce = false; 217 private static final String NONCE_HEADER = "this_invocation_nonce"; 218 private static String nonce_value = "no nonce (yet)"; 219 220 // (This implementation as a public static variable is a bit unclean.) 221 // Number of ignored declarations. 222 public static int omitted_declarations = 0; 223 224 // Logging Categories 225 226 /** 227 * If true, then print the variable name each time the variable's value is first 228 * missing/nonsensical. 229 */ 230 public static boolean debug_missing = false; 231 232 /** Debug tracer for reading. */ 233 public static final Logger debugRead = Logger.getLogger("daikon.FileIO.read"); 234 235 /** Debug tracer for printing. */ 236 public static final Logger debugPrint = Logger.getLogger("daikon.FileIO.printDtrace"); 237 238 /** Debug tracer for printing variable values. */ 239 public static final Logger debugVars = Logger.getLogger("daikon.FileIO.vars"); 240 241 // public static final SimpleLog debug_decl = new SimpleLog(false); 242 243 /** Parents in the ppt/variable hierarchy for a particular program point. */ 244 public static final class ParentRelation implements java.io.Serializable { 245 static final long serialVersionUID = 20060622L; 246 public PptRelationType rel_type; 247 public @Interned String parent_ppt_name; 248 public int id; 249 250 public ParentRelation(PptRelationType rel_type, @Interned String parent_ppt_name, int id) { 251 this.rel_type = rel_type; 252 this.parent_ppt_name = parent_ppt_name; 253 this.id = id; 254 } 255 256 @SideEffectFree 257 @Override 258 public String toString(@GuardSatisfied ParentRelation this) { 259 return parent_ppt_name + "[" + id + "] " + rel_type; 260 } 261 262 /** 263 * Intern the ppt name. 264 * 265 * @param in the input stream from which to read the object 266 * @throws IOException if there is a problem reading the stream 267 * @throws ClassNotFoundException if a class cannot be loaded 268 */ 269 private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { 270 in.defaultReadObject(); 271 if (parent_ppt_name != null) { 272 parent_ppt_name = parent_ppt_name.intern(); 273 } 274 } 275 } 276 277 // Utilities 278 @EnsuresNonNullIf(result = true, expression = "#1") 279 @Pure 280 public static final boolean isComment(@Nullable String s) { 281 return s != null && (s.startsWith("//") || s.startsWith("#")); 282 } 283 284 /** 285 * Returns true if the next line is a comment. 286 * 287 * @param reader the reader whose next line to check 288 * @return true if the next line is a comment 289 */ 290 // Nullness-checking of read_data_trace_record(ParseState) works even 291 // without these two lines, since StringJoiner accepts null values. 292 @SuppressWarnings( 293 "nullness:contracts.conditional.postcondition" // readLine() assertion is ensured by call to 294 // reset() 295 ) 296 @EnsuresNonNullIf(result = true, expression = "#1.readLine()") 297 public static final boolean nextLineIsComment(BufferedReader reader) { 298 boolean result = false; 299 try { 300 reader.mark(10000); 301 String nextline = reader.readLine(); 302 result = isComment(nextline); 303 } catch (IOException e) { 304 result = false; 305 } 306 try { 307 reader.reset(); 308 } catch (IOException e) { 309 throw new UncheckedIOException(e); 310 } 311 return result; 312 } 313 314 /////////////////////////////////////////////////////////////////////////// 315 /// Declaration files 316 /// 317 318 /** 319 * Returns a new PptMap containing declarations read from the files listed in the argument; 320 * connection information (controlling variables and entry ppts) is set correctly upon return. 321 * 322 * @param files files to be read (java.io.File) 323 * @return a new PptMap containing declarations read from the files listed in the argument 324 */ 325 public static PptMap read_declaration_files(Collection<File> files) throws IOException { 326 PptMap all_ppts = new PptMap(); 327 // Read all decls, creating PptTopLevels and VarInfos 328 for (File file : files) { 329 Daikon.progress = "Reading " + file; 330 if (!Daikon.dkconfig_quiet) { 331 System.out.print("."); // show progress 332 } 333 read_declaration_file(file, all_ppts); 334 } 335 return all_ppts; 336 } 337 338 /** Read one decls file; add it to all_ppts. */ 339 public static void read_declaration_file(File filename, PptMap all_ppts) throws IOException { 340 if (Daikon.using_DaikonSimple) { 341 Processor processor = new DaikonSimple.SimpleProcessor(); 342 read_data_trace_file(filename.toString(), all_ppts, processor, true, false); 343 } else { 344 Processor processor = new Processor(); 345 read_data_trace_file(filename.toString(), all_ppts, processor, true, true); 346 } 347 } 348 349 // Read a declaration in the Version 2 format. For Version 1, see 350 // read_declaration. 351 /** 352 * Reads one ppt declaration. The next line should be the ppt record. After completion, the file 353 * pointer will be pointing at the next record (ie, the blank line at the end of the ppt 354 * declaration will have been read in). Returns null if the ppt is excluded/omitted from this 355 * execution of Daikon. 356 */ 357 private static @Nullable PptTopLevel read_ppt_decl(ParseState state, String top_line) 358 throws IOException { 359 360 // process the ppt record 361 String line = top_line; 362 Scanner scanner = new Scanner(line); 363 @Interned String record_name = need(state, scanner, "'ppt'"); 364 if (record_name != "ppt") { // interned 365 decl_error(state, "found '%s' where 'ppt' expected", record_name); 366 } 367 String ppt_name = need(state, scanner, "ppt name"); 368 ppt_name = user_mod_ppt_name(ppt_name); 369 370 /** Information that will populate the new program point. */ 371 Map<String, VarDefinition> varmap = new LinkedHashMap<>(); 372 /** The VarDefinition we are in the middle of reading, or null if we are not. */ 373 VarDefinition vardef = null; 374 List<ParentRelation> ppt_parents = new ArrayList<>(); 375 EnumSet<PptFlags> ppt_flags = EnumSet.noneOf(PptFlags.class); 376 PptType ppt_type = PptType.POINT; 377 378 try { 379 // Read the records that define this program point 380 while ((line = state.reader.readLine()) != null) { 381 // debug_decl.log("read line %s%n", line); 382 line = line.trim(); 383 if (line.length() == 0) { 384 break; 385 } 386 387 scanner = new Scanner(line); 388 @Interned String record = scanner.next().intern(); 389 if (vardef == null) { 390 if (record == "parent") { // interned 391 ppt_parents.add(parse_ppt_parent(state, scanner)); 392 } else if (record == "flags") { // interned 393 parse_ppt_flags(state, scanner, ppt_flags); 394 } else if (record == "variable") { // interned 395 vardef = new VarDefinition(state, scanner); 396 // There is no need to check "varmap.containsKey(vardef.name)" 397 // because this is the first variable. 398 assert varmap.isEmpty(); 399 if (var_included(vardef.name)) varmap.put(vardef.name, vardef); 400 } else if (record == "ppt-type") { // interned 401 ppt_type = parse_ppt_type(state, scanner); 402 } else { 403 decl_error(state, "record '%s' found where %s expected", record, "'parent', 'flags'"); 404 } 405 } else { // there must be a current variable 406 if (record == "var-kind") { // interned 407 vardef.parse_var_kind(scanner); 408 } else if (record == "enclosing-var") { // interned 409 vardef.parse_enclosing_var_name(scanner); 410 } else if (record == "reference-type") { // interned 411 vardef.parse_reference_type(scanner); 412 } else if (record == "array") { // interned 413 vardef.parse_array(scanner); 414 } else if (record == "function-args") { // interned 415 vardef.parse_function_args(scanner); 416 } else if (record == "rep-type") { // interned 417 vardef.parse_rep_type(scanner); 418 } else if (record == "dec-type") { // interned 419 vardef.parse_dec_type(scanner); 420 } else if (record == "flags") { // interned 421 vardef.parse_flags(scanner); 422 } else if (record == "lang-flags") { // interned 423 vardef.parse_lang_flags(scanner); 424 } else if (record == "parent") { // interned 425 vardef.parse_parent(scanner, ppt_parents); 426 } else if (record == "comparability") { // interned 427 vardef.parse_comparability(scanner); 428 } else if (record == "constant") { // interned 429 vardef.parse_constant(scanner); 430 } else if (record == "variable") { // interned 431 try { 432 vardef.checkRep(); // make sure the previous variable is ok 433 } catch (AssertionError e) { 434 decl_error(state, e); 435 } 436 vardef = new VarDefinition(state, scanner); 437 if (varmap.containsKey(vardef.name)) { 438 decl_error(state, "var %s declared twice", vardef.name); 439 } 440 if (var_included(vardef.name)) varmap.put(vardef.name, vardef); 441 } else if (record == "min-value") { // interned 442 vardef.parse_min_value(scanner); 443 } else if (record == "max-value") { // interned 444 vardef.parse_max_value(scanner); 445 } else if (record == "min-length") { // interned 446 vardef.parse_min_length(scanner); 447 } else if (record == "max-length") { // interned 448 vardef.parse_max_length(scanner); 449 } else if (record == "valid-values") { // interned 450 vardef.parse_valid_values(scanner); 451 } else { 452 decl_error(state, "Unexpected variable item '%s' found", record); 453 } 454 } 455 } 456 } catch (Daikon.ParseError pe) { 457 decl_error(state, "%s", pe.getMessage()); 458 throw new Error(); // this can't happen 459 } 460 if (vardef != null) { 461 try { 462 vardef.checkRep(); 463 } catch (AssertionError e) { 464 decl_error(state, e); 465 } 466 } 467 468 // If we are excluding this ppt, just read the data and throw it away 469 if (!ppt_included(ppt_name)) { 470 omitted_declarations++; 471 return null; 472 } 473 474 // Build the var infos from the var definitions. 475 List<VarInfo> vi_list = new ArrayList<>(varmap.size()); 476 for (VarDefinition vd : varmap.values()) { 477 @SuppressWarnings("interning") // about to be used in a new program point 478 @Interned VarInfo vi = new VarInfo(vd); 479 vi_list.add(vi); 480 } 481 VarInfo[] vi_array = vi_list.toArray(new VarInfo[vi_list.size()]); 482 483 // Check to see if the program point is new 484 if (state.all_ppts.containsName(ppt_name)) { 485 @NonNull PptTopLevel existing_ppt = state.all_ppts.get(ppt_name); 486 assert existing_ppt != null : "state.all_ppts.containsName(" + ppt_name + ")"; 487 if (state.ppts_may_be_new) { 488 check_decl_match(state, existing_ppt, vi_array); 489 } else { // ppts are already in the map 490 if (VarInfo.assertionsEnabled()) { 491 for (VarInfo vi : vi_array) { 492 vi.checkRep(); 493 } 494 } 495 return existing_ppt; 496 } 497 } 498 499 // Build the program point 500 PptTopLevel newppt = new PptTopLevel(ppt_name, ppt_type, ppt_parents, ppt_flags, vi_array); 501 502 return newppt; 503 } 504 505 /** Parses a ppt parent hierarchy record and returns it. */ 506 private static ParentRelation parse_ppt_parent(ParseState state, Scanner scanner) { 507 508 PptRelationType rel_type = 509 parse_enum_val(state, scanner, PptRelationType.class, "relation type"); 510 String parent_ppt_name = need(state, scanner, "ppt name"); 511 int id = Integer.parseInt(need(state, scanner, "relation id")); 512 ParentRelation pr = new ParentRelation(rel_type, parent_ppt_name, id); 513 514 need_eol(state, scanner); 515 return pr; 516 } 517 518 /** Parses a program point flag record. Adds any specified flags to to flags. */ 519 private static void parse_ppt_flags(ParseState state, Scanner scanner, EnumSet<PptFlags> flags) { 520 521 flags.add(parse_enum_val(state, scanner, PptFlags.class, "ppt flags")); 522 while (scanner.hasNext()) { 523 flags.add(parse_enum_val(state, scanner, PptFlags.class, "ppt flags")); 524 } 525 } 526 527 /** Parses a ppt-type record and returns the type. */ 528 private static PptType parse_ppt_type(ParseState state, Scanner scanner) { 529 530 PptType ppt_type = parse_enum_val(state, scanner, PptType.class, "ppt type"); 531 need_eol(state, scanner); 532 return ppt_type; 533 } 534 535 // Read a declaration in the Version 1 format. For version 2, see 536 // read_ppt_decl. 537 // The "DECLARE" line has already been read. 538 private static @Nullable PptTopLevel read_declaration(ParseState state) throws IOException { 539 540 // We have just read the "DECLARE" line. 541 String ppt_name = state.reader.readLine(); 542 if (ppt_name == null) { 543 throw new Daikon.UserError( 544 "File ends with \"DECLARE\" with no following program point name", state); 545 } 546 ppt_name = user_mod_ppt_name(ppt_name); 547 ppt_name = ppt_name.intern(); 548 VarInfo[] vi_array = read_VarInfos(state, ppt_name); 549 550 // System.out.printf("Ppt %s with %d variables%n", ppt_name, 551 // vi_array.length); 552 553 // This program point name has already been encountered. 554 if (state.all_ppts.containsName(ppt_name)) { 555 @NonNull PptTopLevel existing_ppt = state.all_ppts.get(ppt_name); 556 assert existing_ppt != null : "state.all_ppts.containsName(" + ppt_name + ")"; 557 if (state.ppts_may_be_new) { 558 check_decl_match(state, existing_ppt, vi_array); 559 } else { // ppts are already in the map 560 return existing_ppt; 561 } 562 } 563 564 // If we are excluding this ppt, just throw it away 565 if (!ppt_included(ppt_name)) { 566 omitted_declarations++; 567 return null; 568 } 569 570 // taking care of visibility information 571 // the information is needed in the variable hierarchy because private methods 572 // should not be linked under the object program point 573 // the ppt name is truncated before putting it in the pptMap because the visibility 574 // information is only present in the decls file and not the dtrace file 575 576 // if (ppt_name.startsWith("public")) { 577 // int position = ppt_name.indexOf("public"); 578 // ppt_name = ppt_name.substring(7); 579 // PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 580 // newppt.ppt_name.setVisibility("public"); 581 // return newppt; 582 // } 583 // if (ppt_name.startsWith("private")) { 584 // int position = ppt_name.indexOf("private"); 585 // ppt_name = ppt_name.substring(8); 586 // PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 587 // newppt.ppt_name.setVisibility("private"); 588 // return newppt; 589 // } 590 // if (ppt_name.startsWith("protected")) { 591 // int position = ppt_name.indexOf("protected"); 592 // ppt_name = ppt_name.substring(10); 593 // PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 594 // newppt.ppt_name.setVisibility("protected"); 595 // return newppt; 596 // } 597 598 // TODO: add a new config variable to turn this accessibility flag processing on? 599 PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 600 // newppt.ppt_name.setVisibility("package-protected"); 601 return newppt; 602 // return new PptTopLevel(ppt_name, vi_array); 603 } 604 605 private static VarInfo[] read_VarInfos(ParseState state, String ppt_name) throws IOException { 606 607 // The var_infos that will populate the new program point 608 List<VarInfo> var_infos = new ArrayList<>(); 609 610 // Each iteration reads a variable name, type, and comparability. 611 // Possibly abstract this out into a separate function?? 612 VarInfo vi; 613 while ((vi = read_VarInfo(state, ppt_name)) != null) { 614 for (VarInfo vi2 : var_infos) { 615 if (vi.name() == vi2.name()) { 616 throw new Daikon.UserError("Duplicate variable name " + vi.name(), state); 617 } 618 } 619 // Can't do this test in read_VarInfo, it seems, because of the test 620 // against null above. 621 if (!var_included(vi.name())) { 622 continue; 623 } 624 var_infos.add(vi); 625 } 626 627 VarInfo[] result = var_infos.toArray(new VarInfo[var_infos.size()]); 628 return result; 629 } 630 631 // So that warning message below is only printed once 632 private static boolean seen_string_rep_type = false; 633 634 /** 635 * Read a variable name, type, and comparability; construct a VarInfo. Return null after reading 636 * the last variable in this program point declaration. 637 * 638 * <p>The resulting VarInfo does not have its ppt field set; the client should arrange to do so. 639 * 640 * @param state the parse state 641 * @param ppt_name the name of the variable's program point; used only for diagnostic messages 642 * @return a new VarInfo read from {@code state.reader} 643 * @throws IOException if there is trouble reading the file 644 */ 645 private static @Nullable VarInfo read_VarInfo(ParseState state, String ppt_name) 646 throws IOException { 647 LineNumberReader file = state.reader; 648 int varcomp_format = state.varcomp_format; 649 String filename = state.filename; 650 651 String line = file.readLine(); 652 if ((line == null) || line.equals("")) { 653 return null; 654 } 655 String varname = line; 656 String proglang_type_string_and_aux = file.readLine(); 657 String file_rep_type_string = file.readLine(); 658 String comparability_string = file.readLine(); 659 if ( // (varname == null) || // already returned null if varname==null 660 (proglang_type_string_and_aux == null) 661 || (file_rep_type_string == null) 662 || (comparability_string == null)) 663 throw new Daikon.UserError( 664 "End of file " 665 + filename 666 + " while reading variable " 667 + varname 668 + " in declaration of program point " 669 + ppt_name); 670 int equals_index = file_rep_type_string.indexOf(" = "); 671 String static_constant_value_string = null; 672 @Interned Object static_constant_value = null; 673 boolean is_static_constant = false; 674 if (equals_index != -1) { 675 is_static_constant = true; 676 static_constant_value_string = file_rep_type_string.substring(equals_index + 3); 677 file_rep_type_string = file_rep_type_string.substring(0, equals_index); 678 } 679 // XXX temporary, for compatibility with older .dtrace files. 12/20/2001 680 if ("String".equals(file_rep_type_string)) { 681 file_rep_type_string = "java.lang.String"; 682 if (!seen_string_rep_type) { 683 seen_string_rep_type = true; 684 System.err.println( 685 "Warning: Malformed trace file. Representation type 'String' should be " 686 + "'java.lang.String' instead on line " 687 + (file.getLineNumber() - 1) 688 + " of " 689 + filename); 690 } 691 } 692 // This is for people who were confused by the above temporary 693 // workaround when it didn't have a warning. But this has never 694 // worked, so it's fatal. 695 else if ("String[]".equals(file_rep_type_string)) { 696 throw new Daikon.UserError( 697 "Representation type 'String[]' should be " 698 + "'java.lang.String[]' instead for variable " 699 + varname, 700 file, 701 filename); 702 } 703 /// XXX 704 705 int hash_position = proglang_type_string_and_aux.indexOf('#'); 706 String aux_string = ""; 707 if (hash_position == -1) { 708 hash_position = proglang_type_string_and_aux.length(); 709 } else { 710 aux_string = 711 proglang_type_string_and_aux.substring( 712 hash_position + 1, proglang_type_string_and_aux.length()); 713 } 714 715 String proglang_type_string = proglang_type_string_and_aux.substring(0, hash_position).trim(); 716 717 ProglangType prog_type; 718 ProglangType file_rep_type; 719 ProglangType rep_type; 720 VarInfoAux aux; 721 try { 722 prog_type = ProglangType.parse(proglang_type_string); 723 file_rep_type = ProglangType.rep_parse(file_rep_type_string); 724 rep_type = file_rep_type.fileTypeToRepType(); 725 aux = VarInfoAux.parse(aux_string); 726 } catch (IOException e) { 727 throw new Daikon.UserError(e, file, filename); 728 } 729 730 if (static_constant_value_string != null) { 731 static_constant_value = rep_type.parse_value(static_constant_value_string, file, filename); 732 // Why can't the value be null? 733 assert static_constant_value != null; 734 } 735 VarComparability comparability = null; 736 try { 737 comparability = VarComparability.parse(varcomp_format, comparability_string, prog_type); 738 } catch (Exception e) { 739 throw new Daikon.UserError( 740 String.format( 741 "Error parsing comparability (%s) at line %d in file %s", 742 e, file.getLineNumber(), filename)); 743 } 744 if (!VarInfo.legalFileRepType(file_rep_type)) { 745 throw new Daikon.UserError( 746 "Unsupported representation type " 747 + file_rep_type.format() 748 + " (parsed as " 749 + rep_type 750 + ")" 751 + " for variable " 752 + varname, 753 file, 754 filename); 755 } 756 if (!VarInfo.legalRepType(rep_type)) { 757 throw new Daikon.UserError( 758 "Unsupported (converted) representation type " 759 + file_rep_type.format() 760 + " for variable " 761 + varname, 762 file, 763 filename); 764 } 765 // COMPARABILITY TEST 766 if (!(comparability.alwaysComparable() 767 || ((VarComparabilityImplicit) comparability).dimensions == file_rep_type.dimensions())) { 768 System.err.println(); 769 throw new Daikon.UserError( 770 "Rep type " 771 + file_rep_type.format() 772 + " has " 773 + file_rep_type.dimensions() 774 + " dimensions," 775 + " but comparability " 776 + comparability 777 + " has " 778 + ((VarComparabilityImplicit) comparability).dimensions 779 + " dimensions," 780 + " for variable " 781 + varname, 782 file, 783 filename); 784 } 785 786 @SuppressWarnings("interning") 787 @Interned VarInfo result = 788 new VarInfo( 789 varname, 790 prog_type, 791 file_rep_type, 792 comparability, 793 is_static_constant, 794 static_constant_value, 795 aux); 796 return result; 797 } 798 799 @RequiresNonNull("FileIO.new_decl_format") 800 private static int read_var_comparability(ParseState state, String line) throws IOException { 801 802 // System.out.printf("read_var_comparability, line = '%s' %b%n", line, 803 // new_decl_format); 804 String comp_str; 805 if (new_decl_format) { 806 Scanner scanner = new Scanner(line); 807 scanner.next(); 808 comp_str = need(state, scanner, "comparability"); 809 need_eol(state, scanner); 810 } else { // old format 811 comp_str = state.reader.readLine(); 812 if (comp_str == null) { 813 throw new Daikon.UserError("Found end of file, expected comparability", state); 814 } 815 } 816 817 if (comp_str.equals("none")) { 818 return VarComparability.NONE; 819 } else if (comp_str.equals("implicit")) { 820 return VarComparability.IMPLICIT; 821 } else { 822 throw new Daikon.UserError("Unrecognized VarComparability '" + comp_str + "'", state); 823 } 824 } 825 826 private static @Interned String read_input_language(ParseState state, String line) 827 throws IOException { 828 829 Scanner scanner = new Scanner(line); 830 scanner.next(); 831 @Interned String input_lang = need(state, scanner, "input language"); 832 need_eol(state, scanner); 833 return input_lang; 834 } 835 836 @EnsuresNonNull("FileIO.new_decl_format") 837 private static void read_decl_version(ParseState state, String line) throws IOException { 838 Scanner scanner = new Scanner(line); 839 scanner.next(); 840 @Interned String version = need(state, scanner, "declaration version number"); 841 need_eol(state, scanner); 842 boolean new_df; 843 if (version == "2.0") // interned 844 new_df = true; 845 else if (version == "1.0") // interned 846 new_df = false; 847 else { 848 decl_error(state, "'%s' found where 1.0 or 2.0 expected", version); 849 throw new Error("Can't get here"); // help out definite assignment analysis 850 } 851 852 // Make sure that if a format was specified previously, it is the same 853 if ((new_decl_format != null) && (new_df != new_decl_format.booleanValue())) { 854 decl_error(state, "decl format '%s' does not match previous setting", version); 855 } 856 857 // System.out.println("setting new_decl_format = " + new_df); 858 new_decl_format = Boolean.valueOf(new_df); 859 } 860 861 // Each line following is the name (in JVM form) of a class that 862 // implements java.util.List. All those lines (including interspersed 863 // comments) are returned. 864 private static String read_list_implementors(LineNumberReader reader) throws IOException { 865 StringJoiner result = new StringJoiner(lineSep); 866 for (; ; ) { 867 String line = reader.readLine(); 868 if (line == null || line.equals("")) { 869 break; 870 } 871 result.add(line); 872 if (isComment(line)) { 873 continue; 874 } 875 ProglangType.list_implementors.add(line.intern()); 876 } 877 return result.toString(); 878 } 879 880 /////////////////////////////////////////////////////////////////////////// 881 /// invocation tracking for dtrace files entry/exit grouping 882 /// 883 884 static final class Invocation implements Comparable<Invocation> { 885 PptTopLevel ppt; // used in printing and in suppressing duplicates 886 // Rather than a valuetuple, place its elements here. 887 @Nullable Object[] vals; 888 int[] mods; 889 890 static Object canonical_hashcode = new Object(); 891 892 Invocation(PptTopLevel ppt, @Nullable Object[] vals, int[] mods) { 893 this.ppt = ppt; 894 this.vals = vals; 895 this.mods = mods; 896 } 897 898 /** 899 * Return a string representation of this. The Invocation is formatted on two lines, indented by 900 * two spaces. The receiver Invocation may be canonicalized or not. 901 * 902 * @return a string representation of this 903 */ 904 String format(@GuardSatisfied @UnknownSignedness Invocation this) { 905 return format(true); 906 } 907 908 /** 909 * Return a string representation of this. The Invocation is formatted on two lines, indented by 910 * two spaces. The receiver Invocation may be canonicalized or not. 911 * 912 * @param show_values if true, show values; otherwise, return just the Ppt name 913 * @return a string representation of this 914 */ 915 String format(@GuardSatisfied @UnknownSignedness Invocation this, boolean show_values) { 916 if (!show_values) { 917 return " " + ppt.ppt_name.getNameWithoutPoint(); 918 } 919 920 StringWriter sw = new StringWriter(); 921 PrintWriter pw = new PrintWriter(sw); 922 923 pw.println(" " + ppt.ppt_name.getNameWithoutPoint()); 924 pw.print(" "); 925 926 // [adonovan] is this sound? Let me know if not (sorry). 927 // assert ppt.var_infos.length == vals.length; 928 929 for (int j = 0; j < vals.length; j++) { 930 if (j != 0) { 931 pw.print(", "); 932 } 933 934 pw.print(ppt.var_infos[j].name() + "="); 935 936 Object val = vals[j]; 937 if (canonical_hashcode.equals( 938 val)) // succeeds only for canonicalized Invocations. Can be an == test, but there is 939 // little point. val can be null, so it cannot be the receiver. 940 pw.print("<hashcode>"); 941 else if (val instanceof int[]) pw.print(Arrays.toString((int[]) val)); 942 else if (val instanceof String) pw.print(StringsPlume.escapeNonASCII((String) val)); 943 else { 944 pw.print(val); 945 } 946 } 947 pw.println(); 948 949 return sw.toString(); 950 } 951 952 /** Change uses of hashcodes to canonical_hashcode. */ 953 public @Interned Invocation canonicalize() { 954 @Nullable Object[] new_vals = new @Nullable Object[vals.length]; 955 System.arraycopy(vals, 0, new_vals, 0, vals.length); 956 VarInfo[] vis = ppt.var_infos; 957 // Warning: abstraction violation! 958 for (VarInfo vi : vis) { 959 if ((vi.value_index != -1) && (vi.file_rep_type == ProglangType.HASHCODE)) { 960 new_vals[vi.value_index] = canonical_hashcode; 961 } 962 } 963 @SuppressWarnings("interning:cast.unsafe.constructor.invocation") 964 @Interned Invocation result = new @Interned Invocation(ppt, new_vals, mods); 965 return result; 966 } 967 968 // Return true if the invocations print the same 969 @EnsuresNonNullIf(result = true, expression = "#1") 970 @Pure 971 @Override 972 public boolean equals(@GuardSatisfied Invocation this, @GuardSatisfied @Nullable Object other) { 973 if (other instanceof FileIO.Invocation) { 974 return this.format().equals(((FileIO.Invocation) other).format()); 975 } else { 976 return false; 977 } 978 } 979 980 @Pure 981 @Override 982 public int compareTo(@GuardSatisfied Invocation this, Invocation other) { 983 return ppt.name().compareTo(other.ppt.name()); 984 } 985 986 @Pure 987 @Override 988 public int hashCode(@GuardSatisfied @UnknownSignedness Invocation this) { 989 return this.format().hashCode(); 990 } 991 } 992 993 // I could save some Object overhead by using two parallel stacks 994 // instead of Invocation objects; but that's not worth it. 995 996 // Map key is a (global, not per-procedure) nonce. 997 // The nonce indicates which returns are associated with which entries. 998 static HashMap<Integer, Invocation> call_hashmap = new HashMap<>(); 999 // call_stack is for procedures without nonces. 1000 static Deque<Invocation> call_stack = new ArrayDeque<Invocation>(); 1001 1002 /** 1003 * Reads data from {@code .dtrace} files. For each record in the files, calls the appropriate 1004 * callback in the processor. 1005 * 1006 * @see #read_data_trace_files(Collection,PptMap,Processor,boolean) 1007 * @see #read_data_trace_file(String,PptMap,Processor,boolean,boolean) 1008 */ 1009 public static void read_data_trace_files(Collection<String> files, PptMap all_ppts) 1010 throws IOException { 1011 1012 Processor processor = new Processor(); 1013 read_data_trace_files(files, all_ppts, processor, true); 1014 } 1015 1016 /** 1017 * Reads data from {@code .dtrace} files. Calls {@link 1018 * #read_data_trace_file(String,PptMap,Processor,boolean,boolean)} for each element of filenames. 1019 * 1020 * @param ppts_may_be_new true if declarations of ppts read from the data trace file are new (and 1021 * thus are not in all_ppts). false if the ppts may already be there. 1022 * @see #read_data_trace_file(String,PptMap,Processor,boolean,boolean) 1023 */ 1024 public static void read_data_trace_files( 1025 Collection<String> files, PptMap all_ppts, Processor processor, boolean ppts_may_be_new) 1026 throws IOException { 1027 1028 for (String filename : files) { 1029 // System.out.printf("processing filename %s%n", filename); 1030 try { 1031 read_data_trace_file(filename, all_ppts, processor, false, ppts_may_be_new); 1032 } catch (Daikon.NormalTermination e) { 1033 throw e; 1034 } catch (Throwable e) { 1035 if (dkconfig_continue_after_file_exception) { 1036 System.out.println(); 1037 System.out.println( 1038 "WARNING: Error while processing trace file; remaining records ignored."); 1039 System.out.print("Ignored backtrace:"); 1040 e.printStackTrace(System.out); 1041 System.out.println(); 1042 } else { 1043 throw e; 1044 } 1045 } 1046 } 1047 if (Daikon.server_dir != null) { 1048 // Yoav: server mode 1049 while (true) { 1050 @SuppressWarnings( 1051 "nullness") // server_dir is a directory; this was checked when the variable was set 1052 String @NonNull [] dir_files = Daikon.server_dir.list(); 1053 Arrays.sort(dir_files); 1054 boolean hasEnd = false; 1055 for (String f : dir_files) { 1056 if (f.endsWith(".end")) { 1057 hasEnd = true; 1058 } 1059 if (f.endsWith(".end") || f.endsWith(".start")) { 1060 continue; 1061 } 1062 if (files.contains(f)) { 1063 continue; 1064 } 1065 files.add(f); 1066 System.out.println("Reading " + f); 1067 read_data_trace_file( 1068 new File(Daikon.server_dir, f).toString(), 1069 all_ppts, 1070 processor, 1071 false, 1072 ppts_may_be_new); 1073 } 1074 if (hasEnd) { 1075 break; 1076 } 1077 try { 1078 Thread.sleep(1000); 1079 } catch (java.lang.InterruptedException e) { 1080 // It's not a problem if the sleep is interrupted. 1081 } 1082 } 1083 } 1084 1085 process_unmatched_procedure_entries(); 1086 1087 warn_if_hierarchy_mismatch(all_ppts); 1088 } 1089 1090 // Determine if dataflow hierarchy should have been used, and print 1091 // warning if this does not match Daikon.use_dataflow_hierarchy. 1092 // Dataflow hierarchy should be used only when all program points 1093 // correspond to points normally found in traces from a 1094 // programming languages. 1095 private static void warn_if_hierarchy_mismatch(PptMap all_ppts) { 1096 1097 boolean some_program_points = false; 1098 boolean all_program_points = true; 1099 1100 // Go through each top level ppt, and make all_program_points 1101 // false if at least one of them is not a program point normally 1102 // found in traces from programming languages. 1103 for (PptTopLevel ppt_top_level : all_ppts.ppt_all_iterable()) { 1104 boolean is_program_point = 1105 (ppt_top_level.ppt_name.isExitPoint() 1106 || ppt_top_level.ppt_name.isEnterPoint() 1107 || ppt_top_level.ppt_name.isThrowsPoint() 1108 || ppt_top_level.ppt_name.isObjectInstanceSynthetic() 1109 || ppt_top_level.ppt_name.isClassStaticSynthetic() 1110 || ppt_top_level.ppt_name.isGlobalPoint()); 1111 1112 all_program_points = all_program_points && is_program_point; 1113 some_program_points = some_program_points || is_program_point; 1114 } 1115 1116 // If all program points correspond to a programming language, 1117 // but the dataflow hierarchy has been turned off, then 1118 // suggest not using the --nohierarchy flag. 1119 // if (all_program_points && (!Daikon.use_dataflow_hierarchy)) { 1120 // System.out.println("Warning: data trace appears to be over" + 1121 // " a program execution, but dataflow" + 1122 // " hierarchy has been turned off," + 1123 // " consider running Daikon without the" + 1124 // " --nohierarchy flag"); 1125 // } 1126 1127 // if some of the program points do not correspond to a 1128 // points from a programming language, and the dataflow 1129 // hierarchy is being used, suggest using the --nohierarchy flag. 1130 if (Daikon.use_dataflow_hierarchy && !all_program_points && some_program_points) { 1131 System.out.println( 1132 "Warning: Daikon is using a dataflow" 1133 + " hierarchy analysis on a data trace" 1134 + " that does not appear to be over a" 1135 + " program execution. Consider running" 1136 + " Daikon with the --nohierarchy flag."); 1137 } 1138 } 1139 1140 /** 1141 * Connect to Chicory. 1142 * 1143 * @return the stream that is connected to Chicory 1144 */ 1145 private static @Owning InputStream connectToChicory() { 1146 1147 // bind to any free port 1148 try (ServerSocket daikonServer = new ServerSocket(0)) { 1149 1150 // tell Chicory what port we have! 1151 System.out.println("DaikonChicoryOnlinePort=" + daikonServer.getLocalPort()); 1152 1153 daikonServer.setReceiveBufferSize(64000); 1154 1155 Socket chicSocket; 1156 try { 1157 daikonServer.setSoTimeout(5000); 1158 1159 // System.out.println("waiting for chicory connection on port " + 1160 // daikonServer.getLocalPort()); 1161 chicSocket = daikonServer.accept(); 1162 } catch (IOException e) { 1163 throw new RuntimeException("Unable to connect to Chicory", e); 1164 } 1165 1166 try { 1167 return chicSocket.getInputStream(); 1168 } catch (IOException e) { 1169 throw new RuntimeException("Unable to get Chicory's input stream", e); 1170 } 1171 } catch (IOException e) { 1172 throw new RuntimeException("Unable to create server", e); 1173 } 1174 } 1175 1176 /** 1177 * A Processor is used to read a dtrace file. A Processor defines callbacks for each record type 1178 * in a dtrace file. As each record is read from a dtrace file, the corresponding callback is 1179 * called. 1180 * 1181 * <p>to use a Processor, pass it to {@link #read_data_trace_files(Collection, PptMap, 1182 * FileIO.Processor, boolean)}. {@code read_data_trace_files} will call {@link 1183 * #process_sample(PptMap,PptTopLevel,ValueTuple,Integer)} once for every sample in the dtrace 1184 * file, and will call other callbacks for other records in the dtrace file. 1185 * 1186 * <p>For an example of how to create and use a Processor, see {@link daikon.tools.ReadTrace}. 1187 * 1188 * @see #read_data_trace_files(Collection, PptMap, FileIO.Processor, boolean) 1189 * @see daikon.tools.ReadTrace 1190 */ 1191 public static class Processor { 1192 /** 1193 * Process a data sample record. This default implementation calls {@link 1194 * FileIO#process_sample(PptMap, PptTopLevel, ValueTuple, Integer)}. 1195 * 1196 * @see FileIO#process_sample(PptMap, PptTopLevel, ValueTuple, Integer) 1197 */ 1198 @RequiresNonNull("FileIO.data_trace_state") 1199 public void process_sample( 1200 PptMap all_ppts, PptTopLevel ppt, ValueTuple vt, @Nullable Integer nonce) { 1201 FileIO.process_sample(all_ppts, ppt, vt, nonce); 1202 } 1203 1204 /** Process a program point declaration record. */ 1205 public void process_decl(PptMap all_ppts, PptTopLevel ppt) {} 1206 1207 /** Process a ppt decl format record. */ 1208 public void process_decl_version(String format) {} 1209 1210 /** Process a VarComparability declaration. */ 1211 public void process_comparability(String comparability) {} 1212 1213 /** Process a ListImplementors declaration. */ 1214 public void process_list_implementors(String implementors) {} 1215 1216 /** Process an input-language declaration. */ 1217 public void process_input_language(String language) {} 1218 1219 /** Process a null record (haven't read anything yet). */ 1220 public void process_null() {} 1221 1222 /** Process a comment. */ 1223 public void process_comment(String comment) {} 1224 1225 /** Process indication of end of file. */ 1226 public void process_eof() {} 1227 1228 /** Process indication of exceeding file size limit. */ 1229 public void process_truncated() {} 1230 1231 /** Process continuable error. */ 1232 public void process_error() {} 1233 } 1234 1235 /** 1236 * Total number of samples passed to process_sample(). Not part of ParseState because it's global 1237 * over all files processed by Daikon. 1238 */ 1239 public static int samples_processed = 0; 1240 1241 /** The type of the record that was most recently read. */ 1242 public enum RecordType { 1243 SAMPLE, // got a sample 1244 1245 DECL, // got a ppt decl 1246 DECL_VERSION, // got an indication of the ppt decl format 1247 COMPARABILITY, // got a VarComparability declaration 1248 LIST_IMPLEMENTORS, // got a ListImplementors declaration 1249 INPUT_LANGUAGE, // got an input-language declaration 1250 1251 NULL, // haven't read anything yet 1252 COMMENT, // got a comment 1253 EOF, // reached end of file 1254 TRUNCATED, // dkconfig_max_line_number reached (without error) 1255 ERROR, // continuable error; fatal errors thrown as exceptions 1256 }; 1257 1258 /** 1259 * ParseState indicates: 1260 * 1261 * <ol> 1262 * <li>Some global information about the state of the parser while reading a decl or dtrace 1263 * file. 1264 * <li>The record that was most recently read; thus, ParseState is essentially a discriminated 1265 * union whose tag is a RecordType. (TODO: These are poor names that should probably be 1266 * swapped!) ParseState is what is returned (actually, side-effected) by method 1267 * read_data_trace_record when it reads a record. 1268 * </ol> 1269 */ 1270 @UsesObjectEquals 1271 @MustCall("close") public static class ParseState implements Closeable { 1272 1273 // 1274 // This is the global information about the state of the parser. 1275 // 1276 1277 /** Name of input file. */ 1278 public String filename; 1279 1280 /** True if the current file is a declaration file. */ 1281 public boolean is_decl_file; 1282 1283 /** 1284 * True if ppts may be new. If a duplicate is seen, it must match a previous point exactly. If 1285 * false, the previous ppt is used without checking for a match. 1286 */ 1287 public boolean ppts_may_be_new; 1288 1289 /** All of the ppts seen so far. */ 1290 public PptMap all_ppts; 1291 1292 /** Input stream. */ 1293 public final @Owning LineNumberReader reader; 1294 1295 /** Total number of lines in the input file. */ 1296 public long total_lines; 1297 1298 /** Comparability format, either VarComparability.IMPLICIT or VarComparability.NONE. */ 1299 public int varcomp_format; 1300 1301 // 1302 // This is the discriminated-union part of the ParseState. 1303 // (Presumably this design was chosen for efficiency, to avoid creating 1304 // & garbage-collecting these values many times.) 1305 // 1306 1307 public RecordType rtype; 1308 1309 /** 1310 * Current ppt. Used when status=DECL or SAMPLE. Can be null if this declaration was skipped 1311 * because of --ppt-select-pattern or --ppt-omit-pattern. 1312 */ 1313 public @Nullable PptTopLevel ppt; 1314 1315 /** The current nonce. Used when status=SAMPLE. */ 1316 public @Nullable Integer nonce; 1317 1318 /** The current set of values. Used when status=SAMPLE. */ 1319 public @Nullable ValueTuple vt; 1320 1321 /** Miscellaneous text in the parsed item. */ 1322 public @Nullable Object payload; // used when status=COMMENT 1323 1324 /** 1325 * Start parsing the given file. 1326 * 1327 * @param raw_filename the file name supplied by the user; may be "-" or "+" 1328 * @param decl_file_p true if the file is a declaration file 1329 * @param ppts_may_be_new true if declarations of ppts read from the data trace file are new 1330 * (and thus are not in all_ppts). false if the ppts may already be there. 1331 * @param ppts the program points 1332 * @throws IOException if there is a problem reading or writing a file 1333 */ 1334 @SuppressWarnings("StaticAssignmentInConstructor") // for progress output 1335 public ParseState( 1336 String raw_filename, boolean decl_file_p, boolean ppts_may_be_new, PptMap ppts) 1337 throws IOException { 1338 // Pretty up raw_filename for use in messages 1339 if (raw_filename.equals("-")) { 1340 filename = "standard input"; 1341 } else if (raw_filename.equals("+")) { 1342 filename = "chicory socket"; 1343 } else { 1344 // Remove directory parts, to make it shorter 1345 filename = raw_filename; 1346 } 1347 1348 is_decl_file = decl_file_p; 1349 this.ppts_may_be_new = ppts_may_be_new; 1350 all_ppts = ppts; 1351 1352 boolean is_url = raw_filename.startsWith("file:") || raw_filename.startsWith("jar:"); 1353 1354 // Do we need to count the lines in the file? 1355 total_lines = 0; 1356 boolean count_lines = dkconfig_count_lines; 1357 if (is_decl_file) { 1358 count_lines = false; 1359 } else if (dkconfig_dtrace_line_count != 0) { 1360 total_lines = dkconfig_dtrace_line_count; 1361 count_lines = false; 1362 } else if (filename.equals("-")) { 1363 count_lines = false; 1364 } else if (is_url) { 1365 count_lines = false; 1366 } else if (Daikon.dkconfig_progress_delay == -1) { 1367 count_lines = false; 1368 } else if (new File(raw_filename).length() == 0) { 1369 // Either it's actually empty, or it's something like a pipe. 1370 count_lines = false; 1371 } 1372 1373 if (count_lines) { 1374 Daikon.progress = "Checking size of " + filename; 1375 total_lines = FilesPlume.countLines(raw_filename); 1376 } else { 1377 // System.out.printf("no count %b %d %s %d %d%n", is_decl_file, 1378 // dkconfig_dtrace_line_count, filename, 1379 // Daikon.dkconfig_progress_delay, (new File(raw_filename)).length()); 1380 } 1381 1382 // Open the reader stream 1383 if (raw_filename.equals("-")) { 1384 // "-" means read from the standard input stream 1385 Reader file_reader = new InputStreamReader(System.in, "ISO-8859-1"); 1386 reader = new LineNumberReader(file_reader); 1387 } else if (raw_filename.equals("+")) { // socket comm with Chicory 1388 InputStream chicoryInput = connectToChicory(); 1389 InputStreamReader chicReader = new InputStreamReader(chicoryInput, UTF_8); 1390 reader = new LineNumberReader(chicReader); 1391 } else if (is_url) { 1392 URL url = URI.create(raw_filename).toURL(); 1393 InputStream stream = null; // dummy initialization for compiler's definite assignment check 1394 try { 1395 stream = url.openStream(); 1396 InputStream gzip_stream = 1397 raw_filename.endsWith(".gz") ? new GZIPInputStream(stream) : stream; 1398 InputStreamReader isr = new InputStreamReader(gzip_stream, UTF_8); 1399 LineNumberReader lnr = new LineNumberReader(isr); 1400 reader = lnr; 1401 } catch (IOException e) { 1402 if (stream != null) { 1403 stream.close(); 1404 } 1405 throw e; 1406 } 1407 } else { 1408 reader = FilesPlume.newLineNumberFileReader(raw_filename); 1409 } 1410 1411 varcomp_format = VarComparability.IMPLICIT; 1412 rtype = RecordType.NULL; 1413 ppt = null; 1414 } 1415 1416 /** Releases resources held by this. */ 1417 @Override 1418 @EnsuresCalledMethods(value = "reader", methods = "close") 1419 public void close(@GuardSatisfied ParseState this) { 1420 try { 1421 reader.close(); 1422 } catch (IOException e) { 1423 throw new BugInDaikon(e); 1424 } 1425 } 1426 1427 /** 1428 * Returns the current line number in the input file, or -1 if not available. 1429 * 1430 * @return the current line number in the input file, or -1 if not available 1431 */ 1432 public int get_linenum() { 1433 return reader.getLineNumber(); 1434 } 1435 1436 private static NumberFormat pctFmt; 1437 1438 static { 1439 pctFmt = NumberFormat.getPercentInstance(); 1440 pctFmt.setMinimumFractionDigits(2); 1441 pctFmt.setMaximumFractionDigits(2); 1442 } 1443 1444 public String reading_message() { 1445 String line; 1446 if (reader == null) { 1447 line = "?"; 1448 } else { 1449 long lineNum = reader.getLineNumber(); 1450 line = String.valueOf(lineNum); 1451 if (total_lines > 0) { 1452 double frac = lineNum / (double) total_lines; 1453 String percent = pctFmt.format(frac); 1454 line = line + ", " + percent; 1455 } 1456 } 1457 return "Reading " + filename + " (line " + line + ") ..."; 1458 } 1459 1460 public String line_file_message() { 1461 return String.format(" at line %d in file %s", reader.getLineNumber(), filename); 1462 } 1463 } 1464 1465 /** Returns the current line number in the input file, or -1 if not available. */ 1466 public static int get_linenum() { 1467 if (FileIO.data_trace_state == null) { 1468 return -1; 1469 } else { 1470 return FileIO.data_trace_state.get_linenum(); 1471 } 1472 } 1473 1474 /** 1475 * Logically, this is a local variable in static method read_data_trace_file. It is used for 1476 * status output, and to give the line number at which a problem was detected. 1477 */ 1478 // The @MonotonicNonNull property is not true globally, but within every 1479 // method it's true, so it is a useful annotation. 1480 public static @MonotonicNonNull ParseState data_trace_state = null; 1481 1482 // The variable is only ever cleared at the end of a routine that set it. 1483 @SuppressWarnings("nullness") // reinitialization 1484 private static void clear_data_trace_state() { 1485 FileIO.data_trace_state = null; 1486 } 1487 1488 /** 1489 * Read only samples from {@code .dtrace} file. Uses the standard data processor which calls 1490 * {@link FileIO#process_sample(PptMap, PptTopLevel, ValueTuple, Integer)} on each record, and 1491 * ignores records other than samples. 1492 */ 1493 public static void read_data_trace_file(String filename, PptMap all_ppts) throws IOException { 1494 Processor processor = new Processor(); 1495 read_data_trace_file(filename, all_ppts, processor, false, true); 1496 } 1497 1498 /** 1499 * Read declarations AND samples (not just sample data as the name might imply) from {@code 1500 * .dtrace} file. For each record read from the file, passes the record to a method of the 1501 * processor. 1502 */ 1503 public static void read_data_trace_file( 1504 String filename, 1505 PptMap all_ppts, 1506 Processor processor, 1507 boolean is_decl_file, 1508 boolean ppts_may_be_new) 1509 throws IOException { 1510 1511 if (debugRead.isLoggable(Level.FINE)) { 1512 debugRead.fine( 1513 "read_data_trace_file " 1514 + filename 1515 + ((Daikon.ppt_regexp != null) ? " " + Daikon.ppt_regexp.pattern() : "") 1516 + ((Daikon.ppt_omit_regexp != null) ? " " + Daikon.ppt_omit_regexp.pattern() : "")); 1517 } 1518 1519 try (ParseState data_trace_state = 1520 new ParseState(filename, is_decl_file, ppts_may_be_new, all_ppts)) { 1521 FileIO.data_trace_state = data_trace_state; 1522 1523 // Used for debugging: write new data trace file. 1524 if (Global.debugPrintDtrace) { 1525 Path p = new File(filename + ".debug").toPath(); 1526 BufferedWriter bw = null; // dummy initialization for compiler's definite assignment check 1527 try { 1528 bw = Files.newBufferedWriter(p, UTF_8); 1529 Global.dtraceWriter = new PrintWriter(bw); 1530 } catch (IOException e) { 1531 if (bw != null) { 1532 bw.close(); 1533 } 1534 throw e; 1535 } 1536 } 1537 1538 while (true) { 1539 read_data_trace_record(data_trace_state); 1540 1541 if (data_trace_state.rtype == RecordType.SAMPLE) { 1542 assert data_trace_state.ppt != null 1543 : "@AssumeAssertion(nullness): dependent: RecordType.SAMPLE"; 1544 assert data_trace_state.vt != null 1545 : "@AssumeAssertion(nullness): dependent: RecordType.SAMPLE"; 1546 // Nonce may be null 1547 samples_processed++; 1548 // Add orig and derived variables; pass to inference (add_and_flow) 1549 try { 1550 processor.process_sample( 1551 data_trace_state.all_ppts, 1552 data_trace_state.ppt, 1553 data_trace_state.vt, 1554 data_trace_state.nonce); 1555 } catch (Error e) { 1556 // e.printStackTrace(); 1557 if (!dkconfig_continue_after_file_exception) { 1558 throw new Daikon.UserError(e, data_trace_state); 1559 } else { 1560 System.out.println(); 1561 System.out.println( 1562 "WARNING: Error while processing trace file; subsequent records ignored."); 1563 System.out.print("Ignored backtrace:"); 1564 e.printStackTrace(System.out); 1565 System.out.println(); 1566 } 1567 } 1568 } else if ((data_trace_state.rtype == RecordType.EOF) 1569 || (data_trace_state.rtype == RecordType.TRUNCATED)) { 1570 break; 1571 } else { 1572 // don't need to do anything explicit for other records found 1573 } 1574 } 1575 1576 if (Global.debugPrintDtrace) { 1577 assert Global.dtraceWriter != null 1578 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 1579 Global.dtraceWriter.close(); 1580 } 1581 1582 Daikon.progress = "Finished reading " + data_trace_state.filename; 1583 1584 clear_data_trace_state(); 1585 } 1586 } 1587 1588 /** 1589 * Like read_data_trace_record, but sets global FileIO.data_trace_state for the duration of the 1590 * call then clears it before returning. Intended for most external callers. 1591 */ 1592 public static void read_data_trace_record_setstate(ParseState state) throws IOException { 1593 1594 FileIO.data_trace_state = state; 1595 read_data_trace_record(state); 1596 clear_data_trace_state(); 1597 } 1598 1599 /** 1600 * Read a single record of ANY type (sample, declaration, comparability, etc.) from a dtrace file. 1601 * If the record is anything but a sample, also processes it. The record is stored by side effect 1602 * into the state argument. 1603 */ 1604 // TODO: For clarity, this should perhaps return its side-effected argument. 1605 @RequiresNonNull("FileIO.data_trace_state") 1606 // not guaranteed: File might be empty EnsuresNonNull("FileIO.new_decl_format") 1607 public static void read_data_trace_record(ParseState state) throws IOException { 1608 1609 // Abstract out the test result into a variable because Java doesn't 1610 // permit suppressing warnings on a statement. Yuck. 1611 boolean stateOK = (state == FileIO.data_trace_state); 1612 assert stateOK; 1613 1614 LineNumberReader reader = state.reader; 1615 1616 for (String line = reader.readLine(); line != null; line = reader.readLine()) { 1617 if (line.equals("")) { 1618 continue; 1619 } 1620 1621 // This cleverness would not be necessary if every comment was followed by 1622 // a blank line. We can't depend on that, though. 1623 if (isComment(line)) { 1624 StringJoiner commentLines = new StringJoiner(lineSep); 1625 commentLines.add(line); 1626 while (nextLineIsComment(reader)) { 1627 commentLines.add(reader.readLine()); 1628 } 1629 state.payload = commentLines.toString(); 1630 state.rtype = RecordType.COMMENT; 1631 return; 1632 } 1633 1634 // stop at a specified point in the file 1635 if ((dkconfig_max_line_number > 0) && (reader.getLineNumber() > dkconfig_max_line_number)) { 1636 state.rtype = RecordType.TRUNCATED; 1637 return; 1638 } 1639 1640 // interning bugfix: no need to intern "line" (after code change to is_declaration_header) 1641 1642 // Check for the file format 1643 if (line.startsWith("decl-version")) { 1644 read_decl_version(state, line); 1645 state.payload = (new_decl_format ? "2.0" : "1.0"); 1646 state.payload = (FileIO.new_decl_format ? "2.0" : "1.0"); 1647 state.rtype = RecordType.DECL_VERSION; 1648 return; 1649 } 1650 1651 // Check for the input language 1652 if (line.startsWith("input-language")) { 1653 String input_language = read_input_language(state, line); 1654 state.payload = input_language; 1655 state.rtype = RecordType.INPUT_LANGUAGE; 1656 return; 1657 } 1658 1659 // If we have gotten to here and new_decl_format is not set, presume 1660 // it is the old format 1661 if (new_decl_format == null) { 1662 // System.out.printf("setting new_decl_format to false%n"); 1663 new_decl_format = Boolean.FALSE; 1664 } 1665 1666 // First look for declarations in the dtrace stream 1667 if (is_declaration_header(line)) { 1668 if (new_decl_format) { 1669 state.ppt = read_ppt_decl(state, line); 1670 } else { 1671 state.ppt = read_declaration(state); 1672 } 1673 // ppt can be null if this declaration was skipped because of 1674 // --ppt-select-pattern or --ppt-omit-pattern. 1675 if (state.ppt != null) { 1676 if (!state.all_ppts.containsName(state.ppt.name())) { 1677 state.all_ppts.add(state.ppt); 1678 assert state.ppt != null : "@AssumeAssertion(nullness)"; 1679 try { 1680 Daikon.init_ppt(state.ppt, state.all_ppts); 1681 } catch (Exception e) { 1682 decl_error(state, e); 1683 } 1684 } 1685 } 1686 state.rtype = RecordType.DECL; 1687 return; 1688 } 1689 if (line.equals("VarComparability") || line.startsWith("var-comparability")) { 1690 state.varcomp_format = read_var_comparability(state, line); 1691 state.rtype = RecordType.COMPARABILITY; 1692 return; 1693 } 1694 if (line.equals("ListImplementors")) { 1695 state.payload = read_list_implementors(reader); 1696 state.rtype = RecordType.LIST_IMPLEMENTORS; 1697 return; 1698 } 1699 String ppt_name = line; 1700 if (new_decl_format) ppt_name = unescape_decl(line); // interning bugfix: no need to intern 1701 ppt_name = user_mod_ppt_name(ppt_name); 1702 if (!ppt_included(ppt_name)) { 1703 // System.out.printf("skipping ppt %s%n", line); 1704 while ((line != null) && !line.equals("")) line = reader.readLine(); 1705 continue; 1706 } 1707 // System.out.printf("Not skipping ppt %s%n", line); 1708 1709 if (state.is_decl_file) { 1710 if (!new_decl_format && line.startsWith("ppt ")) { 1711 throw new Daikon.UserError( 1712 String.format( 1713 "Declaration file %s is not version 2.0, but line %d looks like a version 2.0" 1714 + " declaration: %s%nPerhaps the file is missing a \"decl-version 2.0\"" 1715 + " record at the beginning", 1716 state.filename, state.reader.getLineNumber(), line)); 1717 } 1718 throw new Daikon.UserError( 1719 String.format( 1720 "Declaration files should not contain samples, but file %s does at line %d: %s", 1721 state.filename, state.reader.getLineNumber(), line)); 1722 } 1723 1724 // Parse the ppt name 1725 try { 1726 new PptName(ppt_name); 1727 } catch (Throwable t) { 1728 @SuppressWarnings("nullness") // thrown exception always has a detail message 1729 @NonNull String message = t.getMessage(); 1730 // Augment the message with line number information. 1731 if (!(t instanceof Daikon.UserError)) { 1732 message = String.format("Illegal program point name '%s' (%s)", ppt_name, message); 1733 } 1734 throw new Daikon.UserError(message, reader, state.filename); 1735 } 1736 1737 if (state.all_ppts.size() == 0) { 1738 throw new Daikon.UserError( 1739 "No declarations were provided before the first sample. Perhaps you did not supply" 1740 + " the proper .decls file to Daikon. (Or, there could be a bug in the front end" 1741 + " that created the .dtrace file " 1742 + state.filename 1743 + ".)"); 1744 } 1745 1746 PptTopLevel ppt = state.all_ppts.get(ppt_name); 1747 if (ppt == null) { 1748 throw new Daikon.UserError( 1749 "No declaration was provided for program point " + ppt_name, state); 1750 } 1751 1752 // not vis.length, as that includes constants, derived variables, etc. 1753 // Actually, we do want to leave space for _orig vars. 1754 // And for the time being (and possibly forever), for derived variables. 1755 int vals_array_size = ppt.var_infos.length - ppt.num_static_constant_vars; 1756 1757 // Read an invocation nonce if one exists 1758 Integer nonce; 1759 1760 boolean nonce_exists; 1761 { 1762 String nonce_header_peekahead; 1763 // arbitrary number, hopefully big enough; catch exceptions 1764 reader.mark(1000); 1765 try { 1766 nonce_header_peekahead = reader.readLine(); 1767 } catch (Exception e) { 1768 nonce_header_peekahead = null; 1769 } 1770 reader.reset(); 1771 nonce_exists = NONCE_HEADER.equals(nonce_header_peekahead); 1772 } 1773 if (!nonce_exists) { 1774 nonce = null; 1775 } else { 1776 @SuppressWarnings("nullness") // nonce_exists is true, so readLine() returns non-null 1777 @NonNull String nonce_header = reader.readLine(); // read & discard header 1778 assert NONCE_HEADER.equals(nonce_header); 1779 String nonce_number = reader.readLine(); 1780 if (nonce_number == null) { 1781 throw new Daikon.UserError("File ended while trying to read nonce", state); 1782 } 1783 nonce = Integer.valueOf(nonce_number); 1784 1785 if (Global.debugPrintDtrace) { 1786 to_write_nonce = true; 1787 nonce_value = nonce.toString(); 1788 } 1789 } 1790 1791 @Nullable Object[] vals = new @Nullable Object[vals_array_size]; 1792 int[] mods = new int[vals_array_size]; 1793 1794 // Read a single record from the trace file; 1795 // fills up vals and mods arrays by side effect. 1796 try { 1797 read_vals_and_mods_from_trace_file(reader, state.filename, ppt, vals, mods); 1798 } catch (IOException e) { 1799 String nextLine = reader.readLine(); 1800 if ((e instanceof EOFException) || (nextLine == null)) { 1801 System.out.println(); 1802 System.out.println( 1803 "WARNING: Unexpected EOF while processing " 1804 + "trace file - last record of trace file ignored"); 1805 state.rtype = RecordType.EOF; 1806 return; 1807 } else if (dkconfig_continue_after_file_exception) { 1808 System.out.println(); 1809 System.out.println("WARNING: IOException while processing trace file - record ignored"); 1810 System.out.print("Ignored backtrace:"); 1811 e.printStackTrace(System.out); 1812 System.out.println(); 1813 while (nextLine != null && !nextLine.equals("")) { 1814 // System.out.println("Discarded line " + reader.getLineNumber() 1815 // + ": " + nextLine); 1816 nextLine = reader.readLine(); 1817 } 1818 continue; 1819 } else { 1820 throw e; 1821 } 1822 } 1823 1824 state.ppt = ppt; 1825 state.nonce = nonce; 1826 state.vt = ValueTuple.makeUninterned(vals, mods); 1827 state.rtype = RecordType.SAMPLE; 1828 return; 1829 } 1830 1831 state.rtype = RecordType.EOF; 1832 return; 1833 } 1834 1835 /** 1836 * Add orig() and derived variables to vt (by side effect), then supply it to the program point 1837 * for flowing. 1838 * 1839 * @param vt trace data only; modified by side effect to add derived vars 1840 */ 1841 @RequiresNonNull("FileIO.data_trace_state") 1842 public static void process_sample( 1843 PptMap all_ppts, PptTopLevel ppt, ValueTuple vt, @Nullable Integer nonce) { 1844 1845 // Add orig variables. This must be above the check below because 1846 // it saves away the orig values from enter points for later use 1847 // by exit points. 1848 boolean ignore = compute_orig_variables(ppt, vt.vals, vt.mods, nonce); 1849 if (ignore) { 1850 return; 1851 } 1852 1853 // Only process the leaves of the ppt tree. 1854 // This test assumes that all leaves are numbered exit program points 1855 // -- that is, points of the form foo:::EXIT22 for which isExitPoint() 1856 // is true and isCombinedExitPoint() is false. "Combined" exit points 1857 // of the form foo:::EXIT are not processed -- they are assumed to be 1858 // non-leaves. 1859 if (Daikon.use_dataflow_hierarchy) { 1860 1861 // Rather than defining leaves as :::EXIT54 (numbered exit) 1862 // program points define them as everything except 1863 // ::EXIT (combined), :::ENTER, :::THROWS, :::OBJECT, ::GLOBAL 1864 // and :::CLASS program points. This scheme ensures that arbitrarly 1865 // named program points such as :::POINT (used by convertcsv.pl) 1866 // will be treated as leaves. 1867 1868 if (ppt.ppt_name.isEnterPoint() 1869 || ppt.ppt_name.isThrowsPoint() 1870 || ppt.ppt_name.isObjectInstanceSynthetic() 1871 || ppt.ppt_name.isClassStaticSynthetic() 1872 || ppt.ppt_name.isGlobalPoint()) { 1873 return; 1874 } 1875 1876 if (ppt.ppt_name.isExitPoint() && ppt.ppt_name.isCombinedExitPoint()) { 1877 // not Daikon.UserError; caller has more info (e.g., filename) 1878 throw new RuntimeException( 1879 "Bad program point name " + ppt.name + " is a combined exit point name"); 1880 } 1881 } 1882 1883 // Add derived variables 1884 compute_derived_variables(ppt, vt.vals, vt.mods); 1885 1886 // Causes interning 1887 vt = new ValueTuple(vt.vals, vt.mods); 1888 1889 if (debugRead.isLoggable(Level.FINE)) { 1890 debugRead.fine("Adding ValueTuple to " + ppt.name()); 1891 debugRead.fine(" length is " + vt.vals.length); 1892 } 1893 1894 // If we are only reading the sample, don't process them 1895 if (dkconfig_read_samples_only) { 1896 return; 1897 } 1898 1899 @SuppressWarnings({"UnusedVariable", "nullness:contracts.precondition"}) 1900 Object dummy = ppt.add_bottom_up(vt, 1); 1901 1902 if (debugVars.isLoggable(Level.FINE)) { 1903 debugVars.fine(ppt.name() + " vars: " + Debug.int_vars(ppt, vt)); 1904 } 1905 1906 if (Global.debugPrintDtrace) { 1907 assert Global.dtraceWriter != null 1908 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 1909 Global.dtraceWriter.close(); 1910 } 1911 } 1912 1913 /** Returns true if this procedure has an unmatched entry. */ 1914 static boolean has_unmatched_procedure_entry(PptTopLevel ppt) { 1915 for (Invocation invok : call_hashmap.values()) { 1916 if (invok.ppt == ppt) { 1917 return true; 1918 } 1919 } 1920 for (Invocation invok : call_stack) { 1921 if (invok.ppt == ppt) { 1922 return true; 1923 } 1924 } 1925 return false; 1926 } 1927 1928 /** Print each call that does not have a matching exit. */ 1929 public static void process_unmatched_procedure_entries() { 1930 1931 if (dkconfig_unmatched_procedure_entries_quiet) { 1932 return; 1933 } 1934 1935 int unmatched_count = call_stack.size() + call_hashmap.size(); 1936 1937 if (!call_stack.isEmpty() || !call_hashmap.isEmpty()) { 1938 System.out.println(); 1939 System.out.print( 1940 "No return from procedure observed " 1941 + StringsPlume.nplural(unmatched_count, "time") 1942 + "."); 1943 if (Daikon.use_dataflow_hierarchy) { 1944 System.out.print(" Unmatched entries are ignored!"); 1945 } 1946 System.out.println(); 1947 if (!call_hashmap.isEmpty()) { 1948 // Put the invocations in sorted order for printing. 1949 ArrayList<Invocation> invocations = new ArrayList<>(); 1950 for (@KeyFor("call_hashmap") Integer i : CollectionsPlume.sortedKeySet(call_hashmap)) { 1951 Invocation invok = call_hashmap.get(i); 1952 assert invok != null; 1953 invocations.add(invok); 1954 } 1955 System.out.println("Unterminated calls:"); 1956 if (dkconfig_verbose_unmatched_procedure_entries) { 1957 print_invocations_verbose(invocations); 1958 } else { 1959 print_invocations_grouped(invocations); 1960 } 1961 } 1962 1963 if (!call_stack.isEmpty()) { 1964 if (dkconfig_verbose_unmatched_procedure_entries) { 1965 System.out.println( 1966 "Remaining " 1967 + StringsPlume.nplural(unmatched_count, "stack") 1968 + " call summarized below."); 1969 print_invocations_verbose(call_stack); 1970 } else { 1971 print_invocations_grouped(call_stack); 1972 } 1973 } 1974 System.out.print("End of report for procedures not returned from."); 1975 if (Daikon.use_dataflow_hierarchy) { 1976 System.out.print(" Unmatched entries are ignored!"); 1977 } 1978 System.out.println(); 1979 } 1980 } 1981 1982 /** Print all the invocations in the collection, in order. */ 1983 static void print_invocations_verbose(Collection<Invocation> invocations) { 1984 for (Invocation invok : invocations) { 1985 System.out.println(invok.format()); 1986 } 1987 } 1988 1989 /** Print the invocations in the collection, in order, and coalescing duplicates. */ 1990 static void print_invocations_grouped(Collection<Invocation> invocations) { 1991 Map<@Interned String, Integer> counter = new LinkedHashMap<>(); 1992 1993 for (Invocation invok_noncanonical : invocations) { 1994 @Interned Invocation invok = invok_noncanonical.canonicalize(); 1995 String invokString = invok.format(false).intern(); 1996 if (counter.containsKey(invokString)) { 1997 Integer oldCount = counter.get(invokString); 1998 Integer newCount = oldCount.intValue() + 1; 1999 counter.put(invokString, newCount); 2000 } else { 2001 counter.put(invokString, 1); 2002 } 2003 } 2004 2005 // Print the invocations in sorted order. 2006 for (Map.Entry<@Interned String, Integer> invokEntry : counter.entrySet()) { 2007 System.out.println( 2008 invokEntry.getKey() + " : " + StringsPlume.nplural(invokEntry.getValue(), "invocation")); 2009 } 2010 } 2011 2012 // This procedure reads a single record from a trace file and 2013 // fills up vals and mods by side effect. The ppt name and 2014 // invocation nonce (if any) have already been read. 2015 @RequiresNonNull("FileIO.data_trace_state") 2016 private static void read_vals_and_mods_from_trace_file( 2017 LineNumberReader reader, 2018 String filename, 2019 PptTopLevel ppt, 2020 @Nullable Object[] vals, 2021 int[] mods) 2022 throws IOException { 2023 VarInfo[] vis = ppt.var_infos; 2024 int num_tracevars = ppt.num_tracevars; 2025 2026 /*NNC:@Nullable*/ String[] oldvalue_reps = ppt_to_value_reps.get(ppt); 2027 if (oldvalue_reps == null) { 2028 // We've not encountered this program point before. The nulls in 2029 // this array will compare non-equal to whatever is in the trace 2030 // file, which is the desired behavior. 2031 oldvalue_reps = new /*NNC:@Nullable*/ String[num_tracevars]; 2032 } 2033 2034 if (Global.debugPrintDtrace) { 2035 assert Global.dtraceWriter != null 2036 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 2037 Global.dtraceWriter.println(ppt.name()); 2038 2039 if (to_write_nonce) { 2040 Global.dtraceWriter.println(NONCE_HEADER); 2041 Global.dtraceWriter.println(nonce_value); 2042 to_write_nonce = false; 2043 } 2044 } 2045 2046 for (int vi_index = 0, val_index = 0; val_index < num_tracevars; vi_index++) { 2047 assert vi_index < vis.length 2048 : "Got to vi_index " 2049 + vi_index 2050 + " after " 2051 + val_index 2052 + " of " 2053 + num_tracevars 2054 + " values"; 2055 VarInfo vi = vis[vi_index]; 2056 assert !vi.is_static_constant || (vi.value_index == -1) 2057 // : "Bad value_index " + vi.value_index + " when static_constant_value = " + 2058 // vi.static_constant_value + " for " + vi.repr() + " at " + ppt_name 2059 ; 2060 if (vi.is_static_constant) { 2061 continue; 2062 } 2063 assert val_index == vi.value_index 2064 // : "Differing val_index = " + val_index 2065 // + " and vi.value_index = " + vi.value_index 2066 // + " for " + vi.name + lineSep + vi.repr() 2067 ; 2068 2069 // In errors, say "for program point", not "at program point" as the 2070 // latter confuses Emacs goto-error. 2071 2072 String line = reader.readLine(); 2073 if (line == null) { 2074 throw new Daikon.UserError( 2075 "Unexpected end of file at " 2076 + data_trace_state.filename 2077 + " line " 2078 + reader.getLineNumber() 2079 + lineSep 2080 + " Expected variable " 2081 + vi.name() 2082 + ", got " 2083 + "null" // line 2084 + " for program point " 2085 + ppt.name()); 2086 } 2087 2088 // Read lines until an included variable is found 2089 while ((line != null) && !line.equals("") && !var_included(line)) { 2090 line = reader.readLine(); // value (discard it) 2091 line = reader.readLine(); // modbit 2092 if (line == null || !(line.equals("0") || line.equals("1") || line.equals("2"))) { 2093 throw new Daikon.UserError("Bad modbit '" + line + "'", data_trace_state); 2094 } 2095 line = reader.readLine(); // next variable name 2096 } 2097 if (line == null) { 2098 throw new Daikon.UserError( 2099 "Unexpected end of file at " 2100 + data_trace_state.filename 2101 + " line " 2102 + reader.getLineNumber() 2103 + lineSep 2104 + " Expected to find variable name" 2105 + " for program point " 2106 + ppt.name()); 2107 } 2108 2109 if (!unescape_decl(line.trim()).equals(vi.str_name())) { 2110 throw new Daikon.UserError( 2111 "Mismatch between declaration and trace. Expected variable " 2112 + vi.name() 2113 + ", got " 2114 + line 2115 + " for program point " 2116 + ppt.name(), 2117 data_trace_state); 2118 } 2119 line = reader.readLine(); 2120 if (line == null) { 2121 throw new Daikon.UserError( 2122 "Unexpected end of file at " 2123 + data_trace_state.filename 2124 + " line " 2125 + reader.getLineNumber() 2126 + lineSep 2127 + " Expected value for variable " 2128 + vi.name() 2129 + ", got " 2130 + "null" // line 2131 + " for program point " 2132 + ppt.name()); 2133 } 2134 String value_rep = line; 2135 line = reader.readLine(); 2136 if (line == null) { 2137 throw new Daikon.UserError( 2138 "Unexpected end of file at " 2139 + data_trace_state.filename 2140 + " line " 2141 + reader.getLineNumber() 2142 + lineSep 2143 + " Expected modbit for variable " 2144 + vi.name() 2145 + ", got " 2146 + "null" // line 2147 + " for program point " 2148 + ppt.name()); 2149 } 2150 if (!(line.equals("0") || line.equals("1") || line.equals("2"))) { 2151 throw new Daikon.UserError("Bad modbit `" + line + "'", data_trace_state); 2152 } 2153 int mod = ValueTuple.parseModified(line); 2154 2155 // System.out.println("Mod is " + mod + " at " + data_trace_state.filename + " line " + 2156 // reader.getLineNumber()); 2157 // System.out.pringln(" for variable " + vi.name() 2158 // + " for program point " + ppt.name()); 2159 2160 // MISSING_FLOW is only found during flow algorithm 2161 assert mod != ValueTuple.MISSING_FLOW : "Data trace value can't be missing due to flow"; 2162 2163 if (mod != ValueTuple.MISSING_NONSENSICAL) { 2164 // Set the modbit now, depending on whether the value of the variable 2165 // has been changed or not. 2166 if (value_rep.equals(oldvalue_reps[val_index])) { 2167 if (!dkconfig_add_changed) { 2168 mod = ValueTuple.UNMODIFIED; 2169 } 2170 } else { 2171 mod = ValueTuple.MODIFIED; 2172 } 2173 } 2174 2175 mods[val_index] = mod; 2176 oldvalue_reps[val_index] = value_rep; 2177 2178 if (Global.debugPrintDtrace) { 2179 assert Global.dtraceWriter != null 2180 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 2181 Global.dtraceWriter.println(vi.name()); 2182 Global.dtraceWriter.println(value_rep); 2183 Global.dtraceWriter.println(mod); 2184 } 2185 Debug dbg = Debug.newDebug(FileIO.class, ppt, Debug.vis(vi)); 2186 if (dbg != null) { 2187 dbg.log("Var " + vi.name() + " has value " + value_rep + " mod " + mod); 2188 } 2189 2190 // Both uninit and nonsensical mean missing modbit 2, because 2191 // it doesn't make sense to look at x.y when x is uninitialized. 2192 if (ValueTuple.modIsMissingNonsensical(mod)) { 2193 if (!(value_rep.equals("nonsensical") 2194 // Kvasir still uses "uninit" (it distinguishes between 2195 // uninit and nonsensical), though the Daikon manual does not 2196 // officially permit "uninit" as a value and has not since at 2197 // least 2002. This is fixed in the Kvasir repository as of 2198 // 5/2009, so the following two lines should be removed at 2199 // some point not too long after that. Then Daikon should 2200 // print a warning (or even terminate execution) about uses 2201 // of "uninit". 2202 || value_rep.equals("uninit") 2203 || value_rep.equals("missing"))) { 2204 throw new Daikon.UserError( 2205 "Modbit indicates nonsensical value for variable " 2206 + vi.name() 2207 + " with value \"" 2208 + value_rep 2209 + "\";" 2210 + lineSep 2211 + " text of value should be \"nonsensical\"", 2212 data_trace_state); 2213 } else { 2214 if (debug_missing && !vi.canBeMissing) { 2215 System.out.printf( 2216 "Var %s ppt %s at line %d missing%n", vi, ppt.name(), FileIO.get_linenum()); 2217 System.out.printf("val_index = %d, mods[val_index] = %d%n", val_index, mods[val_index]); 2218 } 2219 vi.canBeMissing = true; 2220 } 2221 vals[val_index] = null; 2222 } else { 2223 // mod is not MISSING_NONSENSICAL 2224 2225 // System.out.println("Mod is " + mod + " (missing=" + 2226 // ValueTuple.MISSING + "), rep=" + value_rep + 2227 // "(modIsMissing=" + ValueTuple.modIsMissing(mod) + ")"); 2228 2229 try { 2230 vals[val_index] = vi.rep_type.parse_value(value_rep, reader, filename); 2231 if (vals[val_index] == null) { 2232 if (debug_missing && !vi.canBeMissing) { 2233 System.out.printf( 2234 "Var %s ppt %s at line %d is null, and modbit is not missing%n", 2235 vi, ppt.name(), FileIO.get_linenum()); 2236 } 2237 // The value in the trace was null even though the modbit was not 2238 // MISSING_NONSENSICAL. Set the modbit to MISSING_NONSENSICAL. 2239 // This can happen for a value like [1 nonsensical 2], because 2240 // if any array value is nonsensical, the whole array is 2241 // treated as nonsensical. 2242 mods[val_index] = ValueTuple.MISSING_NONSENSICAL; 2243 vi.canBeMissing = true; 2244 } 2245 } catch (Daikon.UserError e) { 2246 throw e; 2247 } catch (Throwable e) { 2248 // e.printStackTrace(System.err); // for debugging 2249 throw new Daikon.UserError( 2250 e, 2251 "Error while parsing value " 2252 + value_rep 2253 + " for variable " 2254 + vi.name() 2255 + " of type " 2256 + vi.rep_type 2257 + ": " 2258 + e.getLocalizedMessage(), 2259 reader, 2260 filename); 2261 } 2262 } 2263 val_index++; 2264 } 2265 2266 // Does oldvalue_reps now have no null elements??? 2267 oldvalue_reps = castNonNullDeep(oldvalue_reps); // https://tinyurl.com/cfissue/986 2268 ppt_to_value_reps.put(ppt, oldvalue_reps); 2269 2270 if (Global.debugPrintDtrace) { 2271 assert Global.dtraceWriter != null 2272 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 2273 Global.dtraceWriter.println(); 2274 } 2275 2276 // Expecting the end of a block of values. 2277 String line = reader.readLine(); 2278 // First, we might get some variables that ought to be omitted. 2279 while ((line != null) && !line.equals("") && !var_included(line)) { 2280 line = reader.readLine(); // value 2281 line = reader.readLine(); // modbit 2282 line = reader.readLine(); // next variable name 2283 } 2284 assert (line == null) || line.equals("") 2285 : "Expected blank line in " 2286 + data_trace_state.filename 2287 + " at line " 2288 + reader.getLineNumber() 2289 + ": " 2290 + line; 2291 } 2292 2293 /** 2294 * If this is a function entry ppt, stores the values of all of the variables away for use at the 2295 * exit. If this is an exit, finds the values at enter and adds them as the values of the orig 2296 * variables. Normally returns false. Returns true if this is an exit without a matching enter. 2297 * See dkconfig_ignore_missing_enter for more info. If true is returned, this ppt should be 2298 * ignored by the caller. 2299 */ 2300 @RequiresNonNull("FileIO.data_trace_state") 2301 public static boolean compute_orig_variables( 2302 PptTopLevel ppt, 2303 // HashMap cumulative_modbits, 2304 @Nullable Object[] vals, 2305 int[] mods, 2306 @Nullable Integer nonce) { 2307 assert data_trace_state != null; 2308 2309 VarInfo[] vis = ppt.var_infos; 2310 @Interned String fn_name = ppt.ppt_name.getNameWithoutPoint(); 2311 String ppt_name = ppt.name(); 2312 if (ppt_name.endsWith(enter_tag)) { 2313 Invocation invok = new Invocation(ppt, vals, mods); 2314 if (nonce == null) { 2315 call_stack.push(invok); 2316 } else { 2317 call_hashmap.put(nonce, invok); 2318 } 2319 return false; 2320 } 2321 2322 if (ppt.ppt_name.isExitPoint() || ppt.ppt_name.isThrowsPoint()) { 2323 Invocation invoc; 2324 // Set invoc 2325 { 2326 if (nonce == null) { 2327 if (call_stack.isEmpty()) { 2328 // Not Daikon.UserError: caller knows context such as 2329 // file name and line number. 2330 throw new Error("Function exit without corresponding entry: " + ppt.name()); 2331 } 2332 invoc = call_stack.pop(); 2333 while (invoc.ppt.ppt_name.getNameWithoutPoint() != fn_name) { 2334 // Should also mark as a function that made an exceptional exit 2335 // at run time. 2336 System.err.println( 2337 "Exceptional exit from function " 2338 + fn_name 2339 + ", expected to first exit from " 2340 + invoc.ppt.ppt_name.getNameWithoutPoint() 2341 + ((data_trace_state.filename == null) 2342 ? "" 2343 : "; at " 2344 + data_trace_state.filename 2345 + " line " 2346 + data_trace_state.reader.getLineNumber())); 2347 invoc = call_stack.pop(); 2348 } 2349 } else { 2350 // nonce != null 2351 if (!call_hashmap.containsKey(nonce)) { 2352 if (dkconfig_ignore_missing_enter) { 2353 // System.out.printf("Didn't find call with nonce %d to match %s" + 2354 // " ending at %s line %d%n", nonce, ppt.name(), 2355 // data_trace_state.filename, 2356 // data_trace_state.reader.getLineNumber()); 2357 return true; 2358 } else { 2359 // Not Daikon.UserError: caller knows context such as 2360 // file name and line number. 2361 throw new Error( 2362 String.format( 2363 "Didn't find call with nonce %s to match %s ending at %s line %d", 2364 nonce, 2365 ppt.name(), 2366 data_trace_state.filename, 2367 data_trace_state.reader.getLineNumber())); 2368 } 2369 } 2370 invoc = call_hashmap.get(nonce); 2371 call_hashmap.remove(nonce); 2372 } 2373 } 2374 2375 // Loop through each orig variable and get its value/mod bits from 2376 // the ENTER point. vi_index is the index into var_infos at the 2377 // ENTER point. val_index is the index into vals[] and mods[] at 2378 // ENTER point. Note that vis[] includes static constants but 2379 // vals[] and mods[] do not. Also that we don't create orig versions 2380 // of static constants 2381 int vi_index = 0; 2382 for (int val_index = 0; val_index < ppt.num_orig_vars; val_index++) { 2383 VarInfo vi = vis[ppt.num_tracevars + ppt.num_static_constant_vars + val_index]; 2384 assert !vi.is_static_constant : "orig constant " + vi; 2385 2386 // Skip over constants in the entry point 2387 while (invoc.ppt.var_infos[vi_index].is_static_constant) { 2388 vi_index++; 2389 } 2390 2391 // Copy the vals and mod bits from entry to exit 2392 vals[ppt.num_tracevars + val_index] = invoc.vals[val_index]; 2393 int mod = invoc.mods[val_index]; 2394 mods[ppt.num_tracevars + val_index] = mod; 2395 2396 // If the value was missing, mark this variable as can be missing. 2397 // Carefully check that we have orig version of the variable from 2398 // the ENTER point. 2399 if (ValueTuple.modIsMissingNonsensical(mod)) { 2400 if (debug_missing && !vi.canBeMissing) { 2401 System.out.printf("add_orig: var %s missing[%d/%d]%n", vi, val_index, vi_index); 2402 } 2403 vi.canBeMissing = true; 2404 assert invoc.vals[val_index] == null; 2405 assert vi.name() == invoc.ppt.var_infos[vi_index].prestate_name() 2406 : vi.name() + " != " + invoc.ppt.var_infos[vi_index]; 2407 assert invoc.ppt.var_infos[vi_index].canBeMissing : invoc.ppt.var_infos[vi_index]; 2408 } 2409 vi_index++; 2410 } 2411 } 2412 return false; 2413 } 2414 2415 /** Computes values of derived variables. */ 2416 public static void compute_derived_variables( 2417 PptTopLevel ppt, @Nullable Object[] vals, int[] mods) { 2418 // This ValueTuple is temporary: we're temporarily suppressing interning, 2419 // which we will do after we have all the values available. 2420 ValueTuple partial_vt = ValueTuple.makeUninterned(vals, mods); 2421 int filled_slots = ppt.num_orig_vars + ppt.num_tracevars + ppt.num_static_constant_vars; 2422 for (int i = 0; i < filled_slots; i++) { 2423 assert !ppt.var_infos[i].isDerived(); 2424 } 2425 int num_const = ppt.num_static_constant_vars; 2426 for (int i = filled_slots; i < ppt.var_infos.length; i++) { 2427 assert ppt.var_infos[i].derived != null : "variable not derived: " + ppt.var_infos[i].repr(); 2428 assert ppt.var_infos[i].derived != null : "@AssumeAssertion(nullness): application invariant"; 2429 // Add this derived variable's value 2430 ValueAndModified vm = ppt.var_infos[i].derived.computeValueAndModified(partial_vt); 2431 vals[i - num_const] = vm.value; 2432 mods[i - num_const] = vm.modified; 2433 } 2434 } 2435 2436 /////////////////////////////////////////////////////////////////////////// 2437 /// Serialized PptMap files 2438 /// 2439 2440 /** 2441 * Use a special record type. Saving as one object allows for reference-sharing, easier saves and 2442 * loads, and potential for later overriding of SerialFormat.readObject if the save format changes 2443 * (ick). 2444 */ 2445 static final class SerialFormat implements Serializable { 2446 // We are Serializable, so we specify a version to allow changes to 2447 // method signatures without breaking serialization. If you add or 2448 // remove fields, you should change this number to the current date. 2449 static final long serialVersionUID = 20060905L; 2450 2451 @RequiresNonNull("FileIO.new_decl_format") 2452 public SerialFormat(PptMap map, Configuration config) { 2453 this.map = map; 2454 this.config = config; 2455 this.new_decl_format = FileIO.new_decl_format; 2456 } 2457 2458 public PptMap map; 2459 public Configuration config; 2460 public boolean new_decl_format = false; 2461 } 2462 2463 /** 2464 * Write a serialized PptMap to a file. 2465 * 2466 * @param map a PptMap 2467 * @param file the file to which to write 2468 * @throws IOException if there is trouble writing the file 2469 */ 2470 public static void write_serialized_pptmap(PptMap map, File file) throws IOException { 2471 SerialFormat record = new SerialFormat(map, Configuration.getInstance()); 2472 FilesPlume.writeObject(record, file); 2473 } 2474 2475 /** 2476 * Read either a serialized PptMap or a InvMap and return a PptMap. If an InvMap is specified, it 2477 * is converted to a PptMap. 2478 */ 2479 @EnsuresNonNull("FileIO.new_decl_format") 2480 public static PptMap read_serialized_pptmap(File file, boolean use_saved_config) 2481 throws IOException { 2482 2483 try { 2484 Object obj = FilesPlume.readObject(file); 2485 if (obj instanceof FileIO.SerialFormat) { 2486 SerialFormat record = (SerialFormat) obj; 2487 if (use_saved_config) { 2488 Configuration.getInstance().overlap(record.config); 2489 } 2490 FileIO.new_decl_format = record.new_decl_format; 2491 // System.err.printf("Setting FileIO.new_decl_format to %b%n", 2492 // FileIO.new_decl_format); 2493 return record.map; 2494 } else if (obj instanceof InvMap) { 2495 // System.err.printf("Restoring an InvMap%n"); 2496 InvMap invs = (InvMap) obj; 2497 PptMap ppts = new PptMap(); 2498 for (PptTopLevel ppt : invs.pptIterable()) { 2499 PptTopLevel nppt = new PptTopLevel(ppt.name, ppt.var_infos); 2500 nppt.set_sample_number(ppt.num_samples()); 2501 ppts.add(nppt); 2502 List<Invariant> inv_list = invs.get(ppt); 2503 for (Invariant inv : inv_list) { 2504 PptSlice slice = nppt.get_or_instantiate_slice(inv.ppt.var_infos); 2505 inv.ppt = slice; 2506 slice.addInvariant(inv); 2507 } 2508 } 2509 assert FileIO.new_decl_format != null 2510 : "@AssumeAssertion(nullness): InvMap.readObject() sets FileIO.new_decl_format"; 2511 return ppts; 2512 } else { 2513 throw new IOException("Unexpected serialized file type: " + obj.getClass()); 2514 } 2515 } catch (ClassNotFoundException e) { 2516 throw (IOException) new IOException("Error while loading inv file").initCause(e); 2517 } catch (InvalidClassException e) { 2518 throw new IOException( 2519 "It is likely that the .inv file format has changed, because a Daikon data structure has" 2520 + " been modified, so your old .inv file is no longer readable by Daikon. Please" 2521 + " regenerate your .inv file." 2522 // + lineSep + e.toString() 2523 ); 2524 } 2525 // } catch (StreamCorruptedException e) { // already extends IOException 2526 // } catch (OptionalDataException e) { // already extends IOException 2527 } 2528 2529 /** 2530 * Returns whether or not the specified ppt name should be included in processing. Ppts can be 2531 * excluded because they match the omit_regexp, don't match ppt_regexp, or are greater than 2532 * ppt_max_name. 2533 */ 2534 public static boolean ppt_included(String ppt_name) { 2535 2536 // System.out.println ("ppt_name = '" + ppt_name + "' max name = '" 2537 // + Daikon.ppt_max_name + "'"); 2538 if (((Daikon.ppt_omit_regexp != null) && Daikon.ppt_omit_regexp.matcher(ppt_name).find()) 2539 || ((Daikon.ppt_regexp != null) && !Daikon.ppt_regexp.matcher(ppt_name).find()) 2540 || ((Daikon.ppt_max_name != null) 2541 && ((Daikon.ppt_max_name.compareTo(ppt_name) < 0) 2542 && (ppt_name.indexOf(global_suffix) == -1)))) { 2543 return false; 2544 } else { 2545 return true; 2546 } 2547 } 2548 2549 /** 2550 * Returns true if the given variable is included, according to Daikon's {@code 2551 * --var-select-pattern} and {@code --var-omit-pattern} flags. 2552 */ 2553 public static boolean var_included(String var_name) { 2554 assert !var_name.equals(""); 2555 if (((Daikon.var_omit_regexp != null) && Daikon.var_omit_regexp.matcher(var_name).find()) 2556 || ((Daikon.var_regexp != null) && !Daikon.var_regexp.matcher(var_name).find())) { 2557 return false; 2558 } else { 2559 return true; 2560 } 2561 } 2562 2563 /** 2564 * Checks the specified array of variables to see if it matches exactly the variables in the 2565 * existing ppt. Throws an error if there are any differences. Used to ensure that a new ppt with 2566 * the same name as an existing ppt is exactly the same. 2567 */ 2568 static void check_decl_match(ParseState state, PptTopLevel existing_ppt, VarInfo[] vi_array) { 2569 2570 VarInfo[] existing_vars = existing_ppt.var_infos; 2571 if (existing_ppt.num_declvars != vi_array.length) { 2572 throw new Daikon.UserError( 2573 "Duplicate declaration of program point \"" 2574 + existing_ppt.name() 2575 + "\" with a different number of VarInfo objects: " 2576 + "old VarInfo number=" 2577 + existing_ppt.num_declvars 2578 + ", new VarInfo number=" 2579 + vi_array.length, 2580 state); 2581 } 2582 2583 for (int i = 0; i < vi_array.length; i++) { 2584 String oldName = existing_vars[i].str_name(); 2585 String newName = vi_array[i].str_name(); 2586 if (!oldName.equals(newName)) { 2587 throw new Daikon.UserError( 2588 "Duplicate declaration of program point \"" 2589 + existing_ppt.name() 2590 + "\" with two different VarInfo: old VarInfo=" 2591 + oldName 2592 + ", new VarInfo=" 2593 + newName, 2594 state); 2595 } 2596 } 2597 } 2598 2599 /** 2600 * Converts the declaration record version of a name into its correct version. In the declaration 2601 * record, blanks are encoded as \_ and backslashes as \\. 2602 */ 2603 private static String unescape_decl(String orig) { 2604 StringBuilder sb = new StringBuilder(orig.length()); 2605 // The previous escape character was seen just before this position. 2606 int post_esc = 0; 2607 int this_esc = orig.indexOf('\\'); 2608 while (this_esc != -1) { 2609 if (this_esc == orig.length() - 1) { 2610 sb.append(orig.substring(post_esc, this_esc + 1)); 2611 post_esc = this_esc + 1; 2612 break; 2613 } 2614 switch (orig.charAt(this_esc + 1)) { 2615 case 'n': 2616 sb.append(orig.substring(post_esc, this_esc)); 2617 sb.append('\n'); // not lineSep 2618 post_esc = this_esc + 2; 2619 break; 2620 case 'r': 2621 sb.append(orig.substring(post_esc, this_esc)); 2622 sb.append('\r'); 2623 post_esc = this_esc + 2; 2624 break; 2625 case '_': 2626 sb.append(orig.substring(post_esc, this_esc)); 2627 sb.append(' '); 2628 post_esc = this_esc + 2; 2629 break; 2630 case '\\': 2631 // This is not in the default case because the search would find 2632 // the quoted backslash. Here we incluce the first backslash in 2633 // the output, but not the first. 2634 sb.append(orig.substring(post_esc, this_esc + 1)); 2635 post_esc = this_esc + 2; 2636 break; 2637 2638 default: 2639 // In the default case, retain the character following the 2640 // backslash, but discard the backslash itself. "\*" is just 2641 // a one-character string. 2642 sb.append(orig.substring(post_esc, this_esc)); 2643 post_esc = this_esc + 1; 2644 break; 2645 } 2646 this_esc = orig.indexOf('\\', post_esc); 2647 } 2648 if (post_esc == 0) { 2649 return orig; 2650 } 2651 sb.append(orig.substring(post_esc)); 2652 return sb.toString(); 2653 } 2654 2655 // The reverse of unescape_decl. Test them together. 2656 /** 2657 * Converts a name into its declaration record version. In the declaration record, blanks are 2658 * encoded as \_ and backslashes as \\. 2659 * 2660 * @param orig the name of a declaration 2661 * @return the representation of the name in a declaration file 2662 */ 2663 @SuppressWarnings("UnusedMethod") 2664 private static String escape_decl(String orig) { 2665 return orig.replace("\\", "\\\\") 2666 .replace(" ", "\\_") 2667 .replace("\n", "\\n") // not lineSep 2668 .replace("\r", "\\r"); 2669 } 2670 2671 /** 2672 * Class that holds information from the declaration record (in the file). Once collected, this 2673 * information is used to create a VarInfo. This class is necessary because a VarInfo cannot be 2674 * created until much of this information is present: the constructor requires all the information 2675 * at the time of construction, and some of the fields are final. 2676 * 2677 * <p>In general, each field has a one-to-one relation with the corresponding entry in the 2678 * variable definition block in the trace file. More detailed information about each of the fields 2679 * can be found in the 'Variable declarations' section of the 'File Formats' appendix of the 2680 * Daikon developers manual. Specifics can also be found in the 'parse_[field]' methods of the 2681 * class (eg, parse_var_kind, parse_enclosing_var_name, etc). 2682 */ 2683 @SuppressWarnings( 2684 "nullness") // undocumented class needs documentation before annotating with nullness 2685 public static class VarDefinition implements java.io.Serializable, Cloneable { 2686 static final long serialVersionUID = 20060524L; 2687 2688 /** Current information about input file and previously parsed values. */ 2689 transient ParseState state; 2690 2691 /** Name of the variable (required). */ 2692 public String name; 2693 2694 /** Type of the variable (required). */ 2695 public VarKind kind = null; 2696 2697 /** Name of variable that contains this variable (optional) */ 2698 // seems non-null for arrays/sequences 2699 public @Nullable String enclosing_var_name; 2700 2701 /** the simple (not fully specified) name of this variable (optional) */ 2702 public @Nullable String relative_name = null; 2703 2704 /** Type of reference for structure/class variables. */ 2705 public RefType ref_type = RefType.POINTER; 2706 2707 /** Number of array dimensions (0 or 1). */ 2708 public int arr_dims = 0; 2709 2710 /** 2711 * Non-null iff (vardef.kind == VarKind.FUNCTION). The arguments that were used to create this 2712 * function application. 2713 */ 2714 @SuppressWarnings("serial") 2715 public @Nullable List<String> function_args = null; 2716 2717 /** The type of the variable as stored in the dtrace file (required) */ 2718 public ProglangType rep_type = null; 2719 2720 /** Declared type of the variable as an arbitrary string (required) */ 2721 public ProglangType declared_type = null; 2722 2723 /** Variable flags (optional) */ 2724 public EnumSet<VarFlags> flags = EnumSet.noneOf(VarFlags.class); 2725 2726 /** Language specific variable flags (optional) */ 2727 public EnumSet<LangFlags> lang_flags = EnumSet.noneOf(LangFlags.class); 2728 2729 /** Comparability of this variable (required. */ 2730 @SuppressWarnings("serial") 2731 public VarComparability comparability = null; 2732 2733 /** Parent program points in ppt hierarchy (optional) */ 2734 @SuppressWarnings("serial") 2735 public List<VarParent> parents; 2736 2737 /** Non-null if this 'variable' always has the same value (optional) */ 2738 @SuppressWarnings("serial") 2739 public @Nullable @Interned Object static_constant_value = null; 2740 2741 /** 2742 * Non-null if it is statically known that the value of the variable will be always greater than 2743 * or equal to this value. 2744 */ 2745 public @Nullable String min_value = null; 2746 2747 /** 2748 * Non-null if it is statically known that the value of the variable will be always less than or 2749 * equal to this value. 2750 */ 2751 public @Nullable String max_value = null; 2752 2753 /** Non-null if it is statically known that the array will have at least this many elements. */ 2754 public @Nullable Integer min_length = null; 2755 2756 /** Non-null if it is statically known that the array will have up to this many elements. */ 2757 public @Nullable Integer max_length = null; 2758 2759 /** Non-null if the set of valid values for the variable is statically known. */ 2760 public @Nullable String valid_values = null; 2761 2762 /** Check representation invariants. */ 2763 public void checkRep() { 2764 2765 // Basic checking for sensible input 2766 assert name != null; 2767 if (kind == null) { 2768 throw new AssertionError("missing var-kind information for variable " + name); 2769 } 2770 assert (arr_dims == 0) || (arr_dims == 1) 2771 : String.format( 2772 "array dimensions==%s, should be 0 or 1, for variable %s", arr_dims, name); 2773 assert !rep_type.isArray() || arr_dims == 1 2774 : String.format("array dimensions is 0, should be 1, for variable %s", name); 2775 if (rep_type == null) { 2776 throw new AssertionError("missing rep-type information for variable " + name); 2777 } 2778 if (declared_type == null) { 2779 throw new AssertionError("missing dec-type information for variable " + name); 2780 } 2781 if (comparability == null) { 2782 throw new AssertionError("missing comparability information for variable " + name); 2783 } 2784 assert (kind == VarKind.FUNCTION) || (function_args == null) 2785 : String.format( 2786 "incompatible kind=%s and function_args=%s for VarDefinition %s", 2787 kind, function_args, name); 2788 if ((kind == VarKind.FIELD || kind == VarKind.ARRAY) && enclosing_var_name == null) { 2789 throw new AssertionError("enclosing-var not specified for variable " + name); 2790 } 2791 } 2792 2793 /** Initialize from the 'variable <em>name</em>' record. Scanner should be pointing at name. */ 2794 public VarDefinition(ParseState state, Scanner scanner) { 2795 this.state = state; 2796 this.parents = new ArrayList<VarParent>(); 2797 name = need(scanner, "name"); 2798 need_eol(scanner); 2799 if (state.varcomp_format == VarComparability.IMPLICIT) { 2800 comparability = VarComparabilityImplicit.unknown; 2801 } else { 2802 comparability = VarComparabilityNone.it; 2803 } 2804 } 2805 2806 public VarDefinition(String name, VarKind kind, ProglangType type) { 2807 this.state = null; 2808 this.parents = new ArrayList<VarParent>(); 2809 this.name = name; 2810 this.kind = kind; 2811 this.rep_type = type; 2812 this.declared_type = type; 2813 comparability = VarComparabilityNone.it; 2814 } 2815 2816 @SideEffectFree 2817 @Override 2818 public VarDefinition clone(@GuardSatisfied VarDefinition this) { 2819 try { 2820 return (VarDefinition) super.clone(); 2821 } catch (CloneNotSupportedException e) { 2822 throw new Error("This can't happen: ", e); 2823 } 2824 } 2825 2826 public VarDefinition copy() { 2827 try { 2828 VarDefinition copy = this.clone(); 2829 copy.flags = flags.clone(); 2830 copy.lang_flags = lang_flags.clone(); 2831 return copy; 2832 } catch (Throwable t) { 2833 throw new RuntimeException(t); 2834 } 2835 } 2836 2837 /** Restore interned strings. */ 2838 private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { 2839 in.defaultReadObject(); 2840 name = name.intern(); 2841 if (enclosing_var_name != null) { 2842 enclosing_var_name = enclosing_var_name.intern(); 2843 } 2844 if (relative_name != null) { 2845 relative_name = relative_name.intern(); 2846 } 2847 for (VarParent parent : parents) { 2848 parent.parent_ppt = parent.parent_ppt.intern(); 2849 if (parent.parent_variable != null) { 2850 parent.parent_variable = parent.parent_variable.intern(); 2851 } 2852 } 2853 } 2854 2855 /** Clears the parent relations, if any existed. */ 2856 public void clear_parent_relation() { 2857 parents.clear(); 2858 } 2859 2860 /** Parse a var-kind record. Scanner should be pointing at the variable kind. */ 2861 public void parse_var_kind(Scanner scanner) { 2862 VarKind kind_local = parse_enum_val(scanner, VarKind.class, "variable kind"); 2863 kind = kind_local; 2864 2865 if ((kind == VarKind.FIELD) || (kind == VarKind.FUNCTION)) { 2866 relative_name = need(scanner, "relative name"); 2867 } 2868 need_eol(scanner); 2869 } 2870 2871 /** Parses the enclosing-var record. */ 2872 public void parse_enclosing_var_name(Scanner scanner) { 2873 enclosing_var_name = need(scanner, "enclosing variable name"); 2874 need_eol(scanner); 2875 } 2876 2877 /** Parses the reference-type record. */ 2878 public void parse_reference_type(Scanner scanner) { 2879 RefType ref_type_local = parse_enum_val(scanner, RefType.class, "reference type"); 2880 ref_type = ref_type_local; 2881 need_eol(scanner); 2882 } 2883 2884 /** Parses the array record. */ 2885 public void parse_array(Scanner scanner) { 2886 @Interned String arr_str = need(scanner, "array dimensions"); 2887 if (arr_str == "0") { // interned 2888 arr_dims = 0; 2889 } else if (arr_str == "1") { // interned 2890 arr_dims = 1; 2891 } else { 2892 decl_error(state, "%s found where 0 or 1 expected", arr_str); 2893 } 2894 } 2895 2896 /** Parses the function-args record. */ 2897 public void parse_function_args(Scanner scanner) { 2898 2899 function_args = new ArrayList<String>(); 2900 while (scanner.hasNext()) { 2901 function_args.add(unescape_decl(scanner.next()).intern()); 2902 } 2903 } 2904 2905 public void parse_rep_type(Scanner scanner) { 2906 @Interned String rep_type_str = need(scanner, "rep type"); 2907 need_eol(scanner); 2908 rep_type = ProglangType.rep_parse(rep_type_str); 2909 } 2910 2911 public void parse_dec_type(Scanner scanner) { 2912 @Interned String declared_type_str = need(scanner, "declaration type"); 2913 need_eol(scanner); 2914 declared_type = ProglangType.parse(declared_type_str); 2915 } 2916 2917 /** Parse the flags record. Multiple flags can be specified. */ 2918 public void parse_flags(Scanner scanner) { 2919 2920 flags.add(parse_enum_val(scanner, VarFlags.class, "Flag")); 2921 while (scanner.hasNext()) flags.add(parse_enum_val(scanner, VarFlags.class, "Flag")); 2922 // System.out.printf("flags for %s are %s%n", name, flags); 2923 } 2924 2925 /** Parse the langauge specific flags record. Multiple flags can be specified. */ 2926 public void parse_lang_flags(Scanner scanner) { 2927 2928 lang_flags.add(parse_enum_val(scanner, LangFlags.class, "Language Specific Flag")); 2929 while (scanner.hasNext()) { 2930 lang_flags.add(parse_enum_val(scanner, LangFlags.class, "Language Specific Flag")); 2931 } 2932 } 2933 2934 /** Parses a comparability record. */ 2935 public void parse_comparability(Scanner scanner) { 2936 @Interned String comparability_str = need(scanner, "comparability"); 2937 need_eol(scanner); 2938 comparability = 2939 VarComparability.parse(state.varcomp_format, comparability_str, declared_type); 2940 } 2941 2942 /** Parse a parent ppt record. */ 2943 public void parse_parent(Scanner scanner, List<ParentRelation> ppt_parents) 2944 throws Daikon.ParseError { 2945 2946 String parent_ppt = need(scanner, "parent ppt"); 2947 String parent_relation_id_string = need(scanner, "parent id"); 2948 int parent_relation_id; 2949 try { 2950 parent_relation_id = Integer.parseInt(parent_relation_id_string); 2951 } catch (NumberFormatException nfe) { 2952 throw new Daikon.ParseError("Expected a number, found: " + parent_relation_id_string); 2953 } 2954 String parent_variable = null; 2955 2956 boolean found = false; 2957 for (ParentRelation pr : ppt_parents) { 2958 if ((pr.parent_ppt_name == parent_ppt) && (pr.id == parent_relation_id)) { 2959 found = true; 2960 break; 2961 } 2962 } 2963 if (!found) { 2964 decl_error( 2965 state, 2966 "specified parent ppt '%s[%d]' for variable '%s' is not a parent to this ppt", 2967 parent_ppt, 2968 parent_relation_id, 2969 name); 2970 } 2971 if (scanner.hasNext()) { 2972 parent_variable = need(scanner, "parent variable"); 2973 } 2974 2975 parents.add(new VarParent(parent_ppt, parent_relation_id, parent_variable)); 2976 2977 need_eol(scanner); 2978 } 2979 2980 /** Parse a constant record. */ 2981 public void parse_constant(Scanner scanner) { 2982 @Interned String constant_str = need(scanner, "constant value"); 2983 need_eol(scanner); 2984 try { 2985 static_constant_value = rep_type.parse_value(constant_str, null, "parse_constant"); 2986 } catch (Error e) { 2987 decl_error(state, e); 2988 } 2989 } 2990 2991 /** Parse a minimum value record. */ 2992 public void parse_min_value(Scanner scanner) { 2993 this.min_value = need(scanner, "minimum value"); 2994 need_eol(scanner); 2995 } 2996 2997 /** Parse a maximum value record. */ 2998 public void parse_max_value(Scanner scanner) { 2999 this.max_value = need(scanner, "maximum value"); 3000 need_eol(scanner); 3001 } 3002 3003 /** Parse a minimum length record. */ 3004 public void parse_min_length(Scanner scanner) { 3005 this.min_length = Integer.parseInt(need(scanner, "minimum length")); 3006 need_eol(scanner); 3007 } 3008 3009 /** Parse a maximum length record. */ 3010 public void parse_max_length(Scanner scanner) { 3011 this.max_length = Integer.parseInt(need(scanner, "maximum length")); 3012 need_eol(scanner); 3013 } 3014 3015 /** Parse a valid values record. */ 3016 public void parse_valid_values(Scanner scanner) { 3017 this.valid_values = scanner.nextLine(); 3018 } 3019 3020 /** 3021 * Helper function, returns the next string token unescaped and interned. Throw Daikon.UserError 3022 * if there is no next token. 3023 */ 3024 public @Interned String need(Scanner scanner, String description) { 3025 return FileIO.need(state, scanner, description); 3026 } 3027 3028 /** Throws Daikon.UserError if the scanner is not at end of line */ 3029 public void need_eol(Scanner scanner) { 3030 FileIO.need_eol(state, scanner); 3031 } 3032 3033 /** 3034 * Looks up the next token as a member of enum_class. Throws Daikon.UserError if there is no 3035 * token or if it is not valid member of the class. Enums are presumed to be in in upper case. 3036 */ 3037 public <E extends Enum<E>> E parse_enum_val( 3038 Scanner scanner, Class<E> enum_class, String descr) { 3039 return FileIO.parse_enum_val(state, scanner, enum_class, descr); 3040 } 3041 } 3042 3043 /** 3044 * Helper function, returns the next string token unescaped and interned. Throws Daikon.UserError 3045 * if there is no next token. 3046 */ 3047 public static @Interned String need(ParseState state, Scanner scanner, String description) { 3048 if (!scanner.hasNext()) { 3049 decl_error(state, "end-of-line found where %s expected", description); 3050 } 3051 return unescape_decl(scanner.next()).intern(); 3052 } 3053 3054 /** Throws a Daikon.UserError if the scanner is not at end of line */ 3055 public static void need_eol(ParseState state, Scanner scanner) { 3056 if (scanner.hasNext()) { 3057 decl_error(state, "'%s' found where end-of-line expected", scanner.next()); 3058 } 3059 } 3060 3061 /** 3062 * Looks up the next token as a member of enum_class. Throws Daikon.UserError if there is no token 3063 * or if it is not valid member of the class. Enums are presumed to be in in upper case. 3064 */ 3065 public static <E extends Enum<E>> E parse_enum_val( 3066 ParseState state, Scanner scanner, Class<E> enum_class, String descr) { 3067 3068 @Interned String str = need(state, scanner, descr); 3069 try { 3070 E e = Enum.valueOf(enum_class, str.toUpperCase()); 3071 return e; 3072 } catch (Exception exception) { 3073 @SuppressWarnings( 3074 "nullness") // getEnumConstants returns non-null because enum_class is an enum class 3075 E @NonNull [] all = enum_class.getEnumConstants(); 3076 StringJoiner msg = new StringJoiner(", "); 3077 for (E e : all) { 3078 msg.add(String.format("'%s'", e.name().toLowerCase())); 3079 } 3080 decl_error(state, "'%s' found where %s expected", str, msg); 3081 throw new Error("execution cannot get to here, previous line threw an error"); 3082 } 3083 } 3084 3085 /** 3086 * Call this to indicate a malformed declaration. 3087 * 3088 * @param state the current parse state 3089 * @param format a format string, for the error message 3090 * @param args arguments for the format string 3091 */ 3092 private static void decl_error(ParseState state, String format, @Nullable Object... args) { 3093 @SuppressWarnings({ 3094 "formatter:unneeded.suppression", // temporary? 3095 "formatter:format.string" // https://tinyurl.com/cfissue/2584 3096 }) 3097 String msg = String.format(format, args) + state.line_file_message(); 3098 throw new Daikon.UserError(msg); 3099 } 3100 3101 /** Call this to indicate a malformed declaration. */ 3102 private static void decl_error(ParseState state, Throwable cause) { 3103 String msg = cause.getMessage() + state.line_file_message(); 3104 if (msg.startsWith("null at")) { 3105 msg = msg.substring(5); 3106 } 3107 throw new Daikon.UserError(cause, msg); 3108 } 3109 3110 /** Returns whether the line is the start of a ppt declaration. */ 3111 @RequiresNonNull("FileIO.new_decl_format") 3112 @Pure 3113 private static boolean is_declaration_header(String line) { 3114 if (new_decl_format) { 3115 return line.startsWith("ppt "); 3116 } else { 3117 return line.equals(declaration_header); 3118 } 3119 } 3120 3121 /** 3122 * Handle any possible modifications to the ppt name. For now, just support the Applications 3123 * Communities specific modification to remove duplicate stack entries. But a more generic 3124 * technique could be implemented in the future. 3125 */ 3126 public static String user_mod_ppt_name(String ppt_name) { 3127 3128 if (!dkconfig_rm_stack_dups) { 3129 return ppt_name; 3130 } 3131 3132 // System.out.printf("removing stack dups (%b)in fileio%n", 3133 // dkconfig_rm_stack_dups); 3134 3135 String[] stack = ppt_name.split("[|]"); 3136 List<String> nd_stack = new ArrayList<>(); 3137 for (String si : stack) { 3138 if (nd_stack.contains(si)) { 3139 continue; 3140 } 3141 nd_stack.add(si); 3142 } 3143 return String.join("|", nd_stack).intern(); 3144 } 3145}