001package daikon; 002 003import static daikon.PptRelation.PptRelationType; 004import static daikon.PptTopLevel.PptFlags; 005import static daikon.PptTopLevel.PptType; 006import static daikon.VarInfo.LangFlags; 007import static daikon.VarInfo.RefType; 008import static daikon.VarInfo.VarFlags; 009import static daikon.VarInfo.VarKind; 010import static daikon.tools.nullness.NullnessUtil.castNonNullDeep; 011import static java.nio.charset.StandardCharsets.UTF_8; 012 013import daikon.Daikon.BugInDaikon; 014import daikon.config.Configuration; 015import daikon.derive.ValueAndModified; 016import daikon.diff.InvMap; 017import daikon.inv.Invariant; 018import java.io.BufferedReader; 019import java.io.BufferedWriter; 020import java.io.Closeable; 021import java.io.EOFException; 022import java.io.File; 023import java.io.IOException; 024import java.io.InputStream; 025import java.io.InputStreamReader; 026import java.io.InvalidClassException; 027import java.io.LineNumberReader; 028import java.io.ObjectInputStream; 029import java.io.PrintWriter; 030import java.io.Reader; 031import java.io.Serializable; 032import java.io.StringWriter; 033import java.io.UncheckedIOException; 034import java.net.ServerSocket; 035import java.net.Socket; 036import java.net.URI; 037import java.net.URL; 038import java.nio.file.Files; 039import java.nio.file.Path; 040import java.text.NumberFormat; 041import java.util.ArrayDeque; 042import java.util.ArrayList; 043import java.util.Arrays; 044import java.util.Collection; 045import java.util.Deque; 046import java.util.EnumSet; 047import java.util.HashMap; 048import java.util.LinkedHashMap; 049import java.util.List; 050import java.util.Map; 051import java.util.Scanner; 052import java.util.StringJoiner; 053import java.util.logging.Level; 054import java.util.logging.Logger; 055import java.util.zip.GZIPInputStream; 056import org.checkerframework.checker.calledmethods.qual.EnsuresCalledMethods; 057import org.checkerframework.checker.interning.qual.Interned; 058import org.checkerframework.checker.interning.qual.UsesObjectEquals; 059import org.checkerframework.checker.lock.qual.GuardSatisfied; 060import org.checkerframework.checker.mustcall.qual.MustCall; 061import org.checkerframework.checker.mustcall.qual.Owning; 062import org.checkerframework.checker.nullness.qual.EnsuresNonNull; 063import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; 064import org.checkerframework.checker.nullness.qual.KeyFor; 065import org.checkerframework.checker.nullness.qual.MonotonicNonNull; 066import org.checkerframework.checker.nullness.qual.NonNull; 067import org.checkerframework.checker.nullness.qual.Nullable; 068import org.checkerframework.checker.nullness.qual.RequiresNonNull; 069import org.checkerframework.dataflow.qual.Pure; 070import org.checkerframework.dataflow.qual.SideEffectFree; 071import org.plumelib.util.CollectionsPlume; 072import org.plumelib.util.FilesPlume; 073import org.plumelib.util.StringsPlume; 074 075/** File I/O utilities. */ 076public final class FileIO { 077 078 /** Nobody should ever instantiate a FileIO. */ 079 private FileIO() { 080 throw new Error(); 081 } 082 083 /// Constants 084 085 static final String declaration_header = "DECLARE"; 086 087 // Program point name tags 088 /** String used to append a ppt type to a ppt name. */ 089 public static final String ppt_tag_separator = ":::"; 090 091 /** String used to identify entry ppt names. */ 092 public static final String enter_suffix = "ENTER"; 093 094 /** String used to mark entry ppt names. */ 095 public static final String enter_tag = ppt_tag_separator + enter_suffix; 096 097 // EXIT does not necessarily appear at the end of the program point name; 098 // a number may follow it. 099 /** String used to identify exit ppt names. */ 100 public static final String exit_suffix = "EXIT"; 101 102 /** String used to mark exit ppt names. */ 103 public static final String exit_tag = ppt_tag_separator + exit_suffix; 104 105 /** To be deleted. */ 106 public static final String throws_suffix = "THROWS"; 107 108 /** To be deleted. */ 109 public static final String throws_tag = ppt_tag_separator + throws_suffix; 110 111 public static final String object_suffix = "OBJECT"; 112 113 /** String used to mark object ppt names. */ 114 public static final String object_tag = ppt_tag_separator + object_suffix; 115 116 /** String used to identify class ppt names. */ 117 public static final String class_static_suffix = "CLASS"; 118 119 /** String used to mark class ppt names. */ 120 public static final String class_static_tag = ppt_tag_separator + class_static_suffix; 121 122 /** String used to identify global ppt names. */ 123 public static final String global_suffix = "GLOBAL"; 124 125 private static final String lineSep = Global.lineSep; 126 127 /// Settings 128 129 // Variables starting with dkconfig_ should only be set via the 130 // daikon.config.Configuration interface. 131 132 /** 133 * When true, just ignore exit ppts that don't have a matching enter ppt rather than exiting with 134 * an error. Unmatched exits can occur if only a portion of a dtrace file is processed. 135 */ 136 public static boolean dkconfig_ignore_missing_enter = false; 137 138 /** 139 * Boolean. When false, set modbits to 1 iff the printed representation has changed. When true, 140 * set modbits to 1 if the printed representation has changed; leave other modbits as is. 141 */ 142 public static boolean dkconfig_add_changed = true; 143 144 /** Integer. Maximum number of lines to read from the dtrace file. If 0, reads the entire file. */ 145 public static int dkconfig_max_line_number = 0; 146 147 /** 148 * Boolean. When false, don't count the number of lines in the dtrace file before reading. This 149 * will disable the percentage progress printout. 150 */ 151 public static boolean dkconfig_count_lines = true; 152 153 /** 154 * Boolean. When true, only read the samples, but don't process them. Used to gather timing 155 * information. 156 */ 157 public static boolean dkconfig_read_samples_only = false; 158 159 /** 160 * Boolean. When true, don't print a warning about unmatched procedure entries, which are ignored 161 * by Daikon (unless the {@code --nohierarchy} command-line argument is provided). 162 */ 163 public static boolean dkconfig_unmatched_procedure_entries_quiet = false; 164 165 /** Boolean. If true, prints the unmatched procedure entries verbosely. */ 166 public static boolean dkconfig_verbose_unmatched_procedure_entries = false; 167 168 /** 169 * Boolean. When true, suppress exceptions related to file reading. This permits Daikon to 170 * continue even if there is a malformed trace file. Use this with care: in general, it is better 171 * to fix the problem that caused a bad trace file, rather than to suppress the exception. 172 */ 173 public static boolean dkconfig_continue_after_file_exception = false; 174 175 /** 176 * Long integer. If non-zero, this value will be used as the number of lines in (each) dtrace file 177 * input for the purposes of the progress display, and the counting of the lines in the file will 178 * be suppressed. 179 */ 180 public static long dkconfig_dtrace_line_count = 0; 181 182 /** True if declaration records are in the new format -- that is, decl-version 2.0. */ 183 // Set by read_decl_version; by read_data_trace_record if the file is non-empty; 184 // by read_serialized_pptmap; and by InvMap.readObject. 185 public static @MonotonicNonNull Boolean new_decl_format = null; 186 187 /** 188 * Do not use this routine unless you know what you are doing. This routine breaks the 189 * representation invariant that new_decl_format, once set, is never reset to null. This routine 190 * should be used only if you can guarantee that new_decl_format will be once again set to a 191 * non-null value before any code runs that depends on the fact that new_decl_format is non-null. 192 */ 193 @SuppressWarnings("nullness") // reinitialization 194 public static void resetNewDeclFormat() { 195 FileIO.new_decl_format = null; 196 } 197 198 /** 199 * If true, modified all ppt names to remove duplicate routine names within the ppt name. This is 200 * used when a stack trace (of active methods) is used as the ppt name. The routine names must be 201 * separated by vertical bars (|). 202 */ 203 public static boolean dkconfig_rm_stack_dups = false; 204 205 /// Variables 206 207 // This hashmap maps every program point to an array, which contains the 208 // old values of all variables in scope the last time the program point 209 // was executed. This enables us to determine whether the values have been 210 // modified since this program point was last executed. 211 static HashMap<PptTopLevel, String[]> ppt_to_value_reps = new HashMap<>(); 212 213 // For debugging purposes: printing out a modified trace file with 214 // changed modbits. 215 private static boolean to_write_nonce = false; 216 private static final String NONCE_HEADER = "this_invocation_nonce"; 217 private static String nonce_value = "no nonce (yet)"; 218 219 // (This implementation as a public static variable is a bit unclean.) 220 // Number of ignored declarations. 221 public static int omitted_declarations = 0; 222 223 // Logging Categories 224 225 /** 226 * If true, then print the variable name each time the variable's value is first 227 * missing/nonsensical. 228 */ 229 public static boolean debug_missing = false; 230 231 /** Debug tracer for reading. */ 232 public static final Logger debugRead = Logger.getLogger("daikon.FileIO.read"); 233 234 /** Debug tracer for printing. */ 235 public static final Logger debugPrint = Logger.getLogger("daikon.FileIO.printDtrace"); 236 237 /** Debug tracer for printing variable values. */ 238 public static final Logger debugVars = Logger.getLogger("daikon.FileIO.vars"); 239 240 // public static final SimpleLog debug_decl = new SimpleLog(false); 241 242 /** Parents in the ppt/variable hierarchy for a particular program point. */ 243 public static final class ParentRelation implements java.io.Serializable { 244 static final long serialVersionUID = 20060622L; 245 public PptRelationType rel_type; 246 public @Interned String parent_ppt_name; 247 public int id; 248 249 public ParentRelation(PptRelationType rel_type, @Interned String parent_ppt_name, int id) { 250 this.rel_type = rel_type; 251 this.parent_ppt_name = parent_ppt_name; 252 this.id = id; 253 } 254 255 @SideEffectFree 256 @Override 257 public String toString(@GuardSatisfied ParentRelation this) { 258 return parent_ppt_name + "[" + id + "] " + rel_type; 259 } 260 261 /** 262 * Intern the ppt name. 263 * 264 * @param in the input stream from which to read the object 265 * @throws IOException if there is a problem reading the stream 266 * @throws ClassNotFoundException if a class cannot be loaded 267 */ 268 private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { 269 in.defaultReadObject(); 270 if (parent_ppt_name != null) { 271 parent_ppt_name = parent_ppt_name.intern(); 272 } 273 } 274 } 275 276 // Utilities 277 @EnsuresNonNullIf(result = true, expression = "#1") 278 @Pure 279 public static final boolean isComment(@Nullable String s) { 280 return s != null && (s.startsWith("//") || s.startsWith("#")); 281 } 282 283 /** 284 * Returns true if the next line is a comment. 285 * 286 * @param reader the reader whose next line to check 287 * @return true if the next line is a comment 288 */ 289 // Nullness-checking of read_data_trace_record(ParseState) works even 290 // without these two lines, since StringJoiner accepts null values. 291 @SuppressWarnings( 292 "nullness:contracts.conditional.postcondition" // readLine() assertion is ensured by call to 293 // reset() 294 ) 295 @EnsuresNonNullIf(result = true, expression = "#1.readLine()") 296 public static final boolean nextLineIsComment(BufferedReader reader) { 297 boolean result = false; 298 try { 299 reader.mark(10000); 300 String nextline = reader.readLine(); 301 result = isComment(nextline); 302 } catch (IOException e) { 303 result = false; 304 } 305 try { 306 reader.reset(); 307 } catch (IOException e) { 308 throw new UncheckedIOException(e); 309 } 310 return result; 311 } 312 313 /////////////////////////////////////////////////////////////////////////// 314 /// Declaration files 315 /// 316 317 /** 318 * Returns a new PptMap containing declarations read from the files listed in the argument; 319 * connection information (controlling variables and entry ppts) is set correctly upon return. 320 * 321 * @param files files to be read (java.io.File) 322 * @return a new PptMap containing declarations read from the files listed in the argument 323 */ 324 public static PptMap read_declaration_files(Collection<File> files) throws IOException { 325 PptMap all_ppts = new PptMap(); 326 // Read all decls, creating PptTopLevels and VarInfos 327 for (File file : files) { 328 Daikon.progress = "Reading " + file; 329 if (!Daikon.dkconfig_quiet) { 330 System.out.print("."); // show progress 331 } 332 read_declaration_file(file, all_ppts); 333 } 334 return all_ppts; 335 } 336 337 /** Read one decls file; add it to all_ppts. */ 338 public static void read_declaration_file(File filename, PptMap all_ppts) throws IOException { 339 if (Daikon.using_DaikonSimple) { 340 Processor processor = new DaikonSimple.SimpleProcessor(); 341 read_data_trace_file(filename.toString(), all_ppts, processor, true, false); 342 } else { 343 Processor processor = new Processor(); 344 read_data_trace_file(filename.toString(), all_ppts, processor, true, true); 345 } 346 } 347 348 // Read a declaration in the Version 2 format. For Version 1, see 349 // read_declaration. 350 /** 351 * Reads one ppt declaration. The next line should be the ppt record. After completion, the file 352 * pointer will be pointing at the next record (ie, the blank line at the end of the ppt 353 * declaration will have been read in). Returns null if the ppt is excluded/omitted from this 354 * execution of Daikon. 355 */ 356 private static @Nullable PptTopLevel read_ppt_decl(ParseState state, String top_line) 357 throws IOException { 358 359 // process the ppt record 360 String line = top_line; 361 Scanner scanner = new Scanner(line); 362 @Interned String record_name = need(state, scanner, "'ppt'"); 363 if (record_name != "ppt") { // interned 364 decl_error(state, "found '%s' where 'ppt' expected", record_name); 365 } 366 String ppt_name = need(state, scanner, "ppt name"); 367 ppt_name = user_mod_ppt_name(ppt_name); 368 369 /** Information that will populate the new program point. */ 370 Map<String, VarDefinition> varmap = new LinkedHashMap<>(); 371 /** The VarDefinition we are in the middle of reading, or null if we are not. */ 372 VarDefinition vardef = null; 373 List<ParentRelation> ppt_parents = new ArrayList<>(); 374 EnumSet<PptFlags> ppt_flags = EnumSet.noneOf(PptFlags.class); 375 PptType ppt_type = PptType.POINT; 376 377 try { 378 // Read the records that define this program point 379 while ((line = state.reader.readLine()) != null) { 380 // debug_decl.log("read line %s%n", line); 381 line = line.trim(); 382 if (line.length() == 0) { 383 break; 384 } 385 386 scanner = new Scanner(line); 387 @Interned String record = scanner.next().intern(); 388 if (vardef == null) { 389 if (record == "parent") { // interned 390 ppt_parents.add(parse_ppt_parent(state, scanner)); 391 } else if (record == "flags") { // interned 392 parse_ppt_flags(state, scanner, ppt_flags); 393 } else if (record == "variable") { // interned 394 vardef = new VarDefinition(state, scanner); 395 // There is no need to check "varmap.containsKey(vardef.name)" 396 // because this is the first variable. 397 assert varmap.isEmpty(); 398 if (var_included(vardef.name)) { 399 varmap.put(vardef.name, vardef); 400 } 401 } else if (record == "ppt-type") { // interned 402 ppt_type = parse_ppt_type(state, scanner); 403 } else { 404 decl_error(state, "record '%s' found where %s expected", record, "'parent', 'flags'"); 405 } 406 } else { // there must be a current variable 407 if (record == "var-kind") { // interned 408 vardef.parse_var_kind(scanner); 409 } else if (record == "enclosing-var") { // interned 410 vardef.parse_enclosing_var_name(scanner); 411 } else if (record == "reference-type") { // interned 412 vardef.parse_reference_type(scanner); 413 } else if (record == "array") { // interned 414 vardef.parse_array(scanner); 415 } else if (record == "function-args") { // interned 416 vardef.parse_function_args(scanner); 417 } else if (record == "rep-type") { // interned 418 vardef.parse_rep_type(scanner); 419 } else if (record == "dec-type") { // interned 420 vardef.parse_dec_type(scanner); 421 } else if (record == "flags") { // interned 422 vardef.parse_flags(scanner); 423 } else if (record == "lang-flags") { // interned 424 vardef.parse_lang_flags(scanner); 425 } else if (record == "parent") { // interned 426 vardef.parse_parent(scanner, ppt_parents); 427 } else if (record == "comparability") { // interned 428 vardef.parse_comparability(scanner); 429 } else if (record == "constant") { // interned 430 vardef.parse_constant(scanner); 431 } else if (record == "variable") { // interned 432 try { 433 vardef.checkRep(); // make sure the previous variable is ok 434 } catch (AssertionError e) { 435 decl_error(state, e); 436 } 437 vardef = new VarDefinition(state, scanner); 438 if (varmap.containsKey(vardef.name)) { 439 decl_error(state, "var %s declared twice", vardef.name); 440 } 441 if (var_included(vardef.name)) { 442 varmap.put(vardef.name, vardef); 443 } 444 } else if (record == "min-value") { // interned 445 vardef.parse_min_value(scanner); 446 } else if (record == "max-value") { // interned 447 vardef.parse_max_value(scanner); 448 } else if (record == "min-length") { // interned 449 vardef.parse_min_length(scanner); 450 } else if (record == "max-length") { // interned 451 vardef.parse_max_length(scanner); 452 } else if (record == "valid-values") { // interned 453 vardef.parse_valid_values(scanner); 454 } else { 455 decl_error(state, "Unexpected variable item '%s' found", record); 456 } 457 } 458 } 459 } catch (Daikon.ParseError pe) { 460 decl_error(state, "%s", pe.getMessage()); 461 throw new Error(); // this can't happen 462 } 463 if (vardef != null) { 464 try { 465 vardef.checkRep(); 466 } catch (AssertionError e) { 467 decl_error(state, e); 468 } 469 } 470 471 // If we are excluding this ppt, just read the data and throw it away 472 if (!ppt_included(ppt_name)) { 473 omitted_declarations++; 474 return null; 475 } 476 477 // Build the var infos from the var definitions. 478 List<VarInfo> vi_list = new ArrayList<>(varmap.size()); 479 for (VarDefinition vd : varmap.values()) { 480 @SuppressWarnings("interning") // about to be used in a new program point 481 @Interned VarInfo vi = new VarInfo(vd); 482 vi_list.add(vi); 483 } 484 VarInfo[] vi_array = vi_list.toArray(new VarInfo[0]); 485 486 // Check to see if the program point is new 487 if (state.all_ppts.containsName(ppt_name)) { 488 @NonNull PptTopLevel existing_ppt = state.all_ppts.get(ppt_name); 489 assert existing_ppt != null : "state.all_ppts.containsName(" + ppt_name + ")"; 490 if (state.ppts_may_be_new) { 491 check_decl_match(state, existing_ppt, vi_array); 492 } else { // ppts are already in the map 493 if (VarInfo.assertionsEnabled()) { 494 for (VarInfo vi : vi_array) { 495 vi.checkRep(); 496 } 497 } 498 return existing_ppt; 499 } 500 } 501 502 // Build the program point 503 PptTopLevel newppt = new PptTopLevel(ppt_name, ppt_type, ppt_parents, ppt_flags, vi_array); 504 505 return newppt; 506 } 507 508 /** Parses a ppt parent hierarchy record and returns it. */ 509 private static ParentRelation parse_ppt_parent(ParseState state, Scanner scanner) { 510 511 PptRelationType rel_type = 512 parse_enum_val(state, scanner, PptRelationType.class, "relation type"); 513 String parent_ppt_name = need(state, scanner, "ppt name"); 514 int id = Integer.parseInt(need(state, scanner, "relation id")); 515 ParentRelation pr = new ParentRelation(rel_type, parent_ppt_name, id); 516 517 need_eol(state, scanner); 518 return pr; 519 } 520 521 /** Parses a program point flag record. Adds any specified flags to to flags. */ 522 private static void parse_ppt_flags(ParseState state, Scanner scanner, EnumSet<PptFlags> flags) { 523 524 flags.add(parse_enum_val(state, scanner, PptFlags.class, "ppt flags")); 525 while (scanner.hasNext()) { 526 flags.add(parse_enum_val(state, scanner, PptFlags.class, "ppt flags")); 527 } 528 } 529 530 /** Parses a ppt-type record and returns the type. */ 531 private static PptType parse_ppt_type(ParseState state, Scanner scanner) { 532 533 PptType ppt_type = parse_enum_val(state, scanner, PptType.class, "ppt type"); 534 need_eol(state, scanner); 535 return ppt_type; 536 } 537 538 // Read a declaration in the Version 1 format. For version 2, see 539 // read_ppt_decl. 540 // The "DECLARE" line has already been read. 541 private static @Nullable PptTopLevel read_declaration(ParseState state) throws IOException { 542 543 // We have just read the "DECLARE" line. 544 String ppt_name = state.reader.readLine(); 545 if (ppt_name == null) { 546 throw new Daikon.UserError( 547 "File ends with \"DECLARE\" with no following program point name", state); 548 } 549 ppt_name = user_mod_ppt_name(ppt_name); 550 ppt_name = ppt_name.intern(); 551 VarInfo[] vi_array = read_VarInfos(state, ppt_name); 552 553 // System.out.printf("Ppt %s with %d variables%n", ppt_name, 554 // vi_array.length); 555 556 // This program point name has already been encountered. 557 if (state.all_ppts.containsName(ppt_name)) { 558 @NonNull PptTopLevel existing_ppt = state.all_ppts.get(ppt_name); 559 assert existing_ppt != null : "state.all_ppts.containsName(" + ppt_name + ")"; 560 if (state.ppts_may_be_new) { 561 check_decl_match(state, existing_ppt, vi_array); 562 } else { // ppts are already in the map 563 return existing_ppt; 564 } 565 } 566 567 // If we are excluding this ppt, just throw it away 568 if (!ppt_included(ppt_name)) { 569 omitted_declarations++; 570 return null; 571 } 572 573 // taking care of visibility information 574 // the information is needed in the variable hierarchy because private methods 575 // should not be linked under the object program point 576 // the ppt name is truncated before putting it in the pptMap because the visibility 577 // information is only present in the decls file and not the dtrace file 578 579 // if (ppt_name.startsWith("public")) { 580 // int position = ppt_name.indexOf("public"); 581 // ppt_name = ppt_name.substring(7); 582 // PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 583 // newppt.ppt_name.setVisibility("public"); 584 // return newppt; 585 // } 586 // if (ppt_name.startsWith("private")) { 587 // int position = ppt_name.indexOf("private"); 588 // ppt_name = ppt_name.substring(8); 589 // PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 590 // newppt.ppt_name.setVisibility("private"); 591 // return newppt; 592 // } 593 // if (ppt_name.startsWith("protected")) { 594 // int position = ppt_name.indexOf("protected"); 595 // ppt_name = ppt_name.substring(10); 596 // PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 597 // newppt.ppt_name.setVisibility("protected"); 598 // return newppt; 599 // } 600 601 // TODO: add a new config variable to turn this accessibility flag processing on? 602 PptTopLevel newppt = new PptTopLevel(ppt_name, vi_array); 603 // newppt.ppt_name.setVisibility("package-protected"); 604 return newppt; 605 // return new PptTopLevel(ppt_name, vi_array); 606 } 607 608 private static VarInfo[] read_VarInfos(ParseState state, String ppt_name) throws IOException { 609 610 // The var_infos that will populate the new program point 611 List<VarInfo> var_infos = new ArrayList<>(); 612 613 // Each iteration reads a variable name, type, and comparability. 614 // Possibly abstract this out into a separate function?? 615 VarInfo vi; 616 while ((vi = read_VarInfo(state, ppt_name)) != null) { 617 for (VarInfo vi2 : var_infos) { 618 if (vi.name() == vi2.name()) { 619 throw new Daikon.UserError("Duplicate variable name " + vi.name(), state); 620 } 621 } 622 // Can't do this test in read_VarInfo, it seems, because of the test 623 // against null above. 624 if (!var_included(vi.name())) { 625 continue; 626 } 627 var_infos.add(vi); 628 } 629 630 VarInfo[] result = var_infos.toArray(new VarInfo[0]); 631 return result; 632 } 633 634 // So that warning message below is only printed once 635 private static boolean seen_string_rep_type = false; 636 637 /** 638 * Read a variable name, type, and comparability; construct a VarInfo. Return null after reading 639 * the last variable in this program point declaration. 640 * 641 * <p>The resulting VarInfo does not have its ppt field set; the client should arrange to do so. 642 * 643 * @param state the parse state 644 * @param ppt_name the name of the variable's program point; used only for diagnostic messages 645 * @return a new VarInfo read from {@code state.reader} 646 * @throws IOException if there is trouble reading the file 647 */ 648 private static @Nullable VarInfo read_VarInfo(ParseState state, String ppt_name) 649 throws IOException { 650 LineNumberReader file = state.reader; 651 int varcomp_format = state.varcomp_format; 652 String filename = state.filename; 653 654 String line = file.readLine(); 655 if ((line == null) || line.equals("")) { 656 return null; 657 } 658 String varname = line; 659 String proglang_type_string_and_aux = file.readLine(); 660 String file_rep_type_string = file.readLine(); 661 String comparability_string = file.readLine(); 662 if ( // (varname == null) || // already returned null if varname==null 663 (proglang_type_string_and_aux == null) 664 || (file_rep_type_string == null) 665 || (comparability_string == null)) 666 throw new Daikon.UserError( 667 "End of file " 668 + filename 669 + " while reading variable " 670 + varname 671 + " in declaration of program point " 672 + ppt_name); 673 int equals_index = file_rep_type_string.indexOf(" = "); 674 String static_constant_value_string = null; 675 @Interned Object static_constant_value = null; 676 boolean is_static_constant = false; 677 if (equals_index != -1) { 678 is_static_constant = true; 679 static_constant_value_string = file_rep_type_string.substring(equals_index + 3); 680 file_rep_type_string = file_rep_type_string.substring(0, equals_index); 681 } 682 // XXX temporary, for compatibility with older .dtrace files. 12/20/2001 683 if ("String".equals(file_rep_type_string)) { 684 file_rep_type_string = "java.lang.String"; 685 if (!seen_string_rep_type) { 686 seen_string_rep_type = true; 687 System.err.println( 688 "Warning: Malformed trace file. Representation type 'String' should be " 689 + "'java.lang.String' instead on line " 690 + (file.getLineNumber() - 1) 691 + " of " 692 + filename); 693 } 694 } 695 // This is for people who were confused by the above temporary 696 // workaround when it didn't have a warning. But this has never 697 // worked, so it's fatal. 698 else if ("String[]".equals(file_rep_type_string)) { 699 throw new Daikon.UserError( 700 "Representation type 'String[]' should be " 701 + "'java.lang.String[]' instead for variable " 702 + varname, 703 file, 704 filename); 705 } 706 /// XXX 707 708 int hash_position = proglang_type_string_and_aux.indexOf('#'); 709 String aux_string = ""; 710 if (hash_position == -1) { 711 hash_position = proglang_type_string_and_aux.length(); 712 } else { 713 aux_string = 714 proglang_type_string_and_aux.substring( 715 hash_position + 1, proglang_type_string_and_aux.length()); 716 } 717 718 String proglang_type_string = proglang_type_string_and_aux.substring(0, hash_position).trim(); 719 720 ProglangType prog_type; 721 ProglangType file_rep_type; 722 ProglangType rep_type; 723 VarInfoAux aux; 724 try { 725 prog_type = ProglangType.parse(proglang_type_string); 726 file_rep_type = ProglangType.rep_parse(file_rep_type_string); 727 rep_type = file_rep_type.fileTypeToRepType(); 728 aux = VarInfoAux.parse(aux_string); 729 } catch (IOException e) { 730 throw new Daikon.UserError(e, file, filename); 731 } 732 733 if (static_constant_value_string != null) { 734 static_constant_value = rep_type.parse_value(static_constant_value_string, file, filename); 735 // Why can't the value be null? 736 assert static_constant_value != null; 737 } 738 VarComparability comparability = null; 739 try { 740 comparability = VarComparability.parse(varcomp_format, comparability_string, prog_type); 741 } catch (Exception e) { 742 throw new Daikon.UserError( 743 String.format( 744 "Error parsing comparability (%s) at line %d in file %s", 745 e, file.getLineNumber(), filename)); 746 } 747 if (!VarInfo.legalFileRepType(file_rep_type)) { 748 throw new Daikon.UserError( 749 "Unsupported representation type " 750 + file_rep_type.format() 751 + " (parsed as " 752 + rep_type 753 + ")" 754 + " for variable " 755 + varname, 756 file, 757 filename); 758 } 759 if (!VarInfo.legalRepType(rep_type)) { 760 throw new Daikon.UserError( 761 "Unsupported (converted) representation type " 762 + file_rep_type.format() 763 + " for variable " 764 + varname, 765 file, 766 filename); 767 } 768 // COMPARABILITY TEST 769 if (!(comparability.alwaysComparable() 770 || ((VarComparabilityImplicit) comparability).dimensions == file_rep_type.dimensions())) { 771 System.err.println(); 772 throw new Daikon.UserError( 773 "Rep type " 774 + file_rep_type.format() 775 + " has " 776 + file_rep_type.dimensions() 777 + " dimensions," 778 + " but comparability " 779 + comparability 780 + " has " 781 + ((VarComparabilityImplicit) comparability).dimensions 782 + " dimensions," 783 + " for variable " 784 + varname, 785 file, 786 filename); 787 } 788 789 @SuppressWarnings("interning") 790 @Interned VarInfo result = 791 new VarInfo( 792 varname, 793 prog_type, 794 file_rep_type, 795 comparability, 796 is_static_constant, 797 static_constant_value, 798 aux); 799 return result; 800 } 801 802 @RequiresNonNull("FileIO.new_decl_format") 803 private static int read_var_comparability(ParseState state, String line) throws IOException { 804 805 // System.out.printf("read_var_comparability, line = '%s' %b%n", line, 806 // new_decl_format); 807 String comp_str; 808 if (new_decl_format) { 809 Scanner scanner = new Scanner(line); 810 scanner.next(); 811 comp_str = need(state, scanner, "comparability"); 812 need_eol(state, scanner); 813 } else { // old format 814 comp_str = state.reader.readLine(); 815 if (comp_str == null) { 816 throw new Daikon.UserError("Found end of file, expected comparability", state); 817 } 818 } 819 820 if (comp_str.equals("none")) { 821 return VarComparability.NONE; 822 } else if (comp_str.equals("implicit")) { 823 return VarComparability.IMPLICIT; 824 } else { 825 throw new Daikon.UserError("Unrecognized VarComparability '" + comp_str + "'", state); 826 } 827 } 828 829 private static @Interned String read_input_language(ParseState state, String line) 830 throws IOException { 831 832 Scanner scanner = new Scanner(line); 833 scanner.next(); 834 @Interned String input_lang = need(state, scanner, "input language"); 835 need_eol(state, scanner); 836 return input_lang; 837 } 838 839 @EnsuresNonNull("FileIO.new_decl_format") 840 private static void read_decl_version(ParseState state, String line) throws IOException { 841 Scanner scanner = new Scanner(line); 842 scanner.next(); 843 @Interned String version = need(state, scanner, "declaration version number"); 844 need_eol(state, scanner); 845 boolean new_df; 846 if (version == "2.0") { // interned 847 new_df = true; 848 } else if (version == "1.0") { // interned 849 new_df = false; 850 } else { 851 decl_error(state, "'%s' found where 1.0 or 2.0 expected", version); 852 throw new Error("Can't get here"); // help out definite assignment analysis 853 } 854 855 // Make sure that if a format was specified previously, it is the same 856 if ((new_decl_format != null) && (new_df != new_decl_format.booleanValue())) { 857 decl_error(state, "decl format '%s' does not match previous setting", version); 858 } 859 860 // System.out.println("setting new_decl_format = " + new_df); 861 new_decl_format = Boolean.valueOf(new_df); 862 } 863 864 // Each line following is the name (in JVM form) of a class that 865 // implements java.util.List. All those lines (including interspersed 866 // comments) are returned. 867 private static String read_list_implementors(LineNumberReader reader) throws IOException { 868 StringJoiner result = new StringJoiner(lineSep); 869 for (; ; ) { 870 String line = reader.readLine(); 871 if (line == null || line.equals("")) { 872 break; 873 } 874 result.add(line); 875 if (isComment(line)) { 876 continue; 877 } 878 ProglangType.list_implementors.add(line.intern()); 879 } 880 return result.toString(); 881 } 882 883 /////////////////////////////////////////////////////////////////////////// 884 /// invocation tracking for dtrace files entry/exit grouping 885 /// 886 887 static final class Invocation implements Comparable<Invocation> { 888 PptTopLevel ppt; // used in printing and in suppressing duplicates 889 // Rather than a valuetuple, place its elements here. 890 @Nullable Object[] vals; 891 int[] mods; 892 893 static Object canonical_hashcode = new Object(); 894 895 Invocation(PptTopLevel ppt, @Nullable Object[] vals, int[] mods) { 896 this.ppt = ppt; 897 this.vals = vals; 898 this.mods = mods; 899 } 900 901 /** 902 * Return a string representation of this. The Invocation is formatted on two lines, indented by 903 * two spaces. The receiver Invocation may be canonicalized or not. 904 * 905 * @return a string representation of this 906 */ 907 String format(@GuardSatisfied Invocation this) { 908 return format(true); 909 } 910 911 /** 912 * Return a string representation of this. The Invocation is formatted on two lines, indented by 913 * two spaces. The receiver Invocation may be canonicalized or not. 914 * 915 * @param show_values if true, show values; otherwise, return just the Ppt name 916 * @return a string representation of this 917 */ 918 String format(@GuardSatisfied Invocation this, boolean show_values) { 919 if (!show_values) { 920 return " " + ppt.ppt_name.getNameWithoutPoint(); 921 } 922 923 StringWriter sw = new StringWriter(); 924 PrintWriter pw = new PrintWriter(sw); 925 926 pw.println(" " + ppt.ppt_name.getNameWithoutPoint()); 927 pw.print(" "); 928 929 // [adonovan] is this sound? Let me know if not (sorry). 930 // assert ppt.var_infos.length == vals.length; 931 932 for (int j = 0; j < vals.length; j++) { 933 if (j != 0) { 934 pw.print(", "); 935 } 936 937 pw.print(ppt.var_infos[j].name() + "="); 938 939 Object val = vals[j]; 940 if (canonical_hashcode.equals( 941 val)) // succeeds only for canonicalized Invocations. Can be an == test, but there is 942 // little point. val can be null, so it cannot be the receiver. 943 pw.print("<hashcode>"); 944 else if (val instanceof int[]) { 945 pw.print(Arrays.toString((int[]) val)); 946 } else if (val instanceof String) { 947 pw.print(StringsPlume.escapeNonASCII((String) val)); 948 } else { 949 pw.print(val); 950 } 951 } 952 pw.println(); 953 954 return sw.toString(); 955 } 956 957 /** Change uses of hashcodes to canonical_hashcode. */ 958 public @Interned Invocation canonicalize() { 959 @Nullable Object[] new_vals = new @Nullable Object[vals.length]; 960 System.arraycopy(vals, 0, new_vals, 0, vals.length); 961 VarInfo[] vis = ppt.var_infos; 962 // Warning: abstraction violation! 963 for (VarInfo vi : vis) { 964 if ((vi.value_index != -1) && (vi.file_rep_type == ProglangType.HASHCODE)) { 965 new_vals[vi.value_index] = canonical_hashcode; 966 } 967 } 968 @SuppressWarnings("interning:cast.unsafe.constructor.invocation") 969 @Interned Invocation result = new @Interned Invocation(ppt, new_vals, mods); 970 return result; 971 } 972 973 // Return true if the invocations print the same 974 @EnsuresNonNullIf(result = true, expression = "#1") 975 @Pure 976 @Override 977 public boolean equals(@GuardSatisfied Invocation this, @GuardSatisfied @Nullable Object other) { 978 if (other instanceof FileIO.Invocation) { 979 return this.format().equals(((FileIO.Invocation) other).format()); 980 } else { 981 return false; 982 } 983 } 984 985 @Pure 986 @Override 987 public int compareTo(@GuardSatisfied Invocation this, Invocation other) { 988 return ppt.name().compareTo(other.ppt.name()); 989 } 990 991 @Pure 992 @Override 993 public int hashCode(@GuardSatisfied Invocation this) { 994 return this.format().hashCode(); 995 } 996 } 997 998 // I could save some Object overhead by using two parallel stacks 999 // instead of Invocation objects; but that's not worth it. 1000 1001 // Map key is a (global, not per-procedure) nonce. 1002 // The nonce indicates which returns are associated with which entries. 1003 static HashMap<Integer, Invocation> call_hashmap = new HashMap<>(); 1004 // call_stack is for procedures without nonces. 1005 static Deque<Invocation> call_stack = new ArrayDeque<Invocation>(); 1006 1007 /** 1008 * Reads data from {@code .dtrace} files. For each record in the files, calls the appropriate 1009 * callback in the processor. 1010 * 1011 * @see #read_data_trace_files(Collection,PptMap,Processor,boolean) 1012 * @see #read_data_trace_file(String,PptMap,Processor,boolean,boolean) 1013 */ 1014 public static void read_data_trace_files(Collection<String> files, PptMap all_ppts) 1015 throws IOException { 1016 1017 Processor processor = new Processor(); 1018 read_data_trace_files(files, all_ppts, processor, true); 1019 } 1020 1021 /** 1022 * Reads data from {@code .dtrace} files. Calls {@link 1023 * #read_data_trace_file(String,PptMap,Processor,boolean,boolean)} for each element of filenames. 1024 * 1025 * @param ppts_may_be_new true if declarations of ppts read from the data trace file are new (and 1026 * thus are not in all_ppts). false if the ppts may already be there. 1027 * @see #read_data_trace_file(String,PptMap,Processor,boolean,boolean) 1028 */ 1029 public static void read_data_trace_files( 1030 Collection<String> files, PptMap all_ppts, Processor processor, boolean ppts_may_be_new) 1031 throws IOException { 1032 1033 for (String filename : files) { 1034 // System.out.printf("processing filename %s%n", filename); 1035 try { 1036 read_data_trace_file(filename, all_ppts, processor, false, ppts_may_be_new); 1037 } catch (Daikon.NormalTermination e) { 1038 throw e; 1039 } catch (Throwable e) { 1040 if (dkconfig_continue_after_file_exception) { 1041 System.out.println(); 1042 System.out.println( 1043 "WARNING: Error while processing trace file; remaining records ignored."); 1044 System.out.print("Ignored backtrace:"); 1045 e.printStackTrace(System.out); 1046 System.out.println(); 1047 } else { 1048 throw e; 1049 } 1050 } 1051 } 1052 if (Daikon.server_dir != null) { 1053 // Yoav: server mode 1054 while (true) { 1055 @SuppressWarnings( 1056 "nullness") // server_dir is a directory; this was checked when the variable was set 1057 String @NonNull [] dir_files = Daikon.server_dir.list(); 1058 Arrays.sort(dir_files); 1059 boolean hasEnd = false; 1060 for (String f : dir_files) { 1061 if (f.endsWith(".end")) { 1062 hasEnd = true; 1063 } 1064 if (f.endsWith(".end") || f.endsWith(".start")) { 1065 continue; 1066 } 1067 if (files.contains(f)) { 1068 continue; 1069 } 1070 files.add(f); 1071 System.out.println("Reading " + f); 1072 read_data_trace_file( 1073 new File(Daikon.server_dir, f).toString(), 1074 all_ppts, 1075 processor, 1076 false, 1077 ppts_may_be_new); 1078 } 1079 if (hasEnd) { 1080 break; 1081 } 1082 try { 1083 Thread.sleep(1000); 1084 } catch (java.lang.InterruptedException e) { 1085 // It's not a problem if the sleep is interrupted. 1086 } 1087 } 1088 } 1089 1090 process_unmatched_procedure_entries(); 1091 1092 warn_if_hierarchy_mismatch(all_ppts); 1093 } 1094 1095 // Determine if dataflow hierarchy should have been used, and print 1096 // warning if this does not match Daikon.use_dataflow_hierarchy. 1097 // Dataflow hierarchy should be used only when all program points 1098 // correspond to points normally found in traces from a 1099 // programming languages. 1100 private static void warn_if_hierarchy_mismatch(PptMap all_ppts) { 1101 1102 boolean some_program_points = false; 1103 boolean all_program_points = true; 1104 1105 // Go through each top level ppt, and make all_program_points 1106 // false if at least one of them is not a program point normally 1107 // found in traces from programming languages. 1108 for (PptTopLevel ppt_top_level : all_ppts.ppt_all_iterable()) { 1109 boolean is_program_point = 1110 (ppt_top_level.ppt_name.isExitPoint() 1111 || ppt_top_level.ppt_name.isEnterPoint() 1112 || ppt_top_level.ppt_name.isThrowsPoint() 1113 || ppt_top_level.ppt_name.isObjectInstanceSynthetic() 1114 || ppt_top_level.ppt_name.isClassStaticSynthetic() 1115 || ppt_top_level.ppt_name.isGlobalPoint()); 1116 1117 all_program_points = all_program_points && is_program_point; 1118 some_program_points = some_program_points || is_program_point; 1119 } 1120 1121 // If all program points correspond to a programming language, 1122 // but the dataflow hierarchy has been turned off, then 1123 // suggest not using the --nohierarchy flag. 1124 // if (all_program_points && (!Daikon.use_dataflow_hierarchy)) { 1125 // System.out.println("Warning: data trace appears to be over" + 1126 // " a program execution, but dataflow" + 1127 // " hierarchy has been turned off," + 1128 // " consider running Daikon without the" + 1129 // " --nohierarchy flag"); 1130 // } 1131 1132 // if some of the program points do not correspond to a 1133 // points from a programming language, and the dataflow 1134 // hierarchy is being used, suggest using the --nohierarchy flag. 1135 if (Daikon.use_dataflow_hierarchy && !all_program_points && some_program_points) { 1136 System.out.println( 1137 "Warning: Daikon is using a dataflow" 1138 + " hierarchy analysis on a data trace" 1139 + " that does not appear to be over a" 1140 + " program execution. Consider running" 1141 + " Daikon with the --nohierarchy flag."); 1142 } 1143 } 1144 1145 /** 1146 * Connect to Chicory. 1147 * 1148 * @return the stream that is connected to Chicory 1149 */ 1150 private static @Owning InputStream connectToChicory() { 1151 1152 // bind to any free port 1153 try (ServerSocket daikonServer = new ServerSocket(0)) { 1154 1155 // tell Chicory what port we have! 1156 System.out.println("DaikonChicoryOnlinePort=" + daikonServer.getLocalPort()); 1157 1158 daikonServer.setReceiveBufferSize(64000); 1159 1160 Socket chicSocket; 1161 try { 1162 daikonServer.setSoTimeout(5000); 1163 1164 // System.out.println("waiting for chicory connection on port " + 1165 // daikonServer.getLocalPort()); 1166 chicSocket = daikonServer.accept(); 1167 } catch (IOException e) { 1168 throw new RuntimeException("Unable to connect to Chicory", e); 1169 } 1170 1171 try { 1172 return chicSocket.getInputStream(); 1173 } catch (IOException e) { 1174 throw new RuntimeException("Unable to get Chicory's input stream", e); 1175 } 1176 } catch (IOException e) { 1177 throw new RuntimeException("Unable to create server", e); 1178 } 1179 } 1180 1181 /** 1182 * A Processor is used to read a dtrace file. A Processor defines callbacks for each record type 1183 * in a dtrace file. As each record is read from a dtrace file, the corresponding callback is 1184 * called. 1185 * 1186 * <p>to use a Processor, pass it to {@link #read_data_trace_files(Collection, PptMap, 1187 * FileIO.Processor, boolean)}. {@code read_data_trace_files} will call {@link 1188 * #process_sample(PptMap,PptTopLevel,ValueTuple,Integer)} once for every sample in the dtrace 1189 * file, and will call other callbacks for other records in the dtrace file. 1190 * 1191 * <p>For an example of how to create and use a Processor, see {@link daikon.tools.ReadTrace}. 1192 * 1193 * @see #read_data_trace_files(Collection, PptMap, FileIO.Processor, boolean) 1194 * @see daikon.tools.ReadTrace 1195 */ 1196 public static class Processor { 1197 /** 1198 * Process a data sample record. This default implementation calls {@link 1199 * FileIO#process_sample(PptMap, PptTopLevel, ValueTuple, Integer)}. 1200 * 1201 * @see FileIO#process_sample(PptMap, PptTopLevel, ValueTuple, Integer) 1202 */ 1203 @RequiresNonNull("FileIO.data_trace_state") 1204 public void process_sample( 1205 PptMap all_ppts, PptTopLevel ppt, ValueTuple vt, @Nullable Integer nonce) { 1206 FileIO.process_sample(all_ppts, ppt, vt, nonce); 1207 } 1208 1209 /** Process a program point declaration record. */ 1210 public void process_decl(PptMap all_ppts, PptTopLevel ppt) {} 1211 1212 /** Process a ppt decl format record. */ 1213 public void process_decl_version(String format) {} 1214 1215 /** Process a VarComparability declaration. */ 1216 public void process_comparability(String comparability) {} 1217 1218 /** Process a ListImplementors declaration. */ 1219 public void process_list_implementors(String implementors) {} 1220 1221 /** Process an input-language declaration. */ 1222 public void process_input_language(String language) {} 1223 1224 /** Process a null record (haven't read anything yet). */ 1225 public void process_null() {} 1226 1227 /** Process a comment. */ 1228 public void process_comment(String comment) {} 1229 1230 /** Process indication of end of file. */ 1231 public void process_eof() {} 1232 1233 /** Process indication of exceeding file size limit. */ 1234 public void process_truncated() {} 1235 1236 /** Process continuable error. */ 1237 public void process_error() {} 1238 } 1239 1240 /** 1241 * Total number of samples passed to process_sample(). Not part of ParseState because it's global 1242 * over all files processed by Daikon. 1243 */ 1244 public static int samples_processed = 0; 1245 1246 /** The type of the record that was most recently read. */ 1247 public enum RecordType { 1248 SAMPLE, // got a sample 1249 1250 DECL, // got a ppt decl 1251 DECL_VERSION, // got an indication of the ppt decl format 1252 COMPARABILITY, // got a VarComparability declaration 1253 LIST_IMPLEMENTORS, // got a ListImplementors declaration 1254 INPUT_LANGUAGE, // got an input-language declaration 1255 1256 NULL, // haven't read anything yet 1257 COMMENT, // got a comment 1258 EOF, // reached end of file 1259 TRUNCATED, // dkconfig_max_line_number reached (without error) 1260 ERROR, // continuable error; fatal errors thrown as exceptions 1261 }; 1262 1263 /** 1264 * ParseState indicates: 1265 * 1266 * <ol> 1267 * <li>Some global information about the state of the parser while reading a decl or dtrace 1268 * file. 1269 * <li>The record that was most recently read; thus, ParseState is essentially a discriminated 1270 * union whose tag is a RecordType. (TODO: These are poor names that should probably be 1271 * swapped!) ParseState is what is returned (actually, side-effected) by method 1272 * read_data_trace_record when it reads a record. 1273 * </ol> 1274 */ 1275 @UsesObjectEquals 1276 @MustCall("close") public static class ParseState implements Closeable { 1277 1278 // 1279 // This is the global information about the state of the parser. 1280 // 1281 1282 /** Name of input file. */ 1283 public String filename; 1284 1285 /** True if the current file is a declaration file. */ 1286 public boolean is_decl_file; 1287 1288 /** 1289 * True if ppts may be new. If a duplicate is seen, it must match a previous point exactly. If 1290 * false, the previous ppt is used without checking for a match. 1291 */ 1292 public boolean ppts_may_be_new; 1293 1294 /** All of the ppts seen so far. */ 1295 public PptMap all_ppts; 1296 1297 /** Input stream. */ 1298 public final @Owning LineNumberReader reader; 1299 1300 /** Total number of lines in the input file. */ 1301 public long total_lines; 1302 1303 /** Comparability format, either VarComparability.IMPLICIT or VarComparability.NONE. */ 1304 public int varcomp_format; 1305 1306 // 1307 // This is the discriminated-union part of the ParseState. 1308 // (Presumably this design was chosen for efficiency, to avoid creating 1309 // & garbage-collecting these values many times.) 1310 // 1311 1312 public RecordType rtype; 1313 1314 /** 1315 * Current ppt. Used when status=DECL or SAMPLE. Can be null if this declaration was skipped 1316 * because of --ppt-select-pattern or --ppt-omit-pattern. 1317 */ 1318 public @Nullable PptTopLevel ppt; 1319 1320 /** The current nonce. Used when status=SAMPLE. */ 1321 public @Nullable Integer nonce; 1322 1323 /** The current set of values. Used when status=SAMPLE. */ 1324 public @Nullable ValueTuple vt; 1325 1326 /** Miscellaneous text in the parsed item. */ 1327 public @Nullable Object payload; // used when status=COMMENT 1328 1329 /** 1330 * Start parsing the given file. 1331 * 1332 * @param raw_filename the file name supplied by the user; may be "-" or "+" 1333 * @param decl_file_p true if the file is a declaration file 1334 * @param ppts_may_be_new true if declarations of ppts read from the data trace file are new 1335 * (and thus are not in all_ppts). false if the ppts may already be there. 1336 * @param ppts the program points 1337 * @throws IOException if there is a problem reading or writing a file 1338 */ 1339 @SuppressWarnings("StaticAssignmentInConstructor") // for progress output 1340 public ParseState( 1341 String raw_filename, boolean decl_file_p, boolean ppts_may_be_new, PptMap ppts) 1342 throws IOException { 1343 // Pretty up raw_filename for use in messages 1344 if (raw_filename.equals("-")) { 1345 filename = "standard input"; 1346 } else if (raw_filename.equals("+")) { 1347 filename = "chicory socket"; 1348 } else { 1349 // Remove directory parts, to make it shorter 1350 filename = raw_filename; 1351 } 1352 1353 is_decl_file = decl_file_p; 1354 this.ppts_may_be_new = ppts_may_be_new; 1355 all_ppts = ppts; 1356 1357 boolean is_url = raw_filename.startsWith("file:") || raw_filename.startsWith("jar:"); 1358 1359 // Do we need to count the lines in the file? 1360 total_lines = 0; 1361 boolean count_lines = dkconfig_count_lines; 1362 if (is_decl_file) { 1363 count_lines = false; 1364 } else if (dkconfig_dtrace_line_count != 0) { 1365 total_lines = dkconfig_dtrace_line_count; 1366 count_lines = false; 1367 } else if (filename.equals("-")) { 1368 count_lines = false; 1369 } else if (is_url) { 1370 count_lines = false; 1371 } else if (Daikon.dkconfig_progress_delay == -1) { 1372 count_lines = false; 1373 } else if (new File(raw_filename).length() == 0) { 1374 // Either it's actually empty, or it's something like a pipe. 1375 count_lines = false; 1376 } 1377 1378 if (count_lines) { 1379 Daikon.progress = "Checking size of " + filename; 1380 total_lines = FilesPlume.countLines(raw_filename); 1381 } else { 1382 // System.out.printf("no count %b %d %s %d %d%n", is_decl_file, 1383 // dkconfig_dtrace_line_count, filename, 1384 // Daikon.dkconfig_progress_delay, (new File(raw_filename)).length()); 1385 } 1386 1387 // Open the reader stream 1388 if (raw_filename.equals("-")) { 1389 // "-" means read from the standard input stream 1390 Reader file_reader = new InputStreamReader(System.in, "ISO-8859-1"); 1391 reader = new LineNumberReader(file_reader); 1392 } else if (raw_filename.equals("+")) { // socket comm with Chicory 1393 InputStream chicoryInput = connectToChicory(); 1394 InputStreamReader chicReader = new InputStreamReader(chicoryInput, UTF_8); 1395 reader = new LineNumberReader(chicReader); 1396 } else if (is_url) { 1397 URL url = URI.create(raw_filename).toURL(); 1398 InputStream stream = null; // dummy initialization for compiler's definite assignment check 1399 try { 1400 stream = url.openStream(); 1401 InputStream gzip_stream = 1402 raw_filename.endsWith(".gz") ? new GZIPInputStream(stream) : stream; 1403 InputStreamReader isr = new InputStreamReader(gzip_stream, UTF_8); 1404 LineNumberReader lnr = new LineNumberReader(isr); 1405 reader = lnr; 1406 } catch (IOException e) { 1407 if (stream != null) { 1408 stream.close(); 1409 } 1410 throw e; 1411 } 1412 } else { 1413 reader = FilesPlume.newLineNumberFileReader(raw_filename); 1414 } 1415 1416 varcomp_format = VarComparability.IMPLICIT; 1417 rtype = RecordType.NULL; 1418 ppt = null; 1419 } 1420 1421 /** Releases resources held by this. */ 1422 @Override 1423 @EnsuresCalledMethods(value = "reader", methods = "close") 1424 public void close(@GuardSatisfied ParseState this) { 1425 try { 1426 reader.close(); 1427 } catch (IOException e) { 1428 throw new BugInDaikon(e); 1429 } 1430 } 1431 1432 /** 1433 * Returns the current line number in the input file, or -1 if not available. 1434 * 1435 * @return the current line number in the input file, or -1 if not available 1436 */ 1437 public int get_linenum() { 1438 return reader.getLineNumber(); 1439 } 1440 1441 private static NumberFormat pctFmt; 1442 1443 static { 1444 pctFmt = NumberFormat.getPercentInstance(); 1445 pctFmt.setMinimumFractionDigits(2); 1446 pctFmt.setMaximumFractionDigits(2); 1447 } 1448 1449 public String reading_message() { 1450 String line; 1451 if (reader == null) { 1452 line = "?"; 1453 } else { 1454 long lineNum = reader.getLineNumber(); 1455 line = String.valueOf(lineNum); 1456 if (total_lines > 0) { 1457 double frac = lineNum / (double) total_lines; 1458 String percent = pctFmt.format(frac); 1459 line = line + ", " + percent; 1460 } 1461 } 1462 return "Reading " + filename + " (line " + line + ") ..."; 1463 } 1464 1465 public String line_file_message() { 1466 return String.format(" at line %d in file %s", reader.getLineNumber(), filename); 1467 } 1468 } 1469 1470 /** Returns the current line number in the input file, or -1 if not available. */ 1471 public static int get_linenum() { 1472 if (FileIO.data_trace_state == null) { 1473 return -1; 1474 } else { 1475 return FileIO.data_trace_state.get_linenum(); 1476 } 1477 } 1478 1479 /** 1480 * Logically, this is a local variable in static method read_data_trace_file. It is used for 1481 * status output, and to give the line number at which a problem was detected. 1482 */ 1483 // The @MonotonicNonNull property is not true globally, but within every 1484 // method it's true, so it is a useful annotation. 1485 public static @MonotonicNonNull ParseState data_trace_state = null; 1486 1487 // The variable is only ever cleared at the end of a routine that set it. 1488 @SuppressWarnings("nullness") // reinitialization 1489 private static void clear_data_trace_state() { 1490 FileIO.data_trace_state = null; 1491 } 1492 1493 /** 1494 * Read only samples from {@code .dtrace} file. Uses the standard data processor which calls 1495 * {@link FileIO#process_sample(PptMap, PptTopLevel, ValueTuple, Integer)} on each record, and 1496 * ignores records other than samples. 1497 */ 1498 public static void read_data_trace_file(String filename, PptMap all_ppts) throws IOException { 1499 Processor processor = new Processor(); 1500 read_data_trace_file(filename, all_ppts, processor, false, true); 1501 } 1502 1503 /** 1504 * Read declarations AND samples (not just sample data as the name might imply) from {@code 1505 * .dtrace} file. For each record read from the file, passes the record to a method of the 1506 * processor. 1507 */ 1508 public static void read_data_trace_file( 1509 String filename, 1510 PptMap all_ppts, 1511 Processor processor, 1512 boolean is_decl_file, 1513 boolean ppts_may_be_new) 1514 throws IOException { 1515 1516 if (debugRead.isLoggable(Level.FINE)) { 1517 debugRead.fine( 1518 "read_data_trace_file " 1519 + filename 1520 + ((Daikon.ppt_regexp != null) ? " " + Daikon.ppt_regexp.pattern() : "") 1521 + ((Daikon.ppt_omit_regexp != null) ? " " + Daikon.ppt_omit_regexp.pattern() : "")); 1522 } 1523 1524 try (ParseState data_trace_state = 1525 new ParseState(filename, is_decl_file, ppts_may_be_new, all_ppts)) { 1526 FileIO.data_trace_state = data_trace_state; 1527 1528 // Used for debugging: write new data trace file. 1529 if (Global.debugPrintDtrace) { 1530 Path p = new File(filename + ".debug").toPath(); 1531 BufferedWriter bw = null; // dummy initialization for compiler's definite assignment check 1532 try { 1533 bw = Files.newBufferedWriter(p, UTF_8); 1534 Global.dtraceWriter = new PrintWriter(bw); 1535 } catch (IOException e) { 1536 if (bw != null) { 1537 bw.close(); 1538 } 1539 throw e; 1540 } 1541 } 1542 1543 while (true) { 1544 read_data_trace_record(data_trace_state); 1545 1546 if (data_trace_state.rtype == RecordType.SAMPLE) { 1547 assert data_trace_state.ppt != null 1548 : "@AssumeAssertion(nullness): dependent: RecordType.SAMPLE"; 1549 assert data_trace_state.vt != null 1550 : "@AssumeAssertion(nullness): dependent: RecordType.SAMPLE"; 1551 // Nonce may be null 1552 samples_processed++; 1553 // Add orig and derived variables; pass to inference (add_and_flow) 1554 try { 1555 processor.process_sample( 1556 data_trace_state.all_ppts, 1557 data_trace_state.ppt, 1558 data_trace_state.vt, 1559 data_trace_state.nonce); 1560 } catch (Error e) { 1561 // e.printStackTrace(); 1562 if (!dkconfig_continue_after_file_exception) { 1563 throw new Daikon.UserError(e, data_trace_state); 1564 } else { 1565 System.out.println(); 1566 System.out.println( 1567 "WARNING: Error while processing trace file; subsequent records ignored."); 1568 System.out.print("Ignored backtrace:"); 1569 e.printStackTrace(System.out); 1570 System.out.println(); 1571 } 1572 } 1573 } else if ((data_trace_state.rtype == RecordType.EOF) 1574 || (data_trace_state.rtype == RecordType.TRUNCATED)) { 1575 break; 1576 } else { 1577 // don't need to do anything explicit for other records found 1578 } 1579 } 1580 1581 if (Global.debugPrintDtrace) { 1582 assert Global.dtraceWriter != null 1583 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 1584 Global.dtraceWriter.close(); 1585 } 1586 1587 Daikon.progress = "Finished reading " + data_trace_state.filename; 1588 1589 clear_data_trace_state(); 1590 } 1591 } 1592 1593 /** 1594 * Like read_data_trace_record, but sets global FileIO.data_trace_state for the duration of the 1595 * call then clears it before returning. Intended for most external callers. 1596 */ 1597 public static void read_data_trace_record_setstate(ParseState state) throws IOException { 1598 1599 FileIO.data_trace_state = state; 1600 read_data_trace_record(state); 1601 clear_data_trace_state(); 1602 } 1603 1604 /** 1605 * Read a single record of ANY type (sample, declaration, comparability, etc.) from a dtrace file. 1606 * If the record is anything but a sample, also processes it. The record is stored by side effect 1607 * into the state argument. 1608 */ 1609 // TODO: For clarity, this should perhaps return its side-effected argument. 1610 @RequiresNonNull("FileIO.data_trace_state") 1611 // not guaranteed: File might be empty EnsuresNonNull("FileIO.new_decl_format") 1612 public static void read_data_trace_record(ParseState state) throws IOException { 1613 1614 // Abstract out the test result into a variable because Java doesn't 1615 // permit suppressing warnings on a statement. Yuck. 1616 boolean stateOK = (state == FileIO.data_trace_state); 1617 assert stateOK; 1618 1619 LineNumberReader reader = state.reader; 1620 1621 for (String line = reader.readLine(); line != null; line = reader.readLine()) { 1622 if (line.equals("")) { 1623 continue; 1624 } 1625 1626 // This cleverness would not be necessary if every comment was followed by 1627 // a blank line. We can't depend on that, though. 1628 if (isComment(line)) { 1629 StringJoiner commentLines = new StringJoiner(lineSep); 1630 commentLines.add(line); 1631 while (nextLineIsComment(reader)) { 1632 commentLines.add(reader.readLine()); 1633 } 1634 state.payload = commentLines.toString(); 1635 state.rtype = RecordType.COMMENT; 1636 return; 1637 } 1638 1639 // stop at a specified point in the file 1640 if ((dkconfig_max_line_number > 0) && (reader.getLineNumber() > dkconfig_max_line_number)) { 1641 state.rtype = RecordType.TRUNCATED; 1642 return; 1643 } 1644 1645 // interning bugfix: no need to intern "line" (after code change to is_declaration_header) 1646 1647 // Check for the file format 1648 if (line.startsWith("decl-version")) { 1649 read_decl_version(state, line); 1650 state.payload = (new_decl_format ? "2.0" : "1.0"); 1651 state.payload = (FileIO.new_decl_format ? "2.0" : "1.0"); 1652 state.rtype = RecordType.DECL_VERSION; 1653 return; 1654 } 1655 1656 // Check for the input language 1657 if (line.startsWith("input-language")) { 1658 String input_language = read_input_language(state, line); 1659 state.payload = input_language; 1660 state.rtype = RecordType.INPUT_LANGUAGE; 1661 return; 1662 } 1663 1664 // If we have gotten to here and new_decl_format is not set, presume 1665 // it is the old format 1666 if (new_decl_format == null) { 1667 // System.out.printf("setting new_decl_format to false%n"); 1668 new_decl_format = Boolean.FALSE; 1669 } 1670 1671 // First look for declarations in the dtrace stream 1672 if (is_declaration_header(line)) { 1673 if (new_decl_format) { 1674 state.ppt = read_ppt_decl(state, line); 1675 } else { 1676 state.ppt = read_declaration(state); 1677 } 1678 // ppt can be null if this declaration was skipped because of 1679 // --ppt-select-pattern or --ppt-omit-pattern. 1680 if (state.ppt != null) { 1681 if (!state.all_ppts.containsName(state.ppt.name())) { 1682 state.all_ppts.add(state.ppt); 1683 assert state.ppt != null : "@AssumeAssertion(nullness)"; 1684 try { 1685 Daikon.init_ppt(state.ppt, state.all_ppts); 1686 } catch (Exception e) { 1687 decl_error(state, e); 1688 } 1689 } 1690 } 1691 state.rtype = RecordType.DECL; 1692 return; 1693 } 1694 if (line.equals("VarComparability") || line.startsWith("var-comparability")) { 1695 state.varcomp_format = read_var_comparability(state, line); 1696 state.rtype = RecordType.COMPARABILITY; 1697 return; 1698 } 1699 if (line.equals("ListImplementors")) { 1700 state.payload = read_list_implementors(reader); 1701 state.rtype = RecordType.LIST_IMPLEMENTORS; 1702 return; 1703 } 1704 String ppt_name = line; 1705 if (new_decl_format) { 1706 // interning bugfix: no need to intern 1707 ppt_name = unescape_decl(line); 1708 } 1709 ppt_name = user_mod_ppt_name(ppt_name); 1710 if (!ppt_included(ppt_name)) { 1711 // System.out.printf("skipping ppt %s%n", line); 1712 while ((line != null) && !line.equals("")) line = reader.readLine(); 1713 continue; 1714 } 1715 // System.out.printf("Not skipping ppt %s%n", line); 1716 1717 if (state.is_decl_file) { 1718 if (!new_decl_format && line.startsWith("ppt ")) { 1719 throw new Daikon.UserError( 1720 String.format( 1721 "Declaration file %s is not version 2.0, but line %d looks like a version 2.0" 1722 + " declaration: %s%nPerhaps the file is missing a \"decl-version 2.0\"" 1723 + " record at the beginning", 1724 state.filename, state.reader.getLineNumber(), line)); 1725 } 1726 throw new Daikon.UserError( 1727 String.format( 1728 "Declaration files should not contain samples, but file %s does at line %d: %s", 1729 state.filename, state.reader.getLineNumber(), line)); 1730 } 1731 1732 // Parse the ppt name 1733 try { 1734 new PptName(ppt_name); 1735 } catch (Throwable t) { 1736 @SuppressWarnings("nullness") // thrown exception always has a detail message 1737 @NonNull String message = t.getMessage(); 1738 // Augment the message with line number information. 1739 if (!(t instanceof Daikon.UserError)) { 1740 message = String.format("Illegal program point name '%s' (%s)", ppt_name, message); 1741 } 1742 throw new Daikon.UserError(message, reader, state.filename); 1743 } 1744 1745 if (state.all_ppts.size() == 0) { 1746 throw new Daikon.UserError( 1747 "No declarations were provided before the first sample. Perhaps you did not supply" 1748 + " the proper .decls file to Daikon. (Or, there could be a bug in the front end" 1749 + " that created the .dtrace file " 1750 + state.filename 1751 + ".)"); 1752 } 1753 1754 PptTopLevel ppt = state.all_ppts.get(ppt_name); 1755 if (ppt == null) { 1756 throw new Daikon.UserError( 1757 "No declaration was provided for program point " + ppt_name, state); 1758 } 1759 1760 // not vis.length, as that includes constants, derived variables, etc. 1761 // Actually, we do want to leave space for _orig vars. 1762 // And for the time being (and possibly forever), for derived variables. 1763 int vals_array_size = ppt.var_infos.length - ppt.num_static_constant_vars; 1764 1765 // Read an invocation nonce if one exists 1766 Integer nonce; 1767 1768 boolean nonce_exists; 1769 { 1770 String nonce_header_peekahead; 1771 // arbitrary number, hopefully big enough; catch exceptions 1772 reader.mark(1000); 1773 try { 1774 nonce_header_peekahead = reader.readLine(); 1775 } catch (Exception e) { 1776 nonce_header_peekahead = null; 1777 } 1778 reader.reset(); 1779 nonce_exists = NONCE_HEADER.equals(nonce_header_peekahead); 1780 } 1781 if (!nonce_exists) { 1782 nonce = null; 1783 } else { 1784 @SuppressWarnings("nullness") // nonce_exists is true, so readLine() returns non-null 1785 @NonNull String nonce_header = reader.readLine(); // read & discard header 1786 assert NONCE_HEADER.equals(nonce_header); 1787 String nonce_number = reader.readLine(); 1788 if (nonce_number == null) { 1789 throw new Daikon.UserError("File ended while trying to read nonce", state); 1790 } 1791 nonce = Integer.valueOf(nonce_number); 1792 1793 if (Global.debugPrintDtrace) { 1794 to_write_nonce = true; 1795 nonce_value = nonce.toString(); 1796 } 1797 } 1798 1799 @Nullable Object[] vals = new @Nullable Object[vals_array_size]; 1800 int[] mods = new int[vals_array_size]; 1801 1802 // Read a single record from the trace file; 1803 // fills up vals and mods arrays by side effect. 1804 try { 1805 read_vals_and_mods_from_trace_file(reader, state.filename, ppt, vals, mods); 1806 } catch (IOException e) { 1807 String nextLine = reader.readLine(); 1808 if ((e instanceof EOFException) || (nextLine == null)) { 1809 System.out.println(); 1810 System.out.println( 1811 "WARNING: Unexpected EOF while processing " 1812 + "trace file - last record of trace file ignored"); 1813 state.rtype = RecordType.EOF; 1814 return; 1815 } else if (dkconfig_continue_after_file_exception) { 1816 System.out.println(); 1817 System.out.println("WARNING: IOException while processing trace file - record ignored"); 1818 System.out.print("Ignored backtrace:"); 1819 e.printStackTrace(System.out); 1820 System.out.println(); 1821 while (nextLine != null && !nextLine.equals("")) { 1822 // System.out.println("Discarded line " + reader.getLineNumber() 1823 // + ": " + nextLine); 1824 nextLine = reader.readLine(); 1825 } 1826 continue; 1827 } else { 1828 throw e; 1829 } 1830 } 1831 1832 state.ppt = ppt; 1833 state.nonce = nonce; 1834 state.vt = ValueTuple.makeUninterned(vals, mods); 1835 state.rtype = RecordType.SAMPLE; 1836 return; 1837 } 1838 1839 state.rtype = RecordType.EOF; 1840 return; 1841 } 1842 1843 /** 1844 * Add orig() and derived variables to vt (by side effect), then supply it to the program point 1845 * for flowing. 1846 * 1847 * @param vt trace data only; modified by side effect to add derived vars 1848 */ 1849 @RequiresNonNull("FileIO.data_trace_state") 1850 public static void process_sample( 1851 PptMap all_ppts, PptTopLevel ppt, ValueTuple vt, @Nullable Integer nonce) { 1852 1853 // Add orig variables. This must be above the check below because 1854 // it saves away the orig values from enter points for later use 1855 // by exit points. 1856 boolean ignore = compute_orig_variables(ppt, vt.vals, vt.mods, nonce); 1857 if (ignore) { 1858 return; 1859 } 1860 1861 // Only process the leaves of the ppt tree. 1862 // This test assumes that all leaves are numbered exit program points 1863 // -- that is, points of the form foo:::EXIT22 for which isExitPoint() 1864 // is true and isCombinedExitPoint() is false. "Combined" exit points 1865 // of the form foo:::EXIT are not processed -- they are assumed to be 1866 // non-leaves. 1867 if (Daikon.use_dataflow_hierarchy) { 1868 1869 // Rather than defining leaves as :::EXIT54 (numbered exit) 1870 // program points define them as everything except 1871 // ::EXIT (combined), :::ENTER, :::THROWS, :::OBJECT, ::GLOBAL 1872 // and :::CLASS program points. This scheme ensures that arbitrarly 1873 // named program points such as :::POINT (used by convertcsv.pl) 1874 // will be treated as leaves. 1875 1876 if (ppt.ppt_name.isEnterPoint() 1877 || ppt.ppt_name.isThrowsPoint() 1878 || ppt.ppt_name.isObjectInstanceSynthetic() 1879 || ppt.ppt_name.isClassStaticSynthetic() 1880 || ppt.ppt_name.isGlobalPoint()) { 1881 return; 1882 } 1883 1884 if (ppt.ppt_name.isExitPoint() && ppt.ppt_name.isCombinedExitPoint()) { 1885 // not Daikon.UserError; caller has more info (e.g., filename) 1886 throw new RuntimeException( 1887 "Bad program point name " + ppt.name + " is a combined exit point name"); 1888 } 1889 } 1890 1891 // Add derived variables 1892 compute_derived_variables(ppt, vt.vals, vt.mods); 1893 1894 // Causes interning 1895 vt = new ValueTuple(vt.vals, vt.mods); 1896 1897 if (debugRead.isLoggable(Level.FINE)) { 1898 debugRead.fine("Adding ValueTuple to " + ppt.name()); 1899 debugRead.fine(" length is " + vt.vals.length); 1900 } 1901 1902 // If we are only reading the sample, don't process them 1903 if (dkconfig_read_samples_only) { 1904 return; 1905 } 1906 1907 @SuppressWarnings({"UnusedVariable", "nullness:contracts.precondition"}) 1908 Object dummy = ppt.add_bottom_up(vt, 1); 1909 1910 if (debugVars.isLoggable(Level.FINE)) { 1911 debugVars.fine(ppt.name() + " vars: " + Debug.int_vars(ppt, vt)); 1912 } 1913 1914 if (Global.debugPrintDtrace) { 1915 assert Global.dtraceWriter != null 1916 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 1917 Global.dtraceWriter.close(); 1918 } 1919 } 1920 1921 /** Returns true if this procedure has an unmatched entry. */ 1922 static boolean has_unmatched_procedure_entry(PptTopLevel ppt) { 1923 for (Invocation invok : call_hashmap.values()) { 1924 if (invok.ppt == ppt) { 1925 return true; 1926 } 1927 } 1928 for (Invocation invok : call_stack) { 1929 if (invok.ppt == ppt) { 1930 return true; 1931 } 1932 } 1933 return false; 1934 } 1935 1936 /** Print each call that does not have a matching exit. */ 1937 public static void process_unmatched_procedure_entries() { 1938 1939 if (dkconfig_unmatched_procedure_entries_quiet) { 1940 return; 1941 } 1942 1943 int unmatched_count = call_stack.size() + call_hashmap.size(); 1944 1945 if (!call_stack.isEmpty() || !call_hashmap.isEmpty()) { 1946 System.out.println(); 1947 System.out.print( 1948 "No return from procedure observed " 1949 + StringsPlume.nplural(unmatched_count, "time") 1950 + "."); 1951 if (Daikon.use_dataflow_hierarchy) { 1952 System.out.print(" Unmatched entries are ignored!"); 1953 } 1954 System.out.println(); 1955 if (!call_hashmap.isEmpty()) { 1956 // Put the invocations in sorted order for printing. 1957 ArrayList<Invocation> invocations = new ArrayList<>(); 1958 for (@KeyFor("call_hashmap") Integer i : CollectionsPlume.sortedKeySet(call_hashmap)) { 1959 Invocation invok = call_hashmap.get(i); 1960 assert invok != null; 1961 invocations.add(invok); 1962 } 1963 System.out.println("Unterminated calls:"); 1964 if (dkconfig_verbose_unmatched_procedure_entries) { 1965 print_invocations_verbose(invocations); 1966 } else { 1967 print_invocations_grouped(invocations); 1968 } 1969 } 1970 1971 if (!call_stack.isEmpty()) { 1972 if (dkconfig_verbose_unmatched_procedure_entries) { 1973 System.out.println( 1974 "Remaining " 1975 + StringsPlume.nplural(unmatched_count, "stack") 1976 + " call summarized below."); 1977 print_invocations_verbose(call_stack); 1978 } else { 1979 print_invocations_grouped(call_stack); 1980 } 1981 } 1982 System.out.print("End of report for procedures not returned from."); 1983 if (Daikon.use_dataflow_hierarchy) { 1984 System.out.print(" Unmatched entries are ignored!"); 1985 } 1986 System.out.println(); 1987 } 1988 } 1989 1990 /** Print all the invocations in the collection, in order. */ 1991 static void print_invocations_verbose(Collection<Invocation> invocations) { 1992 for (Invocation invok : invocations) { 1993 System.out.println(invok.format()); 1994 } 1995 } 1996 1997 /** Print the invocations in the collection, in order, and coalescing duplicates. */ 1998 static void print_invocations_grouped(Collection<Invocation> invocations) { 1999 Map<@Interned String, Integer> counter = new LinkedHashMap<>(); 2000 2001 for (Invocation invok_noncanonical : invocations) { 2002 @Interned Invocation invok = invok_noncanonical.canonicalize(); 2003 String invokString = invok.format(false).intern(); 2004 if (counter.containsKey(invokString)) { 2005 Integer oldCount = counter.get(invokString); 2006 Integer newCount = oldCount.intValue() + 1; 2007 counter.put(invokString, newCount); 2008 } else { 2009 counter.put(invokString, 1); 2010 } 2011 } 2012 2013 // Print the invocations in sorted order. 2014 for (Map.Entry<@Interned String, Integer> invokEntry : counter.entrySet()) { 2015 System.out.println( 2016 invokEntry.getKey() + " : " + StringsPlume.nplural(invokEntry.getValue(), "invocation")); 2017 } 2018 } 2019 2020 // This procedure reads a single record from a trace file and 2021 // fills up vals and mods by side effect. The ppt name and 2022 // invocation nonce (if any) have already been read. 2023 @RequiresNonNull("FileIO.data_trace_state") 2024 private static void read_vals_and_mods_from_trace_file( 2025 LineNumberReader reader, 2026 String filename, 2027 PptTopLevel ppt, 2028 @Nullable Object[] vals, 2029 int[] mods) 2030 throws IOException { 2031 VarInfo[] vis = ppt.var_infos; 2032 int num_tracevars = ppt.num_tracevars; 2033 2034 /*NNC:@Nullable*/ String[] oldvalue_reps = ppt_to_value_reps.get(ppt); 2035 if (oldvalue_reps == null) { 2036 // We've not encountered this program point before. The nulls in 2037 // this array will compare non-equal to whatever is in the trace 2038 // file, which is the desired behavior. 2039 oldvalue_reps = new /*NNC:@Nullable*/ String[num_tracevars]; 2040 } 2041 2042 if (Global.debugPrintDtrace) { 2043 assert Global.dtraceWriter != null 2044 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 2045 Global.dtraceWriter.println(ppt.name()); 2046 2047 if (to_write_nonce) { 2048 Global.dtraceWriter.println(NONCE_HEADER); 2049 Global.dtraceWriter.println(nonce_value); 2050 to_write_nonce = false; 2051 } 2052 } 2053 2054 for (int vi_index = 0, val_index = 0; val_index < num_tracevars; vi_index++) { 2055 assert vi_index < vis.length 2056 : "Got to vi_index " 2057 + vi_index 2058 + " after " 2059 + val_index 2060 + " of " 2061 + num_tracevars 2062 + " values"; 2063 VarInfo vi = vis[vi_index]; 2064 assert !vi.is_static_constant || (vi.value_index == -1) 2065 // : "Bad value_index " + vi.value_index + " when static_constant_value = " + 2066 // vi.static_constant_value + " for " + vi.repr() + " at " + ppt_name 2067 ; 2068 if (vi.is_static_constant) { 2069 continue; 2070 } 2071 assert val_index == vi.value_index 2072 // : "Differing val_index = " + val_index 2073 // + " and vi.value_index = " + vi.value_index 2074 // + " for " + vi.name + lineSep + vi.repr() 2075 ; 2076 2077 // In errors, say "for program point", not "at program point" as the 2078 // latter confuses Emacs goto-error. 2079 2080 String line = reader.readLine(); 2081 if (line == null) { 2082 throw new Daikon.UserError( 2083 "Unexpected end of file at " 2084 + data_trace_state.filename 2085 + " line " 2086 + reader.getLineNumber() 2087 + lineSep 2088 + " Expected variable " 2089 + vi.name() 2090 + ", got " 2091 + "null" // line 2092 + " for program point " 2093 + ppt.name()); 2094 } 2095 2096 // Read lines until an included variable is found 2097 while ((line != null) && !line.equals("") && !var_included(line)) { 2098 line = reader.readLine(); // value (discard it) 2099 line = reader.readLine(); // modbit 2100 if (line == null || !(line.equals("0") || line.equals("1") || line.equals("2"))) { 2101 throw new Daikon.UserError("Bad modbit '" + line + "'", data_trace_state); 2102 } 2103 line = reader.readLine(); // next variable name 2104 } 2105 if (line == null) { 2106 throw new Daikon.UserError( 2107 "Unexpected end of file at " 2108 + data_trace_state.filename 2109 + " line " 2110 + reader.getLineNumber() 2111 + lineSep 2112 + " Expected to find variable name" 2113 + " for program point " 2114 + ppt.name()); 2115 } 2116 2117 if (!unescape_decl(line.trim()).equals(vi.str_name())) { 2118 throw new Daikon.UserError( 2119 "Mismatch between declaration and trace. Expected variable " 2120 + vi.name() 2121 + ", got " 2122 + line 2123 + " for program point " 2124 + ppt.name(), 2125 data_trace_state); 2126 } 2127 line = reader.readLine(); 2128 if (line == null) { 2129 throw new Daikon.UserError( 2130 "Unexpected end of file at " 2131 + data_trace_state.filename 2132 + " line " 2133 + reader.getLineNumber() 2134 + lineSep 2135 + " Expected value for variable " 2136 + vi.name() 2137 + ", got " 2138 + "null" // line 2139 + " for program point " 2140 + ppt.name()); 2141 } 2142 String value_rep = line; 2143 line = reader.readLine(); 2144 if (line == null) { 2145 throw new Daikon.UserError( 2146 "Unexpected end of file at " 2147 + data_trace_state.filename 2148 + " line " 2149 + reader.getLineNumber() 2150 + lineSep 2151 + " Expected modbit for variable " 2152 + vi.name() 2153 + ", got " 2154 + "null" // line 2155 + " for program point " 2156 + ppt.name()); 2157 } 2158 if (!(line.equals("0") || line.equals("1") || line.equals("2"))) { 2159 throw new Daikon.UserError("Bad modbit `" + line + "'", data_trace_state); 2160 } 2161 int mod = ValueTuple.parseModified(line); 2162 2163 // System.out.println("Mod is " + mod + " at " + data_trace_state.filename + " line " + 2164 // reader.getLineNumber()); 2165 // System.out.pringln(" for variable " + vi.name() 2166 // + " for program point " + ppt.name()); 2167 2168 // MISSING_FLOW is only found during flow algorithm 2169 assert mod != ValueTuple.MISSING_FLOW : "Data trace value can't be missing due to flow"; 2170 2171 if (mod != ValueTuple.MISSING_NONSENSICAL) { 2172 // Set the modbit now, depending on whether the value of the variable 2173 // has been changed or not. 2174 if (value_rep.equals(oldvalue_reps[val_index])) { 2175 if (!dkconfig_add_changed) { 2176 mod = ValueTuple.UNMODIFIED; 2177 } 2178 } else { 2179 mod = ValueTuple.MODIFIED; 2180 } 2181 } 2182 2183 mods[val_index] = mod; 2184 oldvalue_reps[val_index] = value_rep; 2185 2186 if (Global.debugPrintDtrace) { 2187 assert Global.dtraceWriter != null 2188 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 2189 Global.dtraceWriter.println(vi.name()); 2190 Global.dtraceWriter.println(value_rep); 2191 Global.dtraceWriter.println(mod); 2192 } 2193 Debug dbg = Debug.newDebug(FileIO.class, ppt, Debug.vis(vi)); 2194 if (dbg != null) { 2195 dbg.log("Var " + vi.name() + " has value " + value_rep + " mod " + mod); 2196 } 2197 2198 // Both uninit and nonsensical mean missing modbit 2, because 2199 // it doesn't make sense to look at x.y when x is uninitialized. 2200 if (ValueTuple.modIsMissingNonsensical(mod)) { 2201 if (!(value_rep.equals("nonsensical") 2202 // Kvasir still uses "uninit" (it distinguishes between 2203 // uninit and nonsensical), though the Daikon manual does not 2204 // officially permit "uninit" as a value and has not since at 2205 // least 2002. This is fixed in the Kvasir repository as of 2206 // 5/2009, so the following two lines should be removed at 2207 // some point not too long after that. Then Daikon should 2208 // print a warning (or even terminate execution) about uses 2209 // of "uninit". 2210 || value_rep.equals("uninit") 2211 || value_rep.equals("missing"))) { 2212 throw new Daikon.UserError( 2213 "Modbit indicates nonsensical value for variable " 2214 + vi.name() 2215 + " with value \"" 2216 + value_rep 2217 + "\";" 2218 + lineSep 2219 + " text of value should be \"nonsensical\"", 2220 data_trace_state); 2221 } else { 2222 if (debug_missing && !vi.canBeMissing) { 2223 System.out.printf( 2224 "Var %s ppt %s at line %d missing%n", vi, ppt.name(), FileIO.get_linenum()); 2225 System.out.printf("val_index = %d, mods[val_index] = %d%n", val_index, mods[val_index]); 2226 } 2227 vi.canBeMissing = true; 2228 } 2229 vals[val_index] = null; 2230 } else { 2231 // mod is not MISSING_NONSENSICAL 2232 2233 // System.out.println("Mod is " + mod + " (missing=" + 2234 // ValueTuple.MISSING + "), rep=" + value_rep + 2235 // "(modIsMissing=" + ValueTuple.modIsMissing(mod) + ")"); 2236 2237 try { 2238 vals[val_index] = vi.rep_type.parse_value(value_rep, reader, filename); 2239 if (vals[val_index] == null) { 2240 if (debug_missing && !vi.canBeMissing) { 2241 System.out.printf( 2242 "Var %s ppt %s at line %d is null, and modbit is not missing%n", 2243 vi, ppt.name(), FileIO.get_linenum()); 2244 } 2245 // The value in the trace was null even though the modbit was not 2246 // MISSING_NONSENSICAL. Set the modbit to MISSING_NONSENSICAL. 2247 // This can happen for a value like [1 nonsensical 2], because 2248 // if any array value is nonsensical, the whole array is 2249 // treated as nonsensical. 2250 mods[val_index] = ValueTuple.MISSING_NONSENSICAL; 2251 vi.canBeMissing = true; 2252 } 2253 } catch (Daikon.UserError e) { 2254 throw e; 2255 } catch (Throwable e) { 2256 // e.printStackTrace(System.err); // for debugging 2257 throw new Daikon.UserError( 2258 e, 2259 "Error while parsing value " 2260 + value_rep 2261 + " for variable " 2262 + vi.name() 2263 + " of type " 2264 + vi.rep_type 2265 + ": " 2266 + e.getLocalizedMessage(), 2267 reader, 2268 filename); 2269 } 2270 } 2271 val_index++; 2272 } 2273 2274 // Does oldvalue_reps now have no null elements??? 2275 oldvalue_reps = castNonNullDeep(oldvalue_reps); // https://tinyurl.com/cfissue/986 2276 ppt_to_value_reps.put(ppt, oldvalue_reps); 2277 2278 if (Global.debugPrintDtrace) { 2279 assert Global.dtraceWriter != null 2280 : "@AssumeAssertion(nullness): dependent: set if debugPrintDtrace is true"; 2281 Global.dtraceWriter.println(); 2282 } 2283 2284 // Expecting the end of a block of values. 2285 String line = reader.readLine(); 2286 // First, we might get some variables that ought to be omitted. 2287 while ((line != null) && !line.equals("") && !var_included(line)) { 2288 line = reader.readLine(); // value 2289 line = reader.readLine(); // modbit 2290 line = reader.readLine(); // next variable name 2291 } 2292 assert (line == null) || line.equals("") 2293 : "Expected blank line in " 2294 + data_trace_state.filename 2295 + " at line " 2296 + reader.getLineNumber() 2297 + ": " 2298 + line; 2299 } 2300 2301 /** 2302 * If this is a function entry ppt, stores the values of all of the variables away for use at the 2303 * exit. If this is an exit, finds the values at enter and adds them as the values of the orig 2304 * variables. Normally returns false. Returns true if this is an exit without a matching enter. 2305 * See dkconfig_ignore_missing_enter for more info. If true is returned, this ppt should be 2306 * ignored by the caller. 2307 */ 2308 @RequiresNonNull("FileIO.data_trace_state") 2309 public static boolean compute_orig_variables( 2310 PptTopLevel ppt, 2311 // HashMap cumulative_modbits, 2312 @Nullable Object[] vals, 2313 int[] mods, 2314 @Nullable Integer nonce) { 2315 assert data_trace_state != null; 2316 2317 VarInfo[] vis = ppt.var_infos; 2318 @Interned String fn_name = ppt.ppt_name.getNameWithoutPoint(); 2319 String ppt_name = ppt.name(); 2320 if (ppt_name.endsWith(enter_tag)) { 2321 Invocation invok = new Invocation(ppt, vals, mods); 2322 if (nonce == null) { 2323 call_stack.push(invok); 2324 } else { 2325 call_hashmap.put(nonce, invok); 2326 } 2327 return false; 2328 } 2329 2330 if (ppt.ppt_name.isExitPoint() || ppt.ppt_name.isThrowsPoint()) { 2331 Invocation invoc; 2332 // Set invoc 2333 { 2334 if (nonce == null) { 2335 if (call_stack.isEmpty()) { 2336 // Not Daikon.UserError: caller knows context such as 2337 // file name and line number. 2338 throw new Error("Function exit without corresponding entry: " + ppt.name()); 2339 } 2340 invoc = call_stack.pop(); 2341 while (invoc.ppt.ppt_name.getNameWithoutPoint() != fn_name) { 2342 // Should also mark as a function that made an exceptional exit 2343 // at run time. 2344 System.err.println( 2345 "Exceptional exit from function " 2346 + fn_name 2347 + ", expected to first exit from " 2348 + invoc.ppt.ppt_name.getNameWithoutPoint() 2349 + ((data_trace_state.filename == null) 2350 ? "" 2351 : "; at " 2352 + data_trace_state.filename 2353 + " line " 2354 + data_trace_state.reader.getLineNumber())); 2355 invoc = call_stack.pop(); 2356 } 2357 } else { 2358 // nonce != null 2359 if (!call_hashmap.containsKey(nonce)) { 2360 if (dkconfig_ignore_missing_enter) { 2361 // System.out.printf("Didn't find call with nonce %d to match %s" + 2362 // " ending at %s line %d%n", nonce, ppt.name(), 2363 // data_trace_state.filename, 2364 // data_trace_state.reader.getLineNumber()); 2365 return true; 2366 } else { 2367 // Not Daikon.UserError: caller knows context such as 2368 // file name and line number. 2369 throw new Error( 2370 String.format( 2371 "Didn't find call with nonce %s to match %s ending at %s line %d", 2372 nonce, 2373 ppt.name(), 2374 data_trace_state.filename, 2375 data_trace_state.reader.getLineNumber())); 2376 } 2377 } 2378 invoc = call_hashmap.get(nonce); 2379 call_hashmap.remove(nonce); 2380 } 2381 } 2382 2383 // Loop through each orig variable and get its value/mod bits from 2384 // the ENTER point. vi_index is the index into var_infos at the 2385 // ENTER point. val_index is the index into vals[] and mods[] at 2386 // ENTER point. Note that vis[] includes static constants but 2387 // vals[] and mods[] do not. Also that we don't create orig versions 2388 // of static constants 2389 int vi_index = 0; 2390 for (int val_index = 0; val_index < ppt.num_orig_vars; val_index++) { 2391 VarInfo vi = vis[ppt.num_tracevars + ppt.num_static_constant_vars + val_index]; 2392 assert !vi.is_static_constant : "orig constant " + vi; 2393 2394 // Skip over constants in the entry point 2395 while (invoc.ppt.var_infos[vi_index].is_static_constant) { 2396 vi_index++; 2397 } 2398 2399 // Copy the vals and mod bits from entry to exit 2400 vals[ppt.num_tracevars + val_index] = invoc.vals[val_index]; 2401 int mod = invoc.mods[val_index]; 2402 mods[ppt.num_tracevars + val_index] = mod; 2403 2404 // If the value was missing, mark this variable as can be missing. 2405 // Carefully check that we have orig version of the variable from 2406 // the ENTER point. 2407 if (ValueTuple.modIsMissingNonsensical(mod)) { 2408 if (debug_missing && !vi.canBeMissing) { 2409 System.out.printf("add_orig: var %s missing[%d/%d]%n", vi, val_index, vi_index); 2410 } 2411 vi.canBeMissing = true; 2412 assert invoc.vals[val_index] == null; 2413 assert vi.name() == invoc.ppt.var_infos[vi_index].prestate_name() 2414 : vi.name() + " != " + invoc.ppt.var_infos[vi_index]; 2415 assert invoc.ppt.var_infos[vi_index].canBeMissing : invoc.ppt.var_infos[vi_index]; 2416 } 2417 vi_index++; 2418 } 2419 } 2420 return false; 2421 } 2422 2423 /** Computes values of derived variables. */ 2424 public static void compute_derived_variables( 2425 PptTopLevel ppt, @Nullable Object[] vals, int[] mods) { 2426 // This ValueTuple is temporary: we're temporarily suppressing interning, 2427 // which we will do after we have all the values available. 2428 ValueTuple partial_vt = ValueTuple.makeUninterned(vals, mods); 2429 int filled_slots = ppt.num_orig_vars + ppt.num_tracevars + ppt.num_static_constant_vars; 2430 for (int i = 0; i < filled_slots; i++) { 2431 assert !ppt.var_infos[i].isDerived(); 2432 } 2433 int num_const = ppt.num_static_constant_vars; 2434 for (int i = filled_slots; i < ppt.var_infos.length; i++) { 2435 assert ppt.var_infos[i].derived != null : "variable not derived: " + ppt.var_infos[i].repr(); 2436 assert ppt.var_infos[i].derived != null : "@AssumeAssertion(nullness): application invariant"; 2437 // Add this derived variable's value 2438 ValueAndModified vm = ppt.var_infos[i].derived.computeValueAndModified(partial_vt); 2439 vals[i - num_const] = vm.value; 2440 mods[i - num_const] = vm.modified; 2441 } 2442 } 2443 2444 /////////////////////////////////////////////////////////////////////////// 2445 /// Serialized PptMap files 2446 /// 2447 2448 /** 2449 * Use a special record type. Saving as one object allows for reference-sharing, easier saves and 2450 * loads, and potential for later overriding of SerialFormat.readObject if the save format changes 2451 * (ick). 2452 */ 2453 static final class SerialFormat implements Serializable { 2454 // We are Serializable, so we specify a version to allow changes to 2455 // method signatures without breaking serialization. If you add or 2456 // remove fields, you should change this number to the current date. 2457 static final long serialVersionUID = 20060905L; 2458 2459 @RequiresNonNull("FileIO.new_decl_format") 2460 public SerialFormat(PptMap map, Configuration config) { 2461 this.map = map; 2462 this.config = config; 2463 this.new_decl_format = FileIO.new_decl_format; 2464 } 2465 2466 public PptMap map; 2467 public Configuration config; 2468 public boolean new_decl_format = false; 2469 } 2470 2471 /** 2472 * Write a serialized PptMap to a file. 2473 * 2474 * @param map a PptMap 2475 * @param file the file to which to write 2476 * @throws IOException if there is trouble writing the file 2477 */ 2478 public static void write_serialized_pptmap(PptMap map, File file) throws IOException { 2479 SerialFormat record = new SerialFormat(map, Configuration.getInstance()); 2480 FilesPlume.writeObject(record, file); 2481 } 2482 2483 /** 2484 * Read either a serialized PptMap or a InvMap and return a PptMap. If an InvMap is specified, it 2485 * is converted to a PptMap. 2486 */ 2487 @EnsuresNonNull("FileIO.new_decl_format") 2488 public static PptMap read_serialized_pptmap(File file, boolean use_saved_config) 2489 throws IOException { 2490 2491 try { 2492 Object obj = FilesPlume.readObject(file); 2493 if (obj instanceof FileIO.SerialFormat) { 2494 SerialFormat record = (SerialFormat) obj; 2495 if (use_saved_config) { 2496 Configuration.getInstance().overlap(record.config); 2497 } 2498 FileIO.new_decl_format = record.new_decl_format; 2499 // System.err.printf("Setting FileIO.new_decl_format to %b%n", 2500 // FileIO.new_decl_format); 2501 return record.map; 2502 } else if (obj instanceof InvMap) { 2503 // System.err.printf("Restoring an InvMap%n"); 2504 InvMap invs = (InvMap) obj; 2505 PptMap ppts = new PptMap(); 2506 for (PptTopLevel ppt : invs.pptIterable()) { 2507 PptTopLevel nppt = new PptTopLevel(ppt.name, ppt.var_infos); 2508 nppt.set_sample_number(ppt.num_samples()); 2509 ppts.add(nppt); 2510 List<Invariant> inv_list = invs.get(ppt); 2511 for (Invariant inv : inv_list) { 2512 PptSlice slice = nppt.get_or_instantiate_slice(inv.ppt.var_infos); 2513 inv.ppt = slice; 2514 slice.addInvariant(inv); 2515 } 2516 } 2517 assert FileIO.new_decl_format != null 2518 : "@AssumeAssertion(nullness): InvMap.readObject() sets FileIO.new_decl_format"; 2519 return ppts; 2520 } else { 2521 throw new IOException("Unexpected serialized file type: " + obj.getClass()); 2522 } 2523 } catch (ClassNotFoundException e) { 2524 throw (IOException) new IOException("Error while loading inv file").initCause(e); 2525 } catch (InvalidClassException e) { 2526 throw new IOException( 2527 "It is likely that the .inv file format has changed, because a Daikon data structure has" 2528 + " been modified, so your old .inv file is no longer readable by Daikon. Please" 2529 + " regenerate your .inv file." 2530 // + lineSep + e.toString() 2531 ); 2532 } 2533 // } catch (StreamCorruptedException e) { // already extends IOException 2534 // } catch (OptionalDataException e) { // already extends IOException 2535 } 2536 2537 /** 2538 * Returns whether or not the specified ppt name should be included in processing. Ppts can be 2539 * excluded because they match the omit_regexp, don't match ppt_regexp, or are greater than 2540 * ppt_max_name. 2541 */ 2542 public static boolean ppt_included(String ppt_name) { 2543 2544 // System.out.println ("ppt_name = '" + ppt_name + "' max name = '" 2545 // + Daikon.ppt_max_name + "'"); 2546 if (((Daikon.ppt_omit_regexp != null) && Daikon.ppt_omit_regexp.matcher(ppt_name).find()) 2547 || ((Daikon.ppt_regexp != null) && !Daikon.ppt_regexp.matcher(ppt_name).find()) 2548 || ((Daikon.ppt_max_name != null) 2549 && ((Daikon.ppt_max_name.compareTo(ppt_name) < 0) 2550 && (ppt_name.indexOf(global_suffix) == -1)))) { 2551 return false; 2552 } else { 2553 return true; 2554 } 2555 } 2556 2557 /** 2558 * Returns true if the given variable is included, according to Daikon's {@code 2559 * --var-select-pattern} and {@code --var-omit-pattern} flags. 2560 */ 2561 public static boolean var_included(String var_name) { 2562 assert !var_name.equals(""); 2563 if (((Daikon.var_omit_regexp != null) && Daikon.var_omit_regexp.matcher(var_name).find()) 2564 || ((Daikon.var_regexp != null) && !Daikon.var_regexp.matcher(var_name).find())) { 2565 return false; 2566 } else { 2567 return true; 2568 } 2569 } 2570 2571 /** 2572 * Checks the specified array of variables to see if it matches exactly the variables in the 2573 * existing ppt. Throws an error if there are any differences. Used to ensure that a new ppt with 2574 * the same name as an existing ppt is exactly the same. 2575 */ 2576 static void check_decl_match(ParseState state, PptTopLevel existing_ppt, VarInfo[] vi_array) { 2577 2578 VarInfo[] existing_vars = existing_ppt.var_infos; 2579 if (existing_ppt.num_declvars != vi_array.length) { 2580 throw new Daikon.UserError( 2581 "Duplicate declaration of program point \"" 2582 + existing_ppt.name() 2583 + "\" with a different number of VarInfo objects: " 2584 + "old VarInfo number=" 2585 + existing_ppt.num_declvars 2586 + ", new VarInfo number=" 2587 + vi_array.length, 2588 state); 2589 } 2590 2591 for (int i = 0; i < vi_array.length; i++) { 2592 String oldName = existing_vars[i].str_name(); 2593 String newName = vi_array[i].str_name(); 2594 if (!oldName.equals(newName)) { 2595 throw new Daikon.UserError( 2596 "Duplicate declaration of program point \"" 2597 + existing_ppt.name() 2598 + "\" with two different VarInfo: old VarInfo=" 2599 + oldName 2600 + ", new VarInfo=" 2601 + newName, 2602 state); 2603 } 2604 } 2605 } 2606 2607 /** 2608 * Converts the declaration record version of a name into its correct version. In the declaration 2609 * record, blanks are encoded as \_ and backslashes as \\. 2610 */ 2611 private static String unescape_decl(String orig) { 2612 StringBuilder sb = new StringBuilder(orig.length()); 2613 // The previous escape character was seen just before this position. 2614 int post_esc = 0; 2615 int this_esc = orig.indexOf('\\'); 2616 while (this_esc != -1) { 2617 if (this_esc == orig.length() - 1) { 2618 sb.append(orig.substring(post_esc, this_esc + 1)); 2619 post_esc = this_esc + 1; 2620 break; 2621 } 2622 switch (orig.charAt(this_esc + 1)) { 2623 case 'n': 2624 sb.append(orig.substring(post_esc, this_esc)); 2625 sb.append('\n'); // not lineSep 2626 post_esc = this_esc + 2; 2627 break; 2628 case 'r': 2629 sb.append(orig.substring(post_esc, this_esc)); 2630 sb.append('\r'); 2631 post_esc = this_esc + 2; 2632 break; 2633 case '_': 2634 sb.append(orig.substring(post_esc, this_esc)); 2635 sb.append(' '); 2636 post_esc = this_esc + 2; 2637 break; 2638 case '\\': 2639 // This is not in the default case because the search would find 2640 // the quoted backslash. Here we incluce the first backslash in 2641 // the output, but not the first. 2642 sb.append(orig.substring(post_esc, this_esc + 1)); 2643 post_esc = this_esc + 2; 2644 break; 2645 2646 default: 2647 // In the default case, retain the character following the 2648 // backslash, but discard the backslash itself. "\*" is just 2649 // a one-character string. 2650 sb.append(orig.substring(post_esc, this_esc)); 2651 post_esc = this_esc + 1; 2652 break; 2653 } 2654 this_esc = orig.indexOf('\\', post_esc); 2655 } 2656 if (post_esc == 0) { 2657 return orig; 2658 } 2659 sb.append(orig.substring(post_esc)); 2660 return sb.toString(); 2661 } 2662 2663 // The reverse of unescape_decl. Test them together. 2664 /** 2665 * Converts a name into its declaration record version. In the declaration record, blanks are 2666 * encoded as \_ and backslashes as \\. 2667 * 2668 * @param orig the name of a declaration 2669 * @return the representation of the name in a declaration file 2670 */ 2671 @SuppressWarnings("UnusedMethod") 2672 private static String escape_decl(String orig) { 2673 return orig.replace("\\", "\\\\") 2674 .replace(" ", "\\_") 2675 .replace("\n", "\\n") // not lineSep 2676 .replace("\r", "\\r"); 2677 } 2678 2679 /** 2680 * Class that holds information from the declaration record (in the file). Once collected, this 2681 * information is used to create a VarInfo. This class is necessary because a VarInfo cannot be 2682 * created until much of this information is present: the constructor requires all the information 2683 * at the time of construction, and some of the fields are final. 2684 * 2685 * <p>In general, each field has a one-to-one relation with the corresponding entry in the 2686 * variable definition block in the trace file. More detailed information about each of the fields 2687 * can be found in the 'Variable declarations' section of the 'File Formats' appendix of the 2688 * Daikon developers manual. Specifics can also be found in the 'parse_[field]' methods of the 2689 * class (eg, parse_var_kind, parse_enclosing_var_name, etc). 2690 */ 2691 @SuppressWarnings( 2692 "nullness") // undocumented class needs documentation before annotating with nullness 2693 public static class VarDefinition implements java.io.Serializable, Cloneable { 2694 static final long serialVersionUID = 20060524L; 2695 2696 /** Current information about input file and previously parsed values. */ 2697 transient ParseState state; 2698 2699 /** Name of the variable (required). */ 2700 public String name; 2701 2702 /** Type of the variable (required). */ 2703 public VarKind kind = null; 2704 2705 /** Name of variable that contains this variable (optional) */ 2706 // seems non-null for arrays/sequences 2707 public @Nullable String enclosing_var_name; 2708 2709 /** the simple (not fully specified) name of this variable (optional) */ 2710 public @Nullable String relative_name = null; 2711 2712 /** Type of reference for structure/class variables. */ 2713 public RefType ref_type = RefType.POINTER; 2714 2715 /** Number of array dimensions (0 or 1). */ 2716 public int arr_dims = 0; 2717 2718 /** 2719 * Non-null iff (vardef.kind == VarKind.FUNCTION). The arguments that were used to create this 2720 * function application. 2721 */ 2722 @SuppressWarnings("serial") 2723 public @Nullable List<String> function_args = null; 2724 2725 /** The type of the variable as stored in the dtrace file (required) */ 2726 public ProglangType rep_type = null; 2727 2728 /** Declared type of the variable as an arbitrary string (required) */ 2729 public ProglangType declared_type = null; 2730 2731 /** Variable flags (optional) */ 2732 public EnumSet<VarFlags> flags = EnumSet.noneOf(VarFlags.class); 2733 2734 /** Language specific variable flags (optional) */ 2735 public EnumSet<LangFlags> lang_flags = EnumSet.noneOf(LangFlags.class); 2736 2737 /** Comparability of this variable (required. */ 2738 @SuppressWarnings("serial") 2739 public VarComparability comparability = null; 2740 2741 /** Parent program points in ppt hierarchy (optional) */ 2742 @SuppressWarnings("serial") 2743 public List<VarParent> parents; 2744 2745 /** Non-null if this 'variable' always has the same value (optional) */ 2746 @SuppressWarnings("serial") 2747 public @Nullable @Interned Object static_constant_value = null; 2748 2749 /** 2750 * Non-null if it is statically known that the value of the variable will be always greater than 2751 * or equal to this value. 2752 */ 2753 public @Nullable String min_value = null; 2754 2755 /** 2756 * Non-null if it is statically known that the value of the variable will be always less than or 2757 * equal to this value. 2758 */ 2759 public @Nullable String max_value = null; 2760 2761 /** Non-null if it is statically known that the array will have at least this many elements. */ 2762 public @Nullable Integer min_length = null; 2763 2764 /** Non-null if it is statically known that the array will have up to this many elements. */ 2765 public @Nullable Integer max_length = null; 2766 2767 /** Non-null if the set of valid values for the variable is statically known. */ 2768 public @Nullable String valid_values = null; 2769 2770 /** Check representation invariants. */ 2771 public void checkRep() { 2772 2773 // Basic checking for sensible input 2774 assert name != null; 2775 if (kind == null) { 2776 throw new AssertionError("missing var-kind information for variable " + name); 2777 } 2778 assert (arr_dims == 0) || (arr_dims == 1) 2779 : String.format( 2780 "array dimensions==%s, should be 0 or 1, for variable %s", arr_dims, name); 2781 assert !rep_type.isArray() || arr_dims == 1 2782 : String.format("array dimensions is 0, should be 1, for variable %s", name); 2783 if (rep_type == null) { 2784 throw new AssertionError("missing rep-type information for variable " + name); 2785 } 2786 if (declared_type == null) { 2787 throw new AssertionError("missing dec-type information for variable " + name); 2788 } 2789 if (comparability == null) { 2790 throw new AssertionError("missing comparability information for variable " + name); 2791 } 2792 assert (kind == VarKind.FUNCTION) || (function_args == null) 2793 : String.format( 2794 "incompatible kind=%s and function_args=%s for VarDefinition %s", 2795 kind, function_args, name); 2796 if ((kind == VarKind.FIELD || kind == VarKind.ARRAY) && enclosing_var_name == null) { 2797 throw new AssertionError("enclosing-var not specified for variable " + name); 2798 } 2799 } 2800 2801 /** Initialize from the 'variable <em>name</em>' record. Scanner should be pointing at name. */ 2802 public VarDefinition(ParseState state, Scanner scanner) { 2803 this.state = state; 2804 this.parents = new ArrayList<VarParent>(); 2805 name = need(scanner, "name"); 2806 need_eol(scanner); 2807 if (state.varcomp_format == VarComparability.IMPLICIT) { 2808 comparability = VarComparabilityImplicit.unknown; 2809 } else { 2810 comparability = VarComparabilityNone.it; 2811 } 2812 } 2813 2814 public VarDefinition(String name, VarKind kind, ProglangType type) { 2815 this.state = null; 2816 this.parents = new ArrayList<VarParent>(); 2817 this.name = name; 2818 this.kind = kind; 2819 this.rep_type = type; 2820 this.declared_type = type; 2821 comparability = VarComparabilityNone.it; 2822 } 2823 2824 @SideEffectFree 2825 @Override 2826 public VarDefinition clone(@GuardSatisfied VarDefinition this) { 2827 try { 2828 return (VarDefinition) super.clone(); 2829 } catch (CloneNotSupportedException e) { 2830 throw new Error("This can't happen: ", e); 2831 } 2832 } 2833 2834 public VarDefinition copy() { 2835 try { 2836 VarDefinition copy = this.clone(); 2837 copy.flags = flags.clone(); 2838 copy.lang_flags = lang_flags.clone(); 2839 return copy; 2840 } catch (Throwable t) { 2841 throw new RuntimeException(t); 2842 } 2843 } 2844 2845 /** Restore interned strings. */ 2846 private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { 2847 in.defaultReadObject(); 2848 name = name.intern(); 2849 if (enclosing_var_name != null) { 2850 enclosing_var_name = enclosing_var_name.intern(); 2851 } 2852 if (relative_name != null) { 2853 relative_name = relative_name.intern(); 2854 } 2855 for (VarParent parent : parents) { 2856 parent.parent_ppt = parent.parent_ppt.intern(); 2857 if (parent.parent_variable != null) { 2858 parent.parent_variable = parent.parent_variable.intern(); 2859 } 2860 } 2861 } 2862 2863 /** Clears the parent relations, if any existed. */ 2864 public void clear_parent_relation() { 2865 parents.clear(); 2866 } 2867 2868 /** Parse a var-kind record. Scanner should be pointing at the variable kind. */ 2869 public void parse_var_kind(Scanner scanner) { 2870 VarKind kind_local = parse_enum_val(scanner, VarKind.class, "variable kind"); 2871 kind = kind_local; 2872 2873 if ((kind == VarKind.FIELD) || (kind == VarKind.FUNCTION)) { 2874 relative_name = need(scanner, "relative name"); 2875 } 2876 need_eol(scanner); 2877 } 2878 2879 /** Parses the enclosing-var record. */ 2880 public void parse_enclosing_var_name(Scanner scanner) { 2881 enclosing_var_name = need(scanner, "enclosing variable name"); 2882 need_eol(scanner); 2883 } 2884 2885 /** Parses the reference-type record. */ 2886 public void parse_reference_type(Scanner scanner) { 2887 RefType ref_type_local = parse_enum_val(scanner, RefType.class, "reference type"); 2888 ref_type = ref_type_local; 2889 need_eol(scanner); 2890 } 2891 2892 /** Parses the array record. */ 2893 public void parse_array(Scanner scanner) { 2894 @Interned String arr_str = need(scanner, "array dimensions"); 2895 if (arr_str == "0") { // interned 2896 arr_dims = 0; 2897 } else if (arr_str == "1") { // interned 2898 arr_dims = 1; 2899 } else { 2900 decl_error(state, "%s found where 0 or 1 expected", arr_str); 2901 } 2902 } 2903 2904 /** Parses the function-args record. */ 2905 public void parse_function_args(Scanner scanner) { 2906 2907 function_args = new ArrayList<String>(); 2908 while (scanner.hasNext()) { 2909 function_args.add(unescape_decl(scanner.next()).intern()); 2910 } 2911 } 2912 2913 public void parse_rep_type(Scanner scanner) { 2914 @Interned String rep_type_str = need(scanner, "rep type"); 2915 need_eol(scanner); 2916 rep_type = ProglangType.rep_parse(rep_type_str); 2917 } 2918 2919 public void parse_dec_type(Scanner scanner) { 2920 @Interned String declared_type_str = need(scanner, "declaration type"); 2921 need_eol(scanner); 2922 declared_type = ProglangType.parse(declared_type_str); 2923 } 2924 2925 /** Parse the flags record. Multiple flags can be specified. */ 2926 public void parse_flags(Scanner scanner) { 2927 2928 flags.add(parse_enum_val(scanner, VarFlags.class, "Flag")); 2929 while (scanner.hasNext()) flags.add(parse_enum_val(scanner, VarFlags.class, "Flag")); 2930 // System.out.printf("flags for %s are %s%n", name, flags); 2931 } 2932 2933 /** Parse the langauge specific flags record. Multiple flags can be specified. */ 2934 public void parse_lang_flags(Scanner scanner) { 2935 2936 lang_flags.add(parse_enum_val(scanner, LangFlags.class, "Language Specific Flag")); 2937 while (scanner.hasNext()) { 2938 lang_flags.add(parse_enum_val(scanner, LangFlags.class, "Language Specific Flag")); 2939 } 2940 } 2941 2942 /** Parses a comparability record. */ 2943 public void parse_comparability(Scanner scanner) { 2944 @Interned String comparability_str = need(scanner, "comparability"); 2945 need_eol(scanner); 2946 comparability = 2947 VarComparability.parse(state.varcomp_format, comparability_str, declared_type); 2948 } 2949 2950 /** Parse a parent ppt record. */ 2951 public void parse_parent(Scanner scanner, List<ParentRelation> ppt_parents) 2952 throws Daikon.ParseError { 2953 2954 String parent_ppt = need(scanner, "parent ppt"); 2955 String parent_relation_id_string = need(scanner, "parent id"); 2956 int parent_relation_id; 2957 try { 2958 parent_relation_id = Integer.parseInt(parent_relation_id_string); 2959 } catch (NumberFormatException nfe) { 2960 throw new Daikon.ParseError("Expected a number, found: " + parent_relation_id_string); 2961 } 2962 String parent_variable = null; 2963 2964 boolean found = false; 2965 for (ParentRelation pr : ppt_parents) { 2966 if ((pr.parent_ppt_name == parent_ppt) && (pr.id == parent_relation_id)) { 2967 found = true; 2968 break; 2969 } 2970 } 2971 if (!found) { 2972 decl_error( 2973 state, 2974 "specified parent ppt '%s[%d]' for variable '%s' is not a parent to this ppt", 2975 parent_ppt, 2976 parent_relation_id, 2977 name); 2978 } 2979 if (scanner.hasNext()) { 2980 parent_variable = need(scanner, "parent variable"); 2981 } 2982 2983 parents.add(new VarParent(parent_ppt, parent_relation_id, parent_variable)); 2984 2985 need_eol(scanner); 2986 } 2987 2988 /** Parse a constant record. */ 2989 public void parse_constant(Scanner scanner) { 2990 @Interned String constant_str = need(scanner, "constant value"); 2991 need_eol(scanner); 2992 try { 2993 static_constant_value = rep_type.parse_value(constant_str, null, "parse_constant"); 2994 } catch (Error e) { 2995 decl_error(state, e); 2996 } 2997 } 2998 2999 /** Parse a minimum value record. */ 3000 public void parse_min_value(Scanner scanner) { 3001 this.min_value = need(scanner, "minimum value"); 3002 need_eol(scanner); 3003 } 3004 3005 /** Parse a maximum value record. */ 3006 public void parse_max_value(Scanner scanner) { 3007 this.max_value = need(scanner, "maximum value"); 3008 need_eol(scanner); 3009 } 3010 3011 /** Parse a minimum length record. */ 3012 public void parse_min_length(Scanner scanner) { 3013 this.min_length = Integer.parseInt(need(scanner, "minimum length")); 3014 need_eol(scanner); 3015 } 3016 3017 /** Parse a maximum length record. */ 3018 public void parse_max_length(Scanner scanner) { 3019 this.max_length = Integer.parseInt(need(scanner, "maximum length")); 3020 need_eol(scanner); 3021 } 3022 3023 /** Parse a valid values record. */ 3024 public void parse_valid_values(Scanner scanner) { 3025 this.valid_values = scanner.nextLine(); 3026 } 3027 3028 /** 3029 * Helper function, returns the next string token unescaped and interned. Throw Daikon.UserError 3030 * if there is no next token. 3031 */ 3032 public @Interned String need(Scanner scanner, String description) { 3033 return FileIO.need(state, scanner, description); 3034 } 3035 3036 /** Throws Daikon.UserError if the scanner is not at end of line */ 3037 public void need_eol(Scanner scanner) { 3038 FileIO.need_eol(state, scanner); 3039 } 3040 3041 /** 3042 * Looks up the next token as a member of enum_class. Throws Daikon.UserError if there is no 3043 * token or if it is not valid member of the class. Enums are presumed to be in in upper case. 3044 */ 3045 public <E extends Enum<E>> E parse_enum_val( 3046 Scanner scanner, Class<E> enum_class, String descr) { 3047 return FileIO.parse_enum_val(state, scanner, enum_class, descr); 3048 } 3049 } 3050 3051 /** 3052 * Helper function, returns the next string token unescaped and interned. Throws Daikon.UserError 3053 * if there is no next token. 3054 */ 3055 public static @Interned String need(ParseState state, Scanner scanner, String description) { 3056 if (!scanner.hasNext()) { 3057 decl_error(state, "end-of-line found where %s expected", description); 3058 } 3059 return unescape_decl(scanner.next()).intern(); 3060 } 3061 3062 /** Throws a Daikon.UserError if the scanner is not at end of line */ 3063 public static void need_eol(ParseState state, Scanner scanner) { 3064 if (scanner.hasNext()) { 3065 decl_error(state, "'%s' found where end-of-line expected", scanner.next()); 3066 } 3067 } 3068 3069 /** 3070 * Looks up the next token as a member of enum_class. Throws Daikon.UserError if there is no token 3071 * or if it is not valid member of the class. Enums are presumed to be in in upper case. 3072 */ 3073 public static <E extends Enum<E>> E parse_enum_val( 3074 ParseState state, Scanner scanner, Class<E> enum_class, String descr) { 3075 3076 @Interned String str = need(state, scanner, descr); 3077 try { 3078 E e = Enum.valueOf(enum_class, str.toUpperCase()); 3079 return e; 3080 } catch (Exception exception) { 3081 @SuppressWarnings( 3082 "nullness") // getEnumConstants returns non-null because enum_class is an enum class 3083 E @NonNull [] all = enum_class.getEnumConstants(); 3084 StringJoiner msg = new StringJoiner(", "); 3085 for (E e : all) { 3086 msg.add(String.format("'%s'", e.name().toLowerCase())); 3087 } 3088 decl_error(state, "'%s' found where %s expected", str, msg); 3089 throw new Error("execution cannot get to here, previous line threw an error"); 3090 } 3091 } 3092 3093 /** 3094 * Call this to indicate a malformed declaration. 3095 * 3096 * @param state the current parse state 3097 * @param format a format string, for the error message 3098 * @param args arguments for the format string 3099 */ 3100 private static void decl_error(ParseState state, String format, @Nullable Object... args) { 3101 @SuppressWarnings({ 3102 "formatter:unneeded.suppression", // temporary? 3103 "formatter:format.string" // https://tinyurl.com/cfissue/2584 3104 }) 3105 String msg = String.format(format, args) + state.line_file_message(); 3106 throw new Daikon.UserError(msg); 3107 } 3108 3109 /** Call this to indicate a malformed declaration. */ 3110 private static void decl_error(ParseState state, Throwable cause) { 3111 String msg = cause.getMessage() + state.line_file_message(); 3112 if (msg.startsWith("null at")) { 3113 msg = msg.substring(5); 3114 } 3115 throw new Daikon.UserError(cause, msg); 3116 } 3117 3118 /** Returns whether the line is the start of a ppt declaration. */ 3119 @RequiresNonNull("FileIO.new_decl_format") 3120 @Pure 3121 private static boolean is_declaration_header(String line) { 3122 if (new_decl_format) { 3123 return line.startsWith("ppt "); 3124 } else { 3125 return line.equals(declaration_header); 3126 } 3127 } 3128 3129 /** 3130 * Handle any possible modifications to the ppt name. For now, just support the Applications 3131 * Communities specific modification to remove duplicate stack entries. But a more generic 3132 * technique could be implemented in the future. 3133 */ 3134 public static String user_mod_ppt_name(String ppt_name) { 3135 3136 if (!dkconfig_rm_stack_dups) { 3137 return ppt_name; 3138 } 3139 3140 // System.out.printf("removing stack dups (%b)in fileio%n", 3141 // dkconfig_rm_stack_dups); 3142 3143 String[] stack = ppt_name.split("[|]"); 3144 List<String> nd_stack = new ArrayList<>(); 3145 for (String si : stack) { 3146 if (nd_stack.contains(si)) { 3147 continue; 3148 } 3149 nd_stack.add(si); 3150 } 3151 return String.join("|", nd_stack).intern(); 3152 } 3153}