#!/usr/bin/env perl # # The LLVM Compiler Infrastructure # # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ##===----------------------------------------------------------------------===## # # A script designed to wrap a build so that all calls to gcc are intercepted # and piped to the static analyzer. # ##===----------------------------------------------------------------------===## use strict; use warnings; use FindBin qw($RealBin); use Digest::MD5; use File::Basename; use File::Find; use Term::ANSIColor; use Term::ANSIColor qw(:constants); use Cwd qw/ getcwd abs_path /; use Sys::Hostname; use Data::Dumper; use Config; use File::Copy qw(copy), qw(move); my $Windows = ""; if ($Config{osname} =~ "linux") { $Windows = ""; } my $Verbose = 0; # Verbose output from this script. my $Prog = "post-process"; my $BuildName; my $BuildDate; my $TERM = $ENV{'TERM'}; my $UseColor = (defined $TERM and $TERM =~ 'xterm-.*color' and -t STDOUT and defined $ENV{'SCAN_BUILD_COLOR'}); my $UserName = HtmlEscape(getlogin || getpwuid($<) || 'unknown'); my $HostName = HtmlEscape(hostname() || 'unknown'); my $CurrentDir = HtmlEscape(getcwd()); my $CurrentDirSuffix = basename($CurrentDir); my @PluginsToLoad; my $HtmlTitle = "Clang static analysis report generated by post-process script"; my $Date = localtime(); my $HtmlDir; my $HtmlDirSpecified = 0; my $BlacklistFile; my $BlacklistFileSpecified = 0; my @Blacklist; my $WhitelistFile; my $WhitelistFileSpecified = 0; my @Whitelist; my $AnalyzerStats = 0; my $KeepEmpty = 0; my @filesFound; my $baseDir; sub FileWanted { my $baseDirRegEx = quotemeta $baseDir; my $file = $File::Find::name; if ($file =~ /report-.*\.html$/) { my $relative_file = $file; $relative_file =~ s/$baseDirRegEx//g; push @filesFound, $relative_file; } } ##----------------------------------------------------------------------------## # Diagnostics ##----------------------------------------------------------------------------## sub Diag { if ($UseColor) { print BOLD, MAGENTA "$Prog: @_"; print RESET; } else { print "$Prog: @_"; } } sub ErrorDiag { if ($UseColor) { print STDERR BOLD, RED "$Prog: "; print STDERR RESET, RED @_; print STDERR RESET; } else { print STDERR "$Prog: @_"; } } sub DiagCrashes { my $Dir = shift; Diag ("The analyzer encountered problems on some source files.\n"); Diag ("Preprocessed versions of these sources were deposited in '$Dir/failures'.\n"); Diag ("Please consider submitting a bug report using these files:\n"); Diag (" http://clang-analyzer.llvm.org/filing_bugs.html\n") } sub DieDiag { if ($UseColor) { print STDERR BOLD, RED "$Prog: "; print STDERR RESET, RED @_; print STDERR RESET; } else { print STDERR "$Prog: ", @_; } exit 1; } ##----------------------------------------------------------------------------## # CopyFiles - Copy resource files to target directory. ##----------------------------------------------------------------------------## sub CopyFiles { my $Dir = shift; my $JS = Cwd::realpath("$RealBin/../share/scan-build/sorttable.js"); if($^O eq "MSWin32") { $JS = File::Spec->rel2abs("$RealBin/../share/scan-build/sorttable.js"); } DieDiag("Cannot find 'sorttable.js'.\n") if (! -r $JS); copy $JS, $Dir; DieDiag("Could not copy 'sorttable.js' to '$Dir'.\n") if (! -r "$Dir/sorttable.js"); my $CSS = Cwd::realpath("$RealBin/../share/scan-build/scanview.css"); if($^O eq "MSWin32") { $CSS = File::Spec->rel2abs("$RealBin/../share/scan-build/scanview.css"); } DieDiag("Cannot find 'scanview.css'.\n") if (! -r $CSS); copy $CSS, $Dir; DieDiag("Could not copy 'scanview.css' to '$Dir'.\n") if (! -r $CSS); } ##----------------------------------------------------------------------------## # ScanWhitelist - Scan the whitelist file for unsilenced reporistories ##----------------------------------------------------------------------------## sub ScanWhitelist { # Scan the report file for tags. open(IN, "$WhitelistFile") or DieDiag("Cannot open '$WhitelistFile'\n"); while (my $row = ) { $row =~ s@\\@/@g; chomp $row; push @Whitelist, $row; } close(IN); } ##----------------------------------------------------------------------------## # UpdatePrefix - Compute the common prefix of files. ##----------------------------------------------------------------------------## my $Prefix; sub UpdatePrefix { my $x = shift; print "\nUpdating prefix of $x"; my $y = basename($x); $x =~ s/\Q$y\E$//; if (!defined $Prefix) { $Prefix = $x; return; } chop $Prefix while (!($x =~ /^\Q$Prefix/)); } sub GetPrefix { return $Prefix; } ##----------------------------------------------------------------------------## # UpdateInFilePath - Update the path in the report file. ##----------------------------------------------------------------------------## sub UpdateInFilePath { my $fname = shift; my $regex = shift; my $newtext = shift; open (RIN, $fname) or die "cannot open $fname"; open (ROUT, ">", "$fname.tmp") or die "cannot open $fname.tmp"; while () { s/$regex/$newtext/; print ROUT $_; } close (ROUT); close (RIN); move "$fname.tmp", $fname; } ##----------------------------------------------------------------------------## # ComputeDigest - Compute a digest of the specified file. ##----------------------------------------------------------------------------## sub ComputeDigest { my $FName = shift; DieDiag("Cannot read $FName to compute Digest.\n") if (! -r $FName); # Use Digest::MD5. We don't have to be cryptographically secure. We're # just looking for duplicate files that come from a non-malicious source. # We use Digest::MD5 because it is a standard Perl module that should # come bundled on most systems. open(FILE, $FName) or DieDiag("Cannot open $FName when computing Digest.\n"); binmode FILE; my $Result = Digest::MD5->new->addfile(*FILE)->hexdigest; close(FILE); # Return the digest. return $Result; } ##----------------------------------------------------------------------------## # ScanBlacklist - Scan the blacklist file for silenced bugs. ##----------------------------------------------------------------------------## sub ScanBlacklist { # Scan the report file for tags. open(IN, "$BlacklistFile") or DieDiag("Cannot open '$BlacklistFile'\n"); while (my $row = ) { $row =~ s/^\s+|\s+$//g; push @Blacklist, $row; } close(IN); } ##----------------------------------------------------------------------------## # ShowProgress - Show the progress of post-processing on user terminal. ##----------------------------------------------------------------------------## my $printedPerc = 0; sub ShowProgress { my $numFilesScanned = shift; my $numFilesFound = shift; my $perc = int(($numFilesScanned / $numFilesFound)*100); my $pperc = $perc % 10; if (($pperc == 0) and ($printedPerc != $perc)) { print "..($perc%)"; if ($perc == 100) { print "\n"; } $printedPerc = $perc; } } ##----------------------------------------------------------------------------## # ScanFile - Scan a report file for various identifying attributes. ##----------------------------------------------------------------------------## # Sometimes a source file is scanned more than once, and thus produces # multiple error reports. We use a cache to solve this problem. my %AlreadyScanned; sub ScanFile { my $Index = shift; my $Dir = shift; my $FName = shift; my $Stats = shift; # Compute a digest for the report file. Determine if we have already # scanned a file that looks just like it. my $digest = ComputeDigest("$Dir/$FName"); if (defined $AlreadyScanned{$digest}) { # Redundant file. Remove it. unlink "$Dir/$FName"; return; } $AlreadyScanned{$digest} = 1; # At this point the report file is not world readable. Make it happen. if($^O eq "MSWin32") { chmod("+rw", "$Dir/$FName"); } else { system ("chmod", "644", "$Dir/$FName"); } # Scan the report file for tags. open(IN, "$Dir/$FName") or DieDiag("Cannot open '$Dir/$FName'\n"); my $BugType = ""; my $BugFile = ""; my $FunctionName = ""; my $BugCategory = ""; my $BugDescription = ""; my $BugPathLength = 1; my $BugLine = 0; my $FileName = ""; my $BugDir = ""; if ($Verbose) { Diag("Analyzing '$Dir/$FName'\n"); } while () { last if (/$Windows/); if (/$Windows$/) { $BugType = $1; } elsif (/$Windows$/) { $BugFile = $1; } elsif (/$Windows$/) { $BugPathLength = $1; } elsif (/$Windows$/) { $BugLine = $1; } elsif (/$Windows$/) { $BugCategory = $1; } elsif (/$Windows$/) { $BugDescription = $1; } elsif (/$Windows$/) { $FunctionName = $1; } elsif (/$Windows$/) { $FileName = $1; } } close(IN); if (!defined $BugCategory) { $BugCategory = "Other"; } if($WhitelistFileSpecified) { my $Whitelisted = 0; $BugDir = $BugFile; $BugDir =~ s@\\@/@g; chomp $BugDir; foreach (@Whitelist) { if (index($BugDir, $_) != -1) { $Whitelisted = 1; } } if($Whitelisted == 0) { unlink "$Dir/$FName"; return; } } if (!defined $FunctionName) { $FunctionName = "Global"; } if($BlacklistFileSpecified){ my $BugSummary = "${FileName} ${FunctionName} ${BugDescription}"; if (grep {$_ eq $BugSummary} @Blacklist) { # Redundant file. Remove it. unlink "$Dir/$FName"; return; } } # Don't add internal statistics to the bug reports if ($BugCategory =~ /statistics/i) { AddStatLine($BugDescription, $Stats, $BugFile); return; } push @$Index,[ $FName, $BugCategory, $BugType, $BugFile, $BugLine, $FunctionName, $BugPathLength ]; } ##----------------------------------------------------------------------------## # Diagnostics ##----------------------------------------------------------------------------## sub Postprocess { my $Dir = shift; my $AnalyzerStats = shift; my $KeepEmpty = shift; die "No directory specified." if (!defined $Dir); if (! -d $Dir) { Diag("No bugs found.\n"); return 0; } $baseDir = $Dir . "/"; find({ wanted => \&FileWanted, follow => 0}, $Dir); my $numFilesFound = scalar(@filesFound); if ($numFilesFound == 0 and ! -e "$Dir/failures") { if (! $KeepEmpty) { Diag("Removing directory '$Dir' because it contains no reports.\n"); rmdir $Dir; } Diag("No bugs found.\n"); return 0; } if($WhitelistFileSpecified){ ScanWhitelist(); } # Scan each report file and build an index. my @Index; my @Stats; my $numFilesScanned = 1; if($BlacklistFileSpecified){ ScanBlacklist(); } foreach my $file (@filesFound) { ScanFile(\@Index, $Dir, $file , \@Stats); if (!$Verbose) { ShowProgress($numFilesScanned, $numFilesFound); } ++$numFilesScanned; } # Scan the failures directory and use the information in the .info files # to update the common prefix directory. my @failures; my @attributes_ignored; if (-d "$Dir/failures") { opendir(DIR, "$Dir/failures"); @failures = grep { /[.]info.txt$/ && !/attribute_ignored/; } readdir(DIR); closedir(DIR); opendir(DIR, "$Dir/failures"); @attributes_ignored = grep { /^attribute_ignored/; } readdir(DIR); closedir(DIR); foreach my $file (@failures) { open IN, "$Dir/failures/$file" or DieDiag("cannot open $file\n"); my $Path = ; if (defined $Path) { UpdatePrefix($Path); } close IN; } } # Generate an index.html file. my $FName = "$Dir/index.html"; open(OUT, ">", $FName) or DieDiag("Cannot create file '$FName'\n"); # Print out the header. print OUT < ${HtmlTitle}

${HtmlTitle}

ENDTEXT print OUT "\n" if (defined($BuildName) && defined($BuildDate)); print OUT < ENDTEXT if (scalar(@filesFound)) { # Print out the summary table. my %Totals; for my $row ( @Index ) { my $bug_type = ($row->[2]); my $bug_category = ($row->[1]); my $key = "$bug_category:$bug_type"; if (!defined $Totals{$key}) { $Totals{$key} = [1,$bug_category,$bug_type]; } else { $Totals{$key}->[0]++; } } print OUT "

Bug Summary

"; if (defined $BuildName) { print OUT "\n

Results in this analysis run are based on analyzer build $BuildName.

\n" } my $TotalBugs = scalar(@Index); print OUT <
ENDTEXT my $last_category; for my $key ( sort { my $x = $Totals{$a}; my $y = $Totals{$b}; my $res = $x->[1] cmp $y->[1]; $res = $x->[2] cmp $y->[2] if ($res == 0); $res } keys %Totals ) { my $val = $Totals{$key}; my $category = $val->[1]; if (!defined $last_category or $last_category ne $category) { $last_category = $category; print OUT "\n"; } my $x = lc $key; $x =~ s/[ ,'":\/()]+/_/g; print OUT "\n"; } # Print out the table of errors. print OUT <

Reports

User:${UserName}\@${HostName}
Working Directory:${CurrentDir}
Date:${Date}
Version:${BuildName} (${BuildDate})
Bug TypeQuantityDisplay?
All Bugs$TotalBugs
$category
"; print OUT $val->[2]; print OUT ""; print OUT $val->[0]; print OUT "
ENDTEXT my $prefix = GetPrefix(); my $regex; my $InFileRegex; my $InFilePrefix = "File:"; print OUT ""; print OUT ""; # Update the file prefix. my $fname = $row->[3]; if (defined $regex) { $fname =~ s/$regex//; UpdateInFilePath("$Dir/$ReportFile", $InFileRegex, $InFilePrefix) } print OUT ""; # Print out the quantities. for my $j ( 4 .. 5 ) { print OUT ""; } # Print the rest of the columns. for (my $j = 6; $j <= $#{$row}; ++$j) { print OUT "" } # Emit the "View" link. print OUT ""; # Emit REPORTBUG markers. print OUT "\n\n"; # End the row. print OUT "\n"; } print OUT "\n
Bug Group Bug Type ▾ File Line Path Length
"; if (defined $prefix) { $regex = qr/^\Q$prefix\E/is; $InFileRegex = qr/\Q$InFilePrefix$prefix\E/is; } for my $row ( sort { $a->[2] cmp $b->[2] } @Index ) { my $x = "$row->[1]:$row->[2]"; $x = lc $x; $x =~ s/[ ,'":\/()]+/_/g; my $ReportFile = $row->[0]; print OUT "
"; print OUT $row->[1]; print OUT ""; print OUT $row->[2]; print OUT ""; my @fname = split /\//,$fname; if ($#fname > 0) { while ($#fname >= 0) { my $x = shift @fname; print OUT $x; if ($#fname >= 0) { print OUT " /"; } } } else { print OUT $fname; } print OUT "$row->[$j]$row->[$j]View Report
\n\n"; } if (scalar (@failures) || scalar(@attributes_ignored)) { print OUT "

Analyzer Failures

\n"; if (scalar @attributes_ignored) { print OUT "The analyzer's parser ignored the following attributes:

\n"; print OUT "\n"; print OUT "\n"; foreach my $file (sort @attributes_ignored) { die "cannot demangle attribute name\n" if (! ($file =~ /^attribute_ignored_(.+).txt/)); my $attribute = $1; # Open the attribute file to get the first file that failed. next if (!open (ATTR, "$Dir/failures/$file")); my $ppfile = ; chomp $ppfile; close ATTR; next if (! -e "$Dir/failures/$ppfile"); # Open the info file and get the name of the source file. open (INFO, "$Dir/failures/$ppfile.info.txt") or die "Cannot open $Dir/failures/$ppfile.info.txt\n"; my $srcfile = ; chomp $srcfile; close (INFO); # Print the information in the table. my $prefix = GetPrefix(); if (defined $prefix) { $srcfile =~ s/^\Q$prefix//; } print OUT "\n"; my $ppfile_clang = $ppfile; $ppfile_clang =~ s/[.](.+)$/.clang.$1/; print OUT " \n"; } print OUT "
AttributeSource FilePreprocessed FileSTDERR Output
$attribute$srcfile$ppfile$ppfile.stderr.txt
\n"; } if (scalar @failures) { print OUT "

The analyzer had problems processing the following files:

\n"; print OUT "\n"; print OUT "\n"; foreach my $file (sort @failures) { $file =~ /(.+).info.txt$/; # Get the preprocessed file. my $ppfile = $1; # Open the info file and get the name of the source file. open (INFO, "$Dir/failures/$file") or die "Cannot open $Dir/failures/$file\n"; my $srcfile = ; chomp $srcfile; my $problem = ; chomp $problem; close (INFO); # Print the information in the table. my $prefix = GetPrefix(); if (defined $prefix) { $srcfile =~ s/^\Q$prefix//; } print OUT "\n"; my $ppfile_clang = $ppfile; $ppfile_clang =~ s/[.](.+)$/.clang.$1/; print OUT " \n"; } print OUT "
ProblemSource FilePreprocessed FileSTDERR Output
$problem$srcfile$ppfile$ppfile.stderr.txt
\n"; } print OUT "

Please consider submitting preprocessed files as bug reports.

\n"; } print OUT "\n"; close(OUT); CopyFiles($Dir); # Print statistics print CalcStats(\@Stats) if $AnalyzerStats; my $Num = scalar(@Index); Diag("$Num bugs found.\n"); if ($Num > 0 && -r "$Dir/index.html") { Diag("Open '$Dir/index.html' to examine summarized report.\n"); } DiagCrashes($Dir) if (scalar @failures || scalar @attributes_ignored); return $Num; } ##----------------------------------------------------------------------------## # HtmlEscape - HTML entity encode characters that are special in HTML ##----------------------------------------------------------------------------## sub HtmlEscape { # copy argument to new variable so we don't clobber the original my $arg = shift || ''; my $tmp = $arg; $tmp =~ s/&/&/g; $tmp =~ s//>/g; return $tmp; } ##----------------------------------------------------------------------------## # DisplayHelp - Utility function to display all help options. ##----------------------------------------------------------------------------## sub DisplayHelp { print < [build options] ENDTEXT if (defined $BuildName) { print "ANALYZER BUILD: $BuildName ($BuildDate)\n\n"; } print < Specifies the directory with static analyzer report files. If this option is not specified the program exits. -h --help Display this message. --html-title [title] --html-title=[title] Specify the title used on generated HTML pages. If not specified, a default title will be used. --blacklist-file Specifies the location of the blacklist file. Contents of the file should be row-wise entries in the format: example: test.cpp foo Address of stack memory associated with local variable 'buff' returned to caller [core.StackAddressEscape] --keep-empty Don't remove the build results directory even if no issues were reported. --windows-format The html files were generated on windows machine and hence has '\r' line endings. --verbose Verbose output. --whitelist-file Specifies the location of the whitelist file. Contents of the file should be row-wise entries of directories to be whitelisted for scanning. These entries may be sub-strings of the entire directory of the file. example: modem_proc/my/project qcomm Will ensure that any reports that are in directories containing the substrings above will be displayed while those that don't wont. ENDTEXT } ##----------------------------------------------------------------------------## # Process command-line arguments. ##----------------------------------------------------------------------------## my $RequestDisplayHelp = 0; my $ForceDisplayHelp = 0; if (!@ARGV) { $ForceDisplayHelp = 1; } while (@ARGV) { # Scan for options we recognize. my $arg = $ARGV[0]; if ($arg eq "-h" or $arg eq "--help") { $RequestDisplayHelp = 1; shift @ARGV; next; } if ($arg eq "--verbose") { $Verbose = 1; shift @ARGV; next; } if ($arg eq "--whitelist-file") { shift @ARGV; if (!@ARGV) { DieDiag("'--whitelist-dir' option requires a target filename.\n"); } # Construct an absolute path. Uses the current working directory # as a base if the original path was not absolute. $WhitelistFile = abs_path(shift @ARGV); $WhitelistFileSpecified = 1; next; } if ($arg eq "--report-dir") { shift @ARGV; if (!@ARGV) { DieDiag("'--report-dir' option requires a target directory name.\n"); } # Construct an absolute path. Uses the current working directory # as a base if the original path was not absolute. $HtmlDir = abs_path(shift @ARGV); $HtmlDirSpecified = 1; next; } if ($arg eq "--blacklist-file") { shift @ARGV; if (!@ARGV) { DieDiag("'--blacklist-file' option requires a target file name.\n"); } # Construct an absolute path. Uses the current working directory # as a base if the original path was not absolute. $BlacklistFile = abs_path(shift @ARGV); DieDiag("Cannot find Blacklist file $BlacklistFile\n") if (! -r $BlacklistFile); $BlacklistFileSpecified = 1; next; } if ($arg =~ /^--html-title(=(.+))?$/) { shift @ARGV; if (!defined $2 || $2 eq '') { if (!@ARGV) { DieDiag("'--html-title' option requires a string.\n"); } $HtmlTitle = shift @ARGV; } else { $HtmlTitle = $2; } next; } if ($arg eq "--keep-empty") { shift @ARGV; $KeepEmpty = 1; next; } if ($arg eq "--windows-format") { shift @ARGV; $Windows = "\r"; next; } DieDiag("unrecognized option '$arg'\n"); last; } if(!$RequestDisplayHelp && $HtmlDirSpecified) { if (-d $HtmlDir) { if (! -r $HtmlDir) { DieDiag("directory '$HtmlDir' exists but is not readable.\n"); } } else { DieDiag("report directory does not exist.\n"); $ForceDisplayHelp = 1; } } else { $ForceDisplayHelp = 1; } if ($ForceDisplayHelp || $RequestDisplayHelp) { DisplayHelp(); exit $ForceDisplayHelp; } Diag("Will read the reports from '$HtmlDir'\n"); if (!$KeepEmpty) { Diag("Will remove the report directory if it contains no reports.\n"); } Diag("Scanning started...\n"); my $NumBugs = Postprocess($HtmlDir, $AnalyzerStats, $KeepEmpty); exit 0;