#! /usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
'''This script analyzes a JSON file emitted by DMD.'''
from __future__ import print_function, division
import argparse
import collections
import gzip
import json
import os
import platform
import re
import shutil
import sys
import tempfile
from bisect import bisect_right
# The DMD output version this script handles.
outputVersion = 5
# If --ignore-alloc-fns is specified, stack frames containing functions that
# match these strings will be removed from the *start* of stack traces. (Once
# we hit a non-matching frame, any subsequent frames won't be removed even if
# they do match.)
allocatorFns = [
# Matches malloc, replace_malloc, moz_xmalloc, vpx_malloc, js_malloc, pod_malloc, malloc_zone_*, g_malloc.
'malloc',
# Matches calloc, replace_calloc, moz_xcalloc, vpx_calloc, js_calloc, pod_calloc, malloc_zone_calloc, pod_callocCanGC.
'calloc',
# Matches realloc, replace_realloc, moz_xrealloc, vpx_realloc, js_realloc, pod_realloc, pod_reallocCanGC.
'realloc',
# Matches memalign, posix_memalign, replace_memalign, replace_posix_memalign, moz_xmemalign, vpx_memalign, malloc_zone_memalign.
'memalign',
'operator new(',
'operator new[](',
'g_slice_alloc',
# This one necessary to fully filter some sequences of allocation functions
# that happen in practice. Note that ??? entries that follow non-allocation
# functions won't be stripped, as explained above.
'???',
]
class Record(object):
'''A record is an aggregation of heap blocks that have identical stack
traces. It can also be used to represent the difference between two
records.'''
def __init__(self):
self.numBlocks = 0
self.reqSize = 0
self.slopSize = 0
self.usableSize = 0
self.allocatedAtDesc = None
self.reportedAtDescs = []
self.usableSizes = collections.defaultdict(int)
def isZero(self, args):
return self.numBlocks == 0 and \
self.reqSize == 0 and \
self.slopSize == 0 and \
self.usableSize == 0 and \
len(self.usableSizes) == 0
def negate(self):
self.numBlocks = -self.numBlocks
self.reqSize = -self.reqSize
self.slopSize = -self.slopSize
self.usableSize = -self.usableSize
negatedUsableSizes = collections.defaultdict(int)
for usableSize, count in self.usableSizes.items():
negatedUsableSizes[-usableSize] = count
self.usableSizes = negatedUsableSizes
def subtract(self, r):
# We should only be calling this on records with matching stack traces.
# Check this.
assert self.allocatedAtDesc == r.allocatedAtDesc
assert self.reportedAtDescs == r.reportedAtDescs
self.numBlocks -= r.numBlocks
self.reqSize -= r.reqSize
self.slopSize -= r.slopSize
self.usableSize -= r.usableSize
usableSizes1 = self.usableSizes
usableSizes2 = r.usableSizes
usableSizes3 = collections.defaultdict(int)
for usableSize in usableSizes1:
counts1 = usableSizes1[usableSize]
if usableSize in usableSizes2:
counts2 = usableSizes2[usableSize]
del usableSizes2[usableSize]
counts3 = counts1 - counts2
if counts3 != 0:
if counts3 < 0:
usableSize = -usableSize
counts3 = -counts3
usableSizes3[usableSize] = counts3
else:
usableSizes3[usableSize] = counts1
for usableSize in usableSizes2:
usableSizes3[-usableSize] = usableSizes2[usableSize]
self.usableSizes = usableSizes3
@staticmethod
def cmpByUsableSize(r1, r2):
# Sort by usable size, then by req size.
return cmp(abs(r1.usableSize), abs(r2.usableSize)) or \
Record.cmpByReqSize(r1, r2)
@staticmethod
def cmpByReqSize(r1, r2):
# Sort by req size.
return cmp(abs(r1.reqSize), abs(r2.reqSize))
@staticmethod
def cmpBySlopSize(r1, r2):
# Sort by slop size.
return cmp(abs(r1.slopSize), abs(r2.slopSize))
@staticmethod
def cmpByNumBlocks(r1, r2):
# Sort by block counts, then by usable size.
return cmp(abs(r1.numBlocks), abs(r2.numBlocks)) or \
Record.cmpByUsableSize(r1, r2)
sortByChoices = {
'usable': Record.cmpByUsableSize, # the default
'req': Record.cmpByReqSize,
'slop': Record.cmpBySlopSize,
'num-blocks': Record.cmpByNumBlocks,
}
def parseCommandLine():
# 24 is the maximum number of frames that DMD will produce.
def range_1_24(string):
value = int(string)
if value < 1 or value > 24:
msg = '{:s} is not in the range 1..24'.format(string)
raise argparse.ArgumentTypeError(msg)
return value
description = '''
Analyze heap data produced by DMD.
If one file is specified, analyze it; if two files are specified, analyze the
difference.
Input files can be gzipped.
Write to stdout unless -o/--output is specified.
Stack traces are fixed to show function names, filenames and line numbers
unless --no-fix-stacks is specified; stack fixing modifies the original file
and may take some time. If specified, the BREAKPAD_SYMBOLS_PATH environment
variable is used to find breakpad symbols for stack fixing.
'''
p = argparse.ArgumentParser(description=description)
p.add_argument('-o', '--output', type=argparse.FileType('w'),
help='output file; stdout if unspecified')
p.add_argument('-f', '--max-frames', type=range_1_24, default=8,
help='maximum number of frames to consider in each trace')
p.add_argument('-s', '--sort-by', choices=sortByChoices.keys(),
default='usable',
help='sort the records by a particular metric')
p.add_argument('-a', '--ignore-alloc-fns', action='store_true',
help='ignore allocation functions at the start of traces')
p.add_argument('--no-fix-stacks', action='store_true',
help='do not fix stacks')
p.add_argument('--clamp-contents', action='store_true',
help='for a scan mode log, clamp addresses to the start of live blocks, or zero if not in one')
p.add_argument('--print-clamp-stats', action='store_true',
help='print information about the results of pointer clamping; mostly useful for debugging clamping')
p.add_argument('--filter-stacks-for-testing', action='store_true',
help='filter stack traces; only useful for testing purposes')
p.add_argument('input_file',
help='a file produced by DMD')
p.add_argument('input_file2', nargs='?',
help='a file produced by DMD; if present, it is diff\'d with input_file')
return p.parse_args(sys.argv[1:])
# Fix stacks if necessary: first write the output to a tempfile, then replace
# the original file with it.
def fixStackTraces(inputFilename, isZipped, opener):
# This append() call is needed to make the import statements work when this
# script is installed as a symlink.
sys.path.append(os.path.dirname(__file__))
bpsyms = os.environ.get('BREAKPAD_SYMBOLS_PATH', None)
sysname = platform.system()
if bpsyms and os.path.exists(bpsyms):
import fix_stack_using_bpsyms as fixModule
fix = lambda line: fixModule.fixSymbols(line, bpsyms)
elif sysname == 'Linux':
import fix_linux_stack as fixModule
fix = lambda line: fixModule.fixSymbols(line)
elif sysname == 'Darwin':
import fix_macosx_stack as fixModule
fix = lambda line: fixModule.fixSymbols(line)
else:
fix = None # there is no fix script for Windows
if fix:
# Fix stacks, writing output to a temporary file, and then
# overwrite the original file.
tmpFile = tempfile.NamedTemporaryFile(delete=False)
# If the input is gzipped, then the output (written initially to
# |tmpFile|) should be gzipped as well.
#
# And we want to set its pre-gzipped filename to '' rather than the
# name of the temporary file, so that programs like the Unix 'file'
# utility don't say that it was called 'tmp6ozTxE' (or something like
# that) before it was zipped. So that explains the |filename=''|
# parameter.
#
# But setting the filename like that clobbers |tmpFile.name|, so we
# must get that now in order to move |tmpFile| at the end.
tmpFilename = tmpFile.name
if isZipped:
tmpFile = gzip.GzipFile(filename='', fileobj=tmpFile)
with opener(inputFilename, 'rb') as inputFile:
for line in inputFile:
tmpFile.write(fix(line))
tmpFile.close()
shutil.move(tmpFilename, inputFilename)
def getDigestFromFile(args, inputFile):
# Handle gzipped input if necessary.
isZipped = inputFile.endswith('.gz')
opener = gzip.open if isZipped else open
# Fix stack traces unless otherwise instructed.
if not args.no_fix_stacks:
fixStackTraces(inputFile, isZipped, opener)
if args.clamp_contents:
clampBlockList(args, inputFile, isZipped, opener)
with opener(inputFile, 'rb') as f:
j = json.load(f)
if j['version'] != outputVersion:
raise Exception("'version' property isn't '{:d}'".format(outputVersion))
# Extract the main parts of the JSON object.
invocation = j['invocation']
dmdEnvVar = invocation['dmdEnvVar']
mode = invocation['mode']
blockList = j['blockList']
traceTable = j['traceTable']
frameTable = j['frameTable']
# Insert the necessary entries for unrecorded stack traces. Note that 'ut'
# and 'uf' will not overlap with any keys produced by DMD's
# ToIdStringConverter::Base32() function.
unrecordedTraceID = 'ut'
unrecordedFrameID = 'uf'
traceTable[unrecordedTraceID] = [unrecordedFrameID]
frameTable[unrecordedFrameID] = \
'#00: (no stack trace recorded due to --stacks=partial)'
# For the purposes of this script, 'scan' behaves like 'live'.
if mode == 'scan':
mode = 'live'
if not mode in ['live', 'dark-matter', 'cumulative']:
raise Exception("bad 'mode' property: '{:s}'".format(mode))
# Remove allocation functions at the start of traces.
if args.ignore_alloc_fns:
# Build a regexp that matches every function in allocatorFns.
escapedAllocatorFns = map(re.escape, allocatorFns)
fn_re = re.compile('|'.join(escapedAllocatorFns))
# Remove allocator fns from each stack trace.
for traceKey, frameKeys in traceTable.items():
numSkippedFrames = 0
for frameKey in frameKeys:
frameDesc = frameTable[frameKey]
if re.search(fn_re, frameDesc):
numSkippedFrames += 1
else:
break
if numSkippedFrames > 0:
traceTable[traceKey] = frameKeys[numSkippedFrames:]
# Trim the number of frames.
for traceKey, frameKeys in traceTable.items():
if len(frameKeys) > args.max_frames:
traceTable[traceKey] = frameKeys[:args.max_frames]
def buildTraceDescription(traceTable, frameTable, traceKey):
frameKeys = traceTable[traceKey]
fmt = ' #{:02d}{:}'
if args.filter_stacks_for_testing:
# When running SmokeDMD.cpp, every stack trace should contain at
# least one frame that contains 'DMD.cpp', from either |DMD.cpp| or
# |SmokeDMD.cpp|. (Or 'dmd.cpp' on Windows.) If we see such a
# frame, we replace the entire stack trace with a single,
# predictable frame. There is too much variation in the stack
# traces across different machines and platforms to do more precise
# matching, but this level of matching will result in failure if
# stack fixing fails completely.
for frameKey in frameKeys:
frameDesc = frameTable[frameKey]
if 'DMD.cpp' in frameDesc or 'dmd.cpp' in frameDesc:
return [fmt.format(1, ': ... DMD.cpp ...')]
# The frame number is always '#00' (see DMD.h for why), so we have to
# replace that with the correct frame number.
desc = []
for n, frameKey in enumerate(traceTable[traceKey], start=1):
desc.append(fmt.format(n, frameTable[frameKey][3:]))
return desc
# Aggregate blocks into records. All sufficiently similar blocks go into a
# single record.
if mode in ['live', 'cumulative']:
liveOrCumulativeRecords = collections.defaultdict(Record)
elif mode == 'dark-matter':
unreportedRecords = collections.defaultdict(Record)
onceReportedRecords = collections.defaultdict(Record)
twiceReportedRecords = collections.defaultdict(Record)
heapUsableSize = 0
heapBlocks = 0
recordKeyPartCache = {}
for block in blockList:
# For each block we compute a |recordKey|, and all blocks with the same
# |recordKey| are aggregated into a single record. The |recordKey| is
# derived from the block's 'alloc' and 'reps' (if present) stack
# traces.
#
# We use frame descriptions (e.g. "#00: foo (X.cpp:99)") when comparing
# traces for equality. We can't use trace keys or frame keys because
# they're not comparable across different DMD runs (which is relevant
# when doing diffs).
#
# Using frame descriptions also fits in with the stack trimming done
# for --max-frames, which requires that stack traces with common
# beginnings but different endings to be considered equivalent. E.g. if
# we have distinct traces T1:[A:D1,B:D2,C:D3] and T2:[X:D1,Y:D2,Z:D4]
# and we trim the final frame of each they should be considered
# equivalent because the untrimmed frame descriptions (D1 and D2)
# match.
#
# Having said all that, during a single invocation of dmd.py on a
# single DMD file, for a single frameKey value the record key will
# always be the same, and we might encounter it 1000s of times. So we
# cache prior results for speed.
def makeRecordKeyPart(traceKey):
if traceKey in recordKeyPartCache:
return recordKeyPartCache[traceKey]
recordKeyPart = str(map(lambda frameKey: frameTable[frameKey],
traceTable[traceKey]))
recordKeyPartCache[traceKey] = recordKeyPart
return recordKeyPart
allocatedAtTraceKey = block.get('alloc', unrecordedTraceID)
if mode in ['live', 'cumulative']:
recordKey = makeRecordKeyPart(allocatedAtTraceKey)
records = liveOrCumulativeRecords
elif mode == 'dark-matter':
recordKey = makeRecordKeyPart(allocatedAtTraceKey)
if 'reps' in block:
reportedAtTraceKeys = block['reps']
for reportedAtTraceKey in reportedAtTraceKeys:
recordKey += makeRecordKeyPart(reportedAtTraceKey)
if len(reportedAtTraceKeys) == 1:
records = onceReportedRecords
else:
records = twiceReportedRecords
else:
records = unreportedRecords
record = records[recordKey]
if 'req' not in block:
raise Exception("'req' property missing in block'")
reqSize = block['req']
slopSize = block.get('slop', 0)
if 'num' in block:
num = block['num']
else:
num = 1
usableSize = reqSize + slopSize
heapUsableSize += num * usableSize
heapBlocks += num
record.numBlocks += num
record.reqSize += num * reqSize
record.slopSize += num * slopSize
record.usableSize += num * usableSize
if record.allocatedAtDesc == None:
record.allocatedAtDesc = \
buildTraceDescription(traceTable, frameTable,
allocatedAtTraceKey)
if mode in ['live', 'cumulative']:
pass
elif mode == 'dark-matter':
if 'reps' in block and record.reportedAtDescs == []:
f = lambda k: buildTraceDescription(traceTable, frameTable, k)
record.reportedAtDescs = map(f, reportedAtTraceKeys)
record.usableSizes[usableSize] += num
# All the processed data for a single DMD file is called a "digest".
digest = {}
digest['dmdEnvVar'] = dmdEnvVar
digest['mode'] = mode
digest['heapUsableSize'] = heapUsableSize
digest['heapBlocks'] = heapBlocks
if mode in ['live', 'cumulative']:
digest['liveOrCumulativeRecords'] = liveOrCumulativeRecords
elif mode == 'dark-matter':
digest['unreportedRecords'] = unreportedRecords
digest['onceReportedRecords'] = onceReportedRecords
digest['twiceReportedRecords'] = twiceReportedRecords
return digest
def diffRecords(args, records1, records2):
records3 = {}
# Process records1.
for k in records1:
r1 = records1[k]
if k in records2:
# This record is present in both records1 and records2.
r2 = records2[k]
del records2[k]
r2.subtract(r1)
if not r2.isZero(args):
records3[k] = r2
else:
# This record is present only in records1.
r1.negate()
records3[k] = r1
for k in records2:
# This record is present only in records2.
records3[k] = records2[k]
return records3
def diffDigests(args, d1, d2):
if (d1['mode'] != d2['mode']):
raise Exception("the input files have different 'mode' properties")
d3 = {}
d3['dmdEnvVar'] = (d1['dmdEnvVar'], d2['dmdEnvVar'])
d3['mode'] = d1['mode']
d3['heapUsableSize'] = d2['heapUsableSize'] - d1['heapUsableSize']
d3['heapBlocks'] = d2['heapBlocks'] - d1['heapBlocks']
if d1['mode'] in ['live', 'cumulative']:
d3['liveOrCumulativeRecords'] = \
diffRecords(args, d1['liveOrCumulativeRecords'],
d2['liveOrCumulativeRecords'])
elif d1['mode'] == 'dark-matter':
d3['unreportedRecords'] = diffRecords(args, d1['unreportedRecords'],
d2['unreportedRecords'])
d3['onceReportedRecords'] = diffRecords(args, d1['onceReportedRecords'],
d2['onceReportedRecords'])
d3['twiceReportedRecords'] = diffRecords(args, d1['twiceReportedRecords'],
d2['twiceReportedRecords'])
return d3
def printDigest(args, digest):
dmdEnvVar = digest['dmdEnvVar']
mode = digest['mode']
heapUsableSize = digest['heapUsableSize']
heapBlocks = digest['heapBlocks']
if mode in ['live', 'cumulative']:
liveOrCumulativeRecords = digest['liveOrCumulativeRecords']
elif mode == 'dark-matter':
unreportedRecords = digest['unreportedRecords']
onceReportedRecords = digest['onceReportedRecords']
twiceReportedRecords = digest['twiceReportedRecords']
separator = '#' + '-' * 65 + '\n'
def number(n):
'''Format a number with comma as a separator.'''
return '{:,d}'.format(n)
def perc(m, n):
return 0 if n == 0 else (100 * m / n)
def plural(n):
return '' if n == 1 else 's'
# Prints to stdout, or to file if -o/--output was specified.
def out(*arguments, **kwargs):
print(*arguments, file=args.output, **kwargs)
def printStack(traceDesc):
for frameDesc in traceDesc:
out(frameDesc)
def printRecords(recordKind, records, heapUsableSize):
RecordKind = recordKind.capitalize()
out(separator)
numRecords = len(records)
cmpRecords = sortByChoices[args.sort_by]
sortedRecords = sorted(records.values(), cmp=cmpRecords, reverse=True)
kindBlocks = 0
kindUsableSize = 0
maxRecord = 1000
# First iteration: get totals, etc.
for record in sortedRecords:
kindBlocks += record.numBlocks
kindUsableSize += record.usableSize
# Second iteration: print.
if numRecords == 0:
out('# no {:} heap blocks\n'.format(recordKind))
kindCumulativeUsableSize = 0
for i, record in enumerate(sortedRecords, start=1):
# Stop printing at the |maxRecord|th record.
if i == maxRecord:
out('# {:}: stopping after {:,d} heap block records\n'.
format(RecordKind, i))
break
kindCumulativeUsableSize += record.usableSize
out(RecordKind + ' {')
out(' {:} block{:} in heap block record {:,d} of {:,d}'.
format(number(record.numBlocks),
plural(record.numBlocks), i, numRecords))
out(' {:} bytes ({:} requested / {:} slop)'.
format(number(record.usableSize),
number(record.reqSize),
number(record.slopSize)))
abscmp = lambda (usableSize1, _1), (usableSize2, _2): \
cmp(abs(usableSize1), abs(usableSize2))
usableSizes = sorted(record.usableSizes.items(), cmp=abscmp,
reverse=True)
hasSingleBlock = len(usableSizes) == 1 and usableSizes[0][1] == 1
if not hasSingleBlock:
out(' Individual block sizes: ', end='')
if len(usableSizes) == 0:
out('(no change)', end='')
else:
isFirst = True
for usableSize, count in usableSizes:
if not isFirst:
out('; ', end='')
out('{:}'.format(number(usableSize)), end='')
if count > 1:
out(' x {:,d}'.format(count), end='')
isFirst = False
out()
out(' {:4.2f}% of the heap ({:4.2f}% cumulative)'.
format(perc(record.usableSize, heapUsableSize),
perc(kindCumulativeUsableSize, heapUsableSize)))
if mode in ['live', 'cumulative']:
pass
elif mode == 'dark-matter':
out(' {:4.2f}% of {:} ({:4.2f}% cumulative)'.
format(perc(record.usableSize, kindUsableSize),
recordKind,
perc(kindCumulativeUsableSize, kindUsableSize)))
out(' Allocated at {')
printStack(record.allocatedAtDesc)
out(' }')
if mode in ['live', 'cumulative']:
pass
elif mode == 'dark-matter':
for n, reportedAtDesc in enumerate(record.reportedAtDescs):
again = 'again ' if n > 0 else ''
out(' Reported {:}at {{'.format(again))
printStack(reportedAtDesc)
out(' }')
out('}\n')
return (kindUsableSize, kindBlocks)
def printInvocation(n, dmdEnvVar, mode):
out('Invocation{:} {{'.format(n))
if dmdEnvVar == None:
out(' $DMD is undefined')
else:
out(' $DMD = \'' + dmdEnvVar + '\'')
out(' Mode = \'' + mode + '\'')
out('}\n')
# Print command line. Strip dirs so the output is deterministic, which is
# needed for testing.
out(separator, end='')
out('# ' + ' '.join(map(os.path.basename, sys.argv)) + '\n')
# Print invocation(s).
if type(dmdEnvVar) is not tuple:
printInvocation('', dmdEnvVar, mode)
else:
printInvocation(' 1', dmdEnvVar[0], mode)
printInvocation(' 2', dmdEnvVar[1], mode)
# Print records.
if mode in ['live', 'cumulative']:
liveOrCumulativeUsableSize, liveOrCumulativeBlocks = \
printRecords(mode, liveOrCumulativeRecords, heapUsableSize)
elif mode == 'dark-matter':
twiceReportedUsableSize, twiceReportedBlocks = \
printRecords('twice-reported', twiceReportedRecords, heapUsableSize)
unreportedUsableSize, unreportedBlocks = \
printRecords('unreported', unreportedRecords, heapUsableSize)
onceReportedUsableSize, onceReportedBlocks = \
printRecords('once-reported', onceReportedRecords, heapUsableSize)
# Print summary.
out(separator)
out('Summary {')
if mode in ['live', 'cumulative']:
out(' Total: {:} bytes in {:} blocks'.
format(number(liveOrCumulativeUsableSize),
number(liveOrCumulativeBlocks)))
elif mode == 'dark-matter':
fmt = ' {:15} {:>12} bytes ({:6.2f}%) in {:>7} blocks ({:6.2f}%)'
out(fmt.
format('Total:',
number(heapUsableSize),
100,
number(heapBlocks),
100))
out(fmt.
format('Unreported:',
number(unreportedUsableSize),
perc(unreportedUsableSize, heapUsableSize),
number(unreportedBlocks),
perc(unreportedBlocks, heapBlocks)))
out(fmt.
format('Once-reported:',
number(onceReportedUsableSize),
perc(onceReportedUsableSize, heapUsableSize),
number(onceReportedBlocks),
perc(onceReportedBlocks, heapBlocks)))
out(fmt.
format('Twice-reported:',
number(twiceReportedUsableSize),
perc(twiceReportedUsableSize, heapUsableSize),
number(twiceReportedBlocks),
perc(twiceReportedBlocks, heapBlocks)))
out('}\n')
#############################
# Pretty printer for DMD JSON
#############################
def prettyPrintDmdJson(out, j):
out.write('{\n')
out.write(' "version": {0},\n'.format(j['version']))
out.write(' "invocation": ')
json.dump(j['invocation'], out, sort_keys=True)
out.write(',\n')
out.write(' "blockList": [')
first = True
for b in j['blockList']:
out.write('' if first else ',')
out.write('\n ')
json.dump(b, out, sort_keys=True)
first = False
out.write('\n ],\n')
out.write(' "traceTable": {')
first = True
for k, l in j['traceTable'].iteritems():
out.write('' if first else ',')
out.write('\n "{0}": {1}'.format(k, json.dumps(l)))
first = False
out.write('\n },\n')
out.write(' "frameTable": {')
first = True
for k, v in j['frameTable'].iteritems():
out.write('' if first else ',')
out.write('\n "{0}": {1}'.format(k, json.dumps(v)))
first = False
out.write('\n }\n')
out.write('}\n')
##################################################################
# Code for clamping addresses using conservative pointer analysis.
##################################################################
# Start is the address of the first byte of the block, while end is
# the address of the first byte after the final byte in the block.
class AddrRange:
def __init__(self, block, length):
self.block = block
self.start = int(block, 16)
self.length = length
self.end = self.start + self.length
assert self.start > 0
assert length >= 0
class ClampStats:
def __init__(self):
# Number of pointers already pointing to the start of a block.
self.startBlockPtr = 0
# Number of pointers pointing to the middle of a block. These
# are clamped to the start of the block they point into.
self.midBlockPtr = 0
# Number of null pointers.
self.nullPtr = 0
# Number of non-null pointers that didn't point into the middle
# of any blocks. These are clamped to null.
self.nonNullNonBlockPtr = 0
def clampedBlockAddr(self, sameAddress):
if sameAddress:
self.startBlockPtr += 1
else:
self.midBlockPtr += 1
def nullAddr(self):
self.nullPtr += 1
def clampedNonBlockAddr(self):
self.nonNullNonBlockPtr += 1
def log(self):
sys.stderr.write('Results:\n')
sys.stderr.write(' Number of pointers already pointing to start of blocks: ' + str(self.startBlockPtr) + '\n')
sys.stderr.write(' Number of pointers clamped to start of blocks: ' + str(self.midBlockPtr) + '\n')
sys.stderr.write(' Number of non-null pointers not pointing into blocks clamped to null: ' + str(self.nonNullNonBlockPtr) + '\n')
sys.stderr.write(' Number of null pointers: ' + str(self.nullPtr) + '\n')
# Search the block ranges array for a block that address points into.
# The search is carried out in an array of starting addresses for each blocks
# because it is faster.
def clampAddress(blockRanges, blockStarts, clampStats, address):
i = bisect_right(blockStarts, address)
# Any addresses completely out of the range should have been eliminated already.
assert i > 0
r = blockRanges[i - 1]
assert r.start <= address
if address >= r.end:
assert address < blockRanges[i].start
clampStats.clampedNonBlockAddr()
return '0'
clampStats.clampedBlockAddr(r.start == address)
return r.block
def clampBlockList(args, inputFileName, isZipped, opener):
# XXX This isn't very efficient because we end up reading and writing
# the file multiple times.
with opener(inputFileName, 'rb') as f:
j = json.load(f)
if j['version'] != outputVersion:
raise Exception("'version' property isn't '{:d}'".format(outputVersion))
# Check that the invocation is reasonable for contents clamping.
invocation = j['invocation']
if invocation['mode'] != 'scan':
raise Exception("Log was taken in mode " + invocation['mode'] + " not scan")
sys.stderr.write('Creating block range list.\n')
blockList = j['blockList']
blockRanges = []
for block in blockList:
blockRanges.append(AddrRange(block['addr'], block['req']))
blockRanges.sort(key=lambda r: r.start)
# Make sure there are no overlapping blocks.
prevRange = blockRanges[0]
for currRange in blockRanges[1:]:
assert prevRange.end <= currRange.start
prevRange = currRange
sys.stderr.write('Clamping block contents.\n')
clampStats = ClampStats()
firstAddr = blockRanges[0].start
lastAddr = blockRanges[-1].end
blockStarts = []
for r in blockRanges:
blockStarts.append(r.start)
for block in blockList:
# Small blocks don't have any contents.
if not 'contents' in block:
continue
cont = block['contents']
for i in range(len(cont)):
address = int(cont[i], 16)
if address == 0:
clampStats.nullAddr()
continue
# If the address is before the first block or after the last
# block then it can't be within a block.
if address < firstAddr or address >= lastAddr:
clampStats.clampedNonBlockAddr()
cont[i] = '0'
continue
cont[i] = clampAddress(blockRanges, blockStarts, clampStats, address)
# Remove any trailing nulls.
while len(cont) and cont[-1] == '0':
cont.pop()
if args.print_clamp_stats:
clampStats.log()
sys.stderr.write('Saving file.\n')
tmpFile = tempfile.NamedTemporaryFile(delete=False)
tmpFilename = tmpFile.name
if isZipped:
tmpFile = gzip.GzipFile(filename='', fileobj=tmpFile)
prettyPrintDmdJson(tmpFile, j)
tmpFile.close()
shutil.move(tmpFilename, inputFileName)
def main():
args = parseCommandLine()
digest = getDigestFromFile(args, args.input_file)
if args.input_file2:
digest2 = getDigestFromFile(args, args.input_file2)
digest = diffDigests(args, digest, digest2)
printDigest(args, digest)
if __name__ == '__main__':
main()