Skip to content

Commit

Permalink
Added ./script/summary.py
Browse files Browse the repository at this point in the history 
A full summary of static measurements (code size, stack usage, etc) can now
be found with:

    make summary

This is done through the combination of a new ./scripts/summary.py
script and the ability of existing scripts to merge into existing csv
files, allowing multiple results to be merged either in a pipeline, or
in parallel with a single ./script/summary.py call.

The ./scripts/summary.py script can also be used to quickly compare
different builds or configurations. This is a proper implementation
of a similar but hacky shell script that has already been very useful
for making optimization decisions:

    $ ./scripts/structs.py new.csv -d old.csv --summary
    name (2 added, 0 removed)               code             stack            structs
    TOTAL                                  28648 (-2.7%)      2448               1012

Also some other small tweaks to scripts:

- Removed state saving diff rules. This isn't the most useful way to
  handle comparing changes.

- Added short flags for --summary (-Y) and --files (-F), since these
  are quite often used.
  • Loading branch information
@geky
geky committed Mar 20, 2022
1 parent eb8be9f commit 55b3c53
Show file tree
Hide file tree
Showing 7 changed files with 580 additions and 110 deletions.
34 changes: 15 additions & 19 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ endif
ifdef EXEC
override TESTFLAGS += --exec="$(EXEC)"
endif
ifdef COVERAGE
override TESTFLAGS += --coverage
endif
ifdef BUILDDIR
override TESTFLAGS += --build-dir="$(BUILDDIR:/=)"
override CALLSFLAGS += --build-dir="$(BUILDDIR:/=)"
Expand Down Expand Up @@ -104,41 +107,34 @@ test%: tests/test$$(firstword $$(subst \#, ,%)).toml
code: $(OBJ)
./scripts/code.py $^ -S $(CODEFLAGS)

.PHONY: code-diff
code-diff: $(OBJ)
./scripts/code.py $^ -d $(TARGET).code.csv -o $(TARGET).code.csv $(CODEFLAGS)

.PHONY: data
data: $(OBJ)
./scripts/data.py $^ -S $(DATAFLAGS)

.PHONY: data-diff
data-diff: $(OBJ)
./scripts/data.py $^ -d $(TARGET).data.csv -o $(TARGET).data.csv $(DATAFLAGS)

.PHONY: stack
stack: $(CGI)
./scripts/stack.py $^ -S $(STACKFLAGS)

.PHONY: stack-diff
stack-diff: $(CGI)
./scripts/stack.py $^ -d $(TARGET).stack.csv -o $(TARGET).stack.csv $(STACKFLAGS)

.PHONY: structs
structs: $(OBJ)
./scripts/structs.py $^ -S $(STRUCTSFLAGS)

.PHONY: structs-diff
structs-diff: $(OBJ)
./scripts/structs.py $^ -d $(TARGET).structs.csv -o $(TARGET).structs.csv $(STRUCTSFLAGS)

.PHONY: coverage
coverage:
./scripts/coverage.py $(BUILDDIR)tests/*.toml.info -s $(COVERAGEFLAGS)

.PHONY: coverage-diff
coverage-diff:
./scripts/coverage.py $(BUILDDIR)tests/*.toml.info $(COVERAGEFLAGS)
.PHONY: summary
summary: $(OBJ) $(CGI)
$(strip \
./scripts/code.py $(OBJ) -q -o - $(CODEFLAGS) \
| ./scripts/data.py $(OBJ) -q -m - -o - $(DATAFLAGS) \
| ./scripts/stack.py $(CGI) -q -m - -o - $(STACKFLAGS) \
| ./scripts/structs.py $(OBJ) -q -m - -o - $(STRUCTFLAGS) \
$(if $(COVERAGE),\
| ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \
-q -m - -o - $(COVERAGEFLAGS)) \
| ./scripts/summary.py $(SUMMARYFLAGS))


# rules
-include $(DEP)
Expand Down
73 changes: 55 additions & 18 deletions scripts/code.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,30 @@ def collect(paths, **args):
# map to source files
if args.get('build_dir'):
file = re.sub('%s/*' % re.escape(args['build_dir']), '', file)
# replace .o with .c, different scripts report .o/.c, we need to
# choose one if we want to deduplicate csv files
file = re.sub('\.o$', '.c', file)
# discard internal functions
if not args.get('everything'):
if func.startswith('__'):
continue
# discard .8449 suffixes created by optimizer
func = re.sub('\.[0-9]+', '', func)

flat_results.append((file, func, size))

return flat_results

def main(**args):
def openio(path, mode='r'):
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
else:
return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
else:
return open(path, mode)

# find sizes
if not args.get('use', None):
# find .o files
Expand All @@ -76,13 +89,14 @@ def main(**args):

results = collect(paths, **args)
else:
with open(args['use']) as f:
with openio(args['use']) as f:
r = csv.DictReader(f)
results = [
( result['file'],
result['function'],
result['name'],
int(result['code_size']))
for result in r]
for result in r
if result.get('code_size') not in {None, ''}]

total = 0
for _, _, size in results:
Expand All @@ -91,13 +105,14 @@ def main(**args):
# find previous results?
if args.get('diff'):
try:
with open(args['diff']) as f:
with openio(args['diff']) as f:
r = csv.DictReader(f)
prev_results = [
( result['file'],
result['function'],
result['name'],
int(result['code_size']))
for result in r]
for result in r
if result.get('code_size') not in {None, ''}]
except FileNotFoundError:
prev_results = []

Expand All @@ -107,14 +122,34 @@ def main(**args):

# write results to CSV
if args.get('output'):
with open(args['output'], 'w') as f:
w = csv.writer(f)
w.writerow(['file', 'function', 'code_size'])
for file, func, size in sorted(results):
w.writerow((file, func, size))
merged_results = co.defaultdict(lambda: {})
other_fields = []

# merge?
if args.get('merge'):
try:
with openio(args['merge']) as f:
r = csv.DictReader(f)
for result in r:
file = result.pop('file', '')
func = result.pop('name', '')
result.pop('code_size', None)
merged_results[(file, func)] = result
other_fields = result.keys()
except FileNotFoundError:
pass

for file, func, size in results:
merged_results[(file, func)]['code_size'] = size

with openio(args['output'], 'w') as f:
w = csv.DictWriter(f, ['file', 'name', *other_fields, 'code_size'])
w.writeheader()
for (file, func), result in sorted(merged_results.items()):
w.writerow({'file': file, 'name': func, **result})

# print results
def dedup_entries(results, by='function'):
def dedup_entries(results, by='name'):
entries = co.defaultdict(lambda: 0)
for file, func, size in results:
entry = (file if by == 'file' else func)
Expand Down Expand Up @@ -162,7 +197,7 @@ def print_diff_entry(name, old, new, diff, ratio):
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))

def print_entries(by='function'):
def print_entries(by='name'):
entries = dedup_entries(results, by=by)

if not args.get('diff'):
Expand Down Expand Up @@ -201,7 +236,7 @@ def print_totals():
print_entries(by='file')
print_totals()
else:
print_entries(by='function')
print_entries(by='name')
print_totals()

if __name__ == "__main__":
Expand All @@ -214,12 +249,16 @@ def print_totals():
or a list of paths. Defaults to %r." % OBJ_PATHS)
parser.add_argument('-v', '--verbose', action='store_true',
help="Output commands that run behind the scenes.")
parser.add_argument('-q', '--quiet', action='store_true',
help="Don't show anything, useful with -o.")
parser.add_argument('-o', '--output',
help="Specify CSV file to store results.")
parser.add_argument('-u', '--use',
help="Don't compile and find code sizes, instead use this CSV file.")
parser.add_argument('-d', '--diff',
help="Specify CSV file to diff code size against.")
parser.add_argument('-m', '--merge',
help="Merge with an existing CSV file when writing to output.")
parser.add_argument('-a', '--all', action='store_true',
help="Show all functions, not just the ones that changed.")
parser.add_argument('-A', '--everything', action='store_true',
Expand All @@ -228,13 +267,11 @@ def print_totals():
help="Sort by size.")
parser.add_argument('-S', '--reverse-size-sort', action='store_true',
help="Sort by size, but backwards.")
parser.add_argument('--files', action='store_true',
parser.add_argument('-F', '--files', action='store_true',
help="Show file-level code sizes. Note this does not include padding! "
"So sizes may differ from other tools.")
parser.add_argument('--summary', action='store_true',
parser.add_argument('-Y', '--summary', action='store_true',
help="Only show the total code size.")
parser.add_argument('-q', '--quiet', action='store_true',
help="Don't show anything, useful with -o.")
parser.add_argument('--type', default='tTrRdD',
help="Type of symbols to report, this uses the same single-character "
"type-names emitted by nm. Defaults to %(default)r.")
Expand Down
69 changes: 53 additions & 16 deletions scripts/coverage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ def func_from_lineno(file, lineno):


def main(**args):
def openio(path, mode='r'):
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
else:
return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
else:
return open(path, mode)

# find coverage
if not args.get('use'):
# find *.info files
Expand All @@ -83,14 +92,16 @@ def main(**args):

results = collect(paths, **args)
else:
with open(args['use']) as f:
with openio(args['use']) as f:
r = csv.DictReader(f)
results = [
( result['file'],
result['function'],
result['name'],
int(result['coverage_hits']),
int(result['coverage_count']))
for result in r]
for result in r
if result.get('coverage_hits') not in {None, ''}
if result.get('coverage_count') not in {None, ''}]

total_hits, total_count = 0, 0
for _, _, hits, count in results:
Expand All @@ -100,14 +111,16 @@ def main(**args):
# find previous results?
if args.get('diff'):
try:
with open(args['diff']) as f:
with openio(args['diff']) as f:
r = csv.DictReader(f)
prev_results = [
( result['file'],
result['function'],
result['name'],
int(result['coverage_hits']),
int(result['coverage_count']))
for result in r]
for result in r
if result.get('coverage_hits') not in {None, ''}
if result.get('coverage_count') not in {None, ''}]
except FileNotFoundError:
prev_results = []

Expand All @@ -118,14 +131,36 @@ def main(**args):

# write results to CSV
if args.get('output'):
with open(args['output'], 'w') as f:
w = csv.writer(f)
w.writerow(['file', 'function', 'coverage_hits', 'coverage_count'])
for file, func, hits, count in sorted(results):
w.writerow((file, func, hits, count))
merged_results = co.defaultdict(lambda: {})
other_fields = []

# merge?
if args.get('merge'):
try:
with openio(args['merge']) as f:
r = csv.DictReader(f)
for result in r:
file = result.pop('file', '')
func = result.pop('name', '')
result.pop('coverage_hits', None)
result.pop('coverage_count', None)
merged_results[(file, func)] = result
other_fields = result.keys()
except FileNotFoundError:
pass

for file, func, hits, count in results:
merged_results[(file, func)]['coverage_hits'] = hits
merged_results[(file, func)]['coverage_count'] = count

with openio(args['output'], 'w') as f:
w = csv.DictWriter(f, ['file', 'name', *other_fields, 'coverage_hits', 'coverage_count'])
w.writeheader()
for (file, func), result in sorted(merged_results.items()):
w.writerow({'file': file, 'name': func, **result})

# print results
def dedup_entries(results, by='function'):
def dedup_entries(results, by='name'):
entries = co.defaultdict(lambda: (0, 0))
for file, func, hits, count in results:
entry = (file if by == 'file' else func)
Expand Down Expand Up @@ -197,7 +232,7 @@ def print_diff_entry(name,
'%+d/%+d' % (diff_hits, diff_count),
' (%+.1f%%)' % (100*ratio) if ratio else ''))

def print_entries(by='function'):
def print_entries(by='name'):
entries = dedup_entries(results, by=by)

if not args.get('diff'):
Expand Down Expand Up @@ -245,7 +280,7 @@ def print_totals():
print_entries(by='file')
print_totals()
else:
print_entries(by='function')
print_entries(by='name')
print_totals()

if __name__ == "__main__":
Expand All @@ -266,6 +301,8 @@ def print_totals():
help="Don't do any work, instead use this CSV file.")
parser.add_argument('-d', '--diff',
help="Specify CSV file to diff code size against.")
parser.add_argument('-m', '--merge',
help="Merge with an existing CSV file when writing to output.")
parser.add_argument('-a', '--all', action='store_true',
help="Show all functions, not just the ones that changed.")
parser.add_argument('-A', '--everything', action='store_true',
Expand All @@ -274,9 +311,9 @@ def print_totals():
help="Sort by coverage.")
parser.add_argument('-S', '--reverse-coverage-sort', action='store_true',
help="Sort by coverage, but backwards.")
parser.add_argument('--files', action='store_true',
parser.add_argument('-F', '--files', action='store_true',
help="Show file-level coverage.")
parser.add_argument('--summary', action='store_true',
parser.add_argument('-Y', '--summary', action='store_true',
help="Only show the total coverage.")
parser.add_argument('-q', '--quiet', action='store_true',
help="Don't show anything, useful with -o.")
Expand Down
Loading

0 comments on commit 55b3c53

Please sign in to comment.