Handle XFLASH (D21) and serial (D23) dumps in elf_mem_map, add dump2bin

- Uniformly parse D2/D21/D23 dump types.
- Add dump2bin to parse/convert a dump into metadata and binary.
- Move the parsing into it's own module in order to be shared.

tools/dump2bin

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+import argparse
+import os, sys
+
+from lib.dump import decode_dump
+
+
+def main():
+    # parse the arguments
+    ap = argparse.ArgumentParser(description="""
+        Parse and decode a memory dump obtained from the D2/D21/D23 g-code
+        into readable metadata and binary. The output binary is padded and
+        extended to fit the original address range.
+    """)
+    ap.add_argument('-i', dest='info', action='store_true',
+                    help='display crash info only')
+    ap.add_argument('dump')
+    ap.add_argument('output', nargs='?')
+    args = ap.parse_args()
+
+    # decode the dump data
+    dump = decode_dump(args.dump)
+    if dump is None:
+        return os.EX_DATAERR
+
+    # output descriptors
+    if args.info:
+        o_fd = None
+        o_md = sys.stdout
+    elif args.output is None:
+        o_fd = sys.stdout.buffer
+        o_md = sys.stderr
+    else:
+        o_fd = open(args.output, 'wb')
+        o_md = sys.stdout
+
+    # output binary
+    if o_fd:
+        o_fd.write(dump.data)
+        o_fd.close()
+
+    # metadata
+    print('   dump type: {typ}\n'
+          'crash reason: {reason}\n'
+          '   registers: {regs}\n'
+          '          PC: {pc}\n'
+          '          SP: {sp}\n'
+          '      ranges: {ranges}'.format(
+              typ=dump.typ,
+              reason=dump.reason.name if dump.reason is not None else 'N/A',
+              regs=dump.regs,
+              pc='{:#x}'.format(dump.pc) if dump.pc is not None else 'N/A',
+              sp='{:#x}'.format(dump.sp) if dump.sp is not None else 'N/A',
+              ranges=str(dump.ranges)),
+          file=o_md)
+
+
+if __name__ == '__main__':
+    exit(main())

tools/elf_mem_map

@@ -4,13 +4,10 @@ import elftools.elf.elffile
 import elftools.dwarf.descriptions
 from collections import namedtuple
 from struct import unpack
-import sys
-import re
+import os
 
-SRAM_START = 0x200
-SRAM_OFFSET = 0x800000
-EEPROM_OFFSET = 0x810000
-FILL_BYTE = b'\0'
+from lib.dump import decode_dump
+from lib.avr import *
 
 
 Entry = namedtuple('Entry', ['name', 'loc', 'size', 'declpos'])
@@ -257,59 +254,6 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
     return grefs
 
 
-def decode_dump(path):
-    fd = open(path, 'r')
-    if fd is None:
-        return None
-
-    buf_addr = None # starting address
-    buf_data = None # data
-
-    in_dump = False
-    for line in enumerate(fd):
-        line = (line[0], line[1].rstrip())
-        tokens = line[1].split(maxsplit=1)
-        if not in_dump:
-            if len(tokens) > 0 and tokens[0] in ['D2', 'D23']:
-                in_dump = True
-            continue
-        else:
-            if len(tokens) < 1:
-                print('malformed line {}: {}'.format(*line), file=sys.stderr)
-                continue
-            elif tokens[0] == 'ok':
-                break
-            elif tokens[0] == 'reason:':
-                # ignored
-                continue
-            elif not re.match(r'[0-9a-fA-F]', tokens[0]):
-                print('malformed line {}: {}'.format(*line), file=sys.stderr)
-                continue
-
-        addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big')
-        data = bytes.fromhex(tokens[1])
-
-        if buf_addr is None:
-            buf_addr = addr
-            buf_data = data
-        else:
-            # grow buffer as needed
-            if addr < buf_addr:
-                buf_data = FILL_BYTE * (buf_addr - addr)
-                buf_addr = addr
-            addr_end = addr + len(data)
-            buf_end = buf_addr + len(buf_data)
-            if addr_end > buf_end:
-                buf_data += FILL_BYTE * (addr_end - buf_end)
-
-            # replace new part
-            rep_start = addr - buf_addr
-            rep_end = rep_start + len(data)
-            buf_data = buf_data[:rep_start] + data + buf_data[rep_end:]
-
-    return (buf_addr, buf_data)
-
-
 def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True):
     last_end = None
     for entry in grefs:
@@ -426,8 +370,17 @@ def main():
     elif args.qdirstat:
         print_qdirstat(grefs)
     else:
-        addr, data = decode_dump(args.dump)
-        annotate_refs(grefs, addr, data,
+        # fetch the memory data
+        dump = decode_dump(args.dump)
+        if dump is None:
+            return os.EX_DATAERR
+
+        # strip padding, if present
+        addr_start = dump.ranges[0][0]
+        addr_end = dump.ranges[-1][0]+dump.ranges[-1][1]
+        data = dump.data[addr_start:addr_end]
+
+        annotate_refs(grefs, addr_start, data,
                       width=args.name_width,
                       gaps=not args.no_gaps,
                       overlaps=args.overlaps)

tools/lib/avr.py

@@ -0,0 +1,4 @@
+SRAM_START = 0x200
+SRAM_SIZE = 0x2000
+SRAM_OFFSET = 0x800000
+EEPROM_OFFSET = 0x810000

tools/lib/dump.py

@@ -0,0 +1,165 @@
+import sys
+import re
+import enum
+import struct
+from . import avr
+
+
+FILL_BYTE = b'\0' # used to fill memory gaps in the dump
+
+class CrashReason(enum.IntEnum):
+    MANUAL = 0
+    STACK_ERROR = 1
+    WATCHDOG = 2
+    BAD_ISR = 3
+
+class Dump():
+    def __init__(self, typ, reason, regs, pc, sp, data, ranges):
+        self.typ = typ
+        self.reason = reason
+        self.regs = regs
+        self.pc = pc
+        self.sp = sp
+        self.data = data
+        self.ranges = ranges
+
+
+# expand the buffer identified by addr+data to fill the region start+size
+def region_expand(addr, data, start, size):
+    if start < addr:
+        data = FILL_BYTE * (addr - start) + data
+        addr = start
+    end = start + size
+    data_end = addr + len(data)
+    if end > data_end:
+        data += FILL_BYTE * (data_end - end)
+    return addr, data
+
+
+def merge_ranges(ranges):
+    ranges = list(sorted(ranges, key=lambda x: x[0]))
+    if len(ranges) < 2:
+        return ranges
+
+    ret = [ranges[0]]
+    for cur in ranges[1:]:
+        last = ret[-1]
+        last_end = last[0] + last[1]
+        if last_end < cur[0]:
+            ret.append(cur)
+        else:
+            cur_end = cur[0] + cur[1]
+            last = (last[0], max(last_end, cur_end) - last[0])
+            ret[-1] = last
+    return ret
+
+
+def decode_dump(path):
+    fd = open(path, 'r')
+    if fd is None:
+        return None
+
+    buf_addr = None # starting address
+    buf_data = None # data
+
+    typ = None      # dump type
+    reason = None   # crash reason
+    regs = None     # registers present
+    pc = None       # PC address
+    sp = None       # SP address
+    ranges = []     # dumped ranges
+
+    in_dump = False
+    for line in enumerate(fd):
+        line = (line[0], line[1].rstrip())
+        tokens = line[1].split(maxsplit=1)
+
+        def line_error():
+            print('malformed line {}: {}'.format(*line), file=sys.stderr)
+
+        # handle metadata
+        if not in_dump:
+            if len(tokens) > 0 and tokens[0] in ['D2', 'D21', 'D23']:
+                in_dump = True
+                typ = tokens[0]
+            continue
+        else:
+            if len(tokens) == 0:
+                line_error()
+                continue
+            elif tokens[0] == 'ok':
+                break
+            elif tokens[0] == 'error:' and len(tokens) == 2:
+                values = tokens[1].split(' ')
+                if typ == 'D23' and len(values) >= 3:
+                    reason = CrashReason(int(values[0], 0))
+                    pc = int(values[1], 0)
+                    sp = int(values[2], 0)
+                else:
+                    line_error()
+                continue
+            elif len(tokens) != 2 or not re.match(r'^[0-9a-fA-F]+$', tokens[0]):
+                line_error()
+                continue
+
+        # decode hex data
+        addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big')
+        data = bytes.fromhex(tokens[1])
+        ranges.append((addr, len(data)))
+
+        if buf_addr is None:
+            buf_addr = addr
+            buf_data = data
+        else:
+            # grow buffer as needed
+            buf_addr, buf_data = region_expand(buf_addr, buf_data,
+                                               addr, len(data))
+
+            # replace new part
+            rep_start = addr - buf_addr
+            rep_end = rep_start + len(data)
+            buf_data = buf_data[:rep_start] + data + buf_data[rep_end:]
+
+    # merge continuous ranges
+    ranges = merge_ranges(ranges)
+
+    if typ == 'D2':
+        # D2 doesn't guarantee registers to be present
+        regs = len(ranges) > 0 and \
+            ranges[0][0] == 0 and \
+            ranges[0][1] >= avr.SRAM_START
+
+        # fill to fit for easy loading
+        buf_addr, buf_data = region_expand(
+            buf_addr, buf_data, 0, avr.SRAM_START + avr.SRAM_SIZE)
+
+    elif typ == 'D23':
+        # check if the dump is complete
+        if len(ranges) != 1 or ranges[0][0] != 0 or \
+           ranges[0][1] != avr.SRAM_START + avr.SRAM_SIZE:
+            print('warning: incomplete D23 dump', file=sys.stderr)
+        else:
+            regs = True
+        if reason is None:
+            print('warning: no error line in D23', file=sys.stderr)
+
+    elif typ == 'D21':
+        if len(ranges) != 1 or len(buf_data) != (avr.SRAM_START + avr.SRAM_SIZE + 256):
+            print('error: incomplete D21 dump', file=sys.stderr)
+            return None
+
+        # decode the header structure
+        magic, regs_present, crash_reason, pc, sp = struct.unpack('<LBBLH', buf_data[0:12])
+        if magic != 0x55525547:
+            print('error: invalid dump header in D21', file=sys.stderr)
+            return None
+
+        regs = bool(regs_present)
+        reason = CrashReason(crash_reason)
+
+        # extract the data section
+        buf_addr = 0
+        buf_data = buf_data[256:]
+        ranges[0] = (0, len(buf_data))
+
+    return Dump(typ, reason, regs, pc, sp, buf_data, ranges)