浏览代码

Merge pull request #58 from rauls5382/disasm

Add source information to disassembly reports
Raul Silvera 8 年前
父节点
当前提交
706c7989d0

+ 5
- 4
driver/driver.go 查看文件

@@ -135,10 +135,11 @@ type ObjTool interface {
135 135
 
136 136
 // An Inst is a single instruction in an assembly listing.
137 137
 type Inst struct {
138
-	Addr uint64 // virtual address of instruction
139
-	Text string // instruction text
140
-	File string // source file
141
-	Line int    // source line
138
+	Addr     uint64 // virtual address of instruction
139
+	Text     string // instruction text
140
+	Function string // function name
141
+	File     string // source file
142
+	Line     int    // source line
142 143
 }
143 144
 
144 145
 // An ObjFile is a single object file: a shared library or executable.

+ 14
- 5
internal/binutils/disasm.go 查看文件

@@ -28,6 +28,7 @@ var (
28 28
 	nmOutputRE            = regexp.MustCompile(`^\s*([[:xdigit:]]+)\s+(.)\s+(.*)`)
29 29
 	objdumpAsmOutputRE    = regexp.MustCompile(`^\s*([[:xdigit:]]+):\s+(.*)`)
30 30
 	objdumpOutputFileLine = regexp.MustCompile(`^(.*):([0-9]+)`)
31
+	objdumpOutputFunction = regexp.MustCompile(`^(\S.*)\(\):`)
31 32
 )
32 33
 
33 34
 func findSymbols(syms []byte, file string, r *regexp.Regexp, address uint64) ([]*plugin.Sym, error) {
@@ -83,7 +84,7 @@ func matchSymbol(names []string, start, end uint64, r *regexp.Regexp, address ui
83 84
 // the assembly instructions in a slice.
84 85
 func disassemble(asm []byte) ([]plugin.Inst, error) {
85 86
 	buf := bytes.NewBuffer(asm)
86
-	file, line := "", 0
87
+	function, file, line := "", "", 0
87 88
 	var assembly []plugin.Inst
88 89
 	for {
89 90
 		input, err := buf.ReadString('\n')
@@ -100,10 +101,11 @@ func disassemble(asm []byte) ([]plugin.Inst, error) {
100 101
 			if address, err := strconv.ParseUint(fields[1], 16, 64); err == nil {
101 102
 				assembly = append(assembly,
102 103
 					plugin.Inst{
103
-						Addr: address,
104
-						Text: fields[2],
105
-						File: file,
106
-						Line: line,
104
+						Addr:     address,
105
+						Text:     fields[2],
106
+						Function: function,
107
+						File:     file,
108
+						Line:     line,
107 109
 					})
108 110
 				continue
109 111
 			}
@@ -112,7 +114,14 @@ func disassemble(asm []byte) ([]plugin.Inst, error) {
112 114
 			if l, err := strconv.ParseUint(fields[2], 10, 32); err == nil {
113 115
 				file, line = fields[1], int(l)
114 116
 			}
117
+			continue
118
+		}
119
+		if fields := objdumpOutputFunction.FindStringSubmatch(input); len(fields) == 2 {
120
+			function = fields[1]
121
+			continue
115 122
 		}
123
+		// Reset on unrecognized lines.
124
+		function, file, line = "", "", 0
116 125
 	}
117 126
 
118 127
 	return assembly, nil

+ 12
- 12
internal/binutils/disasm_test.go 查看文件

@@ -116,10 +116,10 @@ func TestFunctionAssembly(t *testing.T) {
116 116
   1003: instruction four
117 117
 `,
118 118
 			[]plugin.Inst{
119
-				{0x1000, "instruction one", "", 0},
120
-				{0x1001, "instruction two", "", 0},
121
-				{0x1002, "instruction three", "", 0},
122
-				{0x1003, "instruction four", "", 0},
119
+				{Addr: 0x1000, Text: "instruction one"},
120
+				{Addr: 0x1001, Text: "instruction two"},
121
+				{Addr: 0x1002, Text: "instruction three"},
122
+				{Addr: 0x1003, Text: "instruction four"},
123 123
 			},
124 124
 		},
125 125
 		{
@@ -128,8 +128,8 @@ func TestFunctionAssembly(t *testing.T) {
128 128
   2001: instruction two
129 129
 `,
130 130
 			[]plugin.Inst{
131
-				{0x2000, "instruction one", "", 0},
132
-				{0x2001, "instruction two", "", 0},
131
+				{Addr: 0x2000, Text: "instruction one"},
132
+				{Addr: 0x2001, Text: "instruction two"},
133 133
 			},
134 134
 		},
135 135
 	}
@@ -137,17 +137,17 @@ func TestFunctionAssembly(t *testing.T) {
137 137
 	const objdump = "testdata/wrapper/objdump"
138 138
 
139 139
 	for _, tc := range testcases {
140
-		insns, err := disassemble([]byte(tc.asm))
140
+		insts, err := disassemble([]byte(tc.asm))
141 141
 		if err != nil {
142 142
 			t.Fatalf("FunctionAssembly: %v", err)
143 143
 		}
144 144
 
145
-		if len(insns) != len(tc.want) {
146
-			t.Errorf("Unexpected number of assembly instructions %d (want %d)\n", len(insns), len(tc.want))
145
+		if len(insts) != len(tc.want) {
146
+			t.Errorf("Unexpected number of assembly instructions %d (want %d)\n", len(insts), len(tc.want))
147 147
 		}
148
-		for i := range insns {
149
-			if insns[i] != tc.want[i] {
150
-				t.Errorf("Expected symbol %v, got %v\n", tc.want[i], insns[i])
148
+		for i := range insts {
149
+			if insts[i] != tc.want[i] {
150
+				t.Errorf("Expected symbol %v, got %v\n", tc.want[i], insts[i])
151 151
 			}
152 152
 		}
153 153
 	}

+ 9
- 9
internal/driver/driver_test.go 查看文件

@@ -1020,18 +1020,18 @@ func (m *mockObjTool) Disasm(file string, start, end uint64) ([]plugin.Inst, err
1020 1020
 	switch start {
1021 1021
 	case 0x1000:
1022 1022
 		return []plugin.Inst{
1023
-			{0x1000, "instruction one", "", 0},
1024
-			{0x1001, "instruction two", "", 0},
1025
-			{0x1002, "instruction three", "", 0},
1026
-			{0x1003, "instruction four", "", 0},
1023
+			{Addr: 0x1000, Text: "instruction one"},
1024
+			{Addr: 0x1001, Text: "instruction two"},
1025
+			{Addr: 0x1002, Text: "instruction three"},
1026
+			{Addr: 0x1003, Text: "instruction four"},
1027 1027
 		}, nil
1028 1028
 	case 0x3000:
1029 1029
 		return []plugin.Inst{
1030
-			{0x3000, "instruction one", "", 0},
1031
-			{0x3001, "instruction two", "", 0},
1032
-			{0x3002, "instruction three", "", 0},
1033
-			{0x3003, "instruction four", "", 0},
1034
-			{0x3004, "instruction five", "", 0},
1030
+			{Addr: 0x3000, Text: "instruction one"},
1031
+			{Addr: 0x3001, Text: "instruction two"},
1032
+			{Addr: 0x3002, Text: "instruction three"},
1033
+			{Addr: 0x3003, Text: "instruction four"},
1034
+			{Addr: 0x3004, Text: "instruction five"},
1035 1035
 		}, nil
1036 1036
 	}
1037 1037
 	return nil, fmt.Errorf("unimplemented")

+ 4
- 4
internal/driver/testdata/pprof.cpu.flat.addresses.disasm 查看文件

@@ -1,14 +1,14 @@
1 1
 Total: 1.12s
2 2
 ROUTINE ======================== line1000
3 3
      1.10s      1.10s (flat, cum) 98.21% of Total
4
-     1.10s      1.10s       1000: instruction one
4
+     1.10s      1.10s       1000: instruction one                         ; line1000 file1000.src:1
5 5
          .          .       1001: instruction two
6 6
          .          .       1002: instruction three
7 7
          .          .       1003: instruction four
8 8
 ROUTINE ======================== line3000
9 9
       10ms      1.12s (flat, cum)   100% of Total
10
-      10ms      1.01s       3000: instruction one
11
-         .      100ms       3001: instruction two
12
-         .       10ms       3002: instruction three
10
+      10ms      1.01s       3000: instruction one                         ; line3000 file3000.src:6
11
+         .      100ms       3001: instruction two                         ; line3000 file3000.src:9
12
+         .       10ms       3002: instruction three                       ; line3000 file3000.src:7
13 13
          .          .       3003: instruction four
14 14
          .          .       3004: instruction five

+ 4
- 4
internal/driver/testdata/pprof.cpu.flat.addresses.weblist 查看文件

@@ -69,7 +69,7 @@ Type: cpu<br>
69 69
 Duration: 10s, Total samples = 1.12s (11.20%)<br>Total: 1.12s</div><h1>line1000</h1>testdata/file1000.src
70 70
 <pre onClick="pprof_toggle_asm(event)">
71 71
   Total:       1.10s      1.10s (flat, cum) 98.21%
72
-<span class=line>      1</span> <span class=deadsrc>       1.10s      1.10s line1 </span><span class=asm>               1.10s      1.10s     1000: instruction one                                  <span class=disasmloc>testdata/file1000.src:1</span>
72
+<span class=line>      1</span> <span class=deadsrc>       1.10s      1.10s line1 </span><span class=asm>               1.10s      1.10s     1000: instruction one                                  <span class=disasmloc>file1000.src:1</span>
73 73
                    .          .     1001: instruction two                                  <span class=disasmloc></span>
74 74
                    .          .     1002: instruction three                                <span class=disasmloc></span>
75 75
                    .          .     1003: instruction four                                 <span class=disasmloc></span>
@@ -88,14 +88,14 @@ Duration: 10s, Total samples = 1.12s (11.20%)<br>Total: 1.12s</div><h1>line1000<
88 88
 <span class=line>      3</span> <span class=nop>           .          . line3 </span>
89 89
 <span class=line>      4</span> <span class=nop>           .          . line4 </span>
90 90
 <span class=line>      5</span> <span class=nop>           .          . line5 </span>
91
-<span class=line>      6</span> <span class=deadsrc>        10ms      1.01s line6 </span><span class=asm>                10ms      1.01s     3000: instruction one                                  <span class=disasmloc>testdata/file3000.src:6</span>
91
+<span class=line>      6</span> <span class=deadsrc>        10ms      1.01s line6 </span><span class=asm>                10ms      1.01s     3000: instruction one                                  <span class=disasmloc>file3000.src:6</span>
92 92
 </span>
93
-<span class=line>      7</span> <span class=deadsrc>           .       10ms line7 </span><span class=asm>                   .       10ms     3002: instruction three                                <span class=disasmloc>testdata/file3000.src:7</span>
93
+<span class=line>      7</span> <span class=deadsrc>           .       10ms line7 </span><span class=asm>                   .       10ms     3002: instruction three                                <span class=disasmloc>file3000.src:7</span>
94 94
                    .          .     3003: instruction four                                 <span class=disasmloc></span>
95 95
                    .          .     3004: instruction five                                 <span class=disasmloc></span>
96 96
 </span>
97 97
 <span class=line>      8</span> <span class=nop>           .          . line8 </span>
98
-<span class=line>      9</span> <span class=deadsrc>           .      100ms line9 </span><span class=asm>                   .      100ms     3001: instruction two                                  <span class=disasmloc>testdata/file3000.src:9</span>
98
+<span class=line>      9</span> <span class=deadsrc>           .      100ms line9 </span><span class=asm>                   .      100ms     3001: instruction two                                  <span class=disasmloc>file3000.src:9</span>
99 99
 </span>
100 100
 <span class=line>     10</span> <span class=nop>           .          . line0 </span>
101 101
 <span class=line>     11</span> <span class=nop>           .          . line1 </span>

+ 5
- 4
internal/plugin/plugin.go 查看文件

@@ -110,10 +110,11 @@ type ObjTool interface {
110 110
 
111 111
 // An Inst is a single instruction in an assembly listing.
112 112
 type Inst struct {
113
-	Addr uint64 // virtual address of instruction
114
-	Text string // instruction text
115
-	File string // source file
116
-	Line int    // source line
113
+	Addr     uint64 // virtual address of instruction
114
+	Text     string // instruction text
115
+	Function string // function name
116
+	File     string // source file
117
+	Line     int    // source line
117 118
 }
118 119
 
119 120
 // An ObjFile is a single object file: a shared library or executable.

+ 84
- 24
internal/report/report.go 查看文件

@@ -303,12 +303,12 @@ func printAssembly(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
303 303
 		flatSum, cumSum := sns.Sum()
304 304
 
305 305
 		// Get the function assembly.
306
-		insns, err := obj.Disasm(s.sym.File, s.sym.Start, s.sym.End)
306
+		insts, err := obj.Disasm(s.sym.File, s.sym.Start, s.sym.End)
307 307
 		if err != nil {
308 308
 			return err
309 309
 		}
310 310
 
311
-		ns := annotateAssembly(insns, sns, s.base)
311
+		ns := annotateAssembly(insts, sns, s.base)
312 312
 
313 313
 		fmt.Fprintf(w, "ROUTINE ======================== %s\n", s.sym.Name[0])
314 314
 		for _, name := range s.sym.Name[1:] {
@@ -318,8 +318,37 @@ func printAssembly(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
318 318
 			rpt.formatValue(flatSum), rpt.formatValue(cumSum),
319 319
 			percentage(cumSum, rpt.total))
320 320
 
321
+		function, file, line := "", "", 0
321 322
 		for _, n := range ns {
322
-			fmt.Fprintf(w, "%10s %10s %10x: %s\n", valueOrDot(n.FlatValue(), rpt), valueOrDot(n.CumValue(), rpt), n.Info.Address, n.Info.Name)
323
+			flat := valueOrDot(n.flatValue(), rpt)
324
+			cum := valueOrDot(n.cumValue(), rpt)
325
+			if n.function == function && n.file == file && n.line == line {
326
+				fmt.Fprintf(w, "%10s %10s %10x: %s\n",
327
+					flat, cum,
328
+					n.address, n.instruction,
329
+				)
330
+				continue
331
+			}
332
+			line := ""
333
+			if n.line > 0 {
334
+				line = fmt.Sprintf(":%d", n.line)
335
+			}
336
+			if len(n.instruction) <= 40 {
337
+				fmt.Fprintf(w, "%10s %10s %10x: %-40s; %s %s%s\n",
338
+					flat, cum,
339
+					n.address, n.instruction,
340
+					n.function, n.file, line,
341
+				)
342
+				continue
343
+			}
344
+			fmt.Fprintf(w, "%75s; %s %s%s\n",
345
+				"",
346
+				n.function, n.file, line,
347
+			)
348
+			fmt.Fprintf(w, "%10s %10s %10x: %s\n",
349
+				flat, cum,
350
+				n.address, n.instruction,
351
+			)
323 352
 		}
324 353
 	}
325 354
 	return nil
@@ -417,43 +446,74 @@ func nodesPerSymbol(ns graph.Nodes, symbols []*objSymbol) map[*objSymbol]graph.N
417 446
 	return symNodes
418 447
 }
419 448
 
449
+type assemblyInstruction struct {
450
+	address         uint64
451
+	instruction     string
452
+	function        string
453
+	file            string
454
+	line            int
455
+	flat, cum       int64
456
+	flatDiv, cumDiv int64
457
+}
458
+
459
+func (a *assemblyInstruction) flatValue() int64 {
460
+	if a.flatDiv != 0 {
461
+		return a.flat / a.flatDiv
462
+	}
463
+	return a.flat
464
+}
465
+
466
+func (a *assemblyInstruction) cumValue() int64 {
467
+	if a.cumDiv != 0 {
468
+		return a.cum / a.cumDiv
469
+	}
470
+	return a.cum
471
+}
472
+
420 473
 // annotateAssembly annotates a set of assembly instructions with a
421 474
 // set of samples. It returns a set of nodes to display. base is an
422 475
 // offset to adjust the sample addresses.
423
-func annotateAssembly(insns []plugin.Inst, samples graph.Nodes, base uint64) graph.Nodes {
476
+func annotateAssembly(insts []plugin.Inst, samples graph.Nodes, base uint64) []assemblyInstruction {
424 477
 	// Add end marker to simplify printing loop.
425
-	insns = append(insns, plugin.Inst{
478
+	insts = append(insts, plugin.Inst{
426 479
 		Addr: ^uint64(0),
427 480
 	})
428 481
 
429 482
 	// Ensure samples are sorted by address.
430 483
 	samples.Sort(graph.AddressOrder)
431 484
 
432
-	var s int
433
-	var asm graph.Nodes
434
-	for ix, in := range insns[:len(insns)-1] {
435
-		n := graph.Node{
436
-			Info: graph.NodeInfo{
437
-				Address: in.Addr,
438
-				Name:    in.Text,
439
-				File:    trimPath(in.File),
440
-				Lineno:  in.Line,
441
-			},
485
+	s := 0
486
+	asm := make([]assemblyInstruction, 0, len(insts))
487
+	for ix, in := range insts[:len(insts)-1] {
488
+		n := assemblyInstruction{
489
+			address:     in.Addr,
490
+			instruction: in.Text,
491
+			function:    in.Function,
492
+			line:        in.Line,
493
+		}
494
+		if in.File != "" {
495
+			n.file = filepath.Base(in.File)
442 496
 		}
443 497
 
444 498
 		// Sum all the samples until the next instruction (to account
445 499
 		// for samples attributed to the middle of an instruction).
446
-		for next := insns[ix+1].Addr; s < len(samples) && samples[s].Info.Address-base < next; s++ {
447
-			n.FlatDiv += samples[s].FlatDiv
448
-			n.Flat += samples[s].Flat
449
-			n.CumDiv += samples[s].CumDiv
450
-			n.Cum += samples[s].Cum
451
-			if samples[s].Info.File != "" {
452
-				n.Info.File = trimPath(samples[s].Info.File)
453
-				n.Info.Lineno = samples[s].Info.Lineno
500
+		for next := insts[ix+1].Addr; s < len(samples) && samples[s].Info.Address-base < next; s++ {
501
+			sample := samples[s]
502
+			n.flatDiv += sample.FlatDiv
503
+			n.flat += sample.Flat
504
+			n.cumDiv += sample.CumDiv
505
+			n.cum += sample.Cum
506
+			if f := sample.Info.File; f != "" && n.file == "" {
507
+				n.file = filepath.Base(f)
508
+			}
509
+			if ln := sample.Info.Lineno; ln != 0 && n.line == 0 {
510
+				n.line = ln
511
+			}
512
+			if f := sample.Info.Name; f != "" && n.function == "" {
513
+				n.function = f
454 514
 			}
455 515
 		}
456
-		asm = append(asm, &n)
516
+		asm = append(asm, n)
457 517
 	}
458 518
 
459 519
 	return asm

+ 33
- 18
internal/report/source.go 查看文件

@@ -204,7 +204,7 @@ func printWebSource(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
204 204
 
205 205
 // sourceCoordinates returns the lowest and highest line numbers from
206 206
 // a set of assembly statements.
207
-func sourceCoordinates(asm map[int]graph.Nodes) (start, end int) {
207
+func sourceCoordinates(asm map[int][]assemblyInstruction) (start, end int) {
208 208
 	for l := range asm {
209 209
 		if start == 0 || l < start {
210 210
 			start = l
@@ -219,8 +219,8 @@ func sourceCoordinates(asm map[int]graph.Nodes) (start, end int) {
219 219
 // assemblyPerSourceLine disassembles the binary containing a symbol
220 220
 // and classifies the assembly instructions according to its
221 221
 // corresponding source line, annotating them with a set of samples.
222
-func assemblyPerSourceLine(objSyms []*objSymbol, rs graph.Nodes, src string, obj plugin.ObjTool) map[int]graph.Nodes {
223
-	assembly := make(map[int]graph.Nodes)
222
+func assemblyPerSourceLine(objSyms []*objSymbol, rs graph.Nodes, src string, obj plugin.ObjTool) map[int][]assemblyInstruction {
223
+	assembly := make(map[int][]assemblyInstruction)
224 224
 	// Identify symbol to use for this collection of samples.
225 225
 	o := findMatchingSymbol(objSyms, rs)
226 226
 	if o == nil {
@@ -228,17 +228,17 @@ func assemblyPerSourceLine(objSyms []*objSymbol, rs graph.Nodes, src string, obj
228 228
 	}
229 229
 
230 230
 	// Extract assembly for matched symbol
231
-	insns, err := obj.Disasm(o.sym.File, o.sym.Start, o.sym.End)
231
+	insts, err := obj.Disasm(o.sym.File, o.sym.Start, o.sym.End)
232 232
 	if err != nil {
233 233
 		return assembly
234 234
 	}
235 235
 
236 236
 	srcBase := filepath.Base(src)
237
-	anodes := annotateAssembly(insns, rs, o.base)
237
+	anodes := annotateAssembly(insts, rs, o.base)
238 238
 	var lineno = 0
239 239
 	for _, an := range anodes {
240
-		if filepath.Base(an.Info.File) == srcBase {
241
-			lineno = an.Info.Lineno
240
+		if filepath.Base(an.file) == srcBase {
241
+			lineno = an.line
242 242
 		}
243 243
 		if lineno != 0 {
244 244
 			assembly[lineno] = append(assembly[lineno], an)
@@ -290,7 +290,7 @@ func printFunctionHeader(w io.Writer, name, path string, flatSum, cumSum int64,
290 290
 }
291 291
 
292 292
 // printFunctionSourceLine prints a source line and the corresponding assembly.
293
-func printFunctionSourceLine(w io.Writer, fn *graph.Node, assembly graph.Nodes, rpt *Report) {
293
+func printFunctionSourceLine(w io.Writer, fn *graph.Node, assembly []assemblyInstruction, rpt *Report) {
294 294
 	if len(assembly) == 0 {
295 295
 		fmt.Fprintf(w,
296 296
 			"<span class=line> %6d</span> <span class=nop>  %10s %10s %s </span>\n",
@@ -309,16 +309,23 @@ func printFunctionSourceLine(w io.Writer, fn *graph.Node, assembly graph.Nodes,
309 309
 	for _, an := range assembly {
310 310
 		var fileline string
311 311
 		class := "disasmloc"
312
-		if an.Info.File != "" {
313
-			fileline = fmt.Sprintf("%s:%d", template.HTMLEscapeString(an.Info.File), an.Info.Lineno)
314
-			if an.Info.Lineno != fn.Info.Lineno {
312
+		if an.file != "" {
313
+			fileline = fmt.Sprintf("%s:%d", template.HTMLEscapeString(an.file), an.line)
314
+			if an.line != fn.Info.Lineno {
315 315
 				class = "unimportant"
316 316
 			}
317 317
 		}
318
+		flat, cum := an.flat, an.cum
319
+		if an.flatDiv != 0 {
320
+			flat = flat / an.flatDiv
321
+		}
322
+		if an.cumDiv != 0 {
323
+			cum = cum / an.cumDiv
324
+		}
318 325
 		fmt.Fprintf(w, " %8s %10s %10s %8x: %-48s <span class=%s>%s</span>\n", "",
319
-			valueOrDot(an.Flat, rpt), valueOrDot(an.Cum, rpt),
320
-			an.Info.Address,
321
-			template.HTMLEscapeString(an.Info.Name),
326
+			valueOrDot(flat, rpt), valueOrDot(cum, rpt),
327
+			an.address,
328
+			template.HTMLEscapeString(an.instruction),
322 329
 			class,
323 330
 			template.HTMLEscapeString(fileline))
324 331
 	}
@@ -411,14 +418,22 @@ func getSourceFromFile(file, sourcePath string, fns graph.Nodes, start, end int)
411 418
 
412 419
 // getMissingFunctionSource creates a dummy function body to point to
413 420
 // the source file and annotates it with the samples in asm.
414
-func getMissingFunctionSource(filename string, asm map[int]graph.Nodes, start, end int) (graph.Nodes, string) {
421
+func getMissingFunctionSource(filename string, asm map[int][]assemblyInstruction, start, end int) (graph.Nodes, string) {
415 422
 	var fnodes graph.Nodes
416 423
 	for i := start; i <= end; i++ {
417
-		lrs := asm[i]
418
-		if len(lrs) == 0 {
424
+		insts := asm[i]
425
+		if len(insts) == 0 {
419 426
 			continue
420 427
 		}
421
-		flat, cum := lrs.Sum()
428
+		var group assemblyInstruction
429
+		for _, insn := range insts {
430
+			group.flat += insn.flat
431
+			group.cum += insn.cum
432
+			group.flatDiv += insn.flatDiv
433
+			group.cumDiv += insn.cumDiv
434
+		}
435
+		flat := group.flatValue()
436
+		cum := group.cumValue()
422 437
 		fnodes = append(fnodes, &graph.Node{
423 438
 			Info: graph.NodeInfo{
424 439
 				Name:   "???",