Fix mean computation during graph creation

When -mean is selected, currently pprof divides the sample value
by value[0], which is expected to be the number of samples. This
is intended to produce mean value per sample. These means cannot
be added. Instead, we should add the value and the number of samples
independently and perform the division at the end.

To do this we will create a separate function to get the number of samples,
and accumulate it independently from the sample value (weigth) and apply
the division after the accumulation is completed.

internal/driver/driver.go

@@ -206,7 +206,7 @@ func aggregate(prof *profile.Profile, v variables) error {
 
 func reportOptions(p *profile.Profile, vars variables) (*report.Options, error) {
 	si, mean := vars["sample_index"].value, vars["mean"].boolValue()
-	value, sample, err := sampleFormat(p, si, mean)
+	value, meanDiv, sample, err := sampleFormat(p, si, mean)
 	if err != nil {
 		return nil, err
 	}
@@ -233,9 +233,10 @@ func reportOptions(p *profile.Profile, vars variables) (*report.Options, error)
 		NodeFraction: vars["nodefraction"].floatValue(),
 		EdgeFraction: vars["edgefraction"].floatValue(),
 
-		SampleValue: value,
-		SampleType:  stype,
-		SampleUnit:  sample.Unit,
+		SampleValue:       value,
+		SampleMeanDivisor: meanDiv,
+		SampleType:        stype,
+		SampleUnit:        sample.Unit,
 
 		OutputUnit: vars["unit"].value,
 
@@ -253,18 +254,20 @@ type sampleValueFunc func([]int64) int64
 
 // sampleFormat returns a function to extract values out of a profile.Sample,
 // and the type/units of those values.
-func sampleFormat(p *profile.Profile, sampleIndex string, mean bool) (sampleValueFunc, *profile.ValueType, error) {
+func sampleFormat(p *profile.Profile, sampleIndex string, mean bool) (value, meanDiv sampleValueFunc, v *profile.ValueType, err error) {
 	if len(p.SampleType) == 0 {
-		return nil, nil, fmt.Errorf("profile has no samples")
+		return nil, nil, nil, fmt.Errorf("profile has no samples")
 	}
 	index, err := locateSampleIndex(p, sampleIndex)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
+	value = valueExtractor(index)
 	if mean {
-		return meanExtractor(index), p.SampleType[index], nil
+		meanDiv = valueExtractor(0)
 	}
-	return valueExtractor(index), p.SampleType[index], nil
+	v = p.SampleType[index]
+	return
 }
 
 func valueExtractor(ix int) sampleValueFunc {
@@ -272,12 +275,3 @@ func valueExtractor(ix int) sampleValueFunc {
 		return v[ix]
 	}
 }
-
-func meanExtractor(ix int) sampleValueFunc {
-	return func(v []int64) int64 {
-		if v[0] == 0 {
-			return 0
-		}
-		return v[ix] / v[0]
-	}
-}

internal/graph/dotgraph.go

@@ -67,11 +67,11 @@ func ComposeDot(w io.Writer, g *Graph, a *DotAttributes, c *DotConfig) {
 	nodeIDMap := make(map[*Node]int)
 	hasNodelets := make(map[*Node]bool)
 
-	maxFlat := float64(abs64(g.Nodes[0].Flat))
+	maxFlat := float64(abs64(g.Nodes[0].FlatValue()))
 	for i, n := range g.Nodes {
 		nodeIDMap[n] = i + 1
-		if float64(abs64(n.Flat)) > maxFlat {
-			maxFlat = float64(abs64(n.Flat))
+		if float64(abs64(n.FlatValue())) > maxFlat {
+			maxFlat = float64(abs64(n.FlatValue()))
 		}
 	}
 
@@ -128,7 +128,7 @@ func (b *builder) addLegend() {
 
 // addNode generates a graph node in DOT format.
 func (b *builder) addNode(node *Node, nodeID int, maxFlat float64) {
-	flat, cum := node.Flat, node.Cum
+	flat, cum := node.FlatValue(), node.CumValue()
 	attrs := b.attributes.Nodes[node]
 
 	// Populate label for node.
@@ -177,8 +177,8 @@ func (b *builder) addNode(node *Node, nodeID int, maxFlat float64) {
 	// Create DOT attribute for node.
 	attr := fmt.Sprintf(`label="%s" fontsize=%d shape=%s tooltip="%s (%s)" color="%s" fillcolor="%s"`,
 		label, fontSize, shape, node.Info.PrintableName(), cumValue,
-		dotColor(float64(node.Cum)/float64(abs64(b.config.Total)), false),
-		dotColor(float64(node.Cum)/float64(abs64(b.config.Total)), true))
+		dotColor(float64(node.CumValue())/float64(abs64(b.config.Total)), false),
+		dotColor(float64(node.CumValue())/float64(abs64(b.config.Total)), true))
 
 	// Add on extra attributes if provided.
 	if attrs != nil {
@@ -230,9 +230,9 @@ func (b *builder) addNodelets(node *Node, nodeID int) bool {
 		ts = ts[:maxNodelets]
 	}
 	for i, t := range ts {
-		w := t.Cum
+		w := t.CumValue()
 		if flatTags {
-			w = t.Flat
+			w = t.FlatValue()
 		}
 		if w == 0 {
 			continue
@@ -259,9 +259,9 @@ func (b *builder) numericNodelets(nts []*Tag, maxNumNodelets int, flatTags bool,
 	// Collapse numeric labels into maxNumNodelets buckets, of the form:
 	// 1MB..2MB, 3MB..5MB, ...
 	for j, t := range collapsedTags(nts, maxNumNodelets, flatTags) {
-		w, attr := t.Cum, ` style="dotted"`
-		if flatTags || t.Flat == t.Cum {
-			w, attr = t.Flat, ""
+		w, attr := t.CumValue(), ` style="dotted"`
+		if flatTags || t.FlatValue() == t.CumValue() {
+			w, attr = t.FlatValue(), ""
 		}
 		if w != 0 {
 			weight := b.config.FormatValue(w)
@@ -278,18 +278,18 @@ func (b *builder) addEdge(edge *Edge, from, to int, hasNodelets bool) {
 	if edge.Inline {
 		inline = `\n (inline)`
 	}
-	w := b.config.FormatValue(edge.Weight)
+	w := b.config.FormatValue(edge.WeightValue())
 	attr := fmt.Sprintf(`label=" %s%s"`, w, inline)
 	if b.config.Total != 0 {
 		// Note: edge.weight > b.config.Total is possible for profile diffs.
-		if weight := 1 + int(min64(abs64(edge.Weight*100/b.config.Total), 100)); weight > 1 {
+		if weight := 1 + int(min64(abs64(edge.WeightValue()*100/b.config.Total), 100)); weight > 1 {
 			attr = fmt.Sprintf(`%s weight=%d`, attr, weight)
 		}
-		if width := 1 + int(min64(abs64(edge.Weight*5/b.config.Total), 5)); width > 1 {
+		if width := 1 + int(min64(abs64(edge.WeightValue()*5/b.config.Total), 5)); width > 1 {
 			attr = fmt.Sprintf(`%s penwidth=%d`, attr, width)
 		}
 		attr = fmt.Sprintf(`%s color="%s"`, attr,
-			dotColor(float64(edge.Weight)/float64(abs64(b.config.Total)), false))
+			dotColor(float64(edge.WeightValue())/float64(abs64(b.config.Total)), false))
 	}
 	arrow := "->"
 	if edge.Residual {
@@ -442,12 +442,12 @@ func tagDistance(t, u *Tag) float64 {
 func tagGroupLabel(g []*Tag) (label string, flat, cum int64) {
 	if len(g) == 1 {
 		t := g[0]
-		return measurement.Label(t.Value, t.Unit), t.Flat, t.Cum
+		return measurement.Label(t.Value, t.Unit), t.FlatValue(), t.CumValue()
 	}
 	min := g[0]
 	max := g[0]
-	f := min.Flat
-	c := min.Cum
+	df, f := min.FlatDiv, min.Flat
+	dc, c := min.CumDiv, min.Cum
 	for _, t := range g[1:] {
 		if v, _ := measurement.Scale(t.Value, t.Unit, min.Unit); int64(v) < min.Value {
 			min = t
@@ -456,7 +456,15 @@ func tagGroupLabel(g []*Tag) (label string, flat, cum int64) {
 			max = t
 		}
 		f += t.Flat
+		df += t.FlatDiv
 		c += t.Cum
+		dc += t.CumDiv
+	}
+	if df != 0 {
+		f = f / df
+	}
+	if dc != 0 {
+		c = c / dc
 	}
 	return measurement.Label(min.Value, min.Unit) + ".." + measurement.Label(max.Value, max.Unit), f, c
 }

internal/graph/dotgraph_test.go

@@ -224,35 +224,40 @@ func TestMultilinePrintableName(t *testing.T) {
 }
 
 func TestTagCollapse(t *testing.T) {
+
+	makeTag := func(name, unit string, value, flat, cum int64) *Tag {
+		return &Tag{name, unit, value, flat, 0, cum, 0}
+	}
+
 	tagSource := []*Tag{
-		{"12mb", "mb", 12, 100, 100},
-		{"1kb", "kb", 1, 1, 1},
-		{"1mb", "mb", 1, 1000, 1000},
-		{"2048mb", "mb", 2048, 1000, 1000},
-		{"1b", "b", 1, 100, 100},
-		{"2b", "b", 2, 100, 100},
-		{"7b", "b", 7, 100, 100},
+		makeTag("12mb", "mb", 12, 100, 100),
+		makeTag("1kb", "kb", 1, 1, 1),
+		makeTag("1mb", "mb", 1, 1000, 1000),
+		makeTag("2048mb", "mb", 2048, 1000, 1000),
+		makeTag("1b", "b", 1, 100, 100),
+		makeTag("2b", "b", 2, 100, 100),
+		makeTag("7b", "b", 7, 100, 100),
 	}
 
 	tagWant := [][]*Tag{
 		[]*Tag{
-			{"1B..2GB", "", 0, 2401, 2401},
+			makeTag("1B..2GB", "", 0, 2401, 2401),
 		},
 		[]*Tag{
-			{"2GB", "", 0, 1000, 1000},
-			{"1B..12MB", "", 0, 1401, 1401},
+			makeTag("2GB", "", 0, 1000, 1000),
+			makeTag("1B..12MB", "", 0, 1401, 1401),
 		},
 		[]*Tag{
-			{"2GB", "", 0, 1000, 1000},
-			{"12MB", "", 0, 100, 100},
-			{"1B..1MB", "", 0, 1301, 1301},
+			makeTag("2GB", "", 0, 1000, 1000),
+			makeTag("12MB", "", 0, 100, 100),
+			makeTag("1B..1MB", "", 0, 1301, 1301),
 		},
 		[]*Tag{
-			{"2GB", "", 0, 1000, 1000},
-			{"1MB", "", 0, 1000, 1000},
-			{"2B..1kB", "", 0, 201, 201},
-			{"1B", "", 0, 100, 100},
-			{"12MB", "", 0, 100, 100},
+			makeTag("2GB", "", 0, 1000, 1000),
+			makeTag("1MB", "", 0, 1000, 1000),
+			makeTag("2B..1kB", "", 0, 201, 201),
+			makeTag("1B", "", 0, 100, 100),
+			makeTag("12MB", "", 0, 100, 100),
 		},
 	}
 

internal/graph/graph.go

@@ -34,10 +34,11 @@ type Graph struct {
 
 // Options encodes the options for constructing a graph
 type Options struct {
-	SampleValue func(s []int64) int64      // Function to compute the value of a sample
-	FormatTag   func(int64, string) string // Function to format a sample tag value into a string
-	ObjNames    bool                       // Always preserve obj filename
-	OrigFnNames bool                       // Preserve original (eg mangled) function names
+	SampleValue       func(s []int64) int64      // Function to compute the value of a sample
+	SampleMeanDivisor func(s []int64) int64      // Function to compute the divisor for mean graphs, or nil
+	FormatTag         func(int64, string) string // Function to format a sample tag value into a string
+	ObjNames          bool                       // Always preserve obj filename
+	OrigFnNames       bool                       // Preserve original (eg mangled) function names
 
 	CallTree     bool // Build a tree instead of a graph
 	DropNegative bool // Drop nodes with overall negative values
@@ -63,7 +64,7 @@ type Node struct {
 
 	// Values associated to this node. Flat is exclusive to this node,
 	// Cum includes all descendents.
-	Flat, Cum int64
+	Flat, FlatDiv, Cum, CumDiv int64
 
 	// In and out Contains the nodes immediately reaching or reached by
 	// this node.
@@ -79,15 +80,40 @@ type Node struct {
 	NumericTags map[string]TagMap
 }
 
+// FlatValue returns the exclusive value for this node, computing the
+// mean if a divisor is available.
+func (n *Node) FlatValue() int64 {
+	if n.FlatDiv == 0 {
+		return n.Flat
+	}
+	return n.Flat / n.FlatDiv
+}
+
+// CumValue returns the inclusive value for this node, computing the
+// mean if a divisor is available.
+func (n *Node) CumValue() int64 {
+	if n.CumDiv == 0 {
+		return n.Cum
+	}
+	return n.Cum / n.CumDiv
+}
+
 // AddToEdge increases the weight of an edge between two nodes. If
 // there isn't such an edge one is created.
-func (n *Node) AddToEdge(to *Node, w int64, residual, inline bool) {
+func (n *Node) AddToEdge(to *Node, v int64, residual, inline bool) {
+	n.AddToEdgeDiv(to, 0, v, residual, inline)
+}
+
+// AddToEdgeDiv increases the weight of an edge between two nodes. If
+// there isn't such an edge one is created.
+func (n *Node) AddToEdgeDiv(to *Node, dv, v int64, residual, inline bool) {
 	if n.Out[to] != to.In[n] {
 		panic(fmt.Errorf("asymmetric edges %v %v", *n, *to))
 	}
 
 	if e := n.Out[to]; e != nil {
-		e.Weight += w
+		e.WeightDiv += dv
+		e.Weight += v
 		if residual {
 			e.Residual = true
 		}
@@ -97,7 +123,7 @@ func (n *Node) AddToEdge(to *Node, w int64, residual, inline bool) {
 		return
 	}
 
-	info := &Edge{Src: n, Dest: to, Weight: w, Residual: residual, Inline: inline}
+	info := &Edge{Src: n, Dest: to, WeightDiv: dv, Weight: v, Residual: residual, Inline: inline}
 	n.Out[to] = info
 	to.In[n] = info
 }
@@ -205,7 +231,8 @@ type EdgeMap map[*Node]*Edge
 type Edge struct {
 	Src, Dest *Node
 	// The summary weight of the edge
-	Weight int64
+	Weight, WeightDiv int64
+
 	// residual edges connect nodes that were connected through a
 	// separate node, which has been removed from the report.
 	Residual bool
@@ -213,13 +240,38 @@ type Edge struct {
 	Inline bool
 }
 
+func (e *Edge) WeightValue() int64 {
+	if e.WeightDiv == 0 {
+		return e.Weight
+	}
+	return e.Weight / e.WeightDiv
+}
+
 // Tag represent sample annotations
 type Tag struct {
-	Name  string
-	Unit  string // Describe the value, "" for non-numeric tags
-	Value int64
-	Flat  int64
-	Cum   int64
+	Name          string
+	Unit          string // Describe the value, "" for non-numeric tags
+	Value         int64
+	Flat, FlatDiv int64
+	Cum, CumDiv   int64
+}
+
+// FlatValue returns the exclusive value for this tag, computing the
+// mean if a divisor is available.
+func (t *Tag) FlatValue() int64 {
+	if t.FlatDiv == 0 {
+		return t.Flat
+	}
+	return t.Flat / t.FlatDiv
+}
+
+// CumValue returns the inclusive value for this tag, computing the
+// mean if a divisor is available.
+func (t *Tag) CumValue() int64 {
+	if t.CumDiv == 0 {
+		return t.Cum
+	}
+	return t.Cum / t.CumDiv
 }
 
 // TagMap is a collection of tags, classified by their name.
@@ -247,8 +299,12 @@ func New(prof *profile.Profile, o *Options) *Graph {
 func newGraph(prof *profile.Profile, o *Options) (*Graph, map[uint64]Nodes) {
 	nodes, locationMap := CreateNodes(prof, o)
 	for _, sample := range prof.Sample {
-		weight := o.SampleValue(sample.Value)
-		if weight == 0 {
+		var w, dw int64
+		w = o.SampleValue(sample.Value)
+		if o.SampleMeanDivisor != nil {
+			dw = o.SampleMeanDivisor(sample.Value)
+		}
+		if dw == 0 && w == 0 {
 			continue
 		}
 		seenNode := make(map[*Node]bool, len(sample.Location))
@@ -271,12 +327,12 @@ func newGraph(prof *profile.Profile, o *Options) (*Graph, map[uint64]Nodes) {
 				// Add cum weight to all nodes in stack, avoiding double counting.
 				if _, ok := seenNode[n]; !ok {
 					seenNode[n] = true
-					n.addSample(weight, labels, sample.NumLabel, o.FormatTag, false)
+					n.addSample(dw, w, labels, sample.NumLabel, o.FormatTag, false)
 				}
 				// Update edge weights for all edges in stack, avoiding double counting.
 				if _, ok := seenEdge[nodePair{n, parent}]; !ok && parent != nil && n != parent {
 					seenEdge[nodePair{n, parent}] = true
-					parent.AddToEdge(n, weight, residual, ni != len(locNodes)-1)
+					parent.AddToEdgeDiv(n, dw, w, residual, ni != len(locNodes)-1)
 				}
 				parent = n
 				residual = false
@@ -284,7 +340,7 @@ func newGraph(prof *profile.Profile, o *Options) (*Graph, map[uint64]Nodes) {
 		}
 		if parent != nil && !residual {
 			// Add flat weight to leaf node.
-			parent.addSample(weight, labels, sample.NumLabel, o.FormatTag, true)
+			parent.addSample(dw, w, labels, sample.NumLabel, o.FormatTag, true)
 		}
 	}
 
@@ -316,8 +372,12 @@ type nodePair struct {
 func newTree(prof *profile.Profile, o *Options) (g *Graph) {
 	parentNodeMap := make(map[*Node]NodeMap, len(prof.Sample))
 	for _, sample := range prof.Sample {
-		weight := o.SampleValue(sample.Value)
-		if weight == 0 {
+		var w, dw int64
+		w = o.SampleValue(sample.Value)
+		if o.SampleMeanDivisor != nil {
+			dw = o.SampleMeanDivisor(sample.Value)
+		}
+		if dw == 0 && w == 0 {
 			continue
 		}
 		var parent *Node
@@ -339,15 +399,15 @@ func newTree(prof *profile.Profile, o *Options) (g *Graph) {
 				if n == nil {
 					continue
 				}
-				n.addSample(weight, labels, sample.NumLabel, o.FormatTag, false)
+				n.addSample(dw, w, labels, sample.NumLabel, o.FormatTag, false)
 				if parent != nil {
-					parent.AddToEdge(n, weight, false, lidx != len(lines)-1)
+					parent.AddToEdgeDiv(n, dw, w, false, lidx != len(lines)-1)
 				}
 				parent = n
 			}
 		}
 		if parent != nil {
-			parent.addSample(weight, labels, sample.NumLabel, o.FormatTag, true)
+			parent.addSample(dw, w, labels, sample.NumLabel, o.FormatTag, true)
 		}
 	}
 
@@ -540,21 +600,25 @@ func (ns Nodes) Sum() (flat int64, cum int64) {
 	return
 }
 
-func (n *Node) addSample(value int64, labels string, numLabel map[string][]int64, format func(int64, string) string, flat bool) {
+func (n *Node) addSample(dw, w int64, labels string, numLabel map[string][]int64, format func(int64, string) string, flat bool) {
 	// Update sample value
 	if flat {
-		n.Flat += value
+		n.FlatDiv += dw
+		n.Flat += w
 	} else {
-		n.Cum += value
+		n.CumDiv += dw
+		n.Cum += w
 	}
 
 	// Add string tags
 	if labels != "" {
 		t := n.LabelTags.findOrAddTag(labels, "", 0)
 		if flat {
-			t.Flat += value
+			t.FlatDiv += dw
+			t.Flat += w
 		} else {
-			t.Cum += value
+			t.CumDiv += dw
+			t.Cum += w
 		}
 	}
 
@@ -571,9 +635,11 @@ func (n *Node) addSample(value int64, labels string, numLabel map[string][]int64
 		for _, v := range nvals {
 			t := numericTags.findOrAddTag(format(v, key), key, v)
 			if flat {
-				t.Flat += value
+				t.FlatDiv += dw
+				t.Flat += w
 			} else {
-				t.Cum += value
+				t.CumDiv += dw
+				t.Cum += w
 			}
 		}
 	}

internal/report/report.go

@@ -141,9 +141,9 @@ func (rpt *Report) selectOutputUnit(g *graph.Graph) {
 	var minValue int64
 
 	for _, n := range g.Nodes {
-		nodeMin := abs64(n.Flat)
+		nodeMin := abs64(n.FlatValue())
 		if nodeMin == 0 {
-			nodeMin = abs64(n.Cum)
+			nodeMin = abs64(n.CumValue())
 		}
 		if nodeMin > 0 && (minValue == 0 || nodeMin < minValue) {
 			minValue = nodeMin
@@ -201,11 +201,12 @@ func (rpt *Report) newGraph(nodes graph.NodeSet) *graph.Graph {
 	}
 
 	gopt := &graph.Options{
-		SampleValue:  o.SampleValue,
-		FormatTag:    formatTag,
-		CallTree:     o.CallTree && (o.OutputFormat == Dot || o.OutputFormat == Callgrind),
-		DropNegative: o.DropNegative,
-		KeptNodes:    nodes,
+		SampleValue:       o.SampleValue,
+		SampleMeanDivisor: o.SampleMeanDivisor,
+		FormatTag:         formatTag,
+		CallTree:          o.CallTree && (o.OutputFormat == Dot || o.OutputFormat == Callgrind),
+		DropNegative:      o.DropNegative,
+		KeptNodes:         nodes,
 	}
 
 	// Only keep binary names for disassembly-based reports, otherwise
@@ -240,7 +241,7 @@ func printTopProto(w io.Writer, rpt *Report) error {
 	}
 	var flatSum int64
 	for i, n := range g.Nodes {
-		name, flat, cum := n.Info.PrintableName(), n.Flat, n.Cum
+		name, flat, cum := n.Info.PrintableName(), n.FlatValue(), n.CumValue()
 
 		flatSum += flat
 		f := &profile.Function{
@@ -319,7 +320,7 @@ func printAssembly(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
 			percentage(cumSum, rpt.total))
 
 		for _, n := range ns {
-			fmt.Fprintf(w, "%10s %10s %10x: %s\n", valueOrDot(n.Flat, rpt), valueOrDot(n.Cum, rpt), n.Info.Address, n.Info.Name)
+			fmt.Fprintf(w, "%10s %10s %10x: %s\n", valueOrDot(n.FlatValue(), rpt), valueOrDot(n.CumValue(), rpt), n.Info.Address, n.Info.Name)
 		}
 	}
 	return nil
@@ -442,7 +443,9 @@ func annotateAssembly(insns []plugin.Inst, samples graph.Nodes, base uint64) gra
 		// Sum all the samples until the next instruction (to account
 		// for samples attributed to the middle of an instruction).
 		for next := insns[ix+1].Addr; s < len(samples) && samples[s].Info.Address-base < next; s++ {
+			n.FlatDiv += samples[s].FlatDiv
 			n.Flat += samples[s].Flat
+			n.CumDiv += samples[s].CumDiv
 			n.Cum += samples[s].Cum
 			if samples[s].Info.File != "" {
 				n.Info.File = trimPath(samples[s].Info.File)
@@ -521,10 +524,10 @@ func printTags(w io.Writer, rpt *Report) error {
 		fmt.Fprintf(w, "%s: Total %d\n", key, total)
 		for _, t := range graph.SortTags(tags, true) {
 			if total > 0 {
-				fmt.Fprintf(w, "  %8d (%s): %s\n", t.Flat,
-					percentage(t.Flat, total), t.Name)
+				fmt.Fprintf(w, "  %8d (%s): %s\n", t.FlatValue(),
+					percentage(t.FlatValue(), total), t.Name)
 			} else {
-				fmt.Fprintf(w, "  %8d: %s\n", t.Flat, t.Name)
+				fmt.Fprintf(w, "  %8d: %s\n", t.FlatValue(), t.Name)
 			}
 		}
 		fmt.Fprintln(w)
@@ -544,7 +547,7 @@ func printText(w io.Writer, rpt *Report) error {
 
 	var flatSum int64
 	for _, n := range g.Nodes {
-		name, flat, cum := n.Info.PrintableName(), n.Flat, n.Cum
+		name, flat, cum := n.Info.PrintableName(), n.FlatValue(), n.CumValue()
 
 		var inline, noinline bool
 		for _, e := range n.In {
@@ -604,11 +607,17 @@ func printTraces(w io.Writer, rpt *Report) error {
 		}
 		sort.Strings(labels)
 		fmt.Fprint(w, strings.Join(labels, ""))
+		var d, v int64
+		v = o.SampleValue(sample.Value)
+		if o.SampleMeanDivisor != nil {
+			d = o.SampleMeanDivisor(sample.Value)
+		}
 		// Print call stack.
+		if d != 0 {
+			v = v / d
+		}
 		fmt.Fprintf(w, "%10s   %s\n",
-			rpt.formatValue(o.SampleValue(sample.Value)),
-			stack[0].Info.PrintableName())
-
+			rpt.formatValue(v), stack[0].Info.PrintableName())
 		for _, s := range stack[1:] {
 			fmt.Fprintf(w, "%10s   %s\n", "", s.Info.PrintableName())
 		}
@@ -648,7 +657,7 @@ func printCallgrind(w io.Writer, rpt *Report) error {
 		}
 
 		addr := callgrindAddress(prevInfo, n.Info.Address)
-		sv, _ := measurement.Scale(n.Flat, o.SampleUnit, o.OutputUnit)
+		sv, _ := measurement.Scale(n.FlatValue(), o.SampleUnit, o.OutputUnit)
 		fmt.Fprintf(w, "%s %d %d\n", addr, n.Info.Lineno, int64(sv))
 
 		// Print outgoing edges.
@@ -772,7 +781,7 @@ func printTree(w io.Writer, rpt *Report) error {
 
 	rx := rpt.options.Symbol
 	for _, n := range g.Nodes {
-		name, flat, cum := n.Info.PrintableName(), n.Flat, n.Cum
+		name, flat, cum := n.Info.PrintableName(), n.FlatValue(), n.CumValue()
 
 		// Skip any entries that do not match the regexp (for the "peek" command).
 		if rx != nil && !rx.MatchString(name) {
@@ -902,7 +911,7 @@ func reportLabels(rpt *Report, g *graph.Graph, origCount, droppedNodes, droppedE
 
 	var flatSum int64
 	for _, n := range g.Nodes {
-		flatSum = flatSum + n.Flat
+		flatSum = flatSum + n.FlatValue()
 	}
 
 	label = append(label, fmt.Sprintf("Showing nodes accounting for %s, %s of %s total", rpt.formatValue(flatSum), strings.TrimSpace(percentage(flatSum, rpt.total)), rpt.formatValue(rpt.total)))
@@ -965,9 +974,10 @@ type Options struct {
 	NodeFraction float64
 	EdgeFraction float64
 
-	SampleValue func(s []int64) int64
-	SampleType  string
-	SampleUnit  string // Unit for the sample data from the profile.
+	SampleValue       func(s []int64) int64
+	SampleMeanDivisor func(s []int64) int64
+	SampleType        string
+	SampleUnit        string // Unit for the sample data from the profile.
 
 	OutputUnit string // Units for data formatting in report.
 
@@ -985,7 +995,7 @@ func New(prof *profile.Profile, o *Options) *Report {
 		}
 		return measurement.ScaledLabel(v, o.SampleUnit, o.OutputUnit)
 	}
-	return &Report{prof, computeTotal(prof, o.SampleValue, !o.PositivePercentages),
+	return &Report{prof, computeTotal(prof, o.SampleValue, o.SampleMeanDivisor, !o.PositivePercentages),
 		o, format}
 }
 
@@ -1010,15 +1020,25 @@ func NewDefault(prof *profile.Profile, options Options) *Report {
 // absolute values to provide a meaningful percentage for both
 // negative and positive values. Otherwise only use positive values,
 // which is useful when comparing profiles from different jobs.
-func computeTotal(prof *profile.Profile, value func(v []int64) int64, includeNegative bool) int64 {
-	var ret int64
+func computeTotal(prof *profile.Profile, value, meanDiv func(v []int64) int64, includeNegative bool) int64 {
+	var div, ret int64
 	for _, sample := range prof.Sample {
-		if v := value(sample.Value); v > 0 {
+		var d, v int64
+		v = value(sample.Value)
+		if meanDiv != nil {
+			d = meanDiv(sample.Value)
+		}
+		if v >= 0 {
 			ret += v
+			div += d
 		} else if includeNegative {
 			ret -= v
+			div += d
 		}
 	}
+	if div != 0 {
+		return ret / div
+	}
 	return ret
 }