Add mechanism to reduce profile size

Add new trimproto option that generates a new profile while removing
symbol information for functions below nodefraction. This reduces the
profile sizes significantly.

internal/driver/commands.go

@@ -93,6 +93,7 @@ var pprofCommands = commands{
 	// Save binary formats to a file
 	"callgrind": {report.Callgrind, awayFromTTY("callgraph.out"), false, "Outputs a graph in callgrind format", reportHelp("callgrind", false, true)},
 	"proto":     {report.Proto, awayFromTTY("pb.gz"), false, "Outputs the profile in compressed protobuf format", ""},
+	"trimproto": {report.TrimProto, awayFromTTY("pb.gz"), false, "Outputs the profile in compressed protobuf format, removing info about infrequent functions to reduce profile size", ""},
 
 	// Generate report in DOT format and postprocess with dot
 	"gif": {report.Dot, invokeDot("gif"), false, "Outputs a graph image in GIF format", reportHelp("gif", false, true)},

internal/graph/graph.go

@@ -204,11 +204,18 @@ func SortTags(t []*Tag, flat bool) []*Tag {
 }
 
 // New summarizes performance data from a profile into a graph.
-func New(prof *profile.Profile, o *Options) (g *Graph) {
+func New(prof *profile.Profile, o *Options) *Graph {
 	if o.CallTree {
 		return newTree(prof, o)
 	}
+	g, _ := newGraph(prof, o)
+	return g
+}
 
+// newGraph computes a graph from a profile. It returns the graph, and
+// a map from the profile location indices to the corresponding graph
+// nodes.
+func newGraph(prof *profile.Profile, o *Options) (*Graph, map[uint64]Nodes) {
 	nodes, locationMap := CreateNodes(prof, o.ObjNames, o.KeptNodes)
 	for _, sample := range prof.Sample {
 		weight := o.SampleValue(sample.Value)
@@ -252,7 +259,7 @@ func New(prof *profile.Profile, o *Options) (g *Graph) {
 		}
 	}
 
-	return selectNodesForGraph(nodes, o.DropNegative)
+	return selectNodesForGraph(nodes, o.DropNegative), locationMap
 }
 
 func selectNodesForGraph(nodes Nodes, dropNegative bool) *Graph {
@@ -340,6 +347,31 @@ func joinLabels(s *profile.Sample) string {
 	return strings.Join(labels, `\n`)
 }
 
+// TrimProfile reduces the size of a profile by removing information
+// about locations that contribute to infrequent graph nodes,
+// determined by the value of nodefraction. The locations are
+// preserved, but their line information is removed.
+func TrimProfile(p *profile.Profile, o *Options, nodeFraction float64) *profile.Profile {
+	g, locationMap := newGraph(p, o)
+
+	totalValue, _ := g.Nodes.Sum()
+	cutoff := abs64(int64(float64(totalValue) * nodeFraction))
+
+	for _, l := range p.Location {
+		nodes := locationMap[l.ID]
+		if len(nodes) == 0 || len(l.Line) != len(nodes) {
+			continue
+		}
+		for i, n := range nodes {
+			if n.Cum < cutoff {
+				l.Line[i] = profile.Line{}
+			}
+		}
+	}
+
+	return p.Compact()
+}
+
 // isNegative returns true if the node is considered as "negative" for the
 // purposes of drop_negative.
 func isNegative(n *Node) bool {
@@ -391,29 +423,41 @@ func (nm NodeMap) findOrInsertLocation(l *profile.Location, keepBinary bool, kep
 		}
 		return Nodes{nm.FindOrInsertNode(ni, kept)}
 	}
-	var locNodes Nodes
-	for _, line := range l.Line {
-		ni := NodeInfo{
-			Address: l.Address,
-			Lineno:  int(line.Line),
+	locNodes := make(Nodes, len(l.Line))
+	for li := range l.Line {
+		if ni := nodeInfo(l, li, objfile, keepBinary); ni != nil {
+			locNodes[li] = nm.FindOrInsertNode(*ni, kept)
 		}
+	}
+	return locNodes
+}
 
-		if line.Function != nil {
-			ni.Name = line.Function.Name
-			ni.OrigName = line.Function.SystemName
-			if fname := line.Function.Filename; fname != "" {
-				ni.File = filepath.Clean(fname)
-			}
-			if keepBinary {
-				ni.StartLine = int(line.Function.StartLine)
-			}
-		}
-		if keepBinary || line.Function == nil {
-			ni.Objfile = objfile
+func nodeInfo(l *profile.Location, li int, objfile string, keepBinary bool) *NodeInfo {
+	if !keepBinary {
+		objfile = ""
+	}
+	line := l.Line[li]
+	if line.Function == nil {
+		if l.Address == 0 {
+			return nil
 		}
-		locNodes = append(locNodes, nm.FindOrInsertNode(ni, kept))
+		return &NodeInfo{Address: l.Address, Objfile: objfile}
 	}
-	return locNodes
+
+	ni := &NodeInfo{
+		Address:  l.Address,
+		Lineno:   int(line.Line),
+		Name:     line.Function.Name,
+		OrigName: line.Function.SystemName,
+		Objfile:  objfile,
+	}
+	if fname := line.Function.Filename; fname != "" {
+		ni.File = filepath.Clean(fname)
+	}
+	if keepBinary {
+		ni.StartLine = int(line.Function.StartLine)
+	}
+	return ni
 }
 
 type tags struct {

internal/report/report.go

@@ -54,6 +54,8 @@ func Generate(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
 		return printTags(w, rpt)
 	case Proto:
 		return rpt.prof.Write(w)
+	case TrimProto:
+		return printTrimmedProto(w, rpt)
 	case TopProto:
 		return printTopProto(w, rpt)
 	case Dis:
@@ -184,7 +186,10 @@ func (rpt *Report) newGraph(nodes graph.NodeSet) *graph.Graph {
 		}
 		s.NumLabel = numLabels
 	}
+	return graph.New(rpt.prof, graphOptions(o, nodes))
+}
 
+func graphOptions(o *Options, nodes graph.NodeSet) *graph.Options {
 	gopt := &graph.Options{
 		SampleValue:  o.SampleValue,
 		FormatTag:    formatTag,
@@ -200,7 +205,7 @@ func (rpt *Report) newGraph(nodes graph.NodeSet) *graph.Graph {
 		gopt.ObjNames = true
 	}
 
-	return graph.New(rpt.prof, gopt)
+	return gopt
 }
 
 func formatTag(v int64, key string) string {
@@ -256,6 +261,17 @@ func printTopProto(w io.Writer, rpt *Report) error {
 	return out.Write(w)
 }
 
+// printTrimmedProto writes a profile in a serialize profile.proto,
+// removing symbol information from infrequent nodes to reduce the
+// size of the profile.
+func printTrimmedProto(w io.Writer, rpt *Report) error {
+	o := rpt.options
+	prof := rpt.prof
+	prof = graph.TrimProfile(prof, graphOptions(o, nil), o.NodeFraction)
+
+	return prof.Write(w)
+}
+
 // printAssembly prints an annotated assembly listing.
 func printAssembly(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
 	o := rpt.options
@@ -844,6 +860,7 @@ const (
 	WebList
 	Callgrind
 	TopProto
+	TrimProto
 )
 
 // Options are the formatting and filtering options used to generate a

profile/merge.go

@@ -22,6 +22,14 @@ import (
 	"strings"
 )
 
+// Compact performs garbage collection on a profile to remove any
+// unreferenced fields. This is useful to reduce the size of a profile
+// after samples or locations have been removed.
+func (p *Profile) Compact() *Profile {
+	p, _ = Merge([]*Profile{p})
+	return p
+}
+
 // Merge merges all the profiles in profs into a single Profile.
 // Returns a new profile independent of the input profiles. The merged
 // profile is compacted to eliminate unused samples, locations,
@@ -220,7 +228,9 @@ func (l *Location) key() locationKey {
 	}
 	lines := make([]string, len(l.Line)*2)
 	for i, line := range l.Line {
-		lines[i*2] = strconv.FormatUint(line.Function.ID, 16)
+		if line.Function != nil {
+			lines[i*2] = strconv.FormatUint(line.Function.ID, 16)
+		}
 		lines[i*2+1] = strconv.FormatInt(line.Line, 16)
 	}
 	key.lines = strings.Join(lines, "|")