support https with certificate (#423)

Support https with server certificate.

doc/README.md

@@ -256,9 +256,10 @@ total of the absolute value of all samples when aggregated at the address level.
 
 # Fetching profiles
 
-pprof can read profiles from a file or directly from a URL over http. Its native
-format is a gzipped profile.proto file, but it can also accept some legacy
-formats generated by [gperftools](https://github.com/gperftools/gperftools).
+pprof can read profiles from a file or directly from a URL over http or https.
+Its native format is a gzipped profile.proto file, but it can
+also accept some legacy formats generated by 
+[gperftools](https://github.com/gperftools/gperftools).
 
 When fetching from a URL handler, pprof accepts options to indicate how much to
 wait for the profile.
@@ -270,6 +271,22 @@ wait for the profile.
   profile over http. If not specified, pprof will use heuristics to determine a
   reasonable timeout.
 
+pprof also accepts options which allow a user to specify TLS certificates to
+use when fetching or symbolizing a profile from a protected endpoint. For more
+information about generating these certificates, see
+https://docs.docker.com/engine/security/https/.
+
+* **-tls\_cert= _/path/to/cert_:** File containing the TLS client certificate
+  to be used when fetching and symbolizing profiles.
+* **-tls\_key= _/path/to/key_:** File containing the TLS private key to be used
+  when fetching and symbolizing profiles.
+* **-tls\_ca= _/path/to/ca_:** File containing the certificate authority to be
+  used when fetching and symbolizing profiles.
+
+pprof also supports skipping verification of the server's certificate chain and
+host name when collecting or symbolizing a profile. To skip this verification, 
+use "https+insecure" in place of "https" in the URL.
+
 If multiple profiles are specified, pprof will fetch them all and merge
 them. This is useful to combine profiles from multiple processes of a
 distributed job. The profiles may be from different programs but must be

driver/driver.go

@@ -17,6 +17,7 @@ package driver
 
 import (
 	"io"
+	"net/http"
 	"regexp"
 	"time"
 
@@ -48,13 +49,14 @@ func (o *Options) internalOptions() *plugin.Options {
 		}
 	}
 	return &plugin.Options{
-		Writer:     o.Writer,
-		Flagset:    o.Flagset,
-		Fetch:      o.Fetch,
-		Sym:        sym,
-		Obj:        obj,
-		UI:         o.UI,
-		HTTPServer: httpServer,
+		Writer:        o.Writer,
+		Flagset:       o.Flagset,
+		Fetch:         o.Fetch,
+		Sym:           sym,
+		Obj:           obj,
+		UI:            o.UI,
+		HTTPServer:    httpServer,
+		HTTPTransport: o.HTTPTransport,
 	}
 }
 
@@ -64,13 +66,14 @@ type HTTPServerArgs plugin.HTTPServerArgs
 
 // Options groups all the optional plugins into pprof.
 type Options struct {
-	Writer     Writer
-	Flagset    FlagSet
-	Fetch      Fetcher
-	Sym        Symbolizer
-	Obj        ObjTool
-	UI         UI
-	HTTPServer func(*HTTPServerArgs) error
+	Writer        Writer
+	Flagset       FlagSet
+	Fetch         Fetcher
+	Sym           Symbolizer
+	Obj           ObjTool
+	UI            UI
+	HTTPServer    func(*HTTPServerArgs) error
+	HTTPTransport http.RoundTripper
 }
 
 // Writer provides a mechanism to write data under a certain name,
@@ -100,12 +103,16 @@ type FlagSet interface {
 	// single flag
 	StringList(name string, def string, usage string) *[]*string
 
-	// ExtraUsage returns any additional text that should be
-	// printed after the standard usage message.
-	// The typical use of ExtraUsage is to show any custom flags
-	// defined by the specific pprof plugins being used.
+	// ExtraUsage returns any additional text that should be printed after the
+	// standard usage message. The extra usage message returned includes all text
+	// added with AddExtraUsage().
+	// The typical use of ExtraUsage is to show any custom flags defined by the
+	// specific pprof plugins being used.
 	ExtraUsage() string
 
+	// AddExtraUsage appends additional text to the end of the extra usage message.
+	AddExtraUsage(eu string)
+
 	// Parse initializes the flags with their values for this run
 	// and returns the non-flag command line arguments.
 	// If an unknown flag is encountered or there are no arguments,

internal/driver/driver_test.go

@@ -130,8 +130,9 @@ func TestParse(t *testing.T) {
 			}
 
 			// First pprof invocation to save the profile into a profile.proto.
-			o1 := setDefaults(nil)
-			o1.Flagset = f
+			// Pass in flag set hen setting defaults, because otherwise default
+			// transport will try to add flags to the default flag set.
+			o1 := setDefaults(&plugin.Options{Flagset: f})
 			o1.Fetch = testFetcher{}
 			o1.Sym = testSymbolizer{}
 			o1.UI = testUI
@@ -167,8 +168,9 @@ func TestParse(t *testing.T) {
 
 			// Second pprof invocation to read the profile from profile.proto
 			// and generate a report.
-			o2 := setDefaults(nil)
-			o2.Flagset = f
+			// Pass in flag set hen setting defaults, because otherwise default
+			// transport will try to add flags to the default flag set.
+			o2 := setDefaults(&plugin.Options{Flagset: f})
 			o2.Sym = testSymbolizeDemangler{}
 			o2.Obj = new(mockObjTool)
 			o2.UI = testUI
@@ -297,6 +299,8 @@ type testFlags struct {
 
 func (testFlags) ExtraUsage() string { return "" }
 
+func (testFlags) AddExtraUsage(eu string) {}
+
 func (f testFlags) Bool(s string, d bool, c string) *bool {
 	if b, ok := f.bools[s]; ok {
 		return &b

internal/driver/fetch.go

@@ -16,7 +16,6 @@ package driver
 
 import (
 	"bytes"
-	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -57,7 +56,7 @@ func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) {
 		})
 	}
 
-	p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI)
+	p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI, o.HTTPTransport)
 	if err != nil {
 		return nil, err
 	}
@@ -123,7 +122,7 @@ func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) {
 	return p, nil
 }
 
-func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) {
+func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	var psrc, pbase *profile.Profile
@@ -133,11 +132,11 @@ func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, o
 	var countsrc, countbase int
 	go func() {
 		defer wg.Done()
-		psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui)
+		psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui, tr)
 	}()
 	go func() {
 		defer wg.Done()
-		pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui)
+		pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui, tr)
 	}()
 	wg.Wait()
 	save := savesrc || savebase
@@ -167,7 +166,7 @@ func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, o
 // chunkedGrab fetches the profiles described in source and merges them into
 // a single profile. It fetches a chunk of profiles concurrently, with a maximum
 // chunk size to limit its memory usage.
-func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, plugin.MappingSources, bool, int, error) {
+func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
 	const chunkSize = 64
 
 	var p *profile.Profile
@@ -180,7 +179,7 @@ func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTo
 		if end > len(sources) {
 			end = len(sources)
 		}
-		chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui)
+		chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui, tr)
 		switch {
 		case chunkErr != nil:
 			return nil, nil, false, 0, chunkErr
@@ -204,13 +203,13 @@ func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTo
 }
 
 // concurrentGrab fetches multiple profiles concurrently
-func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, plugin.MappingSources, bool, int, error) {
+func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
 	wg := sync.WaitGroup{}
 	wg.Add(len(sources))
 	for i := range sources {
 		go func(s *profileSource) {
 			defer wg.Done()
-			s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui)
+			s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui, tr)
 		}(&sources[i])
 	}
 	wg.Wait()
@@ -310,7 +309,7 @@ const testSourceAddress = "pproftest.local"
 // grabProfile fetches a profile. Returns the profile, sources for the
 // profile mappings, a bool indicating if the profile was fetched
 // remotely, and an error.
-func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) {
+func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) {
 	var src string
 	duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second
 	if fetcher != nil {
@@ -321,7 +320,7 @@ func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.Ob
 	}
 	if err != nil || p == nil {
 		// Fetch the profile over HTTP or from a file.
-		p, src, err = fetch(source, duration, timeout, ui)
+		p, src, err = fetch(source, duration, timeout, ui, tr)
 		if err != nil {
 			return
 		}
@@ -461,7 +460,7 @@ mapping:
 // fetch fetches a profile from source, within the timeout specified,
 // producing messages through the ui. It returns the profile and the
 // url of the actual source of the profile for remote profiles.
-func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *profile.Profile, src string, err error) {
+func fetch(source string, duration, timeout time.Duration, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, src string, err error) {
 	var f io.ReadCloser
 
 	if sourceURL, timeout := adjustURL(source, duration, timeout); sourceURL != "" {
@@ -469,7 +468,7 @@ func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *pro
 		if duration > 0 {
 			ui.Print(fmt.Sprintf("Please wait... (%v)", duration))
 		}
-		f, err = fetchURL(sourceURL, timeout)
+		f, err = fetchURL(sourceURL, timeout, tr)
 		src = sourceURL
 	} else if isPerfFile(source) {
 		f, err = convertPerfData(source, ui)
@@ -484,8 +483,12 @@ func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *pro
 }
 
 // fetchURL fetches a profile from a URL using HTTP.
-func fetchURL(source string, timeout time.Duration) (io.ReadCloser, error) {
-	resp, err := httpGet(source, timeout)
+func fetchURL(source string, timeout time.Duration, tr http.RoundTripper) (io.ReadCloser, error) {
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   timeout + 5*time.Second,
+	}
+	resp, err := client.Get(source)
 	if err != nil {
 		return nil, fmt.Errorf("http fetch: %v", err)
 	}
@@ -582,30 +585,3 @@ func adjustURL(source string, duration, timeout time.Duration) (string, time.Dur
 	u.RawQuery = values.Encode()
 	return u.String(), timeout
 }
-
-// httpGet is a wrapper around http.Get; it is defined as a variable
-// so it can be redefined during for testing.
-var httpGet = func(source string, timeout time.Duration) (*http.Response, error) {
-	url, err := url.Parse(source)
-	if err != nil {
-		return nil, err
-	}
-
-	var tlsConfig *tls.Config
-	if url.Scheme == "https+insecure" {
-		tlsConfig = &tls.Config{
-			InsecureSkipVerify: true,
-		}
-		url.Scheme = "https"
-		source = url.String()
-	}
-
-	client := &http.Client{
-		Transport: &http.Transport{
-			Proxy:                 http.ProxyFromEnvironment,
-			TLSClientConfig:       tlsConfig,
-			ResponseHeaderTimeout: timeout + 5*time.Second,
-		},
-	}
-	return client.Get(source)
-}

internal/driver/fetch_test.go

@@ -24,8 +24,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/big"
+	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -39,6 +39,7 @@ import (
 	"github.com/google/pprof/internal/plugin"
 	"github.com/google/pprof/internal/proftest"
 	"github.com/google/pprof/internal/symbolizer"
+	"github.com/google/pprof/internal/transport"
 	"github.com/google/pprof/profile"
 )
 
@@ -173,12 +174,6 @@ func (testFile) Close() error                                                 {
 
 func TestFetch(t *testing.T) {
 	const path = "testdata/"
-
-	// Intercept http.Get calls from HTTPFetcher.
-	savedHTTPGet := httpGet
-	defer func() { httpGet = savedHTTPGet }()
-	httpGet = stubHTTPGet
-
 	type testcase struct {
 		source, execName string
 	}
@@ -188,7 +183,7 @@ func TestFetch(t *testing.T) {
 		{path + "go.nomappings.crash", "/bin/gotest.exe"},
 		{"http://localhost/profile?file=cppbench.cpu", ""},
 	} {
-		p, _, _, err := grabProfile(&source{ExecName: tc.execName}, tc.source, nil, testObj{}, &proftest.TestUI{T: t})
+		p, _, _, err := grabProfile(&source{ExecName: tc.execName}, tc.source, nil, testObj{}, &proftest.TestUI{T: t}, &httpTransport{})
 		if err != nil {
 			t.Fatalf("%s: %s", tc.source, err)
 		}
@@ -449,8 +444,9 @@ func TestFetchWithBase(t *testing.T) {
 			f.args = tc.sources
 
 			o := setDefaults(&plugin.Options{
-				UI:      &proftest.TestUI{T: t, AllowRx: "Local symbolization failed|Some binary filenames not available"},
-				Flagset: f,
+				UI:            &proftest.TestUI{T: t, AllowRx: "Local symbolization failed|Some binary filenames not available"},
+				Flagset:       f,
+				HTTPTransport: transport.New(nil),
 			})
 			src, _, err := parseFlags(o)
 
@@ -503,19 +499,14 @@ func mappingSources(key, source string, start uint64) plugin.MappingSources {
 	}
 }
 
-// stubHTTPGet intercepts a call to http.Get and rewrites it to use
-// "file://" to get the profile directly from a file.
-func stubHTTPGet(source string, _ time.Duration) (*http.Response, error) {
-	url, err := url.Parse(source)
-	if err != nil {
-		return nil, err
-	}
+type httpTransport struct{}
 
-	values := url.Query()
+func (tr *httpTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	values := req.URL.Query()
 	file := values.Get("file")
 
 	if file == "" {
-		return nil, fmt.Errorf("want .../file?profile, got %s", source)
+		return nil, fmt.Errorf("want .../file?profile, got %s", req.URL.String())
 	}
 
 	t := &http.Transport{}
@@ -532,7 +523,7 @@ func closedError() string {
 	return "use of closed"
 }
 
-func TestHttpsInsecure(t *testing.T) {
+func TestHTTPSInsecure(t *testing.T) {
 	if runtime.GOOS == "nacl" || runtime.GOOS == "js" {
 		t.Skip("test assumes tcp available")
 	}
@@ -553,7 +544,8 @@ func TestHttpsInsecure(t *testing.T) {
 	pprofVariables = baseVars.makeCopy()
 	defer func() { pprofVariables = baseVars }()
 
-	tlsConfig := &tls.Config{Certificates: []tls.Certificate{selfSignedCert(t)}}
+	tlsCert, _, _ := selfSignedCert(t, "")
+	tlsConfig := &tls.Config{Certificates: []tls.Certificate{tlsCert}}
 
 	l, err := tls.Listen("tcp", "localhost:0", tlsConfig)
 	if err != nil {
@@ -586,8 +578,9 @@ func TestHttpsInsecure(t *testing.T) {
 		Symbolize: "remote",
 	}
 	o := &plugin.Options{
-		Obj: &binutils.Binutils{},
-		UI:  &proftest.TestUI{T: t, AllowRx: "Saved profile in"},
+		Obj:           &binutils.Binutils{},
+		UI:            &proftest.TestUI{T: t, AllowRx: "Saved profile in"},
+		HTTPTransport: transport.New(nil),
 	}
 	o.Sym = &symbolizer.Symbolizer{Obj: o.Obj, UI: o.UI}
 	p, err := fetchProfiles(s, o)
@@ -600,7 +593,122 @@ func TestHttpsInsecure(t *testing.T) {
 	if len(p.Function) == 0 {
 		t.Fatalf("fetchProfiles(%s) got non-symbolized profile: len(p.Function)==0", address)
 	}
-	if err := checkProfileHasFunction(p, "TestHttpsInsecure"); err != nil {
+	if err := checkProfileHasFunction(p, "TestHTTPSInsecure"); err != nil {
+		t.Fatalf("fetchProfiles(%s) %v", address, err)
+	}
+}
+
+func TestHTTPSWithServerCertFetch(t *testing.T) {
+	if runtime.GOOS == "nacl" || runtime.GOOS == "js" {
+		t.Skip("test assumes tcp available")
+	}
+	saveHome := os.Getenv(homeEnv())
+	tempdir, err := ioutil.TempDir("", "home")
+	if err != nil {
+		t.Fatal("creating temp dir: ", err)
+	}
+	defer os.RemoveAll(tempdir)
+
+	// pprof writes to $HOME/pprof by default which is not necessarily
+	// writeable (e.g. on a Debian buildd) so set $HOME to something we
+	// know we can write to for the duration of the test.
+	os.Setenv(homeEnv(), tempdir)
+	defer os.Setenv(homeEnv(), saveHome)
+
+	baseVars := pprofVariables
+	pprofVariables = baseVars.makeCopy()
+	defer func() { pprofVariables = baseVars }()
+
+	cert, certBytes, keyBytes := selfSignedCert(t, "localhost")
+	cas := x509.NewCertPool()
+	cas.AppendCertsFromPEM(certBytes)
+
+	tlsConfig := &tls.Config{
+		RootCAs:      cas,
+		Certificates: []tls.Certificate{cert},
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		ClientCAs:    cas,
+	}
+
+	l, err := tls.Listen("tcp", "localhost:0", tlsConfig)
+	if err != nil {
+		t.Fatalf("net.Listen: got error %v, want no error", err)
+	}
+
+	donec := make(chan error, 1)
+	go func(donec chan<- error) {
+		donec <- http.Serve(l, nil)
+	}(donec)
+	defer func() {
+		if got, want := <-donec, closedError(); !strings.Contains(got.Error(), want) {
+			t.Fatalf("Serve got error %v, want %q", got, want)
+		}
+	}()
+	defer l.Close()
+
+	outputTempFile, err := ioutil.TempFile("", "profile_output")
+	if err != nil {
+		t.Fatalf("Failed to create tempfile: %v", err)
+	}
+	defer os.Remove(outputTempFile.Name())
+	defer outputTempFile.Close()
+
+	// Get port from the address, so request to the server can be made using
+	// the host name specified in certificates.
+	_, portStr, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		t.Fatalf("cannot get port from URL: %v", err)
+	}
+	address := "https://" + "localhost:" + portStr + "/debug/pprof/goroutine"
+	s := &source{
+		Sources:   []string{address},
+		Seconds:   10,
+		Timeout:   10,
+		Symbolize: "remote",
+	}
+
+	certTempFile, err := ioutil.TempFile("", "cert_output")
+	if err != nil {
+		t.Errorf("cannot create cert tempfile: %v", err)
+	}
+	defer os.Remove(certTempFile.Name())
+	defer certTempFile.Close()
+	certTempFile.Write(certBytes)
+
+	keyTempFile, err := ioutil.TempFile("", "key_output")
+	if err != nil {
+		t.Errorf("cannot create key tempfile: %v", err)
+	}
+	defer os.Remove(keyTempFile.Name())
+	defer keyTempFile.Close()
+	keyTempFile.Write(keyBytes)
+
+	f := &testFlags{
+		strings: map[string]string{
+			"tls_cert": certTempFile.Name(),
+			"tls_key":  keyTempFile.Name(),
+			"tls_ca":   certTempFile.Name(),
+		},
+	}
+	o := &plugin.Options{
+		Obj:           &binutils.Binutils{},
+		UI:            &proftest.TestUI{T: t, AllowRx: "Saved profile in"},
+		Flagset:       f,
+		HTTPTransport: transport.New(f),
+	}
+
+	o.Sym = &symbolizer.Symbolizer{Obj: o.Obj, UI: o.UI, Transport: o.HTTPTransport}
+	p, err := fetchProfiles(s, o)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p.SampleType) == 0 {
+		t.Fatalf("fetchProfiles(%s) got empty profile: len(p.SampleType)==0", address)
+	}
+	if len(p.Function) == 0 {
+		t.Fatalf("fetchProfiles(%s) got non-symbolized profile: len(p.Function)==0", address)
+	}
+	if err := checkProfileHasFunction(p, "TestHTTPSWithServerCertFetch"); err != nil {
 		t.Fatalf("fetchProfiles(%s) %v", address, err)
 	}
 }
@@ -614,7 +722,10 @@ func checkProfileHasFunction(p *profile.Profile, fname string) error {
 	return fmt.Errorf("got %s, want function %q", p.String(), fname)
 }
 
-func selfSignedCert(t *testing.T) tls.Certificate {
+// selfSignedCert generates a self-signed certificate, and returns the
+// generated certificate, and byte arrays containing the certificate and
+// key associated with the certificate.
+func selfSignedCert(t *testing.T, host string) (tls.Certificate, []byte, []byte) {
 	privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
 		t.Fatalf("failed to generate private key: %v", err)
@@ -629,6 +740,8 @@ func selfSignedCert(t *testing.T) tls.Certificate {
 		SerialNumber: big.NewInt(1),
 		NotBefore:    time.Now(),
 		NotAfter:     time.Now().Add(10 * time.Minute),
+		IsCA:         true,
+		DNSNames:     []string{host},
 	}
 
 	b, err = x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, privKey.Public(), privKey)
@@ -641,5 +754,5 @@ func selfSignedCert(t *testing.T) tls.Certificate {
 	if err != nil {
 		t.Fatalf("failed to create TLS key pair: %v", err)
 	}
-	return cert
+	return cert, bc, bk
 }

internal/driver/flags.go

@@ -0,0 +1,78 @@
+package driver
+
+import (
+	"flag"
+	"strings"
+)
+
+// GoFlags implements the plugin.FlagSet interface.
+type GoFlags struct {
+	UsageMsgs []string
+}
+
+// Bool implements the plugin.FlagSet interface.
+func (*GoFlags) Bool(o string, d bool, c string) *bool {
+	return flag.Bool(o, d, c)
+}
+
+// Int implements the plugin.FlagSet interface.
+func (*GoFlags) Int(o string, d int, c string) *int {
+	return flag.Int(o, d, c)
+}
+
+// Float64 implements the plugin.FlagSet interface.
+func (*GoFlags) Float64(o string, d float64, c string) *float64 {
+	return flag.Float64(o, d, c)
+}
+
+// String implements the plugin.FlagSet interface.
+func (*GoFlags) String(o, d, c string) *string {
+	return flag.String(o, d, c)
+}
+
+// BoolVar implements the plugin.FlagSet interface.
+func (*GoFlags) BoolVar(b *bool, o string, d bool, c string) {
+	flag.BoolVar(b, o, d, c)
+}
+
+// IntVar implements the plugin.FlagSet interface.
+func (*GoFlags) IntVar(i *int, o string, d int, c string) {
+	flag.IntVar(i, o, d, c)
+}
+
+// Float64Var implements the plugin.FlagSet interface.
+// the value of the flag.
+func (*GoFlags) Float64Var(f *float64, o string, d float64, c string) {
+	flag.Float64Var(f, o, d, c)
+}
+
+// StringVar implements the plugin.FlagSet interface.
+func (*GoFlags) StringVar(s *string, o, d, c string) {
+	flag.StringVar(s, o, d, c)
+}
+
+// StringList implements the plugin.FlagSet interface.
+func (*GoFlags) StringList(o, d, c string) *[]*string {
+	return &[]*string{flag.String(o, d, c)}
+}
+
+// ExtraUsage implements the plugin.FlagSet interface.
+func (f *GoFlags) ExtraUsage() string {
+	return strings.Join(f.UsageMsgs, "\n")
+}
+
+// AddExtraUsage implements the plugin.FlagSet interface.
+func (f *GoFlags) AddExtraUsage(eu string) {
+	f.UsageMsgs = append(f.UsageMsgs, eu)
+}
+
+// Parse implements the plugin.FlagSet interface.
+func (*GoFlags) Parse(usage func()) []string {
+	flag.Usage = usage
+	flag.Parse()
+	args := flag.Args()
+	if len(args) == 0 {
+		usage()
+	}
+	return args
+}

internal/driver/interactive_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/google/pprof/internal/plugin"
 	"github.com/google/pprof/internal/proftest"
 	"github.com/google/pprof/internal/report"
+	"github.com/google/pprof/internal/transport"
 	"github.com/google/pprof/profile"
 )
 
@@ -41,7 +42,10 @@ func TestShell(t *testing.T) {
 
 	// Random interleave of independent scripts
 	pprofVariables = testVariables(savedVariables)
-	o := setDefaults(nil)
+
+	// pass in HTTPTransport when setting defaults, because otherwise default
+	// transport will try to add flags to the default flag set.
+	o := setDefaults(&plugin.Options{HTTPTransport: transport.New(nil)})
 	o.UI = newUI(t, interleave(script, 0))
 	if err := interactive(p, o); err != nil {
 		t.Error("first attempt:", err)

internal/driver/options.go

@@ -16,7 +16,6 @@ package driver
 
 import (
 	"bufio"
-	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -25,6 +24,7 @@ import (
 	"github.com/google/pprof/internal/binutils"
 	"github.com/google/pprof/internal/plugin"
 	"github.com/google/pprof/internal/symbolizer"
+	"github.com/google/pprof/internal/transport"
 )
 
 // setDefaults returns a new plugin.Options with zero fields sets to
@@ -38,7 +38,7 @@ func setDefaults(o *plugin.Options) *plugin.Options {
 		d.Writer = oswriter{}
 	}
 	if d.Flagset == nil {
-		d.Flagset = goFlags{}
+		d.Flagset = &GoFlags{}
 	}
 	if d.Obj == nil {
 		d.Obj = &binutils.Binutils{}
@@ -46,67 +46,15 @@ func setDefaults(o *plugin.Options) *plugin.Options {
 	if d.UI == nil {
 		d.UI = &stdUI{r: bufio.NewReader(os.Stdin)}
 	}
+	if d.HTTPTransport == nil {
+		d.HTTPTransport = transport.New(d.Flagset)
+	}
 	if d.Sym == nil {
-		d.Sym = &symbolizer.Symbolizer{Obj: d.Obj, UI: d.UI}
+		d.Sym = &symbolizer.Symbolizer{Obj: d.Obj, UI: d.UI, Transport: d.HTTPTransport}
 	}
 	return d
 }
 
-// goFlags returns a flagset implementation based on the standard flag
-// package from the Go distribution. It implements the plugin.FlagSet
-// interface.
-type goFlags struct{}
-
-func (goFlags) Bool(o string, d bool, c string) *bool {
-	return flag.Bool(o, d, c)
-}
-
-func (goFlags) Int(o string, d int, c string) *int {
-	return flag.Int(o, d, c)
-}
-
-func (goFlags) Float64(o string, d float64, c string) *float64 {
-	return flag.Float64(o, d, c)
-}
-
-func (goFlags) String(o, d, c string) *string {
-	return flag.String(o, d, c)
-}
-
-func (goFlags) BoolVar(b *bool, o string, d bool, c string) {
-	flag.BoolVar(b, o, d, c)
-}
-
-func (goFlags) IntVar(i *int, o string, d int, c string) {
-	flag.IntVar(i, o, d, c)
-}
-
-func (goFlags) Float64Var(f *float64, o string, d float64, c string) {
-	flag.Float64Var(f, o, d, c)
-}
-
-func (goFlags) StringVar(s *string, o, d, c string) {
-	flag.StringVar(s, o, d, c)
-}
-
-func (goFlags) StringList(o, d, c string) *[]*string {
-	return &[]*string{flag.String(o, d, c)}
-}
-
-func (goFlags) ExtraUsage() string {
-	return ""
-}
-
-func (goFlags) Parse(usage func()) []string {
-	flag.Usage = usage
-	flag.Parse()
-	args := flag.Args()
-	if len(args) == 0 {
-		usage()
-	}
-	return args
-}
-
 type stdUI struct {
 	r *bufio.Reader
 }

internal/plugin/plugin.go

@@ -41,7 +41,8 @@ type Options struct {
 	//
 	// A common use for a custom HTTPServer is to provide custom
 	// authentication checks.
-	HTTPServer func(args *HTTPServerArgs) error
+	HTTPServer    func(args *HTTPServerArgs) error
+	HTTPTransport http.RoundTripper
 }
 
 // Writer provides a mechanism to write data under a certain name,
@@ -71,12 +72,16 @@ type FlagSet interface {
 	// single flag
 	StringList(name string, def string, usage string) *[]*string
 
-	// ExtraUsage returns any additional text that should be
-	// printed after the standard usage message.
-	// The typical use of ExtraUsage is to show any custom flags
-	// defined by the specific pprof plugins being used.
+	// ExtraUsage returns any additional text that should be printed after the
+	// standard usage message. The extra usage message returned includes all text
+	// added with AddExtraUsage().
+	// The typical use of ExtraUsage is to show any custom flags defined by the
+	// specific pprof plugins being used.
 	ExtraUsage() string
 
+	// AddExtraUsage appends additional text to the end of the extra usage message.
+	AddExtraUsage(eu string)
+
 	// Parse initializes the flags with their values for this run
 	// and returns the non-flag command line arguments.
 	// If an unknown flag is encountered or there are no arguments,

internal/symbolizer/symbolizer.go

@@ -18,7 +18,6 @@
 package symbolizer
 
 import (
-	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -35,8 +34,9 @@ import (
 
 // Symbolizer implements the plugin.Symbolize interface.
 type Symbolizer struct {
-	Obj plugin.ObjTool
-	UI  plugin.UI
+	Obj       plugin.ObjTool
+	UI        plugin.UI
+	Transport http.RoundTripper
 }
 
 // test taps for dependency injection
@@ -85,7 +85,10 @@ func (s *Symbolizer) Symbolize(mode string, sources plugin.MappingSources, p *pr
 		}
 	}
 	if remote {
-		if err = symbolzSymbolize(p, force, sources, postURL, s.UI); err != nil {
+		post := func(source, post string) ([]byte, error) {
+			return postURL(source, post, s.Transport)
+		}
+		if err = symbolzSymbolize(p, force, sources, post, s.UI); err != nil {
 			return err // Ran out of options.
 		}
 	}
@@ -95,25 +98,9 @@ func (s *Symbolizer) Symbolize(mode string, sources plugin.MappingSources, p *pr
 }
 
 // postURL issues a POST to a URL over HTTP.
-func postURL(source, post string) ([]byte, error) {
-	url, err := url.Parse(source)
-	if err != nil {
-		return nil, err
-	}
-
-	var tlsConfig *tls.Config
-	if url.Scheme == "https+insecure" {
-		tlsConfig = &tls.Config{
-			InsecureSkipVerify: true,
-		}
-		url.Scheme = "https"
-		source = url.String()
-	}
-
+func postURL(source, post string, tr http.RoundTripper) ([]byte, error) {
 	client := &http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: tlsConfig,
-		},
+		Transport: tr,
 	}
 	resp, err := client.Post(source, "application/octet-stream", strings.NewReader(post))
 	if err != nil {

internal/symbolizer/symbolizer_test.go

@@ -114,8 +114,8 @@ func TestSymbolization(t *testing.T) {
 	}
 
 	s := Symbolizer{
-		mockObjTool{},
-		&proftest.TestUI{T: t},
+		Obj: mockObjTool{},
+		UI:  &proftest.TestUI{T: t},
 	}
 	for i, tc := range []testcase{
 		{

internal/transport/transport.go

@@ -0,0 +1,131 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package transport provides a mechanism to send requests with https cert,
+// key, and CA.
+package transport
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"sync"
+
+	"github.com/google/pprof/internal/plugin"
+)
+
+type transport struct {
+	cert       *string
+	key        *string
+	ca         *string
+	caCertPool *x509.CertPool
+	certs      []tls.Certificate
+	initOnce   sync.Once
+	initErr    error
+}
+
+const extraUsage = `    -tls_cert             TLS client certificate file for fetching profile and symbols
+    -tls_key              TLS private key file for fetching profile and symbols
+    -tls_ca               TLS CA certs file for fetching profile and symbols`
+
+// New returns a round tripper for making requests with the
+// specified cert, key, and ca. The flags tls_cert, tls_key, and tls_ca are
+// added to the flagset to allow a user to specify the cert, key, and ca. If
+// the flagset is nil, no flags will be added, and users will not be able to
+// use these flags.
+func New(flagset plugin.FlagSet) http.RoundTripper {
+	if flagset == nil {
+		return &transport{}
+	}
+	flagset.AddExtraUsage(extraUsage)
+	return &transport{
+		cert: flagset.String("tls_cert", "", "TLS client certificate file for fetching profile and symbols"),
+		key:  flagset.String("tls_key", "", "TLS private key file for fetching profile and symbols"),
+		ca:   flagset.String("tls_ca", "", "TLS CA certs file for fetching profile and symbols"),
+	}
+}
+
+// initialize uses the cert, key, and ca to initialize the certs
+// to use these when making requests.
+func (tr *transport) initialize() error {
+	var cert, key, ca string
+	if tr.cert != nil {
+		cert = *tr.cert
+	}
+	if tr.key != nil {
+		key = *tr.key
+	}
+	if tr.ca != nil {
+		ca = *tr.ca
+	}
+
+	if cert != "" && key != "" {
+		tlsCert, err := tls.LoadX509KeyPair(cert, key)
+		if err != nil {
+			return fmt.Errorf("could not load certificate/key pair specified by -tls_cert and -tls_key: %v", err)
+		}
+		tr.certs = []tls.Certificate{tlsCert}
+	} else if cert == "" && key != "" {
+		return fmt.Errorf("-tls_key is specified, so -tls_cert must also be specified")
+	} else if cert != "" && key == "" {
+		return fmt.Errorf("-tls_cert is specified, so -tls_key must also be specified")
+	}
+
+	if ca != "" {
+		caCertPool := x509.NewCertPool()
+		caCert, err := ioutil.ReadFile(ca)
+		if err != nil {
+			return fmt.Errorf("could not load CA specified by -tls_ca: %v", err)
+		}
+		caCertPool.AppendCertsFromPEM(caCert)
+		tr.caCertPool = caCertPool
+	}
+
+	return nil
+}
+
+// RoundTrip executes a single HTTP transaction, returning
+// a Response for the provided Request.
+func (tr *transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	tr.initOnce.Do(func() {
+		tr.initErr = tr.initialize()
+	})
+	if tr.initErr != nil {
+		return nil, tr.initErr
+	}
+
+	tlsConfig := &tls.Config{
+		RootCAs:      tr.caCertPool,
+		Certificates: tr.certs,
+	}
+
+	if req.URL.Scheme == "https+insecure" {
+		// Make shallow copy of request, and req.URL, so the request's URL can be
+		// modified.
+		r := *req
+		*r.URL = *req.URL
+		req = &r
+		tlsConfig.InsecureSkipVerify = true
+		req.URL.Scheme = "https"
+	}
+
+	transport := http.Transport{
+		Proxy:           http.ProxyFromEnvironment,
+		TLSClientConfig: tlsConfig,
+	}
+
+	return transport.RoundTrip(req)
+}