Allow exporting via signed URL

gen/proto/swagger/parca/query/v1alpha1/query.swagger.json

@@ -299,7 +299,7 @@
           },
           {
             "name": "reportType",
-            "description": "report_type is the type of report to return\n\n - REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels",
+            "description": "report_type is the type of report to return\n\n - REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels\n - REPORT_TYPE_EXPORT: REPORT_TYPE_EXPORT contains the profile as a downloadable file",
             "in": "query",
             "required": false,
             "type": "string",
@@ -312,7 +312,8 @@
               "REPORT_TYPE_FLAMEGRAPH_ARROW",
               "REPORT_TYPE_SOURCE",
               "REPORT_TYPE_TABLE_ARROW",
-              "REPORT_TYPE_PROFILE_METADATA"
+              "REPORT_TYPE_PROFILE_METADATA",
+              "REPORT_TYPE_EXPORT"
             ],
             "default": "REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED"
           },
@@ -390,6 +391,18 @@
             "in": "query",
             "required": false,
             "type": "boolean"
+          },
+          {
+            "name": "exportFormat",
+            "description": "the format to respond the data in for the export\n\n - FORMAT_UNSPECIFIED: FORMAT_UNSPECIFIED unspecified\n - FORMAT_PPROF: DOWNLOAD_FORMAT_PPROF pprof format",
+            "in": "query",
+            "required": false,
+            "type": "string",
+            "enum": [
+              "FORMAT_UNSPECIFIED",
+              "FORMAT_PPROF"
+            ],
+            "default": "FORMAT_UNSPECIFIED"
           }
         ],
         "tags": [
@@ -581,10 +594,11 @@
         "REPORT_TYPE_FLAMEGRAPH_ARROW",
         "REPORT_TYPE_SOURCE",
         "REPORT_TYPE_TABLE_ARROW",
-        "REPORT_TYPE_PROFILE_METADATA"
+        "REPORT_TYPE_PROFILE_METADATA",
+        "REPORT_TYPE_EXPORT"
       ],
       "default": "REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED",
-      "description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels",
+      "description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels\n - REPORT_TYPE_EXPORT: REPORT_TYPE_EXPORT contains the profile as a downloadable file",
       "title": "ReportType is the type of report to return"
     },
     "metastorev1alpha1Function": {
@@ -813,6 +827,10 @@
             "$ref": "#/definitions/v1alpha1Filter"
           },
           "title": "a set of filter to apply to the query request"
+        },
+        "exportFormat": {
+          "$ref": "#/definitions/v1alpha1Format",
+          "title": "the format to respond the data in for the export"
         }
       },
       "title": "QueryRequest is a request for a profile query"
@@ -853,6 +871,10 @@
           "$ref": "#/definitions/v1alpha1ProfileMetadata",
           "title": "profile_metadata contains metadata about the profile i.e. binaries, labels"
         },
+        "export": {
+          "$ref": "#/definitions/v1alpha1Export",
+          "description": "export is the response for an export request."
+        },
         "total": {
           "type": "string",
           "format": "int64",
@@ -1068,6 +1090,24 @@
       },
       "title": "DiffProfile contains parameters for a profile diff request"
     },
+    "v1alpha1Export": {
+      "type": "object",
+      "properties": {
+        "format": {
+          "$ref": "#/definitions/v1alpha1Format",
+          "description": "The format of the download."
+        },
+        "inline": {
+          "$ref": "#/definitions/v1alpha1InlineExport",
+          "description": "InlineExport is the response for an export request where actual content\nis inlined in the response."
+        },
+        "url": {
+          "$ref": "#/definitions/v1alpha1URLExport",
+          "description": "URL is the response for an export request where actual content\nis to be downloaded from the provided URL."
+        }
+      },
+      "description": "Export is the response for an export request."
+    },
     "v1alpha1Filter": {
       "type": "object",
       "properties": {
@@ -1256,6 +1296,16 @@
       },
       "title": "FlamegraphRootNode is a root node of a flame graph"
     },
+    "v1alpha1Format": {
+      "type": "string",
+      "enum": [
+        "FORMAT_UNSPECIFIED",
+        "FORMAT_PPROF"
+      ],
+      "default": "FORMAT_UNSPECIFIED",
+      "description": "- FORMAT_UNSPECIFIED: FORMAT_UNSPECIFIED unspecified\n - FORMAT_PPROF: DOWNLOAD_FORMAT_PPROF pprof format",
+      "title": "format is the format to offer the data in for the download"
+    },
     "v1alpha1FrameFilter": {
       "type": "object",
       "properties": {
@@ -1289,6 +1339,17 @@
       },
       "title": "GroupBy encapsulates the repeated fields to group by"
     },
+    "v1alpha1InlineExport": {
+      "type": "object",
+      "properties": {
+        "content": {
+          "type": "string",
+          "format": "byte",
+          "description": "Content of the export."
+        }
+      },
+      "description": "InlineExport is the response for an export request where actual content is\ninlined in the response."
+    },
     "v1alpha1LabelSet": {
       "type": "object",
       "properties": {
@@ -1677,6 +1738,16 @@
       },
       "title": "TopNodeMeta is the metadata for a given node"
     },
+    "v1alpha1URLExport": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "description": "The URL to download the content from."
+        }
+      },
+      "description": "URLExport is the response for an export request where actual content is\nto be downloaded from the provided URL."
+    },
     "v1alpha1ValuesResponse": {
       "type": "object",
       "properties": {

gen/proto/swagger/parca/share/v1alpha1/share.swagger.json

@@ -28,10 +28,11 @@
         "REPORT_TYPE_FLAMEGRAPH_ARROW",
         "REPORT_TYPE_SOURCE",
         "REPORT_TYPE_TABLE_ARROW",
-        "REPORT_TYPE_PROFILE_METADATA"
+        "REPORT_TYPE_PROFILE_METADATA",
+        "REPORT_TYPE_EXPORT"
       ],
       "default": "REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED",
-      "description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels",
+      "description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels\n - REPORT_TYPE_EXPORT: REPORT_TYPE_EXPORT contains the profile as a downloadable file",
       "title": "ReportType is the type of report to return"
     },
     "metastorev1alpha1Function": {

parca.yaml

@@ -9,11 +9,11 @@ object_storage:
 # Scraping tends to produce a lot more data than Parca Agent CPU which
 # results in memory usage by the server.
 #
-# scrape_configs:
-#   - job_name: "parca"
-#     scrape_interval: "10s"
-#     static_configs:
-#       - targets: [ '127.0.0.1:7070' ]
+scrape_configs:
+  - job_name: "parca"
+    scrape_interval: "10s"
+    static_configs:
+      - targets: [ '127.0.0.1:7070' ]
 
 # Nested under the job config:
 #

pkg/parca/parca.go

@@ -110,6 +110,8 @@ type Flags struct {
 
 	Storage FlagsStorage `embed:"" prefix:"storage-"`
 
+	ObjectStorageExport bool `default:"false" help:"Export profiles to object storage first before serving them via a signed URL."`
+
 	Symbolizer FlagsSymbolizer `embed:"" prefix:"symbolizer-"`
 
 	Debuginfo  FlagsDebuginfo  `embed:"" prefix:"debuginfo-"`
@@ -424,6 +426,9 @@ func Run(ctx context.Context, logger log.Logger, reg *prometheus.Registry, flags
 			debuginfoBucket,
 			debuginfodClients,
 		),
+		flags.ObjectStorageExport,
+		signedRequestsClient,
+		bucket,
 	)
 
 	t := telemetryservice.NewTelemetry(

pkg/query/columnquery.go

@@ -18,6 +18,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -27,6 +30,7 @@ import (
 	"github.com/apache/arrow/go/v16/arrow/bitutil"
 	"github.com/apache/arrow/go/v16/arrow/math"
 	"github.com/apache/arrow/go/v16/arrow/memory"
+	"github.com/cespare/xxhash/v2"
 	"github.com/go-kit/log"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -62,6 +66,14 @@ type SourceFinder interface {
 	SourceExists(ctx context.Context, ref *pb.SourceReference) (bool, error)
 }
 
+type SignedGetClient interface {
+	SignedGET(ctx context.Context, objectKey string, expiry time.Time) (string, error)
+}
+
+type ObjectStoreUploadClient interface {
+	Upload(ctx context.Context, objectKey string, r io.Reader) error
+}
+
 // ColumnQueryAPI is the read api interface for parca
 // It implements the proto/query/query.proto APIServer interface.
 type ColumnQueryAPI struct {
@@ -77,6 +89,13 @@ type ColumnQueryAPI struct {
 	converter          *parcacol.ArrowToProfileConverter
 
 	sourceFinder SourceFinder
+
+	// Whether to first upload an export to object storage and returning a
+	// signed URL to the client instead of serving the export directly within
+	// the API response.
+	objectStorgeExport bool
+	signedGetClient    SignedGetClient
+	objectStore        ObjectStoreUploadClient
 }
 
 func NewColumnQueryAPI(
@@ -87,6 +106,9 @@ func NewColumnQueryAPI(
 	mem memory.Allocator,
 	converter *parcacol.ArrowToProfileConverter,
 	sourceFinder SourceFinder,
+	objectStorageExport bool,
+	signedGetClient SignedGetClient,
+	objectStore ObjectStoreUploadClient,
 ) *ColumnQueryAPI {
 	return &ColumnQueryAPI{
 		logger:             logger,
@@ -97,6 +119,9 @@ func NewColumnQueryAPI(
 		mem:                mem,
 		converter:          converter,
 		sourceFinder:       sourceFinder,
+		objectStorgeExport: objectStorageExport,
+		signedGetClient:    signedGetClient,
+		objectStore:        objectStore,
 	}
 }
 
@@ -320,16 +345,24 @@ func (q *ColumnQueryAPI) Query(ctx context.Context, req *pb.QueryRequest) (*pb.Q
 		return nil, fmt.Errorf("filtering profile: %w", err)
 	}
 
+	data, err := req.MarshalVT()
+	if err != nil {
+		return nil, fmt.Errorf("marshaling request: %w", err)
+	}
+	requestKey := xxhash.Sum64(data)
+
 	return q.renderReport(
 		ctx,
 		p,
+		strconv.FormatUint(requestKey, 16),
 		req.GetReportType(),
 		req.GetNodeTrimThreshold(),
 		filtered,
 		groupBy,
 		req.GetSourceReference(),
 		source,
 		isDiff,
+		req.GetExportFormat(),
 	)
 }
 
@@ -474,17 +507,20 @@ func filterRecord(
 func (q *ColumnQueryAPI) renderReport(
 	ctx context.Context,
 	p profile.Profile,
+	requestKey string,
 	typ pb.QueryRequest_ReportType,
 	nodeTrimThreshold float32,
 	filtered int64,
 	groupBy []string,
 	sourceReference *pb.SourceReference,
 	source string,
 	isDiff bool,
+	exportFormat pb.Format,
 ) (*pb.QueryResponse, error) {
 	return RenderReport(
 		ctx,
 		q.tracer,
+		requestKey,
 		p,
 		typ,
 		nodeTrimThreshold,
@@ -496,12 +532,17 @@ func (q *ColumnQueryAPI) renderReport(
 		sourceReference,
 		source,
 		isDiff,
+		exportFormat,
+		q.objectStorgeExport,
+		q.signedGetClient,
+		q.objectStore,
 	)
 }
 
 func RenderReport(
 	ctx context.Context,
 	tracer trace.Tracer,
+	requestKey string,
 	p profile.Profile,
 	typ pb.QueryRequest_ReportType,
 	nodeTrimThreshold float32,
@@ -513,6 +554,10 @@ func RenderReport(
 	sourceReference *pb.SourceReference,
 	source string,
 	isDiff bool,
+	exportFormat pb.Format,
+	objectStorageExport bool,
+	signedGetClient SignedGetClient,
+	objectStore ObjectStoreUploadClient,
 ) (*pb.QueryResponse, error) {
 	ctx, span := tracer.Start(ctx, "renderReport")
 	span.SetAttributes(attribute.String("reportType", typ.String()))
@@ -609,6 +654,53 @@ func RenderReport(
 			Filtered: filtered,
 			Report:   &pb.QueryResponse_Pprof{Pprof: buf},
 		}, nil
+	case pb.QueryRequest_REPORT_TYPE_EXPORT:
+		var buf []byte
+
+		switch exportFormat {
+		case pb.Format_FORMAT_PPROF:
+			pp, err := GenerateFlatPprof(ctx, isDiff, p)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to generate pprof: %v", err.Error())
+			}
+
+			buf, err = SerializePprof(pp)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to generate pprof: %v", err.Error())
+			}
+		default:
+			return nil, status.Errorf(codes.InvalidArgument, "requested export format does not exist")
+		}
+
+		if objectStorageExport {
+			key := path.Join(requestKey, "pprof")
+			if err := objectStore.Upload(ctx, key, bytes.NewReader(buf)); err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to upload export to object storage: %v", err.Error())
+			}
+
+			signedURL, err := signedGetClient.SignedGET(ctx, key, time.Now().Add(time.Hour))
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to generate signed URL: %v", err.Error())
+			}
+
+			return &pb.QueryResponse{
+				Report: &pb.QueryResponse_Export{
+					Export: &pb.Export{
+						Format:  exportFormat,
+						Content: &pb.Export_Url{Url: &pb.URLExport{Url: signedURL}},
+					},
+				},
+			}, nil
+		}
+
+		return &pb.QueryResponse{
+			Report: &pb.QueryResponse_Export{
+				Export: &pb.Export{
+					Format:  exportFormat,
+					Content: &pb.Export_Inline{Inline: &pb.InlineExport{Content: buf}},
+				},
+			},
+		}, nil
 	case pb.QueryRequest_REPORT_TYPE_TOP:
 		op, err := converter.Convert(ctx, p)
 		if err != nil {