Merge pull request kubernetes#66623 from deads2k/gc-01-graph

Automatic merge from submit-queue (batch tested with PRs 66623, 66718). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

expose GC graph via debug handler

Many times when debugging GC problems, it's important to understand the state of the GC graph at a given point in time.  This pull adds the ability to dump that graph in DOT format for later consumption.  It does this by exposing an additional debug handler and allowing any controller init function to produce such a handler that is included under debug.

Sample full output
```
curl http://localhost:10252/debug/controllers/garbagecollector/graph
digraph full {
  // Node definitions.
  0 [
    label="uid=8581a030-9043-11e8-ad4a-54e1ad486dd3
namespace=kube-system
Pod.v1/kube-dns-7b479ccbc6-qz468
"
    group=""
    version="v1"
    kind="Pod"
    namespace="kube-system"
    name="kube-dns-7b479ccbc6-qz468"
    uid="8581a030-9043-11e8-ad4a-54e1ad486dd3"
    missing="false"
    beingDeleted="false"
    deletingDependents="false"
    virtual="false"
  ];
  1 [
    label="uid=822052fc-9043-11e8-ad4a-54e1ad486dd3
namespace=kube-system
Deployment.v1.apps/kube-dns
"
    group="apps"
    version="v1"
    kind="Deployment"
    namespace="kube-system"
    name="kube-dns"
    uid="822052fc-9043-11e8-ad4a-54e1ad486dd3"
    missing="false"
    beingDeleted="false"
    deletingDependents="false"
    virtual="false"
  ];
  2 [
    label="uid=857bd8ac-9043-11e8-ad4a-54e1ad486dd3
namespace=kube-system
ReplicaSet.v1.apps/kube-dns-7b479ccbc6
"
    group="apps"
    version="v1"
    kind="ReplicaSet"
    namespace="kube-system"
    name="kube-dns-7b479ccbc6"
    uid="857bd8ac-9043-11e8-ad4a-54e1ad486dd3"
    missing="false"
    beingDeleted="false"
    deletingDependents="false"
    virtual="false"
  ];

  // Edge definitions.
  0 -> 2;
  2 -> 1;
}
```

You can also select via UID and have all transitive dependencies output:
```
curl http://localhost:10252/debug/controllers/garbagecollector/graph?uid=8581a030-9043-11e8-ad4a-54e1ad486dd3
digraph full {
  // Node definitions.
  0 [
    label="uid=822052fc-9043-11e8-ad4a-54e1ad486dd3
namespace=kube-system
Deployment.v1.apps/kube-dns
"
    group="apps"
    version="v1"
    kind="Deployment"
    namespace="kube-system"
    name="kube-dns"
    uid="822052fc-9043-11e8-ad4a-54e1ad486dd3"
    missing="false"
    beingDeleted="false"
    deletingDependents="false"
    virtual="false"
  ];
  1 [
    label="uid=8581a030-9043-11e8-ad4a-54e1ad486dd3
namespace=kube-system
Pod.v1/kube-dns-7b479ccbc6-qz468
"
    group=""
    version="v1"
    kind="Pod"
    namespace="kube-system"
    name="kube-dns-7b479ccbc6-qz468"
    uid="8581a030-9043-11e8-ad4a-54e1ad486dd3"
    missing="false"
    beingDeleted="false"
    deletingDependents="false"
    virtual="false"
  ];
  2 [
    label="uid=857bd8ac-9043-11e8-ad4a-54e1ad486dd3
namespace=kube-system
ReplicaSet.v1.apps/kube-dns-7b479ccbc6
"
    group="apps"
    version="v1"
    kind="ReplicaSet"
    namespace="kube-system"
    name="kube-dns-7b479ccbc6"
    uid="857bd8ac-9043-11e8-ad4a-54e1ad486dd3"
    missing="false"
    beingDeleted="false"
    deletingDependents="false"
    virtual="false"
  ];

  // Edge definitions.
  1 -> 2;
  2 -> 0;
}
```

And with some sample rendering:
```
curl http://localhost:10252/debug/controllers/garbagecollector/graph  | dot -T svg -o project.svg
```

produces

![gc](https://user-images.githubusercontent.com/8225098/43223895-8e33c126-9022-11e8-8ad9-6b2f986fd974.png)


@kubernetes/sig-api-machinery-pr-reviews 
/assign @caesarxuchao @liggitt 

```release-note
NONE
```

cmd/cloud-controller-manager/app/controllermanager.go

@@ -110,15 +110,15 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error {
 	// Start the controller manager HTTP server
 	stopCh := make(chan struct{})
 	if c.SecureServing != nil {
-		handler := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging)
-		handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Authorization, &c.Authentication)
+		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging)
+		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
 		if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
 			return err
 		}
 	}
 	if c.InsecureServing != nil {
-		handler := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging)
-		handler = genericcontrollermanager.BuildHandlerChain(handler, &c.Authorization, &c.Authentication)
+		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging)
+		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
 		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
 			return err
 		}

cmd/controller-manager/app/serve.go

@@ -48,7 +48,7 @@ func BuildHandlerChain(apiHandler http.Handler, authorizationInfo *apiserver.Aut
 }
 
 // NewBaseHandler takes in CompletedConfig and returns a handler.
-func NewBaseHandler(c *componentconfig.DebuggingConfiguration) http.Handler {
+func NewBaseHandler(c *componentconfig.DebuggingConfiguration) *mux.PathRecorderMux {
 	mux := mux.NewPathRecorderMux("controller-manager")
 	healthz.InstallHandler(mux)
 	if c.EnableProfiling {

cmd/kube-controller-manager/app/BUILD

@@ -111,6 +111,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/discovery/cached:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",

cmd/kube-controller-manager/app/apps.go

@@ -23,16 +23,18 @@ package app
 import (
 	"fmt"
 
+	"net/http"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kubernetes/pkg/controller/daemon"
 	"k8s.io/kubernetes/pkg/controller/deployment"
 	"k8s.io/kubernetes/pkg/controller/replicaset"
 	"k8s.io/kubernetes/pkg/controller/statefulset"
 )
 
-func startDaemonSetController(ctx ControllerContext) (bool, error) {
+func startDaemonSetController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}] {
-		return false, nil
+		return nil, false, nil
 	}
 	dsc, err := daemon.NewDaemonSetsController(
 		ctx.InformerFactory.Apps().V1().DaemonSets(),
@@ -42,15 +44,15 @@ func startDaemonSetController(ctx ControllerContext) (bool, error) {
 		ctx.ClientBuilder.ClientOrDie("daemon-set-controller"),
 	)
 	if err != nil {
-		return true, fmt.Errorf("error creating DaemonSets controller: %v", err)
+		return nil, true, fmt.Errorf("error creating DaemonSets controller: %v", err)
 	}
 	go dsc.Run(int(ctx.ComponentConfig.DaemonSetController.ConcurrentDaemonSetSyncs), ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }
 
-func startStatefulSetController(ctx ControllerContext) (bool, error) {
+func startStatefulSetController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"}] {
-		return false, nil
+		return nil, false, nil
 	}
 	go statefulset.NewStatefulSetController(
 		ctx.InformerFactory.Core().V1().Pods(),
@@ -59,25 +61,25 @@ func startStatefulSetController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Apps().V1().ControllerRevisions(),
 		ctx.ClientBuilder.ClientOrDie("statefulset-controller"),
 	).Run(1, ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }
 
-func startReplicaSetController(ctx ControllerContext) (bool, error) {
+func startReplicaSetController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"}] {
-		return false, nil
+		return nil, false, nil
 	}
 	go replicaset.NewReplicaSetController(
 		ctx.InformerFactory.Apps().V1().ReplicaSets(),
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.ClientBuilder.ClientOrDie("replicaset-controller"),
 		replicaset.BurstReplicas,
 	).Run(int(ctx.ComponentConfig.ReplicaSetController.ConcurrentRSSyncs), ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }
 
-func startDeploymentController(ctx ControllerContext) (bool, error) {
+func startDeploymentController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}] {
-		return false, nil
+		return nil, false, nil
 	}
 	dc, err := deployment.NewDeploymentController(
 		ctx.InformerFactory.Apps().V1().Deployments(),
@@ -86,8 +88,8 @@ func startDeploymentController(ctx ControllerContext) (bool, error) {
 		ctx.ClientBuilder.ClientOrDie("deployment-controller"),
 	)
 	if err != nil {
-		return true, fmt.Errorf("error creating Deployment controller: %v", err)
+		return nil, true, fmt.Errorf("error creating Deployment controller: %v", err)
 	}
 	go dc.Run(int(ctx.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs), ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }

cmd/kube-controller-manager/app/autoscaling.go

@@ -21,6 +21,8 @@ limitations under the License.
 package app
 
 import (
+	"net/http"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/scale"
@@ -31,9 +33,9 @@ import (
 	"k8s.io/metrics/pkg/client/external_metrics"
 )
 
-func startHPAController(ctx ControllerContext) (bool, error) {
+func startHPAController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "autoscaling", Version: "v1", Resource: "horizontalpodautoscalers"}] {
-		return false, nil
+		return nil, false, nil
 	}
 
 	if ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients {
@@ -44,7 +46,7 @@ func startHPAController(ctx ControllerContext) (bool, error) {
 	return startHPAControllerWithLegacyClient(ctx)
 }
 
-func startHPAControllerWithRESTClient(ctx ControllerContext) (bool, error) {
+func startHPAControllerWithRESTClient(ctx ControllerContext) (http.Handler, bool, error) {
 	clientConfig := ctx.ClientBuilder.ConfigOrDie("horizontal-pod-autoscaler")
 	metricsClient := metrics.NewRESTMetricsClient(
 		resourceclient.NewForConfigOrDie(clientConfig),
@@ -54,7 +56,7 @@ func startHPAControllerWithRESTClient(ctx ControllerContext) (bool, error) {
 	return startHPAControllerWithMetricsClient(ctx, metricsClient)
 }
 
-func startHPAControllerWithLegacyClient(ctx ControllerContext) (bool, error) {
+func startHPAControllerWithLegacyClient(ctx ControllerContext) (http.Handler, bool, error) {
 	hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler")
 	metricsClient := metrics.NewHeapsterMetricsClient(
 		hpaClient,
@@ -66,7 +68,7 @@ func startHPAControllerWithLegacyClient(ctx ControllerContext) (bool, error) {
 	return startHPAControllerWithMetricsClient(ctx, metricsClient)
 }
 
-func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient metrics.MetricsClient) (bool, error) {
+func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient metrics.MetricsClient) (http.Handler, bool, error) {
 	hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler")
 	hpaClientConfig := ctx.ClientBuilder.ConfigOrDie("horizontal-pod-autoscaler")
 
@@ -75,7 +77,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 	scaleKindResolver := scale.NewDiscoveryScaleKindResolver(hpaClient.Discovery())
 	scaleClient, err := scale.NewForConfig(hpaClientConfig, ctx.RESTMapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver)
 	if err != nil {
-		return false, err
+		return nil, false, err
 	}
 
 	replicaCalc := podautoscaler.NewReplicaCalculator(
@@ -94,5 +96,5 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 		ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
 		ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
 	).Run(ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }

cmd/kube-controller-manager/app/batch.go

@@ -23,33 +23,35 @@ package app
 import (
 	"fmt"
 
+	"net/http"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kubernetes/pkg/controller/cronjob"
 	"k8s.io/kubernetes/pkg/controller/job"
 )
 
-func startJobController(ctx ControllerContext) (bool, error) {
+func startJobController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"}] {
-		return false, nil
+		return nil, false, nil
 	}
 	go job.NewJobController(
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.InformerFactory.Batch().V1().Jobs(),
 		ctx.ClientBuilder.ClientOrDie("job-controller"),
 	).Run(int(ctx.ComponentConfig.JobController.ConcurrentJobSyncs), ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }
 
-func startCronJobController(ctx ControllerContext) (bool, error) {
+func startCronJobController(ctx ControllerContext) (http.Handler, bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "batch", Version: "v1beta1", Resource: "cronjobs"}] {
-		return false, nil
+		return nil, false, nil
 	}
 	cjc, err := cronjob.NewCronJobController(
 		ctx.ClientBuilder.ClientOrDie("cronjob-controller"),
 	)
 	if err != nil {
-		return true, fmt.Errorf("error creating CronJob controller: %v", err)
+		return nil, true, fmt.Errorf("error creating CronJob controller: %v", err)
 	}
 	go cjc.Run(ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }

cmd/kube-controller-manager/app/bootstrap.go

@@ -19,32 +19,34 @@ package app
 import (
 	"fmt"
 
+	"net/http"
+
 	"k8s.io/kubernetes/pkg/controller/bootstrap"
 )
 
-func startBootstrapSignerController(ctx ControllerContext) (bool, error) {
+func startBootstrapSignerController(ctx ControllerContext) (http.Handler, bool, error) {
 	bsc, err := bootstrap.NewBootstrapSigner(
 		ctx.ClientBuilder.ClientOrDie("bootstrap-signer"),
 		ctx.InformerFactory.Core().V1().Secrets(),
 		ctx.InformerFactory.Core().V1().ConfigMaps(),
 		bootstrap.DefaultBootstrapSignerOptions(),
 	)
 	if err != nil {
-		return true, fmt.Errorf("error creating BootstrapSigner controller: %v", err)
+		return nil, true, fmt.Errorf("error creating BootstrapSigner controller: %v", err)
 	}
 	go bsc.Run(ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }
 
-func startTokenCleanerController(ctx ControllerContext) (bool, error) {
+func startTokenCleanerController(ctx ControllerContext) (http.Handler, bool, error) {
 	tcc, err := bootstrap.NewTokenCleaner(
 		ctx.ClientBuilder.ClientOrDie("token-cleaner"),
 		ctx.InformerFactory.Core().V1().Secrets(),
 		bootstrap.DefaultTokenCleanerOptions(),
 	)
 	if err != nil {
-		return true, fmt.Errorf("error creating TokenCleaner controller: %v", err)
+		return nil, true, fmt.Errorf("error creating TokenCleaner controller: %v", err)
 	}
 	go tcc.Run(ctx.Stop)
-	return true, nil
+	return nil, true, nil
 }