Merge pull request #670 from enj/enj/f/impersonator_always_authz

impersonator: always authorize every request

Dockerfile

@@ -3,7 +3,7 @@
 # Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM golang:1.16.4 as build-env
+FROM golang:1.16.5 as build-env
 
 WORKDIR /work
 COPY . .

ROADMAP.md

@@ -33,14 +33,13 @@ The following table includes the current roadmap for Pinniped. If you have any q
 
 
 
-Last Updated: April 2021
+Last Updated: June 2021
 Theme|Description|Timeline|
 |--|--|--|
-|LDAP Support|Extends upstream IDP protocols|May 2021|
-|Improved Documentation|Reorganizing and improving Pinniped docs; new how-to guides and tutorials|May 2021|
-|Device Code Flow|Add support for OAuth 2.0 Device Authorization Grant in the Pinniped CLI and Supervisor|Jun 2021|
+|Remote OIDC login support|Add support for logging in from remote hosts without web browsers in the Pinniped CLI and Supervisor|Jun 2021|
 |AD Support|Extends upstream IDP protocols|Jun 2021|
 |Wider Concierge cluster support|Support for more cluster types in the Concierge|Jul 2021|
+|Improved Documentation|Reorganizing and improving Pinniped docs; new how-to guides and tutorials|Exploring/Ongoing|
 |Improving Security Posture|Offer the best security posture for Kubernetes cluster authentication|Exploring/Ongoing|
 |Improve our CI/CD systems|Upgrade tests; make Kind more efficient and reliable for CI ; Windows tests; performance tests; scale tests; soak tests|Exploring/Ongoing|
 |CLI Improvements|Improving CLI UX for setting up Supervisor IDPs|Exploring/Ongoing|

deploy/concierge/deployment.yaml

@@ -29,6 +29,18 @@ metadata:
   labels: #@ labels()
 ---
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+  namespace: #@ namespace()
+  labels: #@ labels()
+  annotations:
+    #! we need to create this service account before we create the secret
+    kapp.k14s.io/change-group: "impersonation-proxy.concierge.pinniped.dev/serviceaccount"
+secrets: #! make sure the token controller does not create any other secrets
+- name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: #@ defaultResourceNameWithSuffix("config")
@@ -134,6 +146,8 @@ spec:
               mountPath: /etc/config
             - name: podinfo
               mountPath: /etc/podinfo
+            - name: impersonation-proxy
+              mountPath: /var/run/secrets/impersonation-proxy.concierge.pinniped.dev/serviceaccount
           livenessProbe:
             httpGet:
               path: /healthz
@@ -156,6 +170,12 @@ spec:
         - name: config-volume
           configMap:
             name: #@ defaultResourceNameWithSuffix("config")
+        - name: impersonation-proxy
+          secret:
+            secretName: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+            items: #! make sure our pod does not start until the token controller has a chance to populate the secret
+              - key: token
+                path: token
         - name: podinfo
           downwardAPI:
             items:
@@ -265,3 +285,16 @@ spec:
       loadBalancerIP: #@ data.values.impersonation_proxy_spec.service.load_balancer_ip
       #@ end
       annotations: #@ data.values.impersonation_proxy_spec.service.annotations
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+  namespace: #@ namespace()
+  labels: #@ labels()
+  annotations:
+    #! wait until the SA exists to create this secret so that the token controller does not delete it
+    #! we have this secret at the end so that kubectl will create the service account first
+    kapp.k14s.io/change-rule: "upsert after upserting impersonation-proxy.concierge.pinniped.dev/serviceaccount"
+    kubernetes.io/service-account.name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+type: kubernetes.io/service-account-token

deploy/concierge/rbac.yaml

@@ -28,12 +28,6 @@ rules:
     resources: [ securitycontextconstraints ]
     verbs: [ use ]
     resourceNames: [ nonroot ]
-  - apiGroups: [ "" ]
-    resources: [ "users", "groups", "serviceaccounts" ]
-    verbs: [ "impersonate" ]
-  - apiGroups: [ "authentication.k8s.io" ]
-    resources: [ "*" ]  #! What we really want is userextras/* but the RBAC authorizer only supports */subresource, not resource/*
-    verbs: [ "impersonate" ]
   - apiGroups: [ "" ]
     resources: [ nodes ]
     verbs: [ list ]
@@ -64,6 +58,35 @@ roleRef:
   name: #@ defaultResourceNameWithSuffix("aggregated-api-server")
   apiGroup: rbac.authorization.k8s.io
 
+#! Give minimal permissions to impersonation proxy service account
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+  labels: #@ labels()
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "users", "groups", "serviceaccounts" ]
+    verbs: [ "impersonate" ]
+  - apiGroups: [ "authentication.k8s.io" ]
+    resources: [ "*" ]  #! What we really want is userextras/* but the RBAC authorizer only supports */subresource, not resource/*
+    verbs: [ "impersonate" ]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+  labels: #@ labels()
+subjects:
+  - kind: ServiceAccount
+    name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+    namespace: #@ namespace()
+roleRef:
+  kind: ClusterRole
+  name: #@ defaultResourceNameWithSuffix("impersonation-proxy")
+  apiGroup: rbac.authorization.k8s.io
+
 #! Give permission to the kube-cert-agent Pod to run privileged.
 ---
 apiVersion: rbac.authorization.k8s.io/v1

go.mod

@@ -31,7 +31,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	gopkg.in/square/go-jose.v2 v2.5.1
+	gopkg.in/square/go-jose.v2 v2.6.0
 	k8s.io/api v0.21.1
 	k8s.io/apimachinery v0.21.1
 	k8s.io/apiserver v0.21.1

go.sum

@@ -1583,8 +1583,9 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.1.9/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
+gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc=

internal/concierge/impersonator/doc.go

@@ -11,7 +11,9 @@ The specifics of how it is implemented are of interest.  The most novel detail
 about the implementation is that we use the "front-end" of the aggregated API
 server logic, mainly the DefaultBuildHandlerChain func, to handle how incoming
 requests are authenticated, authorized, etc.  The "back-end" of the proxy is a
-reverse proxy that impersonates the user (instead of serving REST APIs).
+reverse proxy that impersonates the user (instead of serving REST APIs).  Since
+impersonation fails open, we impersonate users via a secondary service account
+that has no other permissions on the cluster.
 
 In terms of authentication, we aim to handle every type of authentication that
 the Kubernetes API server supports by delegating most of the checks to it.  We
@@ -19,9 +21,18 @@ also honor client certs from a CA that is specific to the impersonation proxy.
 This approach allows clients to use the Token Credential Request API even when
 we do not have the cluster's signing key.
 
-In terms of authorization, we rely mostly on the Kubernetes API server.  Since we
-impersonate the user, the proxied request will be authorized against that user.
-Thus for all regular REST verbs, we perform no authorization checks.
+The proxy will honor cluster configuration in regards to anonymous authentication.
+When disabled, the proxy will not authenticate these requests. There is one caveat
+in that Pinniped itself provides the Token Credential Request API which is used
+specifically by anonymous users to retrieve credentials.  This API is the single
+API that will remain available even when anonymous authentication is disabled.
+
+In terms of authorization, in addition to the regular checks that the Kubernetes
+API server will make for the impersonated user, we perform the same authorization
+checks via subject access review calls.  This protects us from scenarios where
+we fail to correctly impersonate the user due to some bug in our proxy logic.
+We rely completely on the Kubernetes API server to perform admission checks on
+the impersonated requests.
 
 Nested impersonation is handled by performing the same authorization checks the
 Kubernetes API server would (we get this mostly for free by using the aggregated

internal/concierge/impersonator/impersonator.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"os"
 	"regexp"
 	"strings"
 	"time"
@@ -19,6 +20,7 @@ import (
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -68,7 +70,7 @@ func New(
 	dynamicCertProvider dynamiccert.Private,
 	impersonationProxySignerCA dynamiccert.Public,
 ) (func(stopCh <-chan struct{}) error, error) {
-	return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, nil, nil)
+	return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, nil, nil, nil)
 }
 
 func newInternal( //nolint:funlen // yeah, it's kind of long.
@@ -77,6 +79,7 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 	impersonationProxySignerCA dynamiccert.Public,
 	clientOpts []kubeclient.Option, // for unit testing, should always be nil in production
 	recOpts func(*genericoptions.RecommendedOptions), // for unit testing, should always be nil in production
+	recConfig func(*genericapiserver.RecommendedConfig), // for unit testing, should always be nil in production
 ) (func(stopCh <-chan struct{}) error, error) {
 	var listener net.Listener
 
@@ -101,12 +104,12 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 		// along with the Kube API server's CA.
 		// Note: any changes to the the Authentication stack need to be kept in sync with any assumptions made
 		// by getTransportForUser, especially if we ever update the TCR API to start returning bearer tokens.
-		kubeClient, err := kubeclient.New(clientOpts...)
+		kubeClientUnsafeForProxying, err := kubeclient.New(clientOpts...)
 		if err != nil {
 			return nil, err
 		}
 		kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
-			"client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClient.Kubernetes,
+			"client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClientUnsafeForProxying.Kubernetes,
 		)
 		if err != nil {
 			return nil, err
@@ -136,7 +139,7 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 		// Loopback authentication to this server does not really make sense since we just proxy everything to
 		// the Kube API server, thus we replace loopback connection config with one that does direct connections
 		// the Kube API server. Loopback config is mainly used by post start hooks, so this is mostly future proofing.
-		serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClient.ProtoConfig) // assume proto is safe (hooks can override)
+		serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClientUnsafeForProxying.ProtoConfig) // assume proto is safe (hooks can override)
 		// Remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken.
 		// See sanity checks at the end of this function.
 		serverConfig.LoopbackClientConfig.BearerToken = ""
@@ -151,9 +154,15 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 			sets.NewString("attach", "exec", "proxy", "log", "portforward"),
 		)
 
+		// use the custom impersonation proxy service account credentials when reverse proxying to the API server
+		kubeClientForProxy, err := getReverseProxyClient(clientOpts)
+		if err != nil {
+			return nil, fmt.Errorf("failed to build reverse proxy client: %w", err)
+		}
+
 		// Assume proto config is safe because transport level configs do not use rest.ContentConfig.
 		// Thus if we are interacting with actual APIs, they should be using pre-built clients.
-		impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClient.ProtoConfig))
+		impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClientForProxy.ProtoConfig))
 		if err != nil {
 			return nil, err
 		}
@@ -173,10 +182,6 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 			}))
 			handler = filterlatency.TrackStarted(handler, "impersonationproxy")
 
-			handler = filterlatency.TrackCompleted(handler)
-			handler = deleteKnownImpersonationHeaders(handler)
-			handler = filterlatency.TrackStarted(handler, "deleteimpersonationheaders")
-
 			// The standard Kube handler chain (authn, authz, impersonation, audit, etc).
 			// See the genericapiserver.DefaultBuildHandlerChain func for details.
 			handler = defaultBuildHandlerChainFunc(handler, c)
@@ -198,48 +203,109 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 		serverConfig.AuditPolicyChecker = policy.FakeChecker(auditinternal.LevelMetadata, nil)
 		serverConfig.AuditBackend = &auditfake.Backend{}
 
+		// Probe the API server to figure out if anonymous auth is enabled.
+		anonymousAuthEnabled, err := isAnonymousAuthEnabled(kubeClientUnsafeForProxying.JSONConfig)
+		if err != nil {
+			return nil, fmt.Errorf("could not detect if anonymous authentication is enabled: %w", err)
+		}
+		plog.Debug("anonymous authentication probed", "anonymousAuthEnabled", anonymousAuthEnabled)
+
 		// if we ever start unioning a TCR bearer token authenticator with serverConfig.Authenticator
 		// then we will need to update the related assumption in tokenPassthroughRoundTripper
 
+		delegatingAuthenticator := serverConfig.Authentication.Authenticator
+		blockAnonymousAuthenticator := &comparableAuthenticator{
+			RequestFunc: func(req *http.Request) (*authenticator.Response, bool, error) {
+				resp, ok, err := delegatingAuthenticator.AuthenticateRequest(req)
+
+				// anonymous auth is enabled so no further check is necessary
+				if anonymousAuthEnabled {
+					return resp, ok, err
+				}
+
+				// authentication failed
+				if err != nil || !ok {
+					return resp, ok, err
+				}
+
+				// any other user than anonymous is irrelevant
+				if resp.User.GetName() != user.Anonymous {
+					return resp, ok, err
+				}
+
+				reqInfo, ok := genericapirequest.RequestInfoFrom(req.Context())
+				if !ok {
+					return nil, false, constable.Error("no RequestInfo found in the context")
+				}
+
+				// a TKR is a resource, any request that is not for a resource should not be authenticated
+				if !reqInfo.IsResourceRequest {
+					return nil, false, nil
+				}
+
+				// any resource besides TKR should not be authenticated
+				if !isTokenCredReq(reqInfo) {
+					return nil, false, nil
+				}
+
+				// anonymous authentication is disabled, but we must let an anonymous request
+				// to TKR authenticate as this is the only method to retrieve credentials
+				return resp, ok, err
+			},
+		}
+		// Set our custom authenticator before calling Compete(), which will use it.
+		serverConfig.Authentication.Authenticator = blockAnonymousAuthenticator
+
 		delegatingAuthorizer := serverConfig.Authorization.Authorizer
-		nestedImpersonationAuthorizer := &comparableAuthorizer{
+		customReasonAuthorizer := &comparableAuthorizer{
 			authorizerFunc: func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
+				const baseReason = "decision made by impersonation-proxy.concierge.pinniped.dev"
 				switch a.GetVerb() {
 				case "":
 					// Empty string is disallowed because request info has had bugs in the past where it would leave it empty.
-					return authorizer.DecisionDeny, "invalid verb", nil
-				case "create",
-					"update",
-					"delete",
-					"deletecollection",
-					"get",
-					"list",
-					"watch",
-					"patch",
-					"proxy":
-					// we know these verbs are from the request info parsing which is safe to delegate to KAS
-					return authorizer.DecisionAllow, "deferring standard verb authorization to kube API server", nil
+					return authorizer.DecisionDeny, "invalid verb, " + baseReason, nil
 				default:
-					// assume everything else is internal SAR checks that we need to run against the requesting user
-					// because when KAS does the check, it may run the check against our service account and not the
-					// requesting user.  This also handles the impersonate verb to allow for nested impersonation.
-					return delegatingAuthorizer.Authorize(ctx, a)
+					// Since we authenticate the requesting user, we are in the best position to correctly authorize them.
+					// When KAS does the check, it may run the check against our service account and not the requesting user
+					// (due to a bug in the code or any other internal SAR checks that the request processing does).
+					// This also handles the impersonate verb to allow for nested impersonation.
+					decision, reason, err := delegatingAuthorizer.Authorize(ctx, a)
+
+					// make it easier to detect when the impersonation proxy is authorizing a request vs KAS
+					switch len(reason) {
+					case 0:
+						reason = baseReason
+					default:
+						reason = reason + ", " + baseReason
+					}
+
+					return decision, reason, err
 				}
 			},
 		}
 		// Set our custom authorizer before calling Compete(), which will use it.
-		serverConfig.Authorization.Authorizer = nestedImpersonationAuthorizer
+		serverConfig.Authorization.Authorizer = customReasonAuthorizer
 
-		impersonationProxyServer, err := serverConfig.Complete().New("impersonation-proxy", genericapiserver.NewEmptyDelegate())
+		if recConfig != nil {
+			recConfig(serverConfig)
+		}
+
+		completedConfig := serverConfig.Complete()
+		impersonationProxyServer, err := completedConfig.New("impersonation-proxy", genericapiserver.NewEmptyDelegate())
 		if err != nil {
 			return nil, err
 		}
 
 		preparedRun := impersonationProxyServer.PrepareRun()
 
+		// Sanity check. Make sure that our custom authenticator is still in place and did not get changed or wrapped.
+		if completedConfig.Authentication.Authenticator != blockAnonymousAuthenticator {
+			return nil, fmt.Errorf("invalid mutation of anonymous authenticator detected: %#v", completedConfig.Authentication.Authenticator)
+		}
+
 		// Sanity check. Make sure that our custom authorizer is still in place and did not get changed or wrapped.
-		if preparedRun.Authorizer != nestedImpersonationAuthorizer {
-			return nil, constable.Error("invalid mutation of impersonation authorizer detected")
+		if preparedRun.Authorizer != customReasonAuthorizer {
+			return nil, fmt.Errorf("invalid mutation of impersonation authorizer detected: %#v", preparedRun.Authorizer)
 		}
 
 		// Sanity check. Assert that we have a functioning token file to use and no bearer token.
@@ -262,32 +328,87 @@ func newInternal( //nolint:funlen // yeah, it's kind of long.
 	return result, nil
 }
 
-func deleteKnownImpersonationHeaders(delegate http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// remove known impersonation headers while avoiding mutation of input request
-		// unknown future impersonation headers will still get caught by our later checks
-		if ensureNoImpersonationHeaders(r) != nil {
-			r = r.Clone(r.Context())
+func getReverseProxyClient(clientOpts []kubeclient.Option) (*kubeclient.Client, error) {
+	// just use the overrides given during unit tests
+	if len(clientOpts) != 0 {
+		return kubeclient.New(clientOpts...)
+	}
 
-			impersonationHeaders := []string{
-				transport.ImpersonateUserHeader,
-				transport.ImpersonateGroupHeader,
-			}
+	// this is the magic path where the impersonation proxy SA token is mounted
+	const tokenFile = "/var/run/secrets/impersonation-proxy.concierge.pinniped.dev/serviceaccount/token" //nolint:gosec // this is not a credential
 
-			for k := range r.Header {
-				if !strings.HasPrefix(k, transport.ImpersonateUserExtraHeaderPrefix) {
-					continue
-				}
-				impersonationHeaders = append(impersonationHeaders, k)
-			}
+	// make sure the token file we need exists before trying to use it
+	if _, err := os.Stat(tokenFile); err != nil {
+		return nil, err
+	}
 
-			for _, header := range impersonationHeaders {
-				r.Header.Del(header) // delay mutation until the end when we are done iterating over the map
-			}
-		}
+	// build an in cluster config that uses the impersonation proxy token file
+	impersonationProxyRestConfig, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	impersonationProxyRestConfig = rest.AnonymousClientConfig(impersonationProxyRestConfig)
+	impersonationProxyRestConfig.BearerTokenFile = tokenFile
 
-		delegate.ServeHTTP(w, r)
-	})
+	return kubeclient.New(kubeclient.WithConfig(impersonationProxyRestConfig))
+}
+
+func isAnonymousAuthEnabled(config *rest.Config) (bool, error) {
+	anonymousConfig := rest.AnonymousClientConfig(config)
+
+	// we do not need either of these but RESTClientFor complains if they are not set
+	anonymousConfig.GroupVersion = &schema.GroupVersion{}
+	anonymousConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
+
+	// in case anyone looking at audit logs wants to know who is making the anonymous request
+	anonymousConfig.UserAgent = rest.DefaultKubernetesUserAgent()
+
+	rc, err := rest.RESTClientFor(anonymousConfig)
+	if err != nil {
+		return false, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, errHealthz := rc.Get().AbsPath("/healthz").DoRaw(ctx)
+
+	switch {
+	// 200 ok on healthz clearly indicates authentication success
+	case errHealthz == nil:
+		return true, nil
+
+	// we are authenticated but not authorized. anonymous authentication is enabled
+	case apierrors.IsForbidden(errHealthz):
+		return true, nil
+
+	// failure to authenticate will return unauthorized (http misnomer)
+	case apierrors.IsUnauthorized(errHealthz):
+		return false, nil
+
+	// any other error is unexpected
+	default:
+		return false, errHealthz
+	}
+}
+
+func isTokenCredReq(reqInfo *genericapirequest.RequestInfo) bool {
+	if reqInfo.Resource != "tokencredentialrequests" {
+		return false
+	}
+
+	// pinniped components allow for the group suffix to be customized
+	// rather than wiring in the current configured suffix, checking the prefix is sufficient
+	if !strings.HasPrefix(reqInfo.APIGroup, "login.concierge.") {
+		return false
+	}
+
+	return true
+}
+
+// No-op wrapping around RequestFunc to allow for comparisons.
+type comparableAuthenticator struct {
+	authenticator.RequestFunc
 }
 
 // No-op wrapping around AuthorizerFunc to allow for comparisons.

internal/concierge/impersonator/impersonator_test.go

@@ -5,26 +5,32 @@ package impersonator
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/httpstream"
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/features"
 	genericapiserver "k8s.io/apiserver/pkg/server"
@@ -33,10 +39,13 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd/api"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/utils/pointer"
 
+	loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1"
 	"go.pinniped.dev/internal/certauthority"
 	"go.pinniped.dev/internal/constable"
 	"go.pinniped.dev/internal/dynamiccert"
+	"go.pinniped.dev/internal/groupsuffix"
 	"go.pinniped.dev/internal/here"
 	"go.pinniped.dev/internal/httputil/roundtripper"
 	"go.pinniped.dev/internal/kubeclient"
@@ -72,9 +81,12 @@ func TestImpersonator(t *testing.T) {
 		clientNextProtos                   []string
 		kubeAPIServerClientBearerTokenFile string
 		kubeAPIServerStatusCode            int
+		kubeAPIServerHealthz               http.Handler
+		anonymousAuthDisabled              bool
 		wantKubeAPIServerRequestHeaders    http.Header
 		wantError                          string
 		wantConstructionError              string
+		wantAuthorizerAttributes           []authorizer.AttributesRecord
 	}{
 		{
 			name:                               "happy path",
@@ -89,6 +101,61 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding":   {"gzip"},
 				"X-Forwarded-For":   {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
+		},
+		{
+			name:                               "happy path with forbidden healthz",
+			clientCert:                         newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}),
+			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
+			kubeAPIServerHealthz: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusForbidden)
+				_, _ = w.Write([]byte("no healthz for you"))
+			}),
+			wantKubeAPIServerRequestHeaders: http.Header{
+				"Impersonate-User":  {"test-username"},
+				"Impersonate-Group": {"test-group1", "test-group2", "system:authenticated"},
+				"Authorization":     {"Bearer some-service-account-token"},
+				"User-Agent":        {"test-agent"},
+				"Accept":            {"application/vnd.kubernetes.protobuf,application/json"},
+				"Accept-Encoding":   {"gzip"},
+				"X-Forwarded-For":   {"127.0.0.1"},
+			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
+		},
+		{
+			name:                               "happy path with unauthorized healthz",
+			clientCert:                         newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}),
+			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
+			kubeAPIServerHealthz: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusUnauthorized)
+				_, _ = w.Write([]byte("no healthz for you"))
+			}),
+			anonymousAuthDisabled: true,
+			wantKubeAPIServerRequestHeaders: http.Header{
+				"Impersonate-User":  {"test-username"},
+				"Impersonate-Group": {"test-group1", "test-group2", "system:authenticated"},
+				"Authorization":     {"Bearer some-service-account-token"},
+				"User-Agent":        {"test-agent"},
+				"Accept":            {"application/vnd.kubernetes.protobuf,application/json"},
+				"Accept-Encoding":   {"gzip"},
+				"X-Forwarded-For":   {"127.0.0.1"},
+			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                               "happy path with upgrade",
@@ -114,6 +181,12 @@ func TestImpersonator(t *testing.T) {
 				"Connection":        {"Upgrade"},
 				"Upgrade":           {"spdy/3.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username2", UID: "", Groups: []string{"test-group3", "test-group4", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                               "happy path ignores forwarded header",
@@ -131,6 +204,12 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding":   {"gzip"},
 				"X-Forwarded-For":   {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username2", UID: "", Groups: []string{"test-group3", "test-group4", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                               "happy path ignores forwarded header canonicalization",
@@ -148,6 +227,12 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding":   {"gzip"},
 				"X-Forwarded-For":   {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username2", UID: "", Groups: []string{"test-group3", "test-group4", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                               "user is authenticated but the kube API request returns an error",
@@ -164,6 +249,12 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding":   {"gzip"},
 				"X-Forwarded-For":   {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                               "when there is no client cert on request, it is an anonymous request",
@@ -178,6 +269,12 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding":   {"gzip"},
 				"X-Forwarded-For":   {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:       "when there is no client cert on request but it has basic auth, it is still an anonymous request",
@@ -198,12 +295,19 @@ func TestImpersonator(t *testing.T) {
 				"X-Forwarded-For":   {"127.0.0.1"},
 				"Test":              {"val"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                               "failed client cert authentication",
 			clientCert:                         newClientCert(t, unrelatedCA, "test-username", []string{"test-group1"}),
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Unauthorized",
+			wantAuthorizerAttributes:           nil,
 		},
 		{
 			name:                               "nested impersonation by regular users calls delegating authorizer",
@@ -212,7 +316,14 @@ func TestImpersonator(t *testing.T) {
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			// this fails because the delegating authorizer in this test only allows system:masters and fails everything else
 			wantError: `users "some-other-username" is forbidden: User "test-username" ` +
-				`cannot impersonate resource "users" in API group "" at the cluster scope`,
+				`cannot impersonate resource "users" in API group "" at the cluster scope: ` +
+				`decision made by impersonation-proxy.concierge.pinniped.dev`,
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "some-other-username", ResourceRequest: true, Path: "",
+				},
+			},
 		},
 		{
 			name:       "nested impersonation by admin users calls delegating authorizer",
@@ -258,6 +369,96 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding": {"gzip"},
 				"X-Forwarded-For": {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "fire", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "groups", Subresource: "", Name: "elements", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "iam.gke.io/user-assertion", Name: "good", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "iam.gke.io/user-assertion", Name: "stuff", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "alpha.kubernetes.io/identity/roles", Name: "a-role1", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "alpha.kubernetes.io/identity/roles", Name: "a-role2", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "user-assertion.cloud.google.com", Name: "smaller", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "user-assertion.cloud.google.com", Name: "things", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "colors", Name: "red", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "colors", Name: "orange", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "colors", Name: "blue", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "scopes.authorization.openshift.io", Name: "user:info", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "scopes.authorization.openshift.io", Name: "user:full", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "scopes.authorization.openshift.io", Name: "user:check-access", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "alpha.kubernetes.io/identity/project/name", Name: "a-project-name", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "alpha.kubernetes.io/identity/user/domain/id", Name: "a-domain-id", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "alpha.kubernetes.io/identity/user/domain/name", Name: "a-domain-name", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "alpha.kubernetes.io/identity/project/id", Name: "a-project-id", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "fire", UID: "", Groups: []string{"elements", "system:authenticated"},
+						Extra: map[string][]string{
+							"alpha.kubernetes.io/identity/project/id":       {"a-project-id"},
+							"alpha.kubernetes.io/identity/project/name":     {"a-project-name"},
+							"alpha.kubernetes.io/identity/roles":            {"a-role1", "a-role2"},
+							"alpha.kubernetes.io/identity/user/domain/id":   {"a-domain-id"},
+							"alpha.kubernetes.io/identity/user/domain/name": {"a-domain-name"},
+							"colors":                            {"red", "orange", "blue"},
+							"iam.gke.io/user-assertion":         {"good", "stuff"},
+							"scopes.authorization.openshift.io": {"user:info", "user:full", "user:check-access"},
+							"user-assertion.cloud.google.com":   {"smaller", "things"},
+						},
+					},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                  "nested impersonation by admin users cannot impersonate UID",
@@ -268,6 +469,16 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Internal error occurred: invalid impersonation",
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "some-other-username", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "some-other-username", UID: "", Groups: []string{"system:authenticated"}, Extra: map[string][]string{}},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:                  "nested impersonation by admin users cannot impersonate UID header canonicalization",
@@ -278,6 +489,16 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Internal error occurred: invalid impersonation",
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "some-other-username", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "some-other-username", UID: "", Groups: []string{"system:authenticated"}, Extra: map[string][]string{}},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:       "nested impersonation by admin users cannot use reserved key",
@@ -292,6 +513,33 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Internal error occurred: unimplemented functionality - unable to act as current user",
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "other-user-to-impersonate", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "groups", Subresource: "", Name: "other-peeps", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "something.impersonation-proxy.concierge.pinniped.dev", Name: "bad data", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "key", Name: "good", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "other-user-to-impersonate", UID: "", Groups: []string{"other-peeps", "system:authenticated"},
+						Extra: map[string][]string{
+							"key": {"good"},
+							"something.impersonation-proxy.concierge.pinniped.dev": {"bad data"},
+						},
+					},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:       "nested impersonation by admin users cannot use invalid key",
@@ -305,6 +553,24 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Internal error occurred: unimplemented functionality - unable to act as current user",
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "panda", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "groups", Subresource: "", Name: "other-peeps", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "party~~time", Name: "danger", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "panda", UID: "", Groups: []string{"other-peeps", "system:authenticated"}, Extra: map[string][]string{"party~~time": {"danger"}}},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:       "nested impersonation by admin users can use uppercase key because impersonation is lossy",
@@ -328,10 +594,38 @@ func TestImpersonator(t *testing.T) {
 				"Accept-Encoding": {"gzip"},
 				"X-Forwarded-For": {"127.0.0.1"},
 			},
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "panda", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "groups", Subresource: "", Name: "other-peeps", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "test-admin", UID: "", Groups: []string{"test-group2", "system:masters", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "authentication.k8s.io", APIVersion: "v1", Resource: "userextras", Subresource: "roar", Name: "tiger", ResourceRequest: true, Path: "",
+				},
+				{
+					User: &user.DefaultInfo{Name: "panda", UID: "", Groups: []string{"other-peeps", "system:authenticated"}, Extra: map[string][]string{"roar": {"tiger"}}},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
-			name:                  "no bearer token file in Kube API server client config",
-			wantConstructionError: "invalid impersonator loopback rest config has wrong bearer token semantics",
+			name:                     "no bearer token file in Kube API server client config",
+			wantConstructionError:    "invalid impersonator loopback rest config has wrong bearer token semantics",
+			wantAuthorizerAttributes: nil,
+		},
+		{
+			name: "unexpected healthz response",
+			kubeAPIServerHealthz: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte("broken"))
+			}),
+			wantConstructionError:    `could not detect if anonymous authentication is enabled: an error on the server ("broken") has prevented the request from succeeding`,
+			wantAuthorizerAttributes: nil,
 		},
 		{
 			name:       "header canonicalization user header",
@@ -341,7 +635,14 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError: `users "PANDA" is forbidden: User "test-username" ` +
-				`cannot impersonate resource "users" in API group "" at the cluster scope`,
+				`cannot impersonate resource "users" in API group "" at the cluster scope: ` +
+				`decision made by impersonation-proxy.concierge.pinniped.dev`,
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "impersonate", Namespace: "", APIGroup: "", APIVersion: "", Resource: "users", Subresource: "", Name: "PANDA", ResourceRequest: true, Path: "",
+				},
+			},
 		},
 		{
 			name:       "header canonicalization future UID header",
@@ -351,6 +652,12 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Internal error occurred: invalid impersonation",
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 		{
 			name:       "future UID header",
@@ -360,6 +667,12 @@ func TestImpersonator(t *testing.T) {
 			},
 			kubeAPIServerClientBearerTokenFile: "required-to-be-set",
 			wantError:                          "Internal error occurred: invalid impersonation",
+			wantAuthorizerAttributes: []authorizer.AttributesRecord{
+				{
+					User: &user.DefaultInfo{Name: "test-username", UID: "", Groups: []string{"test-group1", "test-group2", "system:authenticated"}, Extra: nil},
+					Verb: "list", Namespace: "", APIGroup: "", APIVersion: "v1", Resource: "namespaces", Subresource: "", Name: "", ResourceRequest: true, Path: "/api/v1/namespaces",
+				},
+			},
 		},
 	}
 	for _, tt := range tests {
@@ -367,6 +680,9 @@ func TestImpersonator(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
+			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+			t.Cleanup(cancel)
+
 			// we need to create this listener ourselves because the API server
 			// code treats (port == 0 && listener == nil) to mean "do nothing"
 			listener, port, err := genericoptions.CreateListener("", "127.0.0.1:0", net.ListenConfig{})
@@ -384,22 +700,28 @@ func TestImpersonator(t *testing.T) {
 			testKubeAPIServerWasCalled := false
 			var testKubeAPIServerSawHeaders http.Header
 			testKubeAPIServerCA, testKubeAPIServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) {
-				require.Equal(t, http.MethodGet, r.Method)
 				switch r.URL.Path {
 				case "/api/v1/namespaces/kube-system/configmaps":
+					require.Equal(t, http.MethodGet, r.Method)
+
 					// The production code uses NewDynamicCAFromConfigMapController which fetches a ConfigMap,
 					// so treat that differently. It wants to read the Kube API server CA from that ConfigMap
 					// to use it to validate client certs. We don't need it for this test, so return NotFound.
 					http.NotFound(w, r)
 					return
+
 				case "/api/v1/namespaces":
+					require.Equal(t, http.MethodGet, r.Method)
+
 					testKubeAPIServerWasCalled = true
 					testKubeAPIServerSawHeaders = r.Header
 					if tt.kubeAPIServerStatusCode != http.StatusOK {
 						w.WriteHeader(tt.kubeAPIServerStatusCode)
-					} else {
-						w.Header().Add("Content-Type", "application/json; charset=UTF-8")
-						_, _ = w.Write([]byte(here.Doc(`
+						return
+					}
+
+					w.Header().Add("Content-Type", "application/json; charset=UTF-8")
+					_, _ = w.Write([]byte(here.Doc(`
 						{
 							"kind": "NamespaceList",
 							"apiVersion":"v1",
@@ -409,9 +731,61 @@ func TestImpersonator(t *testing.T) {
 							]
 						}
 					`)))
+					return
+
+				case "/probe":
+					require.Equal(t, http.MethodGet, r.Method)
+
+					_, _ = fmt.Fprint(w, "probed")
+					return
+
+				case "/healthz":
+					require.Equal(t, http.MethodGet, r.Method)
+					require.Empty(t, r.Header.Get("Authorization"))
+					require.Contains(t, r.Header.Get("User-Agent"), "kubernetes")
+
+					if tt.kubeAPIServerHealthz != nil {
+						tt.kubeAPIServerHealthz.ServeHTTP(w, r)
+						return
 					}
+
+					// by default just match the KAS /healthz endpoint
+					w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+					w.Header().Set("X-Content-Type-Options", "nosniff")
+					_, _ = fmt.Fprint(w, "ok")
+					return
+
+				case "/apis/login.concierge.pinniped.dev/v1alpha1/tokencredentialrequests":
+					require.Equal(t, http.MethodPost, r.Method)
+
+					w.Header().Add("Content-Type", "application/json; charset=UTF-8")
+					_, _ = w.Write([]byte(`{}`))
+					return
+
+				case "/apis/login.concierge.walrus.tld/v1alpha1/tokencredentialrequests":
+					require.Equal(t, http.MethodPost, r.Method)
+
+					w.Header().Add("Content-Type", "application/json; charset=UTF-8")
+					_, _ = w.Write([]byte(`{}`))
+					return
+
+				case "/apis/not-concierge.walrus.tld/v1/tokencredentialrequests":
+					require.Equal(t, http.MethodGet, r.Method)
+
+					w.Header().Add("Content-Type", "application/json; charset=UTF-8")
+					_, _ = w.Write([]byte(`{"hello": "quack"}`))
+					return
+
+				case "/apis/not-concierge.walrus.tld/v1/ducks":
+					require.Equal(t, http.MethodGet, r.Method)
+
+					w.Header().Add("Content-Type", "application/json; charset=UTF-8")
+					_, _ = w.Write([]byte(`{"hello": "birds"}`))
+					return
+
 				default:
-					require.Fail(t, "fake Kube API server got an unexpected request")
+					require.Fail(t, "fake Kube API server got an unexpected request", "path: %s", r.URL.Path)
+					return
 				}
 			})
 
@@ -433,8 +807,29 @@ func TestImpersonator(t *testing.T) {
 				options.SecureServing.Listener = listener // use our listener with the dynamic port
 			}
 
+			recorder := &attributeRecorder{}
+			defer func() {
+				require.ElementsMatch(t, tt.wantAuthorizerAttributes, recorder.attributes)
+				require.Len(t, recorder.attributes, len(tt.wantAuthorizerAttributes))
+			}()
+
+			// Allow standard REST verbs to be authorized so that tests pass without invasive changes
+			recConfig := func(config *genericapiserver.RecommendedConfig) {
+				authz := config.Authorization.Authorizer.(*comparableAuthorizer)
+				delegate := authz.authorizerFunc
+				authz.authorizerFunc = func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
+					recorder.record(a)
+					switch a.GetVerb() {
+					case "create", "get", "list":
+						return authorizer.DecisionAllow, "standard verbs are allowed in tests", nil
+					default:
+						return delegate(ctx, a)
+					}
+				}
+			}
+
 			// Create an impersonator.  Use an invalid port number to make sure our listener override works.
-			runner, constructionErr := newInternal(-1000, certKeyContent, caContent, clientOpts, recOpts)
+			runner, constructionErr := newInternal(-1000, certKeyContent, caContent, clientOpts, recOpts, recConfig)
 			if len(tt.wantConstructionError) > 0 {
 				require.EqualError(t, constructionErr, tt.wantConstructionError)
 				require.Nil(t, runner)
@@ -485,7 +880,7 @@ func TestImpersonator(t *testing.T) {
 
 			// The fake Kube API server knows how to to list namespaces, so make that request using the client
 			// through the impersonator.
-			listResponse, err := client.Kubernetes.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
+			listResponse, err := client.Kubernetes.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
 			if len(tt.wantError) > 0 {
 				require.EqualError(t, err, tt.wantError)
 				require.Equal(t, &corev1.NamespaceList{}, listResponse)
@@ -505,6 +900,109 @@ func TestImpersonator(t *testing.T) {
 			// of the original request mutated by the impersonator.  Otherwise the headers should be nil.
 			require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders)
 
+			// these authorization checks are caused by the anonymous auth checks below
+			tt.wantAuthorizerAttributes = append(tt.wantAuthorizerAttributes,
+				authorizer.AttributesRecord{
+					User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+					Verb: "create", Namespace: "", APIGroup: "login.concierge.pinniped.dev", APIVersion: "v1alpha1", Resource: "tokencredentialrequests", Subresource: "", Name: "", ResourceRequest: true, Path: "/apis/login.concierge.pinniped.dev/v1alpha1/tokencredentialrequests",
+				},
+				authorizer.AttributesRecord{
+					User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+					Verb: "create", Namespace: "", APIGroup: "login.concierge.walrus.tld", APIVersion: "v1alpha1", Resource: "tokencredentialrequests", Subresource: "", Name: "", ResourceRequest: true, Path: "/apis/login.concierge.walrus.tld/v1alpha1/tokencredentialrequests",
+				},
+			)
+			if !tt.anonymousAuthDisabled {
+				tt.wantAuthorizerAttributes = append(tt.wantAuthorizerAttributes,
+					authorizer.AttributesRecord{
+						User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+						Verb: "get", Namespace: "", APIGroup: "", APIVersion: "", Resource: "", Subresource: "", Name: "", ResourceRequest: false, Path: "/probe",
+					},
+					authorizer.AttributesRecord{
+						User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+						Verb: "list", Namespace: "", APIGroup: "not-concierge.walrus.tld", APIVersion: "v1", Resource: "tokencredentialrequests", Subresource: "", Name: "", ResourceRequest: true, Path: "/apis/not-concierge.walrus.tld/v1/tokencredentialrequests",
+					},
+					authorizer.AttributesRecord{
+						User: &user.DefaultInfo{Name: "system:anonymous", UID: "", Groups: []string{"system:unauthenticated"}, Extra: nil},
+						Verb: "list", Namespace: "", APIGroup: "not-concierge.walrus.tld", APIVersion: "v1", Resource: "ducks", Subresource: "", Name: "", ResourceRequest: true, Path: "/apis/not-concierge.walrus.tld/v1/ducks",
+					},
+				)
+			}
+
+			// anonymous TCR should always work
+
+			tcrRegGroup, err := kubeclient.New(kubeclient.WithConfig(rest.AnonymousClientConfig(clientKubeconfig)))
+			require.NoError(t, err)
+
+			tcrOtherGroup, err := kubeclient.New(kubeclient.WithConfig(rest.AnonymousClientConfig(clientKubeconfig)),
+				kubeclient.WithMiddleware(groupsuffix.New("walrus.tld")))
+			require.NoError(t, err)
+
+			_, errTCR := tcrRegGroup.PinnipedConcierge.LoginV1alpha1().TokenCredentialRequests().Create(ctx, &loginv1alpha1.TokenCredentialRequest{}, metav1.CreateOptions{})
+			require.NoError(t, errTCR)
+
+			_, errTCROtherGroup := tcrOtherGroup.PinnipedConcierge.LoginV1alpha1().TokenCredentialRequests().Create(ctx,
+				&loginv1alpha1.TokenCredentialRequest{
+					Spec: loginv1alpha1.TokenCredentialRequestSpec{
+						Authenticator: corev1.TypedLocalObjectReference{
+							APIGroup: pointer.String("anything.pinniped.dev"),
+						},
+					},
+				}, metav1.CreateOptions{})
+			require.NoError(t, errTCROtherGroup)
+
+			// these calls should only work when anonymous auth is enabled
+
+			anonymousConfig := rest.AnonymousClientConfig(clientKubeconfig)
+			anonymousConfig.GroupVersion = &schema.GroupVersion{
+				Group:   "not-concierge.walrus.tld",
+				Version: "v1",
+			}
+			anonymousConfig.APIPath = "/apis"
+			anonymousConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
+			rc, err := rest.RESTClientFor(anonymousConfig)
+			require.NoError(t, err)
+
+			probeBody, errProbe := rc.Get().AbsPath("/probe").DoRaw(ctx)
+			if tt.anonymousAuthDisabled {
+				require.True(t, errors.IsUnauthorized(errProbe), errProbe)
+				require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401}`+"\n", string(probeBody))
+			} else {
+				require.NoError(t, errProbe)
+				require.Equal(t, "probed", string(probeBody))
+			}
+
+			notTCRBody, errNotTCR := rc.Get().Resource("tokencredentialrequests").DoRaw(ctx)
+			if tt.anonymousAuthDisabled {
+				require.True(t, errors.IsUnauthorized(errNotTCR), errNotTCR)
+				require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401}`+"\n", string(notTCRBody))
+			} else {
+				require.NoError(t, errNotTCR)
+				require.Equal(t, `{"hello": "quack"}`, string(notTCRBody))
+			}
+
+			ducksBody, errDucks := rc.Get().Resource("ducks").DoRaw(ctx)
+			if tt.anonymousAuthDisabled {
+				require.True(t, errors.IsUnauthorized(errDucks), errDucks)
+				require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401}`+"\n", string(ducksBody))
+			} else {
+				require.NoError(t, errDucks)
+				require.Equal(t, `{"hello": "birds"}`, string(ducksBody))
+			}
+
+			// this should always fail as unauthorized (even for TCR) because the cert is not valid
+
+			badCertConfig := rest.AnonymousClientConfig(clientKubeconfig)
+			badCert := newClientCert(t, unrelatedCA, "bad-user", []string{"bad-group"})
+			badCertConfig.TLSClientConfig.CertData = badCert.certPEM
+			badCertConfig.TLSClientConfig.KeyData = badCert.keyPEM
+
+			tcrBadCert, err := kubeclient.New(kubeclient.WithConfig(badCertConfig))
+			require.NoError(t, err)
+
+			_, errBadCert := tcrBadCert.PinnipedConcierge.LoginV1alpha1().TokenCredentialRequests().Create(ctx, &loginv1alpha1.TokenCredentialRequest{}, metav1.CreateOptions{})
+			require.True(t, errors.IsUnauthorized(errBadCert), errBadCert)
+			require.EqualError(t, errBadCert, "Unauthorized")
+
 			// Stop the impersonator server.
 			close(stopCh)
 			exitErr := <-errCh
@@ -1438,114 +1936,6 @@ func requireCanBindToPort(t *testing.T, port int) {
 	require.NoError(t, ln.Close())
 }
 
-func Test_deleteKnownImpersonationHeaders(t *testing.T) {
-	tests := []struct {
-		name          string
-		headers, want http.Header
-	}{
-		{
-			name: "no impersonation",
-			headers: map[string][]string{
-				"a":               {"b"},
-				"Accept-Encoding": {"gzip"},
-				"User-Agent":      {"test-user-agent"},
-			},
-			want: map[string][]string{
-				"a":               {"b"},
-				"Accept-Encoding": {"gzip"},
-				"User-Agent":      {"test-user-agent"},
-			},
-		},
-		{
-			name: "impersonate user header is dropped",
-			headers: map[string][]string{
-				"a":                {"b"},
-				"Impersonate-User": {"panda"},
-				"Accept-Encoding":  {"gzip"},
-				"User-Agent":       {"test-user-agent"},
-			},
-			want: map[string][]string{
-				"a":               {"b"},
-				"Accept-Encoding": {"gzip"},
-				"User-Agent":      {"test-user-agent"},
-			},
-		},
-		{
-			name: "all known impersonate headers are dropped",
-			headers: map[string][]string{
-				"Accept-Encoding":           {"gzip"},
-				"Authorization":             {"Bearer some-service-account-token"},
-				"Impersonate-Extra-Extra-1": {"some", "extra", "stuff"},
-				"Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"},
-				"Impersonate-Group":         {"test-group-1", "test-group-2"},
-				"Impersonate-User":          {"test-user"},
-				"User-Agent":                {"test-user-agent"},
-			},
-			want: map[string][]string{
-				"Accept-Encoding": {"gzip"},
-				"Authorization":   {"Bearer some-service-account-token"},
-				"User-Agent":      {"test-user-agent"},
-			},
-		},
-		{
-			name: "future UID header is not dropped",
-			headers: map[string][]string{
-				"Accept-Encoding":           {"gzip"},
-				"Authorization":             {"Bearer some-service-account-token"},
-				"Impersonate-Extra-Extra-1": {"some", "extra", "stuff"},
-				"Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"},
-				"Impersonate-Group":         {"test-group-1", "test-group-2"},
-				"Impersonate-User":          {"test-user"},
-				"Impersonate-Uid":           {"008"},
-				"User-Agent":                {"test-user-agent"},
-			},
-			want: map[string][]string{
-				"Accept-Encoding": {"gzip"},
-				"Authorization":   {"Bearer some-service-account-token"},
-				"User-Agent":      {"test-user-agent"},
-				"Impersonate-Uid": {"008"},
-			},
-		},
-		{
-			name: "future UID header is not dropped, no other headers",
-			headers: map[string][]string{
-				"Impersonate-Uid": {"009"},
-			},
-			want: map[string][]string{
-				"Impersonate-Uid": {"009"},
-			},
-		},
-	}
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
-			inputReq := (&http.Request{Header: tt.headers}).WithContext(context.Background())
-			inputReqCopy := inputReq.Clone(inputReq.Context())
-
-			var called bool
-			delegate := http.HandlerFunc(func(w http.ResponseWriter, outputReq *http.Request) {
-				called = true
-				require.Nil(t, w)
-
-				// assert only headers mutated
-				outputReqCopy := outputReq.Clone(outputReq.Context())
-				outputReqCopy.Header = tt.headers
-				require.Equal(t, inputReqCopy, outputReqCopy)
-
-				require.Equal(t, tt.want, outputReq.Header)
-
-				if ensureNoImpersonationHeaders(inputReq) == nil {
-					require.True(t, inputReq == outputReq, "expect req to passed through when no modification needed")
-				}
-			})
-
-			deleteKnownImpersonationHeaders(delegate).ServeHTTP(nil, inputReq)
-			require.Equal(t, inputReqCopy, inputReq) // assert no mutation occurred
-			require.True(t, called)
-		})
-	}
-}
-
 func Test_withBearerTokenPreservation(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -1623,3 +2013,14 @@ func Test_withBearerTokenPreservation(t *testing.T) {
 		})
 	}
 }
+
+type attributeRecorder struct {
+	lock       sync.Mutex
+	attributes []authorizer.AttributesRecord
+}
+
+func (r *attributeRecorder) record(attributes authorizer.Attributes) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	r.attributes = append(r.attributes, *attributes.(*authorizer.AttributesRecord))
+}

site/content/docs/howto/install-cli.md

@@ -44,10 +44,10 @@ Click Open to allow the command to proceed.
 
 ## Install a specific version via script
 
-For example, to install v0.9.0 on Linux/amd64:
+For example, to install v0.9.1 on Linux/amd64:
 
 ```sh
-curl -Lso pinniped https://get.pinniped.dev/v0.9.0/pinniped-cli-linux-amd64 \
+curl -Lso pinniped https://get.pinniped.dev/v0.9.1/pinniped-cli-linux-amd64 \
   && chmod +x pinniped \
   && sudo mv pinniped /usr/local/bin/pinniped
 ```

site/content/docs/howto/install-concierge.md

@@ -26,9 +26,9 @@ Warning: the default configuration may create a public LoadBalancer Service on y
 
 1. Install the Concierge into the `pinniped-concierge` namespace with default options:
 
-   - `kubectl apply -f https://get.pinniped.dev/v0.9.0/install-pinniped-concierge.yaml`
+   - `kubectl apply -f https://get.pinniped.dev/v0.9.1/install-pinniped-concierge.yaml`
 
-      *Replace v0.9.0 with your preferred version number.*
+      *Replace v0.9.1 with your preferred version number.*
   
 ## With custom options
 

site/content/docs/howto/install-supervisor.md

@@ -25,9 +25,9 @@ You should have a supported Kubernetes cluster with working HTTPS ingress capabi
 
 1. Install the Supervisor into the `pinniped-supervisor` namespace with default options:
 
-   - `kubectl apply -f https://get.pinniped.dev/v0.9.0/install-pinniped-supervisor.yaml`
+   - `kubectl apply -f https://get.pinniped.dev/v0.9.1/install-pinniped-supervisor.yaml`
 
-     *Replace v0.9.0 with your preferred version number.*
+     *Replace v0.9.1 with your preferred version number.*
 
 ## With custom options
 

site/content/docs/tutorials/concierge-only-demo.md

@@ -79,7 +79,7 @@ as the authenticator.
    see [deploy/local-user-authenticator/README.md](https://github.com/vmware-tanzu/pinniped/blob/main/deploy/local-user-authenticator/README.md)
    for instructions on how to deploy using `ytt`.
 
-   If you prefer to install a specific version, replace `latest` in the URL with the version number such as `v0.9.0`.
+   If you prefer to install a specific version, replace `latest` in the URL with the version number such as `v0.9.1`.
 
 1. Create a test user named `pinny-the-seal` in the local-user-authenticator namespace.
 

test/integration/concierge_impersonation_proxy_test.go

@@ -39,12 +39,15 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
+	"k8s.io/apiserver/pkg/authentication/user"
 	k8sinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -53,6 +56,7 @@ import (
 	"k8s.io/client-go/util/certificate/csr"
 	"k8s.io/client-go/util/keyutil"
 	"k8s.io/client-go/util/retry"
+	"k8s.io/utils/pointer"
 
 	conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1"
 	identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1"
@@ -562,7 +566,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			require.True(t, k8serrors.IsForbidden(err), err)
 			require.EqualError(t, err, fmt.Sprintf(
 				`users "other-user-to-impersonate" is forbidden: `+
-					`User "%s" cannot impersonate resource "users" in API group "" at the cluster scope`,
+					`User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+
+					`decision made by impersonation-proxy.concierge.pinniped.dev`,
 				env.TestUser.ExpectedUsername))
 
 			// impersonate the GC service account instead which can read anything (the binding to edit allows this)
@@ -610,7 +615,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			require.True(t, k8serrors.IsForbidden(err), err)
 			require.EqualError(t, err, fmt.Sprintf(
 				`userextras.authentication.k8s.io "with a dangerous value" is forbidden: `+
-					`User "%s" cannot impersonate resource "userextras/some-fancy-key" in API group "authentication.k8s.io" at the cluster scope`,
+					`User "%s" cannot impersonate resource "userextras/some-fancy-key" in API group "authentication.k8s.io" at the cluster scope: `+
+					`decision made by impersonation-proxy.concierge.pinniped.dev`,
 				env.TestUser.ExpectedUsername))
 		})
 
@@ -668,7 +674,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			// the impersonated user lacks the RBAC to perform this call
 			require.True(t, k8serrors.IsForbidden(err), err)
 			require.EqualError(t, err, fmt.Sprintf(
-				`secrets "%s" is forbidden: User "other-user-to-impersonate" cannot get resource "secrets" in API group "" in the namespace "%s"`,
+				`secrets "%s" is forbidden: User "other-user-to-impersonate" cannot get resource "secrets" in API group "" in the namespace "%s": `+
+					`decision made by impersonation-proxy.concierge.pinniped.dev`,
 				impersonationProxyTLSSecretName(env), env.ConciergeNamespace,
 			))
 
@@ -698,7 +705,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 				clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM,
 				&rest.ImpersonationConfig{
 					UserName: "other-user-to-impersonate",
-					Groups:   []string{"other-group-1", "other-group-2"},
+					Groups:   []string{"other-group-1", "other-group-2", "system:masters"}, // impersonate system:masters so we get past authorization checks
 					Extra: map[string][]string{
 						"this-good-key": {"to this good value"},
 						"something.impersonation-proxy.concierge.pinniped.dev": {"super sneaky value"},
@@ -740,7 +747,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			require.True(t, k8serrors.IsForbidden(err), err)
 			require.EqualError(t, err, fmt.Sprintf(
 				`serviceaccounts "root-ca-cert-publisher" is forbidden: `+
-					`User "%s" cannot impersonate resource "serviceaccounts" in API group "" in the namespace "kube-system"`,
+					`User "%s" cannot impersonate resource "serviceaccounts" in API group "" in the namespace "kube-system": `+
+					`decision made by impersonation-proxy.concierge.pinniped.dev`,
 				serviceaccount.MakeUsername(namespaceName, saName)))
 
 			// webhook authorizer deny cache TTL is 10 seconds so we need to wait long enough for it to drain
@@ -800,15 +808,21 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			).PinnipedConcierge
 			whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests().
 				Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
-			require.NoError(t, err)
-			require.Equal(t,
-				expectedWhoAmIRequestResponse(
-					"system:anonymous",
-					[]string{"system:unauthenticated"},
-					nil,
-				),
-				whoAmI,
-			)
+
+			// we expect the impersonation proxy to match the behavior of KAS in regards to anonymous requests
+			if env.HasCapability(library.AnonymousAuthenticationSupported) {
+				require.NoError(t, err)
+				require.Equal(t,
+					expectedWhoAmIRequestResponse(
+						"system:anonymous",
+						[]string{"system:unauthenticated"},
+						nil,
+					),
+					whoAmI,
+				)
+			} else {
+				require.True(t, k8serrors.IsUnauthorized(err), library.Sdump(err))
+			}
 
 			// Test using a service account token.
 			namespaceName := createTestNamespace(t, adminClient)
@@ -1193,6 +1207,244 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create.
 			require.Equal(t, *wantConfigMap, actualConfigMap)
 		})
+
+		t.Run("honors anonymous authentication of KAS", func(t *testing.T) {
+			t.Parallel()
+
+			impersonationProxyAnonymousClient := newAnonymousImpersonationProxyClient(
+				t, impersonationProxyURL, impersonationProxyCACertPEM, nil,
+			)
+
+			copyConfig := rest.CopyConfig(impersonationProxyAnonymousClient.JSONConfig)
+			copyConfig.GroupVersion = &schema.GroupVersion{}
+			copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
+			impersonationProxyAnonymousRestClient, err := rest.RESTClientFor(copyConfig)
+			require.NoError(t, err)
+
+			adminClientRestConfig := library.NewClientConfig(t)
+			clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig)
+			impersonationProxyAdminClientAsAnonymousConfig := newImpersonationProxyClientWithCredentials(t,
+				clusterAdminCredentials,
+				impersonationProxyURL, impersonationProxyCACertPEM,
+				&rest.ImpersonationConfig{UserName: user.Anonymous}).
+				JSONConfig
+			impersonationProxyAdminClientAsAnonymousConfigCopy := rest.CopyConfig(impersonationProxyAdminClientAsAnonymousConfig)
+			impersonationProxyAdminClientAsAnonymousConfigCopy.GroupVersion = &schema.GroupVersion{}
+			impersonationProxyAdminClientAsAnonymousConfigCopy.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
+			impersonationProxyAdminRestClientAsAnonymous, err := rest.RESTClientFor(impersonationProxyAdminClientAsAnonymousConfigCopy)
+			require.NoError(t, err)
+
+			t.Run("anonymous authentication irrelevant", func(t *testing.T) {
+				t.Parallel()
+
+				// - hit the token credential request endpoint with an empty body
+				//   - through the impersonation proxy
+				//   - should succeed as an invalid request whether anonymous authentication is enabled or disabled
+				//   - should not reject as unauthorized
+				t.Run("token credential request", func(t *testing.T) {
+					t.Parallel()
+
+					tkr, err := impersonationProxyAnonymousClient.PinnipedConcierge.LoginV1alpha1().TokenCredentialRequests().
+						Create(ctx, &loginv1alpha1.TokenCredentialRequest{
+							Spec: loginv1alpha1.TokenCredentialRequestSpec{
+								Authenticator: corev1.TypedLocalObjectReference{APIGroup: pointer.String("anything.pinniped.dev")},
+							},
+						}, metav1.CreateOptions{})
+					require.True(t, k8serrors.IsInvalid(err), library.Sdump(err))
+					require.Equal(t, `.login.concierge.pinniped.dev "" is invalid: spec.token.value: Required value: token must be supplied`, err.Error())
+					require.Equal(t, &loginv1alpha1.TokenCredentialRequest{}, tkr)
+				})
+
+				// - hit the healthz endpoint (non-resource endpoint)
+				//   - through the impersonation proxy
+				//   - as cluster admin, impersonating anonymous user
+				//   - should succeed, authentication happens as cluster-admin
+				//   - whoami should confirm we are using impersonation
+				//   - healthz should succeed, anonymous users can request this endpoint
+				//   - healthz/log should fail, forbidden anonymous
+				t.Run("non-resource request while impersonating anonymous - nested impersonation", func(t *testing.T) {
+					t.Parallel()
+
+					whoami, errWho := impersonationProxyAdminRestClientAsAnonymous.Post().Body([]byte(`{}`)).AbsPath("/apis/identity.concierge." + env.APIGroupSuffix + "/v1alpha1/whoamirequests").DoRaw(ctx)
+					require.NoError(t, errWho, library.Sdump(errWho))
+					require.True(t, strings.HasPrefix(string(whoami), `{"kind":"WhoAmIRequest","apiVersion":"identity.concierge.`+env.APIGroupSuffix+`/v1alpha1","metadata":{"creationTimestamp":null},"spec":{},"status":{"kubernetesUserInfo":{"user":{"username":"system:anonymous","groups":["system:unauthenticated"],"extra":{"original-user-info.impersonation-proxy.concierge.pinniped.dev":["{\"username\":`), string(whoami))
+
+					healthz, errHealth := impersonationProxyAdminRestClientAsAnonymous.Get().AbsPath("/healthz").DoRaw(ctx)
+					require.NoError(t, errHealth, library.Sdump(errHealth))
+					require.Equal(t, "ok", string(healthz))
+
+					healthzLog, errHealthzLog := impersonationProxyAdminRestClientAsAnonymous.Get().AbsPath("/healthz/log").DoRaw(ctx)
+					require.True(t, k8serrors.IsForbidden(errHealthzLog), "%s\n%s", library.Sdump(errHealthzLog), string(healthzLog))
+					require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"forbidden: User \"system:anonymous\" cannot get path \"/healthz/log\": decision made by impersonation-proxy.concierge.pinniped.dev","reason":"Forbidden","details":{},"code":403}`+"\n", string(healthzLog))
+				})
+			})
+
+			t.Run("anonymous authentication enabled", func(t *testing.T) {
+				library.IntegrationEnv(t).WithCapability(library.AnonymousAuthenticationSupported)
+				t.Parallel()
+
+				// anonymous auth enabled
+				// - hit the healthz endpoint (non-resource endpoint)
+				//   - through the impersonation proxy
+				//   - should succeed 200
+				//   - should respond "ok"
+				t.Run("non-resource request", func(t *testing.T) {
+					t.Parallel()
+
+					healthz, errHealth := impersonationProxyAnonymousRestClient.Get().AbsPath("/healthz").DoRaw(ctx)
+					require.NoError(t, errHealth, library.Sdump(errHealth))
+					require.Equal(t, "ok", string(healthz))
+				})
+
+				// - hit the pods endpoint (a resource endpoint)
+				//   - through the impersonation proxy
+				//   - should fail forbidden
+				//   - system:anonymous cannot get pods
+				t.Run("resource", func(t *testing.T) {
+					t.Parallel()
+
+					pod, err := impersonationProxyAnonymousClient.Kubernetes.CoreV1().Pods(metav1.NamespaceSystem).
+						Get(ctx, "does-not-matter", metav1.GetOptions{})
+					require.True(t, k8serrors.IsForbidden(err), library.Sdump(err))
+					require.EqualError(t, err, `pods "does-not-matter" is forbidden: User "system:anonymous" cannot get resource "pods" in API group "" in the namespace "kube-system": `+
+						`decision made by impersonation-proxy.concierge.pinniped.dev`, library.Sdump(err))
+					require.Equal(t, &corev1.Pod{}, pod)
+				})
+
+				// - request to whoami (pinniped resource endpoing)
+				//   - through the impersonation proxy
+				//   - should succeed 200
+				//   - should respond "you are system:anonymous"
+				t.Run("pinniped resource request", func(t *testing.T) {
+					t.Parallel()
+
+					whoAmI, err := impersonationProxyAnonymousClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
+						Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
+					require.NoError(t, err)
+					require.Equal(t,
+						expectedWhoAmIRequestResponse(
+							"system:anonymous",
+							[]string{"system:unauthenticated"},
+							nil,
+						),
+						whoAmI,
+					)
+				})
+			})
+
+			t.Run("anonymous authentication disabled", func(t *testing.T) {
+				library.IntegrationEnv(t).WithoutCapability(library.AnonymousAuthenticationSupported)
+				t.Parallel()
+
+				// - hit the healthz endpoint (non-resource endpoint)
+				//   - through the impersonation proxy
+				//   - should fail unauthorized
+				//   - kube api server should reject it
+				t.Run("non-resource request", func(t *testing.T) {
+					t.Parallel()
+
+					healthz, err := impersonationProxyAnonymousRestClient.Get().AbsPath("/healthz").DoRaw(ctx)
+					require.True(t, k8serrors.IsUnauthorized(err), library.Sdump(err))
+					require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401}`+"\n", string(healthz))
+				})
+
+				// - hit the pods endpoint (a resource endpoint)
+				//   - through the impersonation proxy
+				//   - should fail unauthorized
+				//   - kube api server should reject it
+				t.Run("resource", func(t *testing.T) {
+					t.Parallel()
+
+					pod, err := impersonationProxyAnonymousClient.Kubernetes.CoreV1().Pods(metav1.NamespaceSystem).
+						Get(ctx, "does-not-matter", metav1.GetOptions{})
+					require.True(t, k8serrors.IsUnauthorized(err), library.Sdump(err))
+					require.Equal(t, &corev1.Pod{}, pod)
+				})
+
+				// - request to whoami (pinniped resource endpoing)
+				//   - through the impersonation proxy
+				//   - should fail unauthorized
+				//   - kube api server should reject it
+				t.Run("pinniped resource request", func(t *testing.T) {
+					t.Parallel()
+
+					whoAmI, err := impersonationProxyAnonymousClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
+						Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
+					require.True(t, k8serrors.IsUnauthorized(err), library.Sdump(err))
+					require.Equal(t, &identityv1alpha1.WhoAmIRequest{}, whoAmI)
+				})
+			})
+		})
+	})
+
+	t.Run("assert correct impersonator service account is being used", func(t *testing.T) {
+		// pick an API that everyone can access but always make invalid requests to it
+		// we can tell that the request is reaching KAS because only it has the validation logic
+		impersonationProxySSRRClient := impersonationProxyKubeClient(t).AuthorizationV1().SelfSubjectRulesReviews()
+		crbClient := adminClient.RbacV1().ClusterRoleBindings()
+		impersonationProxyName := env.ConciergeAppName + "-impersonation-proxy"
+		saFullName := serviceaccount.MakeUsername(env.ConciergeNamespace, impersonationProxyName)
+		invalidSSRR := &authorizationv1.SelfSubjectRulesReview{}
+
+		// sanity check default expected error message
+		_, err := impersonationProxySSRRClient.Create(ctx, invalidSSRR, metav1.CreateOptions{})
+		require.True(t, k8serrors.IsBadRequest(err), library.Sdump(err))
+		require.EqualError(t, err, "no namespace on request")
+
+		// remove the impersonation proxy SA's permissions
+		crb, err := crbClient.Get(ctx, impersonationProxyName, metav1.GetOptions{})
+		require.NoError(t, err)
+
+		// sanity check the subject
+		require.Len(t, crb.Subjects, 1)
+		sub := crb.Subjects[0].DeepCopy()
+		require.Equal(t, &rbacv1.Subject{
+			Kind:      "ServiceAccount",
+			APIGroup:  "",
+			Name:      impersonationProxyName,
+			Namespace: env.ConciergeNamespace,
+		}, sub)
+
+		crb.Subjects = nil
+		_, err = crbClient.Update(ctx, crb, metav1.UpdateOptions{})
+		require.NoError(t, err)
+
+		// make sure to put the permissions back at the end
+		t.Cleanup(func() {
+			crbEnd, errEnd := crbClient.Get(ctx, impersonationProxyName, metav1.GetOptions{})
+			require.NoError(t, errEnd)
+
+			crbEnd.Subjects = []rbacv1.Subject{*sub}
+			_, errUpdate := crbClient.Update(ctx, crbEnd, metav1.UpdateOptions{})
+			require.NoError(t, errUpdate)
+
+			library.WaitForUserToHaveAccess(t, saFullName, nil, &authorizationv1.ResourceAttributes{
+				Verb:     "impersonate",
+				Resource: "users",
+			})
+		})
+
+		// assert that the impersonation proxy stops working when we remove its permissions
+		library.RequireEventuallyWithoutError(t, func() (bool, error) {
+			_, errCreate := impersonationProxySSRRClient.Create(ctx, invalidSSRR, metav1.CreateOptions{})
+
+			switch {
+			case errCreate == nil:
+				return false, fmt.Errorf("unexpected nil error for test user create invalid SSRR")
+
+			case k8serrors.IsBadRequest(errCreate) && errCreate.Error() == "no namespace on request":
+				t.Log("waiting for impersonation proxy service account to lose impersonate permissions")
+				return false, nil // RBAC change has not rolled out yet
+
+			case k8serrors.IsForbidden(errCreate) && errCreate.Error() ==
+				`users "`+env.TestUser.ExpectedUsername+`" is forbidden: User "`+saFullName+
+					`" cannot impersonate resource "users" in API group "" at the cluster scope`:
+				return true, nil // expected RBAC error
+
+			default:
+				return false, fmt.Errorf("unexpected error for test user create invalid SSRR: %w", errCreate)
+			}
+		}, time.Minute, time.Second)
 	})
 
 	t.Run("adding an annotation reconciles the LoadBalancer service", func(t *testing.T) {

test/integration/rbac_test.go

@@ -0,0 +1,146 @@
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package integration
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	authorizationv1 "k8s.io/api/authorization/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authentication/serviceaccount"
+	"k8s.io/apiserver/pkg/authentication/user"
+	v1 "k8s.io/client-go/kubernetes/typed/authorization/v1"
+	"k8s.io/client-go/rest"
+
+	"go.pinniped.dev/test/library"
+)
+
+func TestServiceAccountPermissions(t *testing.T) {
+	// TODO: update this test to check the permissions of all service accounts
+	//  For now it just checks the permissions of the impersonation proxy SA
+
+	env := library.IntegrationEnv(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+
+	// impersonate the SA since it is easier than fetching a token and lets us control the group memberships
+	config := rest.CopyConfig(library.NewClientConfig(t))
+	config.Impersonate = rest.ImpersonationConfig{
+		UserName: serviceaccount.MakeUsername(env.ConciergeNamespace, env.ConciergeAppName+"-impersonation-proxy"),
+		// avoid permissions assigned to system:serviceaccounts by explicitly impersonating system:serviceaccounts:<namespace>
+		// as not all clusters will have the system:service-account-issuer-discovery binding
+		// system:authenticated is required for us to create selfsubjectrulesreviews
+		// TODO remove this once we stop supporting Kube clusters before v1.19
+		Groups: []string{serviceaccount.MakeNamespaceGroupName(env.ConciergeNamespace), user.AllAuthenticated},
+	}
+
+	ssrrClient := library.NewKubeclient(t, config).Kubernetes.AuthorizationV1().SelfSubjectRulesReviews()
+
+	// the impersonation proxy SA has the same permissions for all checks because it should only be authorized via cluster role bindings
+
+	expectedResourceRules := []authorizationv1.ResourceRule{
+		// system:basic-user is bound to system:authenticated by default
+		{Verbs: []string{"create"}, APIGroups: []string{"authorization.k8s.io"}, Resources: []string{"selfsubjectaccessreviews", "selfsubjectrulesreviews"}},
+
+		// the expected impersonation permissions
+		{Verbs: []string{"impersonate"}, APIGroups: []string{""}, Resources: []string{"users", "groups", "serviceaccounts"}},
+		{Verbs: []string{"impersonate"}, APIGroups: []string{"authentication.k8s.io"}, Resources: []string{"*"}},
+
+		// we bind these to system:authenticated
+		{Verbs: []string{"create", "list"}, APIGroups: []string{"login.concierge." + env.APIGroupSuffix}, Resources: []string{"tokencredentialrequests"}},
+		{Verbs: []string{"create", "list"}, APIGroups: []string{"identity.concierge." + env.APIGroupSuffix}, Resources: []string{"whoamirequests"}},
+	}
+
+	if otherPinnipedGroupSuffix := getOtherPinnipedGroupSuffix(t); len(otherPinnipedGroupSuffix) > 0 {
+		expectedResourceRules = append(expectedResourceRules,
+			// we bind these to system:authenticated in the other instance of pinniped
+			authorizationv1.ResourceRule{Verbs: []string{"create", "list"}, APIGroups: []string{"login.concierge." + otherPinnipedGroupSuffix}, Resources: []string{"tokencredentialrequests"}},
+			authorizationv1.ResourceRule{Verbs: []string{"create", "list"}, APIGroups: []string{"identity.concierge." + otherPinnipedGroupSuffix}, Resources: []string{"whoamirequests"}},
+		)
+	}
+
+	expectedNonResourceRules := []authorizationv1.NonResourceRule{
+		// system:public-info-viewer is bound to system:authenticated and system:unauthenticated by default
+		{Verbs: []string{"get"}, NonResourceURLs: []string{"/healthz", "/livez", "/readyz", "/version", "/version/"}},
+
+		// system:discovery is bound to system:authenticated by default
+		{Verbs: []string{"get"}, NonResourceURLs: []string{"/api", "/api/*", "/apis", "/apis/*",
+			"/healthz", "/livez", "/openapi", "/openapi/*", "/readyz", "/version", "/version/",
+		}},
+	}
+
+	// check permissions in concierge namespace
+	testPermissionsInNamespace(ctx, t, ssrrClient, env.ConciergeNamespace, expectedResourceRules, expectedNonResourceRules)
+
+	// check permissions in supervisor namespace
+	testPermissionsInNamespace(ctx, t, ssrrClient, env.SupervisorNamespace, expectedResourceRules, expectedNonResourceRules)
+
+	// check permissions in kube-system namespace
+	testPermissionsInNamespace(ctx, t, ssrrClient, metav1.NamespaceSystem, expectedResourceRules, expectedNonResourceRules)
+
+	// check permissions in kube-public namespace
+	testPermissionsInNamespace(ctx, t, ssrrClient, metav1.NamespacePublic, expectedResourceRules, expectedNonResourceRules)
+
+	// check permissions in default namespace
+	testPermissionsInNamespace(ctx, t, ssrrClient, metav1.NamespaceDefault, expectedResourceRules, expectedNonResourceRules)
+
+	// we fake a cluster scoped selfsubjectrulesreviews check by picking a nonsense namespace
+	testPermissionsInNamespace(ctx, t, ssrrClient, "some-namespace-invalid-name||pandas-are-the-best", expectedResourceRules, expectedNonResourceRules)
+}
+
+func testPermissionsInNamespace(ctx context.Context, t *testing.T, ssrrClient v1.SelfSubjectRulesReviewInterface, namespace string,
+	expectedResourceRules []authorizationv1.ResourceRule, expectedNonResourceRules []authorizationv1.NonResourceRule) {
+	t.Helper()
+
+	ssrr, err := ssrrClient.Create(ctx, &authorizationv1.SelfSubjectRulesReview{
+		Spec: authorizationv1.SelfSubjectRulesReviewSpec{Namespace: namespace},
+	}, metav1.CreateOptions{})
+	assert.NoError(t, err)
+
+	assert.ElementsMatch(t, expectedResourceRules, ssrr.Status.ResourceRules)
+	assert.ElementsMatch(t, expectedNonResourceRules, ssrr.Status.NonResourceRules)
+}
+
+func getOtherPinnipedGroupSuffix(t *testing.T) string {
+	t.Helper()
+
+	env := library.IntegrationEnv(t)
+
+	var resources []*metav1.APIResourceList
+
+	library.RequireEventuallyWithoutError(t, func() (bool, error) {
+		// we need a complete discovery listing for the check we are trying to make below
+		// loop since tests like TestAPIServingCertificateAutoCreationAndRotation can break discovery
+		_, r, err := library.NewKubernetesClientset(t).Discovery().ServerGroupsAndResources()
+		if err != nil {
+			t.Logf("retrying due to partial discovery failure: %v", err)
+			return false, nil
+		}
+
+		resources = r
+		return true, nil
+	}, 3*time.Minute, time.Second)
+
+	var otherPinnipedGroupSuffix string
+
+	for _, resource := range resources {
+		gv, err := schema.ParseGroupVersion(resource.GroupVersion)
+		require.NoError(t, err)
+		for _, apiResource := range resource.APIResources {
+			if apiResource.Name == "tokencredentialrequests" && gv.Group != "login.concierge."+env.APIGroupSuffix {
+				require.Empty(t, otherPinnipedGroupSuffix, "only expected at most one other instance of pinniped")
+				otherPinnipedGroupSuffix = strings.TrimPrefix(gv.Group, "login.concierge.")
+			}
+		}
+	}
+
+	return otherPinnipedGroupSuffix
+}