Merge #155216

155216: cli, security: disable root user and auth changes for debug zip user r=souravcrl a=souravcrl

**cli, security: add --disallow-root-login flag to disable root user**
We introduce a `--disallow-root-login` flag which now disables root user
 from access to both sql and rpc apis.

**rpc: enable debugzip user for privileged access**
This commit introduces support for the debug_user certificate
as a privileged user for RPC authentication, similar to the existing
root and node users. The debug_user is specifically designed
for collecting debug information (debug zip) and requires privileged
access to serverpb admin and status endpoints.

Modified `pkg/rpc/auth.go` to allow `debug_user` scope in
`checkRootOrNodeInScope()`, treating it as a privileged user
alongside root and node. Enhanced `pkg/rpc/auth_test.go` with
comprehensive test cases for `debug_user` authentication and
authorization across various scenarios.

The `debug_user` is not subject to the disallow-root-login flag and
should always be allowed for debugging purposes.

Release note (security update): 
We will be adding a new flag `--disallow-root-login` to the cockroach start 
command to explicitly allowcrestricting the root user from logging into the system. 
This change affects the [unstated, unchangeable root access rule](https://www.cockroachlabs.com/docs/stable/security-reference/authentication#the-unstated-unchangeable-root-access-rule) as part of
compliance requirements.  This flag is currently experimental and also needs an
additional user setup for debug zip collection as disabling the root user affects 
the debug zip service. We currently do not validate if this user is set up or not.

Note: Care must be taken to ensure none of the certificates that are in use by
the cluster or the SQL/RPC clients have a root in the SAN fields since the flag
will block access to that client.

A new `debug_user` certificate can now be used for privileged RPC access 
to collect debug information. The `debug_user` must be created manually using
the `CREATE USER` command and can be audited using the `SHOW USERS` command.
This user has privileged access to `serverpb` admin and status endpoints required for
debug zip collection.

Fixes: #150845, #152817

Epic: CRDB-49035

Co-authored-by: souravcrl <[email protected]>

Author: craig[bot]

pkg/cli/cliflags/flags.go

@@ -950,6 +950,22 @@ provided for node user if this flag is set.
 `,
 	}
 
+	DisallowRootLogin = FlagInfo{
+		Name: "disallow-root-login",
+		Description: `
+When set, prevents authentication attempts by clients presenting certificates
+with "root" as one of the principals (CommonName or SubjectAlternativeName).
+This applies to both SQL client connections and RPC connections. Authentication
+attempts by root will be rejected with an error.
+<PRE>
+
+</PRE>
+Note: Please ensure none of the certificates that are in use by the cluster or
+the SQL/RPC clients have a root in the SAN fields since the flag will block
+access to that client.
+`,
+	}
+
 	TLSCipherSuites = FlagInfo{
 		Name: "tls-cipher-suites",
 		Description: `

pkg/cli/context.go

@@ -488,6 +488,7 @@ var startCtx struct {
 	serverRootCertDN       string
 	serverNodeCertDN       string
 	serverTLSCipherSuites  []string
+	disallowRootLogin      bool
 
 	// The TLS auto-handshake parameters.
 	initToken             string
@@ -543,6 +544,7 @@ func setStartContextDefaults() {
 	startCtx.serverCertPrincipalMap = nil
 	startCtx.serverRootCertDN = ""
 	startCtx.serverNodeCertDN = ""
+	startCtx.disallowRootLogin = false
 	startCtx.serverListenAddr = ""
 	startCtx.unencryptedLocalhostHTTP = false
 	startCtx.tempDir = ""

pkg/cli/flags.go

@@ -562,6 +562,15 @@ func init() {
 		// Node cert distinguished name
 		cliflagcfg.StringFlag(f, &startCtx.serverNodeCertDN, cliflags.NodeCertDistinguishedName)
 
+		// We add the disallow root login flag for disabling the root user from rpc
+		// and sql access for compliance reasons. We currently mark it as hidden
+		// since the flag behavior is experimental and subject to change.
+		//
+		// NB: a user needs to be configured for collecting debug zips if this
+		// flag is enabled, which we currently do not validate.
+		cliflagcfg.BoolFlag(f, &startCtx.disallowRootLogin, cliflags.DisallowRootLogin)
+		_ = f.MarkHidden(cliflags.DisallowRootLogin.Name)
+
 		// TLS Cipher Suites configured
 		cliflagcfg.StringSliceFlag(f, &startCtx.serverTLSCipherSuites, cliflags.TLSCipherSuites)
 
@@ -1151,6 +1160,7 @@ func extraServerFlagInit(cmd *cobra.Command) error {
 	if err := security.SetNodeSubject(startCtx.serverNodeCertDN); err != nil {
 		return err
 	}
+	security.SetDisallowRootLogin(startCtx.disallowRootLogin)
 	// Currently we don't handle the case where we are setting the --insecure flag
 	// as well as providing the --tls-cipher-suites, we should probably error out
 	// if both are set, issue: #144935.
@@ -1160,6 +1170,7 @@ func extraServerFlagInit(cmd *cobra.Command) error {
 	serverCfg.User = username.NodeUserName()
 	serverCfg.Insecure = startCtx.serverInsecure
 	serverCfg.SSLCertsDir = startCtx.serverSSLCertsDir
+	serverCfg.DisallowRootLogin = startCtx.disallowRootLogin
 
 	fs := cliflagcfg.FlagSetForCmd(cmd)
 

pkg/rpc/BUILD.bazel

@@ -148,6 +148,7 @@ go_test(
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",
         "//pkg/security/username",
+        "//pkg/server/serverpb",
         "//pkg/settings",
         "//pkg/settings/cluster",
         "//pkg/spanconfig",

pkg/rpc/auth.go

@@ -490,21 +490,36 @@ func (a kvAuth) selectAuthzMethod(ar authnResult) (requiredAuthzMethod, error) {
 // checkRootOrNodeInScope checks that the root or node principals are
 // present in the cert user scopes.
 func checkRootOrNodeInScope(clientCert *x509.Certificate, serverTenantID roachpb.TenantID) error {
+	rootLoginDisabledValidate := false
+	debugUserScopedCert := false
+
 	containsFn := func(scope security.CertificateUserScope) bool {
 		// Only consider global scopes or scopes that match this server.
 		if !(scope.Global || scope.TenantID == serverTenantID) {
 			return false
 		}
 
+		if security.CheckRootLoginDisallowed() && scope.Username == username.RootUser {
+			rootLoginDisabledValidate = true
+		}
+
 		// If we get a scope that matches the Node user, immediately return.
 		if scope.Username == username.NodeUser || scope.Username == username.RootUser {
 			return true
 		}
 
+		if scope.Username == username.DebugUser {
+			debugUserScopedCert = true
+		}
+
 		return false
 	}
 	ok, err := security.CertificateUserScopeContainsFunc(clientCert, containsFn)
-	if ok || err != nil {
+	if ok || err != nil || debugUserScopedCert {
+		if rootLoginDisabledValidate {
+			return authError("failed to perform RPC, as root login has been disallowed")
+		}
+
 		return err
 	}
 	certUserScope, err := security.GetCertificateUserScope(clientCert)

pkg/rpc/auth_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/security"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/spanconfig"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
@@ -137,13 +138,15 @@ func testAuthenticateTenant(t *testing.T, enableDRPC bool) {
 		{systemID: stid, ous: append([]string{"foo"}, correctOU...), commonName: "other", expErr: `could not parse tenant ID from Common Name`},
 		{systemID: stid, ous: nil, commonName: "root"},
 		{systemID: stid, ous: nil, commonName: "node"},
+		{systemID: stid, ous: nil, commonName: "debug_user"},
 		{systemID: stid, ous: nil, commonName: "root", tenantScope: 10,
 			expErr: `need root or node client cert to perform RPCs on this server \(this is tenant system; cert is valid for "root" on tenantID 10\)`},
 		{systemID: tenTen, ous: correctOU, commonName: "10", expTenID: roachpb.TenantID{}},
 		{systemID: tenTen, ous: correctOU, commonName: "123", expErr: `client tenant identity \(123\) does not match server`},
 		{systemID: tenTen, ous: correctOU, commonName: "1", expErr: `invalid tenant ID 1 in Common Name \(CN\)`},
 		{systemID: tenTen, ous: nil, commonName: "root"},
 		{systemID: tenTen, ous: nil, commonName: "node"},
+		{systemID: tenTen, ous: nil, commonName: "debug_user"},
 
 		// Passing a client ID in metadata instead of relying only on the TLS cert.
 		{clientTenantInMD: "invalid", expErr: `could not parse tenant ID from (gRPC|drpc) metadata`},
@@ -168,13 +171,17 @@ func testAuthenticateTenant(t *testing.T, enableDRPC bool) {
 		// Server is KV node: expect tenant authorization.
 		{clientTenantInMD: "10",
 			systemID: stid, ous: nil, commonName: "node", expTenID: tenTen},
+		{clientTenantInMD: "10",
+			systemID: stid, ous: nil, commonName: "debug_user", expTenID: tenTen},
 		// tenant ID present in MD, but not in client cert. However,
 		// client cert is valid. Use MD tenant ID.
 		// Server is secondary tenant: do not do additional tenant authorization.
 		{clientTenantInMD: "10",
 			systemID: tenTen, ous: nil, commonName: "root", expTenID: roachpb.TenantID{}},
 		{clientTenantInMD: "10",
 			systemID: tenTen, ous: nil, commonName: "node", expTenID: roachpb.TenantID{}},
+		{clientTenantInMD: "10",
+			systemID: tenTen, ous: nil, commonName: "debug_user", expTenID: roachpb.TenantID{}},
 		// tenant ID present in MD, but not in client cert. Use MD tenant ID.
 		// Server tenant ID does not match client tenant ID.
 		{clientTenantInMD: "123",
@@ -206,8 +213,17 @@ func testAuthenticateTenant(t *testing.T, enableDRPC bool) {
 		{systemID: stid, ous: nil, commonName: "foo", subjectRequired: true, nodeDNString: "CN=foo"},
 		{systemID: stid, ous: nil, commonName: "foo", subjectRequired: true,
 			rootDNString: "CN=foo", nodeDNString: "CN=bar"},
+		// Test case for disallow root login functionality
+		{systemID: stid, ous: nil, commonName: "root", expErr: "root login has been disallowed"},
+		{systemID: stid, ous: nil, commonName: "foo", certDNSName: "root", expErr: "root login has been disallowed"},
 	} {
 		t.Run(fmt.Sprintf("from %v to %v (md %q)", tc.commonName, tc.systemID, tc.clientTenantInMD), func(t *testing.T) {
+			// Enable root login blocking for the specific test case
+			if tc.expErr == "root login has been disallowed" {
+				security.SetDisallowRootLogin(true)
+				defer security.SetDisallowRootLogin(false)
+			}
+
 			err := security.SetCertPrincipalMap(strings.Split(tc.certPrincipalMap, ","))
 			if err != nil {
 				t.Fatal(err)
@@ -1272,3 +1288,125 @@ func (m mockAuthorizer) IsExemptFromRateLimiting(context.Context, roachpb.Tenant
 }
 
 type contextKey struct{}
+
+// TestServerpbEndpointAccess ensures root user access to serverpb admin and status endpoints
+// works correctly with authentication (considering disallow-root-login flag) and authorization.
+func TestServerpbEndpointAccess(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+
+	systemTenantID := roachpb.SystemTenantID
+	const noError = ""
+
+	// Helper to create a certificate with a given common name
+	createCert := func(t *testing.T, commonName string) *x509.Certificate {
+		cert := &x509.Certificate{
+			Subject: pkix.Name{
+				CommonName: commonName,
+			},
+		}
+		var err error
+		cert.RawSubject, err = asn1.Marshal(cert.Subject.ToRDNSequence())
+		require.NoError(t, err)
+		return cert
+	}
+
+	// Helper to create a context with peer certificate
+	createContextWithCert := func(cert *x509.Certificate) context.Context {
+		tlsInfo := credentials.TLSInfo{
+			State: tls.ConnectionState{
+				PeerCertificates: []*x509.Certificate{cert},
+			},
+		}
+		p := peer.Peer{AuthInfo: tlsInfo}
+		return peer.NewContext(context.Background(), &p)
+	}
+
+	t.Run("authentication", func(t *testing.T) {
+		// Test root user authentication with disallow-root-login flag
+		for _, disallowRoot := range []bool{false, true} {
+			testName := "disallow_root_login_disabled"
+			expectedError := noError
+			if disallowRoot {
+				testName = "disallow_root_login_enabled"
+				expectedError = "failed to perform RPC, as root login has been disallowed"
+			}
+
+			t.Run(testName, func(t *testing.T) {
+				if disallowRoot {
+					security.SetDisallowRootLogin(true)
+					defer security.SetDisallowRootLogin(false)
+				}
+
+				cert := createCert(t, "root")
+				ctx := createContextWithCert(cert)
+
+				sv := &settings.Values{}
+				sv.Init(ctx, settings.TestOpaque)
+
+				// Perform authentication - this is where the root login check happens
+				_, err := rpc.TestingAuthenticateTenant(ctx, systemTenantID, sv, false /* enableDRPC */)
+
+				if expectedError == noError {
+					require.NoError(t, err)
+				} else {
+					require.Error(t, err)
+					require.Equal(t, codes.Unauthenticated, status.Code(err))
+					require.Regexp(t, expectedError, err)
+				}
+			})
+		}
+
+		// Test debug_user authentication - should always succeed
+		t.Run("debug_user_authentication", func(t *testing.T) {
+			cert := createCert(t, "debug_user")
+			ctx := createContextWithCert(cert)
+
+			sv := &settings.Values{}
+			sv.Init(ctx, settings.TestOpaque)
+
+			// Perform authentication - debug_user should always be allowed
+			_, err := rpc.TestingAuthenticateTenant(ctx, systemTenantID, sv, false /* enableDRPC */)
+			require.NoError(t, err)
+		})
+	})
+
+	t.Run("authorization", func(t *testing.T) {
+		// Test root and debug_user authorization to access serverpb endpoints
+		endpoints := map[string]interface{}{
+			"/cockroach.server.serverpb.Admin/Liveness":      &serverpb.LivenessRequest{},
+			"/cockroach.server.serverpb.Status/Nodes":        &serverpb.NodesRequest{},
+			"/cockroach.server.serverpb.Status/TenantRanges": &serverpb.TenantRangesRequest{},
+			"/cockroach.server.serverpb.Status/Gossip":       &serverpb.GossipRequest{},
+			"/cockroach.server.serverpb.Status/Ranges":       &serverpb.RangesRequest{},
+			"/cockroach.server.serverpb.Status/EngineStats":  &serverpb.EngineStatsRequest{},
+		}
+
+		for method, req := range endpoints {
+			t.Run(strings.ReplaceAll(method, "/", "_"), func(t *testing.T) {
+				// Test both root and debug_user
+				for _, user := range []string{"root", "debug_user"} {
+					t.Run(user, func(t *testing.T) {
+						cert := createCert(t, user)
+						ctx := createContextWithCert(cert)
+
+						sv := &settings.Values{}
+						sv.Init(ctx, settings.TestOpaque)
+
+						// Test authorization with mock authorizer
+						err := rpc.TestingAuthorizeTenantRequest(ctx, sv, systemTenantID, method, req, mockAuthorizer{
+							hasCrossTenantRead:                 true,
+							hasCapabilityForBatch:              true,
+							hasNodestatusCapability:            true,
+							hasTSDBQueryCapability:             true,
+							hasNodelocalStorageCapability:      true,
+							hasExemptFromRateLimiterCapability: true,
+							hasTSDBAllCapability:               true,
+						})
+
+						require.NoError(t, err)
+					})
+				}
+			})
+		}
+	})
+}

pkg/security/auth.go

@@ -67,6 +67,12 @@ func (c *userCertDistinguishedNameMu) setDNWithString(dnString string) error {
 
 var rootSubjectMu, nodeSubjectMu userCertDistinguishedNameMu
 
+// disallowRootLoginMu controls whether root login should be disallowed.
+var disallowRootLoginMu struct {
+	syncutil.RWMutex
+	disallowed bool
+}
+
 func SetRootSubject(rootDNString string) error {
 	return rootSubjectMu.setDNWithString(rootDNString)
 }
@@ -83,6 +89,20 @@ func UnsetNodeSubject() {
 	nodeSubjectMu.unsetDN()
 }
 
+// SetDisallowRootLogin sets whether root login should be disallowed.
+func SetDisallowRootLogin(disallow bool) {
+	disallowRootLoginMu.Lock()
+	defer disallowRootLoginMu.Unlock()
+	disallowRootLoginMu.disallowed = disallow
+}
+
+// CheckRootLoginDisallowed returns whether root login is currently disallowed.
+func CheckRootLoginDisallowed() bool {
+	disallowRootLoginMu.RLock()
+	defer disallowRootLoginMu.RUnlock()
+	return disallowRootLoginMu.disallowed
+}
+
 // CertificateUserScope indicates the scope of a user certificate i.e. which
 // tenant the user is allowed to authenticate on. Older client certificates
 // without a tenant scope are treated as global certificates which can
@@ -490,6 +510,11 @@ func ValidateUserScope(
 		return roleSubject.Equal(certSubject)
 	}
 	for _, scope := range certUserScope {
+		// Check if root login is disallowed and certificate contains "root" as a principal
+		if CheckRootLoginDisallowed() && scope.Username == username.RootUser {
+			return false
+		}
+
 		if scope.Username == user {
 			// If username matches, allow authentication to succeed if
 			// the tenantID is a match or if the certificate scope is global.