Skip to content

Commit e59c1b3

Browse files
craig[bot]cthumuluru-crdb
craig[bot]
and
cthumuluru-crdb
committed
Merge #157993
157993: rpc, server: add DRPC support for node join RPC r=cthumuluru-crdb a=cthumuluru-crdb DRPC is currently disabled by default and can be enabled via the `rpc.experimental_drpc.enabled` cluster setting. Some code paths, such as command-line tools, don't have access to this setting. This change introduces a `--use-new-drpc` flag for start commands to enable DRPC in those cases. This change enables the use of DRPC for the node join RPC when the `--use-new-rpc` flag is set, and defaults to gRPC otherwise. Epic: CRDB-55200, CRDB-51459 Fixes: #155592, #149600 Release note: None Co-authored-by: Chandra Thumuluru <[email protected]>
2 parents cd899f9 + 2f25e7a commit e59c1b3

File tree

15 files changed

+289
-98
lines changed

15 files changed

+289
-98
lines changed

‎DEPS.bzl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10900,10 +10900,10 @@ def go_deps():
1090010900
name = "io_storj_drpc",
1090110901
build_file_proto_mode = "disable_global",
1090210902
importpath = "storj.io/drpc",
10903-
sha256 = "98b44a51f82873f93f77da80230212ab40f35044e8d38645cb1392ae03462f0b",
10904-
strip_prefix = "github.com/cockroachdb/[email protected]20250924114114-78d4e121902a",
10903+
sha256 = "3dbe010ef17497f2f3087f6d0ab565c1c469c0fc25ba031159fb107e6969715a",
10904+
strip_prefix = "github.com/cockroachdb/[email protected]20251122104303-6d3d35fe35b3",
1090510905
urls = [
10906-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250924114114-78d4e121902a.zip",
10906+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20251122104303-6d3d35fe35b3.zip",
1090710907
],
1090810908
)
1090910909
go_repository(

‎build/bazelutil/distdir_files.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ DISTDIR_FILES = {
344344
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlfmt/com_github_cockroachdb_crlfmt-v0.0.0-20221214225007-b2fc5c302548.zip": "fedc01bdd6d964da0425d5eaac8efadc951e78e13f102292cc0774197f09ab63",
345345
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20251122031428-fe658a2dbda1.zip": "fa361e52b072ce18ac1d103e2556851906477361b5fe688745d29b1efabff3cb",
346346
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/datadriven/com_github_cockroachdb_datadriven-v1.0.3-0.20251006155849-f84f9e519edd.zip": "a7ffcef0b264d9c28c36b2f9b737ff739542f472d7614938ae507e2da269f6c2",
347-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250924114114-78d4e121902a.zip": "98b44a51f82873f93f77da80230212ab40f35044e8d38645cb1392ae03462f0b",
347+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20251122104303-6d3d35fe35b3.zip": "3dbe010ef17497f2f3087f6d0ab565c1c469c0fc25ba031159fb107e6969715a",
348348
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/errors/com_github_cockroachdb_errors-v1.12.1-0.20251010171200-64801262cd6f.zip": "4df66cc44791d4290071696abf179dc6df7b94b4cb5d29a20f39c6bf522c60ee",
349349
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
350350
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gogoproto/com_github_cockroachdb_gogoproto-v1.3.3-0.20241216150617-2358cdb156a1.zip": "bf052c9a7f9e23fb3ec7e9f3b7201cfc264c18ed6da0d662952d276dbc339003",

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ replace github.com/docker/docker => github.com/moby/moby v24.0.6+incompatible
527527

528528
replace github.com/gogo/protobuf => github.com/cockroachdb/gogoproto v1.3.3-0.20241216150617-2358cdb156a1
529529

530-
replace storj.io/drpc => github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a
530+
replace storj.io/drpc => github.com/cockroachdb/drpc v0.0.0-20251122104303-6d3d35fe35b3
531531

532532
// Note: This forked dependency adds a commit that opens up some
533533
// private APIs to enable us to make some perf improvements to

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -556,8 +556,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
556556
github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
557557
github.com/cockroachdb/datadriven v1.0.3-0.20251006155849-f84f9e519edd h1:vpWCe7VvdQbQ/9wGtlH3i+Oj+9OggKci3lsASL1ydvg=
558558
github.com/cockroachdb/datadriven v1.0.3-0.20251006155849-f84f9e519edd/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g=
559-
github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a h1:zXCfk52Hpu2IoejmDm4Bkxmb5Nh9vxwaYOCiqA6f3YA=
560-
github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a/go.mod h1:Ag2/Yfl22WZ8ywFUasRQ2brdltpX5QvY63jnYTZ3N5U=
559+
github.com/cockroachdb/drpc v0.0.0-20251122104303-6d3d35fe35b3 h1:g9feJVqL6mBMZ/odZZsNl2XrsAxRaI6dP/y1PP+lf6s=
560+
github.com/cockroachdb/drpc v0.0.0-20251122104303-6d3d35fe35b3/go.mod h1:Ag2/Yfl22WZ8ywFUasRQ2brdltpX5QvY63jnYTZ3N5U=
561561
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f h1:lUmJxzb2/ukuRIvKTaNkvuj5LwlX4u/KxnI3zmx1SSw=
562562
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g=
563563
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 h1:YqzBA7tf8Gv8Oz0BbBsPenqkyjiohS7EUIwi7p1QJCU=

‎pkg/base/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,9 @@ type Config struct {
475475
// RPCHearbeatTimeout is the timeout for Ping requests.
476476
RPCHeartbeatTimeout time.Duration
477477

478+
// UseDRPC indicates whether to use DRPC as the RPC framework instead of gRPC.
479+
UseDRPC bool
480+
478481
// ApplicationInternalRPCPortMin/PortMax define the range of TCP ports
479482
// used to start the internal RPC service for application-level
480483
// servers. This service is used for node-to-node RPC traffic and to

‎pkg/cli/cliflags/flags.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -719,6 +719,14 @@ information.
719719
`,
720720
}
721721

722+
UseNewRPC = FlagInfo{
723+
Name: "use-new-rpc",
724+
Description: `
725+
Use the new RPC framework for internode communication instead of gRPC. This is
726+
a preview feature and is intended for non-production use only.
727+
`,
728+
}
729+
722730
LocalityAdvertiseAddr = FlagInfo{
723731
Name: "locality-advertise-addr",
724732
Description: `

‎pkg/cli/flags.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,9 @@ func init() {
464464

465465
cliflagcfg.BoolFlag(f, &serverCfg.AcceptProxyProtocolHeaders, cliflags.AcceptProxyProtocolHeaders)
466466

467+
cliflagcfg.BoolFlag(f, &baseCfg.UseDRPC, cliflags.UseNewRPC)
468+
_ = f.MarkHidden(cliflags.UseNewRPC.Name)
469+
467470
// Certificates directory. Use a server-specific flag and value to ignore environment
468471
// variables, but share the same default.
469472
cliflagcfg.StringFlag(f, &startCtx.serverSSLCertsDir, cliflags.ServerCertsDir)

‎pkg/cli/flags_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1889,3 +1889,37 @@ func TestWALFailoverWrapperRoundtrip(t *testing.T) {
18891889
return buf.String()
18901890
})
18911891
}
1892+
1893+
func TestUseNewRPC(t *testing.T) {
1894+
defer leaktest.AfterTest(t)()
1895+
defer log.Scope(t).Close(t)
1896+
1897+
// Avoid leaking configuration changes after the test ends.
1898+
defer initCLIDefaults()
1899+
1900+
testCases := []struct {
1901+
args []string
1902+
expectedVal bool
1903+
}{
1904+
{[]string{}, false}, // Default value
1905+
{[]string{"--use-new-rpc"}, true}, // Flag enabled
1906+
{[]string{"--use-new-rpc=true"}, true}, // Flag explicitly enabled
1907+
{[]string{"--use-new-rpc=false"}, false}, // Flag explicitly disabled
1908+
}
1909+
1910+
for _, tc := range testCases {
1911+
t.Run(fmt.Sprintf("%v", tc.args), func(t *testing.T) {
1912+
// Reset to defaults before each test
1913+
initCLIDefaults()
1914+
1915+
f := startCmd.Flags()
1916+
if err := f.Parse(tc.args); err != nil {
1917+
t.Fatal(err)
1918+
}
1919+
1920+
if baseCfg.UseDRPC != tc.expectedVal {
1921+
t.Errorf("expected UseDRPC=%v, but got %v", tc.expectedVal, baseCfg.UseDRPC)
1922+
}
1923+
})
1924+
}
1925+
}

‎pkg/rpc/BUILD.bazel‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,15 +82,13 @@ go_library(
8282
"@io_opentelemetry_go_otel//attribute",
8383
"@io_storj_drpc//:drpc",
8484
"@io_storj_drpc//drpcclient",
85-
"@io_storj_drpc//drpcconn",
8685
"@io_storj_drpc//drpcctx",
8786
"@io_storj_drpc//drpcmanager",
8887
"@io_storj_drpc//drpcmetadata",
8988
"@io_storj_drpc//drpcmigrate",
9089
"@io_storj_drpc//drpcmux",
9190
"@io_storj_drpc//drpcpool",
9291
"@io_storj_drpc//drpcserver",
93-
"@io_storj_drpc//drpcstream",
9492
"@io_storj_drpc//drpcwire",
9593
"@org_golang_google_grpc//:grpc",
9694
"@org_golang_google_grpc//backoff",

‎pkg/rpc/context.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,9 @@ type ContextOptions struct {
425425

426426
// Locality stores the locality of this node.
427427
Locality roachpb.Locality
428+
429+
// UseDRPC indicates if DRPC must be used for internode communication.
430+
UseDRPC bool
428431
}
429432

430433
// DefaultContextOptions are mostly used in tests.
@@ -458,6 +461,7 @@ func ServerContextOptionsFromBaseConfig(cfg *base.Config) ContextOptions {
458461
AdvertiseAddrH: &cfg.AdvertiseAddrH,
459462
SQLAdvertiseAddrH: &cfg.SQLAdvertiseAddrH,
460463
DisableTLSForHTTP: cfg.DisableTLSForHTTP,
464+
UseDRPC: cfg.UseDRPC,
461465
}
462466
}
463467

@@ -1451,6 +1455,17 @@ func (rpcCtx *Context) GRPCDialOptions(
14511455
return rpcCtx.grpcDialOptionsInternal(ctx, target, class, transport, onNetworkDial)
14521456
}
14531457

1458+
// DRPCDialOptions is same as GRPCDialOptions but for drpc connections.
1459+
func (rpcCtx *Context) DRPCDialOptions(
1460+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1461+
) ([]drpcclient.DialOption, error) {
1462+
transport := tcpTransport
1463+
if rpcCtx.ContextOptions.AdvertiseAddr == target && rpcCtx.canLoopbackDial() {
1464+
transport = loopbackTransport
1465+
}
1466+
return rpcCtx.drpcDialOptionsInternal(ctx, target, class, transport)
1467+
}
1468+
14541469
// grpcDialOptions produces dial options suitable for connecting to the given target and class.
14551470
func (rpcCtx *Context) grpcDialOptionsInternal(
14561471
ctx context.Context,
@@ -2102,6 +2117,34 @@ func (rpcCtx *Context) grpcDialRaw(
21022117
return grpc.DialContext(ctx, target, dialOpts...)
21032118
}
21042119

2120+
// drpcDialRaw is similar to grpcDialRaw but for drpc connections.
2121+
//
2122+
//lint:ignore U1000 used in the future commits.
2123+
func (rpcCtx *Context) drpcDialRaw(
2124+
ctx context.Context,
2125+
target string,
2126+
class rpcbase.ConnectionClass,
2127+
additionalOpts ...drpcclient.DialOption,
2128+
) (*drpcclient.ClientConn, error) {
2129+
transport := tcpTransport
2130+
if rpcCtx.ContextOptions.AdvertiseAddr == target && rpcCtx.canLoopbackDial() {
2131+
transport = loopbackTransport
2132+
}
2133+
drpcDialOpts, err := rpcCtx.drpcDialOptionsInternal(ctx, target, class, transport)
2134+
if err != nil {
2135+
return nil, err
2136+
}
2137+
2138+
drpcDialOpts = append(drpcDialOpts, additionalOpts...)
2139+
2140+
drpcConn, err := drpcclient.DialContext(ctx, target, drpcDialOpts...)
2141+
if err != nil {
2142+
return nil, err
2143+
}
2144+
2145+
return drpcclient.NewClientConnWithOptions(ctx, drpcConn, drpcDialOpts...)
2146+
}
2147+
21052148
// GRPCUnvalidatedDial uses GRPCDialNode and disables validation of the
21062149
// node ID between client and server. This function should only be
21072150
// used with the gossip client and CLI commands which can talk to any

0 commit comments

Comments
 (0)