Merge pull request #151866 from rail/backport24.3-146755-147973

Author: rail

.bazelrc

@@ -92,32 +92,32 @@ build:cross --stamp
 # `--workspace_status_command`; if using these `base` configs, you need to
 # specify an appropriate `--workspace_status_command`. These `base` configs are
 # used by the release process which needs to have more control over stamping.
-build:crosslinux '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu'
+build:crosslinux '--workspace_status_command=./build/bazelutil/stamp.sh -t x86_64-pc-linux-gnu'
 build:crosslinux --config=crosslinuxbase
 build:crosslinuxbase --platforms=//build/toolchains:cross_linux
 build:crosslinuxbase --config=cross
-build:crosslinuxfips '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu'
+build:crosslinuxfips '--workspace_status_command=./build/bazelutil/stamp.sh -t x86_64-pc-linux-gnu'
 build:crosslinuxfips --config=crosslinuxfipsbase
 build:crosslinuxfipsbase --platforms=//build/toolchains:cross_linux
 build:crosslinuxfipsbase --config=cross
 build:crosslinuxfipsbase --@io_bazel_rules_go//go/toolchain:sdk_version=1.22.12fips
-build:crosswindows '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32'
+build:crosswindows '--workspace_status_command=./build/bazelutil/stamp.sh -t x86_64-w64-mingw32'
 build:crosswindows --config=crosswindowsbase
 build:crosswindowsbase --platforms=//build/toolchains:cross_windows
 build:crosswindowsbase --config=cross
-build:crossmacos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin21.2'
+build:crossmacos '--workspace_status_command=./build/bazelutil/stamp.sh -t x86_64-apple-darwin21.2'
 build:crossmacos --config=crossmacosbase
 build:crossmacosbase --platforms=//build/toolchains:cross_macos
 build:crossmacosbase --config=cross
-build:crossmacosarm '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-apple-darwin21.2'
+build:crossmacosarm '--workspace_status_command=./build/bazelutil/stamp.sh -t aarch64-apple-darwin21.2'
 build:crossmacosarm --config=crossmacosarmbase
 build:crossmacosarmbase --platforms=//build/toolchains:cross_macos_arm
 build:crossmacosarmbase --config=cross
-build:crosslinuxarm '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu'
+build:crosslinuxarm '--workspace_status_command=./build/bazelutil/stamp.sh -t aarch64-unknown-linux-gnu'
 build:crosslinuxarm --config=crosslinuxarmbase
 build:crosslinuxarmbase --platforms=//build/toolchains:cross_linux_arm
 build:crosslinuxarmbase --config=cross
-build:crosslinuxs390x '--workspace_status_command=./build/bazelutil/stamp.sh s390x-unknown-linux-gnu'
+build:crosslinuxs390x '--workspace_status_command=./build/bazelutil/stamp.sh -t s390x-unknown-linux-gnu'
 build:crosslinuxs390x --config=crosslinuxs390xbase
 build:crosslinuxs390xbase --platforms=//build/toolchains:cross_linux_s390x
 build:crosslinuxs390xbase --config=cross

build/bazelutil/stamp.sh

@@ -9,14 +9,15 @@
 # This command is used by bazel as the workspace_status_command
 # to implement build stamping with git information.
 
-# Usage: stamp.sh [target-triple] [build-channel] [build-type]
+# Usage: stamp.sh [-t target-triple] [-c build-channel] [-b build-type] [-g build-tag] [-d telemetry-disabled]
 # All arguments are optional and have appropriate defaults. In this way,
 # stamp.sh with no arguments is appropriate as the `workplace_status_command`
 # for a development build.
-#  target-triple: defaults to the value of `cc -dumpmachine`
-#  build-channel: defaults to `unknown`, but can be `official-binary`
-#  build-type: defaults to `development`, but can be `release`
-#  build-tag: will default to an appropriate value if not passed in, but can be overridden
+#  -t target-triple: defaults to the value of `cc -dumpmachine`
+#  -c build-channel: defaults to `unknown`, but can be `official-binary`
+#  -b build-type: defaults to `development`, but can be `release`
+#  -g build-tag: will default to an appropriate value if not passed in, but can be overridden
+#  -d telemetry-disabled: defaults to `false`, but can be set to `true`.
 
 set -euo pipefail
 
@@ -29,56 +30,38 @@ set -euo pipefail
 # For details, see the "Possible timestamp problems with diff-files?" thread on
 # the Git mailing list (http://marc.info/?l=git&m=131687596307197).
 
-# Handle target-triple.
-if [ -z "${1+x}" ]
-then
-    TARGET_TRIPLE=$(cc -dumpmachine)
-else
-    TARGET_TRIPLE="$1"
-    shift 1
-fi
+# Default values
+TARGET_TRIPLE="$(cc -dumpmachine)"
+BUILD_CHANNEL="unknown"
+BUILD_TYPE="development"
+BUILD_TAG=""
+TELEMETRY_DISABLED=false
 
-# Handle build-channel.
-if [ -z "${1+x}" ]
-then
-    BUILD_CHANNEL="unknown"
-else
-    BUILD_CHANNEL="$1"
-    shift 1
-fi
-
-# Handle build-type.
-if [ -z "${1+x}" ]
-then
-    BUILD_TYPE="development"
-else
-    BUILD_TYPE="$1"
-    shift 1
-fi
+# Parse command line arguments
+while getopts "t:c:b:g:d:" opt; do
+  case $opt in
+    t) TARGET_TRIPLE="$OPTARG" ;;
+    c) BUILD_CHANNEL="$OPTARG" ;;
+    b) BUILD_TYPE="$OPTARG" ;;
+    g) BUILD_TAG="$OPTARG" ;;
+    d) TELEMETRY_DISABLED="$OPTARG" ;;
+    \?) echo "Invalid option -$OPTARG" >&2; exit 1 ;;
+  esac
+done
 
 if [ "$BUILD_TYPE" = "release" ]
 then
-    CRASH_REPORT_ENV=$(cat ./pkg/build/version.txt)
+    CRASH_REPORT_ENV="$(cat ./pkg/build/version.txt)"
 else
     CRASH_REPORT_ENV="development"
 fi
 
-# Handle build-tag.
-if [ -z "${1+x}" ]
-then
-    BUILD_TAG=
-else
-    BUILD_TAG="$1"
-    shift 1
-fi
-
-BUILD_REV=$(git describe --match="" --always --abbrev=40)
+BUILD_REV="$(git describe --match="" --always --abbrev=40)"
 if [[ -n "$(git status -s --ignore-submodules --untracked-files=no)" ]]; then
-    BUILD_REV=$BUILD_REV-dirty
+    BUILD_REV="$BUILD_REV-dirty"
 fi
 
-BUILD_UTCTIME=$(date -u '+%Y/%m/%d %H:%M:%S')
-
+BUILD_UTCTIME="$(date -u '+%Y/%m/%d %H:%M:%S')"
 
 # Variables beginning with "STABLE" will be written to stable-status.txt, and
 # others will be written to volatile-status.txt.
@@ -88,11 +71,12 @@ BUILD_UTCTIME=$(date -u '+%Y/%m/%d %H:%M:%S')
 # * https://docs.bazel.build/versions/main/user-manual.html#workspace_status
 # * https://github.com/bazelbuild/rules_go/blob/master/go/core.rst#defines-and-stamping
 cat <<EOF
-STABLE_BUILD_CHANNEL ${BUILD_CHANNEL-}
-STABLE_BUILD_TAG ${BUILD_TAG-}
-STABLE_BUILD_TARGET_TRIPLE ${TARGET_TRIPLE-}
-STABLE_BUILD_TYPE ${BUILD_TYPE-}
-STABLE_CRASH_REPORT_ENV ${CRASH_REPORT_ENV-}
-BUILD_REV ${BUILD_REV-}
-BUILD_UTCTIME ${BUILD_UTCTIME-}
+STABLE_BUILD_CHANNEL $BUILD_CHANNEL
+STABLE_BUILD_TAG $BUILD_TAG
+STABLE_BUILD_TARGET_TRIPLE $TARGET_TRIPLE
+STABLE_BUILD_TYPE $BUILD_TYPE
+STABLE_CRASH_REPORT_ENV $CRASH_REPORT_ENV
+STABLE_TELEMETRY_DISABLED $TELEMETRY_DISABLED
+BUILD_REV $BUILD_REV
+BUILD_UTCTIME $BUILD_UTCTIME
 EOF

build/release/teamcity-support.sh

@@ -66,6 +66,7 @@ verify_docker_image(){
   local expected_sha=$3
   local expected_build_tag=$4
   local fips_build=$5
+  local telemetry_disabled=$6
   local error=0
 
   docker rmi "$img" || true
@@ -107,6 +108,18 @@ verify_docker_image(){
       error=1
     fi
   fi
+  if [[ $docker_platform == "linux/amd64" ]]; then
+    # Running arm64 `cockroach demo` on amd64 times out.
+    telemetry_setting="$(docker run -t --platform="$docker_platform" "$img" demo --no-example-database -e 'SHOW CLUSTER SETTING diagnostics.reporting.enabled;' --format json | grep "diagnostics.reporting.enabled" | cut -d: -f2 | cut -d'"' -f2)"
+    if [[ $telemetry_disabled == true && $telemetry_setting != "f" ]]; then
+      echo "ERROR: expected telemetry to be disabled, but it is enabled"
+      error=1
+    fi
+    if [[ $telemetry_disabled == false && $telemetry_setting != "t" ]]; then
+      echo "ERROR: expected telemetry to be enabled, but it is disabled"
+      error=1
+    fi
+  fi
   return $error
 }
 

build/teamcity/internal/cockroach/release/publish/publish-staged-cockroach-release-no-telemetry.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+
+set -euxo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname $(dirname "${0}"))))))"
+source "$dir/teamcity-support.sh"  # For log_into_gcloud
+
+tc_start_block "Variable Setup"
+version=$(grep -v "^#" "$dir/../pkg/build/version.txt" | head -n1)
+cockroach_archive_prefix="${COCKROACH_ARCHIVE_PREFIX:?COCKROACH_ARCHIVE_PREFIX must be set and not set to 'cockroach'}"
+if [[ $cockroach_archive_prefix == "cockroach" ]]; then
+  echo "COCKROACH_ARCHIVE_PREFIX must be set to a non-default value"
+  exit 1
+fi
+
+if [[ -z "${DRY_RUN}" ]] ; then
+  # TODO: use different buckets here maybe?
+  gcs_bucket="cockroach-release-artifacts-prod"
+  gcs_staged_bucket="cockroach-release-artifacts-staged-prod"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_PROD"
+else
+  gcs_bucket="cockroach-release-artifacts-dryrun"
+  gcs_staged_bucket="cockroach-release-artifacts-staged-dryrun"
+  # export the variable to avoid shell escaping
+  export gcs_credentials="$GCS_CREDENTIALS_DEV"
+fi
+tc_end_block "Variable Setup"
+
+
+tc_start_block "Copy binaries"
+log_into_gcloud
+for platform in linux-amd64 linux-arm64; do
+  archive="${cockroach_archive_prefix}-${version}.${platform}.tgz"
+  gsutil cp "gs://$gcs_staged_bucket/$archive" "gs://$gcs_bucket/$archive"
+  gsutil cp "gs://$gcs_staged_bucket/$archive.sha256sum" "gs://$gcs_bucket/$archive.sha256sum"
+done
+tc_end_block "Copy binaries"

build/teamcity/internal/cockroach/release/publish/publish-staged-cockroach-release.sh

@@ -71,7 +71,7 @@ tc_start_block "Verify binaries SHA"
 # Make sure that the linux/amd64 source docker image is built using the same version and SHA. 
 # This is a quick check and it assumes that the docker image was built correctly and based on the tarball binaries.
 docker_login_gcr "$gcr_staged_repository" "$gcr_staged_credentials"
-verify_docker_image "${gcr_staged_repository}:${version}" "linux/amd64" "$BUILD_VCS_NUMBER" "$version" false
+verify_docker_image "${gcr_staged_repository}:${version}" "linux/amd64" "$BUILD_VCS_NUMBER" "$version" false false
 tc_end_block "Verify binaries SHA"
 
 tc_start_block "Check remote tag and tag"
@@ -242,7 +242,7 @@ fi
 for img in "${images[@]}"; do
   for platform_name in amd64 arm64; do
     tc_start_block "Verify $img on $platform_name"
-    if ! verify_docker_image "$img" "linux/$platform_name" "$BUILD_VCS_NUMBER" "$version" false; then
+    if ! verify_docker_image "$img" "linux/$platform_name" "$BUILD_VCS_NUMBER" "$version" false false; then
       error=1
     fi
     tc_end_block "Verify $img on $platform_name"
@@ -252,7 +252,7 @@ done
 images=("${dockerhub_tag_fips}" "${gcr_tag_fips}")
 for img in "${images[@]}"; do
   tc_start_block "Verify $img"
-  if ! verify_docker_image "$img" "linux/amd64" "$BUILD_VCS_NUMBER" "$version" true; then
+  if ! verify_docker_image "$img" "linux/amd64" "$BUILD_VCS_NUMBER" "$version" true false; then
     error=1
   fi
   tc_end_block "Verify $img"

build/teamcity/internal/release/process/build-cockroach-release-cloud-only.sh

@@ -79,7 +79,7 @@ tc_start_block "Verify docker images"
 error=0
 for arch in amd64 arm64; do
     tc_start_block "Verify $manifest on $arch"
-    if ! verify_docker_image "$manifest" "linux/$arch" "$BUILD_VCS_NUMBER" "$version" false; then
+    if ! verify_docker_image "$manifest" "linux/$arch" "$BUILD_VCS_NUMBER" "$version" false false; then
       error=1
     fi
     tc_end_block "Verify $manifest on $arch"

build/teamcity/internal/release/process/build-cockroach-release-docker-no-telemetry.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+
+TELEMETRY_DISABLED=true ./build/teamcity/internal/release/process/build-cockroach-release-docker.sh

build/teamcity/internal/release/process/build-cockroach-release-docker.sh

@@ -12,13 +12,20 @@ dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
 source "$dir/release/teamcity-support.sh"
 
 tc_start_block "Variable Setup"
+telemetry_disabled="${TELEMETRY_DISABLED:-false}"
+cockroach_archive_prefix="${COCKROACH_ARCHIVE_PREFIX:-cockroach}"
+if [[ $telemetry_disabled == true && $cockroach_archive_prefix == "cockroach" ]]; then
+  echo "COCKROACH_ARCHIVE_PREFIX must be set to a non-default value when telemetry is disabled"
+  exit 1
+fi
+
 version=$(grep -v "^#" "$dir/../pkg/build/version.txt" | head -n1)
 if [[ -z "${DRY_RUN}" ]] ; then
   gcr_credentials="$GCS_CREDENTIALS_PROD"
-  gcr_staged_repository="us-docker.pkg.dev/releases-prod/cockroachdb-staged-releases/cockroach"
+  gcr_staged_repository="us-docker.pkg.dev/releases-prod/cockroachdb-staged-releases/${cockroach_archive_prefix}"
 else
   gcr_credentials="$GCS_CREDENTIALS_DEV"
-  gcr_staged_repository="us-docker.pkg.dev/releases-dev-356314/cockroachdb-staged-releases/cockroach"
+  gcr_staged_repository="us-docker.pkg.dev/releases-dev-356314/cockroachdb-staged-releases/${cockroach_archive_prefix}"
 fi
 tc_end_block "Variable Setup"
 
@@ -33,7 +40,7 @@ tc_start_block "Verify docker images"
 error=0
 for arch in amd64 arm64; do
     tc_start_block "Verify $gcr_tag on $arch"
-    if ! verify_docker_image "$gcr_tag" "linux/$arch" "$BUILD_VCS_NUMBER" "$version" false; then
+    if ! verify_docker_image "$gcr_tag" "linux/$arch" "$BUILD_VCS_NUMBER" "$version" false "$telemetry_disabled"; then
       error=1
     fi
     tc_end_block "Verify $gcr_tag on $arch"

build/teamcity/internal/release/process/build-cockroach-release-linux-amd64-no-telemetry.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+
+PLATFORM=linux-amd64 TELEMETRY_DISABLED=true ./build/teamcity/internal/release/process/build-cockroach-release-per-platform.sh

build/teamcity/internal/release/process/build-cockroach-release-linux-arm64-no-telemetry.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+
+PLATFORM=linux-arm64 TELEMETRY_DISABLED=true ./build/teamcity/internal/release/process/build-cockroach-release-per-platform.sh