From db50235995ae14dba5bf2beb3253f68222b9a53c Mon Sep 17 00:00:00 2001 From: Scott Hart Date: Mon, 20 Jul 2026 17:25:10 -0400 Subject: [PATCH 1/5] ci: update rotate-keys for additional key --- ci/cloudbuild/builds/rotate-keys.sh | 112 +++++++++++++++------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/ci/cloudbuild/builds/rotate-keys.sh b/ci/cloudbuild/builds/rotate-keys.sh index 6d42c1cf03011..fddbcf56c6e4d 100755 --- a/ci/cloudbuild/builds/rotate-keys.sh +++ b/ci/cloudbuild/builds/rotate-keys.sh @@ -25,11 +25,10 @@ # `ci/cloudbuild/build.sh --distro fedora rotate-keys --local` # # The purpose of this build is to rotate the service-account keys used by some -# integration tests, such as GCS and Bigtable (see lib/integration.sh). The -# idea is to have p12 and json keys stored in the +# integration tests, such as GCS, Bigtable, and Observability (see lib/integration.sh). +# The idea is to have p12 and json keys stored in the # gs://cloud-cpp-testing-resources-secrets bucket that are named for each -# month. For example, there may be a key named `key-2021-04.p12`. That key will -# be used for the month of April 2021 only. +# month. For example, there may be a key named `key-2021-04.p12` and `observability-key-2026-07.json`. # # We need to create the keys for the next month sometime before that month # starts, and we need to delete the old keys sometime after we're sure they're @@ -37,8 +36,8 @@ # # The strategy is to: # 1. Make sure a keyfile exists for the *current* month. -# 2. Make sure a keyfile exists for the month that's, say, 2 weeks in the future -# 3. Delete any keyfile from the month that's, say, 45 days ago +# 2. Make sure a keyfile exists for the month that's, say, 2 weeks in the future. +# 3. Delete any keyfile from the month that's, say, 45 days ago. # 4. Delete any key that's over 90 days old. # # We can schedule this script to run every night or weekly and it will create @@ -49,56 +48,63 @@ set -euo pipefail source "$(dirname "$0")/../../lib/init.sh" source module /ci/lib/io.sh -io::log_h2 "Current service account keys" -# Note: For historical reasons this keyfile is named with "storage" in the -# name, though it is used for more than just GCS. One day we may rename this -# service account to something more generic. -account="storage-key-file-sa@cloud-cpp-testing-resources.iam.gserviceaccount.com" -gcloud iam service-accounts keys list \ - --iam-account="${account}" \ - --managed-by=user \ - --format="table(CREATED_AT, EXPIRES_AT)" - -io::log_h2 "Checking for the expected active keys" bucket="gs://cloud-cpp-testing-resources-secrets" -active_key_bases=( - "key-$(date +"%Y-%m")" - "key-$(date +"%Y-%m" --date="now + 2 weeks")" + +# Configuration format: "SA_EMAIL|KEY_PREFIX|FILE_TYPES" +accounts_config=( + "storage-key-file-sa@cloud-cpp-testing-resources.iam.gserviceaccount.com|key|json p12" + "observability-sa@cloud-cpp-testing-resources.iam.gserviceaccount.com|observability-key|json" ) -for key_base in "${active_key_bases[@]}"; do - for filetype in "json" "p12"; do - bucket_path="${bucket}/${key_base}.${filetype}" - io::log "Checking for active key at ${bucket_path}" - if ! gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then - io::log "Not found. Creating ${bucket_path}" - gcloud iam service-accounts keys create - \ - --iam-account="${account}" \ - --key-file-type="${filetype}" | - gcloud storage cp - "${bucket_path}" - fi + +for entry in "${accounts_config[@]}"; do + IFS="|" read -r sa_email prefix filetypes <<<"${entry}" + + io::log_h2 "Current service account keys for ${sa_email}" + gcloud iam service-accounts keys list \ + --iam-account="${sa_email}" \ + --managed-by=user \ + --format="table(CREATED_AT, EXPIRES_AT)" + + io::log_h2 "Checking for the expected active keys (${prefix})" + active_key_bases=( + "${prefix}-$(date +"%Y-%m")" + "${prefix}-$(date +"%Y-%m" --date="now + 2 weeks")" + ) + for key_base in "${active_key_bases[@]}"; do + for filetype in ${filetypes}; do + bucket_path="${bucket}/${key_base}.${filetype}" + io::log "Checking for active key at ${bucket_path}" + if ! gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then + io::log "Not found. Creating ${bucket_path}" + gcloud iam service-accounts keys create - \ + --iam-account="${sa_email}" \ + --key-file-type="${filetype}" | + gcloud storage cp - "${bucket_path}" + fi + done done -done -io::log_h2 "Checking for stale keyfiles" -stale_key_base="key-$(date +"%Y-%m" --date="now - 45 days")" -for filetype in "json" "p12"; do - bucket_path="${bucket}/${stale_key_base}.${filetype}" - io::log "Checking for stale key at ${bucket_path}" - if gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then - io::log "Removing ${bucket_path}" - gcloud storage rm "${bucket_path}" - fi -done + io::log_h2 "Checking for stale keyfiles (${prefix})" + stale_key_base="${prefix}-$(date +"%Y-%m" --date="now - 45 days")" + for filetype in ${filetypes}; do + bucket_path="${bucket}/${stale_key_base}.${filetype}" + io::log "Checking for stale key at ${bucket_path}" + if gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then + io::log "Removing ${bucket_path}" + gcloud storage rm "${bucket_path}" + fi + done -io::log_h2 "Checking for keys over 90-days old" -args=( - "--iam-account=${account}" - "--managed-by=user" - "--filter=CREATED_AT<-p90d" - "--format=value(KEY_ID)" -) -for old_key in $(gcloud iam service-accounts keys list "${args[@]}"); do - io::log "Deleting key: ${old_key}" - gcloud iam service-accounts keys delete "${old_key}" --iam-account="${account}" + io::log_h2 "Checking for keys over 90-days old for ${sa_email}" + args=( + "--iam-account=${sa_email}" + "--managed-by=user" + "--filter=CREATED_AT<-p90d" + "--format=value(KEY_ID)" + ) + for old_key in $(gcloud iam service-accounts keys list "${args[@]}"); do + io::log "Deleting key: ${old_key}" + gcloud iam service-accounts keys delete "${old_key}" --iam-account="${sa_email}" --quiet + done + echo done -echo From ffe5ffa70f8980d1bdaa8bbd23d500670e0d1bfa Mon Sep 17 00:00:00 2001 From: Scott Hart Date: Tue, 21 Jul 2026 10:26:39 -0400 Subject: [PATCH 2/5] Update ci/cloudbuild/builds/rotate-keys.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/cloudbuild/builds/rotate-keys.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/rotate-keys.sh b/ci/cloudbuild/builds/rotate-keys.sh index fddbcf56c6e4d..e9ea7138583c3 100755 --- a/ci/cloudbuild/builds/rotate-keys.sh +++ b/ci/cloudbuild/builds/rotate-keys.sh @@ -102,7 +102,8 @@ for entry in "${accounts_config[@]}"; do "--filter=CREATED_AT<-p90d" "--format=value(KEY_ID)" ) - for old_key in $(gcloud iam service-accounts keys list "${args[@]}"); do + old_keys=$(gcloud iam service-accounts keys list "${args[@]}") + for old_key in ${old_keys}; do io::log "Deleting key: ${old_key}" gcloud iam service-accounts keys delete "${old_key}" --iam-account="${sa_email}" --quiet done From f2d375f231f7698bad1e4d104dae6129db21ca1b Mon Sep 17 00:00:00 2001 From: Scott Hart Date: Tue, 21 Jul 2026 10:27:01 -0400 Subject: [PATCH 3/5] Update ci/cloudbuild/builds/rotate-keys.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/cloudbuild/builds/rotate-keys.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/rotate-keys.sh b/ci/cloudbuild/builds/rotate-keys.sh index e9ea7138583c3..85eb3b636bae4 100755 --- a/ci/cloudbuild/builds/rotate-keys.sh +++ b/ci/cloudbuild/builds/rotate-keys.sh @@ -57,7 +57,8 @@ accounts_config=( ) for entry in "${accounts_config[@]}"; do - IFS="|" read -r sa_email prefix filetypes <<<"${entry}" + IFS="|" read -r sa_email prefix filetypes_str <<<"${entry}" + read -r -a filetypes <<<"${filetypes_str}" io::log_h2 "Current service account keys for ${sa_email}" gcloud iam service-accounts keys list \ From d4207177c4b6d2365828b3ec8e905a7d6ce4b531 Mon Sep 17 00:00:00 2001 From: Scott Hart Date: Tue, 21 Jul 2026 10:27:19 -0400 Subject: [PATCH 4/5] Update ci/cloudbuild/builds/rotate-keys.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/cloudbuild/builds/rotate-keys.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/rotate-keys.sh b/ci/cloudbuild/builds/rotate-keys.sh index 85eb3b636bae4..3584150f6f99b 100755 --- a/ci/cloudbuild/builds/rotate-keys.sh +++ b/ci/cloudbuild/builds/rotate-keys.sh @@ -87,7 +87,7 @@ for entry in "${accounts_config[@]}"; do io::log_h2 "Checking for stale keyfiles (${prefix})" stale_key_base="${prefix}-$(date +"%Y-%m" --date="now - 45 days")" - for filetype in ${filetypes}; do + for filetype in "${filetypes[@]}"; do bucket_path="${bucket}/${stale_key_base}.${filetype}" io::log "Checking for stale key at ${bucket_path}" if gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then From 5a03dda1da5eb07923a5789ce4a3fda183cb6534 Mon Sep 17 00:00:00 2001 From: Scott Hart Date: Tue, 21 Jul 2026 10:27:29 -0400 Subject: [PATCH 5/5] Update ci/cloudbuild/builds/rotate-keys.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/cloudbuild/builds/rotate-keys.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/rotate-keys.sh b/ci/cloudbuild/builds/rotate-keys.sh index 3584150f6f99b..d88ebb9852bf9 100755 --- a/ci/cloudbuild/builds/rotate-keys.sh +++ b/ci/cloudbuild/builds/rotate-keys.sh @@ -72,7 +72,7 @@ for entry in "${accounts_config[@]}"; do "${prefix}-$(date +"%Y-%m" --date="now + 2 weeks")" ) for key_base in "${active_key_bases[@]}"; do - for filetype in ${filetypes}; do + for filetype in "${filetypes[@]}"; do bucket_path="${bucket}/${key_base}.${filetype}" io::log "Checking for active key at ${bucket_path}" if ! gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then