mirror of
https://github.com/vmware-tanzu/pinniped.git
synced 2026-08-25 23:07:32 +00:00
130 lines
4.5 KiB
Bash
Executable File
130 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright 2020-2026 the Pinniped contributors. All Rights Reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Sometimes something goes wrong with the Kind pool CI jobs and a Kind
|
|
# VM gets orphaned, meaning that it is still running but the pool repo
|
|
# is not tracking it anymore. Also, an individual test job could create
|
|
# its own Kind VM without adding it to the pool, and if that job fails
|
|
# to clean it up correctly then the VM would be orphaned.
|
|
#
|
|
# Find and delete all orphaned Kind cluster VMs by deleting VMs which:
|
|
# 1. Are running in GCP with a name that indicates that it is a kind VM,
|
|
# 1. Do not exist in the pool repo,
|
|
# 2. And are older than some number of hours since their creation time.
|
|
#
|
|
# These rules should avoid deleting Kind cluster VMs which were created
|
|
# by an individual test job without being added to the pool repo, assuming
|
|
# that those jobs always finish in less than the threshold number of hours.
|
|
#
|
|
# Params are INSTANCE_ZONE, GCP_PROJECT, GCP_USERNAME, and GCP_JSON_KEY.
|
|
|
|
set -euo pipefail
|
|
|
|
gcloud auth activate-service-account \
|
|
"$GCP_USERNAME" \
|
|
--key-file <(echo "$GCP_JSON_KEY") \
|
|
--project "$GCP_PROJECT"
|
|
|
|
# The pinniped-ci-pool resource is now optional so this script can support
|
|
# cleaning up old kind VMs without knowing about any resource pools.
|
|
all_local=()
|
|
if [[ -d pinniped-ci-pool ]]; then
|
|
pushd pinniped-ci-pool >/dev/null
|
|
all_local=($(find . -name 'kind-worker-*' -type f -exec basename {} ';' | sort))
|
|
popd >/dev/null
|
|
fi
|
|
|
|
# Search for old kind-worker VMs which are currently running.
|
|
# Also look for kind-node-builder VMs, because those could also get orphaned by mistake.
|
|
all_cloud=($(gcloud compute instances list \
|
|
--zones "$INSTANCE_ZONE" --project "$GCP_PROJECT" \
|
|
--filter 'name~kind-worker-[a-f0-9]+|kind-node-builder-[a-f0-9]+' --format 'table[no-heading](name)' | sort))
|
|
|
|
exists_in_local_but_not_cloud=()
|
|
for i in "${all_local[@]}"; do
|
|
found=
|
|
for j in "${all_cloud[@]}"; do
|
|
if [[ "$i" == "$j" ]]; then
|
|
found=yes
|
|
break
|
|
fi
|
|
done
|
|
if [[ "$found" != "yes" ]]; then
|
|
exists_in_local_but_not_cloud+=("$i")
|
|
fi
|
|
done
|
|
|
|
exists_in_cloud_but_not_local=()
|
|
exists_in_cloud_but_not_local_relative_path=()
|
|
for i in "${all_cloud[@]}"; do
|
|
found=
|
|
for j in "${all_local[@]}"; do
|
|
if [[ "$i" == "$j" ]]; then
|
|
found=yes
|
|
break
|
|
fi
|
|
done
|
|
if [[ "$found" != "yes" ]]; then
|
|
exists_in_cloud_but_not_local+=("$i")
|
|
fi
|
|
done
|
|
|
|
if [[ -d pinniped-ci-pool ]]; then
|
|
pushd pinniped-ci-pool >/dev/null
|
|
echo
|
|
echo "All pool repo kind cluster files which do not have a running VM instance:"
|
|
for i in "${exists_in_local_but_not_cloud[@]}"; do
|
|
echo -n "$i "
|
|
relative_path=$(find . -name "$i" -type f)
|
|
exists_in_cloud_but_not_local_relative_path+=("$relative_path")
|
|
git --no-pager log -n1 --pretty=format:"%h%x09%an%x09%ad%x09%s" "$relative_path"
|
|
echo
|
|
done
|
|
if [[ ${#exists_in_local_but_not_cloud[@]} -eq 0 ]]; then
|
|
echo "none"
|
|
fi
|
|
popd >/dev/null
|
|
fi
|
|
|
|
now_in_seconds_since_epoch=$(date +"%s")
|
|
hours_ago_to_delete=2
|
|
vms_to_remove=()
|
|
|
|
echo
|
|
echo "All VM instances with no corresponding pool repo file (with creation time in UTC):"
|
|
for i in "${exists_in_cloud_but_not_local[@]}"; do
|
|
creation_time=$(gcloud compute instances describe "$i" \
|
|
--zone "$INSTANCE_ZONE" --project "$GCP_PROJECT" \
|
|
--format 'table[no-heading](creationTimestamp.date(tz=UTC))')
|
|
# UTC date format example: 2022-04-01T17:01:59
|
|
if [[ "$creation_time" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$ ]]; then
|
|
# Note: on MacOS this date command would be: date -ju -f '%Y-%m-%dT%H:%M:%S' "$creation_time" '+%s'
|
|
creation_time_seconds_since_epoch=$(date -u -d "$creation_time" '+%s')
|
|
if (($((now_in_seconds_since_epoch - creation_time_seconds_since_epoch)) > $((hours_ago_to_delete * 60 * 60)))); then
|
|
vms_to_remove+=("$i")
|
|
echo "$i $creation_time (older than $hours_ago_to_delete hours)"
|
|
else
|
|
echo "$i $creation_time (less than $hours_ago_to_delete hours old)"
|
|
fi
|
|
else
|
|
echo "VM creation time not in expected time format: $creation_time"
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ ${#exists_in_cloud_but_not_local[@]} -eq 0 ]]; then
|
|
echo "none"
|
|
fi
|
|
|
|
echo
|
|
if [[ ${#vms_to_remove[@]} -eq 0 ]]; then
|
|
echo "No old orphaned VMs found to remove."
|
|
else
|
|
echo "Removing ${#vms_to_remove[@]} VM(s) which are older than $hours_ago_to_delete hours in $INSTANCE_ZONE: ${vms_to_remove[*]} ..."
|
|
gcloud compute instances delete --zone "${INSTANCE_ZONE}" --delete-disks all --quiet ${vms_to_remove[*]}
|
|
fi
|
|
|
|
echo
|
|
echo "Done!"
|