You've already forked tribes-supertest
self
7f6224d866
Allow ACME certificate repair to converge after DNS and cluster membership updates by polling edge certificate SANs for up to seven minutes.
541 lines
16 KiB
Bash
Executable File
541 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
dry_run=0
|
|
all_keys=0
|
|
providers="hetzner scaleway ovh"
|
|
failures=0
|
|
|
|
scaleway_zones=(
|
|
fr-par-1 fr-par-2 fr-par-3
|
|
nl-ams-1 nl-ams-2 nl-ams-3
|
|
pl-waw-1 pl-waw-2 pl-waw-3
|
|
it-mil-1
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: scripts/cleanup-cloud-resources [OPTIONS]
|
|
|
|
Delete cloud resources and DNS zones from the provider accounts used by supertest.
|
|
This does not read, modify, or remove Legion state or supertest artifacts.
|
|
|
|
Options:
|
|
--dry-run Print delete commands without running them.
|
|
--provider NAME Limit cleanup to one provider: hetzner, scaleway, ovh.
|
|
Can be repeated.
|
|
--all-keys Delete all provider SSH keys too. By default only
|
|
obvious supertest/Legion test keys are deleted.
|
|
-h, --help Show this help.
|
|
|
|
Examples:
|
|
scripts/cleanup-cloud-resources
|
|
scripts/cleanup-cloud-resources --dry-run
|
|
scripts/cleanup-cloud-resources --provider scaleway
|
|
USAGE
|
|
}
|
|
|
|
log() {
|
|
printf '%s\n' "$*"
|
|
}
|
|
|
|
warn() {
|
|
printf 'warning: %s\n' "$*" >&2
|
|
}
|
|
|
|
fail() {
|
|
failures=$((failures + 1))
|
|
printf 'error: %s\n' "$*" >&2
|
|
}
|
|
|
|
have_provider() {
|
|
local needle="$1"
|
|
[[ " $providers " == *" $needle "* ]]
|
|
}
|
|
|
|
have_command() {
|
|
command -v "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
is_test_name() {
|
|
local name="${1:-}"
|
|
[[ "$name" == st-* || "$name" == *tribes-supertest* || "$name" == *legion-supertest* ]]
|
|
}
|
|
|
|
normalize_dns_name() {
|
|
local name="${1:-}"
|
|
name="${name%.}"
|
|
printf '%s' "${name,,}"
|
|
}
|
|
|
|
json_array() {
|
|
jq -c 'if type == "array" then . elif . == null then [] else [.] end'
|
|
}
|
|
|
|
run_delete() {
|
|
if (( dry_run )); then
|
|
printf '+'
|
|
printf ' %q' "$@"
|
|
printf '\n'
|
|
return 0
|
|
fi
|
|
|
|
printf '+'
|
|
printf ' %q' "$@"
|
|
printf '\n'
|
|
"$@"
|
|
}
|
|
|
|
run_ovh_api_post() {
|
|
local path="$1"
|
|
|
|
if (( dry_run )); then
|
|
printf '+ ovh-api POST %q\n' "$path"
|
|
return 0
|
|
fi
|
|
|
|
printf '+ ovh-api POST %q\n' "$path"
|
|
node - "$path" <<'NODE'
|
|
const crypto = require("node:crypto")
|
|
|
|
async function main() {
|
|
const path = process.argv[2]
|
|
const endpoint = (process.env.OVH_ENDPOINT || "ovh-eu").trim()
|
|
const appKey = (process.env.OVH_APPLICATION_KEY || process.env.OVH_APP_KEY || "").trim()
|
|
const appSecret = (process.env.OVH_APPLICATION_SECRET || process.env.OVH_APP_SECRET || "").trim()
|
|
const consumerKey = (process.env.OVH_CONSUMER_KEY || "").trim()
|
|
const host = endpoint === "ovh-ca" ? "ca.api.ovh.com" : "eu.api.ovh.com"
|
|
const baseUrl = `https://${host}/1.0`
|
|
|
|
if (!appKey || !appSecret || !consumerKey) {
|
|
throw new Error("OVH application key/secret or consumer key is not set.")
|
|
}
|
|
|
|
const timeResponse = await fetch(`${baseUrl}/auth/time`, {
|
|
headers: { "X-Ovh-Application": appKey }
|
|
})
|
|
if (!timeResponse.ok) {
|
|
throw new Error(`OVH auth time failed: ${timeResponse.status} ${await timeResponse.text()}`)
|
|
}
|
|
const timestamp = (await timeResponse.text()).trim()
|
|
const method = "POST"
|
|
const body = ""
|
|
const url = `${baseUrl}${path}`
|
|
const signaturePayload = [appSecret, consumerKey, method, url, body, timestamp].join("+")
|
|
const signature = `$1$${crypto.createHash("sha1").update(signaturePayload).digest("hex")}`
|
|
const response = await fetch(url, {
|
|
method,
|
|
headers: {
|
|
"X-Ovh-Application": appKey,
|
|
"X-Ovh-Consumer": consumerKey,
|
|
"X-Ovh-Signature": signature,
|
|
"X-Ovh-Timestamp": timestamp
|
|
}
|
|
})
|
|
if (!response.ok) {
|
|
throw new Error(`OVH API POST ${path} failed: ${response.status} ${await response.text()}`)
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error instanceof Error ? error.message : String(error))
|
|
process.exit(1)
|
|
})
|
|
NODE
|
|
}
|
|
|
|
delete_hcloud_collection() {
|
|
local resource="$1"
|
|
local label="$2"
|
|
local list
|
|
|
|
if ! list="$(hcloud "$resource" list -o json)"; then
|
|
fail "failed to list Hetzner $label"
|
|
return
|
|
fi
|
|
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Hetzner $label: ${name:-$id} ($id)"
|
|
run_delete hcloud "$resource" delete "$id" || fail "failed to delete Hetzner $label $id"
|
|
done < <(printf '%s\n' "$list" | json_array | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
}
|
|
|
|
cleanup_hetzner() {
|
|
if [[ -z "${HCLOUD_TOKEN:-}" ]]; then
|
|
warn "skipping Hetzner cleanup; HCLOUD_TOKEN is not set"
|
|
return
|
|
fi
|
|
if ! have_command hcloud; then
|
|
warn "skipping Hetzner cleanup; hcloud is not in PATH"
|
|
return
|
|
fi
|
|
|
|
log "== Hetzner =="
|
|
delete_hcloud_collection server "server"
|
|
delete_hcloud_collection load-balancer "load balancer"
|
|
delete_hcloud_collection volume "volume"
|
|
delete_hcloud_collection floating-ip "floating IP"
|
|
delete_hcloud_collection primary-ip "primary IP"
|
|
delete_hcloud_collection firewall "firewall"
|
|
delete_hcloud_collection network "network"
|
|
delete_hcloud_collection zone "DNS zone"
|
|
|
|
local list
|
|
if ! list="$(hcloud ssh-key list -o json)"; then
|
|
fail "failed to list Hetzner SSH keys"
|
|
return
|
|
fi
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
if (( all_keys )) || is_test_name "$name"; then
|
|
log "Deleting Hetzner SSH key: ${name:-$id} ($id)"
|
|
run_delete hcloud ssh-key delete "$id" || fail "failed to delete Hetzner SSH key $id"
|
|
else
|
|
log "Keeping Hetzner SSH key: ${name:-$id} ($id)"
|
|
fi
|
|
done < <(printf '%s\n' "$list" | json_array | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
}
|
|
|
|
scw_list_zone() {
|
|
local zone="$1"
|
|
shift
|
|
scw "$@" list zone="$zone" -o json 2>/dev/null | json_array
|
|
}
|
|
|
|
cleanup_scaleway_zone() {
|
|
local zone="$1"
|
|
local list
|
|
|
|
if list="$(scw_list_zone "$zone" instance server)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway server in $zone: ${name:-$id} ($id)"
|
|
run_delete scw instance server delete "$id" with-volumes=all with-ip=true force-shutdown=true zone="$zone" -w ||
|
|
fail "failed to delete Scaleway server $id in $zone"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list Scaleway servers in $zone"
|
|
fi
|
|
|
|
if list="$(scw_list_zone "$zone" instance volume)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway instance volume in $zone: ${name:-$id} ($id)"
|
|
run_delete scw instance volume delete "$id" zone="$zone" ||
|
|
fail "failed to delete Scaleway instance volume $id in $zone"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list Scaleway instance volumes in $zone"
|
|
fi
|
|
|
|
if list="$(scw_list_zone "$zone" block snapshot)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway block snapshot in $zone: ${name:-$id} ($id)"
|
|
run_delete scw block snapshot delete "$id" zone="$zone" ||
|
|
fail "failed to delete Scaleway block snapshot $id in $zone"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list Scaleway block snapshots in $zone"
|
|
fi
|
|
|
|
if list="$(scw_list_zone "$zone" block volume)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway block volume in $zone: ${name:-$id} ($id)"
|
|
run_delete scw block volume delete "$id" zone="$zone" ||
|
|
fail "failed to delete Scaleway block volume $id in $zone"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list Scaleway block volumes in $zone"
|
|
fi
|
|
|
|
if list="$(scw_list_zone "$zone" instance ip)"; then
|
|
while IFS=$'\t' read -r id address; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway flexible IP in $zone: ${address:-$id} ($id)"
|
|
run_delete scw instance ip delete "$id" zone="$zone" ||
|
|
fail "failed to delete Scaleway flexible IP $id in $zone"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.address // .ip // "")] | @tsv')
|
|
else
|
|
warn "failed to list Scaleway flexible IPs in $zone"
|
|
fi
|
|
|
|
if list="$(scw_list_zone "$zone" instance placement-group)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway placement group in $zone: ${name:-$id} ($id)"
|
|
run_delete scw instance placement-group delete "$id" zone="$zone" ||
|
|
fail "failed to delete Scaleway placement group $id in $zone"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list Scaleway placement groups in $zone"
|
|
fi
|
|
|
|
if list="$(scw_list_zone "$zone" instance security-group)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting Scaleway security group in $zone: ${name:-$id} ($id)"
|
|
run_delete scw instance security-group delete "$id" zone="$zone" ||
|
|
fail "failed to delete Scaleway security group $id in $zone"
|
|
done < <(
|
|
printf '%s\n' "$list" |
|
|
jq -r '.[] |
|
|
select((.organization_default // false | not) and
|
|
(.project_default // false | not) and
|
|
((.name // "") != "Default security group")) |
|
|
[.id, (.name // "")] | @tsv'
|
|
)
|
|
else
|
|
warn "failed to list Scaleway security groups in $zone"
|
|
fi
|
|
}
|
|
|
|
cleanup_scaleway_dns_zones() {
|
|
local domain
|
|
domain="$(normalize_dns_name "${SUPERTEST_DNS_DOMAIN:-}")"
|
|
|
|
if [[ -z "$domain" ]]; then
|
|
warn "skipping Scaleway DNS zone cleanup; Scaleway external-domain API has no list command and SUPERTEST_DNS_DOMAIN is not set"
|
|
return
|
|
fi
|
|
|
|
log "Trying to delete Scaleway external DNS zone: $domain"
|
|
if (( dry_run )); then
|
|
run_delete scw domain external-domain delete domain="$domain"
|
|
return
|
|
fi
|
|
|
|
local output
|
|
if output="$(run_delete scw domain external-domain delete domain="$domain" 2>&1)"; then
|
|
printf '%s\n' "$output"
|
|
return
|
|
fi
|
|
|
|
if grep -qi 'domain not found' <<<"$output"; then
|
|
log "Scaleway external DNS zone not found: $domain"
|
|
return
|
|
fi
|
|
|
|
printf '%s\n' "$output" >&2
|
|
fail "failed to delete Scaleway external DNS zone $domain"
|
|
}
|
|
|
|
cleanup_scaleway() {
|
|
if [[ -z "${SCW_ACCESS_KEY:-}" || -z "${SCW_SECRET_KEY:-}" || -z "${SCW_DEFAULT_PROJECT_ID:-}" ]]; then
|
|
warn "skipping Scaleway cleanup; SCW_ACCESS_KEY, SCW_SECRET_KEY, or SCW_DEFAULT_PROJECT_ID is not set"
|
|
return
|
|
fi
|
|
if ! have_command scw; then
|
|
warn "skipping Scaleway cleanup; scw is not in PATH"
|
|
return
|
|
fi
|
|
|
|
log "== Scaleway =="
|
|
for zone in "${scaleway_zones[@]}"; do
|
|
cleanup_scaleway_zone "$zone"
|
|
done
|
|
cleanup_scaleway_dns_zones
|
|
}
|
|
|
|
ovh_project_ids() {
|
|
ovhcloud cloud project list -o json | json_array | jq -r '.[] | .project_id // .projectId // .id // empty'
|
|
}
|
|
|
|
ovh_list_project() {
|
|
local project="$1"
|
|
shift
|
|
ovhcloud cloud "$@" list --cloud-project "$project" -o json 2>/dev/null | json_array
|
|
}
|
|
|
|
cleanup_ovh_project() {
|
|
local project="$1"
|
|
local list
|
|
|
|
log "OVH project $project"
|
|
|
|
if list="$(ovh_list_project "$project" instance)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting OVH instance: ${name:-$id} ($id)"
|
|
run_delete ovhcloud cloud instance delete "$id" --cloud-project "$project" ||
|
|
fail "failed to delete OVH instance $id"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list OVH instances for project $project"
|
|
fi
|
|
|
|
if list="$(ovh_list_project "$project" loadbalancer)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting OVH load balancer: ${name:-$id} ($id)"
|
|
run_delete ovhcloud cloud loadbalancer delete "$id" --cloud-project "$project" ||
|
|
fail "failed to delete OVH load balancer $id"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list OVH load balancers for project $project"
|
|
fi
|
|
|
|
if list="$(ovh_list_project "$project" instance snapshot)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting OVH instance snapshot: ${name:-$id} ($id)"
|
|
run_delete ovhcloud cloud instance snapshot delete "$id" --cloud-project "$project" ||
|
|
fail "failed to delete OVH instance snapshot $id"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list OVH instance snapshots for project $project"
|
|
fi
|
|
|
|
if list="$(ovh_list_project "$project" storage block)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
log "Deleting OVH block volume: ${name:-$id} ($id)"
|
|
run_delete ovhcloud cloud storage block delete "$id" --cloud-project "$project" ||
|
|
fail "failed to delete OVH block volume $id"
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list OVH block volumes for project $project"
|
|
fi
|
|
|
|
if list="$(ovh_list_project "$project" network private)"; then
|
|
while IFS=$'\t' read -r id name region; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
if is_test_name "$name"; then
|
|
log "Deleting OVH private network: ${name:-$id} ($id)"
|
|
run_delete ovhcloud cloud network private delete "$id" --cloud-project "$project" --region "$region" ||
|
|
fail "failed to delete OVH private network $id"
|
|
else
|
|
log "Keeping OVH private network: ${name:-$id} ($id)"
|
|
fi
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // ""), (.region // "")] | @tsv')
|
|
else
|
|
warn "failed to list OVH private networks for project $project"
|
|
fi
|
|
|
|
if list="$(ovh_list_project "$project" ssh-key)"; then
|
|
while IFS=$'\t' read -r id name; do
|
|
[[ -n "${id:-}" ]] || continue
|
|
if (( all_keys )) || is_test_name "$name"; then
|
|
log "Deleting OVH SSH key: ${name:-$id} ($id)"
|
|
run_delete ovhcloud cloud ssh-key delete "$id" --cloud-project "$project" ||
|
|
fail "failed to delete OVH SSH key $id"
|
|
else
|
|
log "Keeping OVH SSH key: ${name:-$id} ($id)"
|
|
fi
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | [.id, (.name // "")] | @tsv')
|
|
else
|
|
warn "failed to list OVH SSH keys for project $project"
|
|
fi
|
|
}
|
|
|
|
cleanup_ovh_dns_zones() {
|
|
local list
|
|
|
|
if ! list="$(ovhcloud domain-zone list -o json 2>/dev/null | json_array)"; then
|
|
fail "failed to list OVH DNS zones"
|
|
return
|
|
fi
|
|
|
|
while IFS= read -r zone; do
|
|
[[ -n "$zone" ]] || continue
|
|
local encoded_zone
|
|
encoded_zone="$(jq -rn --arg value "$zone" '$value | @uri')"
|
|
log "Requesting OVH DNS zone termination: $zone"
|
|
run_ovh_api_post "/domain/zone/${encoded_zone}/terminate" ||
|
|
fail "failed to request OVH DNS zone termination for $zone"
|
|
log "OVH requires email or Manager confirmation before DNS zone $zone is permanently deleted."
|
|
done < <(printf '%s\n' "$list" | jq -r '.[] | .name // .zoneName // .id // empty')
|
|
}
|
|
|
|
cleanup_ovh() {
|
|
if [[ -z "${OVH_APP_KEY:-}${OVH_APPLICATION_KEY:-}" ||
|
|
-z "${OVH_APP_SECRET:-}${OVH_APPLICATION_SECRET:-}" ||
|
|
-z "${OVH_CONSUMER_KEY:-}" ]]; then
|
|
warn "skipping OVH cleanup; OVH app key/secret or OVH_CONSUMER_KEY is not set"
|
|
return
|
|
fi
|
|
if ! have_command ovhcloud; then
|
|
warn "skipping OVH cleanup; ovhcloud is not in PATH"
|
|
return
|
|
fi
|
|
|
|
log "== OVH =="
|
|
local projects
|
|
if ! projects="$(ovh_project_ids)"; then
|
|
fail "failed to list OVH cloud projects"
|
|
return
|
|
fi
|
|
while IFS= read -r project; do
|
|
[[ -n "$project" ]] || continue
|
|
cleanup_ovh_project "$project"
|
|
done <<<"$projects"
|
|
cleanup_ovh_dns_zones
|
|
}
|
|
|
|
selected_providers=()
|
|
while (($#)); do
|
|
case "$1" in
|
|
--dry-run)
|
|
dry_run=1
|
|
shift
|
|
;;
|
|
--provider)
|
|
[[ $# -ge 2 ]] || { printf '%s\n' "--provider needs a value" >&2; exit 2; }
|
|
selected_providers+=("$2")
|
|
shift 2
|
|
;;
|
|
--provider=*)
|
|
selected_providers+=("${1#--provider=}")
|
|
shift
|
|
;;
|
|
--all-keys)
|
|
all_keys=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
printf 'Unknown option: %s\n\n' "$1" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if ((${#selected_providers[@]})); then
|
|
providers="${selected_providers[*]}"
|
|
fi
|
|
|
|
for provider in $providers; do
|
|
case "$provider" in
|
|
hetzner|scaleway|ovh) ;;
|
|
*)
|
|
printf 'Unknown provider: %s\n' "$provider" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
cd "$repo_root"
|
|
|
|
if (( dry_run )); then
|
|
log "dry run: no resources will be deleted"
|
|
fi
|
|
|
|
have_provider hetzner && cleanup_hetzner
|
|
have_provider scaleway && cleanup_scaleway
|
|
have_provider ovh && cleanup_ovh
|
|
|
|
if (( failures > 0 )); then
|
|
printf 'cloud cleanup completed with %d failure(s)\n' "$failures" >&2
|
|
exit 1
|
|
fi
|
|
|
|
log "cloud cleanup completed"
|