From 7731bee96568913b2cd71d4d09f3d8c65c467b33 Mon Sep 17 00:00:00 2001
From: Sahil Batra <shlbatra123@gmail.com>
Date: Sat, 8 Aug 2026 23:42:55 -0400
Subject: [PATCH] Fix silent no-op when destroying the Vertex AI Feature Store
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

destroy_resources.sh guarded its deletes with
`if gcloud ai feature-online-stores describe ... >/dev/null 2>&1`.
That command group is beta-only and absent from a stock gcloud install,
so gcloud exits 2 with "Invalid choice" — which the guard misread as
"resource doesn't exist". The script printed "not found — skipping" and
"Feature Store torn down" while ml_online_store kept running a Bigtable
node and billing.

Talk to the Vertex AI REST API directly instead, matching how the store
is created (feature_store/setup.py uses the Python SDK, not gcloud).
Deletes now poll the returned LRO to completion rather than returning
before the resource is actually gone.

Also delete every feature view in the store instead of only the
hardcoded $FEATURE_VIEW_ID — any leftover view blocks the store delete
with FAILED_PRECONDITION.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 scripts/destroy_resources.sh | 91 +++++++++++++++++++++++++++++-------
 1 file changed, 75 insertions(+), 16 deletions(-)

diff --git a/scripts/destroy_resources.sh b/scripts/destroy_resources.sh
index 4bf23e3..62c80e9 100755
--- a/scripts/destroy_resources.sh
+++ b/scripts/destroy_resources.sh
@@ -76,12 +76,66 @@ run() {
     fi
 }
 
+# --- Vertex AI Feature Store REST helpers ---
+# `gcloud ai feature-online-stores` is beta-only and absent from a stock gcloud
+# install, so it can't be used here: it exits 2 ("Invalid choice") and a
+# `if gcloud ... 2>/dev/null` guard silently misreads that as "doesn't exist",
+# leaving a Bigtable node billing forever. Creation already goes straight to the
+# REST API via the Python SDK (feature_store/setup.py), so deletion does too.
+FS_API="https://${FS_REGION}-aiplatform.googleapis.com/v1"
+FS_PARENT="projects/${PROJECT_ID}/locations/${FS_REGION}"
+
+fs_curl() {  # fs_curl <method> <url> -> body on stdout, HTTP status as exit-code proxy
+    curl -sS -X "$1" \
+        -H "Authorization: Bearer $(gcloud auth print-access-token --project="$PROJECT_ID")" \
+        -H "Content-Type: application/json" \
+        -w '\n%{http_code}' "$2"
+}
+
+# fs_get <url> — succeeds only on HTTP 200; prints the body.
+fs_get() {
+    local out status
+    out=$(fs_curl GET "$1") || return 1
+    status=$(printf '%s' "$out" | tail -n1)
+    [ "$status" = "200" ] || return 1
+    printf '%s' "$out" | sed '$d'
+}
+
+# fs_delete <url> <label> — DELETE, then poll the returned LRO to completion.
+fs_delete() {
+    local url="$1" label="$2" out status body op done_at
+    out=$(fs_curl DELETE "$url") || { echo "  Delete request for $label failed."; return 1; }
+    status=$(printf '%s' "$out" | tail -n1)
+    body=$(printf '%s' "$out" | sed '$d')
+    if [ "$status" != "200" ]; then
+        echo "  Delete of $label returned HTTP $status:"
+        echo "$body" | head -20
+        return 1
+    fi
+
+    op=$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("name",""))')
+    [ -n "$op" ] || { echo "  $label: delete accepted (no operation returned)."; return 0; }
+
+    # Poll the long-running operation — store deletes take a minute or two.
+    for _ in $(seq 1 60); do
+        done_at=$(fs_get "${FS_API}/${op}" 2>/dev/null \
+            | python3 -c 'import json,sys; d=json.load(sys.stdin); print("done" if d.get("done") else "")' 2>/dev/null) || true
+        if [ "$done_at" = "done" ]; then
+            echo "  $label deleted."
+            return 0
+        fi
+        sleep 5
+    done
+    echo "  Timed out waiting for $label deletion — check the console."
+    return 1
+}
+
 echo "=== DESTROY POC resources in project: $PROJECT_ID ==="
 echo ""
 echo "The following will be PERMANENTLY deleted:"
 $SKIP_COMPOSER          || echo "  - Composer env      : $COMPOSER_ENV ($COMPOSER_REGION) + its GKE cluster"
 $SKIP_COMPOSER          || { $KEEP_BUCKET || echo "  - Composer DAGs bucket (leftover after env delete)"; }
-$SKIP_FEATURE_STORE     || echo "  - Feature view      : $FEATURE_VIEW_ID"
+$SKIP_FEATURE_STORE     || echo "  - Feature views     : ALL views in $ONLINE_STORE_ID (e.g. $FEATURE_VIEW_ID)"
 $SKIP_FEATURE_STORE     || echo "  - Online store      : $ONLINE_STORE_ID ($FS_REGION, Bigtable)"
 $SKIP_ARTIFACT_REGISTRY || echo "  - Artifact repo     : $AR_LOCATION/$AR_REPOSITORY (ALL images)"
 echo ""
@@ -106,24 +160,29 @@ if [ "$SKIP_FEATURE_STORE" != "true" ]; then
     echo ""
     echo "--- Feature Store ---"
 
-    if gcloud ai feature-online-stores feature-views describe "$FEATURE_VIEW_ID" \
-        --feature-online-store="$ONLINE_STORE_ID" \
-        --region="$FS_REGION" --project="$PROJECT_ID" >/dev/null 2>&1; then
-        echo "Deleting feature view '$FEATURE_VIEW_ID'..."
-        run gcloud ai feature-online-stores feature-views delete "$FEATURE_VIEW_ID" \
-            --feature-online-store="$ONLINE_STORE_ID" \
-            --region="$FS_REGION" --project="$PROJECT_ID" --quiet
+    STORE_URL="${FS_API}/${FS_PARENT}/featureOnlineStores/${ONLINE_STORE_ID}"
+
+    if ! fs_get "$STORE_URL" >/dev/null 2>&1; then
+        echo "Online store '$ONLINE_STORE_ID' not found — skipping."
     else
-        echo "Feature view '$FEATURE_VIEW_ID' not found — skipping."
-    fi
+        # Delete every view in the store, not just $FEATURE_VIEW_ID — any
+        # leftover view blocks the store delete with FAILED_PRECONDITION.
+        VIEWS=$(fs_get "${STORE_URL}/featureViews" 2>/dev/null \
+            | python3 -c 'import json,sys; print("\n".join(v["name"] for v in json.load(sys.stdin).get("featureViews",[])))' \
+            || true)
+
+        if [ -z "${VIEWS:-}" ]; then
+            echo "No feature views in '$ONLINE_STORE_ID'."
+        else
+            while IFS= read -r view; do
+                [ -n "$view" ] || continue
+                echo "Deleting feature view '${view##*/}'..."
+                run fs_delete "${FS_API}/${view}" "feature view '${view##*/}'"
+            done <<< "$VIEWS"
+        fi
 
-    if gcloud ai feature-online-stores describe "$ONLINE_STORE_ID" \
-        --region="$FS_REGION" --project="$PROJECT_ID" >/dev/null 2>&1; then
         echo "Deleting online store '$ONLINE_STORE_ID' (stops Bigtable node billing)..."
-        run gcloud ai feature-online-stores delete "$ONLINE_STORE_ID" \
-            --region="$FS_REGION" --project="$PROJECT_ID" --quiet
-    else
-        echo "Online store '$ONLINE_STORE_ID' not found — skipping."
+        run fs_delete "$STORE_URL" "online store '$ONLINE_STORE_ID'"
     fi
     echo "Feature Store torn down."
 fi