Week 3: Failover and Chaos Testing

Test Matrix Week 3

┌────┬────────────────────────────────────────────────────────┬─────────┐
│ ID │ Failover Test                                          │ Pass    │
├────┼────────────────────────────────────────────────────────┼─────────┤
│3.1 │ ExpressRoute circuit failure (primary → secondary)    │ <30s    │
│3.2 │ Azure VPN Gateway failover (primary → secondary inst) │ <60s    │
│3.3 │ GCP Cloud Interconnect failure                        │ Reroute │
│3.4 │ Hub site MPLS failure (MPLS → DIA)                    │ <15s    │
│3.5 │ Branch DIA failure (DIA → LTE)                        │ <20s    │
│3.6 │ SD-WAN hub failure (Mumbai → Chennai hub)             │ <45s    │
│3.7 │ Umbrella PoP failure (primary → secondary PoP)        │ <10s    │
│3.8 │ Private endpoint failover (SQL HA test)               │ <30s    │
└────┴────────────────────────────────────────────────────────┴─────────┘

---

TEST 3.1: ExpressRoute Circuit Failure and Failover

#!/bin/bash

# er_failover_test.sh


echo "=== TEST 3.1: EXPRESSROUTE FAILOVER ==="
echo "WARNING: This test causes brief disruption to O365/Teams traffic"
echo "Recommended: Run during off-peak hours (maintenance window)"

echo ""
echo "--- BASELINE MEASUREMENT ---"
## Record current path and latency for Teams

BASELINE_LAT=$(ping -c 5 teams.microsoft.com 2>/dev/null | tail -1 | awk -F/ '{printf "%.0f", $5}')
BASELINE_PATH=$(ssh admin@mum-hub-01 "show ip route 52.112.0.0 | grep via" 2>/dev/null)
echo "Baseline Teams latency: ${BASELINE_LAT}ms"
echo "Baseline route: $BASELINE_PATH"

echo ""
echo "--- STEP 1: Disable PRIMARY ExpressRoute interface ---"
T_START=$(date +%s%3N)
ssh admin@mum-hub-01 "configure terminal; interface TenGigabitEthernet0/0/2.4001; shutdown; end"
echo "Primary ER disabled at: $(date)"

echo ""
echo "--- STEP 2: Monitor BGP failover ---"
for i in $(seq 1 10); do
  ELAPSED=$(( $(date +%s%3N) - T_START ))
  PRIMARY=$(ssh admin@mum-hub-01 "show bgp ipv4 unicast summary | grep 169.254.200.1" 2>/dev/null | awk '{print $10}')
  SECONDARY=$(ssh admin@mum-hub-01 "show bgp ipv4 unicast summary | grep 169.254.200.5" 2>/dev/null | awk '{print $10}')
  NEW_ROUTE=$(ssh admin@mum-hub-01 "show ip route 52.112.0.0 | grep via" 2>/dev/null)

  echo "  [${ELAPSED}ms] Primary BGP: $PRIMARY | Secondary BGP: $SECONDARY"

  if echo "$NEW_ROUTE" | grep -q "169.254.200.5"; then
    T_FAILOVER=$(( $(date +%s%3N) - T_START ))
    echo ""
    echo "  ✅ FAILOVER COMPLETE in ${T_FAILOVER}ms"
    echo "  New route: $NEW_ROUTE"
    break
  fi
  sleep 3
done

## Verify Teams still works via secondary path

FAILOVER_LAT=$(ping -c 5 teams.microsoft.com 2>/dev/null | tail -1 | awk -F/ '{printf "%.0f", $5}')
echo ""
echo "--- STEP 3: Verify service continuity ---"
echo "  Teams latency during failover: ${FAILOVER_LAT}ms (baseline: ${BASELINE_LAT}ms)"
[[ "$FAILOVER_LAT" -lt 50 ]] && echo "  ✅ Teams still operational via secondary ER" \
  || echo "  ⚠️  Teams latency degraded during failover"

## Restore primary

echo ""
echo "--- STEP 4: Restore primary interface ---"
ssh admin@mum-hub-01 "configure terminal; interface TenGigabitEthernet0/0/2.4001; no shutdown; end"
sleep 35
RESTORED_ROUTE=$(ssh admin@mum-hub-01 "show ip route 52.112.0.0 | grep via" 2>/dev/null)
echo "$RESTORED_ROUTE" | grep -q "169.254.200.1" && echo "  ✅ Primary path restored (LOCAL_PREF 200 wins)" \
  || echo "  ⚠️  Primary route not restored: $RESTORED_ROUTE"

Pass Criteria: - Failover complete within 30 seconds (BGP hold-time 30s) - Teams voice MOS does not drop below 3.5 during failover - Primary route restoration automatic within 35 seconds

---

TEST 3.2: Azure VPN Gateway Failover

#!/bin/bash

## azure_vpn_failover_test.sh


echo "=== TEST 3.2: AZURE VIRTUAL WAN VPN GATEWAY FAILOVER ==="

echo ""
echo "--- Baseline: Azure SQL reachable via primary tunnel ---"
ssh admin@blr-isr-01 "show ip route 10.100.10.10" 2>/dev/null
BASELINE=$(ssh admin@blr-isr-01 "ping vrf 1 10.100.10.10 repeat 5 | tail -1" 2>/dev/null)
echo "Baseline: $BASELINE"

echo ""
echo "--- Disable primary tunnel to Azure VPN GW ---"
T_START=$(date +%s%3N)
ssh admin@blr-isr-01 "configure terminal; interface Tunnel200; shutdown; end"

echo "--- Monitor failover to secondary tunnel ---"
for i in $(seq 1 8); do
  ELAPSED=$(( $(date +%s%3N) - T_START ))
  T201=$(ssh admin@blr-isr-01 "show interfaces Tunnel201 | grep 'line protocol'" 2>/dev/null | grep -c "up")
  ROUTE=$(ssh admin@blr-isr-01 "show ip route 10.100.10.10 | grep via" 2>/dev/null)

  echo "  [${ELAPSED}ms] T201: $( [[ $T201 -ge 1 ]] && echo 'UP' || echo 'down') | Route: $ROUTE"

  if echo "$ROUTE" | grep -q "Tunnel201"; then
    T_FAILOVER=$(( $(date +%s%3N) - T_START ))
    echo "  ✅ Failover to Tunnel201 in ${T_FAILOVER}ms"
    break
  fi
  sleep 8
done

## Verify SQL still reachable

sleep 5
FAILOVER_PING=$(ssh admin@blr-isr-01 "ping vrf 1 10.100.10.10 repeat 5 | tail -1" 2>/dev/null)
echo "  Azure SQL during failover: $FAILOVER_PING"

## Restore

ssh admin@blr-isr-01 "configure terminal; interface Tunnel200; no shutdown; end"
sleep 30
echo "  ✅ Tunnel200 restored"

---

TEST 3.4: Hub MPLS Failure (MPLS → DIA)

#!/bin/bash

## mpls_failover_test.sh


echo "=== TEST 3.4: MPLS → DIA FAILOVER ==="

HUB="mum-hub-01"

## Baseline: all apps using MPLS for corporate

echo "--- Baseline corporate traffic path ---"
CORP_PATH=$(ssh admin@$HUB "show sdwan app-fwd cflowd flows | grep 10.252 | head -3" 2>/dev/null)
echo "$CORP_PATH"

## Disable MPLS transport

T_START=$(date +%s%3N)
echo "--- Disabling MPLS transport (color: mpls) ---"
ssh admin@$HUB "sdwan; interface TenGigabitEthernet0/0/0; no tunnel-interface; exit; exit" 2>/dev/null
## (Alternatively: shut the physical interface)

## ssh admin@$HUB "configure terminal; interface TenGigabitEthernet0/0/0; shutdown; end"


## Monitor BFD re-convergence

for i in $(seq 1 5); do
  sleep 5
  ELAPSED=$(( $(date +%s%3N) - T_START ))
  DIA_FLOWS=$(ssh admin@$HUB "show sdwan app-fwd cflowd flows | grep 10.252 | grep dia | wc -l" 2>/dev/null)
  echo "  [${ELAPSED}ms] Corporate flows via DIA: $DIA_FLOWS"
  [[ $DIA_FLOWS -gt 0 ]] && { echo "  ✅ Traffic moved to DIA"; break; }
done

## Verify latency impact (DIA will be slower than MPLS)

FAILOVER_LATENCY=$(ssh admin@$HUB "ping vrf 1 10.252.32.1 repeat 5 | tail -1" 2>/dev/null)
echo "  Latency to NJ hub via DIA: $FAILOVER_LATENCY"

## Restore MPLS

ssh admin@$HUB "configure terminal; interface TenGigabitEthernet0/0/0; no shutdown; end"
sleep 15
echo "  ✅ MPLS restored"

---

TEST 3.5: Branch DIA Failure (DIA → LTE)

#!/bin/bash

## branch_dia_lte_failover.sh


echo "=== TEST 3.5: BRANCH DIA → LTE FAILOVER ==="

BRANCH="blr-isr-01"

## Baseline

echo "--- Baseline: Branch using DIA ---"
ssh admin@$BRANCH "show sdwan interface | grep -E '(GigabitEthernet0|Cellular)'" 2>/dev/null

## Disable DIA

T_START=$(date +%s%3N)
echo "--- Shutting DIA interface ---"
ssh admin@$BRANCH "configure terminal; interface GigabitEthernet0/0/0; shutdown; end" 2>/dev/null

## Monitor LTE takeover

for i in $(seq 1 8); do
  sleep 5
  ELAPSED=$(( $(date +%s%3N) - T_START ))
  LTE=$(ssh admin@$BRANCH "show sdwan bfd sessions | grep lte | grep Up | wc -l" 2>/dev/null)
  CTRL=$(ssh admin@$BRANCH "show sdwan control connections | grep ESTABLISHED | wc -l" 2>/dev/null)
  echo "  [${ELAPSED}ms] LTE BFD sessions: $LTE | Control connections: $CTRL"
  [[ $LTE -gt 0 ]] && { echo "  ✅ Branch operating on LTE backup"; break; }
done

## Verify internet still working via Umbrella over LTE

ssh admin@$BRANCH "ping 208.67.222.222 repeat 3 | tail -1" 2>/dev/null | \
  grep -qv "0 success" && echo "  ✅ Umbrella DNS reachable via LTE" || echo "  ⚠️  Umbrella unreachable"

## Restore DIA

ssh admin@$BRANCH "configure terminal; interface GigabitEthernet0/0/0; no shutdown; end" 2>/dev/null
sleep 20
echo "  ✅ DIA restored — SD-WAN will prefer DIA over LTE automatically"

---