Week 2: Integration Testing

Test Matrix Week 2

┌────┬──────────────────────────────────────────────────────┬──────────┐
│ ID │ Test                                                 │ Pass     │
├────┼──────────────────────────────────────────────────────┼──────────┤
│2.1 │ GCP→Azure cross-cloud routing                       │ <50ms    │
│2.2 │ Branch→Azure SQL (end-to-end data path)             │ Port open│
│2.3 │ Branch→GCP Vertex AI API call                       │ HTTP 200 │
│2.4 │ Teams voice quality (MOS via ThousandEyes)          │ MOS >4.0 │
│2.5 │ O365 Exchange via ExpressRoute (not internet)       │ ER path  │
│2.6 │ Salesforce via best-probe path                      │ <100ms   │
│2.7 │ Branch Guest VPN isolation                          │ No leak  │
│2.8 │ SD-Access VN → SD-WAN VPN routing                   │ Routes OK│
│2.9 │ AppDynamics traces: branch→Azure SQL transaction    │ Traced   │
│2.10│ Splunk: all telemetry sources flowing               │ All 17   │
└────┴──────────────────────────────────────────────────────┴──────────┘

---

TEST 2.1: GCP to Azure Cross-Cloud Routing

Objective: Verify that a GCP VM (Vertex AI subnet) can reach Azure resources via the hybrid path: GCP → Cloud Interconnect → Mumbai hub → ExpressRoute → Azure VNet

# Run from GCP-AI-TEST VM (10.128.0.10 in GCP asia-south1)


echo "=== TEST 2.1: GCP→AZURE CROSS-CLOUD ROUTING ==="

## Step 1: Check routing table on GCP VM

ip route show | grep "10.100"
## Expected: 10.100.0.0/16 via GCP internal gateway (next-hop toward Cloud Interconnect)


## Step 2: Traceroute from GCP to Azure app server

echo ""
echo "--- Traceroute: GCP asia-south1 → Azure app server (10.100.2.10) ---"
traceroute -n -q 1 10.100.2.10
## Expected path:

##   1. GCP internal gateway (10.128.x.1)

##   2. Cloud Interconnect hop (169.254.0.1 — Google side)

##   3. Mumbai hub (169.254.0.2 — C8500 side)

##   4. ExpressRoute (169.254.200.1 — Microsoft side)

##   5. Azure VNet gateway (10.200.x.x)

##   6. Target (10.100.2.10)

## Hops MUST include Cloud Interconnect (169.254.0.x) and ExpressRoute (169.254.200.x)

## If traffic goes via public internet: FAIL


## Step 3: Latency measurement

echo ""
echo "--- Latency: GCP → Azure (via hybrid path) ---"
ping -c 10 10.100.2.10 | tail -3
## Expected: <50ms (Mumbai → ER → Azure Central India)

## If >50ms: May be routing via internet — check hub routing


## Step 4: Verify no public internet transit

echo ""
echo "--- Confirm no internet hops ---"
traceroute -n 10.100.2.10 | grep -v "10\." | grep -v "172\." | grep -v "169\." | grep -v "*"
## Expected: No public IP hops (no 203.x.x.x, no 8.x.x.x etc.)

Pass Criteria: - Traceroute includes 169.254.0.x (Cloud Interconnect) AND 169.254.200.x (ExpressRoute) hops - Latency < 50ms GCP asia-south1 → Azure Central India - Zero public internet hops

---

TEST 2.2: Branch to Azure SQL End-to-End

#!/bin/bash

## branch_azure_sql_test.sh


echo "=== TEST 2.2: BRANCH→AZURE SQL END-TO-END ==="

## Test from multiple branch sites

BRANCHES=("blr-user-01:192.168.50.100" "del-user-01:192.168.51.100" "hyd-user-01:192.168.52.100")
SQL_PRIVATE_IP="10.100.10.10"
SQL_PORT="1433"

for BRANCH_ENTRY in "${BRANCHES[@]}"; do
  BRANCH_HOST=$(echo $BRANCH_ENTRY | cut -d: -f1)
  BRANCH_IP=$(echo $BRANCH_ENTRY | cut -d: -f2)

  echo ""
  echo "--- Testing from: $BRANCH_HOST ($BRANCH_IP) ---"

## Test 1: Port connectivity

  RESULT=$(ssh $BRANCH_HOST "timeout 5 bash -c \"echo '' > /dev/tcp/$SQL_PRIVATE_IP/$SQL_PORT\" && echo 'OPEN' || echo 'CLOSED'" 2>/dev/null)
  [[ "$RESULT" == "OPEN" ]] && echo "  ✅ Port $SQL_PORT OPEN (private endpoint reachable)" \
    || echo "  ❌ Port $SQL_PORT CLOSED"

## Test 2: DNS resolves to private IP

  DNS_RESULT=$(ssh $BRANCH_HOST "nslookup abhavtech-sql.database.windows.net 10.252.1.100 2>/dev/null | grep 'Address:' | tail -1 | awk '{print \$2}'" 2>/dev/null)
  [[ "$DNS_RESULT" == "10.100.10."* ]] && echo "  ✅ DNS resolves to private: $DNS_RESULT" \
    || echo "  ❌ DNS returned: $DNS_RESULT (expected 10.100.10.x)"

## Test 3: Traceroute (confirm via SD-WAN hub, not internet)

  HOP_COUNT=$(ssh $BRANCH_HOST "traceroute -n -q 1 -w 2 10.100.10.10 2>/dev/null | grep -c '10\.'")
  echo "  Path: $HOP_COUNT private hops (expected: branch→hub→azure = ~4-6 hops)"

## Test 4: SQL Server connection test (if sqlcmd available)

  if ssh $BRANCH_HOST "which sqlcmd" > /dev/null 2>&1; then
    SQL_RESULT=$(ssh $BRANCH_HOST "sqlcmd -S 10.100.10.10 -U testuser -P 'TestPassword123!' -Q 'SELECT @@VERSION' 2>&1 | head -1")
    [[ "$SQL_RESULT" == *"Microsoft SQL"* ]] && echo "  ✅ SQL query successful" \
      || echo "  ⚠️  SQL connection result: $SQL_RESULT"
  fi
done

---

TEST 2.3: Branch to GCP Vertex AI API

#!/bin/bash

## branch_gcp_vertex_test.sh


echo "=== TEST 2.3: BRANCH→GCP VERTEX AI API ==="

## Test from Bangalore branch via hub backhaul to GCP

## Path: Branch → SD-WAN MPLS → Mumbai hub → Cloud Interconnect → GCP


echo ""
echo "--- From MUM-TEST-01 (Mumbai Hub → GCP) ---"
ssh mum-test-01 "
## Fetch a GCP access token

  TOKEN=\$(curl -s 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token' \
    -H 'Metadata-Flavor: Google' 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)[\"access_token\"])' 2>/dev/null)

  if [[ -z \"\$TOKEN\" ]]; then
## Not on GCP — use service account key

    TOKEN=\$(gcloud auth print-access-token 2>/dev/null)
  fi

  echo 'Testing Vertex AI API from Mumbai hub...'
  START=\$(date +%s%3N)
  HTTP_STATUS=\$(curl -s -o /dev/null -w '%{http_code}' \
    -H \"Authorization: Bearer \$TOKEN\" \
    -H 'Content-Type: application/json' \
    'https://asia-south1-aiplatform.googleapis.com/v1/projects/abhavtech-ai/locations/asia-south1/models')
  END=\$(date +%s%3N)
  LATENCY=\$((END - START))

  echo \"HTTP Status: \$HTTP_STATUS\"
  echo \"Latency: \${LATENCY}ms\"
  [[ \"\$HTTP_STATUS\" =~ ^[234] ]] && echo '✅ PASS: API reachable' || echo '⚠️  Check status'
  [[ \$LATENCY -lt 30 ]] && echo '✅ PASS: Latency OK (<30ms via interconnect)' \
    || echo '⚠️  High latency (\${LATENCY}ms) — may be routing via internet'
"

echo ""
echo "--- Cross-region: GCP asia-south1 → Azure Central India ---"
ssh gcp-ai-test "
  curl -s -o /dev/null -w 'Azure app server latency: %{time_total}s\nHTTP: %{http_code}\n' \
    http://10.100.2.10:8080/health
"

---

TEST 2.4: Microsoft Teams Voice Quality (MOS via ThousandEyes)

Objective: Validate Teams MOS score ≥ 4.0 from all hub sites using ThousandEyes RTP Stream test

#!/bin/bash

## teams_mos_test.sh


echo "=== TEST 2.4: TEAMS VOICE QUALITY VIA THOUSANDEYES ==="

TE_TOKEN="<TE_BEARER_TOKEN>"
TE_BASE="https://api.thousandeyes.com/v7"

## Get test results for Teams voice tests

curl -s -X GET "$TE_BASE/test-results?testType=rtp-stream&appName=teams" \
  -H "Authorization: Bearer $TE_TOKEN" | \
  python3 -c "
import json, sys
data = json.load(sys.stdin)

print('Agent               | MOS   | Loss%  | Jitter | Latency | Status')
print('-' * 70)

for result in data.get('results', []):
    agent = result.get('agentName', 'Unknown')[:20]
    mos = result.get('mos', 0)
    loss = result.get('loss', 0)
    jitter = result.get('jitter', 0)
    latency = result.get('latency', 0)

    if mos >= 4.0 and loss <= 0.5 and jitter <= 10:
        status = '✅ PASS'
    elif mos >= 3.5:
        status = '⚠️  WARN'
    else:
        status = '❌ FAIL'

    print(f'{agent:20} | {mos:5.2f} | {loss:5.1f}% | {jitter:4.0f}ms | {latency:5.0f}ms | {status}')
"

echo ""
echo "Pass Criteria: MOS ≥ 4.0, Loss ≤ 0.5%, Jitter ≤ 10ms from all 6 hub agents"

Manual Verification — Check DSCP marking on ExpressRoute:

! On MUM-HUB-01 — confirm Teams marked EF and using ExpressRoute
show sdwan app-fwd cflowd flows | grep -i teams | head -5
! Expected: ms-teams-media flows with dscp=46 (EF), color=azure-expressroute

! Verify QoS EF class is counting packets
show policy-map interface TenGigabitEthernet0/0/2.4001 class QOS-EF
! Expected: Packet count incrementing, no drops

---

TEST 2.5: O365 Exchange Path Verification (ExpressRoute, Not Internet)

#!/bin/bash

## o365_path_test.sh


echo "=== TEST 2.5: O365 EXCHANGE VIA EXPRESSROUTE ==="

for HUB in mum-hub-01 lon-hub-01 nj-hub-01; do
  echo ""
  echo "--- $HUB ---"

## Check routing for Exchange Online prefix

  ROUTE=$(ssh admin@$HUB "show ip route 40.107.0.0" 2>/dev/null)

  if echo "$ROUTE" | grep -q "169.254.20"; then
    echo "  ✅ Exchange (40.107.0.0/17): Routes via ExpressRoute peer"
  else
    echo "  ❌ Exchange: NOT via ExpressRoute"
    echo "  Route: $ROUTE"
  fi

## Verify not using internet DIA for O365

  DIA_FLOW=$(ssh admin@$HUB "show sdwan app-fwd cflowd flows | grep office365-exchange | grep dia" 2>/dev/null | wc -l)
  ER_FLOW=$(ssh admin@$HUB "show sdwan app-fwd cflowd flows | grep office365-exchange | grep expressroute" 2>/dev/null | wc -l)

  echo "  Exchange flows: ExpressRoute=$ER_FLOW, DIA=$DIA_FLOW"
  [[ $ER_FLOW -gt 0 && $DIA_FLOW -eq 0 ]] && echo "  ✅ All Exchange traffic via ExpressRoute" \
    || [[ $ER_FLOW -gt 0 ]] && echo "  ⚠️  Some Exchange via DIA (check AAR policy)" \
    || echo "  ⚠️  No active Exchange flows (test during business hours)"
done

## ThousandEyes: Compare O365 latency across paths

echo ""
echo "--- ThousandEyes O365 Latency (2-day average) ---"
curl -s "$TE_BASE/test-results?testType=http-server&url=outlook.office365.com" \
  -H "Authorization: Bearer $TE_TOKEN" | \
  python3 -c "
import json, sys
data = json.load(sys.stdin)
for r in data.get('results', []):
    lat = r.get('responseTime', 0)
    agent = r.get('agentName', '?')[:20]
    status = '✅' if lat < 20 else ('⚠️' if lat < 50 else '❌')
    print(f'{status} {agent:20} {lat:5}ms (threshold: <20ms via ExpressRoute)')
"

---

TEST 2.6–2.7: SaaS Probe and Guest Isolation

## TEST 2.6: Salesforce Cloud OnRamp Best-Path Selection

echo "=== TEST 2.6: SALESFORCE CLOUD ONRAMP ==="

## Get SaaS probe results from vManage

curl -k -s -b /tmp/cookies.txt \
  "https://vmanage.abhavtech.com/dataservice/cloudonramp/saas/gateway/apps" | \
  python3 -c "
import json, sys
data = json.load(sys.stdin)
for entry in data.get('data', []):
    if 'salesforce' in entry.get('appName','').lower():
        print(f'Site: {entry.get(\"deviceName\",\"?\")} | Latency: {entry.get(\"latency\",0)}ms | Path: {entry.get(\"bestPath\",\"?\")}')
"
## Expected: All sites <100ms latency to Salesforce via best-probe path


echo ""
echo "=== TEST 2.7: GUEST VPN ISOLATION ==="

## Run from a simulated guest host on VPN 3

## Test 1: Guest cannot reach corporate

CORP_RESULT=$(ssh admin@mum-hub-01 "ping vrf 3 10.252.1.100 repeat 3 timeout 1 | tail -1" 2>/dev/null)
echo "$CORP_RESULT" | grep -q "0 success" && echo "  ✅ Corporate unreachable from Guest VPN 3" \
  || echo "  ❌ CRITICAL: Corporate reachable from Guest VPN!"

## Test 2: Guest cannot reach Azure VNet

AZURE_RESULT=$(ssh admin@mum-hub-01 "ping vrf 3 10.100.10.10 repeat 3 timeout 1 | tail -1" 2>/dev/null)
echo "$AZURE_RESULT" | grep -q "0 success" && echo "  ✅ Azure VNet unreachable from Guest VPN 3" \
  || echo "  ❌ CRITICAL: Azure VNet reachable from Guest VPN!"

## Test 3: Guest CAN reach internet (via DIA)

INET_RESULT=$(ssh admin@mum-hub-01 "ping vrf 3 8.8.8.8 repeat 3 | tail -1" 2>/dev/null)
echo "$INET_RESULT" | grep -qv "0 success" && echo "  ✅ Internet reachable from Guest VPN 3" \
  || echo "  ❌ Internet not reachable from Guest (DIA issue)"

---

TEST 2.8–2.10: SD-Access Handoff, AppDynamics, Splunk

## TEST 2.8: SD-Access VN → SD-WAN VPN Routing

echo "=== TEST 2.8: SD-ACCESS HANDOFF VERIFICATION ==="

## On MUM-HUB-01: BGP from SD-Access border

ssh admin@mum-hub-01 "show bgp vpnv4 unicast vrf 1 summary | grep 65001"
## Expected: 10.240.1.1 in ESTABLISHED state with active prefix count


ssh admin@mum-hub-01 "show bgp vpnv4 unicast vrf 1 | include 10.100"
## Expected: All Mumbai SD-Access subnets (10.100.1.0/24 etc.) present in VRF 1


## From London hub, can we reach Mumbai corporate LAN via OMP?

ssh admin@lon-hub-01 "ping vrf 1 10.100.1.50 repeat 5" 2>/dev/null | tail -2
## Expected: 5/5 success (London hub → SD-WAN fabric → Mumbai hub VPN 1 → SD-Access fabric)


## TEST 2.9: AppDynamics Transaction Tracing

echo ""
echo "=== TEST 2.9: APPDYNAMICS TRANSACTION TRACE ==="

## Trigger a test transaction: Bangalore branch user → Azure SQL

## This should appear as a traced business transaction in AppDynamics


## Check AppDynamics API for recent transactions

curl -s "https://abhavtech.saas.appdynamics.com/controller/rest/applications/MultiCloud-App/events" \
  -u "admin@abhavtech:<PASSWORD>" \
  -G --data-urlencode "time-range-type=BEFORE_NOW" \
  --data-urlencode "duration-in-mins=30" \
  -H "Accept: application/json" | \
  python3 -c "
import json, sys
data = json.load(sys.stdin)
events = data if isinstance(data, list) else data.get('events', [])
print(f'Events in last 30 min: {len(events)}')
for e in events[:5]:
    print(f'  {e.get(\"eventTime\",\"?\")} | {e.get(\"type\",\"?\")} | {e.get(\"message\",\"?\")[:60]}')
"

## TEST 2.10: Splunk Telemetry Sources

echo ""
echo "=== TEST 2.10: SPLUNK TELEMETRY COVERAGE ==="

SPLUNK_HOST="splunk.abhavtech.com"
SPLUNK_TOKEN="<SPLUNK_HEC_TOKEN>"

SOURCES=(
  "index=dnac sourcetype=dnac:assurance" "DNAC Assurance"
  "index=sdwan sourcetype=vmanage" "vManage SD-WAN"
  "index=thousandeyes sourcetype=thousandeyes:test:result" "ThousandEyes"
  "index=appdynamics sourcetype=appdynamics:apm" "AppDynamics"
  "index=ise sourcetype=cisco:ise:syslog" "ISE"
  "index=umbrella sourcetype=cisco:umbrella:dns" "Umbrella DNS"
)

i=0
while [[ $i -lt ${#SOURCES[@]} ]]; do
  SEARCH="${SOURCES[$i]}"
  LABEL="${SOURCES[$((i+1))]}"

  COUNT=$(curl -s -k -u "admin:<PASSWORD>" \
    "https://$SPLUNK_HOST:8089/services/search/jobs/oneshot" \
    --data-urlencode "search=$SEARCH earliest=-15m | stats count" \
    -d "output_mode=json" | \
    python3 -c "import json,sys; d=json.load(sys.stdin); print(d['results'][0]['count'] if d.get('results') else '0')" 2>/dev/null)

  [[ "$COUNT" -gt 0 ]] && echo "  ✅ $LABEL: $COUNT events (last 15 min)" \
    || echo "  ❌ $LABEL: No events — check source integration"

  i=$((i+2))
done

---