VMware vSphere Knowledge Base

Check current admission control policy and slot sizes

# PowerCLI — view HA admission control settings:
Get-Cluster "ClusterName" | Select Name, HAAdmissionControlEnabled, HAFailoverLevel

# Check slot size information:
Get-Cluster "ClusterName" | Get-View | Select -ExpandProperty Summary |
  Select CurrentFailoverLevel, NumVmotionsCopy

Identify VMs with large reservations inflating slot size

# PowerCLI — find VMs with CPU/memory reservations:
Get-VM -Location "ClusterName" | Where-Object {
    $_.ExtensionData.ResourceConfig.CpuAllocation.Reservation -gt 0 -or
    $_.ExtensionData.ResourceConfig.MemoryAllocation.Reservation -gt 0
} | Select Name,
  @{N='CPUResMHz';E={$_.ExtensionData.ResourceConfig.CpuAllocation.Reservation}},
  @{N='MemResMB';E={$_.ExtensionData.ResourceConfig.MemoryAllocation.Reservation}} |
  Sort-Object MemResMB -DescendingCopy

Review cluster capacity vs. reservation

# PowerCLI — cluster capacity overview:
$cluster = Get-Cluster "ClusterName"
$hosts = $cluster | Get-VMHost
$hosts | Select Name, ConnectionState,
  @{N='CPUTotalMHz';E={$_.CpuTotalMHz}},
  @{N='CPUUsageMHz';E={$_.CpuUsageMHz}},
  @{N='MemTotalGB';E={[math]::Round($_.MemoryTotalGB,1)}},
  @{N='MemUsageGB';E={[math]::Round($_.MemoryUsageGB,1)}} |
  Format-Table -AutoSizeCopy

Adjust admission control policy to percentage-based

# vSphere Client:
# Cluster → Configure → vSphere Availability → Edit
# Admission Control Policy: "Cluster resource percentage"
# Set CPU/Memory to match your N+1 requirements (e.g., 25% for 4-host cluster)

# PowerCLI:
$spec = New-Object VMware.Vim.ClusterConfigSpecEx
$spec.DasConfig = New-Object VMware.Vim.ClusterDasConfigInfo
$spec.DasConfig.AdmissionControlEnabled = $true
# Apply via Set-Cluster or Reconfigure-ClusterCopy

Temporary: Disable admission control to power on critical VMs

# WARNING: Only use temporarily — re-enable immediately after
# vSphere Client:
# Cluster → Configure → vSphere Availability → Edit
# Uncheck "Enable Admission Control"

# Power on the VM, then RE-ENABLE admission control:
# Cluster → Configure → vSphere Availability → Edit
# Check "Enable Admission Control"Copy

Check FDM status and logs on each host

# SSH to each ESXi host:
/etc/init.d/fdm status

# Check FDM election logs:
cat /var/log/fdm.log | grep -i "master\|elect\|error\|fail" | tail -40

# Check if FDM is listening:
esxcli network ip connection list | grep 8182Copy

Verify management network connectivity between hosts

# From each ESXi host, ping other hosts on management network:
vmkping -I vmk0 <other-host-mgmt-ip>

# Test FDM port connectivity:
nc -z <other-host-mgmt-ip> 8182

# Check firewall rules:
esxcli network firewall ruleset list | grep fdmCopy

Check DNS resolution for all cluster hosts

# Verify each host can resolve other hosts:
nslookup esxi-host01.domain.local
nslookup esxi-host02.domain.local

# Check /etc/hosts for local overrides:
cat /etc/hosts

# Verify DNS settings:
esxcli network ip dns server listCopy

Restart management agents on all hosts

# SSH to each ESXi host:
/etc/init.d/hostd restart
/etc/init.d/vpxa restart
/etc/init.d/fdm restart

# Verify FDM is running:
/etc/init.d/fdm status
# Expected: "VMware Fault Domain Manager is running"Copy

Reconfigure HA on the entire cluster

# vSphere Client:
# 1. Cluster → Configure → vSphere Availability → Edit
# 2. Uncheck "Turn ON vSphere HA" → OK
# 3. Wait for all hosts to show "HA Disabled"
# 4. Re-enable: Check "Turn ON vSphere HA" → OK

# PowerCLI alternative:
Set-Cluster "ClusterName" -HAEnabled:$false -Confirm:$false
Start-Sleep -Seconds 30
Set-Cluster "ClusterName" -HAEnabled:$true -Confirm:$falseCopy

Check current heartbeat datastore configuration

# vSphere Client:
# Cluster → Configure → vSphere Availability → Heartbeat Datastores
# Verify at least 2 datastores are listed

# PowerCLI — list heartbeat datastores:
$cluster = Get-Cluster "ClusterName"
$clusterView = $cluster | Get-View
$clusterView.RetrieveDasAdvancedRuntimeInfo().HeartbeatDatastoreInfo |
  Select Datastore, @{N='Name';E={
    (Get-View $_.Datastore).Name
  }}Copy

Verify shared datastore visibility across all hosts

# PowerCLI — find datastores shared by all hosts:
$hosts = Get-Cluster "ClusterName" | Get-VMHost
$allDS = $hosts | ForEach-Object { ($_ | Get-Datastore).Name }
$shared = $allDS | Group-Object | Where-Object { $_.Count -eq $hosts.Count }
$shared | Select Name, Count | Format-Table -AutoSizeCopy

Add shared datastores and configure heartbeat selection

# vSphere Client:
# Cluster → Configure → vSphere Availability → Heartbeat Datastores → Edit
# Select: "Use datastores from the specified list and complement automatically"
# Choose at least 2 shared VMFS/NFS datastores accessible from all hosts

# Ensure datastores are on different storage arrays for true redundancyCopy

Verify heartbeat configuration after changes

# After adding datastores, reconfigure HA:
# Right-click each host → Reconfigure for vSphere HA

# Verify in FDM log on each host:
cat /var/log/fdm.log | grep -i "heartbeat" | tail -20
# Expected: "Using datastore [DatastoreName] for heartbeating"Copy

Check FDM status and crash logs

# SSH to the affected ESXi host:
/etc/init.d/fdm status

# Check for recent FDM crash/error entries:
cat /var/log/fdm.log | grep -i "error\|crash\|fatal\|exception" | tail -30

# Check for core dumps:
ls -la /var/core/ | grep fdm

# Check ramdisk usage:
vdf -hCopy

Verify FDM configuration integrity

# Check FDM config directory:
ls -la /etc/opt/vmware/fdm/

# Verify SSL certificates:
ls -la /etc/vmware/ssl/
openssl x509 -in /etc/vmware/ssl/rui.crt -noout -dates

# Check management VMkernel adapter:
esxcli network ip interface list | grep -A5 vmk0Copy

Restart FDM agent and clear stale config

# Stop FDM:
/etc/init.d/fdm stop

# Clear stale FDM state (safe to remove — vCenter will repush):
rm -rf /etc/opt/vmware/fdm/*

# Restart management agents:
/etc/init.d/hostd restart
/etc/init.d/vpxa restart

# Start FDM:
/etc/init.d/fdm start
/etc/init.d/fdm statusCopy

Reconfigure HA on the cluster from vCenter

# vSphere Client:
# Right-click affected host → Reconfigure for vSphere HA

# If that fails, disable/re-enable HA on the cluster:
# Cluster → Configure → vSphere Availability → Edit
# Uncheck HA → OK → Wait → Re-check HA → OK

# Verify agent status after reconfiguration:
# Host → Monitor → vSphere HA → Heartbeat
# Status should show "Connected" and "Agent running"Copy

Enable VMCP on the cluster

# vSphere Client:
# Cluster → Configure → vSphere Availability → Edit
# Under "Failure conditions and VM response":
#   → Datastore with PDL: "Power off and restart VMs"
#   → Datastore with APD:
#       Response: "Power off and restart VMs (conservative)"
#       Delay: 180 seconds (default)
#       VM Storage Protection: "Disabled" / "Enabled"

# PowerCLI — check current VMCP settings:
Get-Cluster "ClusterName" | Get-View |
  Select -ExpandProperty ConfigurationEx |
  Select -ExpandProperty DasConfig |
  Select -ExpandProperty DefaultVmSettings |
  Select -ExpandProperty VmComponentProtectionSettingsCopy

Configure advanced APD/PDL settings

# Advanced HA settings for VMCP tuning:
# Cluster → Configure → vSphere Availability → Advanced Options

# Key settings:
# das.config.fdm.reportFailed.APDTimeout = 140
#   (seconds before APD is reported — default 140)
# vpxd.das.config.fdm.storageVMotion.APDTimeout = 180
#   (seconds to wait before taking action on APD VMs)

# Verify on ESXi host:
esxcli storage core device list | grep -A3 "Status\|APD"Copy

Test VMCP response with non-production VMs

# Test APD scenario (simulate by blocking storage paths):
# 1. Identify test datastore and test VM
# 2. Block iSCSI/FC paths at switch level (controlled test)
# 3. Monitor: Cluster → Monitor → vSphere HA → VM Protection
# 4. Verify VM is restarted on a host with storage access
# 5. Restore paths and verify datastore reconnects

# Check VMCP events:
# vCenter → Monitor → Events → filter "VMCP\|APD\|PDL"Copy

Configure FT logging network and enable FT

# 1. Create a dedicated FT logging VMkernel adapter:
#    Host → Configure → Networking → VMkernel adapters → Add
#    Enable "Fault Tolerance logging" service
#    Assign to a dedicated 10 GbE portgroup

# 2. Enable FT on the VM:
#    Right-click VM → Fault Tolerance → Turn On Fault Tolerance

# PowerCLI — check FT compatibility:
Get-VM "CriticalVM" | Get-View |
  Select @{N='FTCompatible';E={$_.Runtime.FaultToleranceState}}Copy

Troubleshoot FT logging latency and bandwidth

# Check FT latency on the host:
esxtop
# Press 'n' for network view
# Look at FT logging VMkernel: check %DRPTX, %DRPRX, MbRX/MbTX

# Check FT status in vCenter:
# VM → Summary → Fault Tolerance section
# "Secondary VM" should show "Running" and "Up to date"

# If FT latency > 8ms, check:
# - Network bandwidth (10 GbE recommended)
# - NIC teaming / load balancing policy
# - Physical switch MTU (jumbo frames if needed)Copy

Resolve common FT compatibility issues

# FT not available? Check these VM settings:
# 1. Remove all snapshots: VM → Snapshots → Manage Snapshots → Delete All
# 2. Convert disk to thick eager zeroed:
vmkfstools -i "thin.vmdk" -d eagerzeroedthick "thick.vmdk"

# 3. Verify CPU compatibility:
esxcli hardware cpu global get

# 4. Disable features incompatible with FT:
# - No USB controllers, no PCI passthrough
# - No Storage DRS, no VM encryption (pre-vSphere 7 U2)
# - Remove CD/DVD devices if connected to client deviceCopy

Detect network partition and multiple masters

# Check FDM logs for partition events:
cat /var/log/fdm.log | grep -i "partition\|split\|multiple master\|isolat" | tail -30

# Check which host is HA master:
# On each host:
cat /var/log/fdm.log | grep "I am the master" | tail -5

# If more than one host claims master → split brain active!Copy

Check isolation address and network connectivity

# Verify isolation address configuration:
vim-cmd hostsvc/advopt/view das.isolationAddress0
vim-cmd hostsvc/advopt/view das.useDefaultIsolationAddress

# Test isolation address reachability:
vmkping -I vmk0 <isolation-address-ip>

# Check management network NIC status:
esxcli network nic list
esxcli network vswitch standard listCopy

Restore network connectivity immediately

# Priority 1: Restore the physical network
# - Check switch ports, VLANs, uplinks for management network
# - Verify trunk ports are passing management VLAN
# - Check NIC teaming failover status on each host

# From affected host — verify connectivity:
vmkping -I vmk0 <vcenter-ip>
vmkping -I vmk0 <other-host-mgmt-ip>

# Restart networking if needed:
esxcli network restartCopy

Resolve split-brain VMs and reconfigure HA

# If a VM is running on two hosts:
# 1. Identify which instance has the latest state (check VM console)
# 2. Power off the STALE instance via direct host connection
#    (DCUI or direct ESXi vSphere Client)
# 3. Verify VMDK lock is released:
vmkfstools -D /vmfs/volumes/datastore/vm/vm.vmdk

# After network is restored — reconfigure HA:
# Cluster → Right-click → Reconfigure for vSphere HA

# Verify single master:
cat /var/log/fdm.log | grep "I am the master" | tail -5Copy

Configure isolation address and response to prevent recurrence

# Set proper isolation address (NOT the default gateway alone):
# Cluster → Configure → vSphere Availability → Advanced Options
# Add:
#   das.isolationAddress0 = 10.0.0.1  (gateway)
#   das.isolationAddress1 = 10.0.0.2  (secondary reliable IP)
#   das.useDefaultIsolationAddress = false

# Set isolation response to "Shut down and restart":
# Cluster → Configure → vSphere Availability → Edit
# Host Isolation Response: "Shut down and restart VMs"Copy

Install health provider and enable Proactive HA

# 1. Install the vendor health provider plugin in vCenter:
#    Administration → Solutions → vCenter Server Extensions
#    Deploy Dell OMIVV / HPE Amplifier / Lenovo XClarity

# 2. Enable Proactive HA:
#    Cluster → Configure → Proactive HA → Edit
#    Check "Proactive HA Enabled"
#    Automation Level: "Automated" (recommended)
#    Remediation: "Quarantine mode" or "Maintenance mode"

# 3. Verify health provider registration:
#    Cluster → Configure → Proactive HA → Providers
#    Status should show "Registered" for your vendorCopy

Configure failure conditions and host monitoring

# Proactive HA failure conditions:
# Cluster → Configure → Proactive HA → Failure Conditions
# Map hardware component types to responses:
#   - Fan failure → Quarantine mode (moderate)
#   - Power supply failure → Maintenance mode (severe)
#   - Memory error → Maintenance mode (severe)
#   - Storage controller → Maintenance mode (severe)

# Verify DRS is set to "Fully Automated" for evacuation:
Get-Cluster "ClusterName" | Select DrsEnabled, DrsAutomationLevel
# DrsAutomationLevel should be "FullyAutomated"Copy

Test and validate Proactive HA response

# Simulate hardware degradation (vendor-specific):
# Dell: OMIVV → Test Hardware Alert
# HPE: iLO → simulate fan failure via RIBCL

# Monitor response:
# vCenter → Monitor → Tasks → look for DRS migration events
# Host → Monitor → Hardware Status → check component health

# Verify evacuation:
Get-VMHost "affected-host" | Get-VM | Select Name, PowerState, VMHost
# All VMs should have migrated to other hostsCopy

Set isolation response policy and custom addresses

# vSphere Client:
# Cluster → Configure → vSphere Availability → Edit
# Host Isolation Response: "Shut down and restart VMs" (recommended)

# Configure custom isolation addresses:
# Cluster → Configure → vSphere Availability → Advanced Options → Add:
das.isolationAddress0 = 10.0.0.1     # Default gateway
das.isolationAddress1 = 10.0.0.2     # Secondary switch/router
das.isolationAddress2 = 10.0.0.254   # Out-of-band management IP
das.useDefaultIsolationAddress = false  # Use custom onlyCopy

Configure per-VM isolation response overrides

# For VMs that should NOT be shut down during isolation:
# Cluster → Configure → vSphere Availability → VM Overrides → Add
# Select VM → Host Isolation Response: "Leave powered on" or "Disabled"

# Use overrides for:
# - VMs that handle their own clustering (e.g., Oracle RAC, WSFC)
# - Infrastructure VMs that must stay running (DNS, AD)

# PowerCLI — list current VM overrides:
$cluster = Get-Cluster "ClusterName"
(Get-View $cluster).ConfigurationEx.DasVmConfig |
  Select @{N='VM';E={(Get-View $_.Key).Name}},
    @{N='IsolationResponse';E={$_.DasSettings.IsolationResponse}} |
  Format-Table -AutoSizeCopy

Verify isolation address reachability from all hosts

# SSH to each ESXi host and verify:
vmkping -I vmk0 10.0.0.1     # Isolation address 0
vmkping -I vmk0 10.0.0.2     # Isolation address 1
vmkping -I vmk0 10.0.0.254   # Isolation address 2

# Verify settings applied:
vim-cmd hostsvc/advopt/view das.isolationAddress0
vim-cmd hostsvc/advopt/view das.isolationAddress1
vim-cmd hostsvc/advopt/view das.useDefaultIsolationAddressCopy

Set VM restart priorities via VM overrides

# vSphere Client:
# Cluster → Configure → vSphere Availability → VM Overrides → Add
# Select VM → VM Restart Priority:
#   AD/DNS server → "Highest"
#   Database server → "High"
#   App servers → "Medium" (default)
#   Dev/Test VMs → "Low"
#   Template VMs → "Disabled"

# PowerCLI — set restart priority:
$cluster = Get-Cluster "ClusterName"
$vm = Get-VM "DC01"
$spec = New-Object VMware.Vim.ClusterConfigSpecEx
$vmConfig = New-Object VMware.Vim.ClusterDasVmConfigSpec
$vmConfig.Operation = "add"
$vmConfig.Info = New-Object VMware.Vim.ClusterDasVmConfigInfo
$vmConfig.Info.Key = $vm.ExtensionData.MoRef
$vmConfig.Info.DasSettings = New-Object VMware.Vim.ClusterDasVmSettings
$vmConfig.Info.DasSettings.RestartPriority = "highest"
$spec.DasVmConfigSpec += $vmConfig
$cluster.ExtensionData.ReconfigureComputeResource($spec, $true)Copy

Enable VM Monitoring for guest-level health checks

# vSphere Client:
# Cluster → Configure → vSphere Availability → Edit
# VM Monitoring → "VM Monitoring Only" or "VM and Application Monitoring"
# Monitoring sensitivity: "Medium" (60 second heartbeat window)

# VM Monitoring settings:
#   Failure interval: 30 seconds (time between heartbeat checks)
#   Minimum uptime: 120 seconds (grace period after power on)
#   Maximum per-VM resets: 3 (within the time window)
#   Maximum resets time window: 1 hour

# Verify VMware Tools heartbeat is working:
Get-VM "CriticalVM" | Select Name,
  @{N='ToolsStatus';E={$_.ExtensionData.Guest.ToolsStatus}},
  @{N='Heartbeat';E={$_.ExtensionData.GuestHeartbeatStatus}}Copy

Configure orchestrated restart with VM dependencies

# For VMs with startup dependencies (e.g., DB must start before app):
# Cluster → Configure → VM/Host Rules → Add Rule

# Example orchestrated restart order:
#   1. "Highest" priority: DC01, DNS01 (start first)
#   2. "High" priority: SQL01 (waits for Highest VMs to be running)
#   3. "Medium" priority: AppServer01 (waits for High VMs)

# Use VM-VM dependency rules:
# Cluster → Configure → vSphere Availability → VM Overrides
# Set "Orchestrated Restart" dependencies:
#   AppServer01 depends on SQL01
#   SQL01 depends on DC01

# Verify restart order:
# vCenter → Monitor → vSphere HA → Configuration Issues
# Should show no warnings about restart orderCopy