VMware vSphere Knowledge Base

Check FDM agent status on each host

# SSH to each ESXi host:
/etc/init.d/fdm status

# Check FDM logs for errors:
cat /var/log/fdm.log | grep -i "error\|fail\|isolat" | tail -30Copy

Verify HA advanced settings and isolation address

# Check HA isolation response and address:
esxcli system settings advanced list -o /das/

# Key settings to verify:
# das.isolationAddress0 — should be a reliable gateway IP
# das.useDefaultIsolationAddress — verify true/falseCopy

Check heartbeat datastore configuration

# From vSphere Client:
# Cluster → Configure → vSphere Availability → Heartbeat Datastores
# Verify at least 2 heartbeat datastores are listed

# PowerCLI:
Get-Cluster "ClusterName" | Get-HASetting | Select *heartbeat*Copy

Restart FDM agent on affected hosts

# SSH to the ESXi host:
/etc/init.d/fdm restart

# Verify agent is running:
/etc/init.d/fdm status
# Expected: "VMware Fault Domain Manager is running"Copy

Reconfigure HA on the cluster

# PowerCLI — force HA reconfiguration on all hosts:
$cluster = Get-Cluster "ClusterName"
$cluster | Get-VMHost | ForEach-Object {
    $_ | Set-VMHost -State "Connected" -Confirm:$false
}

# Or from vSphere Client:
# Right-click host → Reconfigure for vSphere HACopy

Configure isolation address and admission control

# Set a reliable isolation address (e.g., default gateway):
# Cluster → Configure → vSphere Availability → Advanced Options
# Add: das.isolationAddress0 = 10.0.0.1

# Adjust admission control:
# Cluster → Configure → vSphere Availability → Admission Control
# Set "Host failures cluster tolerates" = 1
# Choose "Cluster resource percentage" or "Dedicated failover hosts"Copy

Check DRS configuration and recommendations

# PowerCLI — Check DRS settings:
Get-Cluster "ClusterName" | Select Name, DrsEnabled, DrsAutomationLevel

# Get pending DRS recommendations:
Get-DrsRecommendation -Cluster "ClusterName" | Format-Table -AutoSizeCopy

Review DRS faults in vCenter events

# PowerCLI — Check for DRS faults:
Get-VIEvent -Entity (Get-Cluster "ClusterName") -MaxSamples 100 |
  Where-Object { $_.FullFormattedMessage -match "DRS|migration" } |
  Select CreatedTime, FullFormattedMessage | Format-Table -Wrap

# vCenter UI: Cluster → Monitor → Events → Filter "DRS"Copy

Check affinity rules and resource pool limits

# PowerCLI — List all DRS rules:
Get-DrsRule -Cluster "ClusterName" | Format-Table Name, Enabled, Type, VMIds

# Check resource pool limits:
Get-ResourcePool -Location (Get-Cluster "ClusterName") |
  Select Name, CpuLimitMHz, MemLimitGB, CpuReservationMHz, MemReservationGBCopy

Set DRS to Fully Automated with appropriate threshold

# PowerCLI:
Set-Cluster "ClusterName" -DrsAutomationLevel FullyAutomated -Confirm:$false

# vCenter UI:
# Cluster → Configure → vSphere DRS → Edit
# Automation Level: Fully Automated
# Migration Threshold: 3 (default, recommended for most environments)Copy

Review and adjust affinity rules

# List conflicting rules:
Get-DrsRule -Cluster "ClusterName" | Where { $_.Enabled } |
  Format-Table Name, Type, @{N="VMs";E={($_.VMIds | ForEach { (Get-VM -Id $_).Name }) -join ", "}}

# Disable overly restrictive rules:
Get-DrsRule -Cluster "ClusterName" -Name "Rule-Name" |
  Set-DrsRule -Enabled $false -Confirm:$falseCopy

Remove resource pool limits blocking migration

# PowerCLI — Remove CPU/Memory limits from resource pools:
Get-ResourcePool -Location (Get-Cluster "ClusterName") -Name "Problem-RP" |
  Set-ResourcePool -CpuLimitMHz $null -MemLimitGB $null -Confirm:$false

# Verify vMotion network is functional:
# Test from ESXi: vmkping -I vmk1 <destination_host_vmotion_ip>Copy

Verify vMotion VMkernel exists on both hosts

# On each ESXi host:
esxcli network ip interface list
# Look for a vmk with "VMotion" service enabled

# Check which services are enabled:
esxcli network ip interface tag get -i vmk1
# Should show: VMotionCopy

Test vMotion connectivity and MTU

# Ping with jumbo frames from source to destination:
vmkping -I vmk1 -d -s 8972 <destination_vmotion_ip>
# -d = don't fragment  -s 8972 = test 9000 MTU

# If this fails, check physical switch MTU and VLAN trunk:
esxcfg-vmknic -l
# Compare MTU values across all hostsCopy

Check CPU compatibility and EVC mode

# PowerCLI — Check EVC mode:
Get-Cluster "ClusterName" | Select Name, EVCMode

# Check individual host CPU:
Get-VMHost | Select Name, ProcessorType, MaxEVCMode

# From vCenter UI:Cluster → Configure → VMware EVC
# All hosts must support the configured EVC baselineCopy

Create or fix vMotion VMkernel adapter

# If vMotion vmk is missing, create one:
esxcli network ip interface add -i vmk2 -p "vMotion"

# Enable vMotion service on existing vmk:
# vCenter UI: Host → Configure → Networking → VMkernel adapters
# Edit vmk → Enable "vMotion" service

# PowerCLI:
New-VMHostNetworkAdapter -VMHost "esxi01" -PortGroup "vMotion-PG" `
  -VirtualSwitch "vSwitch1" -IP "10.10.20.11" -SubnetMask "255.255.255.0" `
  -VMotionEnabled $trueCopy

Fix MTU mismatch across the path

# Set consistent MTU on all ESXi hosts:
esxcli network ip interface set -i vmk1 -m 9000

# Set on vSwitch or dvSwitch:
esxcli network vswitch standard set -v vSwitch1 -m 9000

# IMPORTANT: Also configure physical switches:
# Cisco: interface GigabitEthernet1/0/1 → mtu 9216
# Verify end-to-end with vmkping -d -s 8972Copy

Enable EVC mode for mixed-CPU clusters

# PowerCLI — Set EVC mode:
Set-Cluster "ClusterName" -EVCMode "intel-broadwell" -Confirm:$false
# Choose the baseline matching your OLDEST host CPU generation

# Prerequisites:
# - All VMs must be powered off or at a compatible CPU level
# - DRS must be enabled on the cluster
# - No VM can expose CPU features above the EVC baselineCopy

Check vpxa agent status and restart if needed

# SSH to the ESXi host:
/etc/init.d/vpxa status
/etc/init.d/vpxa restart

# Check vpxa logs:
tail -50 /var/log/vpxa.log
# Look for: "SSL handshake failed", "Connection refused", "DNS lookup failed"Copy

Verify management network and DNS

# Check management VMkernel:
esxcli network ip interface list
# vmk0 should have correct IP and be Up

# Test DNS resolution:
nslookup vcenter.domain.com
# Must resolve to the correct vCenter IP

# Test connectivity to vCenter:
nc -z vcenter.domain.com 443
nc -z vcenter.domain.com 902Copy

Check active network connections

# List all active connections:
esxcli network ip connection list | grep -E "443|902"
# Verify connections to vCenter IP exist

# Check firewall rules:
esxcli network firewall ruleset list | grep -i vpxa
esxcli network firewall ruleset set -e true -r vpxHeartbeatsCopy

Reconnect host from vCenter

# vCenter UI: Right-click the disconnected host → Connection → Connect
# If prompted, re-enter root credentials

# PowerCLI:
Set-VMHost -VMHost "esxi-host01.domain.com" -State "Connected" -Confirm:$falseCopy

Fix DNS and network connectivity

# Fix DNS on the ESXi host:
esxcli network ip dns server add -s 10.0.0.10
esxcli network ip dns server list

# Fix management network if IP is wrong:
esxcli network ip interface ipv4 set -i vmk0 -I 10.0.1.50 -N 255.255.255.0 -t static

# Verify default gateway:
esxcfg-route -l
esxcfg-route 10.0.1.1Copy

Re-trust certificates if SSL handshake fails

# Remove and re-add the host to vCenter (last resort):
# 1. Disconnect host in vCenter
# 2. Remove host from inventory
# 3. Re-add host with FQDN

# Or reset vpxa certificate trust:
/etc/init.d/vpxa stop
rm /etc/vmware/vpxa/vpxa.cfg
/etc/init.d/vpxa start
# Then reconnect from vCenter (will re-push config)Copy

Collect and verify the coredump after reboot

# Check coredump configuration:
esxcli system coredump file list
esxcli system coredump partition list

# List available core dumps:
ls -la /var/core/
# Look for vmkernel-zdump.* files

# Copy coredump for analysis:
vm-support
# This generates a support bundle including the coredumpCopy

Analyze the PSOD backtrace

# Extract the stack trace from the coredump:
esxcli system coredump extract --corefile /var/core/vmkernel-zdump.1

# Key information from the PSOD screen:
# 1. "Module" line — identifies the faulting driver (e.g., nmlx5_core, ixgben)
# 2. Backtrace — shows the call stack at crash time
# 3. Build number — confirms ESXi version

# Check driver version:
esxcli software vib list | grep -i "nmlx5\|ixgbe\|qfle3\|lsi\|hpsa"Copy

Check hardware health after reboot

# Check for memory errors:
esxcli hardware memory get
# Look for "Uncorrectable ECC Errors"

# Check system event log (IPMI/iLO/iDRAC):
# Access BMC → System Event Log → Filter for "Memory", "CPU", "PCI"

# Verify firmware versions:
esxcli hardware platform get
esxcli software vib list | grep -i firmwareCopy

Update the faulting driver/firmware

# If the PSOD backtrace points to a driver (e.g., nmlx5_core):

# Check VMware HCL for the correct driver version:
# https://www.vmware.com/resources/compatibility/search.php

# Update driver VIB:
esxcli software vib install -d /tmp/driver-bundle.zip

# Or use vLCM (vSphere Lifecycle Manager):
# Cluster → Updates → Image → Edit → Add vendor addon/driverCopy

Apply ESXi patches if it's a known VMkernel bug

# Check VMware KB for your PSOD signature:
# Search KB for the module name + "PSOD"

# Apply the latest ESXi patch:
esxcli software profile update -p ESXi-8.0U2b-XXXXX-standard \
  -d https://hostupdate.vmware.com/software/VUM/PRODUCTION/main/vmw-depot-index.xml

# Or via vLCM baseline:
# Cluster → Updates → Baselines → Attach → RemediateCopy

Replace hardware if memory/CPU errors detected

# If BMC event log shows memory DIMM errors:
# 1. Identify the failing DIMM slot from BMC logs
# 2. Place host in maintenance mode (evacuate VMs)
# 3. Power off and replace the DIMM
# 4. Run vendor diagnostics before returning to production

# Configure coredump persistence for future analysis:
esxcli system coredump file add -d /vmfs/volumes/datastore1 -f coredump-esxi01
esxcli system coredump file set -p /vmfs/volumes/datastore1/coredump-esxi01
esxcli system coredump file set -e trueCopy

Attempt power-on and capture the error

# CLI power-on with verbose output:
vim-cmd vmsvc/power.on <vmid>

# Get VM ID first:
vim-cmd vmsvc/getallvms | grep "vm-name"

# Check vpxd.log for admission control details:
# On VCSA:
grep -i "admission\|insufficient\|power.on" /var/log/vmware/vpxd/vpxd.log | tail -30Copy

Check datastore free space and swap file requirements

# Check datastore space:
esxcli storage filesystem list
# Compare "Free" column — swap file needs (VM_Memory - Reservation) space

# PowerCLI:
Get-Datastore | Select Name,
  @{N="FreeGB";E={[math]::Round($_.FreeSpaceGB,2)}},
  @{N="CapacityGB";E={[math]::Round($_.CapacityGB,2)}},
  @{N="PercentFree";E={[math]::Round(($_.FreeSpaceGB/$_.CapacityGB)*100,1)}}Copy

Check HA admission control policy

# PowerCLI:
Get-Cluster "ClusterName" | Select Name, HAEnabled, HAAdmissionControlEnabled,
  HAFailoverLevel, @{N="Policy";E={$_.ExtensionData.Configuration.DasConfig.AdmissionControlPolicy.GetType().Name}}

# Check current cluster capacity vs reserved:
Get-VMHost -Location (Get-Cluster "ClusterName") |
  Select Name, @{N="CPU_Used%";E={[math]::Round($_.CpuUsageMhz/$_.CpuTotalMhz*100)}},
  @{N="Mem_Used%";E={[math]::Round($_.MemoryUsageGB/$_.MemoryTotalGB*100)}}Copy

Free datastore space or move VM to a datastore with space

# Find and remove orphaned VMDKs, old snapshots, ISO files:
# vCenter → Datastore → Files → Sort by size

# Storage vMotion the VM:
# PowerCLI:
Move-VM -VM "vm-name" -Datastore "datastore-with-space" -Confirm:$false

# Or reclaim space by consolidating snapshots:
vim-cmd vmsvc/snapshot.removeall <vmid>Copy

Adjust HA admission control

# Reduce the reserved capacity:
# Cluster → Configure → vSphere Availability → Edit
# Change from "Dedicated failover hosts" to "Cluster resource percentage"
# Set CPU/Memory percentage to a reasonable value (e.g., 25%)

# PowerCLI:
$spec = New-Object VMware.Vim.ClusterConfigSpecEx
$spec.DasConfig = New-Object VMware.Vim.ClusterDasConfigInfo
$spec.DasConfig.AdmissionControlEnabled = $true
# Adjust percentage-based policy as neededCopy

Add memory reservation to reduce swap file size

# Set 100% memory reservation (eliminates .vswp file):
# VM → Edit Settings → Resources → Memory → Reservation = Max

# PowerCLI:
Get-VM "vm-name" | Get-VMResourceConfiguration |
  Set-VMResourceConfiguration -MemReservationGB 16
# Set to match the VM's configured memoryCopy

Check VCSA service status and disk space

# SSH to VCSA (or use VAMI shell):
service-control --status
# Or list all services:
vmon-cli --list

# Check disk usage:
df -h
# Critical partitions: /storage/log, /storage/db, /storage/seat
# If any are at 95%+, that's likely the causeCopy

Review vpxd crash logs

# Check vpxd log for errors:
tail -100 /var/log/vmware/vpxd/vpxd.log
# Look for: "PANIC", "signal 11", "Cannot connect to database", "Out of memory"

# Check for certificate expiration:
for store in MACHINE_SSL_CERT TRUSTED_ROOT_CRLS; do
  /usr/lib/vmware-vmafd/bin/vecs-cli entry list --store "$store" --text | grep -A2 "Not After"
doneCopy

Check PostgreSQL database health

# Check if PostgreSQL is running:
service-control --status vmware-vpostgres

# Check DB connections:
/opt/vmware/vpostgres/current/bin/psql -U postgres -d VCDB \
  -c "SELECT count(*) FROM pg_stat_activity;"

# Check DB size:
/opt/vmware/vpostgres/current/bin/psql -U postgres -d VCDB \
  -c "SELECT pg_size_pretty(pg_database_size('VCDB'));"Copy

Free disk space on VCSA

# Clear old log files:
find /var/log/vmware/ -name "*.log" -mtime +7 -exec rm {} \;
find /storage/log/ -name "*.gz" -mtime +14 -exec rm {} \;

# Clear core dumps:
rm -f /storage/core/*

# Truncate large active logs (if disk is 100% full):
> /var/log/vmware/vpxd/vpxd-1.log
> /var/log/vmware/vpxd/vpxd-2.log

# Verify space freed:
df -h /storage/logCopy

Restart vCenter services

# Restart all services in correct order:
service-control --stop --all
service-control --start --all

# Or restart only vpxd:
service-control --stop vpxd
service-control --start vpxd

# Monitor startup:
tail -f /var/log/vmware/vpxd/vpxd.log
# Wait for: "Server is RUNNING"Copy

Renew expired certificates (if STS cert expired)

# Check STS certificate expiration:
/usr/lib/vmware-vmafd/bin/dir-cli trustedcert list --login administrator@vsphere.local

# Use the VMware certificate fix script:
# Download and run fixsts.sh from VMware KB 79248
python /usr/lib/vmware-vmdir/share/config/ls_update_certs.py \
  --url https://vcenter.domain.com/lookupservice/sdk \
  --fingerprint <thumbprint> \
  --certfile /tmp/new-sts-cert.pem --admin administrator@vsphere.localCopy

Identify orphaned snapshot files

# SSH to the ESXi host running the VM:
cd /vmfs/volumes/<datastore>/<vm-folder>/

# List all VMDK files and their sizes:
ls -lah *.vmdk
# Delta files: vmname-000001.vmdk, vmname-000002.vmdk, etc.
# These should not exist if snapshot manager shows no snapshots

# Check the snapshot descriptor:
cat vmname.vmsd
# If empty but delta files exist → orphaned snapshotsCopy

Check for locked files and release locks

# Find which host holds the lock:
vmkfstools -D /vmfs/volumes/<datastore>/<vm-folder>/vmname-000001-delta.vmdk
# Look for "owner" UUID — maps to an ESXi host

# If lock is held by a different host, check if a backup process is running
# on that host. Kill the stale backup process first.

# Find lock owners across all hosts:
# grep the MAC address from the lock owner in:
esxcfg-info | grep -i "mac address"Copy

Consolidate disks

# Try consolidation from vCenter first:
# Right-click VM → Snapshots → Consolidate

# CLI alternative:
vim-cmd vmsvc/snapshot.removeall <vmid>

# If consolidation fails due to CID mismatch:
# 1. Check CID chain:
grep -i "CID\|parentCID" /vmfs/volumes/<datastore>/<vm-folder>/*.vmdk

# 2. Fix CID chain — ensure each delta's parentCID matches parent's CID
# Edit the .vmdk descriptor (NOT the -flat.vmdk data file)Copy

Identify what's blocking maintenance mode entry/exit

# Check for VMs still on the host:
esxcli vm process list
# All VMs must be migrated or powered off

# Check recent tasks in vCenter:
# Host → Monitor → Tasks & Events
# Look for stuck "Enter Maintenance Mode" or "Reconfigure for HA" tasks

# PowerCLI — Find VMs on this host:
Get-VMHost "esxi01" | Get-VM | Select Name, PowerState, VMHostCopy

Exit maintenance mode via CLI

# SSH to the ESXi host:
esxcli system maintenanceMode set -e false

# Alternative:
vim-cmd hostsvc/maintenance_mode_exit

# If the above fails, try from PowerCLI:
Set-VMHost -VMHost "esxi01.domain.com" -State "Connected" -Confirm:$false

# Force exit (use with caution):
vim-cmd hostsvc/maintenance_mode_exit 0 1
# Second parameter 1 = forceCopy

Resolve stuck HA reconfiguration

# If HA reconfiguration is pending:
# 1. Disable HA on the cluster temporarily
# 2. Exit maintenance mode
# 3. Re-enable HA

# PowerCLI:
Set-Cluster "ClusterName" -HAEnabled:$false -Confirm:$false
Set-VMHost "esxi01" -State Connected -Confirm:$false
Set-Cluster "ClusterName" -HAEnabled:$true -Confirm:$false

# Or restart FDM on the host:
/etc/init.d/fdm restartCopy

Reset all triggered alarms in bulk

# PowerCLI — Reset all triggered alarms to green:
$alarmMgr = Get-View AlarmManager
$rootFolder = Get-View (Get-View ServiceInstance).Content.RootFolder

# Get all triggered alarms:
$triggeredAlarms = $rootFolder.TriggeredAlarmState
Write-Host "Found $($triggeredAlarms.Count) triggered alarms"

# Reset each alarm:
foreach ($alarm in $triggeredAlarms) {
    $alarmMgr.AcknowledgeAlarm($alarm.Alarm, $alarm.Entity)
}
Write-Host "All alarms acknowledged"Copy

Disable non-critical and noisy alarms

# PowerCLI — List all alarm definitions with their enabled state:
Get-AlarmDefinition | Select Name, Enabled, Description |
  Sort-Object Name | Format-Table -Wrap

# Disable specific noisy alarms:
Get-AlarmDefinition -Name "Virtual machine CPU usage" |
  Set-AlarmDefinition -Enabled $false

# Disable alarm actions (email/SNMP) without disabling the alarm:
Get-AlarmDefinition -Name "Host memory usage" | Get-AlarmAction | Remove-AlarmAction -Confirm:$false

# Batch disable all alarm actions during maintenance:
Get-AlarmDefinition | Where { $_.Enabled } |
  ForEach { Get-AlarmAction -AlarmDefinition $_ | Remove-AlarmAction -Confirm:$false }Copy

Tune alarm thresholds for your environment

# PowerCLI — Adjust alarm threshold example:
# Change "Datastore usage on disk" alarm from 75%/85% to 85%/95%:
$alarm = Get-AlarmDefinition -Name "Datastore usage on disk"
# Edit via vCenter UI: Alarm → Edit → Conditions
# Set Warning threshold: 85%
# Set Critical threshold: 95%

# Review alarm action configuration:
Get-AlarmDefinition | Where { $_.Enabled } |
  ForEach {
    $actions = Get-AlarmAction -AlarmDefinition $_
    if ($actions) {
      [PSCustomObject]@{
        Alarm = $_.Name
        ActionCount = $actions.Count
        ActionTypes = ($actions | Select -Expand ActionType) -join ", "
      }
    }
  } | Format-Table -AutoSizeCopy