Skip to content

Instantly share code, notes, and snippets.

@AlexTalker
Created January 12, 2022 13:15
Show Gist options
  • Save AlexTalker/767bfe3ec577f1de140fe883bf1c4262 to your computer and use it in GitHub Desktop.
Save AlexTalker/767bfe3ec577f1de140fe883bf1c4262 to your computer and use it in GitHub Desktop.
Show performance information about NVMe PCie drives or any PCIe sysfs device, based on https://community.mellanox.com/s/article/understanding-pcie-configuration-for-maximum-performance
#!/usr/bin/env bash
# BEGIN: PCIe-related things
_read_attr() {
local dir="$1" name="$2"
local path=$(printf "%s/%s" "$dir" "$name")
cat "$path"
}
_read_device_current_link_width() {
_read_attr "$1" "current_link_width"
}
_read_device_current_link_speed() {
_read_attr "$1" "current_link_speed" | egrep -o '[0-9\.]*'
}
_read_device_max_link_width() {
_read_attr "$1" "max_link_width"
}
_read_device_max_link_speed() {
_read_attr "$1" "max_link_speed" | egrep -o '[0-9\.]*'
}
_calc_pcie_gen_by_speed() {
local pcie_speed="$1"
local pcie_gen=''
case "$pcie_speed" in
2.5)
pcie_gen="1"
;;
5)
pcie_gen="2"
;;
8)
pcie_gen="3"
;;
16)
pcie_gen="4"
;;
esac
echo -n "${pcie_gen}"
}
_calc_pcie_encoding_by_gen() {
local pcie_gen="$1"
local pcie_encoding=''
case "$pcie_gen" in
1)
pcie_encoding="1/5" # 8b/10b
;;
2)
pcie_encoding="1/5" # 8b/10b
;;
3)
pcie_encoding="2/130" # 128b/130b
;;
4)
pcie_encoding="2/130" # 128b/130b
;;
esac
echo -n "${pcie_encoding}"
}
_calc_math() {
awk "BEGIN{print $*}";
}
_calc_pcie_throughput() {
local pcie_width="$1" pcie_speed="$2" pcie_ecc_overhead="$3"
local pcie_headers_overhead=1
_calc_math "${pcie_speed} * ${pcie_width} * (1 - ${pcie_ecc_overhead}) - ${pcie_headers_overhead}"
}
_calc_device_info() {
local pdevice="${1}"
DEVICE_CUR_WIDTH=$(_read_device_current_link_width "$pdevice")
DEVICE_CUR_SPEED=$(_read_device_current_link_speed "$pdevice")
DEVICE_CUR_GEN=$(_calc_pcie_gen_by_speed "${DEVICE_CUR_SPEED}")
DEVICE_CUR_ECC_OVERHEAD=$(_calc_pcie_encoding_by_gen "${DEVICE_CUR_GEN}")
DEVICE_CUR_THROUGHPUT=$(_calc_pcie_throughput "${DEVICE_CUR_WIDTH}" "${DEVICE_CUR_SPEED}" "${DEVICE_CUR_ECC_OVERHEAD}")
DEVICE_MAX_WIDTH=$(_read_device_max_link_width "$pdevice")
DEVICE_MAX_SPEED=$(_read_device_max_link_speed "$pdevice")
DEVICE_MAX_GEN=$(_calc_pcie_gen_by_speed "${DEVICE_MAX_SPEED}")
DEVICE_MAX_ECC_OVERHEAD=$(_calc_pcie_encoding_by_gen "${DEVICE_MAX_GEN}")
DEVICE_MAX_THROUGHPUT=$(_calc_pcie_throughput "${DEVICE_MAX_WIDTH}" "${DEVICE_MAX_SPEED}" "${DEVICE_MAX_ECC_OVERHEAD}")
}
_clear_device_info() {
unset DEVICE_CUR_WIDTH DEVICE_CUR_SPEED DEVICE_MAX_GEN DEVICE_CUR_ECC_OVERHEAD DEVICE_CUR_THROUGHPUT
unset DEVICE_MAX_WIDTH DEVICE_MAX_SPEED DEVICE_MAX_GEN DEVICE_MAX_ECC_OVERHEAD DEVICE_MAX_THROUGHPUT
}
_print_device_info() {
local device_path="$1" log="$2"
local _log='echo -e' _prefix=''
if [[ $(type -t "${log}") == "function" ]]; then
_log="${log}"
else
_prefix="${log}"
fi
_calc_device_info "$device_path"
${_log} "${_prefix}PCIe generation: ${DEVICE_CUR_GEN} (max.: ${DEVICE_MAX_GEN})"
${_log} "${_prefix}PCIe width: x${DEVICE_CUR_WIDTH} (max.: x${DEVICE_MAX_WIDTH})"
${_log} "${_prefix}PCIe speed: ${DEVICE_CUR_SPEED} GT/s (max.: ${DEVICE_MAX_SPEED} GT/s)"
${_log} "${_prefix}PCIe throughput: ${DEVICE_CUR_THROUGHPUT} Gigabit/s (max.: ${DEVICE_MAX_THROUGHPUT} Gigabit/s)"
[ "${DEVICE_CUR_THROUGHPUT}" != "${DEVICE_MAX_THROUGHPUT}" ] && ${_log} "${_prefix}Device throughput is downgraded!"
_clear_device_info
}
# END: PCIe-related things
# BEGIN NVMe subsystem & device
_read_subsys_model() {
_read_attr "$1" "model"
}
_read_subsys_serial() {
_read_attr "$1" "serial"
}
_read_subsys_nqn() {
_read_attr "$1" "subsysnqn"
}
_read_subsys_fw_rev() {
_read_attr "$1" "firmware_rev"
}
_read_nvme_transport() {
_read_attr "$1" "transport"
}
_calc_device_path() {
printf "%s/device" "$@"
}
# END: NVMe subsystem & device
# BEGIN: Logging
_log() {
echo -e "$@"
}
log_subsys() {
_log "$(printf '[Subsystem: %s] %s' "${SUBSYSTEM_NAME}" "$@")"
}
log_nvme() {
log_subsys "$(printf '[NVMe: %s] %s' "${NVME_NAME}" "$@")"
}
# END: Logging
# BEGIN: Main
# Allow dumping information about custom sysfs PCIe devices(i.e. Mellanox adapters)
if [[ -n "$@" ]]; then
_print_device_info "$@"
exit 0
fi
# Fallback to default iteration over NVMe subsystems
for psubsys in /sys/class/nvme-subsystem/*; do
SUBSYSTEM_NAME=$(basename $psubsys)
log_subsys "Model: $(_read_subsys_model "${psubsys}")"
log_subsys "Serial: $(_read_subsys_serial "${psubsys}")"
log_subsys "NQN: $(_read_subsys_nqn "${psubsys}")"
log_subsys "Firmware revision: $(_read_subsys_fw_rev "${psubsys}")"
for pnvme in "$psubsys"/nvme*; do
NVME_NAME=$(basename "$pnvme")
NVME_TRANSPORT=$(_read_nvme_transport "$pnvme")
log_subsys "Found NVMe: ${NVME_NAME} (transport: ${NVME_TRANSPORT})"
[[ "$NVME_TRANSPORT" -ne "pcie" ]] && {
log_nvme "${NVME_TRANSPORT} is not the PCIe transport, skipping..."
continue
}
pdevice=$(_calc_device_path "$pnvme")
_print_device_info "$pdevice" "log_nvme"
done
_log
done
# END: Main
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment