Skip to content

Instantly share code, notes, and snippets.

@snizovtsev

snizovtsev/Vagrantfile

Last active April 9, 2020 15:25
Show Gist options
  • Save snizovtsev/0a01324c96357d1a1bf0f77d5fd8a959 to your computer and use it in GitHub Desktop.
Save snizovtsev/0a01324c96357d1a1bf0f77d5fd8a959 to your computer and use it in GitHub Desktop.
Download ZIP
TRex traffic generator in Vagrant
Raw
faq.md

FAQ

How to configure host

Passwordless access to qemu system session

Create /etc/polkit-1/rules.d/50-allow-libvirt.rules with

/* Allow users in kvm group to manage the libvirt
 * daemon without authentication */
polkit.addRule(function(action, subject) {
        if (action.id == "org.libvirt.unix.manage" &&
                subject.isInGroup("yourgroup")) {
                return polkit.Result.YES;
        }
});

where yourgroup is the unix group you belong to.

VM boots significantly slower when physical NIC passed through

This is probably due to Intel Boot Agent (PXE). Boot with graphics and check. In Vagrant, the only way to disable boot ROMs is to flash your NIC's persistent memory. If you brave enough do the following:

  1. Download and unpack Intel Preboot utility for Linux
  2. Reboot your host with IOMMU disabled
  3. Go to APPS/BootUtil/Linux_x64
  4. Install and load kernel module using sudo DRIVER/install
  5. Run sudo ./bootutil64e and find your NIC numbers
  6. You should see a lines like
Port Network Address Location Series  WOL Flash Firmware                Version
==== =============== ======== ======= === ============================= =======
  N   XXXXXXXXXXXX     3:00.0 Gigabit YES UEFI,PXE Enabled              1.5.88
  1. Run sudo ./bootutil64e -NIC=N -BOOTENABLE=DISABLED
  2. Run sudo ./bootutil64e -NIC=N -FD
  3. Reboot to IOMMU again and here it is.
Raw
traffgen.yml
---
- hosts: all
tasks:
- name: add trex rpm repository
environment: "{{ proxy_env }}"
yum_repository:
name: trex
includepkgs: trex-core
enablegroups: false
skip_if_unavailable: no
description: "Copr repo fo trex traffic generator"
baseurl: "https://copr-be.cloud.fedoraproject.org/results/snizovtsev/trex/epel-8-$basearch"
gpgkey: "https://copr-be.cloud.fedoraproject.org/results/snizovtsev/trex/pubkey.gpg"
- name: install required packages
environment: "{{ proxy_env }}"
notify: system upgraded
dnf:
state: latest
name:
- kernel # force update for trex
- trex-core # cisco traffic generator
- kernel-core # upgrade to match kernel-devel
- tuned # sysctl and boot options
- nmap # nping tests
- wireshark-cli # for debugging
- driverctl # binding/unbinding drivers
- dnsmasq # pxe boot
- name: mkdir /etc/tuned/traffgen
file: path=/etc/tuned/traffgen state=directory
- name: install tuned profile
notify: switch tuned profile
copy:
dest: /etc/tuned/traffgen/tuned.conf
content: |
[main]
include=network-latency
[sysctl]
vm.nr_hugepages=256
[bootloader]
[sysfs]
- name: trex configuration file
copy:
dest: /etc/trex_cfg.yaml
content: |
- version: 2
interfaces: ["03:00.0", "04:00.0"]
port_limit: 2
limit_memory: 512
low_end: true
handlers:
- name: switch tuned profile
shell: tuned-adm profile traffgen
- name: system upgraded
reboot:
Raw
Vagrantfile
#!/usr/bin/env ruby
ENV["PYTHONUNBUFFERED"] = "1"
Vagrant.configure("2") do |config|
# Better defaults
config.vm.provider :libvirt do |libvirt|
libvirt.qemu_use_session = false
libvirt.cpu_mode = 'host-passthrough' # Prefer performance to reproducibility here
libvirt.machine_type = 'q35' # better support device passthrough since ICH9 uses a PCI-E bus
# whereas the I440FX only supports a PCI bus
# Avoid unpredictable memory latencies for better reproducibility
libvirt.memorybacking :hugepages
libvirt.memorybacking :nosharepages
libvirt.memorybacking :locked # Forbids swapping
libvirt.memorybacking :source, :type => 'file'
libvirt.memorybacking :allocation, :mode => 'immediate'
end
config.vm.define :traffgen do |node|
node.vm.box = "centos/8"
node.vm.hostname = "traffgen"
node.vm.synced_folder './shared', '/vagrant', type: 'rsync'
node.vm.provider :libvirt do |qemu|
qemu.graphics_type = 'none' # use serial
qemu.cpus = 3 # Recommended value from TRex FAQ
qemu.cputopology :sockets => 1, :cores => 3, :threads => 1
qemu.memory = 1024 # Reduced value for debugging
#qemu.memory = 4096 # Recommended value from TRex FAQ
# HostNIC-1/p0 (PCI 01:00.0) <-cable-> HostNIC-2/p0 (enp3s0f0)
qemu.pci :bus => '01', :slot => '00', :function => '0'
# HostNIC-1/p1 (PCI 01:00.1) <-cable-> HostNIC-2/p2 (enp3s0f2)
qemu.pci :bus => '01', :slot => '00', :function => '1'
end
node.vm.provision :ansible do |ansible|
ansible.playbook = "./traffgen.yml"
ansible.become = true
ansible.compatibility_mode = "2.0"
ansible.verbose = "vv"
end
end
########################################################################
# System-under-test
#
# Should be connected using physical cable to traffic generator VM.
# We use macvtap because currently KOS dosn't handle passed-through NICs
# properly (bug #3980834).
########################################################################
config.vm.define :sut do |node|
node.vm.provider :libvirt do |qemu|
qemu.graphics_type = 'none' # use serial
qemu.kernel = "#{__dir__}/kernel"
qemu.machine_type = 'pc'
qemu.cpus = 1
qemu.memory = 1024
qemu.mgmt_attach = false # useless on KOS
# Use OVMF UEFI instead of SeaBIOS
#qemu.loader = "/usr/share/OVMF/OVMF_CODE.fd"
#qemu.storage :file, :device => :cdrom, :bus => "sata", :path => "#{__dir__}/uboot-efi.img"
# IOMMU
# qemu.features = ['acpi', 'apic', 'pae', 'ioapic'] # 3 vagrant default + ioapic
# qemu.qemuargs :value => "-device"
# qemu.qemuargs :value => "intel-iommu"
end
node.vm.network :public_network,
:dev => "enp3s0f0",
:mode => "private",
:model_type => "e1000",
:trust_guest_rx_filters => true
node.vm.network :public_network,
:dev => "enp3s0f2",
:mode => "private",
:model_type => "e1000",
:trust_guest_rx_filters => true
end
########################################################################
# NetBSD reference system
########################################################################
config.vm.define :netbsd do |node|
node.vm.box = "generic/netbsd8"
node.vm.provider :libvirt do |qemu|
# qemu.graphics_type should be non-none or netbsd won't boot
qemu.cpus = 1
qemu.memory = 1024
qemu.machine_type = 'pc'
end
node.vm.network :public_network,
:dev => "enp3s0f0",
:mode => "private",
:model_type => "rtl8139",
:trust_guest_rx_filters => true
node.vm.network :public_network,
:dev => "enp3s0f2",
:mode => "private",
:model_type => "rtl8139",
:trust_guest_rx_filters => true
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment