Iptables – proxmox KVM routed network with multiple public IPs

hetznerip-routingiptableskvm-virtualizationproxmox

I have a dedicated hosting with hetzner.
Additionally i have bought a 6IP subnet.

My main IP is: 148.111.111.200
My main subnet is: 255.255.255.224

My additional IPs are 148.222.222.1 to 148.222.222.6.

My usage scenario is as follows:
Several instances will have public facing IPs (webservers, etc)
All instances will have a second nic setup for internal LAN, so I can have mysql server, memcached, etc on private network locked from outside.

Webserver will be online on 148.222.222.1 and will have a second NIC with ip 10.10.10.10

Currently, i have setup the internal lan. All the instances can connect and ping one another via internal IP (10.10.10.X), but my webserver cannot reach the internet.

I cannot use bridged mode, since hetzner does not allow multiple MACs on same external ip, so I have to use routing mode.
Here is my /etc/network/interfaces file for the host:

# network interface settings
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet static

    address  148.111.111.200
    netmask  255.255.255.255
    pointopoint 148.111.111.193
    gateway  148.111.111.193
    broadcast  148.111.111.223
    post-up echo 1 > /proc/sys/net/ipv4/conf/eth0/proxy_arp
#commentedout
    #up route add -net 148.111.111.192 netmask 255.255.255.224 gw 148.111.111.193 eth0


    # default route to access subnet

auto vmbr0
iface vmbr0 inet static

    address  148.111.111.200
    netmask  255.255.255.255
    bridge_ports none
    bridge_stp off
    bridge_fd 0

    up ip route add 148.222.222.1/32 dev vmbr0
    up ip route add 148.222.222.2/32 dev vmbr0

auto vmbr1
iface vmbr1 inet static

    address 10.10.10.1
    netmask 255.255.255.0
    bridge_ports none
    bridge_stp off
    bridge_fd 0

    post-up echo 1 > /proc/sys/net/ipv4/ip_forward
    post-up   iptables -t nat -A POSTROUTING -s '10.10.10.0/24' -o eth0 -j MASQUERADE
    post-down iptables -t nat -D POSTROUTING -s '10.10.10.0/24' -o eth0 -j MASQUERADE

auto vmbr2
iface vmbr2 inet static

    address  148.222.222.1
    netmask  255.255.255.248
    bridge_ports none
    bridge_stp off
    bridge_fd 0

And here is my kvm interfaces file:

auto eth0
iface eth0 inet static
    address 148.222.222.1
    netmask 255.255.255.255
    pointopoint 148.111.111.200
    gateway 148.111.111.200
    dns-nameservers 8.8.8.8 8.8.4.4

auto eth1
    address 10.10.10.12
    netmask 255.255.255.0
    network 10.10.10.0
    broadcast 10.10.10.255

Currently, KVM instances can ping oneanother, but nothing else works
Host can ping instances, but nothing else
And i cannot get internet access on my instances.

What do I need to change in my configuration in order for this to work.

P.S. NIC type is set to virtio in proxmox

Best Answer

I solved the problem by using the following config:

auto eth0
iface eth0 inet static
    address  148.111.111.200
    netmask  255.255.255.255
    pointopoint 148.111.111.193
    gateway  148.111.111.193
    broadcast  148.111.111.193

# default route to access subnet

auto vmbr0
iface vmbr0 inet static
    address  148.111.111.200
    netmask  255.255.255.255
    bridge_ports none
    bridge_stp off
    bridge_fd 0
    bridge_maxwait 0

    #subnet
    up ip route add 148.222.222.0/32 dev vmbr0
    up ip route add 148.222.222.1/32 dev vmbr0
    up ip route add 148.222.222.2/32 dev vmbr0
    up ip route add 148.222.222.3/32 dev vmbr0
    up ip route add 148.222.222.4/32 dev vmbr0
    up ip route add 148.222.222.5/32 dev vmbr0
    up ip route add 148.222.222.6/32 dev vmbr0
    up ip route add 148.222.222.7/32 dev vmbr0

auto vmbr1
iface vmbr1 inet static
    address 10.10.10.1
    netmask 255.255.255.0
    bridge_ports none
    bridge_stp off
    bridge_fd 0

    post-up echo 1 > /proc/sys/net/ipv4/ip_forward
    post-up   iptables -t nat -A POSTROUTING -s '10.10.10.0/24' -o eth0 -j MASQUERADE
    post-down iptables -t nat -D POSTROUTING -s '10.10.10.0/24' -o eth0 -j MASQUERADE

On your client machines you need to define one ore two network interfaces

  • One for VMBR0 (public facing IP)
  • One for VMBR1 (private IP - 10.10.10.X)

Sample config for ubuntu /etc/network/interfaces

#vmbr0
auto eth0
iface eth0 inet static
  address 148.222.222.1
  netmask 255.255.255.255
  pointopoint 148.111.111.200
  gateway 148.111.111.200 #public IP for the proxmox node
  dns-nameservers 8.8.8.8 8.8.4.4

#vmbr1
auto eth1
iface eth1 inet static
  address 10.10.10.20
  netmask 255.255.255.0
  network 10.10.10.0
  broadcast 10.10.10.255