Building a NixOS router for AAISP – L2TP Failover over 4G
Following on from setting up the basics of our router, here's how I add a second WAN connection via a 4G router, and use Andrews & Arnold's L2TP service to keep my public IPv4 and IPv6 networks available in the event of a failure of my fibre service.
You can simplify the configuration here to just use the 4G connection directly if you don't have an ISP that's quite so fancy. Let me know if you'd like me to write about that.
The 4G router is one I picked up a while ago from the Three network, a ZTE MF286D. I do nothing special to it's configuration, and have it running in router mode. It does offer a bridge mode where the external IPs are passed to a connected device, but we don't need that here.
I have an 80Gb/month prepaid 26 month SIM (affiliate link) from Safecom that uses the Three network. It was £60 when I bought it, and is more expensive now, but it's worth checking out there various options over time to get the best prices. I reckon it should cover 2-4 days in a month, so long as we avoid watching a lot of streaming TV.
Things are configured such that when the fibre's PPPoE connection ends, the L2TP session is initiated. And when the PPPoE connection starts, the L2TP session is terminated. Notably this doesn't cover the router booting whilst the fibre is down, which is something to fix.
In Linux L2TP is typically implemented with a combination xl2tpd and pppd. There's a NixOS module for xl2tpd, but it makes some assumptions that mean it won't work for this use case. Typically L2TP is used with IPsec for some old school VPN solutions, rather than on it's own, as we need it.
So the first job is to create a new NixOS module, which we can base on the upstream version and just chop it up a bit to pass more configurability up the stack. I'll be the first to admit my Nix-fu is not strong yet, so I'm sure there are better ways to do this. But it works for me!
# modules/nixos/xl2tpd-flexible.nix
{ config, pkgs, lib, ... }:
with lib;
{
options = {
services.xl2tpd-flexible = {
enable = mkEnableOption "xl2tpd, with more flexible configuration";
xl2tpOptions = mkOption {
type = types.lines;
description = "xl2tpd configuration file content";
default = "";
};
pppOptions = mkOption {
type = types.lines;
description = "ppp options file content";
default = "";
};
};
};
config = mkIf config.services.xl2tpd-flexible.enable {
systemd.services.xl2tpd-flexible = let
cfg = config.services.xl2tpd-flexible;
pppd-options = pkgs.writeText "ppp-options-xl2tpd.conf" cfg.pppOptions;
xl2tpd-conf = pkgs.writeText "xl2tpd.conf" ''
${cfg.xl2tpOptions}
pppoptfile = ${pppd-options}
'';
xl2tpd-ppp-wrapped = pkgs.stdenv.mkDerivation {
name = "xl2tpd-ppp-wrapped";
phases = [ "installPhase" ];
nativeBuildInputs = with pkgs; [ makeWrapper ];
installPhase = ''
mkdir -p $out/bin
makeWrapper ${pkgs.ppp}/sbin/pppd $out/bin/pppd \
--set LD_PRELOAD "${pkgs.libredirect}/lib/libredirect.so" \
--set NIX_REDIRECTS "/etc/ppp=/etc/xl2tpd/ppp"
makeWrapper ${pkgs.xl2tpd}/bin/xl2tpd $out/bin/xl2tpd \
--set LD_PRELOAD "${pkgs.libredirect}/lib/libredirect.so" \
--set NIX_REDIRECTS "${pkgs.ppp}/sbin/pppd=$out/bin/pppd"
'';
};
in {
description = "xl2tpd flexible server";
requires = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
preStart = ''
mkdir -p -m 700 /etc/xl2tpd/ppp
mkdir -p -m 700 /run/xl2tpd
'';
serviceConfig = {
ExecStart = "${xl2tpd-ppp-wrapped}/bin/xl2tpd -D -c ${xl2tpd-conf} -p /run/xl2tpd/pid -C /run/xl2tpd/control";
KillMode = "process";
Restart = "on-success";
Type = "simple";
PIDFile = "/run/xl2tpd/pid";
};
};
};
}
We need to configure our additional WAN port, which builds on our configuration from the previous post. xl2tpd doesn't support connecting to an IPv6 endpoint, and although my SIM gives me a /64 network, I chose not to configure it here to keep things simple.
# nixos/hermes/default.nix
{ config, pkgs, ... }:
{
systemd.network = {
links = {
"20-wan-4g" = {
matchConfig = {
MACAddress = "aa:aa:aa:aa:aa:02"; # port 2
};
linkConfig = {
Name = "wan-4g";
};
};
};
networks = {
"20-wan-4g" = {
matchConfig.Name = "wan-4g";
networkConfig = {
DHCP = "ipv4";
IPv6AcceptRA = "no";
};
dhcpV4Config = {
ClientIdentifier = "mac";
RouteMetric = 200;
};
linkConfig.RequiredForOnline = "no";
};
};
};
}
Note the RouteMetric for the wan-4g is 200, where I used 100 for pppoe-aaisp. This means we won't use the 4G connection's default route so long as the fibre is up.
We'll use our xl2tpd-flexible module and some additional PPP up/down scripts to configure the L2TP service, and modify our PPPoE PPP scripts to trigger the connection. And we'll add a new reference to our CHAP secrets specifically for the pppd used by xl2tpd.
# nixos/hermes/default.nix
{ config, pkgs, ... }:
{
# l2tp failover
age.secrets.xl2tpd-chap-secrets = {
file = ../../secrets/pppoe-chap-secrets.age;
path = "/etc/xl2tpd/ppp/chap-secrets";
owner = "root";
group = "root";
mode = "0600";
};
services.xl2tpd-flexible = {
enable = true;
xl2tpOptions = ''
[global]
max retries = 36
[lac aaisp]
lns = 194.4.172.12
require authentication = no
redial = yes
redial timeout = 10
'';
pppOptions = ''
+ipv6
ipv6cp-use-ipaddr
name <USERNAME>
noauth
ifname l2tp-aaisp
'';
};
# routing (pppoe)
environment.etc.ppp-up = {
enable = true;
target = "ppp/ip-up";
mode = "0755";
text = ''
#!${pkgs.bash}/bin/bash
${pkgs.logger}/bin/logger "$1 is up"
if [ $IFNAME = "pppoe-aaisp" ]; then
${pkgs.logger}/bin/logger "AAISP - FTTP PPPoE online"
${pkgs.logger}/bin/logger "Bring L2TP down and remove routes"
echo "d aaisp" > /run/xl2tpd/control
${pkgs.logger}/bin/logger "Add default routes via PPPoE"
${pkgs.iproute2}/bin/ip route add default dev pppoe-aaisp scope link metric 100
${pkgs.iproute2}/bin/ip -6 route add default dev pppoe-aaisp scope link metric 100
fi
'';
};
environment.etc.ppp-down = {
enable = true;
target = "ppp/ip-down";
mode = "0755";
text = ''
#!${pkgs.bash}/bin/bash
${pkgs.logger}/bin/logger "$1 is down"
if [ $IFNAME = "pppoe-aaisp" ]; then
${pkgs.logger}/bin/logger "AAISP - FTTP PPPoE offline"
${pkgs.logger}/bin/logger "Remove default routes via PPPoE"
${pkgs.iproute2}/bin/ip route del default dev pppoe-aaisp scope link metric 100
${pkgs.iproute2}/bin/ip -6 route del default dev pppoe-aaisp scope link metric 100
${pkgs.logger}/bin/logger "Connect via LT2P failover"
echo "c aaisp" > /run/xl2tpd/control
fi
'';
};
# routing (l2tp)
environment.etc.l2tp-up = {
enable = true;
target = "xl2tpd/ppp/ip-up";
mode = "0755";
text = ''
#!${pkgs.bash}/bin/bash
${pkgs.logger}/bin/logger "$1 is up"
if [ $IFNAME = "l2tp-aaisp" ]; then
${pkgs.logger}/bin/logger "AAISP - L2TP failover online"
${pkgs.logger}/bin/logger "Add default routes via L2TP"
${pkgs.iproute2}/bin/ip route add 194.4.172.12/32 via 192.168.100.1 metric 10
${pkgs.iproute2}/bin/ip route add default dev l2tp-aaisp scope link metric 100
${pkgs.iproute2}/bin/ip -6 route add default dev l2tp-aaisp scope link metric 100
fi
'';
};
environment.etc.l2tp-down = {
enable = true;
target = "xl2tpd/ppp/ip-down";
mode = "0755";
text = ''
#!${pkgs.bash}/bin/bash
${pkgs.logger}/bin/logger "$1 is down"
if [ $IFNAME = "l2tp-aaisp" ]; then
${pkgs.logger}/bin/logger "AAISP - L2TP failover offline"
${pkgs.logger}/bin/logger "Remove default route via L2TP"
${pkgs.iproute2}/bin/ip route del default dev l2tp-aaisp scope link metric 100
${pkgs.iproute2}/bin/ip -6 route del default dev l2tp-aaisp scope link metric 100
${pkgs.iproute2}/bin/ip route del 194.4.172.12/32 via 192.168.100.1 metric 10
fi
'';
};
}
Changes you might want to make include references to 194.4.172.12, which is the L2TP service endpoint for Andrews & Arnold, and 192.168.100.1 which is IP of the 4G router. We add a static route for the L2TP endpoint via this address to prevent it inceptioning.
Lastly, we need to modify our firewall rules to allow traffic through the new l2tp-aaisp interface and to NAT traffic on the way out.
{ config, pkgs, ... }:
{
networking.nftables.ruleset = ''
table inet firewall {
chain rpfilter {
type filter hook prerouting priority mangle + 10; policy drop;
meta nfproto ipv4 udp sport . udp dport { 68 . 67, 67 . 68 } accept comment "DHCPv4 client/server"
fib saddr . mark oif exists accept
}
chain input {
type filter hook input priority filter; policy drop;
# assuming we trust our LAN clients
iifname { "lo", "lan" } accept comment "trusted interfaces"
# handle packets according to connection state
ct state vmap { invalid : drop, established : accept, related : accept, new : jump input-allow, untracked : jump input-allow }
# if we make it here, block and log
tcp flags syn / fin,syn,rst,ack log prefix "refused connection: " level info
}
chain input-allow {
# make your own choice on whether to allow SSH from outside
tcp dport 22 accept comment "ssh from anywhere"
icmp type echo-request accept comment "allow ping"
icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4."
ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client"
}
chain forward {
type filter hook forward priority 0; policy drop;
# no internet egress to RFC1918 IPs
oifname { "pppoe-aaisp", "l2tp-aaisp" } ip daddr { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 } reject with icmp type net-unreachable comment "outbound rfc1918 not permitted"
# established/related allowed, invalid dropped
ct state vmap { established : accept, related : accept, invalid : drop }
# internal interfaces outbound allowed
iifname "lan" oifname { "pppoe-aaisp", "l2tp-aaisp" } accept comment "internal networks out via ISP"
# allow icmp
icmp type echo-request accept comment "allow ping"
icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4."
# log anything that was blocked
tcp flags syn / fin,syn,rst,ack log prefix "refused forward: " level info
}
}
table ip nat {
chain pre {
type nat hook prerouting priority dstnat; policy accept;
# we'll add rules for our 1:1 NAT here later
}
chain post {
type nat hook postrouting priority srcnat; policy accept;
iifname "lan" oifname { "pppoe-aaisp", "l2tp-aaisp" } masquerade comment "LAN NAT to FTTP"
}
chain out {
type nat hook output priority mangle; policy accept;
# we'll add rules for our 1:1 NAT here later
}
}
'';
}
And that's it! I'm able to bring down the FTTP link by pulling the cable or powering off the ONT, and in under 10 seconds, everything is connected again, albeit with much less bandwidth and much higher latency!
In the next post, I'll add VLANs for IoT devices, guests, self hosted services, and private clients/services that I prefer to reach the Internet via Mullvad's VPN service.