Git Sagar
2f8925d439
All wan.net.vmix@* instances created their veth pair with the same temporary peer name 'vhost' in the host namespace before moving it into their netns. Parallel starts at boot could steal each other's peer ends, pairing a host-side vn-<ns> with another namespace's vhost (mismatched /30s, dead links) or leaving the pair stranded in the host namespace. Use a per-namespace temporary name (vh-<ns>) and rename to vhost only after the move, making concurrent creation collision-free. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
296 lines
14 KiB
Nix
296 lines
14 KiB
Nix
{ config, pkgs, lib, vmixLib, ... }:
|
|
with vmixLib.network;
|
|
let
|
|
vmixCfg = config.vmix;
|
|
# creates a /30 network from available range for veth-pair wan interfaces
|
|
mkVethIPv4Range = index: availableIPv4Range:
|
|
let
|
|
vethIPv4RangeLength = 30;
|
|
in
|
|
(calc.cidr.subnet (vethIPv4RangeLength - (calc.cidr.length availableIPv4Range)) index availableIPv4Range);
|
|
|
|
namespaceGlobalService = {
|
|
"ns.net.vmix@" = {
|
|
description = "network namespace %I for vmix";
|
|
before = [ "network.target" ];
|
|
path = with pkgs; [ iproute2 utillinux ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
PrivateMounts = false;
|
|
PrivateNetwork = true;
|
|
ExecStart = (pkgs.writeShellScript "ns.net.vmix-start" ''
|
|
spaceName="$1.vmix"
|
|
ip netns add $spaceName
|
|
umount /var/run/netns/$spaceName
|
|
mount --bind /proc/self/ns/net /var/run/netns/$spaceName
|
|
'') + " %I";
|
|
ExecStop = "${pkgs.iproute2}/bin/ip netns del %I.vmix";
|
|
};
|
|
};
|
|
};
|
|
|
|
mkLanDomainName = spaceName: lanName: lanCfg:
|
|
if (lanCfg.domain != null) then lanCfg.domain else "${lanName}.${spaceName}.vmix";
|
|
|
|
mkLan = spaceName: staticRoutes: wanCfg: lanName: cfg:
|
|
let
|
|
lanCfg = cfg // { name = lanName; spaceName = "${spaceName}"; };
|
|
lanInterfaceName = "brx-${lanCfg.name}";
|
|
lanInterfaceIPAddress =
|
|
if (lanCfg.ipv4.address != null)
|
|
then lanCfg.ipv4.address
|
|
else (calc.cidr.host 1 lanCfg.ipv4.range);
|
|
netmask = calc.cidr.netmask lanCfg.ipv4.range;
|
|
networkPrefix = builtins.elemAt (lib.splitString "/" lanCfg.ipv4.range) 1;
|
|
|
|
dhcpStartAddress =
|
|
if (lanCfg.ipv4.dhcp.startAddress != null)
|
|
then lanCfg.ipv4.dhcp.startAddress
|
|
else (calc.cidr.host 2 lanCfg.ipv4.range);
|
|
|
|
dhcpEndAddress =
|
|
if (lanCfg.ipv4.dhcp.endAddress != null)
|
|
then lanCfg.ipv4.dhcp.endAddress
|
|
else (calc.cidr.host ((calc.cidr.capacity lanCfg.ipv4.range) - 2) lanCfg.ipv4.range);
|
|
|
|
createLanInterface = ''
|
|
ip link add ${lanInterfaceName} type bridge
|
|
ip address add ${lanInterfaceIPAddress}/${networkPrefix} dev ${lanInterfaceName}
|
|
ip link set ${lanInterfaceName} up
|
|
'';
|
|
|
|
deleteLanInterface = ''
|
|
ip link del ${lanInterfaceName}
|
|
'';
|
|
|
|
lanDomainName = mkLanDomainName spaceName lanName lanCfg;
|
|
|
|
lanDnsmasqConf = ''
|
|
# lan ${lanName}
|
|
domain=${lanDomainName},${lanInterfaceName}
|
|
''
|
|
+ (lib.optionalString lanCfg.ipv4.dhcp.enable ''
|
|
dhcp-range=${lanInterfaceName},${dhcpStartAddress},${dhcpEndAddress},${netmask},12h
|
|
dhcp-option=${lanInterfaceName},option:classless-static-route,${lib.concatMapStringsSep "," (route: "${route},${lanInterfaceIPAddress}") ([ "0.0.0.0/0" ] ++ (builtins.filter (route: route != lanCfg.ipv4.range) staticRoutes))}
|
|
'')
|
|
+ (lib.optionalString (lanCfg.ipv4.dhcp.enable && (!wanCfg.dns.resolver.enable) && (lanCfg.ipv4.dhcp.dns.addresses != null) && (lanCfg.ipv4.dhcp.dns.addresses != []))
|
|
("dhcp-option=${lanInterfaceName},option:dns-server,${(lib.concatStringsSep "," lanCfg.ipv4.dhcp.dns.addresses)}\n"));
|
|
|
|
staticIPsListedOnVMs = with lib; concatMapAttrs (vmName: vmCfg: optionalAttrs ((vmCfg ? networks.lans.${lanName}.ip) && (vmCfg.networks.lans.${lanName}.ip != null)) { ${vmCfg.networks.lans.${lanName}.mac} = vmCfg.networks.lans.${lanName}.ip; }) config.vmix.namespaces.${spaceName}.vms;
|
|
in
|
|
lanCfg // {
|
|
createIface = createLanInterface;
|
|
deleteIface = deleteLanInterface;
|
|
dnsmasqConf = lanDnsmasqConf;
|
|
staticHostsFileContents = (with lib; concatStringsSep "\n" (mapAttrsToList (mac: ipv4: "${mac},${ipv4},infinite") (lanCfg.ipv4.dhcp.statics // staticIPsListedOnVMs)));
|
|
domain = lanDomainName;
|
|
};
|
|
|
|
mkLansService = spaceName: wanCfg: lansCfg:
|
|
let
|
|
lansTmpDir = "/tmp/vmix/${spaceName}";
|
|
dhcpDynamicLeaseFileForDnsmasq="${lansTmpDir}/lans.dhcp.dynamic.leases";
|
|
dhcpStaticHostsFileForDnsmasq="${lansTmpDir}/lans.dhcp.static.hosts";
|
|
staticRoutes = [ wanCfg.ipv4.range ] ++ (builtins.map (lanCfg: lanCfg.ipv4.range) (lib.attrValues lansCfg));
|
|
lansList = lib.attrValues(lib.mapAttrs (mkLan spaceName staticRoutes wanCfg) lansCfg);
|
|
dnsmasqConf = pkgs.writeText "dnsmasq-${spaceName}.conf" (''
|
|
dhcp-host=*:*:*:*:*:*,id:*
|
|
except-interface=lo
|
|
dhcp-authoritative
|
|
${lib.optionalString (!wanCfg.dns.resolver.enable) "port=0"}
|
|
localise-queries
|
|
no-hosts
|
|
expand-hosts
|
|
dhcp-leasefile=${dhcpDynamicLeaseFileForDnsmasq}
|
|
dhcp-hostsfile=${dhcpStaticHostsFileForDnsmasq}
|
|
filter-AAAA
|
|
address=/host/${calc.cidr.host 1 wanCfg.ipv4.range}
|
|
${lib.optionalString (wanCfg.dns.resolver.enable && wanCfg.dns.resolver.useHostResolvConf) "no-resolv"}
|
|
${lib.concatMapStringsSep "\n" (nameserver: "server=${nameserver}") (lib.optionals wanCfg.dns.resolver.enable wanCfg.dns.resolver.upstream)}
|
|
'' + (lib.concatMapStrings (lan: lan.dnsmasqConf) lansList)
|
|
);
|
|
|
|
createLansInterfaces = pkgs.writeShellScript "create-lans-${spaceName}-vmix" (''
|
|
# for dnsmasq temp files
|
|
mkdir -m 700 -p /tmp/vmix/${spaceName}
|
|
'' + (lib.concatMapStrings (lan: lan.createIface) lansList)
|
|
);
|
|
|
|
deleteLansInterfaces = pkgs.writeShellScript "delete-lans-${spaceName}-vmix" (''
|
|
# for dnsmasq temp files
|
|
rm -rf /tmp/vmix/${spaceName}
|
|
'' + lib.concatMapStrings (lan: lan.deleteIface) lansList
|
|
);
|
|
|
|
staticHostsFile = pkgs.writeText "dnsmasq-${spaceName}-static-hostsfile" (lib.concatMapStringsSep "\n" (lan: lan.staticHostsFileContents) lansList);
|
|
in
|
|
{
|
|
"lans.net.vmix@${spaceName}" = rec {
|
|
bindsTo = [ "ns.net.vmix@${spaceName}.service" ];
|
|
after = bindsTo ++ [ "lans.static-leases-hostsfile.net.vmix@${spaceName}.service" ];
|
|
wantedBy = [ "net.vmix@${spaceName}.target" ];
|
|
unitConfig.JoinsNamespaceOf = "ns.net.vmix@${spaceName}.service";
|
|
path = with pkgs; [ iproute2 ];
|
|
reloadTriggers = [ staticHostsFile ];
|
|
serviceConfig = {
|
|
ExecStartPre = createLansInterfaces;
|
|
ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${dnsmasqConf}";
|
|
ExecReload = pkgs.writeShellScript "reload-dnsmasq" "kill -HUP $MAINPID";
|
|
ExecStopPost = deleteLansInterfaces;
|
|
Restart = "on-failure";
|
|
RestartSec = "5";
|
|
PrivateTmp = true;
|
|
ProtectSystem = true;
|
|
ProtectHome = true;
|
|
PrivateNetwork = true;
|
|
};
|
|
};
|
|
|
|
"lans.static-leases-hostsfile.net.vmix@${spaceName}" = rec {
|
|
bindsTo = [ "ns.net.vmix@${spaceName}.service" ];
|
|
after = bindsTo;
|
|
requiredBy = [ "sysinit-reactivation.target" ];
|
|
before = requiredBy;
|
|
wantedBy = [ "net.vmix@${spaceName}.target" ];
|
|
unitConfig.JoinsNamespaceOf = "ns.net.vmix@${spaceName}.service";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecCondition = pkgs.writeShellScript "check-if-symlink-already-present.sh" "! [ ${staticHostsFile} -ef ${dhcpStaticHostsFileForDnsmasq} ]";
|
|
ExecStart = pkgs.writeShellScript "create-hostsfile-symlink.sh" "mkdir -m 700 -p ${lansTmpDir}; ln -sf ${staticHostsFile} ${dhcpStaticHostsFileForDnsmasq};";
|
|
Restart = "on-failure";
|
|
RestartSec = "5";
|
|
PrivateTmp = true;
|
|
ProtectSystem = true;
|
|
ProtectHome = true;
|
|
PrivateNetwork = true;
|
|
};
|
|
};
|
|
};
|
|
|
|
mkWanService = spaceName: cfg:
|
|
let
|
|
wanCfg = cfg // { spaceName = spaceName; };
|
|
vethInNSToHost.iface = "vhost";
|
|
# Temporary peer name, unique per namespace. The peer briefly exists in the
|
|
# host namespace before being moved; a shared name ("vhost") lets parallel
|
|
# wan.net.vmix@* starts steal each other's peer ends, cross-wiring namespaces.
|
|
vethInNSToHost.tempIface = "vh-${wanCfg.spaceName}";
|
|
vethOnHostToNS.iface = "vn-${wanCfg.spaceName}";
|
|
vethOnHostToNS.ipv4.address = calc.cidr.host 1 wanCfg.ipv4.range;
|
|
vethInNSToHost.ipv4.address = calc.cidr.host 2 wanCfg.ipv4.range;
|
|
networkPrefix = builtins.elemAt (lib.splitString "/" wanCfg.ipv4.range) 1;
|
|
iptablesMark = builtins.toString (ipv4ToInt vethOnHostToNS.ipv4.address);
|
|
portForwardRules = lib.concatStringsSep "\n" (lib.mapAttrsToList (hostIPnPort: nsPort: "iptables -t nat -A PREROUTING -p tcp --dport ${hostIPnPort} -j DNAT --to-destination ${vethInNSToHost.ipv4.address}:${toString nsPort}") wanCfg.forwardPorts);
|
|
|
|
createWanCommands = ''
|
|
ip link add ${vethOnHostToNS.iface} type veth peer name ${vethInNSToHost.tempIface}
|
|
ip link set ${vethInNSToHost.tempIface} netns ${wanCfg.spaceName}.vmix
|
|
ip netns exec ${wanCfg.spaceName}.vmix ip link set ${vethInNSToHost.tempIface} name ${vethInNSToHost.iface}
|
|
|
|
ip address add ${vethOnHostToNS.ipv4.address}/${networkPrefix} dev ${vethOnHostToNS.iface}
|
|
ip netns exec ${wanCfg.spaceName}.vmix ip address add ${vethInNSToHost.ipv4.address}/${networkPrefix} dev ${vethInNSToHost.iface}
|
|
|
|
iptables -A FORWARD -i ${vethOnHostToNS.iface} -j ACCEPT
|
|
iptables -A FORWARD -o ${vethOnHostToNS.iface} -j ACCEPT
|
|
${lib.optionalString (!wanCfg.host.reachable) "iptables -I INPUT 1 -i ${vethOnHostToNS.iface} -j DROP"}
|
|
|
|
${lib.optionalString wanCfg.masquerade "iptables -t mangle -A PREROUTING -i ${vethOnHostToNS.iface} -j MARK --set-mark ${iptablesMark}"}
|
|
${lib.optionalString wanCfg.masquerade "iptables -t nat -A POSTROUTING -m mark --mark ${iptablesMark} -j MASQUERADE"}
|
|
|
|
${portForwardRules}
|
|
|
|
ip link set ${vethOnHostToNS.iface} up
|
|
ip netns exec ${wanCfg.spaceName}.vmix ip link set ${vethInNSToHost.iface} up
|
|
ip netns exec ${wanCfg.spaceName}.vmix ip r add default via ${vethOnHostToNS.ipv4.address}
|
|
|
|
${lib.optionalString wanCfg.host.addNSLansRoutes (lib.concatMapStrings (lanRange: "ip r add ${lanRange} via ${vethInNSToHost.ipv4.address} \n") wanCfg.lanRanges)}
|
|
'';
|
|
|
|
createWan = pkgs.writeShellScript "create-wan-${wanCfg.spaceName}-vmix" createWanCommands;
|
|
|
|
deleteWan =
|
|
let
|
|
createdIptablesRules = lib.filter (line: (lib.hasPrefix "iptables" line)) (lib.splitString "\n" createWanCommands);
|
|
delIptablesRules = builtins.map (
|
|
rule: lib.replaceStrings [ "-I INPUT 1" "-A" ] [ "-D INPUT" "-D" ] rule
|
|
) createdIptablesRules;
|
|
in
|
|
pkgs.writeShellScript "delete-wan-${wanCfg.spaceName}-vmix" (''
|
|
ip link del ${vethOnHostToNS.iface}
|
|
'' + (lib.concatStringsSep "\n" delIptablesRules));
|
|
in
|
|
{
|
|
"wan.net.vmix@${wanCfg.spaceName}" = rec {
|
|
bindsTo = [ "ns.net.vmix@${wanCfg.spaceName}.service" ];
|
|
after = bindsTo;
|
|
wantedBy = [ "net.vmix@${wanCfg.spaceName}.target" ];
|
|
path = with pkgs; [ iproute2 iptables ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = createWan;
|
|
ExecStop = deleteWan;
|
|
};
|
|
};
|
|
};
|
|
|
|
mkVmGraphicsForwardPorts = spaceName:
|
|
let
|
|
vms = vmixCfg.namespaces.${spaceName}.vms;
|
|
|
|
vmGraphicsPortMappings = lib.concatLists (lib.mapAttrsToList (vmName: vmCfg:
|
|
(lib.optional (vmCfg.vnc.enable && vmCfg.vnc.forwardHostPort != null) {
|
|
hostPort = toString vmCfg.vnc.forwardHostPort;
|
|
nsPort = vmCfg.vnc.port;
|
|
reason = "vnc";
|
|
inherit vmName;
|
|
})
|
|
++ (lib.optional (vmCfg.spice.enable && vmCfg.spice.forwardHostPort != null) {
|
|
hostPort = toString vmCfg.spice.forwardHostPort;
|
|
nsPort = vmCfg.spice.port;
|
|
reason = "spice";
|
|
inherit vmName;
|
|
})
|
|
) vms);
|
|
|
|
addForwardPortMapping = acc: mapping:
|
|
if builtins.hasAttr mapping.hostPort acc then
|
|
throw "Duplicate vmix auto-forward host port ${mapping.hostPort} in namespace '${spaceName}' (at least '${mapping.vmName}' ${mapping.reason} conflicts with another VM mapping)."
|
|
else
|
|
acc // { ${mapping.hostPort} = mapping.nsPort; };
|
|
in
|
|
builtins.foldl' addForwardPortMapping {} vmGraphicsPortMappings;
|
|
|
|
mkNetworkServices = spaceName: cfg:
|
|
let
|
|
netCfg = cfg // { name = spaceName; };
|
|
vethIPv4RangeForWan = mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range;
|
|
vmGraphicsForwardPorts = mkVmGraphicsForwardPorts spaceName;
|
|
manualForwardPorts = netCfg.wan.forwardPorts;
|
|
wanCfg = netCfg.wan // {
|
|
ipv4.range = vethIPv4RangeForWan;
|
|
lanRanges = builtins.map (lan: lan.ipv4.range) (lib.attrValues netCfg.lans);
|
|
forwardPorts = vmGraphicsForwardPorts // manualForwardPorts;
|
|
};
|
|
in
|
|
(mkLansService netCfg.name wanCfg netCfg.lans)
|
|
// (lib.optionalAttrs wanCfg.enable (mkWanService netCfg.name wanCfg));
|
|
|
|
spaceNames = builtins.attrNames vmixCfg.namespaces;
|
|
|
|
networkServices = lib.mergeAttrsList (lib.imap0 (index: spaceName: (mkNetworkServices spaceName (vmixCfg.namespaces.${spaceName}.networks // { inherit index;}))) spaceNames);
|
|
networkTargets = lib.mergeAttrsList (builtins.map (spaceName: {
|
|
"net.vmix@${spaceName}" = {
|
|
description = "Network ${spaceName} for vmix";
|
|
bindsTo = [ "ns.net.vmix@${spaceName}.service" "lans.net.vmix@${spaceName}.service" ]
|
|
++ lib.optionals vmixCfg.namespaces.${spaceName}.networks.wan.enable [ "wan.net.vmix@${spaceName}.service" ];
|
|
};
|
|
}) spaceNames);
|
|
in
|
|
{
|
|
config.systemd.services = namespaceGlobalService // networkServices;
|
|
config.systemd.targets = networkTargets;
|
|
config.boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1;
|
|
config.boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = lib.mkForce true;
|
|
}
|