From 4254ebabaa2e8f4b8a5170dbd07a6726ef448a75 Mon Sep 17 00:00:00 2001 From: Sagar Ch Date: Mon, 10 Jun 2024 22:27:39 +0000 Subject: [PATCH] improvements in networking - macvtaps working - only 1 dnsmasq service per namespace - vms binds to networking services - lans with domains - vms no longer assigned same ip (machine id issues) - --- lib/images/commons/scripts-n-files.nix | 16 +++++- lib/images/debian/templates.nix | 4 +- nixos/network/config.nix | 78 ++++++++++++++++++-------- nixos/network/options.nix | 6 ++ nixos/vm/config.nix | 26 +++++---- 5 files changed, 93 insertions(+), 37 deletions(-) diff --git a/lib/images/commons/scripts-n-files.nix b/lib/images/commons/scripts-n-files.nix index 6540f71..1d104b8 100644 --- a/lib/images/commons/scripts-n-files.nix +++ b/lib/images/commons/scripts-n-files.nix @@ -11,12 +11,24 @@ ''; # dhcp for eth0 - eth0-dhcp-network = pkgs.writeText "eth0-network" '' + dhcp-network-for-iface = { iface, routeMetric ? 1024, useDNS ? true }: pkgs.writeText "${iface}-network" '' [Match] - Name=eth0 + Name=${iface} [Network] DHCP=yes + + [DHCP] + RouteMetric=${toString routeMetric} + + ${if useDNS then '' + [DHCPv4] + UseDNS=True + UseDomains=True + '' else '' + [DHCPv4] + UseDNS=false + ''} ''; # generate ssh host keys before starting sshd diff --git a/lib/images/debian/templates.nix b/lib/images/debian/templates.nix index 37d5fcb..7eaa3f8 100644 --- a/lib/images/debian/templates.nix +++ b/lib/images/debian/templates.nix @@ -10,11 +10,13 @@ with scriptsNFiles; upload ${grub-ifnames-0}:/etc/default/grub.d/90-ifnames-0.cfg upload ${grub-disable-microcode}:/etc/default/grub.d/00-disable-microcode.cfg run-command mount /boot/efi && update-grub - upload ${eth0-dhcp-network}:/etc/systemd/network/00-eth0-dhcp.network + upload ${dhcp-network-for-iface { iface = "eth0"; }}:/etc/systemd/network/00-eth0-dhcp.network run ${ssh-service-override-conf-create} upload ${grow-root-sh}:/usr/local/sbin/grow-root.sh upload ${grow-root-service}:/etc/systemd/system/grow-root.service run-command systemctl enable grow-root.service + truncate /etc/machine-id + delete /var/lib/dbus/machine-id ''; }; diff --git a/nixos/network/config.nix b/nixos/network/config.nix index a60699b..bc6f9e0 100644 --- a/nixos/network/config.nix +++ b/nixos/network/config.nix @@ -29,59 +29,87 @@ let }; }; - mkLanService = networkName: lanName: cfg: + mkLanDomainName = networkName: lanName: lanCfg: + if (lanCfg.domain != null) then lanCfg.domain else "${lanName}.${networkName}.vmix"; + + mkLan = networkName: lanName: cfg: let lanCfg = cfg // { name = lanName; namespace = "${networkName}"; }; lanInterfaceName = "brx-${lanCfg.name}"; lanInterfaceIPAddress = calc.cidr.host 1 lanCfg.ipv4.range; netmask = calc.cidr.netmask lanCfg.ipv4.range; networkPrefix = builtins.elemAt (lib.splitString "/" lanCfg.ipv4.range) 1; + dhcpStartAddress = if (lanCfg.ipv4.dhcp.startAddress != null) then lanCfg.ipv4.dhcp.startAddress else (calc.cidr.host 2 lanCfg.ipv4.range); + dhcpEndAddress = if (lanCfg.ipv4.dhcp.endAddress != null) then lanCfg.ipv4.dhcp.endAddress else (calc.cidr.host ((calc.cidr.capacity lanCfg.ipv4.range) - 2) lanCfg.ipv4.range); - createLanInterface = pkgs.writeShellScript "create-lan-${lanCfg.name}-vmix" '' + createLanInterface = '' ip link add ${lanInterfaceName} type bridge ip address add ${lanInterfaceIPAddress}/${networkPrefix} dev ${lanInterfaceName} ip link set ${lanInterfaceName} up ''; - deleteLanInterface = pkgs.writeShellScript "delete-lan-${lanCfg.name}-vmix" "ip link del ${lanInterfaceName}"; - lanDomainName = "${lanCfg.name}.vmix"; - lanDnsmasqConf = pkgs.writeText "dnsmasq-${lanCfg.name}.conf" ('' - listen-address=${lanInterfaceIPAddress} - dhcp-range=${dhcpStartAddress},${dhcpEndAddress},${netmask},12h - interface=${lanInterfaceName} - bind-interfaces + deleteLanInterface = '' + ip link del ${lanInterfaceName} + ''; + + lanDomainName = mkLanDomainName networkName lanName lanCfg; + + lanDnsmasqConf = '' + dhcp-range=${lanInterfaceName},${dhcpStartAddress},${dhcpEndAddress},${netmask},12h + domain=${lanDomainName},${lanInterfaceName} + '' + (lib.optionalString (lanCfg.ipv4.dns.upstream != []) ("dhcp-option=${lanInterfaceName},option:dns-server,${(lib.concatStringsSep "," lanCfg.ipv4.dns.upstream)}\n")); + in + lanCfg // { + createIface = createLanInterface; + deleteIface = deleteLanInterface; + dnsmasqConf = lanDnsmasqConf; + domain = lanDomainName; + }; + + mkLansService = networkName: lansCfg: + let + dhcpLeaseFile="/tmp/vmix/lans.${networkName}.dhcp.leases"; + lansList = lib.attrValues(lib.mapAttrs (mkLan networkName) lansCfg); + dnsmasqConf = pkgs.writeText "dnsmasq-${networkName}.conf" ('' except-interface=lo dhcp-authoritative - domain=${lanDomainName} - domain-needed localise-queries no-hosts expand-hosts - dhcp-leasefile=/tmp/${lanCfg.name}.vmix.dhcp.leases - '' + - lib.concatStringsSep "\n" (lib.optionals (lanCfg.ipv4.dns.upstream != []) ([ "no-resolv" ] ++ (builtins.map (dnsServer: "server=${dnsServer}") lanCfg.ipv4.dns.upstream))) + dhcp-leasefile=${dhcpLeaseFile} + filter-AAAA + '' + (lib.concatMapStrings (lan: lan.dnsmasqConf) lansList) ); + + createLansInterfaces = pkgs.writeShellScript "create-lans-${networkName}-vmix" ('' + # for dnsmasq temp files + mkdir -p /tmp/vmix + rm -f ${dhcpLeaseFile} + '' + (lib.concatMapStrings (lan: lan.createIface) lansList) + ); + + deleteLansInterfaces = pkgs.writeShellScript "delete-lans-${networkName}-vmix" (lib.concatMapStrings (lan: lan.deleteIface) lansList); in { - "lan.net.vmix@${lanCfg.name}.${lanCfg.namespace}" = rec { - bindsTo = [ "ns.net.vmix@${lanCfg.namespace}.service" ]; + "lans.net.vmix@${networkName}" = rec { + bindsTo = [ "ns.net.vmix@${networkName}.service" ]; after = bindsTo; - wantedBy = [ "net.vmix@${lanCfg.namespace}.target" ]; - unitConfig.JoinsNamespaceOf = "ns.net.vmix@${lanCfg.namespace}.service"; + wantedBy = [ "net.vmix@${networkName}.target" ]; + unitConfig.JoinsNamespaceOf = "ns.net.vmix@${networkName}.service"; path = with pkgs; [ iproute2 ]; serviceConfig = { - ExecStartPre = createLanInterface; - ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${lanDnsmasqConf}"; + ExecStartPre = createLansInterfaces; + ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${dnsmasqConf}"; ExecReload = pkgs.writeShellScript "reload-dnsmasq" "kill -HUP $MAINPID"; - ExecStopPost = deleteLanInterface; + ExecStopPost = deleteLansInterfaces; Restart = "on-failure"; RestartSec = "5"; PrivateTmp = true; @@ -119,6 +147,8 @@ let ip link set ${vethOnHostToNS.iface} up ip netns exec ${wanCfg.namespace}.vmix ip link set ${vethInNSToHost.iface} up ip netns exec ${wanCfg.namespace}.vmix ip r add default via ${vethOnHostToNS.ipv4.address} + + ${lib.concatMapStrings (lanRange: "ip r add ${lanRange} via ${vethInNSToHost.ipv4.address} \n") wanCfg.lanRanges} ''; createWan = pkgs.writeShellScript "create-wan-${wanCfg.namespace}-vmix" createWanCommands; @@ -154,8 +184,8 @@ let let netCfg = cfg // { name = networkName; }; in - (lib.concatMapAttrs (mkLanService netCfg.name) netCfg.lans) - // (mkWanService netCfg.name (netCfg.wan // { ipv4.range = (mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range); })) + (mkLansService netCfg.name netCfg.lans) + // (mkWanService netCfg.name (netCfg.wan // { ipv4.range = (mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range); lanRanges = builtins.map (lan: lan.ipv4.range) (lib.attrValues netCfg.lans); })) // (lib.concatMapAttrs (mkMacvlanService netCfg.name) netCfg.bridges.macvlans); networkNames = builtins.attrNames vmixCfg.networks; @@ -164,7 +194,7 @@ let networkTargets = lib.concatMapAttrs (networkName: netCfg: { "net.vmix@${networkName}" = { description = "Network ${networkName} for vmix"; - bindsTo = [ "ns.net.vmix@${networkName}.service" ]; + bindsTo = [ "ns.net.vmix@${networkName}.service" "lans.net.vmix@${networkName}.service" "wan.net.vmix@${networkName}.service" ]; }; }) vmixCfg.networks; in diff --git a/nixos/network/options.nix b/nixos/network/options.nix index 9dc1c8a..fda4ba9 100644 --- a/nixos/network/options.nix +++ b/nixos/network/options.nix @@ -97,6 +97,12 @@ with vmixLib.network; lans = mkOption { type = types.attrsOf (types.submodule { + options.domain = mkOption { + type = types.nullOr types.str; + default = null; + description = "Domain name for the hosts of this lan."; + }; + options.ipv4 = { range = mkOption { type = types.strMatching regex.cidr4; diff --git a/nixos/vm/config.nix b/nixos/vm/config.nix index bc3a647..f521cd0 100644 --- a/nixos/vm/config.nix +++ b/nixos/vm/config.nix @@ -41,6 +41,7 @@ let create = '' ip link add link ${macvtapNetworkCfg.uplink.iface} name ${macvtapInterfaceName} type macvtap mode bridge ip link set ${macvtapInterfaceName} netns ${netName}.vmix + ip netns exec ${netName}.vmix ip link set dev ${macvtapInterfaceName} up ''; delete = '' ip netns exec ${netName}.vmix ip link del ${macvtapInterfaceName} @@ -66,7 +67,13 @@ let concatStringsSep "\n" (builtins.map (macvtap: macvtap.delete) allMacvtaps) ); - osImage = vmixLib.customizeImage vmCfg.disks.os.file { name = vmCfg.name; }; + osImage = vmixLib.customizeImage vmCfg.disks.os.file { + name = vmCfg.name; + commands = '' + truncate /etc/machine-id + run-command systemd-machine-id-setup + ''; + }; qemuStartVMScript = pkgs.writeShellScript "${vmCfg.name}-qemu-vmix" '' exec qemu-system-${vmCfg.arch} \ @@ -96,20 +103,19 @@ let -device virtio-net-pci,netdev=user \ "} \ ${optionalString (vmCfg.boot.menu == true) "-boot menu=on"} \ - #${optionalString (length vmCfg.boot.order > 0) "-boot order=${concatStringsSep "," vmCfg.boot.order}"} - - # ${concatMapStrings (macvtap: '' - # -device virtio-net-pci,netdev=macvtap-${macvtap.name} \ - # -netdev tap,id=macvtap-${macvtap.name},ifname=${macvtap.iface},script=no,downscript=no \ - # '') allMacvtaps} \ + ${concatStrings (imap1 (i: macvtap: '' + -device virtio-net-pci,netdev=macvtap-${macvtap.name},mac=$(ip l show ${macvtap.iface} | awk '/link\/ether/{print $2}') \ + -netdev tap,id=macvtap-${macvtap.name},fd=${toString (i+2)} ${toString (i+2)}<>/dev/tap$(ip l show ${macvtap.iface} | awk -F':' '/${macvtap.iface}/{print $1}') \ + '') allMacvtaps)} \ + #${optionalString (length vmCfg.boot.order > 0) "-boot order=${concatStringsSep "," vmCfg.boot.order}"} \ ''; in { "vm.vmix@${vmCfg.name}" = rec { - requires = [ "net.vmix@${netCfg.name}.target" "macvtaps.vm.vmix@${vmCfg.name}.service" ]; + bindsTo = [ "net.vmix@${netCfg.name}.target" "macvtaps.vm.vmix@${vmCfg.name}.service" ]; unitConfig.JoinsNamespaceOf = "ns.net.vmix@${netCfg.name}.service"; - after = requires; - path = with pkgs; [ iproute2 qemu ]; + after = bindsTo; + path = with pkgs; [ iproute2 qemu gawk ]; serviceConfig = { ExecStartPre = createTapsforLansScript; ExecStart = qemuStartVMScript;