improvements in networking

- macvtaps working
- only 1 dnsmasq service per namespace
- vms binds to networking services
- lans with domains
- vms no longer assigned same ip (machine id issues)
-

lib/images/commons/scripts-n-files.nix

@@ -11,12 +11,24 @@
   '';
 
   # dhcp for eth0
-  eth0-dhcp-network = pkgs.writeText "eth0-network" ''
+  dhcp-network-for-iface = { iface, routeMetric ? 1024, useDNS ? true }: pkgs.writeText "${iface}-network" ''
     [Match]
-    Name=eth0
+    Name=${iface}
 
     [Network]
     DHCP=yes
+
+    [DHCP]
+    RouteMetric=${toString routeMetric}
+
+    ${if useDNS then ''
+    [DHCPv4]
+    UseDNS=True
+    UseDomains=True
+    '' else ''
+    [DHCPv4]
+    UseDNS=false
+    ''}
   '';
 
   # generate ssh host keys before starting sshd

lib/images/debian/templates.nix

@@ -10,11 +10,13 @@ with scriptsNFiles;
       upload ${grub-ifnames-0}:/etc/default/grub.d/90-ifnames-0.cfg
       upload ${grub-disable-microcode}:/etc/default/grub.d/00-disable-microcode.cfg
       run-command mount /boot/efi && update-grub
-      upload ${eth0-dhcp-network}:/etc/systemd/network/00-eth0-dhcp.network
+      upload ${dhcp-network-for-iface { iface = "eth0"; }}:/etc/systemd/network/00-eth0-dhcp.network
       run ${ssh-service-override-conf-create}
       upload ${grow-root-sh}:/usr/local/sbin/grow-root.sh
       upload ${grow-root-service}:/etc/systemd/system/grow-root.service
       run-command systemctl enable grow-root.service
+      truncate /etc/machine-id
+      delete /var/lib/dbus/machine-id
     '';
   };
 

nixos/network/config.nix

@@ -29,59 +29,87 @@ let
     };
   };
 
-  mkLanService = networkName: lanName: cfg:
+  mkLanDomainName = networkName: lanName: lanCfg:
+    if (lanCfg.domain != null) then lanCfg.domain else "${lanName}.${networkName}.vmix";
+
+  mkLan = networkName: lanName: cfg:
     let
       lanCfg = cfg // { name = lanName; namespace = "${networkName}"; };
       lanInterfaceName = "brx-${lanCfg.name}";
       lanInterfaceIPAddress = calc.cidr.host 1 lanCfg.ipv4.range;
       netmask =  calc.cidr.netmask lanCfg.ipv4.range;
       networkPrefix = builtins.elemAt (lib.splitString "/" lanCfg.ipv4.range) 1;
+
       dhcpStartAddress =
         if (lanCfg.ipv4.dhcp.startAddress != null)
         then lanCfg.ipv4.dhcp.startAddress
         else (calc.cidr.host 2 lanCfg.ipv4.range);
+
       dhcpEndAddress =
         if (lanCfg.ipv4.dhcp.endAddress != null)
         then lanCfg.ipv4.dhcp.endAddress
         else (calc.cidr.host ((calc.cidr.capacity lanCfg.ipv4.range) - 2) lanCfg.ipv4.range);
 
-      createLanInterface = pkgs.writeShellScript "create-lan-${lanCfg.name}-vmix" ''
+      createLanInterface = ''
         ip link add ${lanInterfaceName} type bridge
         ip address add ${lanInterfaceIPAddress}/${networkPrefix} dev ${lanInterfaceName}
         ip link set ${lanInterfaceName} up
       '';
-      deleteLanInterface = pkgs.writeShellScript "delete-lan-${lanCfg.name}-vmix" "ip link del ${lanInterfaceName}";
 
-      lanDomainName = "${lanCfg.name}.vmix";
+      deleteLanInterface = ''
-      lanDnsmasqConf = pkgs.writeText "dnsmasq-${lanCfg.name}.conf" (''
+        ip link del ${lanInterfaceName}
-        listen-address=${lanInterfaceIPAddress}
+      '';
-        dhcp-range=${dhcpStartAddress},${dhcpEndAddress},${netmask},12h
+
-        interface=${lanInterfaceName}
+      lanDomainName = mkLanDomainName networkName lanName lanCfg;
-        bind-interfaces
+
+      lanDnsmasqConf = ''
+        dhcp-range=${lanInterfaceName},${dhcpStartAddress},${dhcpEndAddress},${netmask},12h
+        domain=${lanDomainName},${lanInterfaceName}
+      '' + (lib.optionalString (lanCfg.ipv4.dns.upstream != []) ("dhcp-option=${lanInterfaceName},option:dns-server,${(lib.concatStringsSep "," lanCfg.ipv4.dns.upstream)}\n"));
+    in
+    lanCfg // {
+      createIface = createLanInterface;
+      deleteIface = deleteLanInterface;
+      dnsmasqConf = lanDnsmasqConf;
+      domain = lanDomainName;
+    };
+
+  mkLansService = networkName: lansCfg:
+    let
+      dhcpLeaseFile="/tmp/vmix/lans.${networkName}.dhcp.leases";
+      lansList = lib.attrValues(lib.mapAttrs (mkLan networkName) lansCfg);
+      dnsmasqConf = pkgs.writeText "dnsmasq-${networkName}.conf" (''
         except-interface=lo
         dhcp-authoritative
-        domain=${lanDomainName}
-        domain-needed
         localise-queries
         no-hosts
         expand-hosts
-        dhcp-leasefile=/tmp/${lanCfg.name}.vmix.dhcp.leases
+        dhcp-leasefile=${dhcpLeaseFile}
-      '' +
+        filter-AAAA
-      lib.concatStringsSep "\n" (lib.optionals (lanCfg.ipv4.dns.upstream != []) ([ "no-resolv" ] ++ (builtins.map (dnsServer: "server=${dnsServer}") lanCfg.ipv4.dns.upstream)))
+      '' + (lib.concatMapStrings (lan: lan.dnsmasqConf) lansList)
       );
+
+      createLansInterfaces = pkgs.writeShellScript "create-lans-${networkName}-vmix" (''
+        # for dnsmasq temp files
+        mkdir -p /tmp/vmix
+        rm -f ${dhcpLeaseFile}
+        '' + (lib.concatMapStrings (lan: lan.createIface) lansList)
+      );
+
+      deleteLansInterfaces = pkgs.writeShellScript "delete-lans-${networkName}-vmix" (lib.concatMapStrings (lan: lan.deleteIface) lansList);
     in
     {
-      "lan.net.vmix@${lanCfg.name}.${lanCfg.namespace}" = rec {
+      "lans.net.vmix@${networkName}" = rec {
-        bindsTo = [ "ns.net.vmix@${lanCfg.namespace}.service" ];
+        bindsTo = [ "ns.net.vmix@${networkName}.service" ];
         after = bindsTo;
-        wantedBy = [ "net.vmix@${lanCfg.namespace}.target" ];
+        wantedBy = [ "net.vmix@${networkName}.target" ];
-        unitConfig.JoinsNamespaceOf = "ns.net.vmix@${lanCfg.namespace}.service";
+        unitConfig.JoinsNamespaceOf = "ns.net.vmix@${networkName}.service";
         path = with pkgs; [ iproute2 ];
         serviceConfig = {
-          ExecStartPre = createLanInterface;
+          ExecStartPre = createLansInterfaces;
-          ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${lanDnsmasqConf}";
+          ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${dnsmasqConf}";
           ExecReload = pkgs.writeShellScript "reload-dnsmasq" "kill -HUP $MAINPID";
-          ExecStopPost = deleteLanInterface;
+          ExecStopPost = deleteLansInterfaces;
           Restart = "on-failure";
           RestartSec = "5";
           PrivateTmp = true;
@@ -119,6 +147,8 @@ let
         ip link set ${vethOnHostToNS.iface} up
         ip netns exec ${wanCfg.namespace}.vmix ip link set ${vethInNSToHost.iface} up
         ip netns exec ${wanCfg.namespace}.vmix ip r add default via ${vethOnHostToNS.ipv4.address}
+
+        ${lib.concatMapStrings (lanRange: "ip r add ${lanRange} via ${vethInNSToHost.ipv4.address} \n") wanCfg.lanRanges}
       '';
 
       createWan = pkgs.writeShellScript "create-wan-${wanCfg.namespace}-vmix" createWanCommands;
@@ -154,8 +184,8 @@ let
     let
       netCfg = cfg // { name = networkName; };
     in
-    (lib.concatMapAttrs (mkLanService netCfg.name) netCfg.lans)
+    (mkLansService netCfg.name netCfg.lans)
-    // (mkWanService netCfg.name (netCfg.wan // { ipv4.range = (mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range); }))
+    // (mkWanService netCfg.name (netCfg.wan // { ipv4.range = (mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range); lanRanges = builtins.map (lan: lan.ipv4.range) (lib.attrValues netCfg.lans); }))
     // (lib.concatMapAttrs (mkMacvlanService netCfg.name) netCfg.bridges.macvlans);
 
   networkNames = builtins.attrNames vmixCfg.networks;
@@ -164,7 +194,7 @@ let
   networkTargets = lib.concatMapAttrs (networkName: netCfg: {
     "net.vmix@${networkName}" = {
       description = "Network ${networkName} for vmix";
-      bindsTo = [ "ns.net.vmix@${networkName}.service" ];
+      bindsTo = [ "ns.net.vmix@${networkName}.service" "lans.net.vmix@${networkName}.service" "wan.net.vmix@${networkName}.service" ];
     };
   }) vmixCfg.networks;
 in

nixos/network/options.nix

@@ -97,6 +97,12 @@ with vmixLib.network;
 
     lans = mkOption {
       type = types.attrsOf (types.submodule {
+        options.domain = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = "Domain name for the hosts of this lan.";
+        };
+
         options.ipv4 = {
           range = mkOption {
             type = types.strMatching regex.cidr4;

nixos/vm/config.nix

@@ -41,6 +41,7 @@ let
         create = ''
           ip link add link ${macvtapNetworkCfg.uplink.iface} name ${macvtapInterfaceName} type macvtap mode bridge
           ip link set ${macvtapInterfaceName} netns ${netName}.vmix
+          ip netns exec ${netName}.vmix ip link set dev ${macvtapInterfaceName} up
         '';
         delete = ''
           ip netns exec ${netName}.vmix ip link del ${macvtapInterfaceName}
@@ -66,7 +67,13 @@ let
         concatStringsSep "\n" (builtins.map (macvtap: macvtap.delete) allMacvtaps)
       );
 
-      osImage = vmixLib.customizeImage vmCfg.disks.os.file { name = vmCfg.name; };
+      osImage = vmixLib.customizeImage vmCfg.disks.os.file { 
+        name = vmCfg.name;
+        commands = ''
+          truncate /etc/machine-id
+          run-command systemd-machine-id-setup
+        '';
+      };
 
       qemuStartVMScript = pkgs.writeShellScript "${vmCfg.name}-qemu-vmix" ''
         exec qemu-system-${vmCfg.arch} \
@@ -96,20 +103,19 @@ let
             -device virtio-net-pci,netdev=user \
           "} \
           ${optionalString (vmCfg.boot.menu == true) "-boot menu=on"} \
-          #${optionalString (length vmCfg.boot.order > 0) "-boot order=${concatStringsSep "," vmCfg.boot.order}"}
+          ${concatStrings (imap1 (i: macvtap: ''
-
+            -device virtio-net-pci,netdev=macvtap-${macvtap.name},mac=$(ip l show ${macvtap.iface} | awk '/link\/ether/{print $2}') \
-          # ${concatMapStrings (macvtap: ''
+            -netdev tap,id=macvtap-${macvtap.name},fd=${toString (i+2)} ${toString (i+2)}<>/dev/tap$(ip l show ${macvtap.iface} | awk -F':' '/${macvtap.iface}/{print $1}') \
-          #   -device virtio-net-pci,netdev=macvtap-${macvtap.name} \
+          '') allMacvtaps)} \
-          #   -netdev tap,id=macvtap-${macvtap.name},ifname=${macvtap.iface},script=no,downscript=no \
+          #${optionalString (length vmCfg.boot.order > 0) "-boot order=${concatStringsSep "," vmCfg.boot.order}"} \
-          # '') allMacvtaps} \
       '';
   in
   {
     "vm.vmix@${vmCfg.name}" = rec {
-      requires = [ "net.vmix@${netCfg.name}.target" "macvtaps.vm.vmix@${vmCfg.name}.service" ];
+      bindsTo = [ "net.vmix@${netCfg.name}.target" "macvtaps.vm.vmix@${vmCfg.name}.service" ];
       unitConfig.JoinsNamespaceOf = "ns.net.vmix@${netCfg.name}.service";
-      after = requires;
+      after = bindsTo;
-      path = with pkgs; [ iproute2 qemu ];
+      path = with pkgs; [ iproute2 qemu gawk ];
       serviceConfig = {
         ExecStartPre = createTapsforLansScript;
         ExecStart = qemuStartVMScript;