improvements in networking
- macvtaps working - only 1 dnsmasq service per namespace - vms binds to networking services - lans with domains - vms no longer assigned same ip (machine id issues) -
This commit is contained in:
parent
3d27f32c03
commit
4254ebabaa
5 changed files with 93 additions and 37 deletions
|
|
@ -11,12 +11,24 @@
|
|||
'';
|
||||
|
||||
# dhcp for eth0
|
||||
eth0-dhcp-network = pkgs.writeText "eth0-network" ''
|
||||
dhcp-network-for-iface = { iface, routeMetric ? 1024, useDNS ? true }: pkgs.writeText "${iface}-network" ''
|
||||
[Match]
|
||||
Name=eth0
|
||||
Name=${iface}
|
||||
|
||||
[Network]
|
||||
DHCP=yes
|
||||
|
||||
[DHCP]
|
||||
RouteMetric=${toString routeMetric}
|
||||
|
||||
${if useDNS then ''
|
||||
[DHCPv4]
|
||||
UseDNS=True
|
||||
UseDomains=True
|
||||
'' else ''
|
||||
[DHCPv4]
|
||||
UseDNS=false
|
||||
''}
|
||||
'';
|
||||
|
||||
# generate ssh host keys before starting sshd
|
||||
|
|
|
|||
|
|
@ -10,11 +10,13 @@ with scriptsNFiles;
|
|||
upload ${grub-ifnames-0}:/etc/default/grub.d/90-ifnames-0.cfg
|
||||
upload ${grub-disable-microcode}:/etc/default/grub.d/00-disable-microcode.cfg
|
||||
run-command mount /boot/efi && update-grub
|
||||
upload ${eth0-dhcp-network}:/etc/systemd/network/00-eth0-dhcp.network
|
||||
upload ${dhcp-network-for-iface { iface = "eth0"; }}:/etc/systemd/network/00-eth0-dhcp.network
|
||||
run ${ssh-service-override-conf-create}
|
||||
upload ${grow-root-sh}:/usr/local/sbin/grow-root.sh
|
||||
upload ${grow-root-service}:/etc/systemd/system/grow-root.service
|
||||
run-command systemctl enable grow-root.service
|
||||
truncate /etc/machine-id
|
||||
delete /var/lib/dbus/machine-id
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -29,59 +29,87 @@ let
|
|||
};
|
||||
};
|
||||
|
||||
mkLanService = networkName: lanName: cfg:
|
||||
mkLanDomainName = networkName: lanName: lanCfg:
|
||||
if (lanCfg.domain != null) then lanCfg.domain else "${lanName}.${networkName}.vmix";
|
||||
|
||||
mkLan = networkName: lanName: cfg:
|
||||
let
|
||||
lanCfg = cfg // { name = lanName; namespace = "${networkName}"; };
|
||||
lanInterfaceName = "brx-${lanCfg.name}";
|
||||
lanInterfaceIPAddress = calc.cidr.host 1 lanCfg.ipv4.range;
|
||||
netmask = calc.cidr.netmask lanCfg.ipv4.range;
|
||||
networkPrefix = builtins.elemAt (lib.splitString "/" lanCfg.ipv4.range) 1;
|
||||
|
||||
dhcpStartAddress =
|
||||
if (lanCfg.ipv4.dhcp.startAddress != null)
|
||||
then lanCfg.ipv4.dhcp.startAddress
|
||||
else (calc.cidr.host 2 lanCfg.ipv4.range);
|
||||
|
||||
dhcpEndAddress =
|
||||
if (lanCfg.ipv4.dhcp.endAddress != null)
|
||||
then lanCfg.ipv4.dhcp.endAddress
|
||||
else (calc.cidr.host ((calc.cidr.capacity lanCfg.ipv4.range) - 2) lanCfg.ipv4.range);
|
||||
|
||||
createLanInterface = pkgs.writeShellScript "create-lan-${lanCfg.name}-vmix" ''
|
||||
createLanInterface = ''
|
||||
ip link add ${lanInterfaceName} type bridge
|
||||
ip address add ${lanInterfaceIPAddress}/${networkPrefix} dev ${lanInterfaceName}
|
||||
ip link set ${lanInterfaceName} up
|
||||
'';
|
||||
deleteLanInterface = pkgs.writeShellScript "delete-lan-${lanCfg.name}-vmix" "ip link del ${lanInterfaceName}";
|
||||
|
||||
lanDomainName = "${lanCfg.name}.vmix";
|
||||
lanDnsmasqConf = pkgs.writeText "dnsmasq-${lanCfg.name}.conf" (''
|
||||
listen-address=${lanInterfaceIPAddress}
|
||||
dhcp-range=${dhcpStartAddress},${dhcpEndAddress},${netmask},12h
|
||||
interface=${lanInterfaceName}
|
||||
bind-interfaces
|
||||
deleteLanInterface = ''
|
||||
ip link del ${lanInterfaceName}
|
||||
'';
|
||||
|
||||
lanDomainName = mkLanDomainName networkName lanName lanCfg;
|
||||
|
||||
lanDnsmasqConf = ''
|
||||
dhcp-range=${lanInterfaceName},${dhcpStartAddress},${dhcpEndAddress},${netmask},12h
|
||||
domain=${lanDomainName},${lanInterfaceName}
|
||||
'' + (lib.optionalString (lanCfg.ipv4.dns.upstream != []) ("dhcp-option=${lanInterfaceName},option:dns-server,${(lib.concatStringsSep "," lanCfg.ipv4.dns.upstream)}\n"));
|
||||
in
|
||||
lanCfg // {
|
||||
createIface = createLanInterface;
|
||||
deleteIface = deleteLanInterface;
|
||||
dnsmasqConf = lanDnsmasqConf;
|
||||
domain = lanDomainName;
|
||||
};
|
||||
|
||||
mkLansService = networkName: lansCfg:
|
||||
let
|
||||
dhcpLeaseFile="/tmp/vmix/lans.${networkName}.dhcp.leases";
|
||||
lansList = lib.attrValues(lib.mapAttrs (mkLan networkName) lansCfg);
|
||||
dnsmasqConf = pkgs.writeText "dnsmasq-${networkName}.conf" (''
|
||||
except-interface=lo
|
||||
dhcp-authoritative
|
||||
domain=${lanDomainName}
|
||||
domain-needed
|
||||
localise-queries
|
||||
no-hosts
|
||||
expand-hosts
|
||||
dhcp-leasefile=/tmp/${lanCfg.name}.vmix.dhcp.leases
|
||||
'' +
|
||||
lib.concatStringsSep "\n" (lib.optionals (lanCfg.ipv4.dns.upstream != []) ([ "no-resolv" ] ++ (builtins.map (dnsServer: "server=${dnsServer}") lanCfg.ipv4.dns.upstream)))
|
||||
dhcp-leasefile=${dhcpLeaseFile}
|
||||
filter-AAAA
|
||||
'' + (lib.concatMapStrings (lan: lan.dnsmasqConf) lansList)
|
||||
);
|
||||
|
||||
createLansInterfaces = pkgs.writeShellScript "create-lans-${networkName}-vmix" (''
|
||||
# for dnsmasq temp files
|
||||
mkdir -p /tmp/vmix
|
||||
rm -f ${dhcpLeaseFile}
|
||||
'' + (lib.concatMapStrings (lan: lan.createIface) lansList)
|
||||
);
|
||||
|
||||
deleteLansInterfaces = pkgs.writeShellScript "delete-lans-${networkName}-vmix" (lib.concatMapStrings (lan: lan.deleteIface) lansList);
|
||||
in
|
||||
{
|
||||
"lan.net.vmix@${lanCfg.name}.${lanCfg.namespace}" = rec {
|
||||
bindsTo = [ "ns.net.vmix@${lanCfg.namespace}.service" ];
|
||||
"lans.net.vmix@${networkName}" = rec {
|
||||
bindsTo = [ "ns.net.vmix@${networkName}.service" ];
|
||||
after = bindsTo;
|
||||
wantedBy = [ "net.vmix@${lanCfg.namespace}.target" ];
|
||||
unitConfig.JoinsNamespaceOf = "ns.net.vmix@${lanCfg.namespace}.service";
|
||||
wantedBy = [ "net.vmix@${networkName}.target" ];
|
||||
unitConfig.JoinsNamespaceOf = "ns.net.vmix@${networkName}.service";
|
||||
path = with pkgs; [ iproute2 ];
|
||||
serviceConfig = {
|
||||
ExecStartPre = createLanInterface;
|
||||
ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${lanDnsmasqConf}";
|
||||
ExecStartPre = createLansInterfaces;
|
||||
ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq -d -C ${dnsmasqConf}";
|
||||
ExecReload = pkgs.writeShellScript "reload-dnsmasq" "kill -HUP $MAINPID";
|
||||
ExecStopPost = deleteLanInterface;
|
||||
ExecStopPost = deleteLansInterfaces;
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5";
|
||||
PrivateTmp = true;
|
||||
|
|
@ -119,6 +147,8 @@ let
|
|||
ip link set ${vethOnHostToNS.iface} up
|
||||
ip netns exec ${wanCfg.namespace}.vmix ip link set ${vethInNSToHost.iface} up
|
||||
ip netns exec ${wanCfg.namespace}.vmix ip r add default via ${vethOnHostToNS.ipv4.address}
|
||||
|
||||
${lib.concatMapStrings (lanRange: "ip r add ${lanRange} via ${vethInNSToHost.ipv4.address} \n") wanCfg.lanRanges}
|
||||
'';
|
||||
|
||||
createWan = pkgs.writeShellScript "create-wan-${wanCfg.namespace}-vmix" createWanCommands;
|
||||
|
|
@ -154,8 +184,8 @@ let
|
|||
let
|
||||
netCfg = cfg // { name = networkName; };
|
||||
in
|
||||
(lib.concatMapAttrs (mkLanService netCfg.name) netCfg.lans)
|
||||
// (mkWanService netCfg.name (netCfg.wan // { ipv4.range = (mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range); }))
|
||||
(mkLansService netCfg.name netCfg.lans)
|
||||
// (mkWanService netCfg.name (netCfg.wan // { ipv4.range = (mkVethIPv4Range netCfg.index vmixCfg.global.net.wan.ipv4.range); lanRanges = builtins.map (lan: lan.ipv4.range) (lib.attrValues netCfg.lans); }))
|
||||
// (lib.concatMapAttrs (mkMacvlanService netCfg.name) netCfg.bridges.macvlans);
|
||||
|
||||
networkNames = builtins.attrNames vmixCfg.networks;
|
||||
|
|
@ -164,7 +194,7 @@ let
|
|||
networkTargets = lib.concatMapAttrs (networkName: netCfg: {
|
||||
"net.vmix@${networkName}" = {
|
||||
description = "Network ${networkName} for vmix";
|
||||
bindsTo = [ "ns.net.vmix@${networkName}.service" ];
|
||||
bindsTo = [ "ns.net.vmix@${networkName}.service" "lans.net.vmix@${networkName}.service" "wan.net.vmix@${networkName}.service" ];
|
||||
};
|
||||
}) vmixCfg.networks;
|
||||
in
|
||||
|
|
|
|||
|
|
@ -97,6 +97,12 @@ with vmixLib.network;
|
|||
|
||||
lans = mkOption {
|
||||
type = types.attrsOf (types.submodule {
|
||||
options.domain = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Domain name for the hosts of this lan.";
|
||||
};
|
||||
|
||||
options.ipv4 = {
|
||||
range = mkOption {
|
||||
type = types.strMatching regex.cidr4;
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ let
|
|||
create = ''
|
||||
ip link add link ${macvtapNetworkCfg.uplink.iface} name ${macvtapInterfaceName} type macvtap mode bridge
|
||||
ip link set ${macvtapInterfaceName} netns ${netName}.vmix
|
||||
ip netns exec ${netName}.vmix ip link set dev ${macvtapInterfaceName} up
|
||||
'';
|
||||
delete = ''
|
||||
ip netns exec ${netName}.vmix ip link del ${macvtapInterfaceName}
|
||||
|
|
@ -66,7 +67,13 @@ let
|
|||
concatStringsSep "\n" (builtins.map (macvtap: macvtap.delete) allMacvtaps)
|
||||
);
|
||||
|
||||
osImage = vmixLib.customizeImage vmCfg.disks.os.file { name = vmCfg.name; };
|
||||
osImage = vmixLib.customizeImage vmCfg.disks.os.file {
|
||||
name = vmCfg.name;
|
||||
commands = ''
|
||||
truncate /etc/machine-id
|
||||
run-command systemd-machine-id-setup
|
||||
'';
|
||||
};
|
||||
|
||||
qemuStartVMScript = pkgs.writeShellScript "${vmCfg.name}-qemu-vmix" ''
|
||||
exec qemu-system-${vmCfg.arch} \
|
||||
|
|
@ -96,20 +103,19 @@ let
|
|||
-device virtio-net-pci,netdev=user \
|
||||
"} \
|
||||
${optionalString (vmCfg.boot.menu == true) "-boot menu=on"} \
|
||||
#${optionalString (length vmCfg.boot.order > 0) "-boot order=${concatStringsSep "," vmCfg.boot.order}"}
|
||||
|
||||
# ${concatMapStrings (macvtap: ''
|
||||
# -device virtio-net-pci,netdev=macvtap-${macvtap.name} \
|
||||
# -netdev tap,id=macvtap-${macvtap.name},ifname=${macvtap.iface},script=no,downscript=no \
|
||||
# '') allMacvtaps} \
|
||||
${concatStrings (imap1 (i: macvtap: ''
|
||||
-device virtio-net-pci,netdev=macvtap-${macvtap.name},mac=$(ip l show ${macvtap.iface} | awk '/link\/ether/{print $2}') \
|
||||
-netdev tap,id=macvtap-${macvtap.name},fd=${toString (i+2)} ${toString (i+2)}<>/dev/tap$(ip l show ${macvtap.iface} | awk -F':' '/${macvtap.iface}/{print $1}') \
|
||||
'') allMacvtaps)} \
|
||||
#${optionalString (length vmCfg.boot.order > 0) "-boot order=${concatStringsSep "," vmCfg.boot.order}"} \
|
||||
'';
|
||||
in
|
||||
{
|
||||
"vm.vmix@${vmCfg.name}" = rec {
|
||||
requires = [ "net.vmix@${netCfg.name}.target" "macvtaps.vm.vmix@${vmCfg.name}.service" ];
|
||||
bindsTo = [ "net.vmix@${netCfg.name}.target" "macvtaps.vm.vmix@${vmCfg.name}.service" ];
|
||||
unitConfig.JoinsNamespaceOf = "ns.net.vmix@${netCfg.name}.service";
|
||||
after = requires;
|
||||
path = with pkgs; [ iproute2 qemu ];
|
||||
after = bindsTo;
|
||||
path = with pkgs; [ iproute2 qemu gawk ];
|
||||
serviceConfig = {
|
||||
ExecStartPre = createTapsforLansScript;
|
||||
ExecStart = qemuStartVMScript;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue