diff --git a/ncm-network/pom.xml b/ncm-network/pom.xml index 920558ec7c..e392d15986 100644 --- a/ncm-network/pom.xml +++ b/ncm-network/pom.xml @@ -9,7 +9,7 @@ org.quattor.maven build-profile - 1.59 + 1.62 @@ -37,33 +37,7 @@ - - maven-resources-plugin - 2.4.3 - - - filter-tt-sources - process-sources - - copy-resources - - - UTF-8 - ${project.build.directory}/share/templates/quattor/${project.artifactId} - - - src/main/resources - true - - **/*.tt - - - - - - - - + org.codehaus.mojo rpm-maven-plugin @@ -71,20 +45,6 @@ bridge-utils net-tools - - - /usr/share/templates/ - 644 - root - root - - - ${project.build.directory}/share/templates/ - - - false - - diff --git a/ncm-network/src/main/perl/nmstate.pm b/ncm-network/src/main/perl/nmstate.pm index 50b608503d..77dff5ed0f 100644 --- a/ncm-network/src/main/perl/nmstate.pm +++ b/ncm-network/src/main/perl/nmstate.pm @@ -31,9 +31,12 @@ use Readonly; Readonly my $NMSTATECTL => '/usr/bin/nmstatectl'; Readonly my $NMCLI_CMD => '/usr/bin/nmcli'; +# pick a config name for nmstate yml to configure dns-resolver: settings. if nm_manage_dns=true +Readonly my $NM_RESOLV_YML => "/etc/nmstate/resolv.yml"; use constant IFCFG_DIR => "/etc/nmstate"; + sub iface_filename { my ($self, $iface) = @_; @@ -102,30 +105,27 @@ sub disable_nm_manage_dns } } -# return hasref of policy rule. interface.tt module uses this to create the rule in nmstate file. +# return hasref of policy rule. sub make_nm_ip_rule { my ($self, $device, $rules, $routing_table_hash) = @_; - my @text; - my $idx = 0; + my @rule_entry; foreach my $rule (@$rules) { + my %thisrule; my $priority = 100; $priority = $rule->{priority} if $rule->{priority}; - if (!$rule->{table_id}) { - $rule->{table_id} = "$routing_table_hash->{$rule->{table}}" if $rule->{table}; - push(@text, $rule->{table_id}); - } - if (!$rule->{priority}){ - $rule->{priority} = $priority; - push(@text, $rule->{priority}); - } + $thisrule{priority} = $priority; + $thisrule{'route-table'} = "$routing_table_hash->{$rule->{table}}" if $rule->{table}; + $thisrule{'ip-to'} = $rule->{to} if $rule->{to}; + $thisrule{'ip-from'} = $rule->{from} if $rule->{from}; + push (@rule_entry, \%thisrule); } - return \@text; + return \@rule_entry; } # construct all routes found into array of hashref -# return array of hashref, used by interface.tt module. +# return array of hashref sub make_nm_ip_route { my ($self, $device, $routes, $routing_table_hash) = @_; @@ -143,9 +143,9 @@ sub make_nm_ip_route $rt{destination} = $route->{address}."/32"; } } - $rt{table_id} = "$routing_table_hash->{$route->{table}}" if $route->{table}; - $rt{next_hop_interface} = $device; - $rt{next_hop_address} = $route->{gateway} if $route->{gateway}; + $rt{'table-id'} = "$routing_table_hash->{$route->{table}}" if $route->{table}; + $rt{'next-hop-interface'} = $device; + $rt{'next-hop-address'} = $route->{gateway} if $route->{gateway}; push (@rt_entry, \%rt); } @@ -167,10 +167,13 @@ sub get_bonded_eth return \@data; } -# wirtes the nmstate yml file, uses nmstate/interface.tt module. +# writes the nmstate yml file, using yaml module. sub nmstate_file_dump { my ($self, $filename, $ifaceconfig) = @_; + # ATM interfaces hash will only have one entry per interface. so looking at first entry is fine. long as file isn't resolv.yml + my $iface = $ifaceconfig->{'interfaces'}[0] if ($filename ne $NM_RESOLV_YML); + my $changes = 0; my $func = "nmstate_file_dump"; @@ -179,49 +182,61 @@ sub nmstate_file_dump $self->warn("Failed to cleanup testcfg $testcfg before file_dump: $self->{fail}"); } - if (!$self->file_exists($filename) || $self->mk_bu($filename, $testcfg)) { - - my $trd = EDG::WP4::CCM::TextRender->new('yaml', $ifaceconfig, relpath => 'network'); - if (! defined($trd->get_text())) { - $self->error ("Unable to generate network config $filename: $trd->{fail}"); - return; - }; - my $fh = $trd->filewriter($testcfg, - header => "# File generated by " . __PACKAGE__ . ". Do not edit", - log => $self); - my $filestatus; - if ($fh->close()) { - if ($self->file_exists($filename)) { - $self->info("$func: file $filename has newer version scheduled."); - $filestatus = $UPDATED; - } else { - $self->info("$func: new file $filename scheduled."); - $filestatus = $NEW; - } - } else { - my $is_active = is_active_interface($self, $ifaceconfig->{name}); - if (( $is_active != 1 ) && ($ifaceconfig->{enabled}) eq "true"){ - # if we find no active connection for interface we are managing, lets attempt to start it. - # mark the enterface schedule to be updated. - # this will allow nm to report issues with config on every run or if someone deletes the conneciton. - # if no changes to file, then this will never get applied again. - $self->info("$func: file $filename has no active conneciton, scheduled for update."); - $filestatus = $UPDATED; + if (!$self->file_exists($filename) || $self->mk_bu($filename, $testcfg)) + { + my $trd = EDG::WP4::CCM::TextRender->new('yaml', $ifaceconfig, relpath => 'network'); + if (! defined($trd->get_text())) + { + $self->error ("Unable to generate network config $filename: $trd->{fail}"); + return; + }; + my $fh = $trd->filewriter($testcfg, + header => "# File generated by " . __PACKAGE__ . ". Do not edit", + log => $self); + my $filestatus; + if ($fh->close()) { + if ($self->file_exists($filename)) { + $self->info("$func: file $filename has newer version scheduled."); + $filestatus = $UPDATED; + } else { + $self->info("$func: new file $filename scheduled."); + $filestatus = $NEW; + } } else { - $filestatus = $NOCHANGES; - # they're equal, remove backup files - $self->verbose("$func: no changes scheduled for file $filename. Cleaning up."); - $self->cleanup_backup_test($filename); - } - }; + if ($filename ne $NM_RESOLV_YML) + { + # if it's interface file, lets check if there is a active connection. + my $is_active = is_active_interface($self, $iface->{name}); + if (( $is_active != 1 ) && ($iface->{state}) eq "up") { + # if we find no active connection for interface we are managing, lets attempt to start it. + # mark the enterface schedule to be updated. + # this will allow nm to report issues with config on every run instead of just first run when change is made. + # or if someone deletes the conneciton. + # if no changes to file, then this will never get applied again. + $self->info("$func: file $filename has no active connection, scheduled for update."); + $filestatus = $UPDATED; + } else { + $filestatus = $NOCHANGES; + # they're equal, remove backup files + $self->verbose("$func: no changes scheduled for file $filename. Cleaning up."); + $self->cleanup_backup_test($filename); + } + } else { + $filestatus = $NOCHANGES; + # they're equal, remove backup files + $self->verbose("$func: no changes scheduled for file $filename. Cleaning up."); + $self->cleanup_backup_test($filename); + } + }; return $filestatus; } else { return; } } -# generates the hasrefs for interface used by nmstate/interface.tt module. +# generates the hasrefs for interface in yaml file format needed by nmstate. # bulk of the config settings needed by the nmstate yml is done here. +# to add additional options, it should be constructed here. sub generate_nmstate_config { my ($self, $name, $net, $ipv6, $routing_table) = @_; @@ -238,27 +253,27 @@ sub generate_nmstate_config # create hash of interface entries that will be used by nmstate config. my $ifaceconfig->{name} = $name; - $ifaceconfig->{device} = $device; if ($is_eth) { $ifaceconfig->{type} = "ethernet"; if ($is_bond_eth) { # no ipv4 address for bonded eth, plus in nmstate bonded eth is controlled by controller. no config is required. - $ifaceconfig->{enabled} = "false"; + $ifaceconfig->{ipv4}->{enabled} = "false"; $ifaceconfig->{state} = "up"; } } elsif ($is_vlan_eth) { my $vlan_id = $name; - # replace eveytthing upto and include . to get vlan id of the interface. + # replace eveything upto and including . to get vlan id of the interface. + # TODO: instead of this, should perhaps add valid-id in schema? but may not be backward compatible for existing host entreis, aqdb will need updating? $vlan_id =~ s/^[^.]*.//;; $ifaceconfig->{type} = "vlan"; - $ifaceconfig->{vlan}->{base_iface} = $iface->{physdev}; - $ifaceconfig->{vlan}->{vlan_id} = $vlan_id; + $ifaceconfig->{vlan}->{'base-iface'} = $iface->{physdev}; + $ifaceconfig->{vlan}->{'id'} = $vlan_id; } else { # if bond device $ifaceconfig->{type} = "bond"; - $ifaceconfig->{link_aggregation} = $iface->{link_aggregation}; + $ifaceconfig->{'link-aggregation'} = $iface->{link_aggregation}; if ($bonded_eth){ - $ifaceconfig->{link_aggregation}->{port} = $bonded_eth; + $ifaceconfig->{'link-aggregation'}->{port} = $bonded_eth; } } @@ -274,33 +289,35 @@ sub generate_nmstate_config } my $ip_list=(); $ip_list->{ip} = $ip->addr; - $ip_list->{prefix} = $ip->masklen; + $ip_list->{'prefix-length'} = $ip->masklen; # TODO: append alias ip to ip_list as array, providing ips as array of hashref. $ifaceconfig->{ipv4}->{address} = [$ip_list]; - $ifaceconfig->{enabled} = "true"; + $ifaceconfig->{ipv4}->{enabled} = "true"; } else { # TODO: configure IPV6 enteries if ($iface->{ipv6addr}) { $self->warn("ipv6 addr found but not supported"); + $ifaceconfig->{ipv6}->{enabled} = "false"; # TODO create ipv6.address entries here. i.e #$ifaceconfig->{ipv6}->{address} = [$ipv6_list]; - #.tt module support is added. } else { $self->verbose("no ipv6 entries"); } } } elsif (($eth_bootproto eq "none") && (!$is_bond_eth)) { # no ip on interface and is not a bond eth, assume not managed so disable eth. - $ifaceconfig->{enabled} = "false"; + $ifaceconfig->{ipv4}->{enabled} = "false"; + $ifaceconfig->{ipv6}->{enabled} = "false"; $ifaceconfig->{state} = "down"; } # create default route entry. my %default_rt; if (defined($iface->{gateway})){ $default_rt{destination} = '0.0.0.0/0'; - $default_rt{next_hop_address} = $iface->{gateway}; - $default_rt{next_hop_interface} = $device; + $default_rt{'next-hop-address'} = $iface->{gateway}; + $default_rt{'next-hop-interface'} = $device; } + # combined default route with any policy routing/rule, if any # combination of default route, plus any additional policy routes. # read and set by tt module as @@ -320,27 +337,52 @@ sub generate_nmstate_config push @$routes, \%default_rt if scalar %default_rt; } + my $policy_rule; if (defined($iface->{rule})) { my $rule = $iface->{rule}; - $self->make_nm_ip_rule($iface, $rule, $routing_table); + $policy_rule = $self->make_nm_ip_rule($iface, $rule, $routing_table); $self->verbose("policy rule found, nmstate will manage it"); } + # return hash construct that will match what nmstate yml needs. + my $interface->{interfaces} = [$ifaceconfig]; if (scalar $routes){ - $ifaceconfig->{routes}->{config} = $routes; + $interface->{routes}->{config} = $routes; + } + if (scalar $policy_rule){ + $interface->{'route-rules'}->{config} = $policy_rule; } - #print (YAML::XS::Dump($ifaceconfig)); + #print (YAML::XS::Dump($interface)); # TODO: ethtool settings to add in config file? setting via cmd cli working as is. - # TODO: aliases ip addresses + # TODO: add aliases ip addresses # TODO: bridge_options # TODO: veth, anymore? - return $ifaceconfig; + return $interface; }; +# Genareate hash of dns-resolver config for nmstate. +# only used if nm_manage_dns = true. +sub generate_nm_resolver_config +{ + my ($self, $net, $manage) = @_; + # resolver content will be empty if mange_dns is false + my $nm_dns_config->{'dns-resolver'}->{config}->{search} = []; + $nm_dns_config->{'dns-resolver'}->{config}->{server} = []; + if ($manage) + { + # TODO: adding nameservers and domainname from network path, maybe we need to consider similar approach to ncm-resolver? + my $searchpath; + push @$searchpath, $net->{domainname}; + my $dnsservers = $net->{nameserver}; + $nm_dns_config->{'dns-resolver'}->{config}->{search} = $searchpath; + $nm_dns_config->{'dns-resolver'}->{config}->{server} = $dnsservers; + } + return $nm_dns_config +} # enable NetworkManager service -# without enabled network service, this component is pointless +# without enabled NetworkManager, this component is pointless # sub enable_network_service { @@ -350,7 +392,8 @@ sub enable_network_service } # keep nmstate service disbaled (vendor preset anyway), we will apply config ncm component. -# nmstate service applies all files found in /etc/nmstate and changes to .applied, which will keep change if component is managing the .yml file. +# nmstate service applies all files found in /etc/nmstate and changes to .applied, which will keep changing if component is managing the .yml file. +# we don't need this. # sub disable_nmstate_service { @@ -360,17 +403,25 @@ sub disable_nmstate_service } # check to see if we have active connection for interface we manage. -# this allow ability to start a connection again that may have failed with nmstate apply +# this allow ability to start a connection again if last config run failed to nmstate apply. sub is_active_interface { my ($self, $ifacename) = @_; - my $output = $self->runrun([$NMCLI_CMD, "-f", "name", "conn", "show", "--active"]); + my $output = $self->runrun([$NMCLI_CMD, "-t", "-f", "name,device", "conn", "show", "--active"]); + # outoput returned by nmclie -t is colon seperated + # i.e eth0:eth0 my @existing_conn = split('\n', $output); - my %current_conn; my $found = 0; foreach my $conn_name (@existing_conn) { - $conn_name =~ s/\s+$//; - if ($conn_name eq $ifacename){ + my ($name, $dev) = split(':', $conn_name); + # trim + if ("$dev" eq "$ifacename") { + # ncm-network will set connection same as interface name, if this doesn't match, + # it means this connection existed before nmstate did its first apply. + # doesn't break anything as nmstate resuses the conn, but worth a warning to highlight it? + if ("$name" ne "$ifacename"){ + $self->warn("connection name '$name' doesn't match $ifacename for device $dev, possible connection reuse occured") + } $found = 1; return $found ; }; @@ -379,12 +430,13 @@ sub is_active_interface } # check for existing connections, will clear the default connections created by 'NM with Wired connecton x' +# good to have. sub clear_default_nm_connections { my ($self) = @_; # NM creates auto connections with Wired connection x # Delete all connections with name 'Wired connection', everything ncm-network creates will have connection name set to interface name. - my $output = $self->runrun([$NMCLI_CMD, "-f", "name", "conn"]); + my $output = $self->runrun([$NMCLI_CMD, "-t", "-f", "name", "conn"]); my @existing_conn = split('\n', $output); my %current_conn; foreach my $conn_name (@existing_conn) { @@ -399,18 +451,12 @@ sub clear_default_nm_connections sub nmstate_apply { - my ($self, $exifiles, $ifup, $nwsrv) = @_; + my ($self, $exifiles, $ifup, $ifdown, $nwsrv) = @_; my @ifaces = sort keys %$ifup; - my $nwstate = $exifiles->{$NETWORKCFG}; - + my @ifaces_down = sort keys %$ifdown; my $action; - $self->verbose("Apply config using nmstatectl for each interface"); - if (($nwstate == $UPDATED) || ($nwstate == $NEW)) { - # Do not need to start networking in nmstate. - #$self->verbose($NETWORKCFG, ($nwstate == $NEW ? 'NEW' : 'UPDATED'), " starting network"); - $action = 1; - } + if (@ifaces) { $self->info("Applying changes using $NMSTATECTL ", join(', ', @ifaces)); my @cmds; @@ -423,8 +469,8 @@ sub nmstate_apply push(@cmds, [$NMSTATECTL, "apply", $ymlfile]); push(@cmds, [qw(sleep 10)]) if ($iface =~ m/bond/); } else { - # do we down the interface? - $self->verbose("$ymlfile does not exist, not applying"); + # TODO: perhaps try down the interface? it's done later anyway + $self->verbose("$ymlfile does not exist for $iface, not applying"); } } $action = 1; @@ -434,17 +480,55 @@ sub nmstate_apply $self->verbose('Nothing to apply'); $action = 0; } - + # apply resolver config if exists. + # this will exist at this stage if nm_manage_dns is set to true. + my $resolv_state = $exifiles->{$NM_RESOLV_YML} || 0; + if ($self->file_exists($NM_RESOLV_YML)) + { + my $nwstate = $exifiles->{$NM_RESOLV_YML}; + my @cmds; + if (($nwstate == $UPDATED) || ($nwstate == $NEW)) { + $self->verbose($NM_RESOLV_YML, ($nwstate == $NEW ? 'NEW' : 'UPDATED'), " Apply config"); + push(@cmds, [$NMSTATECTL, "apply", $NM_RESOLV_YML]); + my $out = $self->runrun(@cmds); + $self->verbose($out); + $nwsrv->reload(); + $action = 1; + } + } + # check if we need to stop any interface whose config has been removed. + if (@ifaces_down) { + my @cmds; + foreach my $iface (@ifaces_down) + { + # nmcli down: all devices that are in ifdown + # and have state of REMOVE + my $cfg_filename = $self->iface_filename($iface); + if (exists($exifiles->{"$cfg_filename"}) && + $exifiles->{"$cfg_filename"} == $REMOVE) + { + $self->verbose("REMOVE connection for interface $iface"); + push(@cmds, [$NMCLI_CMD, "connection", "delete", $iface]); + } + } + $action = 1; + my $out = $self->runrun(@cmds); + $self->verbose($out); + } return $action; } - sub get_current_config_post { my ($self) = @_; - # output of nmstate - return $self->runrun([$NMSTATECTL, "show"]); + # Full output of nmstate + my $output = $self->runrun([$NMSTATECTL, "show"]); + + # few useful outputs from nmcli + $output .= $self->runrun([$NMCLI_CMD, "dev", "status"]); + $output .= $self->runrun([$NMCLI_CMD, "connection"]); + return $output; } @@ -474,49 +558,17 @@ sub Configure my $comp_tree = $config->getTree($self->prefix()); my $nwtree = $config->getTree($NETWORK_PATH); - # no backup, restart or anything else required - $self->routing_table($nwtree->{routing_table}); - - # main network config - # TODO: aka7, what is the role of /etc/systconfig/network in networkmanager/nmstate managed OS? - return if ! defined($self->mk_bu($NETWORKCFG)); - my $hostname = $nwtree->{realhostname} || "$nwtree->{hostname}.$nwtree->{domainname}"; + my $manage_dns = $nwtree->{nm_manage_dns} || 0; - # TODO: aka7, targeted OS is EL9, you can assume hostnamectl exists - my $use_hostnamectl = $self->_is_executable($HOSTNAME_CMD); - # if hostnamectl exists, do not set it via the network config file - # systemd rpm --script can remove it anyway - my $nwcfg_hostname = $use_hostnamectl ? undef : $hostname; - - my ($text, $ipv6) = $self->make_network_cfg($nwtree, $net, $nwcfg_hostname); - $exifiles->{$NETWORKCFG} = $self->file_dump($NETWORKCFG, $text); - - # TODO: aka7 this can be removed as well. this is some piece of legacy code you don't need - if ($exifiles->{$NETWORKCFG} == $UPDATED && $use_hostnamectl) { - # Network config was updated, check if it was due to removal of HOSTNAME - # when hostnamectl is present. - my ($hntext, $hnipv6) = $self->make_network_cfg($nwtree, $net, $hostname); - $self->legacy_keeps_state($NETWORKCFG, $hntext, $exifiles); - }; - + my $ipv6 = $nwtree->{ipv6}; foreach my $ifacename (sort keys %$ifaces) { my $iface = $ifaces->{$ifacename}; my $nmstate_cfg = generate_nmstate_config($self, $ifacename, $net, $ipv6, $nwtree->{routing_table}); my $file_name = $self->iface_filename($ifacename); $exifiles->{$file_name} = $self->nmstate_file_dump($file_name, $nmstate_cfg); - # TODO: not sure about what is going on here, keeping it out for now - #$self->default_broadcast_keeps_state($file_name, $ifacename, $iface, $exifiles, 0); $self->ethtool_opts_keeps_state($file_name, $ifacename, $iface, $exifiles); - - # TODO: aka7, this is legacy code, it can go away - if ($exifiles->{$file_name} == $UPDATED) { - # interface configuration was changed - # check if this was due to addition of resolv_mods / peerdns - my $no_resolv = $self->make_ifcfg($ifacename, $iface, $ipv6, resolv_mods => 0, peerdns => 0); - $self->legacy_keeps_state($file_name, $no_resolv, $exifiles); - } } my $dev2mac = $self->make_dev2mac(); @@ -538,9 +590,10 @@ sub Configure # $self->enable_network_service(); - # TODO: not tested with nmstate. leaving it here. + # TODO: not tested with nmstate. leaving it here. needs work.lol y $self->start_openvswitch($ifaces, $ifup); - + + # TODO: This can be set with nmstate config but we doing the triditional way using hostnamectl $self->set_hostname($hostname); # TODO: ethtool options are set using cli, but do we need to update in nmstate config? works for now @@ -561,8 +614,10 @@ sub Configure my $nwsrv = CAF::Service->new(['NetworkManager'], log => $self); # NetworkManager manages dns by default, but we manage dns with e.g. ncm-resolver, new option to enable/disable it. - $self->disable_nm_manage_dns($nwtree->{nm_manage_dns} || 0, $nwsrv); + $self->disable_nm_manage_dns($manage_dns, $nwsrv); + my $dnsconfig = $self->generate_nm_resolver_config($nwtree, $manage_dns); + $exifiles->{$NM_RESOLV_YML} = $self->nmstate_file_dump($NM_RESOLV_YML, $dnsconfig); # nmstate files are applied uinsg nmstate apply via this componant. We don't want nmstate svc to manage it. # If nmstate svc manages the files, it will apply the config for any files found in /etc/nmstate with .yml extension. Once the config is applied, # the file name changes to .applied, which won't be ideal if ncm-component is managing .yml files. @@ -576,20 +631,26 @@ sub Configure # (most likely in this scenario saved by ifup-post) # and leave a system without configured DNS (which ncm-network can't recover from, # as it does not manage /etc/resolv.conf). Without working DNS, the ccm-fetch network test will probably fail. - $self->move($RESOLV_CONF_SAVE, $RESOLV_CONF_SAVE.$RESOLV_SUFFIX); + # if nm is allowed to manage_dns, set dns-resolver: using nmstate. + if (!$manage_dns) { + $self->move($RESOLV_CONF_SAVE, $RESOLV_CONF_SAVE.$RESOLV_SUFFIX); + } # only need to deploy config. my $config_changed = $self->deploy_config($exifiles); # Save/Restore last known working (i.e. initial) /etc/resolv.conf - # TODO: @aka7, hmmm, if nm is allowed to manage dns, then this should be allowed to have changed - $resolv_conf_fh->close(); + # if nm is allowed to manage dns, then this should be allowed to have changed + # TODO: @stdweird still reverts back to orignal resolv.conf when manage_dns=true, why? + if (!$manage_dns) { + $resolv_conf_fh->close(); + } # Since there's per interface reload, interface changes will be applied via nmstatectl. # nmstatectl manages rollback too when options are misconfigured in yml config # This is still used to marke interfaces to apply any changes via nmstatectl - # TODO apply changes if exisitng connection is not active but we manage the file. - my $stopstart += $self->nmstate_apply($exifiles, $ifup, $nwsrv); + # This will also down/delete any interface conection for which config was removed. + my $stopstart += $self->nmstate_apply($exifiles, $ifup, $ifdown, $nwsrv); $init_config .= "\nPOST APPLY\n"; $init_config .= $self->get_current_config(); @@ -639,7 +700,6 @@ sub Configure } # remove all broken links: use file_exists - # TODO: why is there no try/recover for the symlinks? foreach my $link (sort keys %$exilinks) { if (! $self->file_exists($link)) { if ($self->cleanup($link)) {