Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ncm-spma: apt: Finish implementation #1258

Merged
merged 11 commits into from
Jul 28, 2023
Merged

ncm-spma: apt: Finish implementation #1258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
925f4dc
ncm-spma: apt: Use latest release of CAF
jrha Jul 24, 2023
8a44cfc
ncm-spma: apt: Handle exit status of commands
jrha Jan 24, 2018
3c49bc4
ncm-spma: apt: Output error if rendering apt config fails
jrha Jan 24, 2018
422211d
ncm-spma: apt: Prefix debugging with subroutine name
jrha Jul 26, 2023
931adb5
ncm-spma: apt: Evaluate details as hashref (bugfix)
jrha Jan 24, 2018
e330301
ncm-spma: apt: Log profile trees in debug 5
jrha Jan 24, 2018
6b59160
ncm-spma: apt: Apply version and architectures in a seperate step
jrha Jan 24, 2018
9b3c9ea
ncm-spma: apt: invert gpgcheck logic to define forced trust
stdweird Apr 27, 2018
d13b23a
ncm-spma: apt: Return undef on errors
jrha Jul 25, 2023
0701eb9
ncm-spma: apt: Log main steps in Configure
jrha Jul 25, 2023
49ef1d3
ncm-spma: apt: Warn if any packages appear to be unavailable
jrha Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 102 additions & 47 deletions ncm-spma/src/main/perl/spma/apt.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
#${PMpre} NCM::Component::spma::apt${PMpost}

use Data::Dumper;
$Data::Dumper::Indent = 0; # Supress indentation and new-lines
$Data::Dumper::Terse = 1; # Output values only, supress variable names if possible

=head1 NAME

C<NCM::Component::spma::apt> - NCM SPMA backend for apt
Expand Down Expand Up @@ -46,7 +50,7 @@ Packages listed under C</software/packages> will be installed, version and archi
=cut

use parent qw(NCM::Component CAF::Path);
use CAF::Path 17.3.1;
use CAF::Path 21.12.1;
use CAF::Process;
use CAF::FileWriter;
use CAF::FileEditor;
Expand All @@ -67,23 +71,42 @@ Readonly my $TREE_SOURCES => "/software/repositories";
Readonly my $TREE_PKGS => "/software/packages";
Readonly my $BIN_APT_GET => "/usr/bin/apt-get";
Readonly my $BIN_APT_MARK => "/usr/bin/apt-mark";
Readonly my $BIN_APT_CACHE => "/usr/bin/apt-cache";
Readonly my $BIN_DPKG_QUERY => "/usr/bin/dpkg-query";
Readonly my $CMD_APT_UPDATE => [$BIN_APT_GET, qw(-qq update)];
Readonly my $CMD_APT_UPGRADE => [$BIN_APT_GET, qw(-qq dist-upgrade)];
Readonly my $CMD_APT_INSTALL => [$BIN_APT_GET, qw(-qq install)];
Readonly my $CMD_APT_AUTOREMOVE => [$BIN_APT_GET, qw(-qq autoremove)];
Readonly my $CMD_APT_MARK => [$BIN_APT_MARK, qw(-qq)];
Readonly my $CMD_APT_AVAILABLE => [$BIN_APT_CACHE, qw(pkgnames)];
Readonly my $CMD_DPKG_QUERY => [$BIN_DPKG_QUERY, qw(-W -f=${db:Status-Abbrev};${Package}\n)];

our $NoActionSupported = 1;

# Wrapper function for calling apt commands
sub _call_apt
{
my ($self, $cmd, $ok) = @_;
$self->debug(5, '_call_apt: Called with args ', Dumper($cmd));

my $proc = CAF::Process->new($cmd);
my $output = $proc->output();
my $exitstatus = $? >> 8; # Get exit status from highest 8-bits
$self->debug(5, "_call_apt: $proc exited with $exitstatus");
if ($exitstatus > 0) {
$output =~ tr{\n}{ };
my $method = $ok ? 'warn' : 'error';
$self->$method("_call_apt: $proc failed with \"$output\"");
}
return $ok || $exitstatus == 0;
}

# If user specified sources (userrepos) are not allowed, removes any
# sources present in the system that are not listed in $allowed_sources.
sub cleanup_old_sources
{
my ($self, $sources_dir, $allowed_sources) = @_;
$self->debug(5, 'Entered cleanup_old_sources()');
$self->debug(5, "cleanup_old_sources: Called with args ", $sources_dir, $allowed_sources);

if ($self->directory_exists($sources_dir)) {
my $current = Set::Scalar->new(@{$self->listdir($sources_dir, filter => qr{\.list$}, adddir => 1)});
Expand All @@ -108,7 +131,7 @@ sub cleanup_old_sources
sub initialize_sources_dir
{
my ($self, $sources_dir) = @_;
$self->debug(5, 'Entered initialize_sources_dir()');
$self->debug(5, "initialize_sources_dir: Called with args($sources_dir)");

if (! $self->directory($sources_dir)) {
$self->error("Unable to create source dir $sources_dir: $self->{fail}");
Expand All @@ -124,7 +147,7 @@ sub initialize_sources_dir
sub generate_sources
{
my ($self, $sources_dir, $sources, $template) = @_;
$self->debug(5, 'Entered generate_sources()');
$self->debug(5, "generate_sources: Called with args($sources_dir, $sources, $template)");

my $changes = 0;

Expand All @@ -137,7 +160,7 @@ sub generate_sources
$changes += $fh->close() || 0; # handle undef
} else {
$self->error("Invalid template '$template' passed to generate_sources");
return 0;
return;
}
}

Expand All @@ -148,26 +171,27 @@ sub generate_sources
sub configure_apt
{
my ($self, $config) = @_;
$self->debug(5, 'Entered configure_apt()');
$self->debug(5, 'configure_apt: Called with args', Dumper($config));

my $tr = EDG::WP4::CCM::TextRender->new($TEMPLATE_CONFIG, $config, relpath => 'spma');
if ($tr) {
my $fh = $tr->filewriter($FILE_CONFIG);
return $fh->close() || 0; # handle undef
} else {
return 0;
}
$self->error('configure_apt: TextRender failed to render configuration');
return;
}

# Returns a set of all installed packages
sub get_installed_pkgs
{
my $self = shift;
$self->debug(5, 'Entered get_installed_pkgs()');
my ($self) = @_;
$self->debug(5, 'get_installed_pkgs: Called');

my $out = CAF::Process->new($CMD_DPKG_QUERY, keeps_state => 1) ->output();
if ($?) {
$self->debug(5, "dpkg command returned $?");
my $exitstatus = $? >> 8; # Get exit status from highest 8-bits
if ($exitstatus) {
$self->debug(5, "dpkg command returned $exitstatus");
return 0;
}
# db:Status-Abbrev is three characters, we are looking for
Expand All @@ -179,34 +203,53 @@ sub get_installed_pkgs
return Set::Scalar->new(@pkgs);
}

# Returns a set of all available package names
sub get_available_pkgs
{
my ($self) = @_;
$self->debug(5, 'get_available_pkgs: Called');

my $out = CAF::Process->new($CMD_APT_AVAILABLE, keeps_state => 1) ->output();
my $exitstatus = $? >> 8; # Get exit status from highest 8-bits
if ($exitstatus) {
$self->debug(5, "dpkg command returned $exitstatus");
return 0;
}
my @pkgs = split("\n", $out);

return Set::Scalar->new(@pkgs);
}

# For a given package name, extract version and architecture from tree passed in details
# returns an arrayref of packages formatted with name, version and architecture for use with apt
sub get_package_version_arch
{
my ($self, $name, $details) = @_;
$self->debug(5, 'Entered get_package_version_arch()');
$self->debug(5, "get_package_version_arch: Called with args($name, ", Dumper($details), ")");

my @versions;

if ($details) {
if (defined($details) and %$details) {
foreach my $version (sort keys %$details) {
my $params = $details->{$version};
$version = unescape($version);
if ($params->{arch}) {
foreach my $arch (sort keys %{ $params->{arch} }) {
$self->debug(5, ' Adding package ', $name, ' with version ', $version, ' and architecture ', $arch, ' to list');
$self->debug(4, 'get_package_version_arch: Adding package ', $name, ' with version ', $version, ' and architecture ', $arch, ' to list');
push(@versions, sprintf('%s:%s=%s', $name, $arch, $version));
}
} else {
$self->debug(5, ' Adding package ', $name, ' with version ', $version, ' but without architecture to list');
$self->debug(4, 'get_package_version_arch: Adding package ', $name, ' with version ', $version, ' but without architecture to list');
push(@versions, sprintf('%s=%s', $name, $version));
}
}
} else {
$self->debug(5, ' Adding package ', $name, ' without version or architecture to list');
$self->debug(4, 'get_package_version_arch: Adding package ', $name, ' without version or architecture to list');
push(@versions, $name);
}

$self->debug(5, 'get_package_version_arch: returning arrayref:', Dumper(\@versions));

return \@versions;
}

Expand All @@ -216,7 +259,7 @@ sub apply_package_version_arch
{
my ($self, $packagelist, $packagetree) = @_;

$self->debug(5, 'Entered apply_package_version_arch()');
$self->debug(5, "apply_package_version_arch: Called with args", $packagelist, Dumper($packagetree));

my @results;
my @notfound;
Expand All @@ -240,7 +283,7 @@ sub apply_package_version_arch
sub get_desired_pkgs
{
my ($self, $pkgs) = @_;
$self->debug(5, 'Entered get_desired_pkgs()');
$self->debug(5, "get_desired_pkgs: Called with args", Dumper($pkgs));

my $packages = Set::Scalar->new();

Expand All @@ -259,33 +302,32 @@ sub get_desired_pkgs
# Update package metadata from upstream sourcesitories
sub resynchronize_package_index
{
my $self = shift;
$self->debug(5, 'Entered resynchronize_package_index()');
my ($self) = @_;
$self->debug(5, 'resynchronize_package_index: Called');

my $cmd = CAF::Process->new($CMD_APT_UPDATE, keeps_state => 1);
return $cmd->execute() ? 1 : undef;
return $self->_call_apt($CMD_APT_UPDATE);
}


# Upgrade existing packages
sub upgrade_packages
{
my ($self) = @_;
$self->debug(5, 'Entered upgrade_packages()');
$self->debug(5, 'upgrade_packages: Called');

my $cmd = CAF::Process->new($CMD_APT_UPGRADE) ;
return $cmd->execute() ? 1 : undef;
# it's ok if this produces errors (eg unfinished stuff)
# TODO: add support for 'apt --fix-broken install' and things like that
return $self->_call_apt($CMD_APT_UPGRADE, 1);
}


# Install packages
sub install_packages
{
my ($self, $packages) = @_;
$self->debug(5, 'Entered install_packages()');
$self->debug(5, 'install_packages: Called with args', Dumper($packages));

my $cmd = CAF::Process->new([@$CMD_APT_INSTALL, @$packages]) ;
return $cmd->execute() ? 1 : undef;
return $self->_call_apt([@$CMD_APT_INSTALL, @$packages]);
}


Expand All @@ -294,70 +336,83 @@ sub install_packages
sub mark_packages_auto
{
my ($self, $packages) = @_;
$self->debug(5, 'Entered mark_packages_auto()');
$self->debug(5, "mark_packages_auto: Called with args", Dumper($packages));

my $cmd = CAF::Process->new([@$CMD_APT_MARK, 'auto', @$packages]) ;
return $cmd->execute() ? 1 : undef;
return $self->_call_apt([@$CMD_APT_MARK, 'auto', @$packages]);
}


# Remove automatically installed packages
sub autoremove_packages
{
my ($self) = @_;
$self->debug(5, 'Entered autoremove_packages()');
$self->debug(5, 'autoremove_packages: Called');

my $cmd = CAF::Process->new([@$CMD_APT_AUTOREMOVE]) ;
return $cmd->execute() ? 1 : undef;
return $self->_call_apt([@$CMD_APT_AUTOREMOVE]);
}


sub Configure
{
my ($self, $config) = @_;
$self->debug(5, 'Entered Configure()');

# Get configuration trees
my $tree_sources = $config->getTree($TREE_SOURCES);
$self->debug(5, 'TREE_SOURCES ', $TREE_SOURCES, Dumper $tree_sources);
my $tree_pkgs = $config->getTree($TREE_PKGS);
$self->debug(5, 'TREE_PKGS ', $TREE_PKGS, Dumper $tree_pkgs);
my $tree_component = $config->getTree($self->prefix());
$self->debug(5, 'tree_component ', $self->prefix, Dumper $tree_component);

$self->configure_apt($tree_component) or return 0;
defined($self->configure_apt($tree_component)) or return 0;

$self->initialize_sources_dir($DIR_SOURCES) or return 0;
defined($self->initialize_sources_dir($DIR_SOURCES)) or return 0;

# Remove unknown sources if allow_user_sources is not set
if (! $tree_component->{usersources}) {
$self->info('Removing unknown source lists');
$self->cleanup_old_sources($DIR_SOURCES, $tree_sources) or return 0;
};

$self->generate_sources(
$self->info('Generating ', scalar(@$tree_sources), ' source lists');
defined($self->generate_sources(
$DIR_SOURCES,
$tree_sources,
$TEMPLATE_SOURCES,
) or return 0;
)) or return 0;

$self->info('Synchronizing package index');
$self->resynchronize_package_index() or return 0;

$self->info('Applying upgrades to installed packages');
$self->upgrade_packages() or return 0;

my $packages_installed = $self->get_installed_pkgs() or return 0;
my $packages_available = $self->get_available_pkgs() or return 0;
my $packages_desired = $self->get_desired_pkgs($tree_pkgs) or return 0;
my $packages_unwanted = $packages_installed->difference($packages_desired);

$self->debug(5, 'Installed packages:', $packages_installed);
$self->debug(5, 'Desired packages:', $packages_desired);
$self->debug(5, 'Packages installed but unwanted:', $packages_unwanted);
my $packages_unwanted = $packages_installed->difference($packages_desired);
my $packages_to_install = $packages_desired->difference($packages_installed);
my $packages_unavailable = $packages_desired->difference($packages_available);

my $packages_to_install = $self->apply_package_version_arch($packages_desired, $tree_pkgs) or return 0;
if ($packages_unavailable->size > 0) {
$self->warn('The following packages are unavailable, they may have been renamed or virtual: ', $packages_unavailable);
}

$self->debug(5, 'Packages to install ', $packages_to_install);
$self->debug(4, 'Installed packages: ', $packages_installed);
$self->debug(4, 'Desired packages: ', $packages_desired);
$self->debug(4, 'Unavailable packages: ', $packages_unavailable);
$self->debug(4, 'Packages installed but unwanted: ', $packages_unwanted);
$self->debug(4, 'Packages to install (desired but not installed): ', $packages_to_install);

$self->install_packages($packages_to_install) or return 0;
my $apt_packages_to_install = $self->apply_package_version_arch($packages_to_install, $tree_pkgs);
$self->info('Installing ', $packages_to_install->size,' missing packages');
$self->install_packages($apt_packages_to_install) or return 0;

# If user installed packages are not permitted, mark all unlisted packages as automatically installed and
# ask apt to remove any of these that are not required to satisfy dependencies of the desired package list
if (! $tree_component->{userpkgs}) {
$self->info('Marking ', $packages_unwanted->size, ' packages as unwanted and removing any that are not dependencies of installed packages');
$self->mark_packages_auto($packages_unwanted) or return 0;
$self->autoremove_packages() or return 0;
}
Expand Down
2 changes: 1 addition & 1 deletion ncm-spma/src/main/resources/apt/source.tt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
# Name: [% name %]
# Owner: [% owner %]

[%- trust = gpgcheck ? "trusted=yes " : "" -%]
[%- trust = gpgcheck ? "" : "[trusted=yes] " -%]
[%- FOREACH source IN protocols %]
# Protocol type: [% source.name %]
deb [% trust %][% source.url %]
Expand Down
Loading