From 2831d7b8f49b3a19ccc70fa6a8b5de1ac14dcedc Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Tue, 7 Nov 2017 15:38:21 +0100 Subject: [PATCH 1/4] Add DB migration for the new SLES15 product tree --- Makefile | 2 +- .../100-modify-ProductExtensions-schema.sql | 4 ++++ .../200-update-ProductExtensions-entries.sql | 1 + .../300-add-new-ProductExtensions-indices.sql | 2 ++ db/schemas/mysql/latest/100-smt-tables.sql | 14 ++++++++------ package/smt.spec | 2 +- www/perl-lib/SMT.pm | 2 +- 7 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 db/schemas/mysql/3.05-3.06/100-modify-ProductExtensions-schema.sql create mode 100644 db/schemas/mysql/3.05-3.06/200-update-ProductExtensions-entries.sql create mode 100644 db/schemas/mysql/3.05-3.06/300-add-new-ProductExtensions-indices.sql diff --git a/Makefile b/Makefile index 55b6c168..2d99047f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ NAME = smt -VERSION = 3.0.30 +VERSION = 3.0.31 DESTDIR = / PERL ?= perl PERLMODDIR = $(shell $(PERL) -MConfig -e 'print $$Config{installvendorlib};') diff --git a/db/schemas/mysql/3.05-3.06/100-modify-ProductExtensions-schema.sql b/db/schemas/mysql/3.05-3.06/100-modify-ProductExtensions-schema.sql new file mode 100644 index 00000000..2b4a5dfd --- /dev/null +++ b/db/schemas/mysql/3.05-3.06/100-modify-ProductExtensions-schema.sql @@ -0,0 +1,4 @@ +alter table ProductExtensions add column ROOTPRODUCTID int not null; +alter table ProductExtensions add column RECOMMENDED bool default false; +alter table ProductExtensions drop index ProductExtensions_pdid_extid_uq; +alter table ProductExtensions drop index ProductExtensions_pdid_extid_src_idx; diff --git a/db/schemas/mysql/3.05-3.06/200-update-ProductExtensions-entries.sql b/db/schemas/mysql/3.05-3.06/200-update-ProductExtensions-entries.sql new file mode 100644 index 00000000..0a2f98db --- /dev/null +++ b/db/schemas/mysql/3.05-3.06/200-update-ProductExtensions-entries.sql @@ -0,0 +1 @@ +update ProductExtensions set ROOTPRODUCTID = PRODUCTID; diff --git a/db/schemas/mysql/3.05-3.06/300-add-new-ProductExtensions-indices.sql b/db/schemas/mysql/3.05-3.06/300-add-new-ProductExtensions-indices.sql new file mode 100644 index 00000000..89401c7d --- /dev/null +++ b/db/schemas/mysql/3.05-3.06/300-add-new-ProductExtensions-indices.sql @@ -0,0 +1,2 @@ +alter table ProductExtensions ADD UNIQUE KEY `ProductExtensions_pdid_extid_rtpdid_uq` (`PRODUCTID`,`EXTENSIONID`, `ROOTPRODUCTID`); +alter table ProductExtensions ADD KEY `ProductExtensions_pdid_extid_rtpdid_src_idx` (`PRODUCTID`,`EXTENSIONID`,`ROOTPRODUCTID`,`SRC`); diff --git a/db/schemas/mysql/latest/100-smt-tables.sql b/db/schemas/mysql/latest/100-smt-tables.sql index 283b7b15..7e4fe009 100644 --- a/db/schemas/mysql/latest/100-smt-tables.sql +++ b/db/schemas/mysql/latest/100-smt-tables.sql @@ -171,12 +171,14 @@ create table Products ( ) ENGINE=InnoDB AUTO_INCREMENT = 100000; -create table ProductExtensions ( - PRODUCTID integer NOT NULL, - EXTENSIONID integer NOT NULL, - SRC CHAR(1) DEFAULT 'S', - UNIQUE INDEX ProductExtensions_pdid_extid_uq (PRODUCTID, EXTENSIONID), - INDEX ProductExtensions_pdid_extid_src_idx (PRODUCTID, EXTENSIONID, SRC) +CREATE TABLE `ProductExtensions` ( + `PRODUCTID` int(11) NOT NULL, + `EXTENSIONID` int(11) NOT NULL, + `SRC` char(1) DEFAULT 'S', + `ROOTPRODUCTID` int(11) NOT NULL, + `RECOMMENDED` tinyint(1) DEFAULT '0', + UNIQUE KEY `ProductExtensions_pdid_extid_rtpdid_uq` (`PRODUCTID`,`EXTENSIONID`,`ROOTPRODUCTID`), + KEY `ProductExtensions_pdid_extid_rtpdid_src_idx` (`PRODUCTID`,`EXTENSIONID`,`ROOTPRODUCTID`,`SRC`) ) ENGINE=InnoDB; create table ProductMigrations ( diff --git a/package/smt.spec b/package/smt.spec index 4bc0fcac..77e919bf 100644 --- a/package/smt.spec +++ b/package/smt.spec @@ -17,7 +17,7 @@ Name: smt -Version: 3.0.30 +Version: 3.0.31 Release: 0 Summary: Subscription Management Tool License: GPL-2.0+ diff --git a/www/perl-lib/SMT.pm b/www/perl-lib/SMT.pm index d01619c0..0b3ee126 100644 --- a/www/perl-lib/SMT.pm +++ b/www/perl-lib/SMT.pm @@ -5,6 +5,6 @@ use warnings; use vars qw($SCHEMA_VERSION); -$SCHEMA_VERSION = 3.05; +$SCHEMA_VERSION = 3.06; 1; From 01d5b303143da9a7cc4e30e0c602f727dc73fd09 Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Tue, 7 Nov 2017 15:53:36 +0100 Subject: [PATCH 2/4] Add syncing for the new product tree; allow to activate products without repositories --- www/perl-lib/SMT/Rest/SCCAPIv1.pm | 5 --- www/perl-lib/SMT/Rest/SCCAPIv2.pm | 5 --- www/perl-lib/SMT/Rest/SCCAPIv4.pm | 5 --- www/perl-lib/SMT/SCCSync.pm | 60 +++++++++++++++++++++++-------- 4 files changed, 46 insertions(+), 29 deletions(-) diff --git a/www/perl-lib/SMT/Rest/SCCAPIv1.pm b/www/perl-lib/SMT/Rest/SCCAPIv1.pm index 4874d8e2..20a30149 100644 --- a/www/perl-lib/SMT/Rest/SCCAPIv1.pm +++ b/www/perl-lib/SMT/Rest/SCCAPIv1.pm @@ -380,11 +380,6 @@ sub products my @pidarr = keys %{$existingregs}; my $catalogs = SMT::Registration::findCatalogs($self->request(), $self->dbh(), $target, \@pidarr); - if ( (keys %{$catalogs}) == 0) - { - return (Apache2::Const::HTTP_UNPROCESSABLE_ENTITY, "No repositories found"); - } - # TODO: get status - from SMT? # return result diff --git a/www/perl-lib/SMT/Rest/SCCAPIv2.pm b/www/perl-lib/SMT/Rest/SCCAPIv2.pm index 004832ad..29ab005e 100644 --- a/www/perl-lib/SMT/Rest/SCCAPIv2.pm +++ b/www/perl-lib/SMT/Rest/SCCAPIv2.pm @@ -356,11 +356,6 @@ sub products my @pidarr = keys %{$existingregs}; my $catalogs = SMT::Registration::findCatalogs($self->request(), $self->dbh(), $target, \@pidarr); - if ( (keys %{$catalogs}) == 0) - { - return (Apache2::Const::HTTP_UNPROCESSABLE_ENTITY, "No repositories found"); - } - # TODO: get status - from SMT? # diff --git a/www/perl-lib/SMT/Rest/SCCAPIv4.pm b/www/perl-lib/SMT/Rest/SCCAPIv4.pm index 6b5bf583..dbed52d5 100644 --- a/www/perl-lib/SMT/Rest/SCCAPIv4.pm +++ b/www/perl-lib/SMT/Rest/SCCAPIv4.pm @@ -383,11 +383,6 @@ sub delete_single_product my @pidarr = keys %{$existingregs}; my $catalogs = SMT::Registration::findCatalogs($self->request(), $self->dbh(), $target, \@pidarr); - if ( (keys %{$catalogs}) == 0) - { - return (Apache2::Const::HTTP_UNPROCESSABLE_ENTITY, "No repositories found"); - } - foreach my $param ( qw{identifier version arch} ) { unless ( exists $c->{$param} && $c->{$param} ) { return (Apache2::Const::HTTP_UNPROCESSABLE_ENTITY, "Missing required parameter: $param"); diff --git a/www/perl-lib/SMT/SCCSync.pm b/www/perl-lib/SMT/SCCSync.pm index c1135819..9d828744 100644 --- a/www/perl-lib/SMT/SCCSync.pm +++ b/www/perl-lib/SMT/SCCSync.pm @@ -675,6 +675,7 @@ sub _updateProducts { my $self = shift; my $product = shift; + my $root_product_id = shift; my $statement = ""; my $ret = 0; @@ -913,11 +914,14 @@ EOS } $ret += $retprd; + $root_product_id = $product->{id} if ($product->{product_type} eq 'base'); + foreach my $ext (@{$product->{extensions}}) { - $ret += $self->_updateProducts($ext); - $self->_collectExtensions($product->{id}, $ext->{id}); + $ret += $self->_updateProducts($ext, $root_product_id); + $self->_collectExtensions($product->{id}, $ext->{id}, $root_product_id, $ext->{recommended}); } + if (exists $product->{predecessor_ids}) { $self->_collectMigrations($product->{id}, $product->{predecessor_ids}); @@ -990,8 +994,15 @@ sub _collectExtensions my $self = shift || return; my $prdid = shift || return; my $extid = shift || return; - - $self->{EXTS}->{"$prdid-$extid"} = {productid => $prdid, extensionid => $extid}; + my $root_product_id = shift || return; + my $recommended = shift; + + $self->{EXTS}->{"$prdid-$extid-$root_product_id"} = { + productid => $prdid, + extensionid => $extid, + root_product_id => $root_product_id, + recommended => $recommended + }; } sub _updateExtensions @@ -1000,14 +1011,14 @@ sub _updateExtensions my $err = 0; my $href = {}; - my $sql = "select PRODUCTID, EXTENSIONID from ProductExtensions where SRC = 'S'"; + my $sql = "select PRODUCTID, EXTENSIONID, ROOTPRODUCTID, RECOMMENDED from ProductExtensions where SRC = 'S'"; printLog($self->{LOG}, $self->vblevel(), LOG_DEBUG, "STATEMENT: $sql"); eval { my $ref = $self->{DBH}->selectall_arrayref($sql, {Slice => {}}); foreach my $v (@{$ref}) { - $href->{$v->{PRODUCTID}."-".$v->{EXTENSIONID}} = $v; + $href->{$v->{PRODUCTID}."-".$v->{EXTENSIONID}."-".$v->{ROOTPRODUCTID}} = $v; } }; if($@) @@ -1022,10 +1033,15 @@ sub _updateExtensions $self->{LOG}, $self->vblevel()); my $extid = SMT::Utils::lookupProductIdByDataId($self->{DBH}, $self->{EXTS}->{$key}->{extensionid}, 'S', $self->{LOG}, $self->vblevel()); + my $root_product_id = SMT::Utils::lookupProductIdByDataId($self->{DBH}, $self->{EXTS}->{$key}->{root_product_id}, 'S', + $self->{LOG}, $self->vblevel()); + + my $recommended = $self->{EXTS}->{$key}->{recommended}; - if (exists $href->{$prdid."-".$extid}) + my $hash_key = $prdid."-".$extid."-".$root_product_id; + if (exists $href->{$hash_key} && $href->{$hash_key}->{RECOMMENDED} == $recommended) { - delete $href->{$prdid."-".$extid}; + delete $href->{$hash_key}; next; } else @@ -1042,9 +1058,22 @@ sub _updateExtensions $err += 1; next; } - my $sql = sprintf("INSERT INTO ProductExtensions VALUES ( %s, %s, 'S')", - $self->{DBH}->quote($prdid), - $self->{DBH}->quote($extid)); + if(!$root_product_id) + { + printLog($self->{LOG}, $self->vblevel(), LOG_ERROR, "Root product not found for: ".$self->{EXTS}->{$key}->{root_product_id}); + $err += 1; + next; + } + my $sql = sprintf( + "INSERT INTO ProductExtensions (PRODUCTID, EXTENSIONID, ROOTPRODUCTID, RECOMMENDED, SRC) + VALUES ( %s, %s, %s, %s, 'S') + ON DUPLICATE KEY UPDATE recommended = %s", + $self->{DBH}->quote($prdid), + $self->{DBH}->quote($extid), + $self->{DBH}->quote($root_product_id), + $self->{DBH}->quote($recommended), + $self->{DBH}->quote($recommended) # on duplicate key + ); printLog($self->{LOG}, $self->vblevel(), LOG_DEBUG, "STATEMENT: $sql"); eval { $self->{DBH}->do($sql); @@ -1054,14 +1083,17 @@ sub _updateExtensions printLog($self->{LOG}, $self->vblevel(), LOG_ERROR, "$@"); return 1; } + delete $href->{$hash_key}; } } # remove obsolete entries foreach my $key (keys %{$href}) { - $sql = sprintf("DELETE FROM ProductExtensions WHERE PRODUCTID = %s AND EXTENSIONID = %s", - $self->{DBH}->quote($href->{$key}->{PRODUCTID}), - $self->{DBH}->quote($href->{$key}->{EXTENSIONID})); + $sql = sprintf("DELETE FROM ProductExtensions WHERE PRODUCTID = %s AND EXTENSIONID = %s AND ROOTPRODUCTID = %s", + $self->{DBH}->quote($href->{$key}->{PRODUCTID}), + $self->{DBH}->quote($href->{$key}->{EXTENSIONID}), + $self->{DBH}->quote($href->{$key}->{ROOTPRODUCTID}) + ); printLog($self->{LOG}, $self->vblevel(), LOG_DEBUG, "STATEMENT: $sql"); eval { $self->{DBH}->do($sql); From 0cd956489dd0441c00eef60117423cd05b33a7b4 Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Mon, 20 Nov 2017 16:03:10 +0100 Subject: [PATCH 3/4] Add recommended attribute to product JSON --- www/perl-lib/SMT/Rest/SCCAPIv3.pm | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/www/perl-lib/SMT/Rest/SCCAPIv3.pm b/www/perl-lib/SMT/Rest/SCCAPIv3.pm index eee52217..d6b7a694 100644 --- a/www/perl-lib/SMT/Rest/SCCAPIv3.pm +++ b/www/perl-lib/SMT/Rest/SCCAPIv3.pm @@ -63,6 +63,7 @@ sub get_extensions p.RELEASE_STAGE release_stage, 1 free, p.PRODUCT_TYPE product_type, + FALSE recommended, (CASE WHEN (SELECT c.DOMIRROR FROM ProductCatalogs pc JOIN Catalogs c ON pc.CATALOGID = c.ID @@ -101,8 +102,9 @@ sub get_extensions $result->{$pdid}->{id} = int($result->{$pdid}->{id}); $result->{$pdid}->{free} = ($result->{$pdid}->{free} eq "0"?JSON::false:JSON::true); $result->{$pdid}->{available} = ($result->{$pdid}->{available} eq "0"?JSON::false:JSON::true); + $result->{$pdid}->{recommended} = ($result->{$pdid}->{recommended} eq "0"?JSON::false:JSON::true); $result->{$pdid}->{repositories} = $self->_repositories_for_product($baseURL, $result->{$pdid}->{id}); - foreach my $ext ( values %{$self->_extensions_for_products([$result->{$pdid}->{id}])}) + foreach my $ext ( values %{$self->_extensions_for_products([$result->{$pdid}->{id}], $pdid)}) { push @{$result->{$pdid}->{extensions}}, $ext; } @@ -181,6 +183,7 @@ sub get_subscriptions_products p.RELEASE_STAGE release_stage, 1 free, p.PRODUCT_TYPE product_type, + FALSE recommended, (CASE WHEN (SELECT c.DOMIRROR FROM ProductCatalogs pc JOIN Catalogs c ON pc.CATALOGID = c.ID @@ -221,8 +224,9 @@ sub get_subscriptions_products $result->{$pdid}->{id} = int($result->{$pdid}->{id}); $result->{$pdid}->{free} = ($result->{$pdid}->{free} eq "0"?JSON::false:JSON::true); $result->{$pdid}->{available} = ($result->{$pdid}->{available} eq "0"?JSON::false:JSON::true); + $result->{$pdid}->{recommended} = ($result->{$pdid}->{recommended} eq "0"?JSON::false:JSON::true); $result->{$pdid}->{repositories} = $self->_repositories_for_product($baseURL, $result->{$pdid}->{id}, 1); - foreach my $ext ( values %{$self->_extensions_for_products([$result->{$pdid}->{id}], 1)}) + foreach my $ext ( values %{$self->_extensions_for_products([$result->{$pdid}->{id}], $pdid, 1)}) { push @{$result->{$pdid}->{extensions}}, $ext; } @@ -246,6 +250,7 @@ sub _extensions_for_products { my $self = shift || return {}; my $productids = shift || return {}; + my $root_product_id = shift || return {}; my $includeRepoAuth = shift || 0; my $result = {}; @@ -275,6 +280,7 @@ sub _extensions_for_products e.RELEASE_STAGE release_stage, 1 free, e.PRODUCT_TYPE product_type, + pe.RECOMMENDED recommended, (CASE WHEN (SELECT c.DOMIRROR FROM ProductCatalogs pc JOIN Catalogs c ON pc.CATALOGID = c.ID @@ -285,10 +291,10 @@ sub _extensions_for_products THEN 0 ELSE 1 END ) available FROM Products p JOIN ProductExtensions pe ON p.ID = pe.PRODUCTID - JOIN Products e ON pe.EXTENSIONID = e.ID + JOIN Products e ON pe.EXTENSIONID = e.ID AND pe.ROOTPRODUCTID = %s WHERE p.ID in (%s) AND e.PRODUCT_LIST = 'Y' - ", join(',', @{$productids})); + ", $root_product_id, join(',', @{$productids})); $self->request()->log->info("STATEMENT: $sql"); eval { @@ -302,9 +308,10 @@ sub _extensions_for_products $result->{$pdid}->{extensions} = []; $result->{$pdid}->{free} = ($result->{$pdid}->{free} eq "0"?JSON::false:JSON::true); $result->{$pdid}->{available} = ($result->{$pdid}->{available} eq "0"?JSON::false:JSON::true); + $result->{$pdid}->{recommended} = ($result->{$pdid}->{recommended} eq "0"?JSON::false:JSON::true); $result->{$pdid}->{id} = int($result->{$pdid}->{id}); $result->{$pdid}->{repositories} = $self->_repositories_for_product($baseURL, $result->{$pdid}->{id}, $includeRepoAuth); - foreach my $ext ( values %{$self->_extensions_for_products([$result->{$pdid}->{id}], $includeRepoAuth)}) + foreach my $ext ( values %{$self->_extensions_for_products([$result->{$pdid}->{id}], $root_product_id, $includeRepoAuth)}) { push @{$result->{$pdid}->{extensions}}, $ext; } @@ -339,6 +346,7 @@ sub _getProduct p.RELEASE_STAGE release_stage, 1 free, p.PRODUCT_TYPE product_type, + FALSE recommended, (CASE WHEN (SELECT c.DOMIRROR FROM ProductCatalogs pc JOIN Catalogs c ON pc.CATALOGID = c.ID @@ -366,6 +374,7 @@ sub _getProduct { $product->{free} = ($product->{free} eq "0"?JSON::false:JSON::true); $product->{available} = ($product->{available} eq "0"?JSON::false:JSON::true); + $product->{recommended} = ($product->{recommended} eq "0"?JSON::false:JSON::true); $product->{id} = int($product->{id}); $product->{repositories} = $self->_repositories_for_product($baseURL, $product->{id}); $result = $product; From cec2988962fa4dc9c88936dd98327de4f419b864 Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Tue, 21 Nov 2017 17:40:39 +0100 Subject: [PATCH 4/4] Update changes file; fix sync to walk through all extensions --- package/smt.changes | 7 +++++++ www/perl-lib/SMT/SCCSync.pm | 16 ++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/package/smt.changes b/package/smt.changes index 7ee2f683..43ffc2a5 100644 --- a/package/smt.changes +++ b/package/smt.changes @@ -1,3 +1,10 @@ +------------------------------------------------------------------- +Tue Nov 21 13:44:36 UTC 2017 - ikapelyukhin@suse.com + +- version 3.0.31 +- Add SLE15 product tree support +- Add DB migration to accomodate SLE15 product data from SCC + ------------------------------------------------------------------- Wed Oct 4 07:43:16 UTC 2017 - ikapelyukhin@suse.com diff --git a/www/perl-lib/SMT/SCCSync.pm b/www/perl-lib/SMT/SCCSync.pm index 9d828744..fad28e8a 100644 --- a/www/perl-lib/SMT/SCCSync.pm +++ b/www/perl-lib/SMT/SCCSync.pm @@ -736,6 +736,14 @@ id="\${mirror:id}" description="\${mirror:name}" type="\${mirror:type}"> EOS ; + $root_product_id = $product->{id} if ($product->{product_type} eq 'base'); + + foreach my $ext (@{$product->{extensions}}) + { + $ret += $self->_updateProducts($ext, $root_product_id); + $self->_collectExtensions($product->{id}, $ext->{id}, $root_product_id, $ext->{recommended}); + } + # we inserted/update this product already in this run # so let's skip it return 0 if(exists $self->{PROD_DONE}->{$product->{id}}); @@ -914,14 +922,6 @@ EOS } $ret += $retprd; - $root_product_id = $product->{id} if ($product->{product_type} eq 'base'); - - foreach my $ext (@{$product->{extensions}}) - { - $ret += $self->_updateProducts($ext, $root_product_id); - $self->_collectExtensions($product->{id}, $ext->{id}, $root_product_id, $ext->{recommended}); - } - if (exists $product->{predecessor_ids}) { $self->_collectMigrations($product->{id}, $product->{predecessor_ids});