Improve SQL queries and performance to check for PTF packages (bsc#12…

…25619) (#9065)

- Do not export PTF flags as they are calculated at import time
- Make Hibernate aware of new attributes and fix junit tests
- Use boolean for storing PTF flags in the DB
- Add missing indexes to the rhnPackage schema definition
- Make 'package_retracted_and_ptf_details' query more efficient

java/code/src/com/redhat/rhn/common/db/datasource/xml/Package_queries.xml

@@ -864,27 +864,15 @@ SELECT P.id, P.summary, PP.name as provider
         SELECT pid as package_id
           FROM suseServerChannelsRetractedPackagesView
          WHERE sid = :sid
-    ), ptfPackages AS (
-        SELECT pp.package_id
-          FROM rhnpackagecapability pc
-                  INNER JOIN rhnpackageprovides pp ON pc.id = pp.capability_id
-         WHERE pc.name = 'ptf()'
-    ), partOfPtfPackages AS (
-        SELECT pp.package_id
-          FROM rhnpackagecapability pc
-                  INNER JOIN rhnpackageprovides pp ON pc.id = pp.capability_id
-         WHERE pc.name = 'ptf-package()'
     )
     SELECT pk.id AS package_id
               -- Convert the ids to 0/1 flags
               , SIGN(COALESCE(rp.package_id, 0)) AS retracted
-              , SIGN(COALESCE(mp.package_id, 0)) AS master_ptf_package
-              , SIGN(COALESCE(pp.package_id, 0)) AS part_of_ptf
+              , SIGN(COALESCE(CAST(pk.is_ptf AS INTEGER), 0))::int AS master_ptf_package
+              , SIGN(COALESCE(CAST(pk.is_part_of_ptf AS INTEGER), 0))::int AS part_of_ptf
       FROM rhnpackage pk
               LEFT JOIN retractedPackages rp ON rp.package_id = pk.id
-              LEFT JOIN ptfPackages mp ON mp.package_id = pk.id
-              LEFT JOIN partOfPtfPackages pp ON pp.package_id = pk.id
-     WHERE pk.id IN (%s) AND ( rp.package_id IS NOT NULL OR mp.package_id IS NOT NULL OR pp.package_id IS NOT NULL )
+     WHERE pk.id IN (%s) AND ( rp.package_id IS NOT NULL OR pk.is_ptf = TRUE OR pk.is_part_of_ptf = TRUE )
 </query>
 
 <mode name="system_available_packages"  class="com.redhat.rhn.frontend.dto.PackageListItem">
@@ -2459,9 +2447,7 @@ WITH ptfInstalled AS (
     SELECT sp.*
       FROM rhnserverpackage sp
                 INNER JOIN rhnpackage pk ON sp.name_id = pk.name_id AND sp.evr_id = pk.evr_id AND sp.package_arch_id = pk.package_arch_id
-                INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
-                INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
-     WHERE sp.server_id = :sid AND pc.name = 'ptf()'
+     WHERE sp.server_id = :sid AND pk.is_ptf = TRUE
 ), highest_id_packages AS (
     SELECT p.name_id, p.evr_id, p.package_arch_id, max(p.id) AS id
       FROM ptfInstalled pi
@@ -2501,9 +2487,7 @@ ORDER BY nvre
 WITH ptf_packages AS (
     SELECT DISTINCT pk.*
       FROM rhnpackage pk
-                INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
-                INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
-     WHERE pc.name = 'ptf()'
+     WHERE pk.is_ptf = TRUE
 )
 SELECT rp.id AS package_id,
        pn.name || '-' || evr_t_as_vre_simple(latest.evr) AS NVRE,

java/code/src/com/redhat/rhn/domain/rhnpackage/Package.java

@@ -55,6 +55,8 @@ public class Package extends BaseDomainHelper {
     private String path;
     private String headerSignature;
     private String copyright;
+    private Boolean isPtfPackage = false;
+    private Boolean isPartOfPtfPackage = false;
     private String cookie;
     private Date lastModified;
     private Boolean lockPending = Boolean.FALSE;
@@ -261,6 +263,34 @@ public void setId(Long i) {
         this.id = i;
     }
 
+    /**
+     * @return Returns the isPtfPackage.
+     */
+    public Boolean getIsPtfPackage() {
+        return isPtfPackage;
+    }
+
+    /**
+     * @param p The isPtfPackage to set.
+     */
+    public void setIsPtfPackage(Boolean p) {
+        this.isPtfPackage = p;
+    }
+
+    /**
+     * @return Returns the isPartOfPtfPackage.
+     */
+    public Boolean getIsPartOfPtfPackage() {
+        return isPartOfPtfPackage;
+    }
+
+    /**
+     * @param p The isPartOfPtfPackage to set.
+     */
+    public void setIsPartOfPtfPackage(Boolean p) {
+        this.isPartOfPtfPackage = p;
+    }
+
     /**
      * @return Returns the lastModified.
      */

java/code/src/com/redhat/rhn/domain/rhnpackage/Package_satellite.hbm.xml

@@ -28,6 +28,8 @@ PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN"
         <property name="path"            type="string" column="path"/>
         <property name="headerSignature" type="string" column="header_sig"/>
         <property name="copyright"       type="string" column="copyright"/>
+        <property name="isPtfPackage"       type="boolean"   column="is_ptf"/>
+        <property name="isPartOfPtfPackage" type="boolean"   column="is_part_of_ptf"/>
         <property name="cookie"          type="string" column="cookie"/>
         <property name="lastModified"    type="timestamp"   column="last_modified"/>
         <property name="created"         type="timestamp"   column="created"/>

java/code/src/com/redhat/rhn/frontend/xmlrpc/system/test/SystemHandlerPtfTest.java

@@ -131,8 +131,10 @@ public void setUp() throws Exception {
         standard = PackageTest.createTestPackage(admin.getOrg());
         standardUpdated = PackageTestUtils.newVersionOfPackage(standard, null, "2.0.0", null, admin.getOrg());
         standardUpdatedPtf = PackageTestUtils.createPtfPackage(standardUpdated, "123456", "1", admin.getOrg());
+        standardUpdatedPtf.setIsPartOfPtfPackage(true);
         ptfMaster = PackageTestUtils.createPtfMaster("123456", "1", admin.getOrg());
         ptfMasterUpdated = PackageTestUtils.newVersionOfPackage(ptfMaster, null, "2", null, admin.getOrg());
+        ptfMasterUpdated.setIsPtfPackage(true);
         ptfPackage = PackageTestUtils.createPtfPackage("123456", "1", admin.getOrg());
         ptfPackageUpdated = PackageTestUtils.createPtfPackage("123456", "2", admin.getOrg());
 

java/code/src/com/redhat/rhn/testing/PackageTestUtils.java

@@ -155,6 +155,7 @@ public static Package createPtfMaster(String ptfNumber, String ptfVersion, Org o
         master.setPackageName(PackageNameTest.createTestPackageName("ptf-" + ptfNumber));
         master.setDescription("Master PTF package of ptf-" + ptfNumber + " for RHN-JAVA unit tests. Please disregard.");
         master.setPackageEvr(PackageEvrFactory.lookupOrCreatePackageEvr(null, ptfVersion, "0", PackageType.RPM));
+        master.setIsPtfPackage(true);
 
         addProvidesHeader(master, findOrCreateCapability(SpecialCapabilityNames.PTF, ptfNumber + "-" + ptfVersion), 8L);
         addProvidesHeader(master, findOrCreateCapability("ptf-" + ptfNumber, ptfVersion + "-0"), 8L);
@@ -175,6 +176,7 @@ public static Package createPtfPackage(String ptfNumber, String ptfVersion, Org
         ptfPackage.setDescription("Package part of ptf-" + ptfNumber + " for RHN-JAVA unit tests. Please disregard.");
         ptfPackage.setPackageEvr(PackageEvrFactory.lookupOrCreatePackageEvr(null, "1",
             "0" + "." + ptfNumber + "." + ptfVersion + ".PTF", PackageType.RPM));
+        ptfPackage.setIsPartOfPtfPackage(true);
 
         addProvidesHeader(ptfPackage, findOrCreateCapability(SpecialCapabilityNames.PTF_PACKAGE), 0L);
         addProvidesHeader(ptfPackage, findOrCreateCapability(ptfPackage.getPackageName().getName(),
@@ -199,6 +201,7 @@ public static Package createPtfPackage(Package original, String ptfNumber, Strin
         PackageEvr evr = original.getPackageEvr();
         ptfPackage.setPackageEvr(PackageEvrFactory.lookupOrCreatePackageEvr(evr.getEpoch(), evr.getVersion(),
             evr.getRelease() + "." + ptfNumber + "." + ptfVersion + ".PTF", original.getPackageType()));
+        ptfPackage.setIsPartOfPtfPackage(true);
 
         addProvidesHeader(ptfPackage, findOrCreateCapability(SpecialCapabilityNames.PTF_PACKAGE), 0L);
         addProvidesHeader(ptfPackage, findOrCreateCapability(ptfPackage.getPackageName().getName(),

java/spacewalk-java.changes.meaksh.master-improve-perforance-to-check-for-ptf-packages

@@ -0,0 +1 @@
+- Improve SQL queries and performance to check for PTF packages (bsc#1225619)

python/spacewalk/satellite_tools/xmlSource.py

@@ -43,7 +43,6 @@
 
 
 class ParseException(Exception):
-
     """general parser exception (generated at this level)."""
 
     pass
@@ -68,7 +67,6 @@ def __init__(self, stream_version, parser_version, *args):
 
 
 class RecoverableParseException(SAXParseException, Exception):
-
     """exception wrapper for a critical, but possibly recoverable, XML parser
     error.
     """
@@ -77,7 +75,6 @@ class RecoverableParseException(SAXParseException, Exception):
 
 
 class FatalParseException(SAXParseException, Exception):
-
     """exception wrapper for a critical XML parser error."""
 
     pass
@@ -107,7 +104,6 @@ def __repr__(self):
 
 # Base class we use as a SAX parsing handler
 class BaseDispatchHandler(ContentHandler, ErrorHandler):
-
     """Base class we use as a SAX parsing handler
 
     We expect the meaningful data to be on the third level.
@@ -831,7 +827,6 @@ class ExtraTagItem(BaseItem):
 
 
 class DependencyItem(BaseItem):
-
     """virtual class - common settings for dependency items"""
 
     item_class = importLib.Dependency
@@ -1466,7 +1461,6 @@ def postprocessItem(self, item):
 
 
 class PackageContainer(IncompletePackageContainer):
-
     """Inherits from IncompletePackageContainer, since we need to postprocess the
     channel information
     """

python/spacewalk/server/importlib/backendLib.py

@@ -57,6 +57,10 @@ class DBint(DBtype):
     pass
 
 
+class DBbool(DBtype):
+    pass
+
+
 class DBstring(DBtype):
     def __init__(self, limit):
         self.limit = limit

python/spacewalk/server/importlib/backendOracle.py

@@ -22,7 +22,15 @@
 # pylint: disable-next=unused-import
 import sys
 from .backend import Backend
-from .backendLib import DBint, DBstring, DBdateTime, Table, TableCollection, DBblob
+from .backendLib import (
+    DBint,
+    DBstring,
+    DBdateTime,
+    Table,
+    TableCollection,
+    DBblob,
+    DBbool,
+)
 from spacewalk.server import rhnSQL
 from spacewalk.server.rhnSQL.const import POSTGRESQL
 from spacewalk.common.rhnConfig import CFG
@@ -234,6 +242,8 @@ class OracleBackend(Backend):
                 "payload_format": DBstring(32),
                 "path": DBstring(1000),
                 "copyright": DBstring(128),
+                "is_ptf": DBbool(),
+                "is_part_of_ptf": DBbool(),
                 "cookie": DBstring(128),
                 "header_start": DBint(),
                 "header_end": DBint(),
@@ -744,7 +754,6 @@ def init(self):
 
 
 class PostgresqlBackend(OracleBackend):
-
     """
     PostgresqlBackend specific implementation. The bulk of the OracleBackend
     is not actually Oracle specific, so we'll re-use as much as we can and just
@@ -770,5 +779,7 @@ def init(self):
 def SQLBackend():
     if CFG.DB_BACKEND == POSTGRESQL:
         backend = PostgresqlBackend()
-    backend.init()
-    return backend
+        backend.init()
+        return backend
+    else:
+        raise ValueError("The selected DB_BACKEND is not supported")

python/spacewalk/server/importlib/packageImport.py

@@ -324,6 +324,16 @@ def _processPackage(self, package):
                 dep["capability"] = nv
                 if nv not in self.capabilities:
                     self.capabilities[nv] = None
+
+        # Check whether package is a PTF or part of PTF
+        package["is_ptf"] = False
+        package["is_part_of_ptf"] = False
+        for cap in package["provides"]:
+            if cap["capability"][0] == "ptf()":
+                package["is_ptf"] = True
+            elif cap["capability"][0] == "ptf-package()":
+                package["is_part_of_ptf"] = True
+
         # Process files too
         fileList = package["files"]
         for f in fileList:

python/spacewalk/spacewalk-backend.changes.meaksh.master-improve-perforance-to-check-for-ptf-packages

@@ -0,0 +1 @@
+- Check and populate PTF attributes at the time of importing packages (bsc#1225619)

schema/spacewalk/common/tables/rhnPackage.sql

@@ -59,6 +59,8 @@ CREATE TABLE rhnPackage
     header_sig       VARCHAR(64),
     copyright        VARCHAR(128),
     cookie           VARCHAR(128),
+    is_ptf           BOOLEAN DEFAULT (FALSE),
+    is_part_of_ptf   BOOLEAN DEFAULT (FALSE),
     last_modified    TIMESTAMPTZ
                          DEFAULT (current_timestamp) NOT NULL,
     created          TIMESTAMPTZ
@@ -85,3 +87,5 @@ CREATE INDEX rhn_package_nid_id_idx
 
 CREATE SEQUENCE rhn_package_id_seq;
 
+CREATE INDEX rhn_package_is_ptf_idx ON rhnPackage (is_ptf);
+CREATE INDEX rhn_package_is_part_of_ptf_idx ON rhnPackage (is_part_of_ptf);

schema/spacewalk/common/views/susePackageExcludingPartOfPtf.sql

@@ -3,14 +3,7 @@
 --
 
 CREATE OR REPLACE VIEW susePackageExcludingPartOfPtf AS
-  WITH ptfPackages AS (
-      SELECT pp.package_id
-        FROM rhnpackagecapability pc
-                  INNER JOIN rhnpackageprovides pp ON pc.id = pp.capability_id
-       WHERE pc.name = 'ptf-package()'
-  )
   SELECT pkg.*
     FROM rhnpackage pkg
-            LEFT JOIN ptfPackages ptf ON pkg.id = ptf.package_id
-   WHERE ptf.package_id IS NULL
+   WHERE pkg.is_part_of_ptf = FALSE
 ;

schema/spacewalk/susemanager-schema.changes.meaksh.master-improve-perforance-to-check-for-ptf-packages

@@ -0,0 +1 @@
+- Introduce new attributes to detect PTF packages (bsc#1225619)

schema/spacewalk/upgrade/susemanager-schema-5.0.8-to-susemanager-schema-5.0.9/300-replace-susePackageExcludingPartOfPtf-view.sql

@@ -0,0 +1,28 @@
+ALTER TABLE rhnPackage
+    ADD COLUMN IF NOT EXISTS is_ptf
+        BOOLEAN DEFAULT (FALSE),
+    ADD COLUMN IF NOT EXISTS is_part_of_ptf
+        BOOLEAN DEFAULT (FALSE);
+
+CREATE INDEX IF NOT EXISTS rhn_package_is_ptf_idx ON rhnPackage (is_ptf);
+CREATE INDEX IF NOT EXISTS rhn_package_is_part_of_ptf_idx ON rhnPackage (is_part_of_ptf);
+
+UPDATE rhnPackage SET is_ptf = TRUE WHERE id IN (
+    SELECT DISTINCT pk.id
+    FROM rhnpackage pk
+        INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
+        INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
+    WHERE pc.name = 'ptf()');
+
+UPDATE rhnPackage SET is_part_of_ptf = TRUE WHERE id IN (
+    SELECT DISTINCT pk.id
+    FROM rhnpackage pk
+        INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
+        INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
+    WHERE pc.name = 'ptf-package()');
+
+CREATE OR REPLACE VIEW susePackageExcludingPartOfPtf AS
+  SELECT pkg.*
+    FROM rhnpackage pkg
+   WHERE pkg.is_part_of_ptf = FALSE
+;