Improve SQL queries and performance to check for PTF packages (bsc#12…

…25619)

Do not export PTF flags as they are calculated at import time

Make Hibernate aware of new attributes and fix junit tests

Use boolean for storing PTF flags in the DB

Add missing indexes to the rhnPackage schema definition

Make 'package_retracted_and_ptf_details' query more efficient

java/code/src/com/redhat/rhn/common/db/datasource/xml/Package_queries.xml

@@ -860,27 +860,15 @@ SELECT P.id, P.summary, PP.name as provider
         SELECT pid as package_id
           FROM suseServerChannelsRetractedPackagesView
          WHERE sid = :sid
-    ), ptfPackages AS (
-        SELECT pp.package_id
-          FROM rhnpackagecapability pc
-                  INNER JOIN rhnpackageprovides pp ON pc.id = pp.capability_id
-         WHERE pc.name = 'ptf()'
-    ), partOfPtfPackages AS (
-        SELECT pp.package_id
-          FROM rhnpackagecapability pc
-                  INNER JOIN rhnpackageprovides pp ON pc.id = pp.capability_id
-         WHERE pc.name = 'ptf-package()'
     )
     SELECT pk.id AS package_id
               -- Convert the ids to 0/1 flags
               , SIGN(COALESCE(rp.package_id, 0)) AS retracted
-              , SIGN(COALESCE(mp.package_id, 0)) AS master_ptf_package
-              , SIGN(COALESCE(pp.package_id, 0)) AS part_of_ptf
+              , SIGN(COALESCE(CAST(pk.is_ptf AS INTEGER), 0))::int AS master_ptf_package
+              , SIGN(COALESCE(CAST(pk.is_part_of_ptf AS INTEGER), 0))::int AS part_of_ptf
       FROM rhnpackage pk
               LEFT JOIN retractedPackages rp ON rp.package_id = pk.id
-              LEFT JOIN ptfPackages mp ON mp.package_id = pk.id
-              LEFT JOIN partOfPtfPackages pp ON pp.package_id = pk.id
-     WHERE pk.id IN (%s) AND ( rp.package_id IS NOT NULL OR mp.package_id IS NOT NULL OR pp.package_id IS NOT NULL )
+     WHERE pk.id IN (%s) AND ( rp.package_id IS NOT NULL OR pk.is_ptf = TRUE OR pk.is_part_of_ptf = TRUE )
 </query>
 
 <mode name="system_available_packages"  class="com.redhat.rhn.frontend.dto.PackageListItem">
@@ -2447,9 +2435,7 @@ WITH ptfInstalled AS (
     SELECT sp.*
       FROM rhnserverpackage sp
                 INNER JOIN rhnpackage pk ON sp.name_id = pk.name_id AND sp.evr_id = pk.evr_id AND sp.package_arch_id = pk.package_arch_id
-                INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
-                INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
-     WHERE sp.server_id = :sid AND pc.name = 'ptf()'
+     WHERE sp.server_id = :sid AND pk.is_ptf = TRUE
 ), highest_id_packages AS (
     SELECT p.name_id, p.evr_id, p.package_arch_id, max(p.id) AS id
       FROM ptfInstalled pi
@@ -2489,9 +2475,7 @@ ORDER BY nvre
 WITH ptf_packages AS (
     SELECT DISTINCT pk.*
       FROM rhnpackage pk
-                INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
-                INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
-     WHERE pc.name = 'ptf()'
+     WHERE pk.is_ptf = TRUE
 )
 SELECT rp.id AS package_id,
        pn.name || '-' || evr_t_as_vre_simple(latest.evr) AS NVRE,

java/code/src/com/redhat/rhn/domain/rhnpackage/Package.java

@@ -55,6 +55,8 @@ public class Package extends BaseDomainHelper {
     private String path;
     private String headerSignature;
     private String copyright;
+    private Boolean isPtfPackage = false;
+    private Boolean isPartOfPtfPackage = false;
     private String cookie;
     private Date lastModified;
     private Boolean lockPending = Boolean.FALSE;
@@ -261,6 +263,34 @@ public void setId(Long i) {
         this.id = i;
     }
 
+    /**
+     * @return Returns the isPtfPackage.
+     */
+    public Boolean getIsPtfPackage() {
+        return isPtfPackage;
+    }
+
+    /**
+     * @param p The isPtfPackage to set.
+     */
+    public void setIsPtfPackage(Boolean p) {
+        this.isPtfPackage = p;
+    }
+
+    /**
+     * @return Returns the isPartOfPtfPackage.
+     */
+    public Boolean getIsPartOfPtfPackage() {
+        return isPartOfPtfPackage;
+    }
+
+    /**
+     * @param p The isPartOfPtfPackage to set.
+     */
+    public void setIsPartOfPtfPackage(Boolean p) {
+        this.isPartOfPtfPackage = p;
+    }
+
     /**
      * @return Returns the lastModified.
      */

java/code/src/com/redhat/rhn/domain/rhnpackage/Package_satellite.hbm.xml

@@ -28,6 +28,8 @@ PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN"
         <property name="path"            type="string" column="path"/>
         <property name="headerSignature" type="string" column="header_sig"/>
         <property name="copyright"       type="string" column="copyright"/>
+        <property name="isPtfPackage"       type="boolean"   column="is_ptf"/>
+        <property name="isPartOfPtfPackage" type="boolean"   column="is_part_of_ptf"/>
         <property name="cookie"          type="string" column="cookie"/>
         <property name="lastModified"    type="timestamp"   column="last_modified"/>
         <property name="created"         type="timestamp"   column="created"/>

java/code/src/com/redhat/rhn/frontend/xmlrpc/system/test/SystemHandlerPtfTest.java

@@ -131,8 +131,10 @@ public void setUp() throws Exception {
         standard = PackageTest.createTestPackage(admin.getOrg());
         standardUpdated = PackageTestUtils.newVersionOfPackage(standard, null, "2.0.0", null, admin.getOrg());
         standardUpdatedPtf = PackageTestUtils.createPtfPackage(standardUpdated, "123456", "1", admin.getOrg());
+        standardUpdatedPtf.setIsPartOfPtfPackage(true);
         ptfMaster = PackageTestUtils.createPtfMaster("123456", "1", admin.getOrg());
         ptfMasterUpdated = PackageTestUtils.newVersionOfPackage(ptfMaster, null, "2", null, admin.getOrg());
+        ptfMasterUpdated.setIsPtfPackage(true);
         ptfPackage = PackageTestUtils.createPtfPackage("123456", "1", admin.getOrg());
         ptfPackageUpdated = PackageTestUtils.createPtfPackage("123456", "2", admin.getOrg());
 

java/code/src/com/redhat/rhn/testing/PackageTestUtils.java

@@ -155,6 +155,7 @@ public static Package createPtfMaster(String ptfNumber, String ptfVersion, Org o
         master.setPackageName(PackageNameTest.createTestPackageName("ptf-" + ptfNumber));
         master.setDescription("Master PTF package of ptf-" + ptfNumber + " for RHN-JAVA unit tests. Please disregard.");
         master.setPackageEvr(PackageEvrFactory.lookupOrCreatePackageEvr(null, ptfVersion, "0", PackageType.RPM));
+        master.setIsPtfPackage(true);
 
         addProvidesHeader(master, findOrCreateCapability(SpecialCapabilityNames.PTF, ptfNumber + "-" + ptfVersion), 8L);
         addProvidesHeader(master, findOrCreateCapability("ptf-" + ptfNumber, ptfVersion + "-0"), 8L);
@@ -175,6 +176,7 @@ public static Package createPtfPackage(String ptfNumber, String ptfVersion, Org
         ptfPackage.setDescription("Package part of ptf-" + ptfNumber + " for RHN-JAVA unit tests. Please disregard.");
         ptfPackage.setPackageEvr(PackageEvrFactory.lookupOrCreatePackageEvr(null, "1",
             "0" + "." + ptfNumber + "." + ptfVersion + ".PTF", PackageType.RPM));
+        ptfPackage.setIsPartOfPtfPackage(true);
 
         addProvidesHeader(ptfPackage, findOrCreateCapability(SpecialCapabilityNames.PTF_PACKAGE), 0L);
         addProvidesHeader(ptfPackage, findOrCreateCapability(ptfPackage.getPackageName().getName(),
@@ -199,6 +201,7 @@ public static Package createPtfPackage(Package original, String ptfNumber, Strin
         PackageEvr evr = original.getPackageEvr();
         ptfPackage.setPackageEvr(PackageEvrFactory.lookupOrCreatePackageEvr(evr.getEpoch(), evr.getVersion(),
             evr.getRelease() + "." + ptfNumber + "." + ptfVersion + ".PTF", original.getPackageType()));
+        ptfPackage.setIsPartOfPtfPackage(true);
 
         addProvidesHeader(ptfPackage, findOrCreateCapability(SpecialCapabilityNames.PTF_PACKAGE), 0L);
         addProvidesHeader(ptfPackage, findOrCreateCapability(ptfPackage.getPackageName().getName(),

java/spacewalk-java.changes.meaksh.master-improve-perforance-to-check-for-ptf-packages

@@ -0,0 +1 @@
+- Improve SQL queries and performance to check for PTF packages (bsc#1225619)

python/spacewalk/satellite_tools/xmlSource.py

@@ -43,7 +43,6 @@
 
 
 class ParseException(Exception):
-
     """general parser exception (generated at this level)."""
 
     pass
@@ -68,7 +67,6 @@ def __init__(self, stream_version, parser_version, *args):
 
 
 class RecoverableParseException(SAXParseException, Exception):
-
     """exception wrapper for a critical, but possibly recoverable, XML parser
     error.
     """
@@ -77,7 +75,6 @@ class RecoverableParseException(SAXParseException, Exception):
 
 
 class FatalParseException(SAXParseException, Exception):
-
     """exception wrapper for a critical XML parser error."""
 
     pass
@@ -107,7 +104,6 @@ def __repr__(self):
 
 # Base class we use as a SAX parsing handler
 class BaseDispatchHandler(ContentHandler, ErrorHandler):
-
     """Base class we use as a SAX parsing handler
 
     We expect the meaningful data to be on the third level.
@@ -831,7 +827,6 @@ class ExtraTagItem(BaseItem):
 
 
 class DependencyItem(BaseItem):
-
     """virtual class - common settings for dependency items"""
 
     item_class = importLib.Dependency
@@ -1466,7 +1461,6 @@ def postprocessItem(self, item):
 
 
 class PackageContainer(IncompletePackageContainer):
-
     """Inherits from IncompletePackageContainer, since we need to postprocess the
     channel information
     """

python/spacewalk/server/importlib/backendLib.py

@@ -57,6 +57,10 @@ class DBint(DBtype):
     pass
 
 
+class DBbool(DBtype):
+    pass
+
+
 class DBstring(DBtype):
     def __init__(self, limit):
         self.limit = limit

python/spacewalk/server/importlib/backendOracle.py

@@ -22,7 +22,15 @@
 # pylint: disable-next=unused-import
 import sys
 from .backend import Backend
-from .backendLib import DBint, DBstring, DBdateTime, Table, TableCollection, DBblob
+from .backendLib import (
+    DBint,
+    DBstring,
+    DBdateTime,
+    Table,
+    TableCollection,
+    DBblob,
+    DBbool,
+)
 from spacewalk.server import rhnSQL
 from spacewalk.server.rhnSQL.const import POSTGRESQL
 from spacewalk.common.rhnConfig import CFG
@@ -234,6 +242,8 @@ class OracleBackend(Backend):
                 "payload_format": DBstring(32),
                 "path": DBstring(1000),
                 "copyright": DBstring(128),
+                "is_ptf": DBbool(),
+                "is_part_of_ptf": DBbool(),
                 "cookie": DBstring(128),
                 "header_start": DBint(),
                 "header_end": DBint(),
@@ -744,7 +754,6 @@ def init(self):
 
 
 class PostgresqlBackend(OracleBackend):
-
     """
     PostgresqlBackend specific implementation. The bulk of the OracleBackend
     is not actually Oracle specific, so we'll re-use as much as we can and just
@@ -770,5 +779,7 @@ def init(self):
 def SQLBackend():
     if CFG.DB_BACKEND == POSTGRESQL:
         backend = PostgresqlBackend()
-    backend.init()
-    return backend
+        backend.init()
+        return backend
+    else:
+        raise ValueError("The selected DB_BACKEND is not supported")

python/spacewalk/server/importlib/packageImport.py

@@ -324,6 +324,16 @@ def _processPackage(self, package):
                 dep["capability"] = nv
                 if nv not in self.capabilities:
                     self.capabilities[nv] = None
+
+        # Check whether package is a PTF or part of PTF
+        package["is_ptf"] = False
+        package["is_part_of_ptf"] = False
+        for cap in package["provides"]:
+            if cap["capability"][0] == "ptf()":
+                package["is_ptf"] = True
+            elif cap["capability"][0] == "ptf-package()":
+                package["is_part_of_ptf"] = True
+
         # Process files too
         fileList = package["files"]
         for f in fileList:

python/spacewalk/spacewalk-backend.changes.meaksh.master-improve-perforance-to-check-for-ptf-packages

@@ -0,0 +1 @@
+- Check and populate PTF attributes at the time of importing packages (bsc#1225619)

schema/spacewalk/common/tables/rhnPackage.sql

@@ -59,6 +59,8 @@ CREATE TABLE rhnPackage
     header_sig       VARCHAR(64),
     copyright        VARCHAR(128),
     cookie           VARCHAR(128),
+    is_ptf           BOOLEAN DEFAULT (FALSE),
+    is_part_of_ptf   BOOLEAN DEFAULT (FALSE),
     last_modified    TIMESTAMPTZ
                          DEFAULT (current_timestamp) NOT NULL,
     created          TIMESTAMPTZ
@@ -85,3 +87,5 @@ CREATE INDEX rhn_package_nid_id_idx
 
 CREATE SEQUENCE rhn_package_id_seq;
 
+CREATE INDEX rhn_package_is_ptf_idx ON rhnPackage (is_ptf);
+CREATE INDEX rhn_package_is_part_of_ptf_idx ON rhnPackage (is_part_of_ptf);

schema/spacewalk/common/views/susePackageExcludingPartOfPtf.sql

@@ -3,14 +3,7 @@
 --
 
 CREATE OR REPLACE VIEW susePackageExcludingPartOfPtf AS
-  WITH ptfPackages AS (
-      SELECT pp.package_id
-        FROM rhnpackagecapability pc
-                  INNER JOIN rhnpackageprovides pp ON pc.id = pp.capability_id
-       WHERE pc.name = 'ptf-package()'
-  )
   SELECT pkg.*
     FROM rhnpackage pkg
-            LEFT JOIN ptfPackages ptf ON pkg.id = ptf.package_id
-   WHERE ptf.package_id IS NULL
+   WHERE pkg.is_part_of_ptf = FALSE
 ;

schema/spacewalk/susemanager-schema.changes.meaksh.master-improve-perforance-to-check-for-ptf-packages

@@ -0,0 +1 @@
+- Introduce new attributes to detect PTF packages (bsc#1225619)

schema/spacewalk/upgrade/susemanager-schema-5.0.8-to-susemanager-schema-5.0.9/300-replace-susePackageExcludingPartOfPtf-view.sql

@@ -0,0 +1,28 @@
+ALTER TABLE rhnPackage
+    ADD COLUMN IF NOT EXISTS is_ptf
+        BOOLEAN DEFAULT (FALSE),
+    ADD COLUMN IF NOT EXISTS is_part_of_ptf
+        BOOLEAN DEFAULT (FALSE);
+
+CREATE INDEX IF NOT EXISTS rhn_package_is_ptf_idx ON rhnPackage (is_ptf);
+CREATE INDEX IF NOT EXISTS rhn_package_is_part_of_ptf_idx ON rhnPackage (is_part_of_ptf);
+
+UPDATE rhnPackage SET is_ptf = TRUE WHERE id IN (
+    SELECT DISTINCT pk.id
+    FROM rhnpackage pk
+        INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
+        INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
+    WHERE pc.name = 'ptf()');
+
+UPDATE rhnPackage SET is_part_of_ptf = TRUE WHERE id IN (
+    SELECT DISTINCT pk.id
+    FROM rhnpackage pk
+        INNER JOIN rhnpackageprovides pp ON pk.id = pp.package_id
+        INNER JOIN rhnpackagecapability pc ON pp.capability_id = pc.id
+    WHERE pc.name = 'ptf-package()');
+
+CREATE OR REPLACE VIEW susePackageExcludingPartOfPtf AS
+  SELECT pkg.*
+    FROM rhnpackage pkg
+   WHERE pkg.is_part_of_ptf = FALSE
+;