Merge branch 'xdmod8.1' into add-cloud-ingestion-to-ingestor-script

classes/DataWarehouse/Query/Query.php

@@ -462,6 +462,31 @@ public function getStatFields()
         return $this->_stat_fields;
     }
 
+    /**
+     * Add a where condition to the query and add the data to the pdo parameters. This
+     * function should be used when the right hand side of the where condition is untrused
+     * user supplied data.
+     *
+     * Note this function does not handle pdo parameterization of 'IN' conditions.
+     */
+    public function addPdoWhereCondition(\DataWarehouse\Query\Model\WhereCondition $where_condition)
+    {
+        // key on the non-parameterized form since the substitution string is different every time.
+        $key = $where_condition->__toString();
+
+        if (isset($this->_where_conditions[$key])) {
+            return;
+        }
+
+        $namedParam = $this->getNamedParameterMarker($where_condition->_right);
+
+        $this->_where_conditions[$key] = new \DataWarehouse\Query\Model\WhereCondition(
+            $where_condition->_left,
+            $where_condition->_operation,
+            $namedParam
+        );
+    }
+
     public function addWhereCondition(\DataWarehouse\Query\Model\WhereCondition $where_condition)
     {
         $this->_where_conditions[$where_condition->__toString()] = $where_condition;
@@ -725,6 +750,22 @@ public function getCountQueryString()
         $data_query .= ") as a WHERE a.total IS NOT NULL";
         return $data_query;
     }
+
+    /**
+     * Store a bound parameter for the query and return the named parameter
+     * marker that should be used in the SQL query.
+     *
+     * @param the value to bind to the query
+     * @return string a named parameter marker.
+     */
+    protected function getNamedParameterMarker($value)
+    {
+        $pdosubst = ':subst' . $this->pdoindex;
+        $this->pdoparams[$pdosubst] = $value;
+        $this->pdoindex += 1;
+        return $pdosubst;
+    }
+
     public function setParameters(array $parameters = array())
     {
         $this->parameters = $parameters;

classes/DataWarehouse/Query/iJobMetadata.php

@@ -0,0 +1,143 @@
+<?php
+/**
+ * Interface defining the JobMetadata API. The JobMetadata API is used to provide the information
+ * about a given job (or tracked entity) that is available to XDMoD but not stored in the datawarehouse
+ * aggregate tables. See the JobDataset API for the access functions to retrieve information about
+ * a job (or tracked entity) from the datawarehouse itself.
+ *
+ * All modules that implement realms that show information in the Job Viewer
+ * tab must provide an implementation of this interface.
+ */
+
+namespace DataWarehouse\Query;
+
+interface iJobMetadata
+{
+    /**
+     * Return information about all of the data available for a given job. The supported
+     * data categories are defined in \DataWarehouse\Query\RawQueryTypes
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() of \DataWarehouse\Query\RawQueryTypes listing all of
+     * the data types available for the requested job. or null if the job does
+     * not exist.
+     */
+    public function getJobMetadata(\XDUser $user, $jobid);
+
+    /**
+     * Return information that will be displayed in the detailed metrics tab of the Job
+     * Viewer. This function will be called if the job has the DETAILED_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() of metrics. Each metric should contain the summary statistics for the metric as well as the documentation, data type and units. Metrics may be nested. For example if a job provided a CPU User metric
+     * then this call would return a php array with the following structure:
+     *
+     *  "cpu": {
+     *      "user": {
+     *          "std": 0.0066031459102555,
+     *          "med": 0.0045116925711787,
+     *          "skw": 0.79910187474516,
+     *          "cov": 0.73753588228371,
+     *          "max": 0.021206549192241,
+     *          "min": 0.0033731199203248,
+     *          "cnt": 16,
+     *          "krt": -0.85522002996351,
+     *          "avg": 0.0089529825854839,
+     *          "documentation": "The CPU usage in user mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
+     *          "type": "instant",
+     *          "unit": "ratio"
+     *      }
+     *  }
+     */
+    public function getJobSummary(\XDUser $user, $jobid);
+
+    /**
+     * Return information that will be displayed in the executable info tab of the Job Viewer
+     * This function will be called if the job has the EXECUTABLE data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() of text with the information about the executable for the job.
+     */
+    public function getJobExecutableInfo(\XDUser $user, $jobid);
+
+    /**
+     * Return metadata about what timeseries information is available for the provided job.
+     * This function will be called if the job has the TIMESERIES_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() containing timeseries metrics metadata. For example:
+     *
+     *  { "tsid": "cpu_user", "name": "CPU Usage", "leaf": false }
+     *
+     *  where tsid is an (internal) timeseries metric identifier. name is the string that will
+     *  be displayed on the gui and leaf is whether the entry is a leaf node in the metric tree.
+     */
+    public function getJobTimeseriesMetaData(\XDUser $user, $jobid);
+
+    /**
+     * Return metadata about what timeseries information is available for the provided metric for a job.
+     * This function will be called if the job has the TIMESERIES_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     * @param $tsid  The identifier for the timeseries metric. This will be the tsid value returned from a previous
+     * call to getJobTimeseriesMetaData().
+     *
+     * @returns array() containing timeseries metric metadata. for a given metric. For example:
+     *
+     * [{ "nodeid": "node0", "name": "Compute Node 0", "leaf": false },
+     *  { "nodeid": "node1", "name": "Compute Node 1", "leaf": false }]
+     *
+     *  where nodeid is an (internal) timeseries metric identifier. name is the string that will
+     *  be displayed on the gui and leaf is whether the entry is a leaf node in the metric tree.
+     */
+    public function getJobTimeseriesMetricMeta(\XDUser $user, $jobid, $tsid);
+
+    /**
+     * Return metadata about what timeseries information is available for the provided metric for a job.
+     * This function will be called if the job has the TIMESERIES_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     * @param $tsid The name of the metric. This will be the tsid value returned from a previous
+     *                call to getJobTimeseriesMetricMeta().
+     * @param $nodeid The name of the metric. This will be the tsid value returned from a previous
+     *                call to getJobTimeseriesMetricMeta()
+     *
+     * @returns array() containing timeseries metric metadata. for a given metric. For example:
+     *
+     * [{"cpuid": "cpu0", "text": "CPU0", "leaf": true },
+     *  {"cpuid": "cpu1", "text": "CPU1", "leaf": true },
+     *  {"cpuid": "cpu2", "text": "CPU2", "leaf": true },
+     *  {"cpuid": "cpu3", "text": "CPU3", "leaf": true }]
+     *
+     *  where cpuid is an (internal) timeseries metric identifier. name is the string that will
+     *  be displayed on the gui and leaf is whether the entry is a leaf node in the metric tree.
+     */
+    public function getJobTimeseriesMetricNodeMeta(\XDUser $user, $jobid, $tsid, $nodeid);
+
+    /**
+     * Return timeseries data for the provided job. The data available is
+     * obtained via the getJobTimeseriesMetaData(),
+     * getJobTimeseriesMetricMeta() and getJobTimeseriesMetricNodeMeta() if the
+     * data returned from any of these functions has leaf: true, then the corresponding call
+     * to getJobTimeseriesData() will return timeseries data.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     * @param $tsid  The timeseires metric identifier.
+     * @param $nodeid The timeseries node identifier.
+     * @param $cpuid  The timeseries cpu identifier.
+     *
+     * @returns array() timeseries data
+     */
+    public function getJobTimeseriesData(\XDUser $user, $jobid, $tsid, $nodeid, $cpuid);
+}

classes/Rest/Controllers/WarehouseControllerProvider.php

@@ -1489,12 +1489,8 @@ private function getJobDataSet(XDUser $user, $realm, $jobId, $action)
             throw new \DataWarehouse\Query\Exceptions\AccessDeniedException;
         }
 
-        $params = array(
-            new \DataWarehouse\Query\Model\Parameter($rawstats['realms'][$realm]['primary_key'], '=', $jobId)
-        );
-
         $QueryClass = "\\DataWarehouse\\Query\\$realm\\JobDataset";
-        $query = new $QueryClass($params, $action);
+        $query = new $QueryClass(array('primary_key' => $jobId), $action);
 
         $allRoles = $user->getAllRoles();
         $query->setMultipleRoleParameters($allRoles, $user);
@@ -2086,15 +2082,17 @@ private function getJobByPrimaryKey(Application $app, \XDUser $user, $realm, $se
             throw new \DataWarehouse\Query\Exceptions\AccessDeniedException;
         }
 
-        if (isset($searchparams['jobref'])) {
+        if (isset($searchparams['jobref']) && is_int($searchparams['jobref'])) {
             $params = array(
-                new \DataWarehouse\Query\Model\Parameter($rawstats['realms'][$realm]['primary_key'], '=', $searchparams['jobref'])
+                'primary_key' => $searchparams['jobref']
             );
-        } else {
+        } elseif (isset($searchparams['resource_id']) && isset($searchparams['local_job_id'])) {
             $params = array(
-                new \DataWarehouse\Query\Model\Parameter("resource_id", "=", $searchparams['resource_id']),
-                new \DataWarehouse\Query\Model\Parameter($rawstats['realms'][$realm]['ident_key'], "=", $searchparams['local_job_id'])
+                'resource_id' => $searchparams['resource_id'],
+                'job_identifier' => $searchparams['local_job_id']
             );
+        } else {
+            throw new BadRequestException('invalid search parameters');
         }
 
         $QueryClass = "\\DataWarehouse\\Query\\$realm\\JobDataset";

configuration/portal_settings.ini

@@ -38,7 +38,6 @@ email_token_expiration = "600"
 [features]
 xsede = "off"
 appkernels = "off"
-singlejobviewer = "off"
 
 ; Enable support for multiple service providers.
 ;