Upload analyzer statistics information

api/v6/report_server.thrift

@@ -72,6 +72,10 @@ enum SortType {
   BUG_PATH_LENGTH,
 }
 
+enum StoreLimitKind {
+  FAILURE_ZIP_SIZE,         // Maximum size of the collected failed zips which can be store on the server.
+  COMPILATION_DATABASE_SIZE // Limit of the compilation database file size.
+}
 
 struct SourceFileData {
   1: i64             fileId,
@@ -515,4 +519,25 @@ service codeCheckerDBAccess {
                    6: list<string> trimPathPrefixes)
                    throws (1: shared.RequestFailed requestError),
 
+  // Returns true if analysis statistics information can be sent to the server,
+  // otherwise it returns false.
+  // PERMISSION: PRODUCT_STORE
+  bool allowsStoringAnalysisStatistics()
+                                       throws (1: shared.RequestFailed requestError),
+
+  // Returns size limit for each server configuration parameter.
+  // The first key of the map is the limit type, the second is the actual limit
+  // value in bytes.
+  // PERMISSION: PRODUCT_STORE
+  map<StoreLimitKind, i64> getAnalysisStatisticsLimits()
+                                                       throws (1: shared.RequestFailed requestError),
+
+  // This function stores analysis statistics information on the server in a
+  // directory which specified in the configuration file of the server. These
+  // information are sent in a ZIP file where the ZIP file has to be compressed
+  // and sent as a base64 encoded string.
+  // PERMISSION: PRODUCT_STORE
+  bool storeAnalysisStatistics(1: string runName
+                               2: string zipfile)
+                               throws (1: shared.RequestFailed requestError),
 }

config/server_config.json

@@ -1,5 +1,12 @@
 {
   "max_run_count": null,
+  "store": {
+    "analysis_statistics_dir": null,
+    "limit": {
+      "failure_zip_size": 52428800,
+      "compilation_database_size": 104857600
+    }
+  },
   "authentication": {
     "enabled" : false,
     "realm_name" : "CodeChecker Privileged server",

docs/server_config.md

@@ -10,6 +10,11 @@ using the package's installed `config/server_config.json` as a template.
 Table of Contents
 =================
 * [Run limitation](#run-limitations)
+* [Storage](#storage)
+  * [Directory of analysis statistics](#directory-of-analysis-statistics)
+  * [Limits](#Limits)
+    * [Maximum size of failure zips](#maximum-size-of-failure-zips)
+    * [Size of the compilation database](#size-of-the-compilation-database)
 * [Authentication](#authentication)
 
 ## Run limitation
@@ -19,6 +24,33 @@ stored on the server for a product.
 If this field is not present in the config file or the value of this field is a
 negative value, run storage becomes unlimited.
 
+## Storage
+The `store` section of the config file controls storage specific options for the
+server and command line.
+
+### Directory of analysis statistics
+The `analysis_statistics_dir` option specifies a directory where analysis
+statistics should be stored. If this option is specified in the config file the
+client will send analysis related information to the server and the server will
+store these information in this directory.
+If this directory is not specified the server will not store any analysis
+statistic information.
+
+### Limits
+The `limit` section controls limitation of analysis statistics.
+
+#### Maximum size of failure zips
+The `failure_zip_size` section of the `limit` controls the maximum size of
+uploadable failure zips in *bytes*.
+
+*Default value*: 52428800 bytes = 50 MB
+
+#### Size of the compilation database
+The `compilation_database_size` section of the `limit` controls the maximum
+size of uploadable compilation database file in *bytes*.
+
+*Default value*: 104857600 bytes = 100 MB
+
 ## Authentication
 For authentication configuration options see the
 [Authentication](authentication.md) documentation.

libcodechecker/libclient/thrift_helper.py

@@ -156,3 +156,15 @@ def getMissingContentHashes(self, file_hashes):
     def massStoreRun(self, name, tag, version, zipdir, force,
                      trim_path_prefixes):
         pass
+
+    @ThriftClientCall
+    def allowsStoringAnalysisStatistics(self):
+        pass
+
+    @ThriftClientCall
+    def getAnalysisStatisticsLimits(self):
+        pass
+
+    @ThriftClientCall
+    def storeAnalysisStatistics(self, run_name, zip_file):
+        pass

libcodechecker/libhandlers/store.py

@@ -21,6 +21,7 @@
 import zipfile
 import zlib
 
+from codeCheckerDBAccess_v6.ttypes import StoreLimitKind
 from shared.ttypes import Permission, RequestFailed, ErrorCode
 
 from libcodechecker import logger
@@ -300,6 +301,111 @@ def collect_file_hashes_from_plist(plist_file):
             map(lambda f_: " - " + f_, missing_source_files)))
 
 
+def get_analysis_statistics(inputs, limits):
+    """
+    Collects analysis statistics information and returns them.
+    """
+    statistics_files = []
+    for input_path in inputs:
+        input_path = os.path.abspath(input_path)
+
+        if not os.path.exists(input_path):
+            raise OSError(errno.ENOENT,
+                          "Input path does not exist", input_path)
+
+        dirs = []
+        if os.path.isfile(input_path):
+            files = [input_path]
+        else:
+            _, dirs, files = next(os.walk(input_path), ([], [], []))
+
+        for inp_f in files:
+            if inp_f == 'compile_cmd.json':
+                compilation_db = os.path.join(input_path, inp_f)
+                compilation_db_size = \
+                    limits.get(StoreLimitKind.COMPILATION_DATABASE_SIZE)
+
+                if os.stat(compilation_db).st_size > compilation_db_size:
+                    LOG.debug("Compilation database is too big (max: %s).",
+                              sizeof_fmt(compilation_db_size))
+                else:
+                    LOG.debug("Copying file '%s' to analyzer statistics "
+                              "ZIP...", compilation_db)
+                    statistics_files.append(compilation_db)
+            elif inp_f in ['compiler_includes.json',
+                           'compiler_target.json',
+                           'metadata.json']:
+                analyzer_file = os.path.join(input_path, inp_f)
+                statistics_files.append(analyzer_file)
+        for inp_dir in dirs:
+            if inp_dir == 'failed':
+                failure_zip_limit = limits.get(StoreLimitKind.FAILURE_ZIP_SIZE)
+
+                failed_dir = os.path.join(input_path, inp_dir)
+                _, _, files = next(os.walk(failed_dir), ([], [], []))
+                failed_files_size = 0
+                for f in files:
+                    failure_zip = os.path.join(failed_dir, f)
+                    failure_zip_size = os.stat(failure_zip).st_size
+                    failed_files_size += failure_zip_size
+
+                    if failed_files_size > failure_zip_limit:
+                        LOG.debug("We reached the limit of maximum uploadable "
+                                  "failure zip size (max: %s).",
+                                  sizeof_fmt(failure_zip_limit))
+                        break
+                    else:
+                        LOG.debug("Copying failure zip file '%s' to analyzer "
+                                  "statistics ZIP...", failure_zip)
+                        statistics_files.append(failure_zip)
+
+        return statistics_files
+
+
+def storing_analysis_statistics(client, inputs, run_name):
+    """
+    Collects and stores analysis statistics information on the server.
+    """
+    _, zip_file = tempfile.mkstemp('.zip')
+    LOG.debug("Will write failed store ZIP to '%s'...", zip_file)
+
+    try:
+        limits = client.getAnalysisStatisticsLimits()
+        statistics_files = get_analysis_statistics(inputs, limits)
+
+        if not statistics_files:
+            LOG.debug("No analyzer statistics information can be found in the "
+                      "report directory.")
+            return False
+
+        # Write statistics files to the ZIP file.
+        with zipfile.ZipFile(zip_file, 'a', allowZip64=True) as zipf:
+            for stat_file in statistics_files:
+                zipf.write(stat_file)
+
+        # Compressing .zip file
+        with open(zip_file, 'rb') as source:
+            compressed = zlib.compress(source.read(),
+                                       zlib.Z_BEST_COMPRESSION)
+
+        with open(zip_file, 'wb') as target:
+            target.write(compressed)
+
+        LOG.debug("[ZIP] Analysis statistics zip written at '%s'", zip_file)
+
+        with open(zip_file, 'rb') as zf:
+            b64zip = base64.b64encode(zf.read())
+
+        # Store analysis statistics on the server
+        return client.storeAnalysisStatistics(run_name, b64zip)
+
+    except Exception as ex:
+        LOG.debug("Storage of analysis statistics zip has been failed: %s", ex)
+
+    finally:
+        os.remove(zip_file)
+
+
 def main(args):
     """
     Store the defect results in the specified input list as bug reports in the
@@ -382,6 +488,10 @@ def main(args):
                             'force' in args,
                             trim_path_prefixes)
 
+        # Storing analysis statistics if the server allows them.
+        if client.allowsStoringAnalysisStatistics():
+            storing_analysis_statistics(client, args.input, args.name)
+
         LOG.info("Storage finished successfully.")
     except RequestFailed as reqfail:
         if reqfail.errorCode == ErrorCode.SOURCE_FILE:

libcodechecker/server/api/report_server.py

@@ -2340,3 +2340,57 @@ def massStoreRun(self, name, tag, version, b64zip, force,
                     "Multiple source code comment can be found with the same "
                     "checker name for same bug!",
                     wrong_src_code_comments)
+
+    @exc_to_thrift_reqfail
+    @timeit
+    def allowsStoringAnalysisStatistics(self):
+        self.__require_store()
+
+        return True if self.__manager.get_analysis_statistics_dir() else False
+
+    @exc_to_thrift_reqfail
+    @timeit
+    def getAnalysisStatisticsLimits(self):
+        self.__require_store()
+
+        cfg = dict()
+
+        # Get the limit of failure zip size.
+        failure_zip_size = self.__manager.get_failure_zip_size()
+        if failure_zip_size:
+            cfg[ttypes.StoreLimitKind.FAILURE_ZIP_SIZE] = failure_zip_size
+
+        # Get the limit of compilation database size.
+        compilation_database_size = \
+            self.__manager.get_compilation_database_size()
+        if compilation_database_size:
+            cfg[ttypes.StoreLimitKind.COMPILATION_DATABASE_SIZE] = \
+                compilation_database_size
+
+        return cfg
+
+    @exc_to_thrift_reqfail
+    @timeit
+    def storeAnalysisStatistics(self, run_name, b64zip):
+        self.__require_store()
+
+        report_dir_store = self.__manager.get_analysis_statistics_dir()
+        if report_dir_store:
+            try:
+                product_dir = os.path.join(report_dir_store,
+                                           self.__product.endpoint)
+
+                # Create report store directory.
+                if not os.path.exists(product_dir):
+                    os.makedirs(product_dir)
+
+                run_zip_file = os.path.join(product_dir, run_name + '.zip')
+                with open(run_zip_file, 'w') as run_zip:
+                    run_zip.write(zlib.decompress(
+                        base64.b64decode(b64zip)))
+                return True
+            except Exception as ex:
+                LOG.error(ex.message)
+                return False
+
+        return False

libcodechecker/server/session_manager.py

@@ -163,6 +163,9 @@ def __init__(self, configuration_file, session_salt,
         self.__max_run_count = scfg_dict['max_run_count'] \
             if 'max_run_count' in scfg_dict else None
 
+        self.__report_dir_store = None
+
+        self.__store_config = scfg_dict.get('store', {})
         self.__auth_config = scfg_dict['authentication']
 
         if force_auth:
@@ -511,6 +514,30 @@ def get_max_run_count(self):
         """
         return self.__max_run_count
 
+    def get_analysis_statistics_dir(self):
+        """
+        Get directory where the compressed analysis statistics files should be
+        stored. If the value is None it means we do not want to store
+        analysis statistics information on the server.
+        """
+
+        return self.__store_config.get('analysis_statistics_dir')
+
+    def get_failure_zip_size(self):
+        """
+        Maximum size of the collected failed zips which can be store on the
+        server.
+        """
+        limit = self.__store_config.get('limit', {})
+        return limit.get('failure_zip_size')
+
+    def get_compilation_database_size(self):
+        """
+        Limit of the compilation database file size.
+        """
+        limit = self.__store_config.get('limit', {})
+        return limit.get('compilation_database_size')
+
     def __get_local_session_from_db(self, token):
         """
         Creates a local session if a valid session token can be found in the