Skip to content

Commit

Permalink
Refactoring instance manager
Browse files Browse the repository at this point in the history
  • Loading branch information
@csordasmarton
csordasmarton committed Sep 3, 2018
1 parent 5ee98f0 commit 42330a1
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 88 deletions.
30 changes: 14 additions & 16 deletions libcodechecker/analyze/log_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,9 @@ def remove_file_if_exists(filename):
os.remove(filename)


def parse_compile_commands_json(logfile, parseLogOptions):
def parse_compile_commands_json(log_data, parseLogOptions):
"""
logfile: is a compile command json
log_data: content of a compile command json.
"""

output_path = parseLogOptions.output_path
Expand All @@ -239,13 +239,11 @@ def parse_compile_commands_json(logfile, parseLogOptions):
actions = []
filtered_build_actions = {}

data = load_json_or_empty(logfile, {})

compiler_includes = {}
compiler_target = {}

counter = 0
for entry in data:
for entry in log_data:
sourcefile = entry['file']

if not os.path.isabs(sourcefile):
Expand Down Expand Up @@ -364,17 +362,17 @@ def parse_log(logfilepath, parseLogOptions):
"""
LOG.debug('Parsing log file: ' + logfilepath)

with open(logfilepath) as logfile:
try:
actions = parse_compile_commands_json(logfile, parseLogOptions)
except (ValueError, KeyError, TypeError) as ex:
if os.stat(logfilepath).st_size == 0:
LOG.error('The compile database is empty.')
else:
LOG.error('The compile database is not valid.')
LOG.debug(traceback.format_exc())
LOG.debug(ex)
sys.exit(1)
try:
data = load_json_or_empty(logfilepath, {})
actions = parse_compile_commands_json(data, parseLogOptions)
except (ValueError, KeyError, TypeError) as ex:
if os.stat(logfilepath).st_size == 0:
LOG.error('The compile database is empty.')
else:
LOG.error('The compile database is not valid.')
LOG.debug(traceback.format_exc())
LOG.debug(ex)
sys.exit(1)

LOG.debug('Parsing log file done.')

Expand Down
75 changes: 40 additions & 35 deletions libcodechecker/server/instance_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,30 +14,31 @@
import getpass
import json
import os
import portalocker
import psutil
import socket
import stat

import portalocker

from libcodechecker.util import load_json_or_empty


def __getInstanceDescriptorPath(folder=None):
def __get_instance_descriptor_path(folder=None):
if not folder:
folder = os.path.expanduser("~")

return os.path.join(folder, ".codechecker.instances.json")


def __makeInstanceDescriptorFile(folder=None):
descriptor = __getInstanceDescriptorPath(folder)
def __make_instance_descriptor_file(folder=None):
descriptor = __get_instance_descriptor_path(folder)
if not os.path.exists(descriptor):
with open(descriptor, 'w') as f:
json.dump([], f)
os.chmod(descriptor, stat.S_IRUSR | stat.S_IWUSR)


def __checkInstance(hostname, pid):
def __check_instance(hostname, pid):
"""Check if the given process on the system is a valid, running CodeChecker
for the current user."""

Expand All @@ -57,32 +58,34 @@ def __checkInstance(hostname, pid):
return False


def __rewriteInstanceFile(append, remove, folder=None):
"""This helper method reads the user's instance descriptor and manages it
eliminating dead records, appending new ones and reserialising the file."""
def __rewrite_instance_file(append, remove, folder=None):
"""
This helper method reads the user's instance descriptor and manages it
eliminating dead records, appending new ones and re-serialising the file.
"""
__make_instance_descriptor_file(folder)

append_pids = [i['pid'] for i in append]

__makeInstanceDescriptorFile(folder)
with open(__getInstanceDescriptorPath(folder), 'r+') as f:
portalocker.lock(f, portalocker.LOCK_EX)
# After reading, check every instance if they are still valid and
# make sure PID does not collide accidentally with the
# to-be-registered instances, if any exists in the append list as it
# would cause duplication.
#
# Also, we remove the records to the given PIDs, if any exists.
instances = [i for i in get_instances(folder)
if i['pid'] not in append_pids and
(i['hostname'] + ":" + str(i['pid'])) not in remove]

# After reading, check every instance if they are still valid and
# make sure PID does not collide accidentally with the
# to-be-registered instances, if any exists in the append list as it
# would cause duplication.
#
# Also, we remove the records to the given PIDs, if any exists.
append_pids = [i['pid'] for i in append]
instances = [i for i in json.load(f)
if i['pid'] not in append_pids and
(i['hostname'] + ":" + str(i['pid'])) not in remove and
__checkInstance(i['hostname'], i['pid'])]
with open(__get_instance_descriptor_path(folder), 'w') as instance_file:
portalocker.lock(instance_file, portalocker.LOCK_EX)

instances = instances + append

f.seek(0)
f.truncate()
json.dump(instances, f, indent=2)
portalocker.unlock(f)
instance_file.seek(0)
instance_file.truncate()
json.dump(instances, instance_file, indent=2)
portalocker.unlock(instance_file)


def register(pid, workspace, port, folder=None):
Expand All @@ -91,12 +94,12 @@ def register(pid, workspace, port, folder=None):
descriptor.
"""

__rewriteInstanceFile([{"pid": pid,
"hostname": socket.gethostname(),
"workspace": workspace,
"port": port}],
[],
folder)
__rewrite_instance_file([{"pid": pid,
"hostname": socket.gethostname(),
"workspace": workspace,
"port": port}],
[],
folder)


def unregister(pid, folder=None):
Expand All @@ -105,15 +108,17 @@ def unregister(pid, folder=None):
descriptor.
"""

__rewriteInstanceFile([], [socket.gethostname() + ":" + str(pid)], folder)
__rewrite_instance_file([],
[socket.gethostname() + ":" + str(pid)],
folder)


def get_instances(folder=None):
"""Returns the list of running servers for the current user."""

# This method does NOT write the descriptor file.

descriptor = __getInstanceDescriptorPath(folder)
descriptor = __get_instance_descriptor_path(folder)
instances = load_json_or_empty(descriptor, {}, lock=True)

return [i for i in instances if __checkInstance(i['hostname'], i['pid'])]
return [i for i in instances if __check_instance(i['hostname'], i['pid'])]
21 changes: 13 additions & 8 deletions libcodechecker/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,21 @@
import argparse
import datetime
import hashlib
import io
import json
import os
import portalocker
import re
import shutil
import signal
import socket
import stat
import subprocess
import sys
import tempfile
import uuid

from threading import Timer

import portalocker
import psutil

from libcodechecker.logger import get_logger
Expand Down Expand Up @@ -570,21 +570,26 @@ def load_json_or_empty(path, default=None, kind=None, lock=False):

ret = default
try:
with open(path, 'r') as handle:
with io.open(path, 'r') as handle:
if lock:
portalocker.lock(handle, portalocker.LOCK_SH)

ret = json.loads(handle.read())

if lock:
portalocker.unlock(handle)
except IOError as ex:
LOG.warning("Failed to open {0} file: {1}"
.format(kind if kind else 'json', path))
except OSError as ex:
LOG.warning("Failed to open %s file: %s",
kind if kind else 'json',
path)
LOG.warning(ex)
except ValueError as ex:
LOG.warning("'{1}' is not a valid {0} file."
.format(kind if kind else 'json', path))
LOG.warning("'%s' is not a valid %s file.",
kind if kind else 'json',
path)
LOG.warning(ex)
except TypeError as ex:
LOG.warning('Failed to process json file: %s', path)
LOG.warning(ex)

return ret
Expand Down
8 changes: 3 additions & 5 deletions tests/unit/test_buildcmd_escaping.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,18 +74,16 @@ def __get_cmp_json(self, buildcmd):
"command": buildcmd + " -c " + self.src_file_path,
"file": self.src_file_path}

compile_cmds = [compile_cmd]
return json.dumps(compile_cmds)
return [compile_cmd]

def __get_comp_actions(self, compile_cmd):
"""
Generate a compilation command json file and parse it
to return the compilation actions.
"""
comp_cmd_json = self.__get_cmp_json(compile_cmd)
with closing(StringIO(comp_cmd_json)) as text:
return log_parser.parse_compile_commands_json(text,
ParseLogOptions())
return log_parser.parse_compile_commands_json(comp_cmd_json,
ParseLogOptions())

def test_buildmgr(self):
"""
Expand Down
46 changes: 22 additions & 24 deletions tests/unit/test_log_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,26 +150,26 @@ def test_omit_preproc(self):
"""
Compiler preprocessor actions should be omitted.
"""
preprocessor_actions = StringIO('''[
preprocessor_actions = [
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -c /tmp/a.cpp",
"file": "/tmp/a.cpp" },
"command": "g++ /tmp/a.cpp -c /tmp/a.cpp",
"file": "/tmp/a.cpp"},
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -E /tmp/a.cpp",
"file": "/tmp/a.cpp" },
"command": "g++ /tmp/a.cpp -E /tmp/a.cpp",
"file": "/tmp/a.cpp"},
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -MT /tmp/a.cpp",
"file": "/tmp/a.cpp" },
"command": "g++ /tmp/a.cpp -MT /tmp/a.cpp",
"file": "/tmp/a.cpp"},
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -MM /tmp/a.cpp",
"file": "/tmp/a.cpp" },
"command": "g++ /tmp/a.cpp -MM /tmp/a.cpp",
"file": "/tmp/a.cpp"},
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -MF /tmp/a.cpp",
"file": "/tmp/a.cpp" },
"command": "g++ /tmp/a.cpp -MF /tmp/a.cpp",
"file": "/tmp/a.cpp"},
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -M /tmp/a.cpp",
"file": "/tmp/a.cpp" }]
''')
"command": "g++ /tmp/a.cpp -M /tmp/a.cpp",
"file": "/tmp/a.cpp"}]

build_actions = \
log_parser.parse_compile_commands_json(preprocessor_actions,
ParseLogOptions())
Expand All @@ -182,11 +182,10 @@ def test_keep_compile_and_dep(self):
""" Keep the compile command if -MD is set.
Dependency generation is done as a side effect of the compilation.
"""
preprocessor_actions = StringIO('''[
preprocessor_actions = [
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -MD /tmp/a.cpp",
"file": "/tmp/a.cpp" }]
''')
"command": "g++ /tmp/a.cpp -MD /tmp/a.cpp",
"file": "/tmp/a.cpp"}]

build_actions = \
log_parser.parse_compile_commands_json(preprocessor_actions,
Expand All @@ -197,14 +196,13 @@ def test_keep_compile_and_dep(self):
def test_omit_dep_with_e(self):
""" Skip the compile command if -MD is set together with -E. """

preprocessor_actions = StringIO('''[
preprocessor_actions = [
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -MD -E /tmp/a.cpp",
"file": "/tmp/a.cpp" },
"command": "g++ /tmp/a.cpp -MD -E /tmp/a.cpp",
"file": "/tmp/a.cpp"},
{"directory": "/tmp",
"command": "g++ /tmp/a.cpp -E -MD /tmp/a.cpp",
"file": "/tmp/a.cpp" } ]
''')
"command": "g++ /tmp/a.cpp -E -MD /tmp/a.cpp",
"file": "/tmp/a.cpp"}]

build_actions = \
log_parser.parse_compile_commands_json(preprocessor_actions,
Expand Down

0 comments on commit 42330a1

Please sign in to comment.