Skip to content

Commit

Permalink
Merge pull request #745 from yalechen-cyw/main
Browse files Browse the repository at this point in the history 
🎨 update tcaql py
  • Loading branch information
@yql70
yql70 authored Jan 11, 2023
2 parents 7f5cdc1 + 2742914 commit a153250
Showing 1 changed file with 110 additions and 78 deletions.
188 changes: 110 additions & 78 deletions client/tool/util/tca_ql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@
from util.api.fileserver import RetryFileServer
from util.configlib import ConfigReader
from util.envset import EnvSet
from util.errcode import E_NODE_TASK_PARAM
from util.exceptions import AnalyzeTaskError, TaskError
from util.textutil import CodecClient
from util.pathfilter import FilterPathUtil
from util.pathlib import PathMgr
from task.codelintmodel import CodeLintModel
from task.scmmgr import SCMMgr
from task.basic.common import subprocc_log
from util.subprocc import SubProcController
from util.zipmgr import Zip
from util.logutil import LogPrinter
Expand All @@ -53,8 +53,6 @@


class TcaQl(CodeLintModel):
""" """

def __init__(self, params):
CodeLintModel.__init__(self, params)

Expand All @@ -71,10 +69,13 @@ def __download_database(self, params, path):
file_server = RetryFileServer(retry_times=2).get_server()
if file_server.download_big_file(file_name, zip_file):
logger.info("下载成功")
Zip.decompress_by_7z(zip_file, db_path)
Zip().decompress_by_7z(zip_file, db_path)
os.remove(zip_file)
return True
else:
return False
except Exception as e:
logger.warning(f"下载失败 {e}")
logger.warning(f"下载失败")
return False

def __upload_database(self, params, path):
Expand Down Expand Up @@ -128,6 +129,33 @@ def __generate_config_file(self, rule_list, work_dir, source_dir, toscans):
tree.write(setting_file, encoding="utf-8")
return setting_file

def __get_zeus_version(self, params):
# 2023/1/10 调整获取Zeus版本的方式
envs = os.environ
ZEUS_HOME = envs.get("ZEUS_HOME", None)
work_dir = params.work_dir
scan_cmd = self.get_cmd(
"Zeus",
[
"--version",
],
)
lang_ext_output = os.path.join(work_dir, "Zeus_version.txt")
spc = SubProcController(
scan_cmd,
cwd=ZEUS_HOME,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
stdout_filepath=lang_ext_output,
)
spc.wait()

fi = open(lang_ext_output, "rb")
# 读取文件内容并解码为字符串
content = CodecClient().decode(fi.read())
fi.close()
return content.split(" ")[-1].strip()

def compile(self, params, lang):
"""
编译函数，指代码生成数据流
Expand All @@ -140,10 +168,10 @@ def compile(self, params, lang):
repo_id = params.repo_id
envs = os.environ
ZEUS_HOME = envs.get("ZEUS_HOME", None)
tool_path = os.path.join(ZEUS_HOME, "Zeus")
scm_revision = params["scm_revision"]
db_name = f"{repo_id}_{scm_revision}_{lang}"
db_path = os.path.join(db_dir, db_name + ".db")
version = self.__get_zeus_version(params)
db_name = f"{repo_id}_{scm_revision}_{lang}_{version}"
db_path = os.path.join(db_dir, f"{db_name}.db")
inc = params["incr_scan"]
want_suffix = lang_map[lang]
logger.info("是否为增量编译: %s" % inc)
Expand All @@ -156,13 +184,15 @@ def compile(self, params, lang):
return
if inc:
last_scm_revision = params["scm_last_revision"]
last_db_name = f"{repo_id}_{last_scm_revision}_{lang}"
last_db_name = f"{repo_id}_{last_scm_revision}_{lang}_{version}"
last_db_path = os.path.join(db_dir, last_db_name + ".db")
logger.info(f"下载上个成功分析版本数据库{last_db_name}")
if not self.__download_database(params, last_db_name):
logger.info("下载全量数据库失败将重新生成，本次分析将只分析增量部分")
else:
elif os.path.exists(last_db_path):
shutil.copyfile(last_db_path, db_path)
else:
logger.error("下载失败，本次将只分析增量文件，建议全量分析下项目。")
diffs = SCMMgr(params).get_scm_diff()
# 增量需要所有增量文件都重新生成，故这里不能过滤文件
toscans = [diff.path.replace(os.sep, "/") for diff in diffs if diff.path.endswith(tuple(want_suffix))]
Expand All @@ -173,65 +203,69 @@ def compile(self, params, lang):
for toscan in toscans:
wf.writelines(toscan)
wf.write("\n")
inc_build_cmd = [
"./Zeus",
"inc_compile",
"-l",
lang,
"-p",
db_name,
"-db",
db_dir,
"-s",
source_dir,
# "-d",
"-f",
file_list,
]
logger.info(inc_build_cmd)
inc_build_cmd = self.get_cmd(tool_path, inc_build_cmd)
inc_build_cmd = self.get_cmd(
"Zeus",
[
"inc_compile",
"-l",
lang,
"-p",
db_name,
"-db",
db_dir,
"-s",
source_dir,
# "-d",
"-f",
file_list,
],
)
logger.info(" ".join(inc_build_cmd))
# cmd_args += toscans
sp = SubProcController(
command=inc_build_cmd,
cwd=ZEUS_HOME,
stdout_line_callback=self.subprocc_log,
stderr_line_callback=self.subprocc_log,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
)
sp.wait()
logger.info(sp.returncode)
self.__upload_database(params, db_name)
return
# 全量编译命令
toscans = [path.replace(os.sep, "/") for path in PathMgr().get_dir_files(source_dir, tuple(want_suffix))]
toscans = [
path.replace(os.sep, "/") for path in PathMgr().get_dir_files(source_dir, want_suffix=tuple(want_suffix))
]
toscans = FilterPathUtil(params).get_include_files(toscans, relpos)
if not toscans:
return []
with open(file_list, "w") as wf:
for toscan in toscans:
wf.writelines(toscan)
wf.write("\n")
full_build_cmd = [
# "./Zeus",
"compile",
"-p",
db_name,
"-cc",
db_dir,
"-l",
lang,
"-s",
source_dir,
"-f",
file_list
# "-d", # 调试使用
]
logger.info(full_build_cmd)
full_build_cmd = self.get_cmd(tool_path, full_build_cmd)
full_build_cmd = self.get_cmd(
"Zeus",
[
"compile",
"-p",
db_name,
"-cc",
db_dir,
"-l",
lang,
"-s",
source_dir,
"-f",
file_list
# "-d", # 调试使用
],
)
logger.info(" ".join(full_build_cmd))
sp = SubProcController(
command=full_build_cmd,
cwd=ZEUS_HOME,
stdout_line_callback=self.subprocc_log,
stderr_line_callback=self.subprocc_log,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
)
sp.wait()
logger.info(sp.returncode)
Expand All @@ -251,10 +285,10 @@ def analyze(self, params, lang):
repo_id = params.repo_id
envs = os.environ
HADES_HOME = envs.get("HADES_HOME", None)
tool_path = os.path.join(HADES_HOME, "Hades")
scm_revision = params["scm_revision"]
db_name = f"{repo_id}_{scm_revision}_{lang}"
db_path = os.path.join(db_dir, db_name + ".db")
version = self.__get_zeus_version(params)
db_name = f"{repo_id}_{scm_revision}_{lang}_{version}"
db_path = os.path.join(db_dir, f"{db_name}.db")
if not os.path.exists(db_path):
if not os.path.exists(db_dir):
os.makedirs(db_dir)
Expand All @@ -274,7 +308,8 @@ def analyze(self, params, lang):
]
else:
toscans = [
path.replace(os.sep, "/")[relpos:] for path in PathMgr().get_dir_files(source_dir, tuple(want_suffix))
path.replace(os.sep, "/")[relpos:]
for path in PathMgr().get_dir_files(source_dir, want_suffix=tuple(want_suffix))
]
# 过滤文件以及过滤文件取相对路径
toscans = FilterPathUtil(params).get_include_files([os.path.join(source_dir, path) for path in toscans], relpos)
Expand All @@ -287,32 +322,33 @@ def analyze(self, params, lang):
return []
output_json = os.path.join(work_dir, "result.json")
setting_file = self.__generate_config_file(rules, work_dir, source_dir, toscans)
analyze_cmd = [
# "./Hades",
"analyze",
"-l",
lang,
"-cc",
db_dir,
"-db",
db_name,
"-o",
output_json,
"-c",
setting_file,
# "-d",
]
analyze_cmd = self.get_cmd(
"Hades",
[
"analyze",
"-l",
lang,
"-cc",
db_dir,
"-db",
db_name,
"-o",
output_json,
"-c",
setting_file,
# "-d",
],
)
logger.info(analyze_cmd)
task_dir = os.path.dirname(os.getcwd())
request_file = os.path.abspath(os.path.join(task_dir, "task_request.json"))
os.environ["TASK_REQUEST"] = request_file
issues = []
analyze_cmd = self.get_cmd(tool_path, analyze_cmd)
sp = SubProcController(
command=analyze_cmd,
cwd=HADES_HOME,
stdout_line_callback=self.subprocc_log,
stderr_line_callback=self.subprocc_log,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
env=EnvSet().get_origin_env(),
)
sp.wait()
Expand All @@ -326,12 +362,8 @@ def analyze(self, params, lang):
# self.__upload_database(params, db_name)
return issues

def subprocc_log(self, line):
""""""
logger.info(line)

def get_cmd(self, tool_path, options):
return __lu__().format_cmd(tool_path, options)
def get_cmd(self, tool_path, args):
return __lu__().format_cmd(tool_path, args)


tool = TcaQl
Expand Down

0 comments on commit a153250

Please sign in to comment.