Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🎨 update tcaql py #745

Merged
merged 2 commits into from
Jan 11, 2023
Merged

🎨 update tcaql py #745

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e761b3
Merge branch 'main' of https://github.com/yalechen-cyw/CodeAnalysis
cyw3 Jan 10, 2023
2742914
:art: update tcaql py
cyw3 Jan 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 110 additions & 78 deletions client/tool/util/tca_ql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@
from util.api.fileserver import RetryFileServer
from util.configlib import ConfigReader
from util.envset import EnvSet
from util.errcode import E_NODE_TASK_PARAM
from util.exceptions import AnalyzeTaskError, TaskError
from util.textutil import CodecClient
from util.pathfilter import FilterPathUtil
from util.pathlib import PathMgr
from task.codelintmodel import CodeLintModel
from task.scmmgr import SCMMgr
from task.basic.common import subprocc_log
from util.subprocc import SubProcController
from util.zipmgr import Zip
from util.logutil import LogPrinter
Expand All @@ -53,8 +53,6 @@


class TcaQl(CodeLintModel):
""" """

def __init__(self, params):
CodeLintModel.__init__(self, params)

Expand All @@ -71,10 +69,13 @@ def __download_database(self, params, path):
file_server = RetryFileServer(retry_times=2).get_server()
if file_server.download_big_file(file_name, zip_file):
logger.info("下载成功")
Zip.decompress_by_7z(zip_file, db_path)
Zip().decompress_by_7z(zip_file, db_path)
os.remove(zip_file)
return True
else:
return False
except Exception as e:
logger.warning(f"下载失败 {e}")
logger.warning(f"下载失败")
return False

def __upload_database(self, params, path):
Expand Down Expand Up @@ -128,6 +129,33 @@ def __generate_config_file(self, rule_list, work_dir, source_dir, toscans):
tree.write(setting_file, encoding="utf-8")
return setting_file

def __get_zeus_version(self, params):
# 2023/1/10 调整获取Zeus版本的方式
envs = os.environ
ZEUS_HOME = envs.get("ZEUS_HOME", None)
work_dir = params.work_dir
scan_cmd = self.get_cmd(
"Zeus",
[
"--version",
],
)
lang_ext_output = os.path.join(work_dir, "Zeus_version.txt")
spc = SubProcController(
scan_cmd,
cwd=ZEUS_HOME,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
stdout_filepath=lang_ext_output,
)
spc.wait()

fi = open(lang_ext_output, "rb")
# 读取文件内容并解码为字符串
content = CodecClient().decode(fi.read())
fi.close()
return content.split(" ")[-1].strip()

def compile(self, params, lang):
"""
编译函数，指代码生成数据流
Expand All @@ -140,10 +168,10 @@ def compile(self, params, lang):
repo_id = params.repo_id
envs = os.environ
ZEUS_HOME = envs.get("ZEUS_HOME", None)
tool_path = os.path.join(ZEUS_HOME, "Zeus")
scm_revision = params["scm_revision"]
db_name = f"{repo_id}_{scm_revision}_{lang}"
db_path = os.path.join(db_dir, db_name + ".db")
version = self.__get_zeus_version(params)
db_name = f"{repo_id}_{scm_revision}_{lang}_{version}"
db_path = os.path.join(db_dir, f"{db_name}.db")
inc = params["incr_scan"]
want_suffix = lang_map[lang]
logger.info("是否为增量编译: %s" % inc)
Expand All @@ -156,13 +184,15 @@ def compile(self, params, lang):
return
if inc:
last_scm_revision = params["scm_last_revision"]
last_db_name = f"{repo_id}_{last_scm_revision}_{lang}"
last_db_name = f"{repo_id}_{last_scm_revision}_{lang}_{version}"
last_db_path = os.path.join(db_dir, last_db_name + ".db")
logger.info(f"下载上个成功分析版本数据库{last_db_name}")
if not self.__download_database(params, last_db_name):
logger.info("下载全量数据库失败将重新生成，本次分析将只分析增量部分")
else:
elif os.path.exists(last_db_path):
shutil.copyfile(last_db_path, db_path)
else:
logger.error("下载失败，本次将只分析增量文件，建议全量分析下项目。")
diffs = SCMMgr(params).get_scm_diff()
# 增量需要所有增量文件都重新生成，故这里不能过滤文件
toscans = [diff.path.replace(os.sep, "/") for diff in diffs if diff.path.endswith(tuple(want_suffix))]
Expand All @@ -173,65 +203,69 @@ def compile(self, params, lang):
for toscan in toscans:
wf.writelines(toscan)
wf.write("\n")
inc_build_cmd = [
"./Zeus",
"inc_compile",
"-l",
lang,
"-p",
db_name,
"-db",
db_dir,
"-s",
source_dir,
# "-d",
"-f",
file_list,
]
logger.info(inc_build_cmd)
inc_build_cmd = self.get_cmd(tool_path, inc_build_cmd)
inc_build_cmd = self.get_cmd(
"Zeus",
[
"inc_compile",
"-l",
lang,
"-p",
db_name,
"-db",
db_dir,
"-s",
source_dir,
# "-d",
"-f",
file_list,
],
)
logger.info(" ".join(inc_build_cmd))
# cmd_args += toscans
sp = SubProcController(
command=inc_build_cmd,
cwd=ZEUS_HOME,
stdout_line_callback=self.subprocc_log,
stderr_line_callback=self.subprocc_log,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
)
sp.wait()
logger.info(sp.returncode)
self.__upload_database(params, db_name)
return
# 全量编译命令
toscans = [path.replace(os.sep, "/") for path in PathMgr().get_dir_files(source_dir, tuple(want_suffix))]
toscans = [
path.replace(os.sep, "/") for path in PathMgr().get_dir_files(source_dir, want_suffix=tuple(want_suffix))
]
toscans = FilterPathUtil(params).get_include_files(toscans, relpos)
if not toscans:
return []
with open(file_list, "w") as wf:
for toscan in toscans:
wf.writelines(toscan)
wf.write("\n")
full_build_cmd = [
# "./Zeus",
"compile",
"-p",
db_name,
"-cc",
db_dir,
"-l",
lang,
"-s",
source_dir,
"-f",
file_list
# "-d", # 调试使用
]
logger.info(full_build_cmd)
full_build_cmd = self.get_cmd(tool_path, full_build_cmd)
full_build_cmd = self.get_cmd(
"Zeus",
[
"compile",
"-p",
db_name,
"-cc",
db_dir,
"-l",
lang,
"-s",
source_dir,
"-f",
file_list
# "-d", # 调试使用
],
)
logger.info(" ".join(full_build_cmd))
sp = SubProcController(
command=full_build_cmd,
cwd=ZEUS_HOME,
stdout_line_callback=self.subprocc_log,
stderr_line_callback=self.subprocc_log,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
)
sp.wait()
logger.info(sp.returncode)
Expand All @@ -251,10 +285,10 @@ def analyze(self, params, lang):
repo_id = params.repo_id
envs = os.environ
HADES_HOME = envs.get("HADES_HOME", None)
tool_path = os.path.join(HADES_HOME, "Hades")
scm_revision = params["scm_revision"]
db_name = f"{repo_id}_{scm_revision}_{lang}"
db_path = os.path.join(db_dir, db_name + ".db")
version = self.__get_zeus_version(params)
db_name = f"{repo_id}_{scm_revision}_{lang}_{version}"
db_path = os.path.join(db_dir, f"{db_name}.db")
if not os.path.exists(db_path):
if not os.path.exists(db_dir):
os.makedirs(db_dir)
Expand All @@ -274,7 +308,8 @@ def analyze(self, params, lang):
]
else:
toscans = [
path.replace(os.sep, "/")[relpos:] for path in PathMgr().get_dir_files(source_dir, tuple(want_suffix))
path.replace(os.sep, "/")[relpos:]
for path in PathMgr().get_dir_files(source_dir, want_suffix=tuple(want_suffix))
]
# 过滤文件以及过滤文件取相对路径
toscans = FilterPathUtil(params).get_include_files([os.path.join(source_dir, path) for path in toscans], relpos)
Expand All @@ -287,32 +322,33 @@ def analyze(self, params, lang):
return []
output_json = os.path.join(work_dir, "result.json")
setting_file = self.__generate_config_file(rules, work_dir, source_dir, toscans)
analyze_cmd = [
# "./Hades",
"analyze",
"-l",
lang,
"-cc",
db_dir,
"-db",
db_name,
"-o",
output_json,
"-c",
setting_file,
# "-d",
]
analyze_cmd = self.get_cmd(
"Hades",
[
"analyze",
"-l",
lang,
"-cc",
db_dir,
"-db",
db_name,
"-o",
output_json,
"-c",
setting_file,
# "-d",
],
)
logger.info(analyze_cmd)
task_dir = os.path.dirname(os.getcwd())
request_file = os.path.abspath(os.path.join(task_dir, "task_request.json"))
os.environ["TASK_REQUEST"] = request_file
issues = []
analyze_cmd = self.get_cmd(tool_path, analyze_cmd)
sp = SubProcController(
command=analyze_cmd,
cwd=HADES_HOME,
stdout_line_callback=self.subprocc_log,
stderr_line_callback=self.subprocc_log,
stdout_line_callback=subprocc_log,
stderr_line_callback=subprocc_log,
env=EnvSet().get_origin_env(),
)
sp.wait()
Expand All @@ -326,12 +362,8 @@ def analyze(self, params, lang):
# self.__upload_database(params, db_name)
return issues

def subprocc_log(self, line):
""""""
logger.info(line)

def get_cmd(self, tool_path, options):
return __lu__().format_cmd(tool_path, options)
def get_cmd(self, tool_path, args):
return __lu__().format_cmd(tool_path, args)


tool = TcaQl
Expand Down