diff --git a/pw_ide/docs.rst b/pw_ide/docs.rst index eb16cb8afc..2663067bb8 100644 --- a/pw_ide/docs.rst +++ b/pw_ide/docs.rst @@ -15,3 +15,10 @@ these options can be configured: (by default this is `.pw_ide` in the project root). This directory shouldn't be committed to your repository or be a directory that is routinely deleted or manipulated by other processes. + +* ``targets``: A list of build targets to use for code analysis, which is likely + to be a subset of the project's total targets. The target name needs to match + the name of the directory that holds the build system artifacts for the + target. For example, GN outputs build artifacts for the + ``pw_strict_host_clang_debug`` target in a directory with that name in the + ``out`` directory. So that becomes the canonical name for that target. diff --git a/pw_ide/py/BUILD.gn b/pw_ide/py/BUILD.gn index d9af1dcb7a..7570575205 100644 --- a/pw_ide/py/BUILD.gn +++ b/pw_ide/py/BUILD.gn @@ -25,9 +25,15 @@ pw_python_package("py") { sources = [ "pw_ide/__init__.py", "pw_ide/__main__.py", + "pw_ide/cpp.py", + "pw_ide/exceptions.py", "pw_ide/settings.py", + "pw_ide/symlinks.py", + ] + tests = [ + "cpp_test.py", + "test_cases.py", ] - tests = [] python_deps = [ "$dir_pw_console/py" ] pylintrc = "$dir_pigweed/.pylintrc" } diff --git a/pw_ide/py/cpp_test.py b/pw_ide/py/cpp_test.py new file mode 100644 index 0000000000..b3557de86e --- /dev/null +++ b/pw_ide/py/cpp_test.py @@ -0,0 +1,640 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Tests for pw_ide.cpp""" + +import json +from pathlib import Path +from typing import List, Optional, Tuple, TypedDict, Union +import unittest +from unittest.mock import Mock, patch + +# pylint: disable=protected-access +from pw_ide.cpp import ( + _COMPDB_FILE_PREFIX, + _COMPDB_FILE_SEPARATOR, + _COMPDB_FILE_EXTENSION, + _COMPDB_CACHE_DIR_PREFIX, + _COMPDB_CACHE_DIR_SEPARATOR, + _target_and_executable_from_command, + compdb_generate_cache_file_path, + compdb_generate_file_path, + compdb_target_from_path, + CppCompilationDatabase, + CppCompileCommand, + CppCompileCommandDict, + InvalidTargetException, + MissingCompDbException, + aggregate_compilation_database_targets, + get_available_compdbs, + get_available_targets, + get_target, + process_compilation_database, + set_target, +) + +from test_cases import PwIdeTestCase + + +class _TargetAndExecutableFromCommandTestCase(TypedDict): + command: str + target: Optional[str] + executable: Optional[str] + + +class TestTargetAndExecutableFromCommand(unittest.TestCase): + """Tests _target_and_executable_from_command""" + def run_test(self, command: str, expected_target: Optional[str], + expected_executable: Optional[str]) -> None: + (target, executable) = _target_and_executable_from_command(command) + self.assertEqual(target, expected_target) + self.assertEqual(executable, expected_executable) + + def test_correct_target_and_executable_with_gn_compile_command( + self) -> None: + """Test output against typical GN-generated compile commands.""" + + cases: List[_TargetAndExecutableFromCommandTestCase] = [ + { + # pylint: disable=line-too-long + 'command': + 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'target': 'stm32f429i_disc1_debug', + 'executable': 'arm-none-eabi-g++', + }, + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/clang++ -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'target': 'pw_strict_host_clang_debug', + 'executable': 'clang++', + }, + { + # pylint: disable=line-too-long + 'command': + "python ../pw_toolchain/py/pw_toolchain/clang_tidy.py --source-exclude 'third_party/.*' --source-exclude '.*packages/mbedtls.*' --source-exclude '.*packages/boringssl.*' --skip-include-path 'mbedtls/include' --skip-include-path 'mbedtls' --skip-include-path 'boringssl/src/include' --skip-include-path 'boringssl' --skip-include-path 'pw_tls_client/generate_test_data' --source-file ../pw_allocator/freelist.cc --source-root '../' --export-fixes pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/freelist.freelist.cc.o.yaml -- ../environment/cipd/packages/pigweed/bin/clang++ END_OF_INVOKER -MMD -MF pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/freelist.freelist.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_containers/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/freelist.cc -o pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/freelist.freelist.cc.o && touch pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/freelist.freelist.cc.o", + # pylint: enable=line-too-long + 'target': None, + 'executable': 'python', + }, + { + 'command': '', + 'target': None, + 'executable': None, + }, + ] + + for case in cases: + self.run_test(case['command'], case['target'], case['executable']) + + +class TestCppCompileCommand(unittest.TestCase): + """Tests CppCompileCommand""" + def test_post_init_frozen_attrs_set(self) -> None: + command_dict = { + # pylint: disable=line-too-long + 'command': + 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + } + + expected_target = 'stm32f429i_disc1_debug' + expected_executable = 'arm-none-eabi-g++' + command = CppCompileCommand(**command_dict) + + self.assertEqual(command.target, expected_target) + self.assertEqual(command.executable, expected_executable) + + +class TestCppCompilationDatabase(PwIdeTestCase): + """Tests CppCompilationDatabase""" + def setUp(self): + self.fixture: List[CppCompileCommandDict] = [ + { + # pylint: disable=line-too-long + 'command': + 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/clang++ -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + ] + + return super().setUp() + + def test_load_from_dicts(self): + compdb = CppCompilationDatabase.load(self.fixture) + self.assertCountEqual(compdb.as_dicts(), self.fixture) + + def test_load_from_json(self): + compdb = CppCompilationDatabase.load(json.dumps(self.fixture)) + self.assertCountEqual(compdb.as_dicts(), self.fixture) + + def test_load_from_path(self): + with self.make_temp_file( + f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_EXTENSION}', + json.dumps(self.fixture)) as (_, file_path): + path = file_path + + compdb = CppCompilationDatabase.load(path) + self.assertCountEqual(compdb.as_dicts(), self.fixture) + + def test_load_from_file_handle(self): + with self.make_temp_file( + f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_EXTENSION}', + json.dumps(self.fixture)) as (file, _): + compdb = CppCompilationDatabase.load(file) + + self.assertCountEqual(compdb.as_dicts(), self.fixture) + + +class TestCompDbGenerateFilePath(unittest.TestCase): + """Tests compdb_generate_file_path""" + def test_with_target_includes_target(self) -> None: + name = 'foo' + actual = str(compdb_generate_file_path('foo')) + expected = (f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + f'{name}{_COMPDB_FILE_EXTENSION}') + self.assertEqual(actual, expected) + + def test_without_target_omits_target(self) -> None: + actual = str(compdb_generate_file_path()) + expected = f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_EXTENSION}' + self.assertEqual(actual, expected) + + +class TestCompDbGenerateCacheFilePath(unittest.TestCase): + """Tests compdb_generate_cache_file_path""" + def test_with_target_includes_target(self) -> None: + name = 'foo' + actual = str(compdb_generate_cache_file_path('foo')) + expected = (f'{_COMPDB_CACHE_DIR_PREFIX}' + f'{_COMPDB_CACHE_DIR_SEPARATOR}{name}') + self.assertEqual(actual, expected) + + def test_without_target_omits_target(self) -> None: + actual = str(compdb_generate_cache_file_path()) + expected = f'{_COMPDB_CACHE_DIR_PREFIX}' + self.assertEqual(actual, expected) + + +class _CompDbTargetFromPathTestCase(TypedDict): + path: str + target: Optional[str] + + +class TestCompDbTargetFromPath(unittest.TestCase): + """Tests compdb_target_from_path""" + def run_test(self, path: Path, expected_target: Optional[str]) -> None: + target = compdb_target_from_path(path) + self.assertEqual(target, expected_target) + + def test_correct_target_from_path(self) -> None: + """Test that the expected target is extracted from the file path.""" + cases: List[_CompDbTargetFromPathTestCase] = [ + { + 'path': + (f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + f'pw_strict_host_clang_debug{_COMPDB_FILE_EXTENSION}'), + 'target': + 'pw_strict_host_clang_debug' + }, + { + 'path': (f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + f'stm32f429i_disc1_debug{_COMPDB_FILE_EXTENSION}'), + 'target': + 'stm32f429i_disc1_debug' + }, + { + 'path': (f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + f'{_COMPDB_FILE_EXTENSION}'), + 'target': + None + }, + { + 'path': 'foompile_barmmands.json', + 'target': None + }, + { + 'path': 'foompile_barmmands_target_x.json', + 'target': None + }, + { + 'path': '', + 'target': None + }, + ] + + for case in cases: + self.run_test(Path(case['path']), case['target']) + + +class TestGetAvailableCompDbs(PwIdeTestCase): + """Tests get_avaliable_compdbs""" + def test_finds_all_compdbs(self) -> None: + """Test that get_available_compdbs finds all compilation databases.""" + + targets = [ + 'pw_strict_host_clang_debug', + 'stm32f429i_disc1_debug', + ] + + # Simulate a dir with n compilation databases, m < n cache dirs, and + # symlinks set up. + files_data: List[Tuple[Union[Path, str], str]] = \ + [(compdb_generate_file_path(target), '') for target in targets] + + files_data.append((compdb_generate_cache_file_path(targets[0]), '')) + files_data.append((compdb_generate_file_path(), '')) + files_data.append((compdb_generate_cache_file_path(), '')) + + expected = [ + (f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + f'{targets[0]}{_COMPDB_FILE_EXTENSION}', + f'{_COMPDB_CACHE_DIR_PREFIX}{_COMPDB_CACHE_DIR_SEPARATOR}' + f'{targets[0]}'), + (f'{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + f'{targets[1]}{_COMPDB_FILE_EXTENSION}', None), + ] + + settings = self.make_ide_settings(targets=targets) + + with self.make_temp_files(files_data): + found_compdbs = get_available_compdbs(settings) + + # Strip out the temp dir path data. + get_name = lambda p: p.name if p is not None else None + found_compdbs_str = [(get_name(file), get_name(cache)) + for file, cache in found_compdbs] + + self.assertCountEqual(found_compdbs_str, expected) + + +class TestGetAvailableTargets(PwIdeTestCase): + """Tests get_available_targets""" + def test_finds_all_targets(self) -> None: + targets = [ + 'pw_strict_host_clang_debug', + 'stm32f429i_disc1_debug', + ] + + settings = self.make_ide_settings(targets=targets) + + with self.make_temp_file(compdb_generate_file_path(targets[0])), \ + self.make_temp_file(compdb_generate_file_path(targets[1])): + + found_targets = get_available_targets(settings) + + self.assertCountEqual(found_targets, targets) + + +class TestGetTarget(PwIdeTestCase): + """Tests get_target""" + @patch('os.readlink') + def test_finds_valid_target(self, mock_readlink: Mock) -> None: + target = 'pw_strict_host_clang_debug' + settings = self.make_ide_settings(targets=[target]) + mock_readlink.return_value = (f'{_COMPDB_FILE_PREFIX}' + f'{_COMPDB_CACHE_DIR_SEPARATOR}' + f'{target}{_COMPDB_FILE_EXTENSION}') + found_target = get_target(settings) + self.assertEqual(found_target, target) + + @patch('os.readlink') + def test_finds_valid_target_nested(self, mock_readlink: Mock) -> None: + target = 'pw_strict_host_clang_debug' + settings = self.make_ide_settings(targets=[target]) + mock_readlink.return_value = (f'/x/y/z/{_COMPDB_FILE_PREFIX}' + f'{_COMPDB_CACHE_DIR_SEPARATOR}' + f'{target}{_COMPDB_FILE_EXTENSION}') + found_target = get_target(settings) + self.assertEqual(found_target, target) + + def test_returns_none_with_no_symlink(self) -> None: + target = 'pw_strict_host_clang_debug' + settings = self.make_ide_settings(targets=[target]) + found_target = get_target(settings) + self.assertIsNone(found_target) + + +class TestSetTarget(PwIdeTestCase): + """Tests set_target""" + @patch('os.remove') + @patch('os.mkdir') + @patch('os.symlink') + def test_sets_valid_target_when_no_target_set(self, mock_symlink: Mock, + mock_mkdir: Mock, + mock_remove: Mock) -> None: + """Test the case where no symlinks have been set.""" + + target = 'pw_strict_host_clang_debug' + settings = self.make_ide_settings(targets=[target]) + compdb_symlink_path = compdb_generate_file_path() + cache_symlink_path = compdb_generate_cache_file_path() + + with self.make_temp_file(compdb_generate_file_path(target)): + set_target(target, settings) + + mock_mkdir.assert_any_call( + self.path_in_temp_dir(compdb_generate_cache_file_path(target))) + + mock_symlink.assert_any_call(*self.paths_in_temp_dir( + compdb_generate_file_path(target), compdb_symlink_path)) + + mock_symlink.assert_any_call(*self.paths_in_temp_dir( + compdb_generate_cache_file_path(target), cache_symlink_path)) + + mock_remove.assert_not_called() + + @patch('os.remove') + @patch('os.mkdir') + @patch('os.symlink') + def test_sets_valid_target_when_target_already_set( + self, mock_symlink: Mock, mock_mkdir: Mock, + mock_remove: Mock) -> None: + """Test the case where symlinks have been set, and now we're setting + them to a different target.""" + + targets = [ + 'pw_strict_host_clang_debug', + 'stm32f429i_disc1_debug', + ] + + settings = self.make_ide_settings(targets=targets) + compdb_symlink_path = compdb_generate_file_path() + cache_symlink_path = compdb_generate_cache_file_path() + + # Set the first target, which should initalize the symlinks. + with self.make_temp_file(compdb_generate_file_path(targets[0])), \ + self.make_temp_file(compdb_generate_file_path(targets[1])): + + set_target(targets[0], settings) + + mock_mkdir.assert_any_call( + self.path_in_temp_dir( + compdb_generate_cache_file_path(targets[0]))) + + mock_remove.assert_not_called() + + # Simulate symlink creation + with self.make_temp_file(compdb_symlink_path), \ + self.make_temp_file(cache_symlink_path): + + # Set the second target, which should replace the symlinks + set_target(targets[1], settings) + + mock_mkdir.assert_any_call( + self.path_in_temp_dir( + compdb_generate_cache_file_path(targets[1]))) + + mock_remove.assert_any_call( + self.path_in_temp_dir(compdb_symlink_path)) + + mock_remove.assert_any_call( + self.path_in_temp_dir(cache_symlink_path)) + + mock_symlink.assert_any_call(*self.paths_in_temp_dir( + compdb_generate_file_path(targets[1]), + compdb_symlink_path)) + + mock_symlink.assert_any_call(*self.paths_in_temp_dir( + compdb_generate_cache_file_path(targets[1]), + cache_symlink_path)) + + @patch('os.remove') + @patch('os.mkdir') + @patch('os.symlink') + def test_sets_valid_target_back_and_forth(self, mock_symlink: Mock, + mock_mkdir: Mock, + mock_remove: Mock) -> None: + """Test the case where symlinks have been set, we set them to a second + target, and now we're setting them back to the first target.""" + + targets = [ + 'pw_strict_host_clang_debug', + 'stm32f429i_disc1_debug', + ] + + settings = self.make_ide_settings(targets=targets) + compdb_symlink_path = compdb_generate_file_path() + cache_symlink_path = compdb_generate_cache_file_path() + + # Set the first target, which should initalize the symlinks + with self.make_temp_file(compdb_generate_file_path(targets[0])), \ + self.make_temp_file(compdb_generate_file_path(targets[1])): + + set_target(targets[0], settings) + + # Simulate symlink creation + with self.make_temp_file(compdb_symlink_path), \ + self.make_temp_file(cache_symlink_path): + + # Set the second target, which should replace the symlinks + set_target(targets[1], settings) + + # Reset mocks to clear events prior to those under test + mock_symlink.reset_mock() + mock_mkdir.reset_mock() + mock_remove.reset_mock() + + # Set the first target again, which should also replace the + # symlinks and reuse the existing cache folder + set_target(targets[0], settings) + + mock_mkdir.assert_any_call( + self.path_in_temp_dir( + compdb_generate_cache_file_path(targets[0]))) + + mock_remove.assert_any_call( + self.path_in_temp_dir(compdb_symlink_path)) + + mock_remove.assert_any_call( + self.path_in_temp_dir(cache_symlink_path)) + + mock_symlink.assert_any_call(*self.paths_in_temp_dir( + compdb_generate_file_path(targets[0]), + compdb_symlink_path)) + + mock_symlink.assert_any_call(*self.paths_in_temp_dir( + compdb_generate_cache_file_path(targets[0]), + cache_symlink_path)) + + @patch('os.symlink') + def test_invalid_target_not_in_defined_targets_raises( + self, mock_symlink: Mock): + target = 'pw_strict_host_clang_debug' + settings = self.make_ide_settings(targets=[target]) + + with self.make_temp_file(compdb_generate_file_path(target)), \ + self.assertRaises(InvalidTargetException): + + set_target('foo', settings) + mock_symlink.assert_not_called() + + @patch('os.symlink') + def test_invalid_target_not_in_available_targets_raises( + self, mock_symlink: Mock): + target = 'pw_strict_host_clang_debug' + settings = self.make_ide_settings(targets=[target]) + + with self.assertRaises(MissingCompDbException): + set_target(target, settings) + mock_symlink.assert_not_called() + + +class TestAggregateCompilationDatabaseTargets(PwIdeTestCase): + """Tests aggregate_compilation_database_targets""" + def test_gets_all_legitimate_targets(self): + """Test compilation target aggregation against a typical sample of raw + output from GN.""" + + targets = [ + 'pw_strict_host_clang_debug', + 'stm32f429i_disc1_debug', + ] + + raw_db: List[CppCompileCommandDict] = [ + { + # pylint: disable=line-too-long + 'command': + 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/clang++ -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + { + # pylint: disable=line-too-long + 'command': + "python ../pw_toolchain/py/pw_toolchain/clang_tidy.py --source-exclude 'third_party/.*' --source-exclude '.*packages/mbedtls.*' --source-exclude '.*packages/boringssl.*' --skip-include-path 'mbedtls/include' --skip-include-path 'mbedtls' --skip-include-path 'boringssl/src/include' --skip-include-path 'boringssl' --skip-include-path 'pw_tls_client/generate_test_data' --source-file ../pw_allocator/block.cc --source-root '../' --export-fixes pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o.yaml -- ../environment/cipd/packages/pigweed/bin/clang++ END_OF_INVOKER -MMD -MF pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o && touch pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o", + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc', + }, + ] + + aggregated_targets = aggregate_compilation_database_targets(raw_db) + self.assertCountEqual(aggregated_targets, targets) + + +class TestProcessCompilationDatabase(PwIdeTestCase): + """Tests process_compilation_database""" + def test_compilation_database_processed_correctly(self): + """Test compilation database processing against a typical sample of + raw output from GN.""" + + targets = [ + 'pw_strict_host_clang_debug', + 'stm32f429i_disc1_debug', + 'isosceles_debug', + ] + + settings = self.make_ide_settings(targets=targets) + + raw_db: List[CppCompileCommandDict] = [ + { + # pylint: disable=line-too-long + 'command': + 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/isosceles-clang++ -MMD -MF isosceles_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/clang++ -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + { + # pylint: disable=line-too-long + 'command': + "python ../pw_toolchain/py/pw_toolchain/clang_tidy.py --source-exclude 'third_party/.*' --source-exclude '.*packages/mbedtls.*' --source-exclude '.*packages/boringssl.*' --skip-include-path 'mbedtls/include' --skip-include-path 'mbedtls' --skip-include-path 'boringssl/src/include' --skip-include-path 'boringssl' --skip-include-path 'pw_tls_client/generate_test_data' --source-file ../pw_allocator/block.cc --source-root '../' --export-fixes pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o.yaml -- ../environment/cipd/packages/pigweed/bin/clang++ END_OF_INVOKER -MMD -MF pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o && touch pw_strict_host_clang_debug.static_analysis/obj/pw_allocator/block.block.cc.o", + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc', + }, + ] + + expected_compdbs = { + 'isosceles_debug': [ + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/isosceles-clang++ -MMD -MF isosceles_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + ], + 'pw_strict_host_clang_debug': [ + { + # pylint: disable=line-too-long + 'command': + '../environment/cipd/packages/pigweed/bin/clang++ -MMD -MF pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o.d -g3 --sysroot=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -Og -Wshadow -Wredundant-decls -Wthread-safety -Wswitch-enum -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -Wextra-semi -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -DPW_STATUS_CFG_CHECK_IF_USED=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/print_and_abort_assert_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o pw_strict_host_clang_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + ], + 'stm32f429i_disc1_debug': [ + { + # pylint: disable=line-too-long + 'command': + 'arm-none-eabi-g++ -MMD -MF stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o.d -Wno-psabi -mabi=aapcs -mthumb --sysroot=../environment/cipd/packages/arm -specs=nano.specs -specs=nosys.specs -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -Wshadow -Wredundant-decls -u_printf_float -fdiagnostics-color -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Wimplicit-fallthrough -Wcast-qual -Wundef -Wpointer-arith -Werror -Wno-error=cpp -Wno-error=deprecated-declarations -ffile-prefix-map=/pigweed/pigweed/out=out -ffile-prefix-map=/pigweed/pigweed/= -ffile-prefix-map=../= -ffile-prefix-map=/pigweed/pigweed/out=out -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register -DPW_ARMV7M_ENABLE_FPU=1 -I../pw_allocator/public -I../pw_assert/public -I../pw_assert/assert_compatibility_public_overrides -I../pw_preprocessor/public -I../pw_assert_basic/public_overrides -I../pw_assert_basic/public -I../pw_span/public -I../pw_polyfill/public -I../pw_polyfill/standard_library_public -I../pw_status/public -c ../pw_allocator/block.cc -o stm32f429i_disc1_debug/obj/pw_allocator/block.block.cc.o', + # pylint: enable=line-too-long + 'directory': '/pigweed/pigweed/out', + 'file': '../pw_allocator/block.cc' + }, + ], + } + + compdbs = process_compilation_database(raw_db, settings) + compdbs_as_dicts = { + target: compdb.as_dicts() + for target, compdb in compdbs.items() + } + self.assertDictEqual(compdbs_as_dicts, expected_compdbs) + + +if __name__ == '__main__': + unittest.main() diff --git a/pw_ide/py/pw_ide/cpp.py b/pw_ide/py/pw_ide/cpp.py new file mode 100644 index 0000000000..d9f8a2d457 --- /dev/null +++ b/pw_ide/py/pw_ide/cpp.py @@ -0,0 +1,439 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Configure C/C++ IDE support for Pigweed projects.""" + +from collections import defaultdict +from dataclasses import dataclass, field +from io import TextIOBase +import json +import os +from pathlib import Path +import re +from typing import (cast, Dict, Generator, List, Optional, Tuple, TypedDict, + Union) + +from pw_ide.exceptions import (BadCompDbException, InvalidTargetException, + MissingCompDbException) + +from pw_ide.settings import IdeSettings +from pw_ide.symlinks import set_symlink + +_COMPDB_FILE_PREFIX = 'compile_commands' +_COMPDB_FILE_SEPARATOR = '_' +_COMPDB_FILE_EXTENSION = '.json' + +_COMPDB_CACHE_DIR_PREFIX = '.cache' +_COMPDB_CACHE_DIR_SEPARATOR = '_' + +_SUPPORTED_TOOLCHAIN_EXECUTABLES = ('clang', 'gcc', 'g++') + +COMPDB_FILE_GLOB = f'{_COMPDB_FILE_PREFIX}*{_COMPDB_FILE_EXTENSION}' +COMPDB_CACHE_DIR_GLOB = f'{_COMPDB_CACHE_DIR_PREFIX}*' + +CLANGD_WRAPPER_FILE_NAME = 'clangd' + + +def _target_and_executable_from_command( + command: str) -> Tuple[Optional[str], Optional[str]]: + """Extract the target and executable name from a compile command.""" + + tokens = command.split(' ') + target: Optional[str] = None + executable: Optional[str] = Path(tokens[0]).name + executable = executable if executable != '' else None + + if len(tokens) > 1: + for token in tokens[1:]: + # Skip all flags and whitespace until we find the first reference to + # the actual file in the command. The top level directory of the + # file is the target name. + # TODO(chadnorvell): This might be too specific to GN. + if not token.startswith('-') and not token.strip() == '': + target = Path(token).parts[0] + break + + # This is indicative of Python wrapper commands, but is also an artifact of + # the unsophisticated way we extract the target here. + if target in ('.', '..'): + target = None + + return (target, executable) + + +class CppCompileCommandDict(TypedDict): + file: str + directory: str + command: str + + +@dataclass(frozen=True) +class CppCompileCommand: + """A representation of a clang compilation database compile command. + + See: https://clang.llvm.org/docs/JSONCompilationDatabase.html + """ + + file: str + directory: str + command: str + target: Optional[str] = field(default=None, init=False) + executable: Optional[str] = field(default=None, init=False) + + def __post_init__(self) -> None: + (target, + executable) = _target_and_executable_from_command(self.command) + + # We want this class to be essentially immutable, accomplished + # by freezing it. But that means we need to resort to this + # to set these attributes during init. + object.__setattr__(self, 'executable', executable) + object.__setattr__(self, 'target', target) + + def as_dict(self) -> CppCompileCommandDict: + return { + "file": self.file, + "directory": self.directory, + "command": self.command, + } + + +LoadableToCppCompilationDatabase = Union[List[CppCompileCommandDict], str, + TextIOBase, Path] + + +class CppCompilationDatabase: + """A representation of a clang compilation database. + + See: https://clang.llvm.org/docs/JSONCompilationDatabase.html + """ + def __init__(self) -> None: + self._db: List[CppCompileCommand] = [] + + def __len__(self) -> int: + return len(self._db) + + def __getitem__(self, index) -> CppCompileCommand: + return self._db[index] + + def __iter__(self) -> Generator[CppCompileCommand, None, None]: + return (compile_command for compile_command in self._db) + + def add(self, command: CppCompileCommand): + """Add a compile command to the compilation database.""" + + self._db.append(command) + + def as_dicts(self) -> List[CppCompileCommandDict]: + return [compile_command.as_dict() for compile_command in self._db] + + def to_json(self) -> str: + """Output the compilation database to a JSON string.""" + + return json.dumps(self.as_dicts(), indent=2, sort_keys=True) + + def to_file(self, path: Path): + """Write the compilation database to a JSON file.""" + + with open(path, 'w') as file: + json.dump(self.as_dicts(), file, indent=2, sort_keys=True) + + @classmethod + def load( + cls, compdb_to_load: LoadableToCppCompilationDatabase + ) -> 'CppCompilationDatabase': + """Load a compilation database. + + You can provide a JSON file handle or path, a JSON string, or a native + Python data structure that matches the format (list of dicts). + """ + + db_as_dicts: List[CppCompileCommandDict] + + if isinstance(compdb_to_load, list): + # The provided data is already in the format we want it to be in, + # probably, and if it isn't we'll find out when we try to + # instantiate the database. + db_as_dicts = compdb_to_load + else: + if isinstance(compdb_to_load, Path): + # The provided data is a path to a file, presumably JSON. + try: + compdb_data = compdb_to_load.read_text() + except FileNotFoundError: + raise MissingCompDbException() + elif isinstance(compdb_to_load, TextIOBase): + # The provided data is a file handle, presumably JSON. + compdb_data = compdb_to_load.read() + elif isinstance(compdb_to_load, str): + # The provided data is a a string, presumably JSON. + compdb_data = compdb_to_load + + db_as_dicts = json.loads(compdb_data) + + compdb = cls() + + try: + compdb._db = [ + CppCompileCommand(**compile_command) + for compile_command in db_as_dicts + ] + except TypeError: + # This will arise if db_as_dicts is not actually a list of dicts + raise BadCompDbException() + + return compdb + + +def compdb_generate_file_path(target: str = '') -> Path: + """Generate a compilation database file path.""" + + path = Path(f'{_COMPDB_FILE_PREFIX}.json') + + if target: + path = path.with_stem(f'{_COMPDB_FILE_PREFIX}' + f'{_COMPDB_FILE_SEPARATOR}{target}') + + return path + + +def compdb_generate_cache_file_path(target: str = '') -> Path: + """Generate a compilation database cache directory path.""" + + path = Path(f'{_COMPDB_CACHE_DIR_PREFIX}') + + if target: + path = path.with_stem(f'{_COMPDB_CACHE_DIR_PREFIX}' + f'{_COMPDB_CACHE_DIR_SEPARATOR}{target}') + + return path + + +def compdb_target_from_path(filename: Path) -> Optional[str]: + """Given a path that contains a compilation database file name, return the + name of the database's compilation target.""" + + # The length of the common compilation database file name prefix + prefix_length = len(_COMPDB_FILE_PREFIX) + len(_COMPDB_FILE_SEPARATOR) + + if len(filename.stem) <= prefix_length: + return None + + if filename.stem[:prefix_length] != (_COMPDB_FILE_PREFIX + + _COMPDB_FILE_SEPARATOR): + return None + + return filename.stem[prefix_length:] + + +def _none_to_empty_str(value: Optional[str]) -> str: + return value if value is not None else '' + + +def _none_if_not_exists(path: Path) -> Optional[Path]: + return path if path.exists() else None + + +def _compdb_cache_path_if_exists(working_dir: Path, + target: Optional[str]) -> Optional[Path]: + return _none_if_not_exists( + working_dir / + compdb_generate_cache_file_path(_none_to_empty_str(target))) + + +def get_available_compdbs( + settings: IdeSettings) -> List[Tuple[Path, Optional[Path]]]: + """Return the paths of all compilations databases and their associated + caches that exist in the working directory as tuples.""" + compdbs_with_targets = ( + (file_path, compdb_target_from_path(file_path)) + for file_path in settings.working_dir.iterdir() + if file_path.match(f'{_COMPDB_FILE_PREFIX}*{_COMPDB_FILE_EXTENSION}')) + + compdbs_with_caches = [] + + for file_path, target in compdbs_with_targets: + if file_path.name != compdb_generate_file_path().name: + compdbs_with_caches.append( + (file_path, + _compdb_cache_path_if_exists(settings.working_dir, target))) + + return compdbs_with_caches + + +def get_available_targets(settings: IdeSettings) -> List[str]: + """Get the names of all targets available for code analysis. + + The presence of compilation database files matching the expected filename + format in the expected directory is the source of truth on what targets + are available. + """ + match_expr = (fr'^{_COMPDB_FILE_PREFIX}{_COMPDB_FILE_SEPARATOR}' + fr'(\w+){_COMPDB_FILE_EXTENSION}$') + + targets = [] + + for filename in settings.working_dir.iterdir(): + match = re.match(match_expr, filename.name) + if match is not None: + targets.append(match.group(1)) + + return targets + + +def get_defined_available_targets(settings: IdeSettings) -> List[str]: + """Get the names of all targets that are both available for code analysis + and defined in the settings file as targets that should be visible to the + user.""" + available_targets = get_available_targets(settings) + + if len(settings.targets) == 0: + return available_targets + + return [ + target for target in available_targets if target in settings.targets + ] + + +def _is_available_target(target: Optional[str], settings: IdeSettings) -> bool: + """Determines if a target is available for code analysis. + + Availability is defined by the presence of a compilation database for the + target in the working directory. + """ + return target is not None and target in get_available_targets(settings) + + +def _is_valid_target(target: Optional[str], settings: IdeSettings) -> bool: + """Determines if a target can be used for code analysis. + + By default, any target is valid. But the project or user settings can + constrain the valid targets to some subset of available targets (e.g. to + hide variations on the same target that are irrelevant to code analysis). + """ + return target is not None and (len(settings.targets) == 0 + or target in settings.targets) + + +def _is_valid_executable(executable: Optional[str]) -> bool: + """Determines if a compiler executable is valid for code analysis. + + We assume it is if the executable name contains the name of one of the + declared supported toolchains. + """ + if executable is None: + return False + + for supported_executable in _SUPPORTED_TOOLCHAIN_EXECUTABLES: + if supported_executable in executable: + return True + + return False + + +def _is_valid_target_and_executable(compile_command: CppCompileCommand, + settings: IdeSettings) -> bool: + """Determines if a compile command has a target and executable combination + that can be used with code analysis.""" + + return _is_valid_target(compile_command.target, + settings) and (_is_valid_executable( + compile_command.executable)) + + +def get_target(settings: IdeSettings) -> Optional[str]: + """Get the name of the current target used for code analysis. + + The presence of a symlink with the expected filename pointing to a + compilation database matching the expected filename format is the source of + truth on what the current target is. + """ + try: + src_file = (settings.working_dir / + compdb_generate_file_path()).readlink() + except (FileNotFoundError, OSError): + # If the symlink doesn't exist, there is no current target. + return None + + return compdb_target_from_path(Path(src_file)) + + +def set_target(target: str, settings: IdeSettings) -> None: + """Set the target that will be used for code analysis.""" + + if not _is_valid_target(target, settings): + raise InvalidTargetException() + + compdb_symlink_path = settings.working_dir / compdb_generate_file_path() + + compdb_target_path = (settings.working_dir / + compdb_generate_file_path(target)) + + if not compdb_target_path.exists(): + raise MissingCompDbException() + + set_symlink(compdb_target_path, compdb_symlink_path) + + cache_symlink_path = (settings.working_dir / + compdb_generate_cache_file_path()) + + cache_target_path = (settings.working_dir / + compdb_generate_cache_file_path(target)) + + if not cache_target_path.exists(): + os.mkdir(cache_target_path) + + set_symlink(cache_target_path, cache_symlink_path) + + +def aggregate_compilation_database_targets( + compdb_file: LoadableToCppCompilationDatabase) -> List[str]: + """Given a clang compilation database, return all unique targets.""" + + compdb = CppCompilationDatabase.load(compdb_file) + targets = set() + + for compile_command in compdb: + if compile_command.target is not None: + targets.add(compile_command.target) + + return list(targets) + + +def process_compilation_database( + compdb_file: LoadableToCppCompilationDatabase, + settings: IdeSettings) -> Dict[str, CppCompilationDatabase]: + """Given a clang compilation database that may have commands for multiple + valid or invalid targets/toolchains, keep only the valid compile commands + and store them in target-specific compilation databases.""" + + raw_compdb = CppCompilationDatabase.load(compdb_file) + clean_compdbs: Dict[str, CppCompilationDatabase] = ( + defaultdict(CppCompilationDatabase)) + + for compile_command in raw_compdb: + if _is_valid_target_and_executable(compile_command, settings): + # If target is None, we won't arrive here. + target = cast(str, compile_command.target) + clean_compdbs[target].add(compile_command) + + return clean_compdbs + + +def write_compilation_databases(compdbs: Dict[str, CppCompilationDatabase], + settings: IdeSettings) -> None: + """Write compilation databases to target-specific JSON files.""" + + for target, compdb in compdbs.items(): + compdb.to_file(settings.working_dir / + compdb_generate_file_path(target)) diff --git a/pw_ide/py/pw_ide/exceptions.py b/pw_ide/py/pw_ide/exceptions.py new file mode 100644 index 0000000000..b805495890 --- /dev/null +++ b/pw_ide/py/pw_ide/exceptions.py @@ -0,0 +1,30 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""pw_ide exceptions.""" + + +class UnsupportedPlatformException(Exception): + """Raised when an action is attempted on an unsupported platform.""" + + +class InvalidTargetException(Exception): + """Exception for invalid compilation targets.""" + + +class BadCompDbException(Exception): + """Exception for compliation databases that don't conform to the format.""" + + +class MissingCompDbException(Exception): + """Exception for missing compilation database files.""" diff --git a/pw_ide/py/pw_ide/settings.py b/pw_ide/py/pw_ide/settings.py index a2a07df6fa..fb7cd40fa1 100644 --- a/pw_ide/py/pw_ide/settings.py +++ b/pw_ide/py/pw_ide/settings.py @@ -15,7 +15,7 @@ import os from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union from pw_console.yaml_config_loader_mixin import YamlConfigLoaderMixin @@ -25,6 +25,7 @@ os.path.expandvars('$PW_PROJECT_ROOT')) / PW_IDE_DIR_NAME _DEFAULT_CONFIG = { + 'targets': [], 'working_dir': _PW_IDE_DEFAULT_DIR, } @@ -61,3 +62,13 @@ def working_dir(self) -> Path: committed to the code repo. """ return Path(self._config.get('working_dir', '')) + + @property + def targets(self) -> List[str]: + """The list of targets that should be made available for code analysis. + + In this case, "target" is analogous to a GN target, i.e., a particular + build configuration. Targets defined here will be used when processing + a compilation database. + """ + return self._config.get('targets', list()) diff --git a/pw_ide/py/pw_ide/symlinks.py b/pw_ide/py/pw_ide/symlinks.py new file mode 100644 index 0000000000..af623b3479 --- /dev/null +++ b/pw_ide/py/pw_ide/symlinks.py @@ -0,0 +1,24 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Tools for managing symlinks.""" + +import os +from pathlib import Path + + +def set_symlink(target_path: Path, symlink_path: Path) -> None: + if symlink_path.exists(): + os.remove(symlink_path) + + os.symlink(target_path, symlink_path) diff --git a/pw_ide/py/test_cases.py b/pw_ide/py/test_cases.py new file mode 100644 index 0000000000..7da04c6c57 --- /dev/null +++ b/pw_ide/py/test_cases.py @@ -0,0 +1,124 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""pw_ide test classes.""" + +from contextlib import contextmanager +from io import TextIOWrapper +from pathlib import Path +import tempfile +from typing import Generator, List, Optional, Tuple, Union +import unittest + +from pw_ide.settings import IdeSettings + + +class TempDirTestCase(unittest.TestCase): + """Run tests that need access to a temporary directory.""" + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.temp_dir_path = Path(self.temp_dir.name) + + def tearDown(self) -> None: + self.temp_dir.cleanup() + return super().tearDown() + + @contextmanager + def make_temp_file( + self, + filename: Union[Path, str], + content: str = '' + ) -> Generator[Tuple[TextIOWrapper, Path], None, None]: + """Create a temp file in the test case's temp dir. + + Returns a tuple containing the file reference and the file's path. + The file can be read immediately. + """ + path = self.temp_dir_path / filename + + with open(path, 'a+', encoding='utf-8') as file: + file.write(content) + file.flush() + file.seek(0) + yield (file, path) + + @contextmanager + def make_temp_files( + self, files_data: List[Tuple[Union[Path, str], str]] + ) -> Generator[List[TextIOWrapper], None, None]: + """Create several temp files in the test case's temp dir. + + Provide a list of file name and content tuples. Saves you the trouble + of excessive `with self.make_temp_file, self.make_temp_file...` + nesting, and allows programmatic definition of multiple temp file + contexts. Files can be read immediately. + """ + files: List[TextIOWrapper] = [] + + for filename, content in files_data: + file = open(self.path_in_temp_dir(filename), + 'a+', + encoding='utf-8') + file.write(content) + file.flush() + file.seek(0) + files.append(file) + + yield files + + for file in files: + file.close() + + def path_in_temp_dir(self, path: Union[Path, str]) -> Path: + """Place a path into the test case's temp dir. + + This only works with a relative path; with an absolute path, this is a + no-op. + """ + return self.temp_dir_path / path + + def paths_in_temp_dir(self, *paths: Union[Path, str]) -> List[Path]: + """Place several paths into the test case's temp dir. + + This only works with relative paths; with absolute paths, this is a + no-op. + """ + return [self.path_in_temp_dir(path) for path in paths] + + +class PwIdeTestCase(TempDirTestCase): + """A test case for testing `pw_ide`. + + Provides a temp dir for testing file system actions and access to IDE + settings that wrap the temp dir. + """ + def make_ide_settings(self, + working_dir: Optional[Union[str, Path]] = None, + targets: Optional[List[str]] = None) -> IdeSettings: + """Make settings that wrap provided paths in the temp path.""" + + if working_dir is not None: + working_dir_path = self.path_in_temp_dir(working_dir) + else: + working_dir_path = self.temp_dir_path + + if targets is None: + targets = [] + + return IdeSettings(False, + False, + False, + default_config={ + 'working_dir': str(working_dir_path), + 'targets': targets, + })