Various Lint fixes (#2041)

ci/iwyu/mappings.imp

@@ -30,6 +30,8 @@
 { "include": ["\"cuda_runtime_api.h\"", "private", "<cuda_runtime.h>", "public"] },
 { "include": ["<driver_types.h>", "private", "<cuda_runtime.h>", "public"] },
 { "include": ["\"driver_types.h\"", "private", "<cuda_runtime.h>", "public"] },
+{ "include": ["\"cuda/__memory_resource/properties.h\"", "private", "<cuda/memory_resource>", "public"] },
+{ "include": ["\"cuda/__memory_resource/resource_ref.h\"", "private", "<cuda/memory_resource>", "public"] },
 
 # gtest
 { "include": ["@<gtest/gtest-.*>", "private", "<gtest/gtest.h>", "public"] },

examples/developer_guide/3_simple_cpp_stage/src/simple_cpp_stage/_lib/pass_thru.hpp

@@ -17,11 +17,11 @@
 
 #pragma once
 
-#include <morpheus/export.h>            // for exporting symbols
+#include <morpheus/export.h>              // for exporting symbols
 #include <morpheus/messages/control.hpp>  // for ControlMessage
-#include <mrc/segment/builder.hpp>      // for Segment Builder
-#include <mrc/segment/object.hpp>       // for Segment Object
-#include <pymrc/node.hpp>               // for PythonNode
+#include <mrc/segment/builder.hpp>        // for Segment Builder
+#include <mrc/segment/object.hpp>         // for Segment Object
+#include <pymrc/node.hpp>                 // for PythonNode
 #include <rxcpp/rx.hpp>
 
 #include <memory>

python/morpheus/morpheus/_lib/include/morpheus/objects/memory_descriptor.hpp

@@ -19,10 +19,9 @@
 
 #include "morpheus/export.h"
 
+#include <cuda/memory_resource>
 #include <rmm/cuda_stream_view.hpp>
 
-#include "cuda/memory_resource"
-
 namespace morpheus {
 /**
  * @addtogroup objects

python/morpheus/morpheus/_lib/include/morpheus/utilities/json_types.hpp

@@ -24,11 +24,12 @@
 #include <pybind11/pytypes.h>           // for object
 #include <pymrc/types.hpp>              // for PyHolder
 
-#include <cstdint>  // for int64_t, uint64_t, uint8_t
-#include <map>      // for map
-#include <string>   // for allocator, string
-#include <utility>  // for pair
-#include <vector>   // for vector
+#include <algorithm>  // for copy (implied usage)
+#include <cstdint>    // for int64_t, uint64_t, uint8_t
+#include <map>        // for map
+#include <string>     // for allocator, string
+#include <utility>    // for pair
+#include <vector>     // for vector
 
 namespace morpheus {
 namespace utilities {

python/morpheus/morpheus/_lib/include/morpheus/utilities/stage_util.hpp

@@ -18,6 +18,7 @@
 #pragma once
 
 #include <algorithm>
+#include <vector>
 
 namespace morpheus {
 /****** Component public free function implementations************/

python/morpheus/morpheus/_lib/src/io/deserializers.cpp

@@ -39,10 +39,10 @@
 // IWYU pragma: no_include <pybind11/gil.h>
 
 namespace {
-const std::regex IndexRegex(R"(^\s*(unnamed: 0|id)\s*$)",
-                            std::regex_constants::ECMAScript | std::regex_constants::icase);
+const std::regex INDEX_REGEX(R"(^\s*(unnamed: 0|id)\s*$)",
+                             std::regex_constants::ECMAScript | std::regex_constants::icase);
 
-const std::regex UnnamedRegex(R"(^\s*unnamed: 0\s*$)", std::regex_constants::ECMAScript | std::regex_constants::icase);
+const std::regex UNNAMED_REGEX(R"(^\s*unnamed: 0\s*$)", std::regex_constants::ECMAScript | std::regex_constants::icase);
 }  // namespace
 
 namespace morpheus {
@@ -118,7 +118,7 @@ int get_index_col_count(const cudf::io::table_with_metadata& data_table)
         const auto& col_name = names[0];
 
         // Check it against some common terms
-        if (std::regex_search(col_name, IndexRegex))
+        if (std::regex_search(col_name, INDEX_REGEX))
         {
             index_col_count = 1;
         }
@@ -136,7 +136,7 @@ int prepare_df_index(cudf::io::table_with_metadata& data_table)
         auto& col_name = data_table.metadata.schema_info[0].name;
 
         // Also, if its the hideous 'Unnamed: 0', then just use an empty string
-        if (std::regex_search(col_name, UnnamedRegex))
+        if (std::regex_search(col_name, UNNAMED_REGEX))
         {
             col_name.clear();
         }

python/morpheus/morpheus/_lib/src/objects/dtype.cpp

@@ -28,7 +28,7 @@
 #include <utility>
 
 namespace {
-const std::map<char, std::map<size_t, morpheus::TypeId>> StrToTypeId = {
+const std::map<char, std::map<size_t, morpheus::TypeId>> STR_TO_TYPE_ID = {
     {'b', {{1, morpheus::TypeId::BOOL8}}},
 
     {'i',
@@ -234,9 +234,9 @@ DType DType::from_numpy(const std::string& numpy_str)
     }
 
     // Now lookup in the map
-    auto found_type = StrToTypeId.find(type_char);
+    auto found_type = STR_TO_TYPE_ID.find(type_char);
 
-    if (found_type == StrToTypeId.end())
+    if (found_type == STR_TO_TYPE_ID.end())
     {
         throw std::invalid_argument(MORPHEUS_CONCAT_STR("Type char '" << type_char << "' not supported"));
     }

python/morpheus/morpheus/_lib/src/objects/mutable_table_ctx_mgr.cpp

@@ -20,6 +20,7 @@
 #include "morpheus/utilities/string_util.hpp"
 
 #include <pybind11/gil.h>
+#include <pybind11/pybind11.h>  // for attribute_error
 #include <pybind11/pytypes.h>
 
 #include <memory>

python/morpheus/morpheus/_lib/src/objects/rmm_tensor.cpp

@@ -29,7 +29,7 @@
 #include <rmm/cuda_stream_view.hpp>  // for cuda_stream_per_thread
 #include <rmm/device_buffer.hpp>
 
-#include <algorithm>  // for copy, transform
+#include <algorithm>   // for copy, transform
 #include <functional>  // for multiplies, plus, minus
 #include <iterator>    // for back_insert_iterator, back_inserter
 #include <memory>

python/morpheus/morpheus/_lib/src/objects/tensor.cpp

@@ -27,6 +27,7 @@
 #include <mrc/cuda/common.hpp>  // for MRC_CHECK_CUDA
 #include <rmm/device_buffer.hpp>
 
+#include <algorithm>  // needed by get_element_stride
 #include <memory>
 #include <string>
 #include <utility>  // for move

python/morpheus/morpheus/_lib/src/utilities/table_util.cpp

@@ -26,8 +26,8 @@
 
 #include <algorithm>  // for find, transform
 #include <filesystem>
-#include <iterator>  // for back_insert_iterator, back_inserter
-#include <memory>    // for unique_ptr
+#include <iterator>   // for back_insert_iterator, back_inserter
+#include <memory>     // for unique_ptr
 #include <ostream>    // needed for logging
 #include <stdexcept>  // for runtime_error
 

python/morpheus/morpheus/_lib/tests/messages/test_message_meta.cpp

@@ -26,8 +26,8 @@
 #include "morpheus/objects/tensor.hpp"      // for Tensor
 #include "morpheus/types.hpp"               // for RangeType
 
-#include <gtest/gtest.h>             // for TestInfo, TEST_F
-#include <pybind11/gil.h>            // for gil_scoped_release
+#include <gtest/gtest.h>   // for TestInfo, TEST_F
+#include <pybind11/gil.h>  // for gil_scoped_release
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
 #include <rmm/cuda_stream_view.hpp>  // for cuda_stream_per_thread

python/morpheus/morpheus/_lib/tests/test_deserializers.cpp

@@ -30,8 +30,8 @@
 using namespace morpheus;
 
 namespace {
-const std::string UserCols{",time,eventID,eventSource,eventName,z_loss\n"};
-const std::string DataRow{"0,1,2,test,test,5\n"};
+const std::string USER_COLS{",time,eventID,eventSource,eventName,z_loss\n"};
+const std::string DATA_ROW{"0,1,2,test,test,5\n"};
 }  // namespace
 
 TEST_CLASS(Deserializers);
@@ -65,7 +65,7 @@ TEST_F(TestDeserializers, GetIndexColCountNoIdxSimilarName)
     for (const auto& col : not_id_cols)
     {
         // Build a list of column names with `col` as our first element followed by the columns in `UserCols`
-        std::string csv_data{col + UserCols + DataRow};
+        std::string csv_data{col + USER_COLS + DATA_ROW};
 
         auto options = cudf::io::csv_reader_options::builder(cudf::io::source_info{csv_data.c_str(), csv_data.size()});
         auto table   = cudf::io::read_csv(options.build());
@@ -83,7 +83,7 @@ TEST_F(TestDeserializers, GetIndexColCountIdx)
     for (const auto& col : id_cols)
     {
         // Build a list of column names with `col` as our first element followed by the columns in `UserCols`
-        std::string csv_data{col + UserCols + DataRow};
+        std::string csv_data{col + USER_COLS + DATA_ROW};
 
         auto options = cudf::io::csv_reader_options::builder(cudf::io::source_info{csv_data.c_str(), csv_data.size()});
         auto table   = cudf::io::read_csv(options.build());
@@ -95,8 +95,8 @@ TEST_F(TestDeserializers, GetIndexColCountIdx)
 TEST_F(TestDeserializers, GetIndexColCountValidNameInvalidType)
 {
     // Construct a csv with two rows, first row will contain the expected int id value second row will be a string
-    std::string data_rows{DataRow + "s,6,7,frog,toad,8\n"};
-    std::string csv_data{"id" + UserCols + data_rows};
+    std::string data_rows{DATA_ROW + "s,6,7,frog,toad,8\n"};
+    std::string csv_data{"id" + USER_COLS + data_rows};
 
     auto options = cudf::io::csv_reader_options::builder(cudf::io::source_info{csv_data.c_str(), csv_data.size()});
     auto table   = cudf::io::read_csv(options.build());

python/morpheus/morpheus/_lib/tests/test_dev_mem_info.cpp

@@ -40,11 +40,11 @@
 using namespace morpheus;
 
 namespace {
-const TensorIndex Rows = 20;
-const TensorIndex Cols = 5;
-const auto Dtype       = DType::create<float>();
+const TensorIndex ROWS = 20;
+const TensorIndex COLS = 5;
+const auto DTYPE       = DType::create<float>();
 
-const auto ByteSize = Rows * Cols * Dtype.item_size();
+const auto BYTE_SIZE = ROWS * COLS * DTYPE.item_size();
 
 template <template <typename> class ResourceT>
 auto make_mem_resource()
@@ -59,31 +59,31 @@ TEST_F(TestDevMemInfo, RmmBufferConstructor)
 {
     // Explicitly passing the non-default values to make sure we don't have a test that accidentally passes
     auto mem_resource = make_mem_resource<rmm::mr::fixed_size_memory_resource>();
-    auto buffer       = std::make_shared<rmm::device_buffer>(ByteSize, rmm::cuda_stream_legacy, mem_resource.get());
+    auto buffer       = std::make_shared<rmm::device_buffer>(BYTE_SIZE, rmm::cuda_stream_legacy, mem_resource.get());
 
     // Set the offset to the second row in the buffer
-    DevMemInfo dm{buffer, Dtype, {Rows - 1, Cols}, {1, Rows}, Dtype.item_size()};
+    DevMemInfo dm{buffer, DTYPE, {ROWS - 1, COLS}, {1, ROWS}, DTYPE.item_size()};
 
-    EXPECT_EQ(dm.bytes(), (Rows - 1) * Cols * Dtype.item_size());
-    EXPECT_EQ(dm.count(), (Rows - 1) * Cols);
-    EXPECT_EQ(dm.offset_bytes(), Dtype.item_size());
+    EXPECT_EQ(dm.bytes(), (ROWS - 1) * COLS * DTYPE.item_size());
+    EXPECT_EQ(dm.count(), (ROWS - 1) * COLS);
+    EXPECT_EQ(dm.offset_bytes(), DTYPE.item_size());
 
-    EXPECT_EQ(dm.dtype(), Dtype);
-    EXPECT_EQ(dm.type_id(), Dtype.type_id());
+    EXPECT_EQ(dm.dtype(), DTYPE);
+    EXPECT_EQ(dm.type_id(), DTYPE.type_id());
 
     EXPECT_EQ(dm.shape().size(), 2);
-    EXPECT_EQ(dm.shape()[0], Rows - 1);
-    EXPECT_EQ(dm.shape(0), Rows - 1);
-    EXPECT_EQ(dm.shape()[1], Cols);
-    EXPECT_EQ(dm.shape(1), Cols);
+    EXPECT_EQ(dm.shape()[0], ROWS - 1);
+    EXPECT_EQ(dm.shape(0), ROWS - 1);
+    EXPECT_EQ(dm.shape()[1], COLS);
+    EXPECT_EQ(dm.shape(1), COLS);
 
     EXPECT_EQ(dm.stride().size(), 2);
     EXPECT_EQ(dm.stride()[0], 1);
     EXPECT_EQ(dm.stride(0), 1);
-    EXPECT_EQ(dm.stride()[1], Rows);
-    EXPECT_EQ(dm.stride(1), Rows);
+    EXPECT_EQ(dm.stride()[1], ROWS);
+    EXPECT_EQ(dm.stride(1), ROWS);
 
-    EXPECT_EQ(dm.data(), static_cast<u_int8_t*>(buffer->data()) + Dtype.item_size());
+    EXPECT_EQ(dm.data(), static_cast<u_int8_t*>(buffer->data()) + DTYPE.item_size());
 
     EXPECT_EQ(dm.memory()->cuda_stream, rmm::cuda_stream_legacy);
     EXPECT_EQ(dm.memory()->memory_resource,
@@ -94,33 +94,33 @@ TEST_F(TestDevMemInfo, VoidPtrConstructor)
 {
     // Explicitly passing the non-default values to make sure we don't have a test that accidentally passes
     auto mem_resource = make_mem_resource<rmm::mr::arena_memory_resource>();
-    auto buffer       = std::make_shared<rmm::device_buffer>(ByteSize, rmm::cuda_stream_legacy, mem_resource.get());
+    auto buffer       = std::make_shared<rmm::device_buffer>(BYTE_SIZE, rmm::cuda_stream_legacy, mem_resource.get());
 
     auto md = std::make_shared<MemoryDescriptor>(rmm::cuda_stream_legacy, mem_resource.get());
 
     // Set the offset to the second row in the buffer
-    DevMemInfo dm{buffer->data(), Dtype, md, {Rows - 1, Cols}, {1, Rows}, Dtype.item_size()};
+    DevMemInfo dm{buffer->data(), DTYPE, md, {ROWS - 1, COLS}, {1, ROWS}, DTYPE.item_size()};
 
-    EXPECT_EQ(dm.bytes(), (Rows - 1) * Cols * Dtype.item_size());
-    EXPECT_EQ(dm.count(), (Rows - 1) * Cols);
-    EXPECT_EQ(dm.offset_bytes(), Dtype.item_size());
+    EXPECT_EQ(dm.bytes(), (ROWS - 1) * COLS * DTYPE.item_size());
+    EXPECT_EQ(dm.count(), (ROWS - 1) * COLS);
+    EXPECT_EQ(dm.offset_bytes(), DTYPE.item_size());
 
-    EXPECT_EQ(dm.dtype(), Dtype);
-    EXPECT_EQ(dm.type_id(), Dtype.type_id());
+    EXPECT_EQ(dm.dtype(), DTYPE);
+    EXPECT_EQ(dm.type_id(), DTYPE.type_id());
 
     EXPECT_EQ(dm.shape().size(), 2);
-    EXPECT_EQ(dm.shape()[0], Rows - 1);
-    EXPECT_EQ(dm.shape(0), Rows - 1);
-    EXPECT_EQ(dm.shape()[1], Cols);
-    EXPECT_EQ(dm.shape(1), Cols);
+    EXPECT_EQ(dm.shape()[0], ROWS - 1);
+    EXPECT_EQ(dm.shape(0), ROWS - 1);
+    EXPECT_EQ(dm.shape()[1], COLS);
+    EXPECT_EQ(dm.shape(1), COLS);
 
     EXPECT_EQ(dm.stride().size(), 2);
     EXPECT_EQ(dm.stride()[0], 1);
     EXPECT_EQ(dm.stride(0), 1);
-    EXPECT_EQ(dm.stride()[1], Rows);
-    EXPECT_EQ(dm.stride(1), Rows);
+    EXPECT_EQ(dm.stride()[1], ROWS);
+    EXPECT_EQ(dm.stride(1), ROWS);
 
-    EXPECT_EQ(dm.data(), static_cast<u_int8_t*>(buffer->data()) + Dtype.item_size());
+    EXPECT_EQ(dm.data(), static_cast<u_int8_t*>(buffer->data()) + DTYPE.item_size());
 
     EXPECT_EQ(dm.memory()->cuda_stream, rmm::cuda_stream_legacy);
     EXPECT_EQ(dm.memory()->memory_resource,
@@ -131,9 +131,9 @@ TEST_F(TestDevMemInfo, MakeNewBuffer)
 {
     // Explicitly passing the non-default values to make sure we don't have a test that accidentally passes
     auto mem_resource = make_mem_resource<rmm::mr::fixed_size_memory_resource>();
-    auto buffer       = std::make_shared<rmm::device_buffer>(ByteSize, rmm::cuda_stream_legacy, mem_resource.get());
+    auto buffer       = std::make_shared<rmm::device_buffer>(BYTE_SIZE, rmm::cuda_stream_legacy, mem_resource.get());
 
-    DevMemInfo dm{buffer, Dtype, {Rows, Cols}, {1, Rows}};
+    DevMemInfo dm{buffer, DTYPE, {ROWS, COLS}, {1, ROWS}};
 
     const std::size_t buff_size = 20;
     auto new_buff               = dm.make_new_buffer(buff_size);

python/morpheus_llm/morpheus_llm/_lib/src/llm/llm_engine.cpp

@@ -29,6 +29,7 @@
 #include <mrc/coroutines/task.hpp>  // IWYU pragma: keep
 #include <nlohmann/json.hpp>
 
+#include <algorithm>  // for max (implied usage)
 #include <coroutine>
 #include <optional>
 #include <stdexcept>

python/morpheus_llm/morpheus_llm/_lib/src/llm/llm_node_runner.cpp

@@ -27,6 +27,7 @@
 #include <mrc/coroutines/task.hpp>  // IWYU pragma: keep
 #include <nlohmann/json.hpp>
 
+#include <algorithm>  // for max (implied usage)
 #include <coroutine>
 #include <set>
 #include <sstream>

python/morpheus_llm/morpheus_llm/_lib/tests/llm/test_llm_task_handler_runner.cpp

@@ -27,16 +27,18 @@
 #include "morpheus/utilities/json_types.hpp"  // for PythonByteContainer
 
 #include <gtest/gtest.h>
-#include <mrc/channel/forward.hpp>
+#include <mrc/channel/forward.hpp>  // IWYU pragma: keep
 #include <mrc/coroutines/sync_wait.hpp>
 #include <mrc/coroutines/task.hpp>  // IWYU pragma: keep
 #include <nlohmann/json.hpp>        // for nlohmann::json
 
+#include <algorithm>  // for max (implied usage)
 #include <coroutine>
 #include <memory>
 #include <optional>  // for optional
 #include <string>
 // IWYU pragma: no_include "morpheus_llm/llm/fwd.hpp"
+// IWYU pragma: no_include <pymrc/utilities/object_wrappers.hpp>
 
 using namespace morpheus;
 using namespace morpheus::test;

tests/_utils/__init__.py

@@ -25,6 +25,7 @@
 
 import numpy as np
 import pytest
+
 from morpheus.io.deserializers import read_file_to_df
 
 from .test_directories import TestDirectories

tests/scripts/data/dfp_old_namespace_data.py

@@ -26,7 +26,7 @@
 # old DFP import patterns
 from dfp.utils.config_generator import ConfigGenerator
 from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
-from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage # no update
+from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage  # no update
 import dfp.stages.dfp_training
 import dfp.stages.dfp_inference_stage as inference_stage
 import dfp
@@ -39,5 +39,5 @@ def empty_imports_function_scope():
     from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
     from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
     from dfp.utils.regex_utils import iso_date_regex
-    from morpheus_dfp.utils.schema_utils import SchemaBuilder # no update
+    from morpheus_dfp.utils.schema_utils import SchemaBuilder  # no update
     from dfp.modules import dfp_data_prep

tests/scripts/data/llm_old_namespace_data.py

@@ -36,8 +36,9 @@
 
 # These should be skipped
 from morpheus.stages.postprocess.filter_detections_stage import FilterDetectionsStage
-from morpheus_llm.service.vdb import milvus_client # no update
-from morpheus_llm.llm import LLMEngine # no update
+from morpheus_llm.service.vdb import milvus_client  # no update
+from morpheus_llm.llm import LLMEngine  # no update
+
 
 def empty_imports_function_scope():
     '''