feat: Minimal DB2 preprocessor emulator (#140)

clients/vscode-hlasmplugin/proc_grps_schema

@@ -59,6 +59,26 @@
                                 "description": "Profile Member to be copied into the source program."
                             }
                         }
+                    },
+                    "preprocessor": {
+                        "description": "Defines optional preprocessor pass for open code.",
+                        "anyOf": [
+                            {
+                                "type": "string",
+                                "description": "Name of the preprocessor.",
+                                "pattern": "DB2"
+                            },
+                            {
+                                "type": "object",
+                                "properties": {
+                                    "name": {
+                                        "type": "string",
+                                        "description": "Name of the preprocessor.",
+                                        "pattern": "DB2"
+                                    }
+                                }
+                            }
+                        ]
                     }
                 },
                 "required": [

parser_library/fuzzer/fuzzer.cpp

@@ -61,6 +61,19 @@ class fuzzer_lib_provider : public parse_lib_provider
         return read_library_name(library).has_value();
     }
 
+    std::optional<std::string> get_library(
+        const std::string& library, const std::string&, std::string* uri) const override
+    {
+        auto lib = read_library_name(library);
+        if (!lib.has_value())
+            return std::nullopt;
+
+        if (uri)
+            *uri = library;
+
+        return files[lib.value()];
+    }
+
     std::vector<std::string> files;
 };
 
@@ -80,7 +93,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     }
     *target = workspaces::file_impl::replace_non_utf8_chars(std::string_view((const char*)data, size));
 
-    analyzer a(source, analyzer_options(&lib));
+    analyzer a(source, analyzer_options(&lib, db2_preprocessor_options()));
     a.analyze();
 
     return 0; // Non-zero return values are reserved for future use.

parser_library/fuzzer/fuzzer.dict

@@ -2071,4 +2071,12 @@
 "G'<.A>'"
 
 #literal
-"=X'00'"
+"=X'00'"
+
+#macro
+"@0"
+"@1"
+"@2"
+
+#macro separator
+"\xff"

parser_library/src/CMakeLists.txt

@@ -29,6 +29,7 @@ target_sources(parser_library PRIVATE
 	error_messages.h
 	lib_config.cpp
 	location.h
+	preprocessor_options.h
 	protocol.cpp
 	workspace_manager.cpp
 	workspace_manager_impl.h

parser_library/src/analyzer.cpp

@@ -15,6 +15,7 @@
 #include "analyzer.h"
 
 #include "parsing/error_strategy.h"
+#include "processing/preprocessor.h"
 
 using namespace hlasm_plugin::parser_library;
 using namespace hlasm_plugin::parser_library::lexing;
@@ -38,14 +39,27 @@ analyzing_context& analyzer_options::get_context()
 
 context::hlasm_context& analyzer_options::get_hlasm_context() { return *get_context().hlasm_ctx; }
 
-workspaces::parse_lib_provider& analyzer_options::get_lib_provider()
+workspaces::parse_lib_provider& analyzer_options::get_lib_provider() const
 {
     if (lib_provider)
         return *lib_provider;
     else
         return workspaces::empty_parse_lib_provider::instance;
 }
 
+std::unique_ptr<processing::preprocessor> analyzer_options::get_preprocessor(
+    processing::library_fetcher lf, processing::diag_reporter dr) const
+{
+    return std::visit(
+        [&lf, &dr](const auto& p) -> std::unique_ptr<processing::preprocessor> {
+            if constexpr (std::is_same_v<std::decay_t<decltype(p)>, std::monostate>)
+                return {};
+            else
+                return processing::preprocessor::create(p, std::move(lf), std::move(dr));
+        },
+        preprocessor_args);
+}
+
 analyzer::analyzer(const std::string& text, analyzer_options opts)
     : diagnosable_ctx(opts.get_hlasm_context())
     , ctx_(std::move(opts.get_context()))
@@ -57,6 +71,18 @@ analyzer::analyzer(const std::string& text, analyzer_options opts)
                 mngr_,
                 src_proc_,
                 opts.file_name,
+                opts.get_preprocessor(
+                    [libs = &opts.get_lib_provider(), program = opts.file_name, &ctx = ctx_](std::string_view library) {
+                        std::string uri;
+
+                        auto result = libs->get_library(std::string(library), program, &uri);
+
+                        if (!uri.empty())
+                            ctx.hlasm_ctx->add_preprocessor_dependency(uri);
+
+                        return result;
+                    },
+                    [this](diagnostic_op d) { this->add_diagnostic(std::move(d)); }),
                 opts.parsing_opencode == file_is_opencode::yes ? opencode_provider_options { true, 10 }
                                                                : opencode_provider_options {}),
           ctx_,

parser_library/src/analyzer.h

@@ -25,6 +25,7 @@
 #include "lexing/token_stream.h"
 #include "lsp/lsp_context.h"
 #include "parsing/parser_error_listener.h"
+#include "preprocessor_options.h"
 #include "processing/processing_manager.h"
 #include "workspaces/parse_lib_provider.h"
 
@@ -51,6 +52,7 @@ class analyzer_options
     collect_highlighting_info collect_hl_info = collect_highlighting_info::no;
     file_is_opencode parsing_opencode = file_is_opencode::no;
     std::optional<context::id_storage> ids_init;
+    preprocessor_options preprocessor_args;
 
     void set(std::string fn) { file_name = std::move(fn); }
     void set(workspaces::parse_lib_provider* lp) { lib_provider = lp; }
@@ -60,10 +62,13 @@ class analyzer_options
     void set(collect_highlighting_info hi) { collect_hl_info = hi; }
     void set(file_is_opencode f_oc) { parsing_opencode = f_oc; }
     void set(context::id_storage ids) { ids_init.emplace(std::move(ids)); }
+    void set(preprocessor_options pp) { preprocessor_args = std::move(pp); }
 
     context::hlasm_context& get_hlasm_context();
     analyzing_context& get_context();
-    workspaces::parse_lib_provider& get_lib_provider();
+    workspaces::parse_lib_provider& get_lib_provider() const;
+    std::unique_ptr<processing::preprocessor> get_preprocessor(
+        processing::library_fetcher, processing::diag_reporter) const;
 
     friend class analyzer;
 
@@ -82,7 +87,9 @@ class analyzer_options
         constexpr auto hi_cnt = (0 + ... + std::is_same_v<std::decay_t<Args>, collect_highlighting_info>);
         constexpr auto f_oc_cnt = (0 + ... + std::is_same_v<std::decay_t<Args>, file_is_opencode>);
         constexpr auto ids_cnt = (0 + ... + std::is_same_v<std::decay_t<Args>, context::id_storage>);
-        constexpr auto cnt = string_cnt + lib_cnt + ao_cnt + ac_cnt + lib_data_cnt + hi_cnt + f_oc_cnt + ids_cnt;
+        constexpr auto pp_cnt = (0 + ... + std::is_convertible_v<std::decay_t<Args>, preprocessor_options>);
+        constexpr auto cnt =
+            string_cnt + lib_cnt + ao_cnt + ac_cnt + lib_data_cnt + hi_cnt + f_oc_cnt + ids_cnt + pp_cnt;
 
         static_assert(string_cnt <= 1, "Duplicate file_name");
         static_assert(lib_cnt <= 1, "Duplicate parse_lib_provider");
@@ -92,8 +99,9 @@ class analyzer_options
         static_assert(hi_cnt <= 1, "Duplicate collect_highlighting_info");
         static_assert(f_oc_cnt <= 1, "Duplicate file_is_opencode");
         static_assert(ids_cnt <= 1, "Duplicate id_storage");
-        static_assert(
-            !(ac_cnt && (ao_cnt || ids_cnt)), "Do not specify both analyzing_context and asm_option or id_storage");
+        static_assert(pp_cnt <= 1, "Duplicate preprocessor_args");
+        static_assert(!(ac_cnt && (ao_cnt || ids_cnt || pp_cnt)),
+            "Do not specify both analyzing_context and asm_option, id_storage or preprocessor_args");
         static_assert(cnt == sizeof...(Args), "Unrecognized argument provided");
 
         (set(std::forward<Args>(args)), ...);

parser_library/src/config/proc_conf.cpp

@@ -59,18 +59,51 @@ void from_json(const nlohmann::json& j, assembler_options& p)
         it->get_to(p.sysparm);
 }
 
+void to_json(nlohmann::json& j, const db2_preprocessor&) { j = "DB2"; }
+void from_json(const nlohmann::json&, db2_preprocessor&) {}
+
+namespace {
+struct preprocessor_visitor
+{
+    nlohmann::json& j;
+
+    void operator()(const std::monostate&) const {}
+    void operator()(const db2_preprocessor& p) const { j = p; }
+};
+} // namespace
+
 void to_json(nlohmann::json& j, const processor_group& p)
 {
     j = nlohmann::json { { "name", p.name }, { "libs", p.libs } };
     if (auto opts = nlohmann::json(p.asm_options); !opts.empty())
         j["asm_options"] = std::move(opts);
+
+    if (!std::holds_alternative<std::monostate>(p.preprocessor.options))
+        std::visit(preprocessor_visitor { j["preprocessor"] }, p.preprocessor.options);
 }
 void from_json(const nlohmann::json& j, processor_group& p)
 {
     j.at("name").get_to(p.name);
     j.at("libs").get_to(p.libs);
     if (auto it = j.find("asm_options"); it != j.end())
         it->get_to(p.asm_options);
+
+    if (auto it = j.find("preprocessor"); it != j.end())
+    {
+        std::string p_name;
+        if (it->is_string())
+            p_name = it->get<std::string>();
+        else if (it->is_object())
+            it->at("name").get_to(p_name);
+        else
+            throw nlohmann::json::other_error::create(501, "Unable to identify requested preprocessor.");
+
+        std::transform(p_name.begin(), p_name.end(), p_name.begin(), [](unsigned char c) { return (char)toupper(c); });
+        if (p_name == "DB2")
+            it->get_to(p.preprocessor.options.emplace<db2_preprocessor>());
+        else
+            throw nlohmann::json::other_error::create(501, "Unable to identify requested preprocessor.");
+    }
 }
 
 void to_json(nlohmann::json& j, const proc_conf& p)

parser_library/src/config/proc_conf.h

@@ -17,6 +17,7 @@
 
 #include <optional>
 #include <string>
+#include <variant>
 #include <vector>
 
 #include "nlohmann/json_fwd.hpp"
@@ -42,11 +43,27 @@ struct assembler_options
 void to_json(nlohmann::json& j, const assembler_options& p);
 void from_json(const nlohmann::json& j, assembler_options& p);
 
+struct db2_preprocessor
+{};
+inline bool operator==(const db2_preprocessor&, const db2_preprocessor&) { return true; }
+inline bool operator!=(const db2_preprocessor& l, const db2_preprocessor& r) { return !(l == r); }
+
+void to_json(nlohmann::json& j, const db2_preprocessor& p);
+void from_json(const nlohmann::json& j, db2_preprocessor& p);
+
+struct preprocessor_options
+{
+    std::variant<std::monostate, db2_preprocessor> options;
+};
+inline bool operator==(const preprocessor_options& l, const preprocessor_options& r) { return l.options == r.options; }
+inline bool operator!=(const preprocessor_options& l, const preprocessor_options& r) { return !(l == r); }
+
 struct processor_group
 {
     std::string name;
     std::vector<library> libs;
     assembler_options asm_options;
+    preprocessor_options preprocessor;
 };
 void to_json(nlohmann::json& j, const processor_group& p);
 void from_json(const nlohmann::json& j, processor_group& p);

parser_library/src/context/copy_member.h

@@ -61,6 +61,14 @@ struct copy_member_invocation
         , copy_member_definition(std::move(copy_member))
         , current_statement(-1)
     {}
+
+    position current_statement_position() const
+    {
+        if (current_statement != -1)
+            return cached_definition->at(current_statement).get_base()->statement_position();
+        else
+            return {};
+    }
 };
 
 } // namespace hlasm_plugin::parser_library::context

parser_library/src/context/hlasm_context.cpp

@@ -347,8 +347,7 @@ processing_stack_t hlasm_context::processing_stack() const
         res.emplace_back(source_stack_[i].current_instruction, scope_stack_.front(), file_processing_type::OPENCODE);
         for (const auto& member : source_stack_[i].copy_stack)
         {
-            location loc(member.cached_definition->at(member.current_statement).get_base()->statement_position(),
-                member.definition_location->file);
+            location loc(member.current_statement_position(), member.definition_location->file);
             res.emplace_back(std::move(loc), scope_stack_.front(), file_processing_type::COPY);
         }
 
@@ -377,8 +376,7 @@ location hlasm_context::current_statement_location() const
         {
             const auto& member = source_stack_.back().copy_stack.back();
 
-            auto pos = member.cached_definition->at(member.current_statement).get_base()->statement_position();
-            return location(pos, member.definition_location->file);
+            return location(member.current_statement_position(), member.definition_location->file);
         }
         else
             return source_stack_.back().current_instruction;
@@ -783,6 +781,8 @@ const hlasm_context::copy_member_storage& hlasm_context::copy_members() { return
 
 void hlasm_context::leave_copy_member() { source_stack_.back().copy_stack.pop_back(); }
 
+void hlasm_context::add_preprocessor_dependency(const std::string& file) { visited_files_.emplace(file); }
+
 void hlasm_context::apply_source_snapshot(source_snapshot snapshot)
 {
     assert(proc_stack_.size() == 1);

parser_library/src/context/hlasm_context.h

@@ -208,6 +208,9 @@ class hlasm_context
     // leaves current copy member
     void leave_copy_member();
 
+    // register preprocessor dependency
+    void add_preprocessor_dependency(const std::string& file);
+
     // creates specified global set symbol
     template<typename T>
     set_sym_ptr create_global_variable(id_index id, bool is_scalar)

parser_library/src/debugging/debug_lib_provider.h

@@ -47,15 +47,13 @@ class debug_lib_provider final : public workspaces::parse_lib_provider
 
     bool has_library(const std::string& library, const std::string& program) const override
     {
-        auto& proc_grp = ws_.get_proc_grp_by_program(program);
-        for (auto&& lib : proc_grp.libraries())
-        {
-            std::shared_ptr<workspaces::processor> found = lib->find_file(library);
-            if (found)
-                return true;
-        }
+        return ws_.has_library(library, program);
+    }
 
-        return false;
+    std::optional<std::string> get_library(
+        const std::string& library, const std::string& program, std::string* uri) const override
+    {
+        return ws_.get_library(library, program, uri);
     }
 };
 

parser_library/src/debugging/debugger.cpp

@@ -142,6 +142,7 @@ class debugger::impl final : public processing::statement_analyzer
                     open_code->get_file_name(),
                     lib_provider ? lib_provider : &debug_provider,
                     workspace.get_asm_options(open_code->get_file_name()),
+                    workspace.get_preprocessor_options(open_code->get_file_name()),
                 });
 
             a.register_stmt_analyzer(this);

parser_library/src/diagnostic.cpp

@@ -1963,6 +1963,26 @@ diagnostic_op diagnostic_op::error_CW001(const range& range)
     return diagnostic_op(diagnostic_severity::warning, "CW001", "Substring count points past string end", range);
 }
 
+diagnostic_op diagnostic_op::error_P0001(const range& range)
+{
+    return diagnostic_op(diagnostic_severity::error, "P0001", "DB2 preprocessor - invalid line continuation", range);
+}
+
+diagnostic_op diagnostic_op::error_P0002(const range& range, std::string_view lib)
+{
+    return diagnostic_op(diagnostic_severity::error,
+        "P0002",
+        std::string("DB2 preprocessor - unable to find library '").append(lib).append("'"),
+        range);
+}
+
+diagnostic_op diagnostic_op::error_P0003(const range& range, std::string_view lib)
+{
+    return diagnostic_op(diagnostic_severity::error,
+        "P0003",
+        std::string("DB2 preprocessor - nested include '").append(lib).append("' requested"),
+        range);
+}
 
 diagnostic_s diagnostic_s::error_W002(const std::string& ws_uri, const std::string& ws_name)
 {