Merge branch 'master' into quadcub

VowpalWabbit · Dec 20, 2022 · a0b03e9 · a0b03e9
2 parents 928d33d + b6f6c15
commit a0b03e9
Show file tree

Hide file tree

Showing 110 changed files with 1,659 additions and 692 deletions.
diff --git a/.github/workflows/native_nugets.yml b/.github/workflows/native_nugets.yml
@@ -61,8 +61,8 @@ jobs:
           -DVW_INSTALL=On
       - name: Build and install
         run: |
-          cmake --build build --config Debug -t vw_io vw_core vw_cli_bin vw_allreduce vw_spanning_tree_bin vw_c_wrapper vw_cache_parser
-          cmake --build build --config Release -t vw_io vw_core vw_cli_bin vw_allreduce vw_spanning_tree_bin vw_c_wrapper vw_cache_parser
+          cmake --build build --config Debug -t vw_io vw_core vw_cli_bin vw_allreduce vw_spanning_tree_bin vw_c_wrapper vw_cache_parser vw_text_parser
+          cmake --build build --config Release -t vw_io vw_core vw_cli_bin vw_allreduce vw_spanning_tree_bin vw_c_wrapper vw_cache_parser vw_text_parser
 
           # Install debug first so that release overwrites the exe and nothing else
           cmake --install build --prefix ./nuget_staging --config Debug

diff --git a/cs/cli/vowpalwabbit.cpp b/cs/cli/vowpalwabbit.cpp
@@ -13,7 +13,7 @@
 #include "vw_builder.h"
 #include "clr_io.h"
 #include "vw/core/reductions/lda_core.h"
-#include "vw/core/parse_example.h"
+#include "vw/text_parser/parse_example_text.h"
 #include "vw/core/parse_example_json.h"
 #include "vw/core/shared_data.h"
 
@@ -424,7 +424,7 @@ VowpalWabbitExample^ VowpalWabbit::ParseLine(String^ line)
 
   try
   { try
-    { VW::read_line(*m_vw, ex->m_example, reinterpret_cast<char*>(valueHandle.AddrOfPinnedObject().ToPointer()));
+    { VW::parsers::text::read_line(*m_vw, ex->m_example, reinterpret_cast<char*>(valueHandle.AddrOfPinnedObject().ToPointer()));
 
       // finalize example
       VW::setup_example(*m_vw, ex->m_example);

diff --git a/cs/cli/vw_example.cpp b/cs/cli/vw_example.cpp
@@ -11,7 +11,7 @@
 #include "vw/core/reductions/gd.h"
 #include <algorithm>
 #include "vw/core/shared_data.h"
-#include "vw/core/parse_example.h"
+#include "vw/text_parser/parse_example_text.h"
 
 namespace VW
 {
@@ -107,7 +107,7 @@ void VowpalWabbitExample::Label::set(ILabel^ label)
 
 void VowpalWabbitExample::MakeEmpty(VowpalWabbit^ vw)
 { char empty = '\0';
-  VW::read_line(*vw->m_vw, m_example, &empty);
+  VW::parsers::text::read_line(*vw->m_vw, m_example, &empty);
 
   VW::setup_example(*vw->m_vw, m_example);
 }

diff --git a/cs/vw.net.native/vw.net.example.cc b/cs/vw.net.native/vw.net.example.cc
@@ -1,9 +1,9 @@
 #include "vw.net.example.h"
 
 #include "vw/core/feature_group.h"
-#include "vw/core/parse_example.h"
 #include "vw/core/reductions/gd.h"
 #include "vw/core/shared_data.h"
+#include "vw/text_parser/parse_example_text.h"
 
 #include <sstream>
 
@@ -183,7 +183,7 @@ API void MakeIntoNewlineExample(vw_net_native::workspace_context* workspace, VW:
 {
   const char empty = '\0';
 
-  VW::read_line(*workspace->vw, example, &empty);
+  VW::parsers::text::read_line(*workspace->vw, example, &empty);
   VW::setup_example(*workspace->vw, example);
 }
 

diff --git a/cs/vw.net.native/vw.net.example.h b/cs/vw.net.native/vw.net.example.h
@@ -24,6 +24,7 @@ extern "C"
   API VW::example* CreateExample(vw_net_native::workspace_context* workspace);
   API void DeleteExample(example* example);
 
+  // This function is deprecated and will be removed in VW 10.
   API int IsRingExample(vw_net_native::workspace_context* workspace, VW::example* example);
   API int IsExampleNewline(VW::example* example);
   API char* ComputeDiffDescriptionExample(

diff --git a/cs/vw.net.native/vw.net.workspace.cc b/cs/vw.net.native/vw.net.workspace.cc
@@ -4,8 +4,8 @@
 #include "vw/common/string_view.h"
 #include "vw/core/best_constant.h"
 #include "vw/core/learner.h"
-#include "vw/core/parse_example.h"
 #include "vw/core/shared_data.h"
+#include "vw/text_parser/parse_example_text.h"
 
 vw_net_native::workspace_context* create_workspace(
     std::string arguments, io_buf* model, trace_message_t trace_listener, void* trace_context)
@@ -255,7 +255,7 @@ API vw_net_native::ERROR_CODE WorkspaceParseSingleLine(vw_net_native::workspace_
 {
   try
   {
-    VW::read_line(*workspace->vw, ex, VW::string_view(line, length));
+    VW::parsers::text::read_line(*workspace->vw, ex, VW::string_view(line, length));
 
     VW::setup_example(*workspace->vw, ex);
 

diff --git a/java/src/main/c++/jni_spark_vw.cc b/java/src/main/c++/jni_spark_vw.cc
@@ -11,10 +11,10 @@
 #include "vw/core/global_data.h"
 #include "vw/core/learner.h"
 #include "vw/core/merge.h"
-#include "vw/core/parse_example.h"
 #include "vw/core/shared_data.h"
 #include "vw/core/simple_label_parser.h"
 #include "vw/core/vw_fwd.h"
+#include "vw/text_parser/parse_example_text.h"
 
 #include <algorithm>
 #include <exception>
@@ -172,7 +172,8 @@ JNIEXPORT jobject JNICALL Java_org_vowpalwabbit_spark_VowpalWabbitNative_learnFr
   {
     VW::multi_ex ex_coll;
     ex_coll.push_back(&VW::get_unused_example(all));
-    all->example_parser->text_reader(all, exampleStringGuard.c_str(), exampleStringGuard.length(), ex_coll);
+    all->example_parser->text_reader(
+        all, VW::string_view(exampleStringGuard.c_str(), exampleStringGuard.length()), ex_coll);
     VW::setup_examples(*all, ex_coll);
     return callLearner<true>(env, all, ex_coll);
   }
@@ -211,7 +212,8 @@ JNIEXPORT jobject JNICALL Java_org_vowpalwabbit_spark_VowpalWabbitNative_predict
   {
     VW::multi_ex ex_coll;
     ex_coll.push_back(&VW::get_unused_example(all));
-    all->example_parser->text_reader(all, exampleStringGuard.c_str(), exampleStringGuard.length(), ex_coll);
+    all->example_parser->text_reader(
+        all, VW::string_view(exampleStringGuard.c_str(), exampleStringGuard.length()), ex_coll);
     VW::setup_examples(*all, ex_coll);
     return callLearner<false>(env, all, ex_coll);
   }
@@ -413,7 +415,7 @@ JNIEXPORT jlong JNICALL Java_org_vowpalwabbit_spark_VowpalWabbitExample_initiali
     if (isEmpty)
     {
       char empty = '\0';
-      VW::read_line(*all, ex, &empty);
+      VW::parsers::text::read_line(*all, ex, &empty);
     }
     else
       all->example_parser->lbl_parser.default_label(ex->l);

diff --git a/library/libsearch.h b/library/libsearch.h
@@ -21,7 +21,7 @@ class SearchTask  // NOLINT
   SearchTask(VW::workspace& vw_obj) : vw_obj(vw_obj), sch(*(Search::search*)vw_obj.searchstr)
   {
     _bogus_example = new VW::example;
-    VW::read_line(vw_obj, _bogus_example, (char*)"1 | x");
+    VW::parsers::text::read_line(vw_obj, _bogus_example, (char*)"1 | x");
     VW::setup_example(vw_obj, _bogus_example);
 
     _trigger.push_back(_bogus_example);

diff --git a/nuget/native/vowpalwabbit-x64.targets b/nuget/native/vowpalwabbit-x64.targets
@@ -8,18 +8,18 @@
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug' And '$(Platform)'=='x64'">
     <Link>
-      <AdditionalDependencies>vw_configd.lib;vw_allreduced.lib;vw_cache_parserd.lib;fmtd.lib;spdlogd.lib;vw_iod.lib;vw_cored.lib;zlibstaticd.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>vw_configd.lib;vw_allreduced.lib;vw_cache_parserd.lib;vw_text_parserd.lib;fmtd.lib;spdlogd.lib;vw_iod.lib;vw_cored.lib;zlibstaticd.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <Lib>
-      <AdditionalDependencies>vw_configd.lib;vw_allreduced.lib;vw_cache_parserd.lib;fmtd.lib;spdlogd.lib;vw_iod.lib;vw_cored.lib;zlibstaticd.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>vw_configd.lib;vw_allreduced.lib;vw_cache_parserd.lib;vw_text_parserd.lib;fmtd.lib;spdlogd.lib;vw_iod.lib;vw_cored.lib;zlibstaticd.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Lib>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)'=='Release' And '$(Platform)'=='x64'">
     <Link>
-      <AdditionalDependencies>vw_config.lib;vw_allreduce.lib;vw_cache_parser.lib;fmt.lib;spdlog.lib;vw_io.lib;vw_core.lib;zlibstatic.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>vw_config.lib;vw_allreduce.lib;vw_cache_parser.lib;vw_text_parser.lib;fmt.lib;spdlog.lib;vw_io.lib;vw_core.lib;zlibstatic.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <Lib>
-      <AdditionalDependencies>vw_config.lib;vw_allreduce.lib;vw_cache_parser.lib;fmt.lib;spdlog.lib;vw_io.lib;vw_core.lib;zlibstatic.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>vw_config.lib;vw_allreduce.lib;vw_cache_parser.lib;vw_text_parser.lib;fmt.lib;spdlog.lib;vw_io.lib;vw_core.lib;zlibstatic.lib;wsock32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Lib>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">

diff --git a/python/pylibvw.cc b/python/pylibvw.cc
@@ -14,14 +14,14 @@
 #include "vw/core/merge.h"
 #include "vw/core/multiclass.h"
 #include "vw/core/multilabel.h"
-#include "vw/core/parse_example.h"
 #include "vw/core/reductions/gd.h"
 #include "vw/core/reductions/search/search.h"
 #include "vw/core/reductions/search/search_hooktask.h"
 #include "vw/core/shared_data.h"
 #include "vw/core/simple_label_parser.h"
 #include "vw/core/slates_label.h"
 #include "vw/core/vw.h"
+#include "vw/text_parser/parse_example_text.h"
 
 // see http://www.boost.org/doc/libs/1_56_0/doc/html/bbv2/installation.html
 #define BOOST_PYTHON_USE_GCC_SYMBOL_VISIBILITY 1
@@ -494,7 +494,7 @@ example_ptr my_empty_example(vw_ptr vw, size_t labelType)
 example_ptr my_read_example(vw_ptr all, size_t labelType, char* str)
 {
   VW::example* ec = my_empty_example0(all, labelType);
-  VW::read_line(*all, ec, str);
+  VW::parsers::text::read_line(*all, ec, str);
   VW::setup_example(*all, ec);
   return boost::shared_ptr<VW::example>(ec, my_delete_example);
 }
@@ -550,7 +550,7 @@ py::list my_parse(vw_ptr& all, char* str)
 {
   VW::multi_ex examples;
   examples.push_back(&VW::get_unused_example(all.get()));
-  all->example_parser->text_reader(all.get(), str, strlen(str), examples);
+  all->example_parser->text_reader(all.get(), VW::string_view(str, strlen(str)), examples);
 
   py::list example_collection;
   for (auto* ex : examples)