Add velocity and acceleration to StylusState

PiperOrigin-RevId: 658058056

ink_stroke_modeler/BUILD.bazel

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//tools/build_defs/testing:bzl_library.bzl", "bzl_library")
+
 package(
     default_visibility = ["//visibility:public"],
 )
@@ -121,3 +123,10 @@ cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
+
+bzl_library(
+    name = "extensions_bzl",
+    srcs = ["extensions.bzl"],
+    parse_tests = False,
+    visibility = ["//visibility:private"],
+)

ink_stroke_modeler/internal/BUILD.bazel

@@ -24,6 +24,7 @@ cc_library(
     hdrs = ["internal_types.h"],
     deps = [
         "//ink_stroke_modeler:types",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
     ],
 )
@@ -68,6 +69,7 @@ cc_test(
         ":type_matchers",
         "//ink_stroke_modeler:numbers",
         "//ink_stroke_modeler:params",
+        "//ink_stroke_modeler:types",
         "@com_google_googletest//:gtest_main",
     ],
 )

ink_stroke_modeler/internal/internal_types.cc

@@ -2,6 +2,7 @@
 
 #include <string>
 
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 
 namespace ink {
@@ -14,9 +15,23 @@ std::string ToFormattedString(const TipState &tip_state) {
 }
 
 std::string ToFormattedString(const StylusState &stylus_state) {
-  return absl::StrFormat(
-      "<StylusState: pressure: %v, tilt: %v, orientation: %v>",
-      stylus_state.pressure, stylus_state.tilt, stylus_state.orientation);
+  std::string formatted = absl::StrFormat(
+      "<StylusState: pressure: %v, tilt: %v, orientation: %v, "
+      "projected_position: %v",
+      stylus_state.pressure, stylus_state.tilt, stylus_state.orientation,
+      stylus_state.projected_position);
+
+  if (stylus_state.projected_velocity.has_value()) {
+    absl::StrAppend(&formatted,
+                    ", projected_velocity: ", *stylus_state.projected_velocity);
+  }
+  if (stylus_state.projected_acceleration.has_value()) {
+    absl::StrAppend(&formatted, ", projected_acceleration: ",
+                    *stylus_state.projected_acceleration);
+  }
+
+  formatted.push_back('>');
+  return formatted;
 }
 
 }  // namespace stroke_model

ink_stroke_modeler/internal/internal_types.h

@@ -17,6 +17,7 @@
 #ifndef INK_STROKE_MODELER_INTERNAL_INTERNAL_TYPES_H_
 #define INK_STROKE_MODELER_INTERNAL_INTERNAL_TYPES_H_
 
+#include <optional>
 #include <string>
 
 #include "ink_stroke_modeler/types.h"
@@ -46,6 +47,9 @@ struct StylusState {
   float pressure = -1;
   float tilt = -1;
   float orientation = -1;
+  Vec2 projected_position = {0, 0};
+  std::optional<Vec2> projected_velocity = std::nullopt;
+  std::optional<Vec2> projected_acceleration = std::nullopt;
 };
 
 bool operator==(const StylusState& lhs, const StylusState& rhs);
@@ -62,7 +66,10 @@ void AbslStringify(Sink& sink, const StylusState& stylus_state) {
 
 inline bool operator==(const StylusState& lhs, const StylusState& rhs) {
   return lhs.pressure == rhs.pressure && lhs.tilt == rhs.tilt &&
-         lhs.orientation == rhs.orientation;
+         lhs.orientation == rhs.orientation &&
+         lhs.projected_position == rhs.projected_position &&
+         lhs.projected_velocity == rhs.projected_velocity &&
+         lhs.projected_acceleration == rhs.projected_acceleration;
 }
 
 }  // namespace stroke_model

ink_stroke_modeler/internal/internal_types_test.cc

@@ -1,5 +1,7 @@
 #include "ink_stroke_modeler/internal/internal_types.h"
 
+#include <optional>
+
 #include "gtest/gtest.h"
 #include "absl/strings/str_format.h"
 #include "ink_stroke_modeler/types.h"
@@ -17,13 +19,51 @@ TEST(InternalTypesTest, TipStateString) {
 }
 
 TEST(InternalTypesTest, StylusStateEquals) {
-  EXPECT_EQ((StylusState{0.1, 0.2, 0.3}), (StylusState{0.1, 0.2, 0.3}));
-  EXPECT_FALSE((StylusState{0.1, 0.2, 0.3}) == (StylusState{0.1, 0.02, 0.3}));
+  EXPECT_EQ((StylusState{0.1,
+                         0.2,
+                         0.3,
+                         {1, 2},
+                         std::optional<Vec2>({3, 4}),
+                         std::optional<Vec2>({5, 6})}),
+            (StylusState{0.1,
+                         0.2,
+                         0.3,
+                         {1, 2},
+                         std::optional<Vec2>({3, 4}),
+                         std::optional<Vec2>({5, 6})}));
+  EXPECT_FALSE((StylusState{0.1,
+                            0.2,
+                            0.3,
+                            {1, 2},
+                            std::optional<Vec2>({3, 4}),
+                            std::optional<Vec2>({5, 6})}) ==
+               (StylusState{0.1,
+                            0.02,
+                            0.3,
+                            {1, 2},
+                            std::optional<Vec2>({3, 4}),
+                            std::optional<Vec2>({5, 6})}));
+  EXPECT_FALSE(
+      (StylusState{0.1,
+                   0.2,
+                   0.3,
+                   {1, 2},
+                   std::optional<Vec2>({3, 4}),
+                   std::optional<Vec2>({5, 6})}) ==
+      (StylusState{
+          0.1, 0.2, 0.3, {1, 2}, std::nullopt, std::optional<Vec2>({5, 6})}));
 }
 
 TEST(InternalTypesTest, StylusStateString) {
-  EXPECT_EQ(absl::StrFormat("%v", StylusState{0.1, 0.2, 0.3}),
-            "<StylusState: pressure: 0.1, tilt: 0.2, orientation: 0.3>");
+  EXPECT_EQ(absl::StrFormat("%v", StylusState{0.1,
+                                              0.2,
+                                              0.3,
+                                              {1, 2},
+                                              std::optional<Vec2>({3, 4}),
+                                              std::optional<Vec2>({5, 6})}),
+            "<StylusState: pressure: 0.1, tilt: 0.2, orientation: 0.3, "
+            "projected_position: (1, 2), projected_velocity: (3, 4), "
+            "projected_acceleration: (5, 6)>");
 }
 
 }  // namespace

ink_stroke_modeler/internal/stylus_state_modeler.cc

@@ -16,6 +16,7 @@
 
 #include <cmath>
 #include <limits>
+#include <optional>
 
 #include "ink_stroke_modeler/internal/internal_types.h"
 #include "ink_stroke_modeler/internal/utils.h"
@@ -25,7 +26,7 @@
 namespace ink {
 namespace stroke_model {
 
-void StylusStateModeler::Update(Vec2 position, const StylusState &state) {
+void StylusStateModeler::Update(double time, const StylusState &state) {
   // Possibly NaN should be prohibited in ValidateInput, but due to current
   // consumers, that can't be tightened for these values currently.
   if (state.pressure < 0 || std::isnan(state.pressure)) {
@@ -38,42 +39,59 @@ void StylusStateModeler::Update(Vec2 position, const StylusState &state) {
     state_.received_unknown_orientation = true;
   }
 
-  if (state_.received_unknown_pressure && state_.received_unknown_tilt &&
-      state_.received_unknown_orientation) {
-    // We've stopped tracking all fields, so there's no need to keep updating.
-    state_.positions_and_stylus_states.clear();
-    return;
+  Vec2 velocity = {0, 0};
+  Vec2 acceleration = {0, 0};
+  if (!state_.stylus_states.empty()) {
+    velocity = (state.projected_position -
+                state_.stylus_states.back().projected_position) /
+               (time - last_time_);
+    acceleration =
+        (velocity - *state_.stylus_states.back().projected_velocity) /
+        (time - last_time_);
   }
 
-  state_.positions_and_stylus_states.push_back({position, state});
+  state_.stylus_states.push_back(
+      {.pressure = state.pressure,
+       .tilt = state.tilt,
+       .orientation = state.orientation,
+       .projected_position = state.projected_position,
+       .projected_velocity = velocity,
+       .projected_acceleration = acceleration});
 
   if (params_.max_input_samples < 0 ||
-      state_.positions_and_stylus_states.size() >
+      state_.stylus_states.size() >
           static_cast<unsigned int>(params_.max_input_samples)) {
-    state_.positions_and_stylus_states.pop_front();
+    state_.stylus_states.pop_front();
   }
+  last_time_ = time;
 }
 
 void StylusStateModeler::Reset(const StylusStateModelerParams &params) {
-  state_.positions_and_stylus_states.clear();
+  state_.stylus_states.clear();
   state_.received_unknown_pressure = false;
   state_.received_unknown_tilt = false;
   state_.received_unknown_orientation = false;
   save_active_ = false;
   params_ = params;
+  last_time_ = 0;
 }
 
 StylusState StylusStateModeler::Query(Vec2 position) const {
-  if (state_.positions_and_stylus_states.empty())
-    return {.pressure = -1, .tilt = -1, .orientation = -1};
+  if (state_.stylus_states.empty())
+    return {.pressure = -1,
+            .tilt = -1,
+            .orientation = -1,
+            .projected_position = {0, 0},
+            .projected_velocity = std::nullopt,
+            .projected_acceleration = std::nullopt};
 
   int closest_segment_index = -1;
   float min_distance = std::numeric_limits<float>::infinity();
   float interp_value = 0;
-  for (decltype(state_.positions_and_stylus_states.size()) i = 0;
-       i < state_.positions_and_stylus_states.size() - 1; ++i) {
-    const Vec2 segment_start = state_.positions_and_stylus_states[i].position;
-    const Vec2 segment_end = state_.positions_and_stylus_states[i + 1].position;
+  for (decltype(state_.stylus_states.size()) i = 0;
+       i < state_.stylus_states.size() - 1; ++i) {
+    const Vec2 segment_start = state_.stylus_states[i].projected_position;
+    const Vec2 segment_end = state_.stylus_states[i + 1].projected_position;
     float param = NearestPointOnSegment(segment_start, segment_end, position);
     float distance =
         Distance(position, Interp(segment_start, segment_end, param));
@@ -85,17 +103,22 @@ StylusState StylusStateModeler::Query(Vec2 position) const {
   }
 
   if (closest_segment_index < 0) {
-    const auto &state = state_.positions_and_stylus_states.front().state;
+    const auto &state = state_.stylus_states.front();
     return {.pressure = state_.received_unknown_pressure ? -1 : state.pressure,
             .tilt = state_.received_unknown_tilt ? -1 : state.tilt,
             .orientation =
-                state_.received_unknown_orientation ? -1 : state.orientation};
+                state_.received_unknown_orientation ? -1 : state.orientation,
+            .projected_position = state.projected_position,
+            .projected_velocity = state.projected_velocity.has_value()
+                                      ? state.projected_velocity
+                                      : std::nullopt,
+            .projected_acceleration = state.projected_acceleration.has_value()
+                                          ? state.projected_acceleration
+                                          : std::nullopt};
   }
 
-  auto from_state =
-      state_.positions_and_stylus_states[closest_segment_index].state;
-  auto to_state =
-      state_.positions_and_stylus_states[closest_segment_index + 1].state;
+  auto from_state = state_.stylus_states[closest_segment_index];
+  auto to_state = state_.stylus_states[closest_segment_index + 1];
   return StylusState{
       .pressure =
           state_.received_unknown_pressure
@@ -107,16 +130,36 @@ StylusState StylusStateModeler::Query(Vec2 position) const {
       .orientation = state_.received_unknown_orientation
                          ? -1
                          : InterpAngle(from_state.orientation,
-                                       to_state.orientation, interp_value)};
+                                       to_state.orientation, interp_value),
+      .projected_position = Interp(from_state.projected_position,
+                                   to_state.projected_position, interp_value),
+      .projected_velocity =
+          (from_state.projected_velocity.has_value() &&
+           to_state.projected_velocity.has_value())
+              ? std::optional<Vec2>(Interp(*from_state.projected_velocity,
+                                           *to_state.projected_velocity,
+                                           interp_value))
+              : std::nullopt,
+      .projected_acceleration =
+          (from_state.projected_acceleration.has_value() &&
+           to_state.projected_acceleration.has_value())
+              ? std::optional<Vec2>(Interp(*from_state.projected_acceleration,
+                                           *to_state.projected_acceleration,
+                                           interp_value))
+              : std::nullopt};
 }
 
 void StylusStateModeler::Save() {
   saved_state_ = state_;
   save_active_ = true;
+  saved_last_time_ = last_time_;
 }
 
 void StylusStateModeler::Restore() {
-  if (save_active_) state_ = saved_state_;
+  if (save_active_) {
+    state_ = saved_state_;
+    last_time_ = saved_last_time_;
+  }
 }
 
 }  // namespace stroke_model

ink_stroke_modeler/internal/stylus_state_modeler.h

@@ -18,7 +18,6 @@
 #define INK_STROKE_MODELER_INTERNAL_STYLUS_STATE_MODELER_H_
 
 #include <deque>
-#include <ostream>
 
 #include "ink_stroke_modeler/internal/internal_types.h"
 #include "ink_stroke_modeler/params.h"
@@ -47,7 +46,7 @@ class StylusStateModeler {
  public:
   // Adds a position and state pair to the model. During stroke modeling, these
   // values will be taken from the raw input.
-  void Update(Vec2 position, const StylusState &state);
+  void Update(double time, const StylusState &state);
 
   // Clear the model and reset.
   void Reset(const StylusStateModelerParams &params);
@@ -68,23 +67,16 @@ class StylusStateModeler {
   void Restore();
 
  private:
-  struct PositionAndStylusState {
-    Vec2 position{0};
-    StylusState state;
-
-    PositionAndStylusState(Vec2 position_in, const StylusState &state_in)
-        : position(position_in), state(state_in) {}
-  };
-
   struct ModelerState {
     bool received_unknown_pressure = false;
     bool received_unknown_tilt = false;
     bool received_unknown_orientation = false;
 
-    std::deque<PositionAndStylusState> positions_and_stylus_states;
+    std::deque<StylusState> stylus_states;
   };
 
   ModelerState state_;
+  double last_time_ = 0;
 
   // Use a ModelerState + bool instead of optional<ModelerState> for
   // performance. ModelerState contains a std::deque, which has a non-trivial
@@ -93,6 +85,7 @@ class StylusStateModeler {
   // std::optional::reset().
   ModelerState saved_state_;
   bool save_active_ = false;
+  double saved_last_time_ = 0;
 
   StylusStateModelerParams params_;
 };