From 8389913cef6514cd4a5454d1da6ac54b864fc977 Mon Sep 17 00:00:00 2001 From: Jon T Moran Date: Thu, 19 Sep 2024 13:33:45 -0700 Subject: [PATCH] Fix a bug in pressure/tilt/orientation stylus modeling and the loop contraction mitigation that resulted in choosing the wrong point to project to on the raw input polyline. This fix has two parts: * Change the parameters of the `StylusStateModeler` to maintain a minimum number of samples and a minimum duration of samples -- this helps prevent cases in which the best projection point has already been discarded, which can happen on devices with a high input rate * When projecting along the stroke normal, track whether the intersection is to the left or the right of the stroke, and if there are multiple intersections choose the one that is to the "outside" of the turn -- this helps prevent edge cases in which the wrong point is chosen at a sharp turn The existing `max_input_samples` parameter was preserved in order maintain back-compatibility PiperOrigin-RevId: 676543657 --- .../internal/stylus_state_modeler.cc | 217 +++-- .../internal/stylus_state_modeler.h | 20 +- .../internal/stylus_state_modeler_test.cc | 768 +++++++++++------- ink_stroke_modeler/internal/utils.cc | 21 + ink_stroke_modeler/internal/utils.h | 11 +- ink_stroke_modeler/internal/utils_test.cc | 51 ++ ink_stroke_modeler/params.cc | 14 +- ink_stroke_modeler/params.h | 44 +- ink_stroke_modeler/params_test.cc | 32 +- ink_stroke_modeler/stroke_modeler.cc | 38 +- .../stroke_modeler_fuzz_test.cc | 8 +- ink_stroke_modeler/stroke_modeler_test.cc | 4 + 12 files changed, 841 insertions(+), 387 deletions(-) diff --git a/ink_stroke_modeler/internal/stylus_state_modeler.cc b/ink_stroke_modeler/internal/stylus_state_modeler.cc index 48420bd..0def530 100644 --- a/ink_stroke_modeler/internal/stylus_state_modeler.cc +++ b/ink_stroke_modeler/internal/stylus_state_modeler.cc @@ -15,6 +15,7 @@ #include "ink_stroke_modeler/internal/stylus_state_modeler.h" #include +#include #include #include @@ -25,6 +26,27 @@ namespace ink { namespace stroke_model { +namespace { + +bool ShouldDropOldestInput( + const StylusStateModelerParams ¶ms, + const std::deque &raw_input_and_stylus_states) { + if (params.use_stroke_normal_projection) { + return raw_input_and_stylus_states.size() > params.min_input_samples && + // Check the difference between the newest and second-oldest inputs + // -- if that's greater than `min_sample_duration` then we can drop + // the oldest without going below `min_sample_duration`. + // Since `min_input_samples` > 0, the clause above guarantees that we + // have at least two inputs. + (raw_input_and_stylus_states.back().time - + raw_input_and_stylus_states[1].time) > params.min_sample_duration; + + } else { + return raw_input_and_stylus_states.size() > params.max_input_samples; + } +} + +} // namespace void StylusStateModeler::Update(Vec2 position, Time time, const StylusState &state) { @@ -69,9 +91,7 @@ void StylusStateModeler::Update(Vec2 position, Time time, .orientation = state.orientation, }); - if (params_.max_input_samples < 0 || - state_.raw_input_and_stylus_states.size() > - static_cast(params_.max_input_samples)) { + while (ShouldDropOldestInput(params_, state_.raw_input_and_stylus_states)) { state_.raw_input_and_stylus_states.pop_front(); } } @@ -85,9 +105,105 @@ void StylusStateModeler::Reset(const StylusStateModelerParams ¶ms) { params_ = params; } -Result StylusStateModeler::Query(Vec2 position, - std::optional stroke_normal, - Time time) const { +namespace { + +// The location of the projection point along the raw input polyline. +struct RawInputProjection { + int segment_index; + float ratio_along_segment; +}; + +std::optional ProjectAlongStrokeNormal( + Vec2 position, Vec2 acceleration, Time time, Vec2 stroke_normal, + const std::deque &raw_input_polyline) { + // We track the best candidate separately for the left and right sides of the + // stroke, in case the closest projection is not in the right direction. + std::optional best_left_projection; + std::optional best_right_projection; + float best_distance_left = std::numeric_limits::infinity(); + float best_distance_right = std::numeric_limits::infinity(); + + // Update `best_projection` and `best_distance` if needed. + auto maybe_update_projection = + [](RawInputProjection candidate, float distance, + std::optional &best_projection, + float &best_distance) { + if (distance < best_distance) { + best_projection = candidate; + best_distance = distance; + } + }; + + for (decltype(raw_input_polyline.size()) i = 0; + i < raw_input_polyline.size() - 1; ++i) { + const Vec2 segment_start = raw_input_polyline[i].position; + const Vec2 segment_end = raw_input_polyline[i + 1].position; + + // Find the intersection of the stroke normal with the polyline segment. + std::optional segment_ratio = ProjectToSegmentAlongNormal( + segment_start, segment_end, position, stroke_normal); + if (!segment_ratio.has_value()) continue; + + Vec2 projection = Interp(segment_start, segment_end, *segment_ratio); + float distance = Distance(position, projection); + + // We update either the best left or the right projection, depending which + // side of the stroke it lies on -- recall that the stroke normal always + // points to the left. + RawInputProjection candidate{.segment_index = static_cast(i), + .ratio_along_segment = *segment_ratio}; + if (Vec2::DotProduct(projection - position, stroke_normal) < 0) { + maybe_update_projection(candidate, distance, best_right_projection, + best_distance_right); + } else { + maybe_update_projection(candidate, distance, best_left_projection, + best_distance_left); + } + } + + if (best_left_projection.has_value() && best_right_projection.has_value()) { + // We have candidate projections on both sides of the stroke, so we want to + // choose the one on the "outside" of the turn. The acceleration will always + // point to the "inside" of the curve, so we can compare it to the stroke + // normal (which always points left) to determine whether to use the left or + // right candidate. + return Vec2::DotProduct(stroke_normal, acceleration) > 0 + ? best_right_projection + : best_left_projection; + } + + // We have at most one projection -- return it if we have it. If we have + // neither, this returns std::nullopt, which is exactly what we want. + return best_right_projection.has_value() ? best_right_projection + : best_left_projection; +} + +std::optional ProjectToClosestPoint( + Vec2 position, const std::deque &raw_input_polyline) { + std::optional best_projection; + float min_distance = std::numeric_limits::infinity(); + for (decltype(raw_input_polyline.size()) i = 0; + i < raw_input_polyline.size() - 1; ++i) { + const Vec2 segment_start = raw_input_polyline[i].position; + const Vec2 segment_end = raw_input_polyline[i + 1].position; + float segment_ratio = + NearestPointOnSegment(segment_start, segment_end, position); + float distance = + Distance(position, Interp(segment_start, segment_end, segment_ratio)); + if (distance <= min_distance) { + best_projection = + RawInputProjection{.segment_index = static_cast(i), + .ratio_along_segment = segment_ratio}; + min_distance = distance; + } + } + return best_projection; +} + +} // namespace + +Result StylusStateModeler::Query(const TipState &tip, + std::optional stroke_normal) const { if (state_.raw_input_and_stylus_states.empty()) return { .position = {0, 0}, @@ -99,72 +215,45 @@ Result StylusStateModeler::Query(Vec2 position, .orientation = -1, }; - int closest_segment_index = -1; - float min_distance = std::numeric_limits::infinity(); - float interp_value = 0; - for (decltype(state_.raw_input_and_stylus_states.size()) i = 0; - i < state_.raw_input_and_stylus_states.size() - 1; ++i) { - const Vec2 segment_start = state_.raw_input_and_stylus_states[i].position; - const Vec2 segment_end = state_.raw_input_and_stylus_states[i + 1].position; - float param = 0; - if (params_.use_stroke_normal_projection && stroke_normal.has_value()) { - std::optional temp_param = ProjectToSegmentAlongNormal( - segment_start, segment_end, position, *stroke_normal); - if (!temp_param.has_value()) continue; - param = *temp_param; - } else { - param = NearestPointOnSegment(segment_start, segment_end, position); - } - float distance = - Distance(position, Interp(segment_start, segment_end, param)); - if (distance <= min_distance) { - closest_segment_index = i; - min_distance = distance; - interp_value = param; - } - } + std::optional projection = + params_.use_stroke_normal_projection && stroke_normal.has_value() + ? ProjectAlongStrokeNormal(tip.position, tip.acceleration, tip.time, + *stroke_normal, + state_.raw_input_and_stylus_states) + : ProjectToClosestPoint(tip.position, + state_.raw_input_and_stylus_states); - if (closest_segment_index < 0) { - const auto &state = + Result projected_result; + if (projection.has_value()) { + projected_result = InterpResult( + state_.raw_input_and_stylus_states[projection->segment_index], + state_.raw_input_and_stylus_states[projection->segment_index + 1], + projection->ratio_along_segment); + } else { + // We didn't find an appropriate projection; fall back to projecting to the + // closest endpoint of the raw input polyline. + projected_result = Distance(state_.raw_input_and_stylus_states.front().position, - position) < + tip.position) < Distance(state_.raw_input_and_stylus_states.back().position, - position) + tip.position) ? state_.raw_input_and_stylus_states.front() : state_.raw_input_and_stylus_states.back(); - return { - .position = state.position, - .velocity = state.velocity, - .acceleration = state.acceleration, - .time = time, - .pressure = state_.received_unknown_pressure ? -1 : state.pressure, - .tilt = state_.received_unknown_tilt ? -1 : state.tilt, - .orientation = - state_.received_unknown_orientation ? -1 : state.orientation, - }; } - auto from_state = state_.raw_input_and_stylus_states[closest_segment_index]; - auto to_state = state_.raw_input_and_stylus_states[closest_segment_index + 1]; + // Correct the time and strip missing fields before returning. + projected_result.time = tip.time; + if (state_.received_unknown_pressure) { + projected_result.pressure = -1; + } + if (state_.received_unknown_tilt) { + projected_result.tilt = -1; + } + if (state_.received_unknown_orientation) { + projected_result.orientation = -1; + } - return Result{ - .position = Interp(from_state.position, to_state.position, interp_value), - .velocity = Interp(from_state.velocity, to_state.velocity, interp_value), - .acceleration = - Interp(from_state.acceleration, to_state.acceleration, interp_value), - .time = time, - .pressure = - state_.received_unknown_pressure - ? -1 - : Interp(from_state.pressure, to_state.pressure, interp_value), - .tilt = state_.received_unknown_tilt - ? -1 - : Interp(from_state.tilt, to_state.tilt, interp_value), - .orientation = state_.received_unknown_orientation - ? -1 - : InterpAngle(from_state.orientation, - to_state.orientation, interp_value), - }; + return projected_result; } void StylusStateModeler::Save() { diff --git a/ink_stroke_modeler/internal/stylus_state_modeler.h b/ink_stroke_modeler/internal/stylus_state_modeler.h index 35d982f..344d6b7 100644 --- a/ink_stroke_modeler/internal/stylus_state_modeler.h +++ b/ink_stroke_modeler/internal/stylus_state_modeler.h @@ -54,19 +54,29 @@ class StylusStateModeler { // Clear the model and reset. void Reset(const StylusStateModelerParams ¶ms); - // Query the model for the `Result` at the given position. During stroke + // Query the model for the `Result` at the given tip state. During stroke // modeling, the position will be taken from the modeled input. // // If no Update() calls have been received since the last Reset(), this will - // return {.pressure = -1, .tilt = -1, .orientation = -1}. + // return {.position = {0, 0}, + // .velocity = {0, 0}, + // .acceleration = {0, 0}, + // .time = Time(0), + // .pressure = -1, + // .tilt = -1, + // .orientation = -1} // // `stroke_normal` is only used if - // `project_to_segment_along_normal_is_enabled` is true in the params. + // `StylusStateModelerParams::use_stroke_normal_projection` is true. // // Note: While this returns a `Result`, the return value does not represent an // end result, but merely a container to hold all the relevant values. - Result Query(Vec2 position, std::optional stroke_normal, - Time time) const; + Result Query(const TipState &tip, std::optional stroke_normal) const; + + // The number of input samples currently held. Exposed for testing. + int InputSampleCount() const { + return state_.raw_input_and_stylus_states.size(); + } // Saves the current state of the stylus state modeler. See comment on // StrokeModeler::Save() for more details. diff --git a/ink_stroke_modeler/internal/stylus_state_modeler_test.cc b/ink_stroke_modeler/internal/stylus_state_modeler_test.cc index 8e0e06a..bd3dc00 100644 --- a/ink_stroke_modeler/internal/stylus_state_modeler_test.cc +++ b/ink_stroke_modeler/internal/stylus_state_modeler_test.cc @@ -39,16 +39,25 @@ const Result kUnknownResult{.position = {0, 0}, .pressure = -1, .tilt = -1, .orientation = -1}; -constexpr StylusStateModelerParams kNormalProjectionParams{ - .max_input_samples = 10, +const StylusStateModelerParams kNormalProjectionParams{ .use_stroke_normal_projection = true, + .min_input_samples = 5, + .min_sample_duration = Duration(.3), }; TEST(StylusStateModelerTest, QueryEmpty) { StylusStateModeler modeler; - EXPECT_EQ(modeler.Query({0, 0}, std::optional({0, 1}), Time(0)), + EXPECT_EQ(modeler.Query({.position = {0, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), kUnknownResult); - EXPECT_EQ(modeler.Query({-5, 3}, std::optional({0, 1}), Time(0.1)), + EXPECT_EQ(modeler.Query({.position = {-5, 3}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.1)}, + Vec2{0, 1}), kUnknownResult); } @@ -56,7 +65,11 @@ TEST(StylusStateModelerTest, QuerySingleInput) { StylusStateModeler modeler; modeler.Update({0, 0}, Time(0), {.pressure = 0.75, .tilt = 0.75, .orientation = 0.75}); - EXPECT_THAT(modeler.Query({0, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {0, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -68,7 +81,11 @@ TEST(StylusStateModelerTest, QuerySingleInput) { .orientation = .75, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, 1}), Time(0.1)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.1)}, + Vec2{0, 1}), ResultNear({.position = {0, 0}, .velocity = {0, 0}, .acceleration = {0, 0}, @@ -84,7 +101,11 @@ TEST(StylusStateModelerTest, QuerySingleInputWithNormalProjection) { modeler.Reset(kNormalProjectionParams); modeler.Update({0, 0}, Time(0), {.pressure = 0.75, .tilt = 0.75, .orientation = 0.75}); - EXPECT_THAT(modeler.Query({0, 0}, std::optional({0, 1.1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {0, 0}, + .velocity = {0, 0}, + .acceleration = {1, 1}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -96,7 +117,11 @@ TEST(StylusStateModelerTest, QuerySingleInputWithNormalProjection) { .orientation = .75, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, 1.1}), Time(0.1)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {1, 1}, + .time = Time(0.1)}, + Vec2{0, 1}), ResultNear({.position = {0, 0}, .velocity = {0, 0}, .acceleration = {0, 0}, @@ -118,7 +143,11 @@ TEST(StylusStateModelerTest, QueryMultipleInputs) { modeler.Update({3.5, 4}, Time(0.3), {.pressure = .2, .tilt = .2, .orientation = .2}); - EXPECT_THAT(modeler.Query({0, 2}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {0, 2}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {.5, 1.5}, @@ -130,7 +159,11 @@ TEST(StylusStateModelerTest, QueryMultipleInputs) { .orientation = .1, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 2}, std::optional({0, -0.5}), Time(0.1)), + EXPECT_THAT(modeler.Query({.position = {1, 2}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.1)}, + Vec2{0, -0.5}), ResultNear({.position = {1, 1.5}, .velocity = {5, 0}, .acceleration = {50, 0}, @@ -139,7 +172,11 @@ TEST(StylusStateModelerTest, QueryMultipleInputs) { .tilt = .7, .orientation = .3}, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({2, 1.5}, std::optional({2, 2}), Time(0.1)), + EXPECT_THAT(modeler.Query({.position = {2, 1.5}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.1)}, + Vec2{2, 2}), ResultNear( { .position = {2, 1.5}, @@ -151,8 +188,11 @@ TEST(StylusStateModelerTest, QueryMultipleInputs) { .orientation = .7, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({2.5, 1.875}, std::optional({-0.25, 0.125}), - Time(0.2)), + EXPECT_THAT(modeler.Query({.position = {2.5, 1.875}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.2)}, + Vec2{-0.25, 0.125}), ResultNear( { .position = {2.25, 2}, @@ -164,8 +204,11 @@ TEST(StylusStateModelerTest, QueryMultipleInputs) { .orientation = .6, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({2.5, 3.125}, std::optional({0.25, -0.125}), - Time(0.22)), + EXPECT_THAT(modeler.Query({.position = {2.5, 3.125}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.22)}, + Vec2{0.25, -0.125}), ResultNear( { .position = {2.75, 3}, @@ -177,33 +220,43 @@ TEST(StylusStateModelerTest, QueryMultipleInputs) { .orientation = .4, }, kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({2.5, 4}, std::optional({0.5, -0.5}), Time(0.25)), - ResultNear( - { - .position = {3, 3.5}, - .velocity = {10, 20}, - .acceleration = {-50, 200}, - .time = Time(0.25), - .pressure = .8, - .tilt = .1, - .orientation = .3, - }, - kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({3, 4}, std::optional({0.25, -0.25}), Time(0.29)), - ResultNear( - { - .position = {3.25, 3.75}, - .velocity = {7.5, 12.5}, - .acceleration = {-50, 25}, - .time = Time(0.29), - .pressure = .5, - .tilt = .15, - .orientation = .25, - }, - kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({4, 4}, std::optional({0, 1}), Time(0.31)), + EXPECT_THAT(modeler.Query({.position = {2.5, 4}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.25)}, + Vec2{0.5, -0.5}), + ResultNear( + { + .position = {3, 3.5}, + .velocity = {10, 20}, + .acceleration = {-50, 200}, + .time = Time(0.25), + .pressure = .8, + .tilt = .1, + .orientation = .3, + }, + kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {3, 4}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.29)}, + Vec2{0.25, -0.25}), + ResultNear( + { + .position = {3.25, 3.75}, + .velocity = {7.5, 12.5}, + .acceleration = {-50, 25}, + .time = Time(0.29), + .pressure = .5, + .tilt = .15, + .orientation = .25, + }, + kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {4, 4}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.31)}, + Vec2{0, 1}), ResultNear( { .position = {3.5, 4}, @@ -229,7 +282,11 @@ TEST(StylusStateModelerTest, QueryMultipleInputsWithNormalProjection) { modeler.Update({3.5, 4}, Time(0.3), {.pressure = .2, .tilt = .2, .orientation = .2}); - EXPECT_THAT(modeler.Query({0, 2}, std::optional({0, 1.1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {0, 2}, + .velocity = {0, 0}, + .acceleration = {0, 1}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {.5, 1.5}, @@ -241,16 +298,24 @@ TEST(StylusStateModelerTest, QueryMultipleInputsWithNormalProjection) { .orientation = .1, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 2}, std::optional({0, -0.55}), Time(0.1)), - ResultNear({.position = {1, 1.5}, - .velocity = {5, 0}, - .acceleration = {50, 0}, + EXPECT_THAT(modeler.Query({.position = {1, 2}, + .velocity = {0, 0}, + .acceleration = {0, -1}, + .time = Time(0.1)}, + Vec2{-1, 1}), + ResultNear({.position = {1.5, 1.5}, + .velocity = {10, 0}, + .acceleration = {100, 0}, .time = Time(0.1), - .pressure = .4, - .tilt = .7, - .orientation = .3}, + .pressure = .5, + .tilt = .6, + .orientation = .5}, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({2, 1.5}, std::optional({2, 2.1}), Time(0.1)), + EXPECT_THAT(modeler.Query({.position = {2, 1.5}, + .velocity = {0, 0}, + .acceleration = {0, -1}, + .time = Time(0.1)}, + Vec2{-1, 2}), ResultNear( { .position = {2, 1.5}, @@ -262,47 +327,50 @@ TEST(StylusStateModelerTest, QueryMultipleInputsWithNormalProjection) { .orientation = .7, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({2.5, 1.875}, std::optional({-0.3, 0.125}), - Time(0.2)), - ResultNear({.position = {2.24138, 1.98276}, - .velocity = {13.79310, 4.82759}, - .acceleration = {101.72414, 48.27586}, + EXPECT_THAT(modeler.Query({.position = {2.5, 2}, + .velocity = {0, 0}, + .acceleration = {0, 2}, + .time = Time(0.2)}, + Vec2{-3, 0}), + ResultNear({.position = {2.25, 2}, + .velocity = {13.75, 5}, + .acceleration = {100, 50}, .time = Time(0.2), - .pressure = 0.648276, - .tilt = 0.403448, - .orientation = 0.603448}, + .pressure = 0.65, + .tilt = 0.4, + .orientation = 0.6}, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({2.5, 3.125}, std::optional({0.3, -0.125}), - Time(0.22)), - ResultNear({.position = {2.75862, 3.01724}, - .velocity = {11.20690, 15.17241}, - .acceleration = {-1.72414, 151.72414}, + EXPECT_THAT(modeler.Query({.position = {2.5, 3}, + .velocity = {0, 0}, + .acceleration = {0, -1}, + .time = Time(0.22)}, + Vec2{-0.5, 0}), + ResultNear({.position = {2.75, 3}, + .velocity = {11.25, 15}, + .acceleration = {0, 150}, .time = Time(0.22), - .pressure = 0.751724, - .tilt = 0.196552, - .orientation = 0.396552}, + .pressure = 0.75, + .tilt = 0.2, + .orientation = 0.4}, kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({2.5, 4}, std::optional({0.5, -0.55}), Time(0.25)), - ResultNear({.position = {2.98387, 3.46774}, - .velocity = {10.08064, 19.67742}, - .acceleration = {-46.77420, 196.77420}, - .time = Time(0.25), - .pressure = 0.796774, - .tilt = 0.106452, - .orientation = 0.306452}, - kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({3, 4}, std::optional({0.3, -0.25}), Time(0.29)), - ResultNear({.position = {3.27273, 3.77273}, - .velocity = {7.27273, 11.81818}, - .acceleration = {-50.00000, 9.09090}, - .time = Time(0.29), - .pressure = 0.472727, - .tilt = 0.154545, - .orientation = 0.245455}, - kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({4, 4}, std::optional({0, 1.1}), Time(0.31)), + EXPECT_THAT(modeler.Query({.position = {3.25, 4}, + .velocity = {0, 0}, + .acceleration = {0, -2}, + .time = Time(0.29)}, + Vec2{0, 0.1}), + ResultNear({.position = {3.25, 3.75}, + .velocity = {7.5, 12.5}, + .acceleration = {-50, 25}, + .time = Time(0.29), + .pressure = 0.5, + .tilt = 0.15, + .orientation = 0.25}, + kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {4, 4}, + .velocity = {0, 0}, + .acceleration = {0, -0.5}, + .time = Time(0.31)}, + Vec2{0, 1}), ResultNear( { .position = {3.5, 4}, @@ -316,148 +384,135 @@ TEST(StylusStateModelerTest, QueryMultipleInputsWithNormalProjection) { kTol, kAccelTol)); } -TEST(StylusStateModelerTest, QueryStaleInputsAreDiscarded) { +TEST(StylusStateModelerTest, + StrokeNormalProjectionChoosesCorrectTargetIfMultipleIntersections) { StylusStateModeler modeler; + modeler.Reset(kNormalProjectionParams); + modeler.Update({0, 0}, Time(0), {.pressure = 0, .tilt = 0, .orientation = 0}); + modeler.Update({0, 4}, Time(0.1), + {.pressure = 0.2, .tilt = 0.2, .orientation = 0.2}); + modeler.Update({2, 4}, Time(0.2), + {.pressure = 0.4, .tilt = 0.4, .orientation = 0.4}); + modeler.Update({2, 0}, Time(0.3), + {.pressure = 0.6, .tilt = 0.6, .orientation = 0.6}); + + // If there are multiple intersections on the same side of the query, take the + // closest. + EXPECT_THAT(modeler.Query({.position = {-1, 2}, + .velocity = {0, 0}, + .acceleration = {0, -0.5}, + .time = Time(0.15)}, + Vec2{1, 0}), + ResultNear({.position = {0, 2}, + .velocity = {0, 20}, + .acceleration = {0, 200}, + .time = Time(0.15), + .pressure = .1, + .tilt = .1, + .orientation = .1}, + kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {3, 2}, + .velocity = {0, 0}, + .acceleration = {0, -0.5}, + .time = Time(0.15)}, + Vec2{1, 0}), + ResultNear({.position = {2, 2}, + .velocity = {10, -20}, + .acceleration = {0, -400}, + .time = Time(0.15), + .pressure = .5, + .tilt = .5, + .orientation = .5}, + kTol, kAccelTol)); + + // If there are intersections on either side of the query, take the one in the + // opposite direction of the acceleration. + EXPECT_THAT(modeler.Query({.position = {1, 2}, + .velocity = {0, 0}, + .acceleration = {1, -1}, + .time = Time(0.15)}, + Vec2{1, 0}), + ResultNear({.position = {0, 2}, + .velocity = {0, 20}, + .acceleration = {0, 200}, + .time = Time(0.15), + .pressure = .1, + .tilt = .1, + .orientation = .1}, + kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {1, 2}, + .velocity = {0, 0}, + .acceleration = {-1, -1}, + .time = Time(0.15)}, + Vec2{1, 0}), + ResultNear({.position = {2, 2}, + .velocity = {10, -20}, + .acceleration = {0, -400}, + .time = Time(0.15), + .pressure = .5, + .tilt = .5, + .orientation = .5}, + kTol, kAccelTol)); +} + +TEST(StylusStateModelerTest, StaleInputsAreDiscardedClosestPointProjection) { + StylusStateModeler modeler; + modeler.Reset( + {.max_input_samples = 3, .use_stroke_normal_projection = false}); + + EXPECT_EQ(modeler.InputSampleCount(), 0); modeler.Update({1, 1}, Time(0), {.pressure = .6, .tilt = .5, .orientation = .4}); + EXPECT_EQ(modeler.InputSampleCount(), 1); modeler.Update({-1, 2}, Time(0.1), {.pressure = .3, .tilt = .7, .orientation = .6}); + EXPECT_EQ(modeler.InputSampleCount(), 2); modeler.Update({-4, 0}, Time(0.2), {.pressure = .9, .tilt = .7, .orientation = .3}); + EXPECT_EQ(modeler.InputSampleCount(), 3); modeler.Update({-6, -3}, Time(0.3), {.pressure = .4, .tilt = .3, .orientation = .5}); + EXPECT_EQ(modeler.InputSampleCount(), 3); modeler.Update({-5, -5}, Time(0.4), {.pressure = .3, .tilt = .3, .orientation = .1}); - modeler.Update({-3, -4}, Time(0.5), - {.pressure = .6, .tilt = .8, .orientation = .3}); - modeler.Update({-6, -7}, Time(0.6), - {.pressure = .9, .tilt = .8, .orientation = .1}); - modeler.Update({-9, -8}, Time(0.7), - {.pressure = .8, .tilt = .2, .orientation = .2}); - modeler.Update({-11, -5}, Time(0.8), - {.pressure = .2, .tilt = .4, .orientation = .7}); - modeler.Update({-10, -2}, Time(0.9), - {.pressure = .7, .tilt = .3, .orientation = .2}); + EXPECT_EQ(modeler.InputSampleCount(), 3); +} - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), - ResultNear( - { - .position = {1, 1}, - .velocity = {0, 0}, - .acceleration = {0, 0}, - .time = Time(0), - .pressure = .6, - .tilt = .5, - .orientation = .4, - }, - kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 3.5}, std::optional({-1, -2}), Time(0.1)), - ResultNear( - { - .position = {0, 1.5}, - .velocity = {-10, 5}, - .acceleration = {-100, 50}, - .time = Time(0.1), - .pressure = .45, - .tilt = .6, - .orientation = .5, - }, - kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({-3, 17. / 6}, std::optional({1, -1.5}), Time(0.2)), - ResultNear( - { - .position = {-2, 4. / 3.}, - .velocity = {-70. / 3., 0}, - .acceleration = {-166.666656, -33.33334}, - .time = Time(0.2), - .pressure = .5, - .tilt = .7, - .orientation = .5, - }, - kTol, kAccelTol)); - - // This causes the point at {1, 1} to be discarded. - modeler.Update({-8, 0}, Time(1), - {.pressure = .6, .tilt = .8, .orientation = .9}); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0.1)), - ResultNear( - { - .position = {-1, 2}, - .velocity = {-20, 10}, - .acceleration = {-200, 100}, - .time = Time(0.1), - .pressure = .3, - .tilt = .7, - .orientation = .6, - }, - kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 3.5}, std::optional({0, 1}), Time(0.2)), - ResultNear( - { - .position = {-1, 2}, - .velocity = {-20, 10}, - .acceleration = {-200, 100}, - .time = Time(0.2), - .pressure = .3, - .tilt = .7, - .orientation = .6, - }, - kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({-3, 17. / 6}, std::optional({1, -1.5}), Time(0.3)), - ResultNear( - { - .position = {-2, 4. / 3.}, - .velocity = {-70. / 3., 0}, - .acceleration = {-166.666656, -33.33334}, - .time = Time(0.3), - .pressure = .5, - .tilt = .7, - .orientation = .5, - }, - kTol, kAccelTol)); - - // This causes the point at {-1, 2} to be discarded. - modeler.Update({-8, 0}, Time(1.1), - {.pressure = .6, .tilt = .8, .orientation = .9}); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0.3)), - ResultNear( - { - .position = {-4, 0}, - .velocity = {-30, -20}, - .acceleration = {-100, -300}, - .time = Time(0.3), - .pressure = .9, - .tilt = .7, - .orientation = .3, - }, - kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({1, 3.5}, std::optional({0, 1}), Time(0.4)), - ResultNear( - { - .position = {-4, 0}, - .velocity = {-30, -20}, - .acceleration = {-100, -300}, - .time = Time(0.4), - .pressure = .9, - .tilt = .7, - .orientation = .3, - }, - kTol, kAccelTol)); - EXPECT_THAT( - modeler.Query({-3, 17. / 6}, std::optional({-6, 2}), Time(0.5)), - ResultNear( - { - .position = {-4, 0}, - .velocity = {-30, -20}, - .acceleration = {-100, -300}, - .time = Time(0.5), - .pressure = .9, - .tilt = .7, - .orientation = .3, - }, - kTol, kAccelTol)); +TEST(StylusStateModelerTest, StaleInputsAreDiscardedStrokeNormalProjection) { + StylusStateModeler modeler; + modeler.Reset({ + .use_stroke_normal_projection = true, + .min_input_samples = 3, + .min_sample_duration = Duration(.5), + }); + + EXPECT_EQ(modeler.InputSampleCount(), 0); + modeler.Update({1, 1}, Time(0), + {.pressure = .6, .tilt = .5, .orientation = .4}); + EXPECT_EQ(modeler.InputSampleCount(), 1); + modeler.Update({-1, 2}, Time(0.1), + {.pressure = .3, .tilt = .7, .orientation = .6}); + EXPECT_EQ(modeler.InputSampleCount(), 2); + modeler.Update({-4, 0}, Time(0.2), + {.pressure = .9, .tilt = .7, .orientation = .3}); + EXPECT_EQ(modeler.InputSampleCount(), 3); + + // We've hit the minimum number of samples, but not the minimum duration. + modeler.Update({-6, -3}, Time(0.3), + {.pressure = .4, .tilt = .3, .orientation = .5}); + EXPECT_EQ(modeler.InputSampleCount(), 4); + + // Now we've hit the minimum duration as well, so we can drop the two oldest + // inputs. + modeler.Update({-5, -5}, Time(1), + {.pressure = .3, .tilt = .3, .orientation = .1}); + EXPECT_EQ(modeler.InputSampleCount(), 3); + + // Even though we meet the minimum duration with just two inputs, we don't + // drop below the minimum number of samples. + modeler.Update({-5, -5}, Time(2), + {.pressure = .3, .tilt = .3, .orientation = .1}); + EXPECT_EQ(modeler.InputSampleCount(), 3); } TEST(StylusStateModelerTest, QueryCyclicOrientationInterpolation) { @@ -469,16 +524,36 @@ TEST(StylusStateModelerTest, QueryCyclicOrientationInterpolation) { modeler.Update({0, 2}, Time(2), {.pressure = 0, .tilt = 0, .orientation = 1.6 * kPi}); - EXPECT_NEAR( - modeler.Query({0, .25}, std::optional({1, 0}), Time(0)).orientation, - 1.9 * kPi, kTol); - EXPECT_NEAR( - modeler.Query({0, .75}, std::optional({1, 0}), Time(1)).orientation, - .1 * kPi, kTol); - EXPECT_NEAR(modeler.Query({0, 1.25}, std::optional({1, 0}), Time(1.5)) + EXPECT_NEAR(modeler + .Query({.position = {0, .25}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{1, 0}) + .orientation, + 1.9 * kPi, kTol); + EXPECT_NEAR(modeler + .Query({.position = {0, .75}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1)}, + Vec2{1, 0}) + .orientation, + .1 * kPi, kTol); + EXPECT_NEAR(modeler + .Query({.position = {0, 1.25}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1.5)}, + Vec2{1, 0}) .orientation, .05 * kPi, kTol); - EXPECT_NEAR(modeler.Query({0, 1.75}, std::optional({1, 0}), Time(2)) + EXPECT_NEAR(modeler + .Query({.position = {0, 1.75}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(2)}, + Vec2{1, 0}) .orientation, 1.75 * kPi, kTol); } @@ -490,27 +565,38 @@ TEST(StylusStateModelerTest, QueryAndReset) { {.pressure = .4, .tilt = .9, .orientation = .1}); modeler.Update({7, 8}, Time(1), {.pressure = .1, .tilt = .2, .orientation = .5}); - EXPECT_THAT( - modeler.Query({10, 12}, std::optional({0.5, -0.5}), Time(0)), - ResultNear( - { - .position = {7, 8}, - .velocity = {3, 3}, - .acceleration = {3, 3}, - .time = Time(0), - .pressure = .1, - .tilt = .2, - .orientation = .5, - }, - kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {10, 12}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0.5, -0.5}), + ResultNear( + { + .position = {7, 8}, + .velocity = {3, 3}, + .acceleration = {3, 3}, + .time = Time(0), + .pressure = .1, + .tilt = .2, + .orientation = .5, + }, + kTol, kAccelTol)); modeler.Reset(StylusStateModelerParams{}); - EXPECT_EQ(modeler.Query({10, 12}, std::optional({0.5, -0.5}), Time(0)), + EXPECT_EQ(modeler.Query({.position = {10, 12}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0.5, -0.5}), kUnknownResult); modeler.Update({-1, 4}, Time(2), {.pressure = .4, .tilt = .6, .orientation = .8}); - EXPECT_THAT(modeler.Query({6, 7}, std::optional({-7, -3}), Time(2)), + EXPECT_THAT(modeler.Query({.position = {6, 7}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(2)}, + Vec2{-7, -3}), ResultNear( { .position = {-1, 4}, @@ -525,7 +611,11 @@ TEST(StylusStateModelerTest, QueryAndReset) { modeler.Update({-3, 0}, Time(3), {.pressure = .7, .tilt = .2, .orientation = .5}); - EXPECT_THAT(modeler.Query({-2, 2}, std::optional({0, 1}), Time(2.5)), + EXPECT_THAT(modeler.Query({.position = {-2, 2}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(2.5)}, + Vec2{0, 1}), ResultNear( { .position = {-2, 2}, @@ -537,7 +627,11 @@ TEST(StylusStateModelerTest, QueryAndReset) { .orientation = .65, }, kTol, kAccelTol)); - EXPECT_THAT(modeler.Query({0, 5}, std::optional({-0.4, 0.2}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {0, 5}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{-0.4, 0.2}), ResultNear( { .position = {-1, 4}, @@ -558,7 +652,11 @@ TEST(StylusStateModelerTest, UpdateWithUnknownState) { {.pressure = .1, .tilt = .2, .orientation = .3}); modeler.Update({2, 3}, Time(1), {.pressure = .3, .tilt = .4, .orientation = .5}); - EXPECT_THAT(modeler.Query({2, 2}, std::optional({-0.5, 0.5}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {2, 2}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{-0.5, 0.5}), ResultNear({.position = {1.5, 2.5}, .velocity = {0.5, 0.5}, .acceleration = {0.5, 0.5}, @@ -569,31 +667,46 @@ TEST(StylusStateModelerTest, UpdateWithUnknownState) { kTol, kAccelTol)); modeler.Update({5, 5}, Time(2), kUnknownState); - EXPECT_EQ(modeler.Query({5, 5}, std::optional({-0.5, 0.5}), Time(1)), + EXPECT_EQ(modeler.Query({.position = {5, 5}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1)}, + Vec2{-0.5, 0.5}), kUnknownResult); modeler.Update({2, 3}, Time(3), {.pressure = .3, .tilt = .4, .orientation = .5}); - EXPECT_EQ(modeler.Query({1, 2}, std::optional({-0.5, 0.5}), Time(2)), + EXPECT_EQ(modeler.Query({.position = {1, 2}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(2)}, + Vec2{-0.5, 0.5}), kUnknownResult); modeler.Update({-1, 3}, Time(4), kUnknownState); - EXPECT_EQ(modeler.Query({7, 9}, std::optional({-0.5, 0.5}), Time(3)), + EXPECT_EQ(modeler.Query({.position = {7, 9}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(3)}, + Vec2{-0.5, 0.5}), kUnknownResult); modeler.Reset(StylusStateModelerParams{}); modeler.Update({3, 3}, Time(5), {.pressure = .7, .tilt = .6, .orientation = .5}); - EXPECT_THAT( - modeler.Query({3, 3}, std::optional({-0.5, 0.5}), Time(0.2)), - ResultNear({.position = {3, 3}, - .velocity = {0, 0}, - .acceleration = {0, 0}, - .time = Time(0.2), - .pressure = .7, - .tilt = .6, - .orientation = .5}, - kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {3, 3}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.2)}, + Vec2{-0.5, 0.5}), + ResultNear({.position = {3, 3}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.2), + .pressure = .7, + .tilt = .6, + .orientation = .5}, + kTol, kAccelTol)); } TEST(StylusStateModelerTest, StrokeNormalIgnored) { @@ -603,10 +716,17 @@ TEST(StylusStateModelerTest, StrokeNormalIgnored) { {.pressure = .4, .tilt = .9, .orientation = .1}); modeler.Update({7, 8}, Time(1), {.pressure = .1, .tilt = .2, .orientation = .5}); - EXPECT_THAT( - modeler.Query({5, 7}, std::optional({0.5, -0.5}), Time(0.2)), - ResultNear(modeler.Query({5, 7}, std::optional({0, 1}), Time(0.2)), - kTol, kAccelTol)); + EXPECT_THAT(modeler.Query({.position = {5, 7}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.2)}, + Vec2{0.5, -0.5}), + ResultNear(modeler.Query({.position = {5, 7}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0.2)}, + Vec2{0, 1}), + kTol, kAccelTol)); } TEST(StylusStateModelerTest, ModelPressureOnly) { @@ -614,7 +734,11 @@ TEST(StylusStateModelerTest, ModelPressureOnly) { modeler.Update({0, 0}, Time(0), {.pressure = .5, .tilt = -2, .orientation = -.1}); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -629,7 +753,11 @@ TEST(StylusStateModelerTest, ModelPressureOnly) { modeler.Update({2, 0}, Time(1), {.pressure = .7, .tilt = -2, .orientation = -.1}); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, -1}), Time(1)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1)}, + Vec2{0, -1}), ResultNear( { .position = {1, 0}, @@ -648,7 +776,11 @@ TEST(StylusStateModelerTest, ModelTiltOnly) { modeler.Update({0, 0}, Time(0), {.pressure = -2, .tilt = .5, .orientation = -.1}); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -663,7 +795,11 @@ TEST(StylusStateModelerTest, ModelTiltOnly) { modeler.Update({2, 0}, Time(1), {.pressure = -2, .tilt = .3, .orientation = -.1}); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, -1}), Time(1)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1)}, + Vec2{0, -1}), ResultNear( { .position = {1, 0}, @@ -682,7 +818,11 @@ TEST(StylusStateModelerTest, ModelOrientationOnly) { modeler.Update({0, 0}, Time(0), {.pressure = -2, .tilt = -.1, .orientation = 1}); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -697,7 +837,11 @@ TEST(StylusStateModelerTest, ModelOrientationOnly) { modeler.Update({2, 0}, Time(1), {.pressure = -2, .tilt = -.3, .orientation = 2}); - EXPECT_THAT(modeler.Query({1, 1}, std::optional({0, -1}), Time(1)), + EXPECT_THAT(modeler.Query({.position = {1, 1}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1)}, + Vec2{0, -1}), ResultNear( { .position = {1, 0}, @@ -716,7 +860,11 @@ TEST(StylusStateModelerTest, DropFieldsOneByOne) { modeler.Update({0, 0}, Time(0), {.pressure = .5, .tilt = .5, .orientation = .5}); - EXPECT_THAT(modeler.Query({1, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {1, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -731,7 +879,11 @@ TEST(StylusStateModelerTest, DropFieldsOneByOne) { modeler.Update({2, 0}, Time(1), {.pressure = .3, .tilt = .7, .orientation = -1}); - EXPECT_THAT(modeler.Query({1, 0}, std::optional({0, 1}), Time(1)), + EXPECT_THAT(modeler.Query({.position = {1, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(1)}, + Vec2{0, 1}), ResultNear( { .position = {1, 0}, @@ -746,7 +898,11 @@ TEST(StylusStateModelerTest, DropFieldsOneByOne) { modeler.Update({4, 0}, Time(2), {.pressure = .1, .tilt = -1, .orientation = 1}); - EXPECT_THAT(modeler.Query({3, 0}, std::optional({0, 1}), Time(2)), + EXPECT_THAT(modeler.Query({.position = {3, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(2)}, + Vec2{0, 1}), ResultNear( { .position = {3, 0}, @@ -761,7 +917,11 @@ TEST(StylusStateModelerTest, DropFieldsOneByOne) { modeler.Update({6, 0}, Time(3), {.pressure = -1, .tilt = .2, .orientation = 0}); - EXPECT_THAT(modeler.Query({5, 0}, std::optional({0, 1}), Time(3)), + EXPECT_THAT(modeler.Query({.position = {5, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(3)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -776,7 +936,11 @@ TEST(StylusStateModelerTest, DropFieldsOneByOne) { modeler.Update({8, 0}, Time(4), {.pressure = .3, .tilt = .4, .orientation = .5}); - EXPECT_THAT(modeler.Query({7, 0}, std::optional({0, 1}), Time(4)), + EXPECT_THAT(modeler.Query({.position = {7, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(4)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -790,12 +954,20 @@ TEST(StylusStateModelerTest, DropFieldsOneByOne) { kTol, kAccelTol)); modeler.Reset(StylusStateModelerParams{}); - EXPECT_THAT(modeler.Query({1, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {1, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear(kUnknownResult, kTol, kAccelTol)); modeler.Update({0, 0}, Time(5), {.pressure = .1, .tilt = .8, .orientation = .3}); - EXPECT_THAT(modeler.Query({1, 0}, std::optional({0, 1}), Time(5)), + EXPECT_THAT(modeler.Query({.position = {1, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(5)}, + Vec2{0, 1}), ResultNear( { .position = {0, 0}, @@ -832,7 +1004,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { modeler.Update({-10, -2}, Time(9), {.pressure = .7, .tilt = .3, .orientation = .2}); - ASSERT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), + ASSERT_THAT(modeler.Query({.position = {2, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {1, 1}, @@ -848,7 +1024,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { // Calling restore with no save should have no effect. modeler.Restore(); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {2, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {1, 1}, @@ -868,7 +1048,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { {.pressure = .6, .tilt = .8, .orientation = .9}); modeler.Update({-8, 0}, Time(11), {.pressure = .6, .tilt = .8, .orientation = .9}); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {2, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {-4, 0}, @@ -883,7 +1067,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { // Restoring should revert the updates. modeler.Restore(); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {2, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {1, 1}, @@ -902,7 +1090,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { {.pressure = .6, .tilt = .8, .orientation = .9}); modeler.Update({-8, 0}, Time(13), {.pressure = .6, .tilt = .8, .orientation = .9}); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {2, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {-4, 0}, @@ -915,7 +1107,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { }, kTol, kAccelTol)); modeler.Restore(); - EXPECT_THAT(modeler.Query({2, 0}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {2, 0}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {1, 1}, @@ -933,7 +1129,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { modeler.Reset(StylusStateModelerParams{}); modeler.Update({-1, 4}, Time(14), {.pressure = .4, .tilt = .6, .orientation = .8}); - EXPECT_THAT(modeler.Query({6, 7}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {6, 7}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {-1, 4}, @@ -946,7 +1146,11 @@ TEST(StylusStateModelerTest, SaveAndRestore) { }, kTol, kAccelTol)); modeler.Restore(); - EXPECT_THAT(modeler.Query({6, 7}, std::optional({0, 1}), Time(0)), + EXPECT_THAT(modeler.Query({.position = {6, 7}, + .velocity = {0, 0}, + .acceleration = {0, 0}, + .time = Time(0)}, + Vec2{0, 1}), ResultNear( { .position = {-1, 4}, diff --git a/ink_stroke_modeler/internal/utils.cc b/ink_stroke_modeler/internal/utils.cc index c4793a0..2cbc075 100644 --- a/ink_stroke_modeler/internal/utils.cc +++ b/ink_stroke_modeler/internal/utils.cc @@ -9,6 +9,27 @@ namespace ink { namespace stroke_model { +Result InterpResult(const Result &start, const Result &end, + float interp_amount) { + return { + .position = Interp(start.position, end.position, interp_amount), + .velocity = Interp(start.velocity, end.velocity, interp_amount), + .acceleration = + Interp(start.acceleration, end.acceleration, interp_amount), + .time = Interp(start.time, end.time, interp_amount), + .pressure = start.pressure < 0 || end.pressure < 0 + ? -1 + : Interp(start.pressure, end.pressure, interp_amount), + .tilt = start.tilt < 0 || end.tilt < 0 + ? -1 + : Interp(start.tilt, end.tilt, interp_amount), + .orientation = + start.orientation < 0 || end.orientation < 0 + ? -1 + : InterpAngle(start.orientation, end.orientation, interp_amount), + }; +} + std::optional GetStrokeNormal(const TipState &tip_state, Time prev_time) { constexpr float kCosineHalfDegree = 0.99996192; diff --git a/ink_stroke_modeler/internal/utils.h b/ink_stroke_modeler/internal/utils.h index 5c3bfda..c636e09 100644 --- a/ink_stroke_modeler/internal/utils.h +++ b/ink_stroke_modeler/internal/utils.h @@ -82,6 +82,15 @@ inline float InterpAngle(float start, float end, float interp_amount) { return normalize_angle(Interp(start, end, interp_amount)); } +// Linearly interpolates all fields of `Result` using the `Interp` function, +// with the exception of `orientation` which uses `InterpAngle`. +// +// If `pressure`, `tilt`, or `orientation` are not present on either `start` or +// `end` (as indicated by a value < 0), then the output will also have an unset +// value for that field, indicated by a -1. +Result InterpResult(const Result& start, const Result& end, + float interp_amount); + // Returns the distance between two points. inline float Distance(Vec2 start, Vec2 end) { return (end - start).Magnitude(); @@ -107,7 +116,7 @@ inline float NearestPointOnSegment(Vec2 segment_start, Vec2 segment_end, // // If velocity and magnitude are both zero, then we cannot compute the normal // direction, and this return `std::nullopt`. -std::optional GetStrokeNormal(const TipState &tip_state, Time prev_time); +std::optional GetStrokeNormal(const TipState& tip_state, Time prev_time); // Projects the given `position` to the segment defined by `segment_start` and // `segment_end` along the given `stroke_normal`. If the projection is not diff --git a/ink_stroke_modeler/internal/utils_test.cc b/ink_stroke_modeler/internal/utils_test.cc index f7a2d67..6fa21a7 100644 --- a/ink_stroke_modeler/internal/utils_test.cc +++ b/ink_stroke_modeler/internal/utils_test.cc @@ -77,6 +77,57 @@ TEST(UtilsTest, InterpAngle) { EXPECT_NEAR(InterpAngle(1.6 * kPi, .4 * kPi, .625), .1 * kPi, 1e-6); } +TEST(UtilsTest, InterpResult) { + Result a{.position = {1, 2}, + .velocity = {3, 4}, + .acceleration = {5, 6}, + .time = Time(1), + .pressure = 0.1, + .tilt = 0.2, + .orientation = 0.3}; + Result b{.position = {7, 8}, + .velocity = {9, 10}, + .acceleration = {11, 12}, + .time = Time(2), + .pressure = 0.4, + .tilt = 0.5, + .orientation = 0.6}; + EXPECT_THAT(InterpResult(a, b, 0), ResultNear(a, 1e-5, 1e-5)); + EXPECT_THAT(InterpResult(a, b, 1), ResultNear(b, 1e-5, 1e-5)); + + EXPECT_THAT(InterpResult(a, b, 0.5), ResultNear({.position = {4, 5}, + .velocity = {6, 7}, + .acceleration = {8, 9}, + .time = Time(1.5), + .pressure = 0.25, + .tilt = 0.35, + .orientation = 0.45}, + 1e-5, 1e-5)); + EXPECT_THAT(InterpResult(a, b, 0.25), ResultNear({.position = {2.5, 3.5}, + .velocity = {4.5, 5.5}, + .acceleration = {6.5, 7.5}, + .time = Time(1.25), + .pressure = 0.175, + .tilt = 0.275, + .orientation = 0.375}, + 1e-5, 1e-5)); +} + +TEST(UtilsTest, InterpResultIgnoresMissingFields) { + EXPECT_EQ(InterpResult({.pressure = 0.3}, {.pressure = -1}, 0.5).pressure, + -1); + EXPECT_EQ(InterpResult({.pressure = -1}, {.pressure = 0.3}, 0.5).pressure, + -1); + EXPECT_EQ(InterpResult({.tilt = 0.3}, {.tilt = -1}, 0.5).tilt, -1); + EXPECT_EQ(InterpResult({.tilt = -1}, {.tilt = 0.3}, 0.5).tilt, -1); + EXPECT_EQ( + InterpResult({.orientation = 0.3}, {.orientation = -1}, 0.5).orientation, + -1); + EXPECT_EQ( + InterpResult({.orientation = -1}, {.orientation = 0.3}, 0.5).orientation, + -1); +} + TEST(UtilsTest, Distance) { EXPECT_FLOAT_EQ(Distance({0, 0}, {1, 0}), 1); EXPECT_FLOAT_EQ(Distance({1, 1}, {-2, 5}), 5); diff --git a/ink_stroke_modeler/params.cc b/ink_stroke_modeler/params.cc index 61a6aea..d1ca809 100644 --- a/ink_stroke_modeler/params.cc +++ b/ink_stroke_modeler/params.cc @@ -137,8 +137,18 @@ absl::Status ValidateSamplingParams(const SamplingParams& params) { absl::Status ValidateStylusStateModelerParams( const StylusStateModelerParams& params) { - return ValidateGreaterThanZero(params.max_input_samples, - "StylusStateModelerParams::max_input_samples"); + if (params.use_stroke_normal_projection) { + RETURN_IF_ERROR( + ValidateGreaterThanZero(params.min_input_samples, + "StylusStateModelerParams::min_input_samples")); + return ValidateGreaterThanZero( + params.min_sample_duration.Value(), + "StylusStateModelerParams::min_sample_duration"); + } else { + return ValidateGreaterThanZero( + params.max_input_samples, + "StylusStateModelerParams::max_input_samples"); + } } absl::Status ValidateWobbleSmootherParams(const WobbleSmootherParams& params) { diff --git a/ink_stroke_modeler/params.h b/ink_stroke_modeler/params.h index 1a624ab..1225e69 100644 --- a/ink_stroke_modeler/params.h +++ b/ink_stroke_modeler/params.h @@ -144,22 +144,48 @@ struct SamplingParams { double max_estimated_angle_to_traverse_per_input = -1; }; -// These parameters are used modeling the state of the stylus once the position -// has been modeled. +// These parameters are used for modeling the non-positional state of the stylus +// (i.e. pressure, tilt, and orientation) once the position has been modeled. +// +// To calculate the non-positional state, we project the modeled position of the +// tip, to a polyline made up of the most recent raw inputs, and then +// interpolate pressure, tilt, and orientation along that raw input polyline. +// These parameters determine the projection method, and how many raw input +// samples to include in the polyline. struct StylusStateModelerParams { - // The maximum number of raw inputs to look at when finding the relevant - // states for interpolation. + // This determines the number of recent raw input samples to use when + // 'use_stroke_normal_projection` is false; we accumulate `max_input_samples` + // samples, then discard old samples as we receive new inputs. + // If `use_stroke_normal_projection` is false, this must be greater than zero. + // If `use_stroke_normal_projection` is true, this will be ignored, and + // `min_input_samples` and `min_sample_duration` will be used instead. int max_input_samples = 10; - // This toggles between the two projection methods available. If - // `use_stroke_normal_projection` is false, a call to - // `StylusStateModeler::Query` will base its calculations on the nearest point - // for the closest segment. If `use_stroke_normal_projection` is true, it use - // the stroke normal of the polyline for its calculations. + // This determines the method used to project to the raw input polyline. + // * If false, we take the point on the polyline closest to the modeled tip + // position. + // * If true, we cast a pair of rays in opposite directions normal to the + // stroke direction from the modeled tip point and find the intersection + // with the raw input polyline. If multiple intersections are found, we use + // a heuristic to determine the best choice. // We recommend enabling this in order to get increased accuracy for pressure, // tilt, and orientation; however, this defaults to false to preserve behavior // for existing uses. bool use_stroke_normal_projection = false; + + // These determine the number of recent raw input samples to use when + // `use_stroke_normal_projection` is true: we accumulate samples until both of + // the following conditions are true: + // * We have at least `min_input_samples` samples + // * The difference in `time` between the first and last sample is greater + // than `min_sample_duration` + // As we receive additional raw inputs, we discard old samples once they are + // no longer required to maintain those conditions. + // If `use_stroke_normal_projection` is true, both of these must be > zero. + // If `use_stroke_normal_projection` is false, these will be ignored, and + // `max_input_samples` will be used instead. + int min_input_samples = -1; + Duration min_sample_duration{-1}; }; // These parameters are used for applying smoothing to the input to reduce diff --git a/ink_stroke_modeler/params_test.cc b/ink_stroke_modeler/params_test.cc index e978283..857a32f 100644 --- a/ink_stroke_modeler/params_test.cc +++ b/ink_stroke_modeler/params_test.cc @@ -59,8 +59,10 @@ const StrokeModelParams kGoodStrokeModelParams{ .sampling_params{.min_output_rate = 3, .end_of_stroke_stopping_distance = 1e-6, .end_of_stroke_max_iterations = 1}, - .stylus_state_modeler_params{.max_input_samples = 7, - .use_stroke_normal_projection = true}, + .stylus_state_modeler_params{.max_input_samples = -1, + .use_stroke_normal_projection = true, + .min_input_samples = 10, + .min_sample_duration = Duration(0.05)}, .prediction_params = StrokeEndPredictorParams{}}; TEST(ParamsTest, ValidatePositionModelerParams) { @@ -169,12 +171,34 @@ TEST(ParamsTest, ValidateSamplingParams) { .ok()); } -TEST(ParamsTest, ValidateStylusStateModelerParams) { +TEST(ParamsTest, ValidateStylusStateModelerParamsDefaultProjection) { EXPECT_TRUE(ValidateStylusStateModelerParams({.max_input_samples = 1}).ok()); EXPECT_EQ(ValidateStylusStateModelerParams({.max_input_samples = 0}).code(), absl::StatusCode::kInvalidArgument); } +TEST(ParamsTest, ValidateStylusStateModelerParamsStrokeNormalProjection) { + EXPECT_TRUE( + ValidateStylusStateModelerParams({.max_input_samples = -1, + .use_stroke_normal_projection = true, + .min_input_samples = 2, + .min_sample_duration = Duration(.05)}) + .ok()); + EXPECT_EQ( + ValidateStylusStateModelerParams({.max_input_samples = -1, + .use_stroke_normal_projection = true, + .min_input_samples = 0, + .min_sample_duration = Duration(0.05)}) + .code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ( + ValidateStylusStateModelerParams({.max_input_samples = -1, + .use_stroke_normal_projection = false, + .min_input_samples = 10, + .min_sample_duration = Duration(-0.1)}) + .code(), + absl::StatusCode::kInvalidArgument); +} TEST(ParamsTest, ValidateWobbleSmootherParams) { EXPECT_TRUE( @@ -313,7 +337,7 @@ TEST(ParamsTest, ValidateStrokeModelParams) { } { auto bad_params = kGoodStrokeModelParams; - bad_params.stylus_state_modeler_params.max_input_samples = 0; + bad_params.stylus_state_modeler_params.min_input_samples = 0; EXPECT_EQ(ValidateStrokeModelParams(bad_params).code(), absl::StatusCode::kInvalidArgument); } diff --git a/ink_stroke_modeler/stroke_modeler.cc b/ink_stroke_modeler/stroke_modeler.cc index dd9a374..2a01782 100644 --- a/ink_stroke_modeler/stroke_modeler.cc +++ b/ink_stroke_modeler/stroke_modeler.cc @@ -39,21 +39,6 @@ namespace ink { namespace stroke_model { namespace { -Result InterpResult(const Result &from, const Result &to, float interp_value) { - return { - .position = Interp(from.position, to.position, interp_value), - .velocity = Interp(from.velocity, to.velocity, interp_value), - .acceleration = Interp(from.acceleration, to.acceleration, interp_value), - .time = Interp(from.time, to.time, interp_value), - .pressure = Interp(from.pressure, to.pressure, interp_value), - .tilt = Interp(from.tilt, to.tilt, interp_value), - .orientation = - from.orientation == -1 || to.orientation == -1 - ? -1 - : InterpAngle(from.orientation, to.orientation, interp_value), - }; -} - Result MakeResultFromTipState(const TipState &tipstate, const Result &stylus_state) { return { @@ -78,8 +63,8 @@ void ModelStylus( loop_contraction_mitigation_modeler.GetInterpolationValue(); for (const auto &tip_state : tip_states) { std::optional stroke_normal = GetStrokeNormal(tip_state, prev_time); - Result projected_state = stylus_state_modeler.Query( - tip_state.position, stroke_normal, tip_state.time); + Result projected_state = + stylus_state_modeler.Query(tip_state, stroke_normal); Result modeled_state = MakeResultFromTipState(tip_state, projected_state); result.push_back( InterpResult(projected_state, modeled_state, interp_value)); @@ -93,7 +78,22 @@ void ModelStylus( absl::Status StrokeModeler::Reset( const StrokeModelParams &stroke_model_params) { - if (auto status = ValidateStrokeModelParams(stroke_model_params); + // TODO: b/368389799 - This is a temporary workaround for a migration problem + // and should be removed. + StrokeModelParams stroke_model_params_copy = stroke_model_params; + StylusStateModelerParams &stylus_modeler_params = + stroke_model_params_copy.stylus_state_modeler_params; + if (stylus_modeler_params.use_stroke_normal_projection) { + if (stylus_modeler_params.min_input_samples < 0) { + stylus_modeler_params.min_input_samples = + stylus_modeler_params.max_input_samples; + } + if (stylus_modeler_params.min_sample_duration < Duration(0)) { + stylus_modeler_params.min_sample_duration = Duration(1e-10); + } + } + + if (auto status = ValidateStrokeModelParams(stroke_model_params_copy); !status.ok()) { return status; } @@ -101,7 +101,7 @@ absl::Status StrokeModeler::Reset( // Note that many of the sub-modelers require some knowledge about the stroke // (e.g. start position, input type) when resetting, and as such are reset in // ProcessTDown() instead. - stroke_model_params_ = stroke_model_params; + stroke_model_params_ = stroke_model_params_copy; ResetInternal(); const PredictionParams &prediction_params = diff --git a/ink_stroke_modeler/stroke_modeler_fuzz_test.cc b/ink_stroke_modeler/stroke_modeler_fuzz_test.cc index 96ca225..d794343 100644 --- a/ink_stroke_modeler/stroke_modeler_fuzz_test.cc +++ b/ink_stroke_modeler/stroke_modeler_fuzz_test.cc @@ -31,6 +31,12 @@ ArbitraryLoopContractionMitigationParameters() { fuzztest::Arbitrary()); } +fuzztest::Domain ArbitraryStylusStateModelerParams() { + return fuzztest::StructOf( + fuzztest::Arbitrary(), fuzztest::Arbitrary(), + fuzztest::Arbitrary(), ArbitraryDuration()); +} + fuzztest::Domain ArbitraryStrokeModelParams() { return fuzztest::StructOf( fuzztest::StructOf( @@ -44,7 +50,7 @@ fuzztest::Domain ArbitraryStrokeModelParams() { fuzztest::Arbitrary(), /*max_outputs_per_call*/ fuzztest::InRange(1000, 100000), fuzztest::Arbitrary()), - fuzztest::Arbitrary(), + ArbitraryStylusStateModelerParams(), fuzztest::VariantOf( fuzztest::Arbitrary(), fuzztest::StructOf( diff --git a/ink_stroke_modeler/stroke_modeler_test.cc b/ink_stroke_modeler/stroke_modeler_test.cc index c05b8a4..760ee9a 100644 --- a/ink_stroke_modeler/stroke_modeler_test.cc +++ b/ink_stroke_modeler/stroke_modeler_test.cc @@ -365,6 +365,8 @@ TEST(StrokeModelerTest, InputRateSlowerThanMinOutputRateNormalProjection) { StrokeModeler modeler; StrokeModelParams params = kDefaultParams; params.stylus_state_modeler_params.use_stroke_normal_projection = true; + params.stylus_state_modeler_params.min_input_samples = 10; + params.stylus_state_modeler_params.min_sample_duration = Duration(0.334); ASSERT_TRUE(modeler.Reset(params).ok()); Time time{0}; @@ -650,6 +652,8 @@ TEST(StrokeModelerTest, const Duration kDeltaTime{1. / 30}; StrokeModelParams params = kDefaultParams; params.stylus_state_modeler_params.use_stroke_normal_projection = true; + params.stylus_state_modeler_params.min_input_samples = 10; + params.stylus_state_modeler_params.min_sample_duration = Duration(0.334); params.position_modeler_params.loop_contraction_mitigation_params = PositionModelerParams::LoopContractionMitigationParameters{ .is_enabled = true,