diff --git a/src/api/internal.rs b/src/api/internal.rs
index 92021efa20..2601c2df7b 100644
--- a/src/api/internal.rs
+++ b/src/api/internal.rs
@@ -726,9 +726,6 @@ impl<T: Pixel> ContextInner<T> {
 
     let coded_data = fi.coded_frame_data.as_mut().unwrap();
 
-    // Save the motion vectors to FrameInvariants.
-    coded_data.lookahead_me_stats = Some(fs.frame_me_stats.clone());
-
     #[cfg(feature = "dump_lookahead_data")]
     {
       use crate::partition::RefType::*;
@@ -1088,6 +1085,7 @@ impl<T: Pixel> ContextInner<T> {
       let output_frame_data =
         self.frame_data.remove(&output_frameno).unwrap().unwrap();
       let fi = &output_frame_data.fi;
+      let fs = &output_frame_data.fs;
 
       let frame = self.frame_q[&fi.input_frameno].as_ref().unwrap();
 
@@ -1104,13 +1102,7 @@ impl<T: Pixel> ContextInner<T> {
       let frame_data = &mut self.frame_data;
       let len = unique_indices.len();
 
-      let lookahead_me_stats = fi
-        .coded_frame_data
-        .as_ref()
-        .unwrap()
-        .lookahead_me_stats
-        .as_ref()
-        .expect("Lookahead ME stats not populated, this is a bug");
+      let lookahead_me_stats = &fs.frame_me_stats;
 
       // Compute and propagate the importance, split evenly between the
       // referenced frames.
diff --git a/src/encoder.rs b/src/encoder.rs
index 380e5d76ea..9114e0568b 100644
--- a/src/encoder.rs
+++ b/src/encoder.rs
@@ -647,12 +647,6 @@ pub struct FrameInvariants<T: Pixel> {
 /// This may change in the future.
 #[derive(Debug, Clone)]
 pub struct CodedFrameData<T: Pixel> {
-  /// Motion vectors to the _original_ reference frames (not reconstructed).
-  /// Used for lookahead purposes.
-  ///
-  /// These objects are very expensive to create, so their creation
-  /// is deferred until it is needed.
-  pub lookahead_me_stats: Option<Arc<[FrameMEStats; REF_FRAMES as usize]>>,
   /// The lookahead version of `rec_buffer`, used for storing and propagating
   /// the original reference frames (rather than reconstructed ones). The
   /// lookahead uses both `rec_buffer` and `lookahead_rec_buffer`, where
@@ -683,7 +677,6 @@ impl<T: Pixel> CodedFrameData<T> {
     let h_in_imp_b = fi.h_in_b / 2;
 
     CodedFrameData {
-      lookahead_me_stats: None,
       lookahead_rec_buffer: ReferenceFramesSet::new(),
       w_in_imp_b,
       h_in_imp_b,