diff --git a/src/components/structures/TimelinePanel.js b/src/components/structures/TimelinePanel.js
index 65fb00c3058..e708fad6a48 100644
--- a/src/components/structures/TimelinePanel.js
+++ b/src/components/structures/TimelinePanel.js
@@ -2,7 +2,7 @@
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
 Copyright 2019 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -146,6 +146,9 @@ const TimelinePanel = createReactClass({
             liveEvents: [],
             timelineLoading: true, // track whether our room timeline is loading
 
+            // the index of the first event that is to be shown
+            firstVisibleEventIndex: 0,
+
             // canBackPaginate == false may mean:
             //
             // * we haven't (successfully) loaded the timeline yet, or:
@@ -333,11 +336,12 @@ const TimelinePanel = createReactClass({
 
             // We can now paginate in the unpaginated direction
             const canPaginateKey = (backwards) ? 'canBackPaginate' : 'canForwardPaginate';
-            const { events, liveEvents } = this._getEvents();
+            const { events, liveEvents, firstVisibleEventIndex } = this._getEvents();
             this.setState({
                 [canPaginateKey]: true,
                 events,
                 liveEvents,
+                firstVisibleEventIndex,
             });
         }
     },
@@ -369,6 +373,11 @@ const TimelinePanel = createReactClass({
             return Promise.resolve(false);
         }
 
+        if (backwards && this.state.firstVisibleEventIndex !== 0) {
+            debuglog("TimelinePanel: won't", dir, "paginate past first visible event");
+            return Promise.resolve(false);
+        }
+
         debuglog("TimelinePanel: Initiating paginate; backwards:"+backwards);
         this.setState({[paginatingKey]: true});
 
@@ -377,12 +386,13 @@ const TimelinePanel = createReactClass({
 
             debuglog("TimelinePanel: paginate complete backwards:"+backwards+"; success:"+r);
 
-            const { events, liveEvents } = this._getEvents();
+            const { events, liveEvents, firstVisibleEventIndex } = this._getEvents();
             const newState = {
                 [paginatingKey]: false,
                 [canPaginateKey]: r,
                 events,
                 liveEvents,
+                firstVisibleEventIndex,
             };
 
             // moving the window in this direction may mean that we can now
@@ -402,7 +412,11 @@ const TimelinePanel = createReactClass({
             // itself into the right place
             return new Promise((resolve) => {
                 this.setState(newState, () => {
-                    resolve(r);
+                    // we can continue paginating in the given direction if:
+                    // - _timelineWindow.paginate says we can
+                    // - we're paginating forwards, or we won't be trying to
+                    //   paginate backwards past the first visible event
+                    resolve(r && (!backwards || firstVisibleEventIndex === 0));
                 });
             });
         });
@@ -476,12 +490,13 @@ const TimelinePanel = createReactClass({
         this._timelineWindow.paginate(EventTimeline.FORWARDS, 1, false).then(() => {
             if (this.unmounted) { return; }
 
-            const { events, liveEvents } = this._getEvents();
+            const { events, liveEvents, firstVisibleEventIndex } = this._getEvents();
             const lastLiveEvent = liveEvents[liveEvents.length - 1];
 
             const updatedState = {
                 events,
                 liveEvents,
+                firstVisibleEventIndex,
             };
 
             let callRMUpdated;
@@ -1115,6 +1130,7 @@ const TimelinePanel = createReactClass({
     // get the list of events from the timeline window and the pending event list
     _getEvents: function() {
         const events = this._timelineWindow.getEvents();
+        const firstVisibleEventIndex = this._checkForPreJoinUISI(events);
 
         // Hold onto the live events separately. The read receipt and read marker
         // should use this list, so that they don't advance into pending events.
@@ -1128,9 +1144,72 @@ const TimelinePanel = createReactClass({
         return {
             events,
             liveEvents,
+            firstVisibleEventIndex,
         };
     },
 
+    /**
+     * Check for undecryptable messages that were sent while the user was not in
+     * the room.
+     *
+     * @param {Array<MatrixEvent>} events The timeline events to check
+     *
+     * @return {Number} The index within `events` of the event after the most recent
+     * undecryptable event that was sent while the user was not in the room.  If no
+     * such events were found, then it returns 0.
+     */
+    _checkForPreJoinUISI: function(events) {
+        const room = this.props.timelineSet.room;
+
+        if (events.length === 0 || !room ||
+            !MatrixClientPeg.get().isRoomEncrypted(room.roomId)) {
+            return 0;
+        }
+
+        const userId = MatrixClientPeg.get().credentials.userId;
+
+        // get the user's membership at the last event by getting the timeline
+        // that the event belongs to, and traversing the timeline looking for
+        // that event, while keeping track of the user's membership
+        const lastEvent = events[events.length - 1];
+        const timeline = room.getTimelineForEvent(lastEvent.getId());
+        const userMembershipEvent =
+              timeline.getState(EventTimeline.FORWARDS).getMember(userId);
+        let userMembership = userMembershipEvent
+            ? userMembershipEvent.membership : "leave";
+        const timelineEvents = timeline.getEvents();
+        for (let i = timelineEvents.length - 1; i >= 0; i--) {
+            const event = timelineEvents[i];
+            if (event.getId() === lastEvent.getId()) {
+                // found the last event, so we can stop looking through the timeline
+                break;
+            } else if (event.getStateKey() === userId
+                       && event.getType() === "m.room.member") {
+                const prevContent = event.getPrevContent();
+                userMembership = prevContent.membership || "leave";
+            }
+        }
+
+        // now go through the events that we have and find the first undecryptable
+        // one that was sent when the user wasn't in the room
+        for (let i = events.length - 1; i >= 0; i--) {
+            const event = events[i];
+            if (event.getStateKey() === userId
+                && event.getType() === "m.room.member") {
+                const prevContent = event.getPrevContent();
+                userMembership = prevContent.membership || "leave";
+            } else if (userMembership === "leave" &&
+                       (event.isDecryptionFailure() || event.isBeingDecrypted())) {
+                // reached an undecryptable message when the user wasn't in
+                // the room -- don't try to load any more
+                // Note: for now, we assume that events that are being decrypted are
+                // not decryptable
+                return i + 1;
+            }
+        }
+        return 0;
+    },
+
     _indexForEventId: function(evId) {
         for (let i = 0; i < this.state.events.length; ++i) {
             if (evId == this.state.events[i].getId()) {
@@ -1323,6 +1402,9 @@ const TimelinePanel = createReactClass({
             this.state.forwardPaginating ||
             ['PREPARED', 'CATCHUP'].includes(this.state.clientSyncState)
         );
+        const events = this.state.firstVisibleEventIndex
+              ? this.state.events.slice(this.state.firstVisibleEventIndex)
+              : this.state.events;
         return (
             <MessagePanel
                 ref={this._messagePanel}
@@ -1331,7 +1413,7 @@ const TimelinePanel = createReactClass({
                 hidden={this.props.hidden}
                 backPaginating={this.state.backPaginating}
                 forwardPaginating={forwardPaginating}
-                events={this.state.events}
+                events={events}
                 highlightedEventId={this.props.highlightedEventId}
                 readMarkerEventId={this.state.readMarkerEventId}
                 readMarkerVisible={this.state.readMarkerVisible}