diff --git a/editor/modes/visual-editor/block.js b/editor/modes/visual-editor/block.js
index 818687ee9a74f..faf6b6b5d7530 100644
--- a/editor/modes/visual-editor/block.js
+++ b/editor/modes/visual-editor/block.js
@@ -29,7 +29,6 @@ import {
 	mergeBlocks,
 	insertBlocks,
 	removeBlocks,
-	clearSelectedBlock,
 	startTyping,
 	stopTyping,
 	replaceBlocks,
@@ -74,6 +73,7 @@ class VisualEditorBlock extends Component {
 		this.onKeyUp = this.onKeyUp.bind( this );
 		this.handleArrowKey = this.handleArrowKey.bind( this );
 		this.toggleMobileControls = this.toggleMobileControls.bind( this );
+		this.toggleToolbar = this.toggleToolbar.bind( this );
 		this.previousOffset = null;
 	}
 
@@ -95,6 +95,10 @@ class VisualEditorBlock extends Component {
 		) {
 			this.previousOffset = this.node.getBoundingClientRect().top;
 		}
+
+		if ( ! newProps.isSelected || newProps.isTyping ) {
+			this.setState( { showToolbar: false } );
+		}
 	}
 
 	componentDidUpdate( prevProps ) {
@@ -185,7 +189,6 @@ class VisualEditorBlock extends Component {
 			previousBlock,
 			onRemove,
 			onFocus,
-			onDeselect,
 		} = this.props;
 
 		// Remove block on backspace.
@@ -203,7 +206,7 @@ class VisualEditorBlock extends Component {
 
 		// Deselect on escape.
 		if ( ESCAPE === keyCode ) {
-			onDeselect();
+			this.setState( { showToolbar: false } );
 		}
 	}
 
@@ -305,6 +308,16 @@ class VisualEditorBlock extends Component {
 		} );
 	}
 
+	toggleToolbar() {
+		if ( ! this.props.isSelected ) {
+			this.props.onSelect();
+		}
+
+		this.setState( {
+			showToolbar: ! this.state.showToolbar,
+		} );
+	}
+
 	render() {
 		const { block, multiSelectedBlockUids } = this.props;
 		const { name: blockName, isValid } = block;
@@ -329,14 +342,17 @@ class VisualEditorBlock extends Component {
 
 		// Generate the wrapper class names handling the different states of the block.
 		const { isHovered, isSelected, isMultiSelected, isFirstMultiSelected, focus } = this.props;
-		const showUI = isSelected && ( ! this.props.isTyping || focus.collapsed === false );
+		const showUI = isSelected && ( this.state.showToolbar || focus.collapsed === false );
+		const duringInput = this.props.isTyping && ( ! focus || focus.collapsed !== false );
 		const { showMobileControls } = this.state;
 		const wrapperClassname = classnames( 'editor-visual-editor__block', {
 			'is-invalid': ! isValid,
-			'is-selected': showUI,
+			'is-selected': isSelected,
 			'is-multi-selected': isMultiSelected,
 			'is-hovered': isHovered,
 			'is-showing-mobile-controls': showMobileControls,
+			'show-ui': showUI,
+			'during-input': duringInput,
 		} );
 
 		const { onMouseLeave, onFocus, onInsertBlocksAfter, onReplace } = this.props;
@@ -368,9 +384,9 @@ class VisualEditorBlock extends Component {
 				aria-label={ blockLabel }
 				{ ...wrapperProps }
 			>
-				{ ( showUI || isHovered ) && <BlockMover uids={ [ block.uid ] } /> }
-				{ ( showUI || isHovered ) && <BlockRightMenu uid={ block.uid } /> }
-				{ showUI && isValid &&
+				{ ! duringInput && ( isHovered || isSelected ) && <BlockMover uids={ [ block.uid ] } /> }
+				{ ( showUI ) && <BlockRightMenu uid={ block.uid } /> }
+				{ isValid && ! duringInput && ( isHovered || isSelected ) &&
 					<CSSTransitionGroup
 						transitionName={ { appear: 'is-appearing', appearActive: 'is-appearing-active' } }
 						transitionAppear={ true }
@@ -380,11 +396,19 @@ class VisualEditorBlock extends Component {
 						component={ FirstChild }
 					>
 						<div className="editor-visual-editor__block-controls">
-							<BlockSwitcher uid={ block.uid } />
-							<Slot name="Formatting.Toolbar" />
-							<Toolbar className="editor-visual-editor__mobile-tools">
-								{ ( showUI || isHovered ) && <BlockMover uids={ [ block.uid ] } /> }
-								{ ( showUI || isHovered ) && <BlockRightMenu uid={ block.uid } /> }
+							{ ! showUI && (
+								<Toolbar className="editor-visual-editor__toolbar-toggle">
+									<IconButton
+										icon="ellipsis"
+										onClick={ this.toggleToolbar }
+									/>
+								</Toolbar>
+							) }
+							{ showUI && <BlockSwitcher uid={ block.uid } /> }
+							{ showUI && <Slot name="Formatting.Toolbar" /> }
+							{ showUI && <Toolbar className="editor-visual-editor__mobile-tools">
+								<BlockMover uids={ [ block.uid ] } />
+								<BlockRightMenu uid={ block.uid } />
 								<IconButton
 									className="editor-visual-editor__mobile-toggle"
 									onClick={ this.toggleMobileControls }
@@ -392,7 +416,7 @@ class VisualEditorBlock extends Component {
 									label={ __( 'Toggle extra controls' ) }
 									icon="ellipsis"
 								/>
-							</Toolbar>
+							</Toolbar> }
 						</div>
 					</CSSTransitionGroup>
 				}
@@ -461,9 +485,6 @@ export default connect(
 				uid: ownProps.uid,
 			} );
 		},
-		onDeselect() {
-			dispatch( clearSelectedBlock() );
-		},
 
 		onStartTyping() {
 			dispatch( startTyping() );
diff --git a/editor/modes/visual-editor/style.scss b/editor/modes/visual-editor/style.scss
index e51453e0669f6..bf9ab4a9a8656 100644
--- a/editor/modes/visual-editor/style.scss
+++ b/editor/modes/visual-editor/style.scss
@@ -30,8 +30,8 @@
 	margin-bottom: 5px;
 	max-width: $visual-editor-max-width + ( 2 * $block-mover-padding-visible );
 	position: relative;
-
 	padding: $block-padding;
+	transition: 0.2s border;
 
 	@include break-small {
 		// The block mover needs to stay inside the block to allow clicks when hovering the block
@@ -42,11 +42,10 @@
 		z-index: z-index( '.editor-visual-editor__block:before' );
 		content: '';
 		position: absolute;
-		outline: 1px solid transparent;
-		transition: 0.2s outline;
+		border: 2px solid transparent;
+		transition: 0.2s border;
 		top: 0;
 		bottom: 0;
-
 		left: 0;
 		right: 0;
 
@@ -73,13 +72,19 @@
 		background-color: rgba( $white, 0.6 );
 	}
 
-	&.is-hovered:before {
-		outline: 1px solid $light-gray-500;
+	&.is-hovered:before,
+	&.is-selected:before {
+		border-left: 2px solid $light-gray-500;
 		transition: 0.2s outline;
 	}
 
-	&.is-selected:before {
-		outline: 2px solid $light-gray-500;
+	&.show-ui:before {
+		border: 2px solid $light-gray-500;
+		transition: 0.2s outline;
+	}
+
+	&.during-input:before {
+		border: 2px solid transparent;
 		transition: 0.2s outline;
 	}
 
@@ -287,6 +292,16 @@
 	}
 }
 
+ul.editor-visual-editor__toolbar-toggle {
+	height: 20px;
+	margin-top: 18px;
+	vertical-align: bottom;
+
+	button {
+		padding: 0 5px;
+	}
+}
+
 .editor-visual-editor__block-controls {
 	display: flex;
 	position: sticky;