diff --git a/superset-frontend/src/dashboard/components/DashboardBuilder/DashboardBuilder.tsx b/superset-frontend/src/dashboard/components/DashboardBuilder/DashboardBuilder.tsx
index d16e76174d7a6..9d12b38600fb5 100644
--- a/superset-frontend/src/dashboard/components/DashboardBuilder/DashboardBuilder.tsx
+++ b/superset-frontend/src/dashboard/components/DashboardBuilder/DashboardBuilder.tsx
@@ -68,6 +68,44 @@ const StyledDiv = styled.div`
   grid-template-columns: auto 1fr;
   grid-template-rows: auto 1fr;
   flex: 1;
+  /* Special cases */
+
+  /* A row within a column has inset hover menu */
+  .dragdroppable-column .dragdroppable-row .hover-menu--left {
+    left: -12px;
+    background: @lightest;
+    border: 1px solid @gray-light;
+  }
+
+  /* A column within a column or tabs has inset hover menu */
+  .dragdroppable-column .dragdroppable-column .hover-menu--top,
+  .dashboard-component-tabs .dragdroppable-column .hover-menu--top {
+    top: -12px;
+    background: @lightest;
+    border: 1px solid @gray-light;
+  }
+
+  /* move Tabs hover menu to top near actual Tabs */
+  .dashboard-component-tabs > .hover-menu--left {
+    top: 0;
+    transform: unset;
+    background: transparent;
+  }
+
+  /* push Chart actions to upper right */
+  .dragdroppable-column .dashboard-component-chart-holder .hover-menu--top,
+  .dragdroppable .dashboard-component-header .hover-menu--top {
+    right: 8px;
+    top: 8px;
+    background: transparent;
+    border: none;
+    transform: unset;
+    left: unset;
+  }
+  div:hover > .hover-menu-container .hover-menu,
+  .hover-menu-container .hover-menu:hover {
+    opacity: 1;
+  }
 `;
 
 // @z-index-above-dashboard-charts + 1 = 11
diff --git a/superset-frontend/src/dashboard/components/gridComponents/Header.jsx b/superset-frontend/src/dashboard/components/gridComponents/Header.jsx
index ef770b48e4320..7ee2f44136d63 100644
--- a/superset-frontend/src/dashboard/components/gridComponents/Header.jsx
+++ b/superset-frontend/src/dashboard/components/gridComponents/Header.jsx
@@ -137,7 +137,6 @@ class Header extends React.PureComponent {
                   <DragHandle position="left" />
                 </HoverMenu>
               )}
-
             <WithPopoverMenu
               onChangeFocus={this.handleChangeFocus}
               menuItems={[
@@ -152,7 +151,6 @@ class Header extends React.PureComponent {
                   value={component.meta.background}
                   onChange={this.handleChangeBackground}
                 />,
-                <DeleteComponentButton onDelete={this.handleDeleteComponent} />,
               ]}
               editMode={editMode}
             >
@@ -164,6 +162,13 @@ class Header extends React.PureComponent {
                   rowStyle.className,
                 )}
               >
+                {editMode && (
+                  <HoverMenu position="top">
+                    <DeleteComponentButton
+                      onDelete={this.handleDeleteComponent}
+                    />
+                  </HoverMenu>
+                )}
                 <EditableTitle
                   title={component.meta.text}
                   canEdit={editMode}
diff --git a/superset-frontend/src/dashboard/components/gridComponents/Markdown.jsx b/superset-frontend/src/dashboard/components/gridComponents/Markdown.jsx
index a21402c3ab24d..6f64978db8ec5 100644
--- a/superset-frontend/src/dashboard/components/gridComponents/Markdown.jsx
+++ b/superset-frontend/src/dashboard/components/gridComponents/Markdown.jsx
@@ -27,6 +27,7 @@ import { MarkdownEditor } from 'src/components/AsyncAceEditor';
 
 import DeleteComponentButton from 'src/dashboard/components/DeleteComponentButton';
 import DragDroppable from 'src/dashboard/components/dnd/DragDroppable';
+import HoverMenu from 'src/dashboard/components/menu/HoverMenu';
 import ResizableContainer from 'src/dashboard/components/resizable/ResizableContainer';
 import MarkdownModeDropdown from 'src/dashboard/components/menu/MarkdownModeDropdown';
 import WithPopoverMenu from 'src/dashboard/components/menu/WithPopoverMenu';
@@ -317,7 +318,6 @@ class Markdown extends React.PureComponent {
                 value={this.state.editorMode}
                 onChange={this.handleChangeEditorMode}
               />,
-              <DeleteComponentButton onDelete={this.handleDeleteComponent} />,
             ]}
             editMode={editMode}
           >
@@ -350,6 +350,11 @@ class Markdown extends React.PureComponent {
                   className="dashboard-component dashboard-component-chart-holder"
                   data-test="dashboard-component-chart-holder"
                 >
+                  <HoverMenu position="top">
+                    <DeleteComponentButton
+                      onDelete={this.handleDeleteComponent}
+                    />
+                  </HoverMenu>
                   {editMode && isEditing
                     ? this.renderEditMode()
                     : this.renderPreviewMode()}
diff --git a/superset-frontend/src/dashboard/components/menu/HoverMenu.tsx b/superset-frontend/src/dashboard/components/menu/HoverMenu.tsx
index 6fa21eca11fc3..810c9441bfce6 100644
--- a/superset-frontend/src/dashboard/components/menu/HoverMenu.tsx
+++ b/superset-frontend/src/dashboard/components/menu/HoverMenu.tsx
@@ -17,6 +17,7 @@
  * under the License.
  */
 import React, { RefObject } from 'react';
+import { styled } from '@superset-ui/core';
 import cx from 'classnames';
 
 interface HoverMenuProps {
@@ -25,6 +26,43 @@ interface HoverMenuProps {
   children: React.ReactNode;
 }
 
+const HoverStyleOverrides = styled.div`
+  .hover-menu {
+    opacity: 0;
+    position: absolute;
+    z-index: @z-index-above-dashboard-charts;
+    font-size: @font-size-m;
+  }
+
+  .hover-menu--left {
+    width: 24px;
+    top: 50%;
+    transform: translate(0, -50%);
+    left: -28px;
+    padding: 8px 0;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+  }
+
+  .hover-menu--left > :nth-child(n):not(:only-child):not(:last-child) {
+    margin-bottom: 12px;
+  }
+
+  .hover-menu--top {
+    height: 24px;
+    top: -24px;
+    left: 50%;
+    transform: translate(-50%);
+    padding: 0 8px;
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
+    align-items: center;
+  }
+`;
+
 export default class HoverMenu extends React.PureComponent<HoverMenuProps> {
   static defaultProps = {
     position: 'left',
@@ -35,16 +73,18 @@ export default class HoverMenu extends React.PureComponent<HoverMenuProps> {
   render() {
     const { innerRef, position, children } = this.props;
     return (
-      <div
-        ref={innerRef}
-        className={cx(
-          'hover-menu',
-          position === 'left' && 'hover-menu--left',
-          position === 'top' && 'hover-menu--top',
-        )}
-      >
-        {children}
-      </div>
+      <HoverStyleOverrides className="hover-menu-container">
+        <div
+          ref={innerRef}
+          className={cx(
+            'hover-menu',
+            position === 'left' && 'hover-menu--left',
+            position === 'top' && 'hover-menu--top',
+          )}
+        >
+          {children}
+        </div>
+      </HoverStyleOverrides>
     );
   }
 }
diff --git a/superset-frontend/src/dashboard/stylesheets/hover-menu.less b/superset-frontend/src/dashboard/stylesheets/hover-menu.less
deleted file mode 100644
index 41ce575eeae92..0000000000000
--- a/superset-frontend/src/dashboard/stylesheets/hover-menu.less
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-.hover-menu {
-  opacity: 0;
-  position: absolute;
-  z-index: @z-index-above-dashboard-charts;
-  font-size: @font-size-m;
-}
-
-.hover-menu--left {
-  width: 24px;
-  top: 50%;
-  transform: translate(0, -50%);
-  left: -28px;
-  padding: 8px 0;
-  display: flex;
-  flex-direction: column;
-  justify-content: center;
-  align-items: center;
-}
-
-.hover-menu--left > :nth-child(n):not(:only-child):not(:last-child) {
-  margin-bottom: 12px;
-}
-
-.hover-menu--top {
-  height: 24px;
-  top: -24px;
-  left: 50%;
-  transform: translate(-50%);
-  padding: 0 8px;
-  display: flex;
-  flex-direction: row;
-  justify-content: center;
-  align-items: center;
-}
-
-/* Special cases */
-
-/* A row within a column has inset hover menu */
-.dragdroppable-column .dragdroppable-row .hover-menu--left {
-  left: -12px;
-  background: @lightest;
-  border: 1px solid @gray-light;
-}
-
-/* A column within a column or tabs has inset hover menu */
-.dragdroppable-column .dragdroppable-column .hover-menu--top,
-.dashboard-component-tabs .dragdroppable-column .hover-menu--top {
-  top: -12px;
-  background: @lightest;
-  border: 1px solid @gray-light;
-}
-
-/* move Tabs hover menu to top near actual Tabs */
-.dashboard-component-tabs > .hover-menu--left {
-  top: 0;
-  transform: unset;
-  background: transparent;
-}
-
-/* push Chart actions to upper right */
-.dragdroppable-column .dashboard-component-chart-holder > .hover-menu--top {
-  right: 8px;
-  top: 8px;
-  background: transparent;
-  border: none;
-  transform: unset;
-  left: unset;
-}
-
-.hover-menu--top > :nth-child(n):not(:only-child):not(:last-child) {
-  margin-right: 12px;
-}
-
-div:hover > .hover-menu,
-.hover-menu:hover {
-  opacity: 1;
-}
diff --git a/superset-frontend/src/dashboard/stylesheets/index.less b/superset-frontend/src/dashboard/stylesheets/index.less
index 6dee17e77cce0..7882b83f92f22 100644
--- a/superset-frontend/src/dashboard/stylesheets/index.less
+++ b/superset-frontend/src/dashboard/stylesheets/index.less
@@ -24,7 +24,6 @@
 @import './dnd.less';
 @import './filter-scope-selector.less';
 @import './grid.less';
-@import './hover-menu.less';
 @import './popover-menu.less';
 @import './resizable.less';
 @import './components/index.less';