diff --git a/mmdet/models/dense_heads/base_dense_head.py b/mmdet/models/dense_heads/base_dense_head.py
index ae10936615f..d3895f7a959 100644
--- a/mmdet/models/dense_heads/base_dense_head.py
+++ b/mmdet/models/dense_heads/base_dense_head.py
@@ -2,6 +2,7 @@
 from abc import ABCMeta, abstractmethod
 
 import torch
+from mmcv.cnn.utils.weight_init import constant_init
 from mmcv.ops import batched_nms
 from mmcv.runner import BaseModule, force_fp32
 
@@ -14,6 +15,14 @@ class BaseDenseHead(BaseModule, metaclass=ABCMeta):
     def __init__(self, init_cfg=None):
         super(BaseDenseHead, self).__init__(init_cfg)
 
+    def init_weights(self):
+        super(BaseDenseHead, self).init_weights()
+        # avoid init_cfg overwrite the initialization of `conv_offset`
+        for m in self.modules():
+            # DeformConv2dPack, ModulatedDeformConv2dPack
+            if hasattr(m, 'conv_offset'):
+                constant_init(m.conv_offset, 0)
+
     @abstractmethod
     def loss(self, **kwargs):
         """Compute losses of the head."""