diff --git a/supar/modules/pretrained.py b/supar/modules/pretrained.py
index d4c2b52c..af334b56 100644
--- a/supar/modules/pretrained.py
+++ b/supar/modules/pretrained.py
@@ -46,12 +46,12 @@ def __init__(self, model, n_layers, n_out=0, stride=256, pooling='mean', pad_ind
         self.n_layers = n_layers or self.bert.config.num_hidden_layers
         self.hidden_size = self.bert.config.hidden_size
         self.n_out = n_out or self.hidden_size
-        self.stride = stride
         self.pooling = pooling
         self.pad_index = pad_index
         self.dropout = dropout
         self.requires_grad = requires_grad
         self.max_len = int(max(0, self.bert.config.max_position_embeddings) or 1e12) - 2
+        self.stride = min(stride, self.max_len)
 
         self.tokenizer = AutoTokenizer.from_pretrained(model)