Add a pretrained guard for the initial meta init on global rank 0 (#1397

)

llmfoundry/models/hf/hf_causal_lm.py

@@ -284,7 +284,7 @@ def _autoset_attn_implementation_monkeypatch(
         # the different processes. To avoid this contention, we first create the model (on meta device) on local rank
         # zero. This will set up the transformers model cache and avoid the future contention.
         if dist.get_local_rank() == 0:
-            if os.path.isdir(pretrained_model_name_or_path):
+            if pretrained and os.path.isdir(pretrained_model_name_or_path):
                 with init_empty_weights(include_buffers=False):
                     with warnings.catch_warnings():
                         warnings.simplefilter('ignore', UserWarning)