Introduce Padding-Free Plugin to FMS-Acceleration (#57)

* basic padding-free plugin Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * modify dataloader collate function Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * addressed comments Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * Update to README and addition of ilab sample config Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * Address additional PR fixes Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * linting fixes Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * add instruct-lab to GH actions workflow matrix Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * added ilab plugin sample config Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> * added basic test module Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> --------- Signed-off-by: 1000960000 user <aaron.chew1@ibm.com> Co-authored-by: Aaron Chew Aaron.Chew1@ibm.com <aaron@cccxl016.pok.ibm.com>
foundation-model-stack · Aug 1, 2024 · a6f6ef0 · a6f6ef0
1 parent 4ea65e0
commit a6f6ef0
Show file tree

Hide file tree

Showing 19 changed files with 1,195 additions and 4 deletions.
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -29,6 +29,7 @@ jobs:
           - "framework"
           - "accelerated-peft"
           - "fused-ops-and-kernels"
+          - "instruct-lab"
 
     steps:
       - uses: actions/checkout@v4

diff --git a/plugins/framework/src/fms_acceleration/constants.py b/plugins/framework/src/fms_acceleration/constants.py
@@ -21,4 +21,4 @@
 # and activated.
 # - hence the plugins that have model loaders should be on top of this list
 
-PLUGINS = ["peft", "foak"]
+PLUGINS = ["peft", "foak", "ilab"]
diff --git a/plugins/framework/src/fms_acceleration/framework.py b/plugins/framework/src/fms_acceleration/framework.py
@@ -193,7 +193,9 @@ def augmentation(
         train_args: TrainingArguments,
         modifiable_args: Tuple,
     ):
-        model_archs = set(model.config.architectures)  # get the config
+        # get the config
+        archs = model.config.architectures
+        model_archs = set(archs if archs is not None else [])
 
         # NOTE: this assumes that augmentation order does not matter
         for plugin_name, plugin in self.active_plugins:

diff --git a/plugins/framework/src/fms_acceleration/framework_plugin.py b/plugins/framework/src/fms_acceleration/framework_plugin.py
@@ -68,7 +68,8 @@ def get_relevant_configuration_sections(configuration: Dict) -> Dict:
             _cfg = relevant_config
             while n > 1:
                 p = path.pop(0)
-                _cfg[p] = {}
+                if p not in _cfg:
+                    _cfg[p] = {}
                 _cfg = _cfg[p]
                 n -= 1
 

diff --git a/plugins/instruct-lab/.isort.cfg b/plugins/instruct-lab/.isort.cfg
@@ -0,0 +1,13 @@
+[settings]
+profile=black
+from_first=true
+import_heading_future=Future
+import_heading_stdlib=Standard
+import_heading_thirdparty=Third Party
+import_heading_firstparty=First Party
+import_heading_localfolder=Local
+known_firstparty=
+known_localfolder=tuning
+
+# skip code imported from unsloth
+skip_glob=**/unsloth*/**