Merge branch 'main' into hengguo/h2o

.github/CODEOWNERS

@@ -35,6 +35,7 @@
 /intel_extension_for_transformers/neural_chat liang1.lv@intel.com
 /workflows/chatbot liang1.lv@intel.com
 /intel_extension_for_transformers/transformers/llm/quantization penghui.cheng@intel.com
+/intel_extension_for_transformers/transformers/modeling/modeling_gaudi xi2.chen@intel.com
 /intel_extension_for_transformers/transformers penghui.cheng@intel.com
 /intel_extension_for_transformers/utils penghui.cheng@intel.com
 /docs wenxin.zhang@intel.com

.github/workflows/Scaner_BDBA.yaml

@@ -0,0 +1,19 @@
+name: Scanner BDBA
+
+on:
+  workflow_dispatch:
+
+permissions: write-all
+jobs:
+  bdba_job:
+    name: BDBA Scan
+    uses: intel-innersource/frameworks.ai.infrastructure.code-scan-tools/.github/workflows/Scanner_Bdba.yml@one-ci-cd
+    with:
+      org: "intel-innersource"
+      repo: ${{ github.event.repository.name }}
+      refs: ${{ github.ref_name }}
+      group: "22"
+      runners: "['inner-source']"
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      BDBA_TOKEN: ${{ secrets.BDBA_TOKEN }}

.github/workflows/Scaner_Coverity.yaml

@@ -0,0 +1,22 @@
+name: Scanner Coverity PYTHON
+
+on:
+    workflow_dispatch:
+
+permissions: write-all
+jobs:
+  coverity_job:
+    name: Coverity
+    runs-on: inner-source
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+      - name: Scan
+        uses: intel-innersource/frameworks.devops.github.actions.coverity@main
+        with:
+          server: 'https://coverityent.devtools.intel.com/prod4'
+          username: ${{secrets.COVERITY_NAME }}
+          password: ${{secrets.COVERITY }}
+          project: IntelExtensionForTransformers
+          stream: 'IntelExtensionForTransformers12cpp'

.github/workflows/Scaner_Trivy.yaml

@@ -0,0 +1,12 @@
+name: Trivy Scan for Containers
+
+on:
+    workflow_dispatch:
+permissions: write-all
+jobs:
+  trivy_container_job:
+    uses: "intel-innersource/frameworks.ai.infrastructure.code-scan-tools/.github/workflows/Scanner_Trivy.yml@one-ci-cd"
+    with:
+      container: ${{ vars.TRIVY_CONTAINER_NAME }}
+      runners: "['inner-source']"
+      lmc: false

.github/workflows/docker/codeScan.dockerfile

@@ -29,7 +29,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     cloc \
     python3.10-venv \
-    git
+    git \
+    wget
 
 RUN ln -sf $(which python3) /usr/bin/python
 
@@ -38,4 +39,8 @@ RUN python -m pip install --no-cache-dir pylint==2.17.5\
     pyspelling\
     pydocstyle
 
+RUN python -m pip install --no-cache-dir bandit==1.7.8
+RUN wget -O /bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64
+RUN chmod +x /bin/hadolint
+
 WORKDIR /

.github/workflows/format_scan.yml

@@ -37,6 +37,7 @@ jobs:
             "cpplint",
             # "pydocstyle",
             #"pyspelling",
+            "hadolint"
           ]
       fail-fast: false
     steps:

.github/workflows/probot.yml

@@ -2,6 +2,7 @@ name: CI-Summary
 
 on:
   pull_request:
+    branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
   workflow_dispatch:
 

.github/workflows/script/formatScan/hadolint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
+
+find . -type f \( -name "Dockerfile*" \) -print -exec hadolint --ignore DL3006 --ignore DL3007 --ignore DL3008 {} \; 2>&1 | tee ${log_dir}/hadolint.log
+
+if [[ $(grep -c "error" ${log_dir}/hadolint.log) != 0 ]]; then
+    $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET
+    exit 1
+fi
+
+$BOLD_PURPLE && echo "Congratulations, Hadolint check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
+exit 0

.github/workflows/script/formatScan/pylint.sh

@@ -31,6 +31,8 @@ fi
 pip install lm-eval
 pip install accelerate nlpaug nltk schema optimum-intel optimum peft
 pip install --upgrade --force-reinstall transformers==4.36.2
+pip install optimum-habana
+pip install deepspeed
 
 echo "[DEBUG] list pipdeptree..."
 pip install pipdeptree
@@ -41,25 +43,15 @@ python -m pylint -f json --disable=R,C,W,E1129 \
     --max-line-length=120 \
     --extension-pkg-whitelist=numpy,nltk \
     --ignored-classes=TensorProto,NodeProto \
-    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,neural_compressor.benchmark,intel_extension_for_transformers.neural_engine_py,intel_extension_for_transformers.qbits,cv2,PIL.Image \
+    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,neural_compressor.benchmark,intel_extension_for_transformers.neural_engine_py,intel_extension_for_transformers.qbits,intel_extension_for_transformers.transformers.modeling.modeling_gaudi.generation.utils,intel_extension_for_transformers.transformers.modeling.modeling_gaudi.models.mixtral.modeling_mixtral,cv2,PIL.Image \
     /intel-extension-for-transformers/intel_extension_for_transformers >${log_dir}/pylint.json
-exit_code1=$?
-
-python -m pylint -f json --disable=R,C,W,E1129 \
-    --enable=line-too-long \
-    --max-line-length=120 \
-    --disable=no-name-in-module,import-error,no-member,undefined-variable,no-value-for-parameter,unexpected-keyword-arg,not-callable,no-self-argument,too-many-format-args,invalid-unary-operand-type,too-many-function-args \
-    --extension-pkg-whitelist=numpy,nltk \
-    --ignored-classes=TensorProto,NodeProto \
-    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,neural_compressor.benchmark,intel_extension_for_transformers.neural_engine_py,intel_extension_for_transformers.qbits,cv2,PIL.Image \
-    /intel-extension-for-transformers/intel_extension_for_transformers >> ${log_dir}/pylint.json
-exit_code2=$?
+exit_code=$?
 
 $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------" && $RESET
 cat ${log_dir}/pylint.json
 $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
 
-if [ ${exit_code1} -ne 0 ] || [ ${exit_code2} -ne 0 ]; then
+if [ ${exit_code} -ne 0 ]; then
     $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET
     exit 1
 fi

.github/workflows/script/formatScan/trellix.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+log_dir=${workspace}/.github/workflows/script/formatScan
+
+
+echo "---Updating definition (DAT) files ---"
+DEFS_URL=https://update.nai.com/products/commonupdater/current/vscandat1000/dat/0000
+echo "Finding latest defs at $DEFS_URL/avvdat.ini..." \
+ && wget -q $DEFS_URL/avvdat.ini \
+ && echo "SUCCESS" || fail
+
+inifile="avvdat.ini"
+filename=`awk -F"=" '$2 ~ /avvdat.*zip/ { print $2 } ' $inifile`
+filename2="$(echo -e "${filename}" | tr -d '[:space:]')"
+
+if [ -z "$filename2" ]
+then
+      echo "Cannot get defs information from INI file:"
+      cat $inifile
+      fail
+fi
+
+echo "Downloading latest defs from $DEFS_URL/$filename2..." \
+ && wget -q $DEFS_URL/$filename2 \
+ && echo "SUCCESS" || fail
+
+echo "Extracting latest defs..." \
+ && unzip -o $filename2 -d /usr/local/uvscan \
+ && echo "SUCCESS" || fail
+
+echo "--- Scanning ---"
+ENV_SCAN_OPTS="--analyze --mime --program --recursive --unzip --threads 4 --summary --verbose --html=${workspace}/.github/workflows/script/formatScan/report.html"
+echo "Scan Options: $ENV_SCAN_OPTS"
+
+rm -r ${workspace}/avvdat*
+rm -r ${workspace}/.git
+uvscan $ENV_SCAN_OPTS ${workspace} 2>&1 | tee ${log_dir}/trellix.log
+
+
+if [[ $(grep "Possibly Infected" ${log_dir}/trellix.log | sed 's/[^0-9]//g') != 0 ]]; then
+    $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET
+    exit 1
+fi
+
+$BOLD_PURPLE && echo "Congratulations, Trellix Scan passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
+exit 0

.github/workflows/script/unitTest/coverage/.engine-coveragerc

@@ -8,9 +8,7 @@ omit =
  */intel_extension_for_transformers/haystack/**
  */intel_extension_for_transformers/llama_index/**
  */intel_extension_for_transformers/transformers/dynamic/**
- */intel_extension_for_transformers/transformers/mixture/**
  */intel_extension_for_transformers/transformers/modeling/**
- */intel_extension_for_transformers/transformers/nas/**
  */intel_extension_for_transformers/transformers/pruner/**
  */intel_extension_for_transformers/transformers/utils/**
  */intel_extension_for_transformers/transformers/llm/**

.github/workflows/script/unitTest/coverage/.neural-chat-coveragerc

@@ -12,9 +12,7 @@ omit =
  */intel_extension_for_transformers/transformers/llm/library/**
  */intel_extension_for_transformers/transformers/llm/operator/**
  */intel_extension_for_transformers/transformers/dynamic/**
- */intel_extension_for_transformers/transformers/mixture/**
  */intel_extension_for_transformers/transformers/modeling/**
- */intel_extension_for_transformers/transformers/nas/**
  */intel_extension_for_transformers/transformers/pruner/**
   */intel_extension_for_transformers/transformers/runtime/**
  */intel_extension_for_transformers/transformers/setfit/**