L10 module improvements

helpers/fix_bins_lnk_emulation.sh

@@ -0,0 +1,102 @@
+#!/bin/bash -p
+
+# EMBA - EMBEDDED LINUX ANALYZER
+#
+# Copyright 2020-2023 Siemens Energy AG
+#
+# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
+# welcome to redistribute it under the terms of the GNU General Public License.
+# See LICENSE file for usage of this software.
+#
+# EMBA is licensed under GPLv3
+#
+# Author(s): Michael Messner
+
+# Description:  Walk through a root directory and fix ELF permissions. Additionally
+#               this script also tries to fix sym links which are regularly broken
+
+BUSYBOX="/busybox"
+
+ROOT_DIR="${1:-}"
+if ! [[ -d "${ROOT_DIR}" ]]; then
+  echo "[-] No valid root directory provided - Fix links and bin permissions failed"
+  exit
+fi
+
+cp "$(command -v busybox)" "$ROOT_DIR"
+chmod +x "$ROOT_DIR"/busybox
+
+echo "[*] Identifying possible executable files"
+mapfile -t POSSIBLE_ELFS < <(find "$ROOT_DIR" -type f -exec file {} \; | grep "ELF\|executable" | cut -d: -f1)
+
+for POSSIBLE_ELF in "${POSSIBLE_ELFS[@]}"; do
+  [[ -x "${POSSIBLE_ELF}" ]] && continue
+  echo "[*] Processing executable $(basename "$POSSIBLE_ELF") - chmod privileges"
+  chmod +x "$POSSIBLE_ELF"
+done
+
+HOME_DIR="$(pwd)"
+if [[ -d "$ROOT_DIR" ]]; then
+  cd "$ROOT_DIR" || exit 1
+else
+  exit 1
+fi
+
+echo ""
+echo "[*] Identifying possible dead symlinks"
+mapfile -t POSSIBLE_DEAD_SYMLNKS < <(find "." -type f) # -exec file {} \; | grep "data\|ASCII\ text" | cut -d: -f1)
+
+for POSSIBLE_DEAD_SYMLNK in "${POSSIBLE_DEAD_SYMLNKS[@]}"; do
+  DIR_ORIG_FILE=""
+  if [[ "$(strings "$POSSIBLE_DEAD_SYMLNK" | wc -l)" -gt 1 ]]; then
+    continue
+  fi
+  if [[ "$(wc -c "$POSSIBLE_DEAD_SYMLNK" | awk '{print $1}')" -gt 200 ]]; then
+    continue
+  fi
+  if ! [[ "$(strings "$POSSIBLE_DEAD_SYMLNK")" =~ ^[a-zA-Z0-9./_~'-']+$ ]]; then
+    continue
+  fi
+
+  DIR_ORIG_FILE=$(dirname "$POSSIBLE_DEAD_SYMLNK")
+  [[ -z "$DIR_ORIG_FILE" ]] && continue
+  if ! [[ -d "${DIR_ORIG_FILE}" ]] && ! [[ -L "${DIR_ORIG_FILE}" ]]; then
+    echo "[*] Directory to unknown detected: $POSSIBLE_DEAD_SYMLNK -> ${DIR_ORIG_FILE}"
+  fi
+
+  TMP_LNK_ORIG=$(strings "$POSSIBLE_DEAD_SYMLNK")
+  [[ -z "$TMP_LNK_ORIG" ]] && TMP_LNK_ORIG=$(cat "$POSSIBLE_DEAD_SYMLNK")
+  [[ -z "$TMP_LNK_ORIG" ]] && continue
+
+  if [[  ${TMP_LNK_ORIG:0:1} == "/" ]]; then
+    # if we have an absolute path we can just use it
+    LNK_TARGET=".""${TMP_LNK_ORIG}"
+    # sometimes the directory of the final dest does not exist - lets check and create it
+    DIR_LNK_TARGET=$(dirname "$LNK_TARGET")
+    if ! [[ -d "${DIR_LNK_TARGET}" ]]; then
+      echo "[*] Creating ${DIR_LNK_TARGET}"
+      chroot . "${BUSYBOX}" mkdir -p "${DIR_LNK_TARGET}"
+    fi
+  else
+    LNK_TARGET="$DIR_ORIG_FILE"/"${TMP_LNK_ORIG}"
+  fi
+
+  if ! [[ -f "${LNK_TARGET}" ]] && ! [[ -d "${LNK_TARGET}" ]] && ! [[ -L "${LNK_TARGET}" ]]; then
+    echo "[*] Unknown or non existent target detected: ${POSSIBLE_DEAD_SYMLNK} -> ${LNK_TARGET}"
+    LNK_TARGET_NAME="$(basename "${LNK_TARGET}")"
+    mapfile -t POSSIBLE_MATCHES < <(find "." -name "$LNK_TARGET_NAME" -exec file {} \; | grep ELF | cut -d: -f1)
+    for MATCH in "${POSSIBLE_MATCHES[@]}"; do
+      echo "[*] Found possible matching file $MATCH"
+    done
+  fi
+
+  LNK_TARGET=${LNK_TARGET/\./}
+  POSSIBLE_SYMLNK_NAME=${POSSIBLE_DEAD_SYMLNK/\./}
+
+  echo -e "[*] Symlink file $POSSIBLE_SYMLNK_NAME - ${LNK_TARGET}"
+  chroot . "$BUSYBOX" rm "$POSSIBLE_DEAD_SYMLNK"
+  chroot . "$BUSYBOX" ln -s "${LNK_TARGET}" "$POSSIBLE_SYMLNK_NAME"
+done
+
+cd "$HOME_DIR" || exit 1
+rm "$ROOT_DIR"/busybox

modules/L10_system_emulation.sh

@@ -258,6 +258,11 @@ create_emulation_filesystem() {
     print_output "[*] Copy extracted root filesystem to new QEMU image"
     cp -prf "$ROOT_PATH"/* "$MNT_POINT"/ || true
 
+    if [[ -f "$HELP_DIR"/fix_bins_lnk_emulation.sh ]]; then
+      print_output "[*] Starting link fixing helper ..."
+      "$HELP_DIR"/fix_bins_lnk_emulation.sh "$MNT_POINT"
+    fi
+
     # ensure that the needed permissions for exec files are set correctly
     # This is needed at some firmwares have corrupted permissions on ELF or sh files
     print_output "[*] Multiple firmwares have broken script and ELF permissions - We fix them now"
@@ -577,6 +582,7 @@ main_emulation() {
         [[ "$F_STARTUP" -eq 0 && "$NETWORK_MODE" == "default" ]] || [[ "$DETECTED_IP" -eq 0 ]]; then
         mv "$LOG_PATH_MODULE"/qemu.initial.serial.log "$LOG_PATH_MODULE"/qemu.initial.serial_"$IMAGE_NAME"_"$INIT_FNAME"_base_init.log
         COUNTING_1st=$(wc -l "$LOG_PATH_MODULE"/qemu.initial.serial_"$IMAGE_NAME"_"$INIT_FNAME"_base_init.log | awk '{print $1}')
+        PORTS_1st=$(grep -a "inet_bind" "$LOG_PATH_MODULE"/qemu.initial.serial_"$IMAGE_NAME"_"$INIT_FNAME"_base_init.log | sort -u | wc -l | awk '{print $1}' || true)
         if [[ "$KINIT" == "rdinit="* ]]; then
           print_output "[*] Warning: Unknown EMBA startup found via rdinit - testing init"
           # strip rd from rdinit
@@ -595,17 +601,28 @@ main_emulation() {
         F_STARTUP=$(grep -a -c "EMBA preInit script starting" "$LOG_PATH_MODULE"/qemu.initial.serial.log || true)
         F_STARTUP=$(( "$F_STARTUP" + "$(grep -a -c "Network configuration - ACTION" "$LOG_PATH_MODULE"/qemu.initial.serial.log || true)" ))
         COUNTING_2nd=$(wc -l "$LOG_PATH_MODULE"/qemu.initial.serial.log | awk '{print $1}')
+        PORTS_2nd=$(grep -a "inet_bind" "$LOG_PATH_MODULE"/qemu.initial.serial.log | sort -u | wc -l | awk '{print $1}' || true)
         # IPS_INT_VLAN is always at least 1 for the default configuration
         if [[ "${#PANICS[@]}" -gt 0 ]] || [[ "$F_STARTUP" -eq 0 && "${#IPS_INT_VLAN[@]}" -lt 2 ]] || \
           [[ "$DETECTED_IP" -eq 0 ]]; then
+          if [[ "$PORTS_1st" -gt "$PORTS_2nd" ]]; then
+            if [[ "$KINIT" == "rdinit="* ]]; then
+              print_output "[*] Warning: switching back to init (identified services - $PORTS_1st / $PORTS_2nd)"
+              # strip rd from rdinit
+              KINIT="${KINIT:2}"
+            else
+              print_output "[*] Warning: switching back to rdinit (identified services - $PORTS_1st / $PORTS_2nd)"
+              # make rdinit from init
+              KINIT="rd""$KINIT"
+            fi
           # we only switch back if the first check has more output generated
-          if [[ "$COUNTING_1st" -gt "$COUNTING_2nd" ]]; then
+          elif [[ "$COUNTING_1st" -gt "$COUNTING_2nd" ]] && [[ "$PORTS_1st" -ge "$PORTS_2nd" ]]; then
             if [[ "$KINIT" == "rdinit="* ]]; then
-              print_output "[*] Warning: switching back to init"
+              print_output "[*] Warning: switching back to init (generated log output)"
               # strip rd from rdinit
               KINIT="${KINIT:2}"
             else
-              print_output "[*] Warning: switching back to rdinit"
+              print_output "[*] Warning: switching back to rdinit (generated log output)"
               # make rdinit from init
               KINIT="rd""$KINIT"
             fi
@@ -1151,7 +1168,7 @@ get_networking_details_emulation() {
     mapfile -t NVRAM < <(grep -a "\[NVRAM\] " "$LOG_PATH_MODULE"/qemu.initial.serial.log | awk '{print $3}' | grep -E '[[:alnum:]]{3,50}' | sort -u || true)
     # mapfile -t NVRAM_SET < <(grep -a "nvram_set" "$LOG_PATH_MODULE"/qemu.initial.serial.log | cut -d: -f2 | sed 's/^\ //g' | cut -d\  -f1 | sed 's/\"//g' | grep -v "^#" | grep -E '[[:alnum:]]{3,50}'| sort -u || true)
     # we check all available qemu logs for services that are started:
-    mapfile -t PORTS < <(grep -a "inet_bind" "$LOG_PATH_MODULE"/qemu.*serial*.log | sed -E 's/.*inet_bind\[PID:\ [0-9]+\ //' | sort -u || true)
+    mapfile -t PORTS < <(grep -a "inet_bind" "$LOG_PATH_MODULE"/qemu.initial.serial.log | sed -E 's/.*inet_bind\[PID:\ [0-9]+\ //' | sort -u || true)
     mapfile -t VLAN_HW_INFO_DEV < <(grep -a -E "adding VLAN [0-9] to HW filter on device eth[0-9]" "$LOG_PATH_MODULE"/qemu.initial.serial.log | awk -F\  '{print $NF}' | sort -u || true)
 
     NVRAM_TMP=( "${NVRAM[@]}" )

modules/P70_unblob.sh

@@ -134,7 +134,7 @@ unblobber() {
     mkdir -p "$OUTPUT_DIR_UNBLOB"
   fi
 
-  timeout --preserve-status --signal SIGINT 300 "$UNBLOB_BIN" -e "$OUTPUT_DIR_UNBLOB" "$FIRMWARE_PATH_" | tee -a "$LOG_FILE" || true
+  timeout --preserve-status --signal SIGINT 300 "$UNBLOB_BIN" -v -k -e "$OUTPUT_DIR_UNBLOB" "$FIRMWARE_PATH_" | tee -a "$LOG_FILE" || true
 
   print_ln
 }

modules/S115_usermode_emulator.sh

@@ -80,6 +80,10 @@ S115_usermode_emulator() {
       print_ln
       NEG_LOG=1
       print_output "[*] Detected root path: $ORANGE$R_PATH$NC"
+      if [[ -f "$HELP_DIR"/fix_bins_lnk_emulation.sh ]]; then
+        print_output "[*] Starting link fixing helper ..."
+        "$HELP_DIR"/fix_bins_lnk_emulation.sh "$R_PATH"
+      fi
       # MD5_DONE_INT is the array of all MD5 checksums for all root paths -> this is needed to ensure that we do not test bins twice
       MD5_DONE_INT=()
       BIN_CNT=0