[12.0 stable] Backport all recent wwan improvements

docs/WIRELESS.md

@@ -151,6 +151,26 @@ connectors depending on your antenna. Other modems like the EM7565 use MHF4 conn
 instead of u.fl. Remember that the longer the cable from your modem to the antenna, the more signal
 you will lose.
 
+### eSIM
+
+An eSIM (embedded SIM) is a form of SIM card that is embedded directly into a device. Instead of
+an integrated circuit located on a removable SIM card, an eSIM consists of software installed
+onto an eUICC chip permanently attached to a device.
+
+Currently, EVE does not support embedded SIM. A missing piece is the Local Profile Assistant (LPA),
+which is responsible for downloading eSIM profiles from an SM-DP+ (Subscription Manager Data
+Preparation) server, installing them in the device's eUICC, and managing switching between profiles
+(the digital equivalent of swapping SIM cards). ModemManager, utilized by EVE for cellular modem
+management, does not have an LPA built in. Consequently, we must explore alternative open-source
+projects that provide an LPA implementation and integrate one of them with our wwan microservice.
+
+At present, EVE can at least accurately report the eSIM status (such as slot index and profile
+presence). However, if the user has not specified the SIM slot to use, EVE prioritizes physical SIM
+cards over eSIM. Specifically, if the device is preconfigured (e.g., by the manufacturer) with eSIM
+as the primary slot and the user has not explicitly set the slot index via the controller, EVE
+automatically designates the first physical slot (with the lowest index) as the primary option.
+EVE will only attempt to connect with the eSIM if the user explicitly selects the eSIM slot.
+
 ## Radio Silence
 
 Radio silence is the act of disabling all radio transmission for safety or security reasons.

pkg/pillar/cmd/nim/nim.go

@@ -1058,6 +1058,7 @@ func (n *nim) includeLastResortPort(ifAttrs netmonitor.IfAttrs) bool {
 		strings.HasPrefix(ifName, "nbu") ||
 		strings.HasPrefix(ifName, "nbo") ||
 		strings.HasPrefix(ifName, "wlan") ||
+		strings.HasPrefix(ifName, "wwan") ||
 		strings.HasPrefix(ifName, "keth")
 	if exclude {
 		return false

pkg/wwan/Dockerfile

@@ -13,6 +13,7 @@ ENV LIBMBIM_VERSION=1.30.0
 ENV LIBQMI_VERSION=1.34.0
 ENV LIBQRTR_VERSION=1.2.2
 ENV PICOCOM_COMMIT=1acf1ddabaf3576b4023c4f6f09c5a3e4b086fb8
+ENV LENOVO_WWAN_UNLOCK_COMMIT=dc9a7eccf72b83e6db528da930cc7f19d6cdd523
 
 ADD --keep-git-dir=true https://gitlab.freedesktop.org/mobile-broadband/libqrtr-glib.git#${LIBQRTR_VERSION} /libqrtr
 WORKDIR /libqrtr
@@ -48,6 +49,14 @@ WORKDIR /picocom
 # Need this patch to build with musl: https://github.com/npat-efault/picocom/commit/1acf1ddabaf3576b4023c4f6f09c5a3e4b086fb8
 RUN make -j "$(getconf _NPROCESSORS_ONLN)" && strip picocom && cp picocom /usr/bin/
 
+# Install FCC-unlock scripts/tools provided by device vendors
+ADD --keep-git-dir=true https://github.com/lenovo/lenovo-wwan-unlock.git#${LENOVO_WWAN_UNLOCK_COMMIT} /lenovo-wwan-unlock
+WORKDIR /lenovo-wwan-unlock
+RUN cp libmbimtools.so /out/usr/lib/ && chmod 444 /out/usr/lib/libmbimtools.so
+RUN mkdir -p /out/opt/lenovo/ && \
+    cp DPR_Fcc_unlock_service /out/opt/lenovo/ && chmod 544 /out/opt/lenovo/DPR_Fcc_unlock_service
+COPY --chown=root:root --chmod=500 fcc-unlock /out/usr/lib/ModemManager/fcc-unlock.d
+
 RUN strip /usr/bin/*cli /usr/libexec/*proxy
 
 RUN mkdir -p /var/run/dbus/

pkg/wwan/README.md

@@ -189,24 +189,46 @@ More information on the topic of IP connectivity setup in cellular modems can be
 ## Architecture
 
 Cellular connectivity in EVE involves multiple components operating across different layers
-in both user-space and kernel-space. It begins with a physical cellular modem connected via a USB
-interface, exchanging control messages with the host over a CDC-WMC channel, operated on the host
-side by the cdc-wdm driver. Kernel modules like qmi_wwan and cdc_mbim use this channel to facilitate
-communication between the user-space of the host and the cellular modem in the language of QMI/MBIM
-protocols, while user-space libraries libqmi and libmbim provide high-level client-facing C-bindings
-for these protocols.
+in both user-space and kernel-space. It begins with a physical cellular modem typically connected
+via a USB interface, exchanging control messages with the host over a CDC-WMC channel, operated
+on the host side by the cdc-wdm driver. Kernel modules like qmi_wwan and cdc_mbim use this channel
+to facilitate communication between the user-space of the host and the cellular modem in the language
+of QMI/MBIM protocols, while user-space libraries libqmi and libmbim provide high-level client-facing
+C-bindings for these protocols.
+
+With modern high-speed 4G and 5G modems, it is more and more common to connect the modem over the PCIe
+bus instead of USB bus. PCIe offers higher speed, lower latency and lower power consumption than
+USB equivalent, making it perfectly suitable for 5G high speed requirements (up to 20Gbps).
+PCI differs from USB in that PCI devices do not offer high level operations and concepts such
+as USB transfers, sub-devices and endpoints. Instead, PCI drivers are built on top of low level
+operations such as memory-mapped I/O and DMA transfers, making them generally more complex.
+To provide something similar to USB interfaces and endpoints, Qualcomm created the Modem-Host
+Interface ([MHI](https://docs.kernel.org/mhi/mhi.html)), which can be used by a host to communicate
+with any PCIe modem implementing this interface. Linux kernel provides drivers implementing the MHI
+interface since version 5.13. For example, `mhi_wwan_ctrl` is used to drive all control channels
+(QMI, MBIM, AT), while `mhi_net` facilitates packet flow between the host and the modem. EVE kernel
+is built with all these drivers included (available in EVE since version 12.0.0).
+
 The ModemManager user-space daemon serves as a central manager, handling modem initialization,
 network registration, and data connection management, ensuring a standardized and unified interface
 for cellular modem communication on Linux systems. It uses libqmi and libmbim libraries and in some
 cases also AT command set to interact with and manage modems.
 On top of that, our `mmagent` acts as an adapter between the declarative EVE API and
 the imperative ModemManager API, plus it manages the IP settings of the wwan network interfaces
 in the Linux network stack.
+
 Moreover, there are some additional tools installed in the wwan container, such as mmcli, qmicli,
 mbimcli, which are provided solely for the troubleshooting purposes.
-Diagram belows depicts hierarchy and placements of all components and how they interact with each other:
 
-![wwan components](./pics/wwan.png)
+Diagram belows depicts hierarchy and placements of all components and how they interact with each other
+in case of USB modems:
+
+![wwan-usb components](./pics/wwan-usb.png)
+
+In case of PCIe modems, both control and data path use different drivers in the kernel, but the components
+and the interaction between them in the user-space is pretty much the same:
+
+![wwan-pcie components](./pics/wwan-pcie.png)
 
 Further information on the topic of cellular connectivity in Linux (not including EVE-specific components):
 
@@ -226,11 +248,10 @@ Further information on the topic of cellular connectivity in Linux (not includin
 * [This video presentation](https://www.youtube.com/watch?v=NPeMqK_vFFc) provides a brief overview of
   all major components and protocols involved in the cellular modem support in Linux, along with their
   evolution. This presentation also touches on the topic of modern high-speed 5G modems connected
-  over the PCI bus instead of the USB bus for even better performance. In this case, data are transmitted
+  over the PCIe bus instead of the USB bus for even better performance. In this case, data are transmitted
   between the host and the modem using the [MHI protocol](https://docs.kernel.org/mhi/mhi.html).
   Control-plane still relies on QMI/MBIM protocols, running on top of MHI, meaning that there is
-  very little change from the user-space perspective. However, it is important to note that testing
-  of 5G modems connected via PCI on EVE is still TBD.
+  very little change from the user-space perspective.
 * [This document](https://modemmanager.org/docs/modemmanager/ip-connectivity-setup-in-lte-modems/)
   describes IP connectivity setup in cellular modems and signaling between modem and network
   during the registration and connect procedures.
@@ -557,10 +578,14 @@ To use these commands, connect to a device over SSH or a console and type:
 
 ```console
 eve enter wwan
-# For QMI device:
+# For QMI device connected over USB bus:
 qmicli -p -d /dev/<cdc-wdm-device> <command>
-# For MBIM device:
+# For QMI device connected over PCIe:
+qmicli -p -d /dev/wwan<index>qmi<index> <command>
+# For MBIM device connected over USB bus:
 mbimcli -p -d /dev/<cdc-wdm-device> <command>
+# For MBIM device connected over PCIe:
+mbimcli -p -d /dev/wwan<index>mbim<index> <command>
 ```
 
 For example, to get the IP settings of the currently established data connection:
@@ -635,8 +660,12 @@ For example, the following may be a reasonable session to initialize your modem
 
 ```console
 eve enter wwan
+# For USB modem:
 picocom -b 115200 /dev/ttyUSB2
 
+# For PCIe modem you would run instead something like:
+# picocom -b 115200 /dev/wwan0at0
+
 ati
 at!entercnd="A710"
 at+cpin?

pkg/wwan/fcc-unlock/1eac:1002

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Quectel EM160R-GL FCC Unlock Script
+
+ARCH="$(uname -m)"
+VENDOR="$(cat /sys/class/dmi/id/sys_vendor)"
+PRODUCT="$(cat /sys/class/dmi/id/product_family)"
+
+# Fallback to the open-source FCC unlock script when vendor-specific unlock tool
+# is not available or fails.
+# shellcheck disable=SC2015
+[ "$ARCH" = "x86_64" ] && [ "$VENDOR" = "LENOVO" ] && [ "$PRODUCT" = "ThinkEdge SE30" ] &&
+  /opt/lenovo/DPR_Fcc_unlock_service || /etc/ModemManager/fcc-unlock.d/1eac "$@"
+
+exit $?

pkg/wwan/fcc-unlock/2c7c:030a

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Quectel EM05-G FCC Unlock Script
+
+ARCH="$(uname -m)"
+VENDOR="$(cat /sys/class/dmi/id/sys_vendor)"
+PRODUCT="$(cat /sys/class/dmi/id/product_family)"
+
+# Fallback to the open-source FCC unlock script when vendor-specific unlock tool
+# is not available or fails.
+# shellcheck disable=SC2015
+[ "$ARCH" = "x86_64" ] && [ "$VENDOR" = "LENOVO" ] && [ "$PRODUCT" = "ThinkEdge SE10" ] &&
+  /opt/lenovo/DPR_Fcc_unlock_service || /etc/ModemManager/fcc-unlock.d/2c7c "$@"
+
+exit $?

pkg/wwan/fcc-unlock/2c7c:0311

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Quectel EM05-G Smart Gateway FCC Unlock Script
+
+ARCH="$(uname -m)"
+VENDOR="$(cat /sys/class/dmi/id/sys_vendor)"
+PRODUCT="$(cat /sys/class/dmi/id/product_family)"
+
+# Fallback to the open-source FCC unlock script when vendor-specific unlock tool
+# is not available or fails.
+# shellcheck disable=SC2015
+[ "$ARCH" = "x86_64" ] && [ "$VENDOR" = "LENOVO" ] && [ "$PRODUCT" = "ThinkEdge SE10" ] &&
+  /opt/lenovo/DPR_Fcc_unlock_service || /etc/ModemManager/fcc-unlock.d/2c7c "$@"
+
+exit $?

pkg/wwan/mm-init.sh

@@ -13,17 +13,20 @@ enable_fcc_unlock() {
   for SCRIPT in "${SOURCE_DIR}"*; do
     if [ -f "$SCRIPT" ]; then
       SCRIPT_NAME="$(basename "$SCRIPT")"
+      case "$SCRIPT_NAME" in
+        "1eac:1002" | "2c7c:030a" | "2c7c:0311")
+          # For these modems we have our own custom scripts.
+          continue
+          ;;
+      esac
       ln -sf "$SCRIPT" "${TARGET_DIR}${SCRIPT_NAME}"
     fi
   done
-
-  # "Quectel EM05G Smart Gateway" (2c7c:0311) is compatible with the same
-  # FCC unlock script as used for the regular "Quectel EM05G" (2c7c:030a).
-  ln -sf "${SOURCE_DIR}2c7c:030a" "${TARGET_DIR}2c7c:0311"
 }
 
 echo "Loading kernel modules used by ModemManager"
-modprobe -a qcserial usb_wwan qmi_wwan cdc_wdm cdc_mbim cdc_acm
+modprobe -a qcserial usb_wwan qmi_wwan cdc_wdm cdc_mbim cdc_acm \
+            wwan mhi mhi_pci_generic mhi_wwan_ctrl mhi_wwan_mbim
 echo "Kernel modules are loaded"
 
 echo "Starting D-Bus daemon"

pkg/wwan/mmagent/go.mod

@@ -1,31 +1,41 @@
 module github.com/lf-edge/eve/pkg/wwan/mmagent
 
-go 1.20
+go 1.21
 
 require (
 	github.com/godbus/dbus/v5 v5.1.0
-	github.com/lf-edge/eve/pkg/pillar v0.0.0-20231027065537-54cee95e20b8
+	github.com/lf-edge/eve/pkg/pillar v0.0.0-20240417234153-53dbc10640ca
 	github.com/miekg/dns v1.1.55
-	github.com/sirupsen/logrus v1.9.0
+	github.com/sirupsen/logrus v1.9.3
 	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
-	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
-	golang.org/x/sys v0.13.0
+	github.com/vishvananda/netlink v1.2.1-beta.2
+	golang.org/x/sys v0.17.0
 )
 
 require (
 	github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/go-containerregistry v0.8.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.15.5 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-containerregistry v0.14.0 // indirect
 	github.com/google/go-tpm v0.3.0 // indirect
-	github.com/lf-edge/eve-api/go v0.0.0-20240322135714-a2b011fedf87 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/lf-edge/eve-api/go v0.0.0-20240405192828-57b8263b8048 // indirect
+	github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d // indirect
 	github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/tools v0.3.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )