lkl: Android ARM (arm/arm64) support

Initial attempt to run an application with hijack library on Android
platform.  Tested mostly on Android 6.x and 7.x.

The build process assumes that the android ndk toolchain is installed in
a host system as circle.yml does in its test.  arm32 build uses
alternate linker, stored in tools/lkl/bin directory, in order to avoid
the link issue (issue torvalds#59).

The CircleCI test infrastructure requires to use ubuntu 14.04 for this
test.

* Limitations
- aarch64 isn't tested on circleci due to difficulties on aarch64
emulator.
- bionic libc on android-24 emulator (arm32) doesn't call destructor, so
some of tests in hijack-test.sh fail.
- net.sh doesn't properly test network related issue.

Fixes torvalds#59.

Signed-off-by: Hajime Tazaki <thehajime@gmail.com>

arch/lkl/Kconfig

@@ -19,6 +19,7 @@ config LKL
 	   select 64BIT if OUTPUT_FORMAT = "pe-x86-64"
        select HAVE_UNDERSCORE_SYMBOL_PREFIX if OUTPUT_FORMAT = "pe-i386"
        select 64BIT if OUTPUT_FORMAT = "elf64-x86-64-freebsd"
+       select 64BIT if OUTPUT_FORMAT = "elf64-littleaarch64"
        select IP_PNP
        select IP_PNP_DHCP
        select TCP_CONG_BBR

arch/lkl/Makefile

@@ -2,7 +2,7 @@ include arch/lkl/auto.conf
 
 KBUILD_CFLAGS += -fno-builtin
 
-ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf64-x86-64-freebsd elf32-littlearm ))
+ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf64-x86-64-freebsd elf32-littlearm elf64-littleaarch64))
 KBUILD_CFLAGS += -fPIC
 else ifneq (,$(filter $(OUTPUT_FORMAT),pe-i386 pe-x86-64 ))
 ifneq ($(OUTPUT_FORMAT),pe-x86-64)

arch/lkl/include/uapi/asm/Kbuild

@@ -25,7 +25,6 @@ generic-y += socket.h
 generic-y += sockios.h
 generic-y += stat.h
 generic-y += statfs.h
-generic-y += swab.h
 generic-y += termbits.h
 generic-y += termios.h
 generic-y += timex.h

arch/lkl/include/uapi/asm/swab.h

@@ -0,0 +1,10 @@
+#ifndef _ASM_LKL_SWAB_H
+#define _ASM_LKL_SWAB_H
+
+#ifndef __arch_swab32
+#define __arch_swab32(x) ___constant_swab32(x)
+#endif
+
+#include <asm-generic/swab.h>
+
+#endif /* _ASM_LKL_SWAB_H */

arch/lkl/scripts/headers_install.py

@@ -172,6 +172,8 @@ def replace(h):
 find_symbols(p, unions)
 p = re.compile("static\s+__inline__(\s+\w+)+\s+(\w+)\([^)]*\)\s")
 find_symbols(p, defines)
+p = re.compile("static\s+__always_inline(\s+\w+)+\s+(\w+)\([^)]*\)\s")
+find_symbols(p, defines)
 p = re.compile("enum\s+(\w*)\s*{([^}]*)}", re.M|re.S)
 q = re.compile("(\w+)\s*(,|=[^,]*|$)", re.M|re.S)
 find_enums(p, q, defines)

circle.yml

@@ -5,20 +5,30 @@ general:
 machine:
   # Add some environment variables
   environment:
-    CROSS_COMPILE: $(case $CIRCLE_NODE_INDEX in 0) host='' ;; 1) host='i686-w64-mingw32-' ;; 2) host='arm-linux-androideabi-' ;; esac; echo $host)
-    MKARG: $(case $CIRCLE_NODE_INDEX in 0) mkarg='dpdk=yes' ;; 1) host='i686-w64-mingw32-' ;; 2) host='arm-linux-androideabi-' ;; esac; echo $mkarg)
-    PATH: /home/ubuntu/android-toolchain/bin:${PATH}
+    CROSS_COMPILE: $(case $CIRCLE_NODE_INDEX in 0) host='' ;; 1) host='i686-w64-mingw32-' ;; 2) host='arm-linux-androideabi-' ;; 3) host='aarch64-linux-android-' ;; esac; echo $host)
+    MKARG: $(case $CIRCLE_NODE_INDEX in 0) mkarg='dpdk=yes' ;; 1)  ;; 2)  ;; 3) ;; esac; echo $mkarg)
+    PATH: /home/ubuntu/aarch64-linux-android-5.4-linaro-2016.06/bin:/home/ubuntu/android-toolchain/bin:${PATH}
     LKL_TEST_DHCP: 1
+    LKL_ANDROID_TEST: $(case $CIRCLE_NODE_INDEX in 2) echo 1 ;; 3) echo 1 ;; esac)
 
 ## Customize dependencies
 dependencies:
   pre:
     # required for 14.04 container
-    # - sudo dpkg --add-architecture i386
+    - sudo dpkg --add-architecture i386
+    - echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
+    - wget https://dl.winehq.org/wine-builds/Release.key ; sudo apt-key add Release.key ; sudo apt-add-repository 'https://dl.winehq.org/wine-builds/ubuntu/'
     - sudo apt-get update; sudo apt-get install bc libfuse-dev libarchive-dev xfsprogs valgrind gcc-mingw-w64-i686 wine qemu-user-static linux-headers-$(uname -r) ccache
-    - /usr/local/android-ndk/build/tools/make-standalone-toolchain.sh --platform=android-21 --install-dir=/home/ubuntu/android-toolchain --arch=arm
+    - wget -q --output-document=android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip && unzip android-ndk.zip
+    - if [ $CIRCLE_NODE_INDEX -eq 2 ] ; then ./android-ndk-r15b/build/tools/make_standalone_toolchain.py --arch arm --api 24 --install-dir /home/ubuntu/android-toolchain ; fi:
+        parallel: true
+    - if [ $CIRCLE_NODE_INDEX -eq 3 ] ; then ./android-ndk-r15b/build/tools/make_standalone_toolchain.py --arch arm64 --api 24 --install-dir /home/ubuntu/android-toolchain ; fi:
+        parallel: true
     - sudo cp tools/lkl/bin/i686-w64-mingw32-* /usr/bin:
         parallel: true
+    # for aarch64 instance
+    - if [ $CIRCLE_NODE_INDEX -eq 3 ] ; then cd $HOME ; wget https://android-git.linaro.org/platform/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-5.4-linaro.git/snapshot/aarch64-linux-android-5.4-linaro-2016.06.tar.gz ; tar xfz aarch64-linux-android-5.4-linaro-2016.06.tar.gz ; cd aarch64-linux-android-5.4-linaro-2016.06 ; ln -s /home/ubuntu/android-toolchain/sysroot ; fi:
+        parallel: true
     - git fetch --tags git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
   override:
     - mkdir -p ~/.ccache
@@ -36,9 +46,13 @@ test:
   pre:
     - tools/lkl/scripts/checkpatch.sh
   override:
-    - cd tools/lkl && if [ $CIRCLE_NODE_INDEX -eq 2 ] ; then make tests/boot-in.o; arm-linux-androideabi-gcc -o tests/boot tests/boot-in.o liblkl.a -static ; fi:
+    - if [ $CIRCLE_NODE_INDEX -eq 2 -o $CIRCLE_NODE_INDEX -eq 3 ] ; then emulator -avd circleci-android24 -no-window ; fi:
+        background: true
+        parallel: true
+    - if [ $CIRCLE_NODE_INDEX -eq 2 -o $CIRCLE_NODE_INDEX -eq 3 ] ; then circle-android wait-for-boot ; fi:
         parallel: true
-    - cd tools/lkl && make test:
+    # can't exec qemu-aarch64 emulator on circleci ...
+    - if [ $CIRCLE_NODE_INDEX -ne 3 ] ; then cd tools/lkl && make test ; fi:
          parallel: true
 
     - ? >

tools/lkl/Makefile

@@ -39,10 +39,11 @@ export srctree
 export CFLAGS += -I$(OUTPUT)/include -Iinclude -Wall -g -O2 -Wextra \
 	 -Wno-unused-parameter \
 	 -Wno-missing-field-initializers -fno-strict-aliasing
+LDFLAGS += -pie
 
 OUTPUT_FORMAT = $(shell $(LD) -r -print-output-format)
 
-ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf32-i386 elf64-x86-64-freebsd elf32-littlearm))
+ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf32-i386 elf64-x86-64-freebsd elf32-littlearm elf64-littleaarch64))
   OUTPUT_DEF = $(shell echo | $(CC) -dM -E -)
   CFLAGS   += -fPIC -pthread
   ifeq (,$(filter $(OUTPUT_DEF),__ANDROID__))
@@ -112,6 +113,9 @@ $(OUTPUT)liblkl$(SOSUF): LDFLAGS += -shared
 $(OUTPUT)liblkl-hijack$(SOSUF): $(OUTPUT)lib/hijack/hijack-in.o $(OUTPUT)liblkl.a
 $(OUTPUT)liblkl-hijack$(SOSUF): LDFLAGS += -shared -nodefaultlibs
 $(OUTPUT)liblkl-hijack$(SOSUF): LDLIBS += -ldl
+ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
+  $(OUTPUT)liblkl-hijack$(SOSUF): LDLIBS += -lgcc -lc
+endif
 
 $(OUTPUT)lklfuse$(EXESUF): $(OUTPUT)lklfuse-in.o $(OUTPUT)liblkl.a
 $(OUTPUT)lklfuse$(EXESUF): LDLIBS += -lfuse
@@ -131,12 +135,15 @@ endif
 TEST_TARGETS        := test valgrind gdb
 
 $(OUTPUT)tests/boot: $(OUTPUT)tests/boot-in.o $(OUTPUT)liblkl.a
+ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
+  $(OUTPUT)tests/boot: LDLIBS += -lc
+endif
 $(OUTPUT)tests/net-test: $(OUTPUT)tests/net-test-in.o $(OUTPUT)liblkl.a
 $(TEST_TARGETS): $(OUTPUT)tests/boot $(OUTPUT)tests/net-test
 
 # because of libdl, liblkl-hijack will not compile on windows
 # fortunately, the test target will handle a missing libhijack.so correctly
-ifeq (,$(filter $(OUTPUT_FORMAT),pe-i386 elf32-littlearm))
+ifeq (,$(filter $(OUTPUT_FORMAT),pe-i386))
   test: liblkl-hijack$(SOSUF)
 endif
 
@@ -145,7 +152,9 @@ $(OUTPUT)%-in.o: $(OUTPUT)lib/lkl.o FORCE
 
 $(OUTPUT)lib/lkl.o:
 	$(Q)$(MAKE) -C ../.. ARCH=lkl $(KOPT) defconfig
-	$(Q)$(MAKE) -C ../.. ARCH=lkl $(KOPT) install INSTALL_PATH=$(OUTPUT)
+# this workaround is for arm32 linker (ld.gold)
+	$(Q)export PATH=$(srctree)/tools/lkl/bin/:${PATH} ;\
+	$(MAKE) -C ../.. ARCH=lkl $(KOPT) install INSTALL_PATH=$(OUTPUT)
 
 $(OUTPUT)liblkl.a: $(OUTPUT)lib/lkl-in.o $(OUTPUT)lib/lkl.o
 	$(QUIET_AR)$(AR) -rc $@ $^
@@ -162,9 +171,7 @@ $(OUTPUT)cpfromfs$(EXESUF): cptofs$(EXESUF)
 ifneq (,$(filter $(OUTPUT_FORMAT),pe-i386))
   all: $(filter-out $(OUTPUT)liblkl-hijack$(SOSUF), $(ALL_LIBRARIES))
 else ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
-  all: $(filter-out $(OUTPUT)liblkl-hijack$(SOSUF), $(ALL_LIBRARIES))
-else ifneq (,$(filter $(OUTPUT_FORMAT),elf32-littlearm))
-  all: $(filter-out $(OUTPUT)liblkl-hijack$(SOSUF), $(ALL_LIBRARIES)) $(ALL_PROGRAMS)
+  all: $(ALL_LIBRARIES)
 else
   all: $(ALL_PROGRAMS) $(ALL_LIBRARIES)
 endif

tools/lkl/bin/arm-linux-androideabi-ld

@@ -0,0 +1 @@
+arm-linux-androideabi-ld.gold

tools/lkl/include/lkl.h

@@ -19,6 +19,10 @@ extern "C" {
 #define strtok_r strtok_s
 #endif
 
+#if defined(__ANDROID__) && __LKL__BITS_PER_LONG == 32
+#define __lkl__NR_fcntl __lkl__NR_fcntl64
+#endif
+
 #if __LKL__BITS_PER_LONG == 64
 #define lkl_sys_stat lkl_sys_newstat
 #define lkl_sys_lstat lkl_sys_newlstat

tools/lkl/lib/endian.h

@@ -5,9 +5,6 @@
 #include <sys/endian.h>
 #elif defined(__ANDROID__)
 #include <sys/endian.h>
-#define le16toh(x) letoh16(x)
-#define le32toh(x) letoh32(x)
-#define le64toh(x) letoh64(x)
 #elif defined(__MINGW32__)
 #include <winsock.h>
 #define le32toh(x) (x)

tools/lkl/lib/hijack/hijack.c

@@ -201,11 +201,18 @@ int socket(int domain, int type, int protocol)
 	if (domain == AF_UNIX || domain == PF_PACKET)
 		return host_socket(domain, type, protocol);
 
+	if (!lkl_running)
+		return host_socket(domain, type, protocol);
+
 	return lkl_call(__lkl__NR_socket, 3, domain, type, protocol);
 }
 
 HOST_CALL(ioctl);
+#ifdef __ANDROID__
+int ioctl(int fd, int req, ...)
+#else
 int ioctl(int fd, unsigned long req, ...)
+#endif
 {
 	va_list vl;
 	long arg;
@@ -367,9 +374,11 @@ void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset)
 	return lkl_sys_mmap(addr, length, prot, flags, fd, offset);
 }
 
+#ifndef __ANDROID__
 HOST_CALL(__xstat64)
 int stat(const char *pathname, struct stat *buf)
 {
 	CHECK_HOST_CALL(__xstat64);
 	return host___xstat64(0, pathname, buf);
 }
+#endif

tools/lkl/lib/hijack/init.c

@@ -17,9 +17,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
-#include <sys/stat.h>
 #include <fcntl.h>
 #include <errno.h>
+#include <signal.h>
 #include <sys/ioctl.h>
 #include <lkl.h>
 #include <lkl_host.h>
@@ -329,6 +329,17 @@ hijack_init(void)
 	if (single_cpu_mode == 1)
 		PinToFirstCpu(&ori_cpu);
 
+#ifdef __ANDROID__
+	struct sigaction sa;
+
+	sa.sa_handler = SIG_IGN;
+	sa.sa_flags = 0;
+	if (sigaction(32, &sa, 0) == -1) {
+		perror("sigaction");
+		exit(1);
+	}
+#endif
+
 	ret = lkl_start_kernel(&lkl_host_ops, boot_cmdline);
 	if (ret) {
 		fprintf(stderr, "can't start kernel: %s\n", lkl_strerror(ret));

tools/lkl/tests/Makefile

@@ -43,11 +43,21 @@ else
 	VALGRIND_CMD=valgrind_test
 endif
 
+ifneq ($(LKL_ANDROID_TEST),1)
 test:
 	$(call run,)
 	$(call lklfuse)
 	$(HIJACK_TEST)
 	./net.sh
+else
+test:
+	dd if=/dev/zero of=disk.img bs=1024 count=102400
+	yes | mkfs.ext4 disk.img >/dev/null
+	adb shell mkdir -p /data/local/tmp/lkl/bin
+	adb push ../bin/lkl-hijack.sh /data/local/tmp/lkl/bin
+	adb push  ../tests ../liblkl-hijack.so /data/local/tmp/lkl/
+	adb shell CROSS_COMPILE=${CROSS_COMPILE} /data/local/tmp/lkl/tests/android-test.sh
+endif
 
 valgrind:
 	$(call for_fs,$(VALGRIND_CMD))

tools/lkl/tests/android-test.sh

@@ -0,0 +1,16 @@
+#!/system/bin/sh
+
+set -e
+
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+export PATH=${script_dir}:${PATH}
+export LKL_ANDROID_TEST=1
+export TMPDIR=/data/local/tmp
+export CONFIG_AUTO_LKL_POSIX_HOST=y
+
+sed -i "s/\/bin\/bash/\/system\/bin\/sh /" ${script_dir}/../bin/lkl-hijack.sh
+
+cd ${script_dir}
+boot -d ${script_dir}/disk.img  -t ext4
+sh -x ${script_dir}/hijack-test.sh
+sh ${script_dir}/net.sh

tools/lkl/tests/hijack-test.sh

@@ -34,12 +34,12 @@ ${hijack_script} ip route
 
 echo "== ping test=="
 cp `which ping` .
-${hijack_script} ./ping 127.0.0.1 -c 1
+${hijack_script} ./ping -c 1 127.0.0.1
 rm ./ping
 
 echo "== ping6 test=="
 cp `which ping6` .
-${hijack_script} ./ping6 ::1 -c 1
+${hijack_script} ./ping6 -c 1 ::1
 rm ./ping6
 
 echo "== Mount/dump tests =="
@@ -50,18 +50,27 @@ ans=$(LKL_HIJACK_MOUNT=proc,sysfs\
   ${hijack_script} ip -h) || true
 # Need to grab the end because something earlier on prints out this
 # number
+# XXX: android-23 doesn't call destructor...
+if [ -z ${LKL_ANDROID_TEST} ] ; then
 echo "$ans" | tail -n 15 | grep "65536" # lo's MTU
 # lo's dev id
 echo "$ans" | grep "0x0"        # lo's dev_id
 # Doesn't really belong in this section, but might as well check for
 # it here.
 ! echo "$ans" | grep "WARN: failed to free"
+fi
 
 # boot_cmdline test
 echo "== boot command line tests =="
 ans=$(LKL_HIJACK_DEBUG=1\
   LKL_HIJACK_BOOT_CMDLINE="mem=100M" ${hijack_script} ip ad)
+if [ -z ${CROSS_COMPILE} ] ; then
 echo "$ans" | grep "100752k"
+elif [ "${CROSS_COMPILE}" = "arm-linux-androideabi-" ] ; then
+echo "$ans" | grep "101424k"
+elif [ "${CROSS_COMPILE}" = "aarch64-linux-android-" ] ; then
+echo "$ans" | grep "100756k"
+fi
 
 echo "== TAP tests =="
 if [ ! -c /dev/net/tun ]; then

tools/lkl/tests/net-test.c

@@ -11,6 +11,9 @@
 #include <arpa/inet.h>
 #include <netinet/ip.h>
 #include <netinet/ip_icmp.h>
+#ifdef __ANDROID__
+#include <linux/icmp.h>
+#endif
 
 #include <lkl.h>
 #include <lkl_host.h>