Merge pull request torvalds#372 from libos-nuse/android-arm32-aarch64

lkl: Android ARM (arm/arm64) support

arch/lkl/Kconfig

@@ -19,6 +19,7 @@ config LKL
 	   select 64BIT if OUTPUT_FORMAT = "pe-x86-64"
        select HAVE_UNDERSCORE_SYMBOL_PREFIX if OUTPUT_FORMAT = "pe-i386"
        select 64BIT if OUTPUT_FORMAT = "elf64-x86-64-freebsd"
+       select 64BIT if OUTPUT_FORMAT = "elf64-littleaarch64"
        select IP_PNP
        select IP_PNP_DHCP
        select TCP_CONG_BBR

arch/lkl/Makefile

@@ -2,7 +2,7 @@ include arch/lkl/auto.conf
 
 KBUILD_CFLAGS += -fno-builtin
 
-ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf64-x86-64-freebsd elf32-littlearm ))
+ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf64-x86-64-freebsd elf32-littlearm elf64-littleaarch64))
 KBUILD_CFLAGS += -fPIC
 else ifneq (,$(filter $(OUTPUT_FORMAT),pe-i386 pe-x86-64 ))
 ifneq ($(OUTPUT_FORMAT),pe-x86-64)

arch/lkl/include/uapi/asm/Kbuild

@@ -25,7 +25,6 @@ generic-y += socket.h
 generic-y += sockios.h
 generic-y += stat.h
 generic-y += statfs.h
-generic-y += swab.h
 generic-y += termbits.h
 generic-y += termios.h
 generic-y += timex.h

arch/lkl/include/uapi/asm/swab.h

@@ -0,0 +1,10 @@
+#ifndef _ASM_LKL_SWAB_H
+#define _ASM_LKL_SWAB_H
+
+#ifndef __arch_swab32
+#define __arch_swab32(x) ___constant_swab32(x)
+#endif
+
+#include <asm-generic/swab.h>
+
+#endif /* _ASM_LKL_SWAB_H */

arch/lkl/scripts/headers_install.py

@@ -172,6 +172,8 @@ def replace(h):
 find_symbols(p, unions)
 p = re.compile("static\s+__inline__(\s+\w+)+\s+(\w+)\([^)]*\)\s")
 find_symbols(p, defines)
+p = re.compile("static\s+__always_inline(\s+\w+)+\s+(\w+)\([^)]*\)\s")
+find_symbols(p, defines)
 p = re.compile("enum\s+(\w*)\s*{([^}]*)}", re.M|re.S)
 q = re.compile("(\w+)\s*(,|=[^,]*|$)", re.M|re.S)
 find_enums(p, q, defines)

circle.yml

@@ -5,20 +5,30 @@ general:
 machine:
   # Add some environment variables
   environment:
-    CROSS_COMPILE: $(case $CIRCLE_NODE_INDEX in 0) host='' ;; 1) host='i686-w64-mingw32-' ;; 2) host='arm-linux-androideabi-' ;; esac; echo $host)
-    MKARG: $(case $CIRCLE_NODE_INDEX in 0) mkarg='dpdk=yes' ;; 1) host='i686-w64-mingw32-' ;; 2) host='arm-linux-androideabi-' ;; esac; echo $mkarg)
-    PATH: /home/ubuntu/android-toolchain/bin:${PATH}
+    CROSS_COMPILE: $(case $CIRCLE_NODE_INDEX in 0) host='' ;; 1) host='i686-w64-mingw32-' ;; 2) host='arm-linux-androideabi-' ;; 3) host='aarch64-linux-android-' ;; esac; echo $host)
+    MKARG: $(case $CIRCLE_NODE_INDEX in 0) mkarg='dpdk=yes' ;; 1)  ;; 2)  ;; 3) ;; esac; echo $mkarg)
+    PATH: /home/ubuntu/aarch64-linux-android-5.4-linaro-2016.06/bin:/home/ubuntu/android-toolchain/bin:${PATH}
     LKL_TEST_DHCP: 1
+    LKL_ANDROID_TEST: $(case $CIRCLE_NODE_INDEX in 2) echo 1 ;; 3) echo 1 ;; esac)
 
 ## Customize dependencies
 dependencies:
   pre:
     # required for 14.04 container
-    # - sudo dpkg --add-architecture i386
+    - sudo dpkg --add-architecture i386
+    - echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
+    - wget https://dl.winehq.org/wine-builds/Release.key ; sudo apt-key add Release.key ; sudo apt-add-repository 'https://dl.winehq.org/wine-builds/ubuntu/'
     - sudo apt-get update; sudo apt-get install bc libfuse-dev libarchive-dev xfsprogs valgrind gcc-mingw-w64-i686 wine qemu-user-static linux-headers-$(uname -r) ccache
-    - /usr/local/android-ndk/build/tools/make-standalone-toolchain.sh --platform=android-21 --install-dir=/home/ubuntu/android-toolchain --arch=arm
+    - wget -q --output-document=android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip && unzip android-ndk.zip
+    - if [ $CIRCLE_NODE_INDEX -eq 2 ] ; then ./android-ndk-r15b/build/tools/make_standalone_toolchain.py --arch arm --api 24 --install-dir /home/ubuntu/android-toolchain ; fi:
+        parallel: true
+    - if [ $CIRCLE_NODE_INDEX -eq 3 ] ; then ./android-ndk-r15b/build/tools/make_standalone_toolchain.py --arch arm64 --api 24 --install-dir /home/ubuntu/android-toolchain ; fi:
+        parallel: true
     - sudo cp tools/lkl/bin/i686-w64-mingw32-* /usr/bin:
         parallel: true
+    # for aarch64 instance
+    - if [ $CIRCLE_NODE_INDEX -eq 3 ] ; then cd $HOME ; wget https://android-git.linaro.org/platform/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-5.4-linaro.git/snapshot/aarch64-linux-android-5.4-linaro-2016.06.tar.gz ; tar xfz aarch64-linux-android-5.4-linaro-2016.06.tar.gz ; cd aarch64-linux-android-5.4-linaro-2016.06 ; ln -s /home/ubuntu/android-toolchain/sysroot ; fi:
+        parallel: true
     - git fetch --tags git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
   override:
     - mkdir -p ~/.ccache
@@ -36,9 +46,13 @@ test:
   pre:
     - tools/lkl/scripts/checkpatch.sh
   override:
-    - cd tools/lkl && if [ $CIRCLE_NODE_INDEX -eq 2 ] ; then make tests/boot-in.o; arm-linux-androideabi-gcc -o tests/boot tests/boot-in.o liblkl.a -static ; fi:
+    - if [ $CIRCLE_NODE_INDEX -eq 2 -o $CIRCLE_NODE_INDEX -eq 3 ] ; then emulator -avd circleci-android24 -no-window ; fi:
+        background: true
+        parallel: true
+    - if [ $CIRCLE_NODE_INDEX -eq 2 -o $CIRCLE_NODE_INDEX -eq 3 ] ; then circle-android wait-for-boot ; fi:
         parallel: true
-    - cd tools/lkl && make test:
+    # can't exec qemu-aarch64 emulator on circleci ...
+    - if [ $CIRCLE_NODE_INDEX -ne 3 ] ; then cd tools/lkl && make test ; fi:
          parallel: true
 
     - ? >

tools/lkl/Makefile

@@ -39,10 +39,11 @@ export srctree
 export CFLAGS += -I$(OUTPUT)/include -Iinclude -Wall -g -O2 -Wextra \
 	 -Wno-unused-parameter \
 	 -Wno-missing-field-initializers -fno-strict-aliasing
+LDFLAGS += -pie
 
 OUTPUT_FORMAT = $(shell $(LD) -r -print-output-format)
 
-ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf32-i386 elf64-x86-64-freebsd elf32-littlearm))
+ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf32-i386 elf64-x86-64-freebsd elf32-littlearm elf64-littleaarch64))
   OUTPUT_DEF = $(shell echo | $(CC) -dM -E -)
   CFLAGS   += -fPIC -pthread
   ifeq (,$(filter $(OUTPUT_DEF),__ANDROID__))
@@ -112,6 +113,9 @@ $(OUTPUT)liblkl$(SOSUF): LDFLAGS += -shared
 $(OUTPUT)liblkl-hijack$(SOSUF): $(OUTPUT)lib/hijack/hijack-in.o $(OUTPUT)liblkl.a
 $(OUTPUT)liblkl-hijack$(SOSUF): LDFLAGS += -shared -nodefaultlibs
 $(OUTPUT)liblkl-hijack$(SOSUF): LDLIBS += -ldl
+ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
+  $(OUTPUT)liblkl-hijack$(SOSUF): LDLIBS += -lgcc -lc
+endif
 
 $(OUTPUT)lklfuse$(EXESUF): $(OUTPUT)lklfuse-in.o $(OUTPUT)liblkl.a
 $(OUTPUT)lklfuse$(EXESUF): LDLIBS += -lfuse
@@ -131,12 +135,15 @@ endif
 TEST_TARGETS        := test valgrind gdb
 
 $(OUTPUT)tests/boot: $(OUTPUT)tests/boot-in.o $(OUTPUT)liblkl.a
+ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
+  $(OUTPUT)tests/boot: LDLIBS += -lc
+endif
 $(OUTPUT)tests/net-test: $(OUTPUT)tests/net-test-in.o $(OUTPUT)liblkl.a
 $(TEST_TARGETS): $(OUTPUT)tests/boot $(OUTPUT)tests/net-test
 
 # because of libdl, liblkl-hijack will not compile on windows
 # fortunately, the test target will handle a missing libhijack.so correctly
-ifeq (,$(filter $(OUTPUT_FORMAT),pe-i386 elf32-littlearm))
+ifeq (,$(filter $(OUTPUT_FORMAT),pe-i386))
   test: liblkl-hijack$(SOSUF)
 endif
 
@@ -145,7 +152,9 @@ $(OUTPUT)%-in.o: $(OUTPUT)lib/lkl.o FORCE
 
 $(OUTPUT)lib/lkl.o:
 	$(Q)$(MAKE) -C ../.. ARCH=lkl $(KOPT) defconfig
-	$(Q)$(MAKE) -C ../.. ARCH=lkl $(KOPT) install INSTALL_PATH=$(OUTPUT)
+# this workaround is for arm32 linker (ld.gold)
+	$(Q)export PATH=$(srctree)/tools/lkl/bin/:${PATH} ;\
+	$(MAKE) -C ../.. ARCH=lkl $(KOPT) install INSTALL_PATH=$(OUTPUT)
 
 $(OUTPUT)liblkl.a: $(OUTPUT)lib/lkl-in.o $(OUTPUT)lib/lkl.o
 	$(QUIET_AR)$(AR) -rc $@ $^
@@ -162,9 +171,7 @@ $(OUTPUT)cpfromfs$(EXESUF): cptofs$(EXESUF)
 ifneq (,$(filter $(OUTPUT_FORMAT),pe-i386))
   all: $(filter-out $(OUTPUT)liblkl-hijack$(SOSUF), $(ALL_LIBRARIES))
 else ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
-  all: $(filter-out $(OUTPUT)liblkl-hijack$(SOSUF), $(ALL_LIBRARIES))
-else ifneq (,$(filter $(OUTPUT_FORMAT),elf32-littlearm))
-  all: $(filter-out $(OUTPUT)liblkl-hijack$(SOSUF), $(ALL_LIBRARIES)) $(ALL_PROGRAMS)
+  all: $(ALL_LIBRARIES)
 else
   all: $(ALL_PROGRAMS) $(ALL_LIBRARIES)
 endif

tools/lkl/bin/arm-linux-androideabi-ld

@@ -0,0 +1 @@
+arm-linux-androideabi-ld.gold

tools/lkl/include/lkl.h

@@ -19,6 +19,10 @@ extern "C" {
 #define strtok_r strtok_s
 #endif
 
+#if defined(__ANDROID__) && __LKL__BITS_PER_LONG == 32
+#define __lkl__NR_fcntl __lkl__NR_fcntl64
+#endif
+
 #if __LKL__BITS_PER_LONG == 64
 #define lkl_sys_stat lkl_sys_newstat
 #define lkl_sys_lstat lkl_sys_newlstat

tools/lkl/lib/Build

@@ -1,5 +1,5 @@
 CFLAGS_posix-host.o += -D_FILE_OFFSET_BITS=64
-CFLAGS_virtio_net_vde.o += $(shell pkg-config --cflags vdeplug)
+CFLAGS_virtio_net_vde.o += $(pkg-config --cflags vdeplug 2>/dev/null)
 CFLAGS_nt-host.o += -D_WIN32_WINNT=0x0600
 
 lkl-y += fs.o
@@ -18,5 +18,5 @@ lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_fd.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_tap.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_raw.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_macvtap.o
-lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET_DPDK) += virtio_net_dpdk.o
-lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET_VDE) += virtio_net_vde.o
+lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_dpdk.o
+lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_vde.o

tools/lkl/lib/endian.h

@@ -5,9 +5,6 @@
 #include <sys/endian.h>
 #elif defined(__ANDROID__)
 #include <sys/endian.h>
-#define le16toh(x) letoh16(x)
-#define le32toh(x) letoh32(x)
-#define le64toh(x) letoh64(x)
 #elif defined(__MINGW32__)
 #include <winsock.h>
 #define le32toh(x) (x)

tools/lkl/lib/hijack/hijack.c

@@ -201,11 +201,18 @@ int socket(int domain, int type, int protocol)
 	if (domain == AF_UNIX || domain == PF_PACKET)
 		return host_socket(domain, type, protocol);
 
+	if (!lkl_running)
+		return host_socket(domain, type, protocol);
+
 	return lkl_call(__lkl__NR_socket, 3, domain, type, protocol);
 }
 
 HOST_CALL(ioctl);
+#ifdef __ANDROID__
+int ioctl(int fd, int req, ...)
+#else
 int ioctl(int fd, unsigned long req, ...)
+#endif
 {
 	va_list vl;
 	long arg;
@@ -367,9 +374,11 @@ void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset)
 	return lkl_sys_mmap(addr, length, prot, flags, fd, offset);
 }
 
+#ifndef __ANDROID__
 HOST_CALL(__xstat64)
 int stat(const char *pathname, struct stat *buf)
 {
 	CHECK_HOST_CALL(__xstat64);
 	return host___xstat64(0, pathname, buf);
 }
+#endif

tools/lkl/lib/hijack/init.c

@@ -17,9 +17,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
-#include <sys/stat.h>
 #include <fcntl.h>
 #include <errno.h>
+#include <signal.h>
 #include <sys/ioctl.h>
 #include <lkl.h>
 #include <lkl_host.h>
@@ -329,6 +329,17 @@ hijack_init(void)
 	if (single_cpu_mode == 1)
 		PinToFirstCpu(&ori_cpu);
 
+#ifdef __ANDROID__
+	struct sigaction sa;
+
+	sa.sa_handler = SIG_IGN;
+	sa.sa_flags = 0;
+	if (sigaction(32, &sa, 0) == -1) {
+		perror("sigaction");
+		exit(1);
+	}
+#endif
+
 	ret = lkl_start_kernel(&lkl_host_ops, boot_cmdline);
 	if (ret) {
 		fprintf(stderr, "can't start kernel: %s\n", lkl_strerror(ret));

tools/lkl/lib/virtio_net_vde.c

@@ -1,6 +1,5 @@
-#ifdef CONFIG_AUTO_LKL_VIRTIO_NET_VDE
-
 #include <stdio.h>
+#ifdef CONFIG_AUTO_LKL_VIRTIO_NET_VDE
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>

tools/lkl/tests/Makefile

@@ -43,11 +43,21 @@ else
 	VALGRIND_CMD=valgrind_test
 endif
 
+ifneq ($(LKL_ANDROID_TEST),1)
 test:
 	$(call run,)
 	$(call lklfuse)
 	$(HIJACK_TEST)
 	./net.sh
+else
+test:
+	dd if=/dev/zero of=disk.img bs=1024 count=102400
+	yes | mkfs.ext4 disk.img >/dev/null
+	adb shell mkdir -p /data/local/tmp/lkl/bin
+	adb push ../bin/lkl-hijack.sh /data/local/tmp/lkl/bin
+	adb push  ../tests ../liblkl-hijack.so /data/local/tmp/lkl/
+	adb shell CROSS_COMPILE=${CROSS_COMPILE} /data/local/tmp/lkl/tests/android-test.sh
+endif
 
 valgrind:
 	$(call for_fs,$(VALGRIND_CMD))

tools/lkl/tests/android-test.sh

@@ -0,0 +1,16 @@
+#!/system/bin/sh
+
+set -e
+
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+export PATH=${script_dir}:${PATH}
+export LKL_ANDROID_TEST=1
+export TMPDIR=/data/local/tmp
+export CONFIG_AUTO_LKL_POSIX_HOST=y
+
+sed -i "s/\/bin\/bash/\/system\/bin\/sh /" ${script_dir}/../bin/lkl-hijack.sh
+
+cd ${script_dir}
+boot -d ${script_dir}/disk.img  -t ext4
+sh -x ${script_dir}/hijack-test.sh
+sh ${script_dir}/net.sh

tools/lkl/tests/hijack-test.sh

@@ -34,12 +34,12 @@ ${hijack_script} ip route
 
 echo "== ping test=="
 cp `which ping` .
-${hijack_script} ./ping 127.0.0.1 -c 1
+${hijack_script} ./ping -c 1 127.0.0.1
 rm ./ping
 
 echo "== ping6 test=="
 cp `which ping6` .
-${hijack_script} ./ping6 ::1 -c 1
+${hijack_script} ./ping6 -c 1 ::1
 rm ./ping6
 
 echo "== Mount/dump tests =="
@@ -50,18 +50,27 @@ ans=$(LKL_HIJACK_MOUNT=proc,sysfs\
   ${hijack_script} ip -h) || true
 # Need to grab the end because something earlier on prints out this
 # number
+# XXX: android-23 doesn't call destructor...
+if [ -z ${LKL_ANDROID_TEST} ] ; then
 echo "$ans" | tail -n 15 | grep "65536" # lo's MTU
 # lo's dev id
 echo "$ans" | grep "0x0"        # lo's dev_id
 # Doesn't really belong in this section, but might as well check for
 # it here.
 ! echo "$ans" | grep "WARN: failed to free"
+fi
 
 # boot_cmdline test
 echo "== boot command line tests =="
 ans=$(LKL_HIJACK_DEBUG=1\
   LKL_HIJACK_BOOT_CMDLINE="mem=100M" ${hijack_script} ip ad)
+if [ -z ${CROSS_COMPILE} ] ; then
 echo "$ans" | grep "100752k"
+elif [ "${CROSS_COMPILE}" = "arm-linux-androideabi-" ] ; then
+echo "$ans" | grep "101424k"
+elif [ "${CROSS_COMPILE}" = "aarch64-linux-android-" ] ; then
+echo "$ans" | grep "100756k"
+fi
 
 echo "== TAP tests =="
 if [ ! -c /dev/net/tun ]; then

tools/lkl/tests/net-test.c

@@ -11,6 +11,9 @@
 #include <arpa/inet.h>
 #include <netinet/ip.h>
 #include <netinet/ip_icmp.h>
+#ifdef __ANDROID__
+#include <linux/icmp.h>
+#endif
 
 #include <lkl.h>
 #include <lkl_host.h>
@@ -135,10 +138,8 @@ static int test_net_init(int argc, char **argv)
 
 	if (iftype && ifname && (strncmp(iftype, "tap", 3) == 0))
 		nd = lkl_netdev_tap_create(ifname, 0);
-#ifdef CONFIG_AUTO_LKL_VIRTIO_NET_DPDK
 	else if (iftype && ifname && (strncmp(iftype, "dpdk", 4) == 0))
 		nd = lkl_netdev_dpdk_create(ifname, 0, NULL);
-#endif /* CONFIG_AUTO_LKL_VIRTIO_NET_DPDK */
 	else if (iftype && ifname && (strncmp(iftype, "raw", 3) == 0))
 		nd = lkl_netdev_raw_create(ifname);
 	else if (iftype && ifname && (strncmp(iftype, "macvtap", 7) == 0))