Merge branch 'main' into aarch64_aes_gcm_htable_opt

aws · Mar 11, 2024 · 5f901da · 5f901da
2 parents 5ebae3b + 37708f5
commit 5f901da
Show file tree

Hide file tree

Showing 437 changed files with 16,719 additions and 7,903 deletions.
diff --git a/.github/workflows/actions-ci.yml b/.github/workflows/actions-ci.yml
@@ -9,8 +9,8 @@ concurrency:
 
 env:
   GOPROXY: https://proxy.golang.org,direct
-  SDE_MIRROR_URL: "https://downloadmirror.intel.com/777395/sde-external-9.21.1-2023-04-24-win.tar.xz"
-  SDE_VERSION_TAG: sde-external-9.21.1-2023-04-24-win
+  SDE_MIRROR_URL: "https://downloadmirror.intel.com/813591/sde-external-9.33.0-2024-01-07-win.tar.xz"
+  SDE_VERSION_TAG: sde-external-9.33.0-2024-01-07-win
   PACKAGE_NAME: aws-lc
   # Used to enable ASAN test dimension.
   AWSLC_NO_ASM_FIPS: 1
@@ -129,30 +129,31 @@ jobs:
           echo ${env:SDEROOT}
           .\tests\ci\run_windows_tests.bat "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 true
 
-  MSVC-SDE-32-bit:
-    needs: [sanity-test-run]
-    runs-on: aws-lc_windows-2019_64-core
-    steps:
-      - name: Git clone the repository
-        uses: actions/checkout@v3
-
-      - name: Build Windows Dependencies
-        run: |
-          choco install ninja --version 1.9.0.20190208 -y &&
-          choco install nasm --version 2.14.02 -y
-
-      - name: Install SDE simulator
-        run: |
-          curl -SL --output temp.tar.xz ${{ env.SDE_MIRROR_URL }}
-          7z x temp.tar.xz
-          7z x temp.tar
-          ren ${{ env.SDE_VERSION_TAG }} windows-sde
-          del temp.tar.xz
-          del temp.tar
-
-      - name: Run Windows SDE Tests for 32 bit
-        run: |
-          $env:SDEROOT = "${PWD}\windows-sde"
-          echo ${env:SDEROOT}
-          .\tests\ci\run_windows_tests.bat "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 true
-
+    # TODO: Investigate sudden hanging tests and failures in GHA runners (P114059413)
+#  MSVC-SDE-32-bit:
+#    needs: [sanity-test-run]
+#    runs-on: aws-lc_windows-2019_64-core
+#    steps:
+#      - name: Git clone the repository
+#        uses: actions/checkout@v3
+#
+#      - name: Build Windows Dependencies
+#        run: |
+#          choco install ninja --version 1.9.0.20190208 -y &&
+#          choco install nasm --version 2.14.02 -y
+#
+#      - name: Install SDE simulator
+#        run: |
+#          curl -SL --output temp.tar.xz ${{ env.SDE_MIRROR_URL }}
+#          7z x temp.tar.xz
+#          7z x temp.tar
+#          ren ${{ env.SDE_VERSION_TAG }} windows-sde
+#          del temp.tar.xz
+#          del temp.tar
+#
+#      - name: Run Windows SDE Tests for 32 bit
+#        run: |
+#          $env:SDEROOT = "${PWD}\windows-sde"
+#          echo ${env:SDEROOT}
+#          .\tests\ci\run_windows_tests.bat "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 true
+#
diff --git a/.github/workflows/aws-lc-rs.yml b/.github/workflows/aws-lc-rs.yml
@@ -1,4 +1,4 @@
-name: aws-lc-rs sanity tests
+name: aws-lc-rs tests
 on:
   push:
     branches: [ '*' ]
@@ -13,7 +13,6 @@ jobs:
   standard:
     runs-on: ubuntu-latest
     steps:
-
       - uses: actions/checkout@v3
         with:
           repository: awslabs/aws-lc-rs
@@ -67,3 +66,8 @@ jobs:
         working-directory: ./aws-lc-rs/aws-lc-rs
         run: |
           cargo test
+      - name: Install cross
+        run: cargo install cross --git https://github.com/cross-rs/cross
+      - name: Cross-compile arm-linux-androideabi
+        working-directory: ./aws-lc-rs/aws-lc-rs
+        run: cross test --release --features bindgen,unstable --target arm-linux-androideabi
diff --git a/.github/workflows/cross-test.yml b/.github/workflows/cross-test.yml
@@ -48,3 +48,60 @@ jobs:
       - uses: actions/checkout@v4
       - name: PPC64LE Build/Test
         run: tests/ci/run_cross_tests.sh ppc64le powerpc64le-unknown-linux-gnu "-DCMAKE_BUILD_TYPE=Release" "-DCMAKE_BUILD_TYPE=Release -DFIPS=1 -DBUILD_SHARED_LIBS=1"
+  riscv64-non-fips-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install qemu
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install qemu-user qemu-user-binfmt
+      - uses: actions/checkout@v4
+      - name: RISC-V 64 Build/Test
+# The flag below is set to avoid the following error with GCC 11.4.0:
+#
+#  /home/runner/work/aws-lc/aws-lc/crypto/pem/pem_lib.c:705:11: error: 'strncmp' of strings of length 1 and 9 and bound of 9 evaluates to nonzero [-Werror=string-compare]
+#  705 |       if (strncmp(buf, "-----END ", 9) == 0) {
+#      |           ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        env:
+          CFLAGS: "-Wno-string-compare"
+        run: tests/ci/run_cross_tests.sh riscv riscv64-unknown-linux-gnu "-DCMAKE_BUILD_TYPE=Release"
+  armv6-non-fips-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install qemu
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install qemu-user qemu-user-binfmt
+      - uses: actions/checkout@v4
+      - name: armv6 Build/Test
+        run: tests/ci/run_cross_tests.sh armv6 armv6-unknown-linux-gnueabi "-DCMAKE_BUILD_TYPE=Release"
+#  TODO: enable once qemu-user-binfmt in ubuntu-latest supports loongarch64
+#        * QEMU added support for loongarch64 in 7.1: https://www.qemu.org/2022/08/30/qemu-7-1-0/
+#        * The next Ubuntu LTS should have a QEMU 8.x: https://packages.ubuntu.com/noble/qemu-user-binfmt
+#  loongarch64-non-fips-build-test:
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Install qemu
+#        run: |
+#          sudo apt-get update
+#          sudo apt-get -y install qemu-user qemu-user-binfmt
+#      - uses: actions/checkout@v4
+#      - name: armv6 Build/Test
+#        run: tests/ci/run_cross_tests.sh loongarch64 loongarch64-unknown-linux-gnu "-DCMAKE_BUILD_TYPE=Release"
+  s390x-non-fips-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install qemu
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install qemu-user qemu-user-binfmt
+      - uses: actions/checkout@v4
+      - name: s390x Build/Test
+# The flag below is set to avoid the following error with GCC 11.4.0:
+#
+#  /home/runner/work/aws-lc/aws-lc/crypto/pem/pem_lib.c:705:11: error: 'strncmp' of strings of length 1 and 9 and bound of 9 evaluates to nonzero [-Werror=string-compare]
+#  705 |       if (strncmp(buf, "-----END ", 9) == 0) {
+#      |           ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        env:
+          CFLAGS: "-Wno-string-compare"
+        run: tests/ci/run_cross_tests.sh s390x s390x-ibm-linux-gnu "-DCMAKE_BUILD_TYPE=Release"
diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml
@@ -21,6 +21,16 @@ jobs:
       - name: Run integration build
         run: |
           ./tests/ci/integration/run_haproxy_integration.sh
+  tpm2-tss:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update && sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev uuid-dev libltdl-dev libusb-1.0-0-dev libftdi-dev
+      - uses: actions/checkout@v3
+      - name: Run integration build
+        run: |
+          ./tests/ci/integration/run_tpm2_tss_integration.sh
   grpc:
     env:
       DEBIAN_FRONTEND: noninteractive
@@ -71,3 +81,59 @@ jobs:
       - name: Run ntp build
         run: |
           ./tests/ci/integration/run_ntp_integration.sh
+  socat:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update && sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang make autoconf pkg-config openssl
+      - uses: actions/checkout@v3
+      - name: Run integration build
+        run: |
+          ./tests/ci/integration/run_socat_integration.sh
+  python-main:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang make
+      - uses: actions/checkout@v3
+      - name: Build AWS-LC, build python, run tests
+        run: |
+          ./tests/ci/integration/run_python_integration.sh main
+  python-releases:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang make
+      - uses: actions/checkout@v3
+      - name: Build AWS-LC, build python, run tests
+        run: |
+          ./tests/ci/integration/run_python_integration.sh 3.10 3.11 3.12
+  bind9:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang make python3 python3-pytest autoconf pkg-config libcmocka-dev liburcu-dev libuv1-dev libnghttp2-dev libcap-dev libprotobuf-c-dev protobuf-c-compiler libfstrm-dev libjemalloc-dev
+      - uses: actions/checkout@v3
+      - name: Run bind9 build
+        run: |
+          ./tests/ci/integration/run_bind9_integration.sh
+  strongswan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y --no-install-recommends install \
+          cmake gcc ninja-build golang make gperf bison flex autogen autoconf \
+          pkg-config libtool gettext libgmp-dev libsystemd-dev
+      - uses: actions/checkout@v4
+      - name: Run strongswan build
+        run: |
+          ./tests/ci/integration/run_strongswan_integration.sh
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ build/
 build32/
 build64/
 build-fips/
+*_BUILD_ROOT/
 ssl/test/runner/runner
 *.pyc
 *.swp

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -47,6 +47,17 @@ if(BUILD_TESTING OR BUILD_LIBSSL)
   enable_language(CXX)
 endif()
 
+if(CMAKE_VERSION VERSION_GREATER "3.1.0")
+  if (NOT DEFINED CMAKE_CXX_STANDARD)
+    set(CMAKE_CXX_STANDARD 11)
+  endif()
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+  if (NOT DEFINED CMAKE_C_STANDARD)
+    set(CMAKE_C_STANDARD 99)
+  endif ()
+  set(CMAKE_C_STANDARD_REQUIRED ON)
+endif ()
+
 if(CMAKE_C_COMPILER_ID MATCHES "Clang")
   set(CLANG 1)
 elseif(CMAKE_C_COMPILER_ID MATCHES "GNU")
@@ -315,7 +326,9 @@ endif ()
 if(GCC OR CLANG)
   # Note clang-cl is odd and sets both CLANG and MSVC. We base our configuration
   # primarily on our normal Clang one.
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  if (NOT CMAKE_C_STANDARD)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  endif ()
 
   # TODO(CryptoAlg-759): enable '-Wpedantic' if awslc has to follow c99 spec.
   if(CLANG OR (GCC AND CMAKE_C_COMPILER_VERSION VERSION_GREATER "4.1.3"))
@@ -384,7 +397,9 @@ if(GCC OR CLANG)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${C_CXX_FLAGS} -Wmissing-declarations")
 
   if(NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    if (NOT CMAKE_CXX_STANDARD)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    endif ()
     if(APPLE)
       set(CMAKE_MACOSX_RPATH 1)
     endif()
@@ -749,6 +764,8 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "powerpc64le|ppc64le")
   set(ARCH "ppc64le")
 elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "riscv64")
   set(ARCH "riscv64")
+elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "s390x")
+  set(ARCH "s390x")
 elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "loongarch64")
   set(ARCH "loongarch64")
 else()

diff --git a/PORTING.md b/PORTING.md
@@ -1,40 +1,21 @@
-# Porting from OpenSSL to BoringSSL
-
-BoringSSL is an OpenSSL derivative and is mostly source-compatible, for the
-subset of OpenSSL retained. Libraries ideally need little to no changes for
-BoringSSL support, provided they do not use removed APIs. In general, see if the
-library compiles and, on failure, consult the documentation in the header files
-and see if problematic features can be removed.
-
-BoringSSL's `OPENSSL_VERSION_NUMBER` matches the OpenSSL version it targets.
-Version checks for OpenSSL should ideally work as-is in BoringSSL. BoringSSL
-also defines upstream's `OPENSSL_NO_*` feature macros corresponding to removed
-features. If the preprocessor is needed, use these version checks or feature
-macros where possible, especially when patching third-party projects. Such
-patches are more generally useful to OpenSSL consumers and thus more
-appropriate to send upstream.
-
-In some cases, BoringSSL-specific code may be necessary. Use the
-`OPENSSL_IS_BORINGSSL` preprocessor macro in `#ifdef`s. However, first contact
-the BoringSSL maintainers about the missing APIs. We will typically add
-compatibility functions for convenience. In particular, *contact BoringSSL
-maintainers before working around missing OpenSSL 1.1.0 accessors*. BoringSSL
-was originally derived from OpenSSL 1.0.2 but now targets OpenSSL 1.1.0. Some
-newer APIs may be missing but can be added on request. (Not all projects have
-been ported to OpenSSL 1.1.0, so BoringSSL also remains largely compatible with
-OpenSSL 1.0.2.)
-
-The `OPENSSL_IS_BORINGSSL` macro may also be used to distinguish OpenSSL from
-BoringSSL in configure scripts. Do not use the presence or absence of particular
-symbols to detect BoringSSL.
-
-Note: BoringSSL does *not* have a stable API or ABI. It must be updated with its
-consumers. It is not suitable for, say, a system library in a traditional Linux
-distribution. For instance, Chromium statically links the specific revision of
-BoringSSL it was built against. Likewise, Android's system-internal copy of
-BoringSSL is not exposed by the NDK and must not be used by third-party
-applications.
+# Introduction
 
+AWS-LC is a fork of BoringSSL, which is a derivative of OpenSSL. AWS-LC aims to be API compatible with the `OPENSSL_VERSION_NUMBER` defined in [`openssl/base.h`](https://github.com/aws/aws-lc/blob/main/include/openssl/base.h) (currently OpenSSL 1.1.1). Version checks against the macro should ideally work as-is with AWS-LC. AWS-LC defines the corresponding `OPENSSL_NO_*` feature macros corresponding to removed features.
+
+There may be missing APIs or macros and subtle behavioral differences when migrating to AWS-LC from OpenSSL. OpenSSL may have underlying behavioral conventions that aren't standardized and there's no guarantee that these will be consistent within AWS-LC. In general, see if the AWS-LC compiles and runs against the tests with your application, consult the documentation available in [the header files](https://github.com/aws/aws-lc/tree/main/include/openssl), and check if problematic features can be removed if possible. **If not, feel free to [**contact us**](https://github.com/aws/aws-lc/issues/new?assignees=&labels=&projects=&template=general-issue.md&title=) about adding new features or missing symbols, we will typically add compatibility for convenience.**
+
+In rarer instances, AWS-LC-specific code may be necessary. The `OPENSSL_IS_AWSLC` preprocessor macro can be used in `#ifdef`s and configure scripts to distinguish OpenSSL from AWS-LC. Please do not use the presence or absence of particular symbols to detect AWS-LC. AWS-LC is commited to having a stable API, but is not ABI stable. Systems cannot directly swap out OpenSSL with AWS-LC without recompiling. This makes it not suitable as a system library in a traditional Linux distribution.
+
+Despite supporting certain OpenSSL APIs, AWS-LC will not behave exactly the same in regards to non-standardized lower level details. Function signatures and parameters generally remain the same across both libraries. AWS-LC attempts to make cryptography and ssl less configurable and hard to misuse. AWS-LC has diverged since the initial fork off of OpenSSL 1.0.2 and does not implement every feature OpenSSL has today. This porting guide compiles a list of known differences and fall into 4 main categories. More details on each difference and the justifications will be outlined in each section.
+
+1. [Preexisting BoringSSL Changes](#preexisting-boringssl-changes)
+2. [No-op Symbols and Configurations](docs/porting/functionality-differences.md) 
+3. [Differences in Configuration Defaults](docs/porting/configuration-differences.md)
+4. Functional Differences (WIP)
+
+# Preexisting BoringSSL Changes
+
+The following callouts are remnants of BoringSSL's original porting guide. 
 
 ## Major API changes
 

diff --git a/PORTING_TO_AWSLC.md b/PORTING_TO_AWSLC.md