diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0aae5516aeb..760d99c55a8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,7 +89,6 @@ jobs: redis-cli -h 127.0.0.1 -p 5000 cluster nodes - name: Running etcd server with TLS - if: startsWith(matrix.os_name, 'linux_openresty') run: | sudo docker run -d -p 12379:12379 -p 12380:12380 \ -e ALLOW_NONE_AUTHENTICATION=yes \ diff --git a/.travis/apisix_cli_test/test_etcd_tls.sh b/.travis/apisix_cli_test/test_etcd_tls.sh new file mode 100755 index 000000000000..8bde2533b835 --- /dev/null +++ b/.travis/apisix_cli_test/test_etcd_tls.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# 'make init' operates scripts and related configuration files in the current directory +# The 'apisix' command is a command in the /usr/local/apisix, +# and the configuration file for the operation is in the /usr/local/apisix/conf + +. ./.travis/apisix_cli_test/common.sh + +# Check etcd tls verify failure +git checkout conf/config.yaml + +echo ' +etcd: + host: + - "https://127.0.0.1:12379" + prefix: "/apisix" + ' > conf/config.yaml + +out=$(make init 2>&1 || true) +if ! echo "$out" | grep "certificate verify failed"; then + echo "failed: apisix should echo \"certificate verify failed\"" + exit 1 +fi + +echo "passed: Show certificate verify failed info successfully" + + +# Check etcd tls without verification +git checkout conf/config.yaml + +echo ' +etcd: + host: + - "https://127.0.0.1:12379" + tls: + verify: false + prefix: "/apisix" + ' > conf/config.yaml + +out=$(make init 2>&1 || true) +if echo "$out" | grep "certificate verify failed"; then + echo "failed: apisix should not echo \"certificate verify failed\"" + exit 1 +fi + +echo "passed: Certificate verification successfully" diff --git a/.travis/apisix_cli_test/test_main.sh b/.travis/apisix_cli_test/test_main.sh index c2f950e14b94..49779fce921a 100755 --- a/.travis/apisix_cli_test/test_main.sh +++ b/.travis/apisix_cli_test/test_main.sh @@ -997,7 +997,7 @@ fi echo "passed: Show connection refused info successfully" -# check etcd auth error +# Check etcd auth error git checkout conf/config.yaml export ETCDCTL_API=3 diff --git a/.travis/linux_apisix_current_luarocks_runner.sh b/.travis/linux_apisix_current_luarocks_runner.sh index 70d2866ba362..2f8796f8de38 100755 --- a/.travis/linux_apisix_current_luarocks_runner.sh +++ b/.travis/linux_apisix_current_luarocks_runner.sh @@ -20,7 +20,7 @@ do_install() { ./utils/linux-install-openresty.sh - ./utils/linux-install-luarocks.sh + OR_PREFIX=/usr/local/openresty-debug ./utils/linux-install-luarocks.sh ./utils/linux-install-etcd-client.sh } diff --git a/.travis/linux_tengine_runner.sh b/.travis/linux_tengine_runner.sh index fb0004a675b5..35dd29072d97 100755 --- a/.travis/linux_tengine_runner.sh +++ b/.travis/linux_tengine_runner.sh @@ -195,6 +195,13 @@ tengine_install() { cp -r ${OPENRESTY_PREFIX}/* build-cache${OPENRESTY_PREFIX} ls build-cache${OPENRESTY_PREFIX} rm -rf openresty-${OPENRESTY_VERSION} + + wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add - + sudo apt-get -y update --fix-missing + sudo apt-get -y install software-properties-common + sudo add-apt-repository -y "deb https://openresty.org/package/ubuntu $(lsb_release -sc) main" + sudo apt-get update + sudo apt-get install openresty-openssl-debug-dev } do_install() { diff --git a/Makefile b/Makefile index b61c24d4744e..834de6e14be0 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ INSTALL ?= install UNAME ?= $(shell uname) OR_EXEC ?= $(shell which openresty || which nginx) LUAROCKS_VER ?= $(shell luarocks --version | grep -E -o "luarocks [0-9]+.") +OR_PREFIX ?= $(shell $(OR_EXEC) -V 2>&1 | grep -Eo 'prefix=(.*)/nginx\s+' | grep -Eo '/.*/') SHELL := /bin/bash -o pipefail @@ -54,8 +55,16 @@ help: default .PHONY: deps deps: default ifeq ($(LUAROCKS_VER),luarocks 3.) - luarocks install --lua-dir=$(LUAJIT_DIR) rockspec/apisix-master-0.rockspec --tree=deps --only-deps --local + mkdir ~/.luarocks || true + luarocks config variables.OPENSSL_LIBDIR $(addprefix $(OR_PREFIX), openssl/lib) + luarocks config variables.OPENSSL_INCDIR $(addprefix $(OR_PREFIX), openssl/include) + luarocks install rockspec/apisix-master-0.rockspec --tree=deps --only-deps --local else + @echo "WARN: You're not using LuaRocks 3.x, please add the following items to your LuaRocks config file:" + @echo "variables = {" + @echo " OPENSSL_LIBDIR=$(addprefix $(OR_PREFIX), openssl/lib)" + @echo " OPENSSL_INCDIR=$(addprefix $(OR_PREFIX), openssl/include)" + @echo "}" luarocks install rockspec/apisix-master-0.rockspec --tree=deps --only-deps --local endif diff --git a/apisix/cli/etcd.lua b/apisix/cli/etcd.lua index 8ce6076cc20e..cf518bc362a0 100644 --- a/apisix/cli/etcd.lua +++ b/apisix/cli/etcd.lua @@ -20,6 +20,7 @@ local dkjson = require("dkjson") local util = require("apisix.cli.util") local file = require("apisix.cli.file") local http = require("socket.http") +local https = require("ssl.https") local ltn12 = require("ltn12") local type = type @@ -27,6 +28,7 @@ local ipairs = ipairs local print = print local tonumber = tonumber local str_format = string.format +local str_sub = string.sub local table_concat = table.concat local _M = {} @@ -90,6 +92,44 @@ local function compare_semantic_version(v1, v2) end +local function request(url, yaml_conf) + local response_body = {} + local single_request = false + if type(url) == "string" then + url = { + url = url, + method = "GET", + sink = ltn12.sink.table(response_body), + } + single_request = true + end + + local res, code + + if str_sub(url.url, 1, 8) == "https://" then + local verify = "peer" + if yaml_conf.etcd.tls and yaml_conf.etcd.tls.verify == false then + verify = "none" + end + + url.verify = verify + res, code = https.request(url) + else + + res, code = http.request(url) + end + + -- In case of failure, request returns nil followed by an error message. + -- Else the first return value is the response body + -- and followed by the response status code. + if single_request and res ~= nil then + return table_concat(response_body), code + end + + return res, code +end + + function _M.init(env) -- read_yaml_conf local yaml_conf, err = file.read_yaml_conf(env.apisix_home) @@ -137,7 +177,7 @@ function _M.init(env) local version_url = host .. "/version" local errmsg - local res, err = http.request(version_url) + local res, err = request(version_url, yaml_conf) -- In case of failure, request returns nil followed by an error message. -- Else the first return value is the response body -- and followed by the response status code. @@ -179,10 +219,15 @@ function _M.init(env) local post_json_auth = dkjson.encode(json_auth) local response_body = {} - local res, err = http.request{url = auth_url, method = "POST", - source = ltn12.source.string(post_json_auth), - sink = ltn12.sink.table(response_body), - headers = {["Content-Length"] = #post_json_auth}} + local res, err = request({ + url = auth_url, + method = "POST", + source = ltn12.source.string(post_json_auth), + sink = ltn12.sink.table(response_body), + headers = { + ["Content-Length"] = #post_json_auth + } + }, yaml_conf) -- In case of failure, request returns nil followed by an error message. -- Else the first return value is just the number 1 -- and followed by the response status code. @@ -219,10 +264,13 @@ function _M.init(env) headers["Authorization"] = auth_token end - local res, err = http.request{url = put_url, method = "POST", - source = ltn12.source.string(post_json), - sink = ltn12.sink.table(response_body), - headers = headers} + local res, err = request({ + url = put_url, + method = "POST", + source = ltn12.source.string(post_json), + sink = ltn12.sink.table(response_body), + headers = headers + }, yaml_conf) if not res then errmsg = str_format("request etcd endpoint \"%s\" error, %s\n", put_url, err) util.die(errmsg) diff --git a/doc/install-dependencies.md b/doc/install-dependencies.md index f8df981f4130..d1325972b585 100644 --- a/doc/install-dependencies.md +++ b/doc/install-dependencies.md @@ -55,7 +55,12 @@ sudo yum install yum-utils sudo yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo # install OpenResty and some compilation tools -sudo yum install -y openresty curl git gcc luarocks lua-devel +sudo yum install -y openresty curl git gcc lua-devel openresty-openssl-devel + +# install LuaRocks +curl https://raw.githubusercontent.com/apache/apisix/master/utils/linux-install-luarocks.sh -sL | sudo bash - + +# configure # start etcd server nohup etcd & @@ -75,7 +80,10 @@ tar -xvf etcd-v3.4.13-linux-amd64.tar.gz && \ sudo cp -a etcd etcdctl /usr/bin/ # install OpenResty and some compilation tools -sudo yum install -y openresty curl git gcc luarocks lua-devel +sudo yum install -y openresty curl git gcc lua-devel openresty-openssl-devel + +# install LuaRocks +curl https://raw.githubusercontent.com/apache/apisix/master/utils/linux-install-luarocks.sh -sL | sudo bash - # start etcd server nohup etcd & @@ -98,7 +106,10 @@ tar -xvf etcd-v3.4.13-linux-amd64.tar.gz && \ sudo cp -a etcd etcdctl /usr/bin/ # install OpenResty and some compilation tools -sudo apt-get install -y git openresty curl luarocks +sudo apt-get install -y git openresty curl openresty-openssl-dev + +# install LuaRocks +curl https://raw.githubusercontent.com/apache/apisix/master/utils/linux-install-luarocks.sh -sL | sudo bash - # start etcd server nohup etcd & @@ -126,7 +137,10 @@ tar -xvf etcd-v3.4.13-linux-amd64.tar.gz && \ sudo cp -a etcd etcdctl /usr/bin/ # install OpenResty and some compilation tools -sudo apt-get install -y git openresty curl luarocks make +sudo apt-get install -y git openresty curl make openresty-openssl-dev + +# install LuaRocks +curl https://raw.githubusercontent.com/apache/apisix/master/utils/linux-install-luarocks.sh -sL | sudo bash - # start etcd server nohup etcd & diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 6f195def71da..ee08a4cf66ef 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -62,6 +62,7 @@ dependencies = { "graphql = 0.0.2", "argparse = 0.7.1-1", "luasocket = 3.0rc1-2", + "luasec = 0.9-1", } build = { @@ -73,6 +74,8 @@ build = { LUA_BINDIR="$(LUA_BINDIR)", LUA_INCDIR="$(LUA_INCDIR)", LUA="$(LUA)", + OPENSSL_INCDIR="$(OPENSSL_INCDIR)", + OPENSSL_LIBDIR="$(OPENSSL_LIBDIR)", }, install_variables = { INST_PREFIX="$(PREFIX)", diff --git a/utils/centos7-ci.sh b/utils/centos7-ci.sh index 7395518ff5d8..3f4c5f5fcff0 100755 --- a/utils/centos7-ci.sh +++ b/utils/centos7-ci.sh @@ -24,17 +24,20 @@ install_dependencies() { export PATH=/usr/local/openresty-debug/nginx/sbin:/usr/local/openresty-debug/bin:$PATH # install development tools - yum install -y wget tar gcc automake autoconf libtool make \ - curl git which + yum install -y wget tar gcc automake autoconf libtool make unzip \ + curl git which sudo # install epel and luarocks wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm rpm -ivh epel-release-latest-7.noarch.rpm - yum install -y luarocks lua-devel + yum install -y lua-devel + + OR_PREFIX=/usr/local/openresty-debug ./apisix/utils/linux-install-luarocks.sh # install openresty yum install -y yum-utils && yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo yum install -y openresty-debug + yum install -y openresty-openssl-debug-devel # install etcdctl wget https://github.com/etcd-io/etcd/releases/download/v3.4.0/etcd-v3.4.0-linux-amd64.tar.gz diff --git a/utils/install-apisix.sh b/utils/install-apisix.sh index 5f5645b0ab8e..92a3dd6e5dd5 100755 --- a/utils/install-apisix.sh +++ b/utils/install-apisix.sh @@ -56,7 +56,7 @@ do_install() { do_remove() { sudo rm -f /usr/bin/apisix - sudo luarocks purge /usr/local/apisix/deps --tree=/usr/local/apisix/deps + sudo luarocks purge --tree /usr/local/apisix/deps } diff --git a/utils/linux-install-luarocks.sh b/utils/linux-install-luarocks.sh index 2432c3665d87..43c97e25706d 100755 --- a/utils/linux-install-luarocks.sh +++ b/utils/linux-install-luarocks.sh @@ -16,11 +16,19 @@ # limitations under the License. # -wget https://github.com/luarocks/luarocks/archive/v2.4.4.tar.gz -tar -xf v2.4.4.tar.gz -cd luarocks-2.4.4 || exit +if [ -z ${OR_PREFIX} ]; then + OR_PREFIX="/usr/local/openresty" +fi + +wget https://github.com/luarocks/luarocks/archive/v3.4.0.tar.gz +tar -xf v3.4.0.tar.gz +cd luarocks-3.4.0 || exit ./configure --prefix=/usr > build.log 2>&1 || (cat build.log && exit 1) make build > build.log 2>&1 || (cat build.log && exit 1) -sudo make install > build.log 2>&1 || (cat build.log && exit 1) +make install > build.log 2>&1 || (cat build.log && exit 1) cd .. || exit -rm -rf luarocks-2.4.4 +rm -rf luarocks-3.4.0 + +mkdir ~/.luarocks || true +luarocks config variables.OPENSSL_LIBDIR ${OR_PREFIX}/openssl/lib +luarocks config variables.OPENSSL_INCDIR ${OR_PREFIX}/openssl/include diff --git a/utils/linux-install-openresty.sh b/utils/linux-install-openresty.sh index 4b4a6a24b724..f9d040644006 100755 --- a/utils/linux-install-openresty.sh +++ b/utils/linux-install-openresty.sh @@ -17,6 +17,13 @@ # set -euo pipefail +wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add - +sudo apt-get -y update --fix-missing +sudo apt-get -y install software-properties-common +sudo add-apt-repository -y "deb https://openresty.org/package/ubuntu $(lsb_release -sc) main" + +sudo apt-get update + if [ "$OPENRESTY_VERSION" == "source" ]; then cd .. @@ -63,22 +70,14 @@ if [ "$OPENRESTY_VERSION" == "source" ]; then make sudo make install - sudo apt-get install lua5.1 liblua5.1-0-dev - + sudo apt-get install lua5.1 liblua5.1-0-dev openresty-openssl-debug-dev exit 0 fi -wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add - -sudo apt-get -y update --fix-missing -sudo apt-get -y install software-properties-common -sudo add-apt-repository -y "deb https://openresty.org/package/ubuntu $(lsb_release -sc) main" - -sudo apt-get update - if [ "$OPENRESTY_VERSION" == "default" ]; then openresty='openresty-debug' else openresty="openresty-debug=$OPENRESTY_VERSION*" fi -sudo apt-get install "$openresty" lua5.1 liblua5.1-0-dev +sudo apt-get install "$openresty" lua5.1 liblua5.1-0-dev openresty-openssl-debug-dev