From 6addbaa737d3c331a0e6655e756bfe4b55ce3519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 8 Sep 2022 14:46:25 +0200 Subject: [PATCH 01/73] minimal working example --- lib/grpc/codec/json.ex | 15 +++++ lib/grpc/server.ex | 75 +++++++++++++++++++--- lib/grpc/server/adapters/cowboy/handler.ex | 38 ++++++----- lib/grpc/server/http_transcode.ex | 11 ++++ lib/grpc/server/stream.ex | 7 ++ lib/grpc/service.ex | 15 +++-- lib/grpc/stub.ex | 3 +- mix.exs | 1 + 8 files changed, 133 insertions(+), 32 deletions(-) create mode 100644 lib/grpc/codec/json.ex create mode 100644 lib/grpc/server/http_transcode.ex diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex new file mode 100644 index 00000000..078bb883 --- /dev/null +++ b/lib/grpc/codec/json.ex @@ -0,0 +1,15 @@ +defmodule GRPC.Codec.JSON do + @behaviour GRPC.Codec + + def name() do + "json" + end + + def encode(struct) do + Protobuf.JSON.encode!(struct) + end + + def decode(binary, module) do + Protobuf.JSON.decode!(binary, module) + end +end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 0cbed721..95d73850 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -34,6 +34,7 @@ defmodule GRPC.Server do require Logger alias GRPC.RPCError + alias GRPC.Server.HTTPTranscode @type rpc_req :: struct | Enumerable.t() @type rpc_return :: struct | any @@ -43,10 +44,12 @@ defmodule GRPC.Server do quote bind_quoted: [opts: opts], location: :keep do service_mod = opts[:service] service_name = service_mod.__meta__(:name) - codecs = opts[:codecs] || [GRPC.Codec.Proto, GRPC.Codec.WebText] + codecs = opts[:codecs] || [GRPC.Codec.Proto, GRPC.Codec.WebText, GRPC.Codec.JSON] compressors = opts[:compressors] || [] + http_transcode = opts[:http_transcode] || false - Enum.each(service_mod.__rpc_calls__, fn {name, _, _} = rpc -> + Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> + IO.inspect(options, pretty: true) func_name = name |> to_string |> Macro.underscore() |> String.to_atom() path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) @@ -64,12 +67,47 @@ defmodule GRPC.Server do unquote(func_name) ) end + + if http_transcode and Map.has_key?(options, :http) do + %{value: http_opts} = Map.fetch!(options, :http) + + http_path = HTTPTranscode.path(http_opts) + http_method = HTTPTranscode.method(http_opts) + + def __call_rpc__(unquote(http_path), stream) do + GRPC.Server.call( + unquote(service_mod), + %{ + stream + | service_name: unquote(service_name), + method_name: unquote(to_string(name)), + grpc_type: unquote(grpc_type), + http_method: unquote(http_method), + http_transcode: unquote(http_transcode) + }, + unquote(Macro.escape(put_elem(rpc, 0, func_name))), + unquote(func_name) + ) + end + + def service_name(unquote(http_path)) do + unquote(service_name) + end + end + + def service_name(unquote(path)) do + unquote(service_name) + end end) def __call_rpc__(_, stream) do raise GRPC.RPCError, status: :unimplemented end + def service_name(_) do + "" + end + def __meta__(:service), do: unquote(service_mod) def __meta__(:codecs), do: unquote(codecs) def __meta__(:compressors), do: unquote(compressors) @@ -82,7 +120,7 @@ defmodule GRPC.Server do def call( _service_mod, stream, - {_, {req_mod, req_stream}, {res_mod, res_stream}} = rpc, + {_, {req_mod, req_stream}, {res_mod, res_stream}, _options} = rpc, func_name ) do request_id = generate_request_id() @@ -116,6 +154,27 @@ defmodule GRPC.Server do end end + defp do_handle_request( + false, + res_stream, + %{ + request_mod: req_mod, + codec: codec, + adapter: adapter, + payload: payload, + http_transcode: true + } = stream, + func_name + ) do + {:ok, data} = adapter.read_body(payload) + request = codec.decode(data, req_mod) + Logger.debug(fn -> + "http transcode request #{inspect(request)}" + end) + + call_with_interceptors(res_stream, func_name, stream, request) + end + defp do_handle_request( false, res_stream, @@ -353,11 +412,11 @@ defmodule GRPC.Server do end @doc false - @spec service_name(String.t()) :: String.t() - def service_name(path) do - ["", name | _] = String.split(path, "/") - name - end + # @spec service_name(String.t()) :: String.t() + # def service_name(path) do + # ["", name | _] = String.split(path, "/") + # name + # end @doc false @spec servers_to_map(module() | [module()]) :: %{String.t() => [module()]} diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index f0f5641b..b6498b60 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -18,6 +18,10 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do def init(req, {endpoint, servers, opts} = state) do path = :cowboy_req.path(req) + Logger.info(fn -> + "path: #{path}" + end) + with {:ok, server} <- find_server(servers, path), {:ok, codec} <- find_codec(req, server), # can be nil @@ -51,37 +55,38 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:cowboy_loop, req, %{pid: pid, handling_timer: timer_ref, pending_reader: nil}} else {:error, error} -> + Logger.error(fn -> inspect(error) end) trailers = HTTP2.server_trailers(error.status, error.message) req = send_error_trailers(req, trailers) {:ok, req, state} end end + # TODO compile routes instead of dynamic dispatch to find which server has + # which route defp find_server(servers, path) do - case Map.fetch(servers, GRPC.Server.service_name(path)) do - s = {:ok, _} -> - s - - _ -> - {:error, RPCError.exception(status: :unimplemented)} + case Enum.find(servers, fn {_name, server} -> server.service_name(path) != "" end) do + nil -> {:error, RPCError.exception(status: :unimplemented)} + {_, server} -> {:ok, server} end end defp find_codec(req, server) do req_content_type = :cowboy_req.header("content-type", req) - {:ok, subtype} = extract_subtype(req_content_type) - codec = Enum.find(server.__meta__(:codecs), nil, fn c -> c.name() == subtype end) - - if codec do + with {:ok, subtype} <- extract_subtype(req_content_type), + codec when not is_nil(codec) <- + Enum.find(server.__meta__(:codecs), nil, fn c -> c.name() == subtype end) do {:ok, codec} else - # TODO: Send grpc-accept-encoding header - {:error, - RPCError.exception( - status: :unimplemented, - message: "No codec registered for content-type #{req_content_type}" - )} + err -> + Logger.error(fn -> inspect(err) end) + + {:error, + RPCError.exception( + status: :unimplemented, + message: "No codec registered for content-type #{req_content_type}" + )} end end @@ -444,6 +449,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end + defp extract_subtype("application/json"), do: {:ok, "json"} defp extract_subtype("application/grpc"), do: {:ok, "proto"} defp extract_subtype("application/grpc+"), do: {:ok, "proto"} defp extract_subtype("application/grpc;"), do: {:ok, "proto"} diff --git a/lib/grpc/server/http_transcode.ex b/lib/grpc/server/http_transcode.ex new file mode 100644 index 00000000..2b1813e7 --- /dev/null +++ b/lib/grpc/server/http_transcode.ex @@ -0,0 +1,11 @@ +defmodule GRPC.Server.HTTPTranscode do + @spec path(term()) :: String.t() + def path(%{pattern: {_method, path}}) do + path + end + + @spec method(term()) :: String.t() + def method(%{pattern: {method, _path}}) do + method + end +end diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index bd349722..d1f7ff4f 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -34,6 +34,9 @@ defmodule GRPC.Server.Stream do # compressor mainly is used in client decompressing, responses compressing should be set by # `GRPC.Server.set_compressor` compressor: module() | nil, + # For http transcoding + http_method: :get | :post | :put | :patch | :delete, + http_transcode: boolean(), __interface__: map() } @@ -51,11 +54,15 @@ defmodule GRPC.Server.Stream do adapter: nil, local: nil, compressor: nil, + http_method: :post, + http_transcode: false, __interface__: %{send_reply: &__MODULE__.send_reply/3} def send_reply(%{adapter: adapter, codec: codec} = stream, reply, opts) do # {:ok, data, _size} = reply |> codec.encode() |> GRPC.Message.to_data() data = codec.encode(reply) + IO.inspect(data, label: "#{__MODULE__}.send_reply data") + IO.inspect(stream.payload, label: "#{__MODULE__}.send_reply payload") adapter.send_reply(stream.payload, data, Keyword.put(opts, :codec, codec)) stream end diff --git a/lib/grpc/service.ex b/lib/grpc/service.ex index 5357b6c2..67ad1596 100644 --- a/lib/grpc/service.ex +++ b/lib/grpc/service.ex @@ -15,7 +15,7 @@ defmodule GRPC.Service do defmacro __using__(opts) do quote do - import GRPC.Service, only: [rpc: 3, stream: 1] + import GRPC.Service, only: [rpc: 4, stream: 1] Module.register_attribute(__MODULE__, :rpc_calls, accumulate: true) @before_compile GRPC.Service @@ -32,9 +32,10 @@ defmodule GRPC.Service do end end - defmacro rpc(name, request, reply) do + defmacro rpc(name, request, reply, options) do quote do - @rpc_calls {unquote(name), unquote(wrap_stream(request)), unquote(wrap_stream(reply))} + @rpc_calls {unquote(name), unquote(wrap_stream(request)), unquote(wrap_stream(reply)), + unquote(options)} end end @@ -54,8 +55,8 @@ defmodule GRPC.Service do quote do: {unquote(param), false} end - def grpc_type({_, {_, false}, {_, false}}), do: :unary - def grpc_type({_, {_, true}, {_, false}}), do: :client_stream - def grpc_type({_, {_, false}, {_, true}}), do: :server_stream - def grpc_type({_, {_, true}, {_, true}}), do: :bidi_stream + def grpc_type({_, {_, false}, {_, false}, _}), do: :unary + def grpc_type({_, {_, true}, {_, false}, _}), do: :client_stream + def grpc_type({_, {_, false}, {_, true}, _}), do: :server_stream + def grpc_type({_, {_, true}, {_, true}, _}), do: :bidi_stream end diff --git a/lib/grpc/stub.ex b/lib/grpc/stub.ex index 6dab7dd7..0ff942bf 100644 --- a/lib/grpc/stub.ex +++ b/lib/grpc/stub.ex @@ -59,7 +59,8 @@ defmodule GRPC.Stub do service_mod = opts[:service] service_name = service_mod.__meta__(:name) - Enum.each(service_mod.__rpc_calls__, fn {name, {_, req_stream}, {_, res_stream}} = rpc -> + Enum.each(service_mod.__rpc_calls__, fn {name, {_, req_stream}, {_, res_stream}, _options} = + rpc -> func_name = name |> to_string |> Macro.underscore() path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) diff --git a/mix.exs b/mix.exs index af6eed0b..476ff26d 100644 --- a/mix.exs +++ b/mix.exs @@ -43,6 +43,7 @@ defmodule GRPC.Mixfile do # This is the same as :gun 2.0.0-rc.2, # but we can't depend on an RC for releases {:gun, "~> 2.0.1", hex: :grpc_gun}, + {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, {:protobuf, "~> 0.10", only: [:dev, :test]}, {:ex_doc, "~> 0.28.0", only: :dev}, From 55e9b719ce0a69c19edeade30cac9d324b0139e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 8 Sep 2022 15:55:18 +0200 Subject: [PATCH 02/73] return correct content-type when codec is JSON --- lib/grpc/transport/http2.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/grpc/transport/http2.ex b/lib/grpc/transport/http2.ex index 246563f0..6cde2371 100644 --- a/lib/grpc/transport/http2.ex +++ b/lib/grpc/transport/http2.ex @@ -12,6 +12,10 @@ defmodule GRPC.Transport.HTTP2 do %{"content-type" => "application/grpc-web-#{codec.name()}"} end + def server_headers(%{codec: GRPC.Codec.JSON = codec}) do + %{"content-type" => "application/#{codec.name()}"} + end + def server_headers(%{codec: codec}) do %{"content-type" => "application/grpc+#{codec.name()}"} end From fd09f3f17cbbfcef64c198fa07be5d6ed145268b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 8 Sep 2022 16:02:38 +0200 Subject: [PATCH 03/73] support default rpc options arg --- lib/grpc/server.ex | 5 +---- lib/grpc/server/stream.ex | 2 -- lib/grpc/service.ex | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 95d73850..a699934a 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -49,7 +49,6 @@ defmodule GRPC.Server do http_transcode = opts[:http_transcode] || false Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> - IO.inspect(options, pretty: true) func_name = name |> to_string |> Macro.underscore() |> String.to_atom() path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) @@ -166,11 +165,9 @@ defmodule GRPC.Server do } = stream, func_name ) do + IO.inspect(res_stream, label: "do_handle_request") {:ok, data} = adapter.read_body(payload) request = codec.decode(data, req_mod) - Logger.debug(fn -> - "http transcode request #{inspect(request)}" - end) call_with_interceptors(res_stream, func_name, stream, request) end diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index d1f7ff4f..6111e81d 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -61,8 +61,6 @@ defmodule GRPC.Server.Stream do def send_reply(%{adapter: adapter, codec: codec} = stream, reply, opts) do # {:ok, data, _size} = reply |> codec.encode() |> GRPC.Message.to_data() data = codec.encode(reply) - IO.inspect(data, label: "#{__MODULE__}.send_reply data") - IO.inspect(stream.payload, label: "#{__MODULE__}.send_reply payload") adapter.send_reply(stream.payload, data, Keyword.put(opts, :codec, codec)) stream end diff --git a/lib/grpc/service.ex b/lib/grpc/service.ex index 67ad1596..39b7edd9 100644 --- a/lib/grpc/service.ex +++ b/lib/grpc/service.ex @@ -15,7 +15,7 @@ defmodule GRPC.Service do defmacro __using__(opts) do quote do - import GRPC.Service, only: [rpc: 4, stream: 1] + import GRPC.Service, only: [rpc: 4, rpc: 3, stream: 1] Module.register_attribute(__MODULE__, :rpc_calls, accumulate: true) @before_compile GRPC.Service @@ -32,7 +32,7 @@ defmodule GRPC.Service do end end - defmacro rpc(name, request, reply, options) do + defmacro rpc(name, request, reply, options \\ quote(do: %{})) do quote do @rpc_calls {unquote(name), unquote(wrap_stream(request)), unquote(wrap_stream(reply)), unquote(options)} From f97bb4497e8263611bf3f30482308bace2bcb62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 10:20:00 +0200 Subject: [PATCH 04/73] add custom protoc plugin --- lib/grpc/protoc/cli.ex | 226 +++++++++++++++++++++++++++ lib/grpc/protoc/generator.ex | 68 ++++++++ lib/grpc/protoc/generator/service.ex | 71 +++++++++ mix.exs | 62 +++++++- mix.lock | 3 +- priv/templates/service.ex.eex | 24 +++ 6 files changed, 451 insertions(+), 3 deletions(-) create mode 100644 lib/grpc/protoc/cli.ex create mode 100644 lib/grpc/protoc/generator.ex create mode 100644 lib/grpc/protoc/generator/service.ex create mode 100644 priv/templates/service.ex.eex diff --git a/lib/grpc/protoc/cli.ex b/lib/grpc/protoc/cli.ex new file mode 100644 index 00000000..84aa2215 --- /dev/null +++ b/lib/grpc/protoc/cli.ex @@ -0,0 +1,226 @@ +defmodule GRPC.Protoc.CLI do + @moduledoc """ + `protoc` plugin for generating Elixir code. + + `protoc-gen-elixir` (this name is important) **must** be in `$PATH`. You are not supposed + to call it directly, but only through `protoc`. + + ## Examples + + $ protoc --elixir_out=./lib your.proto + $ protoc --elixir_out=plugins=grpc:./lib/ *.proto + $ protoc -I protos --elixir_out=./lib protos/namespace/*.proto + + Options: + + * --version Print version of protobuf-elixir + * --help (-h) Print this help + + """ + + alias Protobuf.Protoc.Context + + # Entrypoint for the escript (protoc-gen-elixir). + @doc false + @spec main([String.t()]) :: :ok + def main(args) + + def main(["--version"]) do + {:ok, version} = :application.get_key(:protobuf, :vsn) + IO.puts(version) + end + + def main([opt]) when opt in ["--help", "-h"] do + IO.puts(@moduledoc) + end + + # When called through protoc, all input is passed through stdin. + def main([] = _args) do + Protobuf.load_extensions() + + # See https://groups.google.com/forum/#!topic/elixir-lang-talk/T5enez_BBTI. + :io.setopts(:standard_io, encoding: :latin1) + + # Read the standard input that protoc feeds us. + bin = binread_all!(:stdio) + + request = Protobuf.Decoder.decode(bin, Google.Protobuf.Compiler.CodeGeneratorRequest) + + ctx = + %Context{} + |> parse_params(request.parameter || "") + |> find_types(request.proto_file, request.file_to_generate) + + files = + Enum.flat_map(request.file_to_generate, fn file -> + desc = Enum.find(request.proto_file, &(&1.name == file)) + GRPC.Protoc.Generator.generate(ctx, desc) + end) + + Google.Protobuf.Compiler.CodeGeneratorResponse.new( + file: files, + supported_features: supported_features() + ) + |> Protobuf.encode_to_iodata() + |> IO.binwrite() + end + + def main(_args) do + raise "invalid arguments. See protoc-gen-elixir --help." + end + + def supported_features() do + # The only available feature is proto3 with optional fields. + # This is backwards compatible with proto2 optional fields. + Google.Protobuf.Compiler.CodeGeneratorResponse.Feature.value(:FEATURE_PROTO3_OPTIONAL) + end + + # Made public for testing. + @doc false + def parse_params(%Context{} = ctx, params_str) when is_binary(params_str) do + params_str + |> String.split(",") + |> Enum.reduce(ctx, &parse_param/2) + end + + defp parse_param("plugins=" <> plugins, ctx) do + %Context{ctx | plugins: String.split(plugins, "+")} + end + + defp parse_param("gen_descriptors=" <> value, ctx) do + case value do + "true" -> + %Context{ctx | gen_descriptors?: true} + + other -> + raise "invalid value for gen_descriptors option, expected \"true\", got: #{inspect(other)}" + end + end + + defp parse_param("package_prefix=" <> package, ctx) do + if package == "" do + raise "package_prefix can't be empty" + else + %Context{ctx | package_prefix: package} + end + end + + defp parse_param("transform_module=" <> module, ctx) do + %Context{ctx | transform_module: Module.concat([module])} + end + + defp parse_param("one_file_per_module=" <> value, ctx) do + case value do + "true" -> + %Context{ctx | one_file_per_module?: true} + + other -> + raise "invalid value for one_file_per_module option, expected \"true\", got: #{inspect(other)}" + end + end + + defp parse_param("include_docs=" <> value, ctx) do + case value do + "true" -> + %Context{ctx | include_docs?: true} + + other -> + raise "invalid value for include_docs option, expected \"true\", got: #{inspect(other)}" + end + end + + defp parse_param(_unknown, ctx) do + ctx + end + + # Made public for testing. + @doc false + @spec find_types(Context.t(), [Google.Protobuf.FileDescriptorProto.t()], [String.t()]) :: + Context.t() + def find_types(%Context{} = ctx, descs, files_to_generate) + when is_list(descs) and is_list(files_to_generate) do + global_type_mapping = + Map.new(descs, fn %Google.Protobuf.FileDescriptorProto{name: filename} = desc -> + {filename, find_types_in_proto(ctx, desc, files_to_generate)} + end) + + %Context{ctx | global_type_mapping: global_type_mapping} + end + + defp find_types_in_proto( + %Context{} = ctx, + %Google.Protobuf.FileDescriptorProto{} = desc, + files_to_generate + ) do + # Only take package_prefix into consideration for files that we're directly generating. + package_prefix = + if desc.name in files_to_generate do + ctx.package_prefix + else + nil + end + + ctx = + %Protobuf.Protoc.Context{ + namespace: [], + package_prefix: package_prefix, + package: desc.package + } + |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc) + + find_types_in_descriptor(_types = %{}, ctx, desc.message_type ++ desc.enum_type) + end + + defp find_types_in_descriptor(types_acc, ctx, descs) when is_list(descs) do + Enum.reduce(descs, types_acc, &find_types_in_descriptor(_acc = &2, ctx, _desc = &1)) + end + + defp find_types_in_descriptor( + types_acc, + ctx, + %Google.Protobuf.DescriptorProto{name: name} = desc + ) do + new_ctx = update_in(ctx.namespace, &(&1 ++ [name])) + + types_acc + |> update_types(ctx, name) + |> find_types_in_descriptor(new_ctx, desc.enum_type) + |> find_types_in_descriptor(new_ctx, desc.nested_type) + end + + defp find_types_in_descriptor( + types_acc, + ctx, + %Google.Protobuf.EnumDescriptorProto{name: name} + ) do + update_types(types_acc, ctx, name) + end + + defp update_types(types, %Context{namespace: ns, package: pkg} = ctx, name) do + type_name = Protobuf.Protoc.Generator.Util.mod_name(ctx, ns ++ [name]) + + mapping_name = + ([pkg] ++ ns ++ [name]) + |> Enum.reject(&is_nil/1) + |> Enum.join(".") + + Map.put(types, "." <> mapping_name, %{type_name: type_name}) + end + + if Version.match?(System.version(), "~> 1.13") do + defp binread_all!(device) do + case IO.binread(device, :eof) do + data when is_binary(data) -> data + :eof -> _previous_behavior = "" + other -> raise "reading from #{inspect(device)} failed: #{inspect(other)}" + end + end + else + defp binread_all!(device) do + case IO.binread(device, :all) do + data when is_binary(data) -> data + other -> raise "reading from #{inspect(device)} failed: #{inspect(other)}" + end + end + end +end diff --git a/lib/grpc/protoc/generator.ex b/lib/grpc/protoc/generator.ex new file mode 100644 index 00000000..e7a3085a --- /dev/null +++ b/lib/grpc/protoc/generator.ex @@ -0,0 +1,68 @@ +defmodule GRPC.Protoc.Generator do + @moduledoc false + + alias Protobuf.Protoc.Context + alias Protobuf.Protoc.Generator + + @spec generate(Context.t(), %Google.Protobuf.FileDescriptorProto{}) :: + [Google.Protobuf.Compiler.CodeGeneratorResponse.File.t()] + def generate(%Context{} = ctx, %Google.Protobuf.FileDescriptorProto{} = desc) do + module_definitions = + ctx + |> generate_module_definitions(desc) + |> Enum.reject(&is_nil/1) + + if ctx.one_file_per_module? do + Enum.map(module_definitions, fn {mod_name, content} -> + file_name = Macro.underscore(mod_name) <> ".svc.ex" + + Google.Protobuf.Compiler.CodeGeneratorResponse.File.new( + name: file_name, + content: content + ) + end) + else + # desc.name is the filename, ending in ".proto". + file_name = Path.rootname(desc.name) <> ".svc.ex" + + content = + module_definitions + |> Enum.map(fn {_mod_name, contents} -> [contents, ?\n] end) + |> IO.iodata_to_binary() + |> Generator.Util.format() + + [ + Google.Protobuf.Compiler.CodeGeneratorResponse.File.new( + name: file_name, + content: content + ) + ] + end + end + + defp generate_module_definitions(ctx, %Google.Protobuf.FileDescriptorProto{} = desc) do + ctx = + %Context{ + ctx + | syntax: syntax(desc.syntax), + package: desc.package, + dep_type_mapping: get_dep_type_mapping(ctx, desc.dependency, desc.name) + } + |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc) + + Enum.map(desc.service, &GRPC.Protoc.Generator.Service.generate(ctx, &1)) + end + + defp get_dep_type_mapping(%Context{global_type_mapping: global_mapping}, deps, file_name) do + mapping = + Enum.reduce(deps, %{}, fn dep, acc -> + Map.merge(acc, global_mapping[dep]) + end) + + Map.merge(mapping, global_mapping[file_name]) + end + + defp syntax("proto3"), do: :proto3 + defp syntax("proto2"), do: :proto2 + defp syntax(nil), do: :proto2 +end diff --git a/lib/grpc/protoc/generator/service.ex b/lib/grpc/protoc/generator/service.ex new file mode 100644 index 00000000..f7aeea10 --- /dev/null +++ b/lib/grpc/protoc/generator/service.ex @@ -0,0 +1,71 @@ +defmodule GRPC.Protoc.Generator.Service do + @moduledoc false + + alias Protobuf.Protoc.Context + alias Protobuf.Protoc.Generator.Util + + require EEx + + EEx.function_from_file( + :defp, + :service_template, + Path.expand("./templates/service.ex.eex", :code.priv_dir(:grpc)), + [:assigns] + ) + + @spec generate(Context.t(), Google.Protobuf.ServiceDescriptorProto.t()) :: + {String.t(), String.t()} + def generate(%Context{} = ctx, %Google.Protobuf.ServiceDescriptorProto{} = desc) do + # service can't be nested + mod_name = Util.mod_name(ctx, [Macro.camelize(desc.name)]) + name = Util.prepend_package_prefix(ctx.package, desc.name) + methods = Enum.map(desc.method, &generate_service_method(ctx, &1)) + + descriptor_fun_body = + if ctx.gen_descriptors? do + Util.descriptor_fun_body(desc) + else + nil + end + + {mod_name, + Util.format( + service_template( + module: mod_name, + service_name: name, + methods: methods, + descriptor_fun_body: descriptor_fun_body, + version: Util.version(), + module_doc?: ctx.include_docs? + ) + )} + end + + defp generate_service_method(ctx, method) do + input = service_arg(Util.type_from_type_name(ctx, method.input_type), method.client_streaming) + + output = + service_arg(Util.type_from_type_name(ctx, method.output_type), method.server_streaming) + + options = + method.options + |> opts() + |> inspect(limit: :infinity) + + {method.name, input, output, options} + end + + defp service_arg(type, _streaming? = true), do: "stream(#{type})" + defp service_arg(type, _streaming?), do: type + + defp opts(%Google.Protobuf.MethodOptions{__pb_extensions__: extensions}) + when extensions == %{} do + %{} + end + + defp opts(%Google.Protobuf.MethodOptions{__pb_extensions__: extensions}) do + for {{type, field}, value} <- extensions, into: %{} do + {field, %{type: type, value: value}} + end + end +end diff --git a/mix.exs b/mix.exs index 476ff26d..4265275d 100644 --- a/mix.exs +++ b/mix.exs @@ -13,6 +13,8 @@ defmodule GRPC.Mixfile do start_permanent: Mix.env() == :prod, deps: deps(), package: package(), + escript: escript(), + aliases: aliases(), description: "The Elixir implementation of gRPC", docs: [ extras: ["README.md"], @@ -37,6 +39,10 @@ defmodule GRPC.Mixfile do [extra_applications: [:logger]] end + def escript do + [main_module: GRPC.Protoc.CLI, name: "protoc-gen-grpc_elixir"] + end + defp deps do [ {:cowboy, "~> 2.9"}, @@ -45,9 +51,15 @@ defmodule GRPC.Mixfile do {:gun, "~> 2.0.1", hex: :grpc_gun}, {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, - {:protobuf, "~> 0.10", only: [:dev, :test]}, + {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", only: [:dev, :test]}, {:ex_doc, "~> 0.28.0", only: :dev}, - {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false} + {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false}, + {:googleapis, + github: "googleapis/googleapis", + branch: "master", + app: false, + compile: false, + only: [:dev, :test]} ] end @@ -60,6 +72,52 @@ defmodule GRPC.Mixfile do } end + defp aliases do + [ + gen_bootstrap_protos: [&build_protobuf_escript/1, &gen_bootstrap_protos/1], + build_protobuf_escript: &build_protobuf_escript/1 + ] + end + defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] + + defp build_protobuf_escript(_args) do + path = Mix.Project.deps_paths().protobuf + + File.cd!(path, fn -> + with 0 <- Mix.shell().cmd("mix deps.get"), + 0 <- Mix.shell().cmd("mix escript.build") do + :ok + else + other -> + Mix.raise("build_protobuf_escript/1 exited with non-zero status: #{other}") + end + end) + end + # https://github.com/elixir-protobuf/protobuf/blob/cdf3acc53f619866b4921b8216d2531da52ceba7/mix.exs#L140 + defp gen_bootstrap_protos(_args) do + proto_src = Mix.Project.deps_paths().googleapis + + protoc!("-I \"#{proto_src}\"", "./lib", [ + "google/api/http.proto", + "google/api/annotations.proto" + ]) + end + + defp protoc!(args, elixir_out, files_to_generate) + when is_binary(args) and is_binary(elixir_out) and is_list(files_to_generate) do + args = + [ + ~s(protoc), + ~s(--plugin=./deps/protobuf/protoc-gen-elixir), + ~s(--elixir_out="#{elixir_out}"), + args + ] ++ files_to_generate + + case Mix.shell().cmd(Enum.join(args, " ")) do + 0 -> Mix.Task.rerun("format", [Path.join([elixir_out, "**", "*.pb.ex"])]) + other -> Mix.raise("'protoc' exited with non-zero status: #{other}") + end + end end diff --git a/mix.lock b/mix.lock index 43cfde44..3ff0ba5b 100644 --- a/mix.lock +++ b/mix.lock @@ -6,12 +6,13 @@ "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, + "googleapis": {:git, "https://github.com/googleapis/googleapis.git", "f0e2be46a5602ad903800811c9583f9e4458de3c", [branch: "master"]}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, - "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, + "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/priv/templates/service.ex.eex b/priv/templates/service.ex.eex new file mode 100644 index 00000000..bbea50b6 --- /dev/null +++ b/priv/templates/service.ex.eex @@ -0,0 +1,24 @@ +defmodule <%= @module %>.Service do + <%= unless @module_doc? do %> + @moduledoc false + <% end %> + use GRPC.Service, name: <%= inspect(@service_name) %>, protoc_gen_elixir_version: "<%= @version %>" + + <%= if @descriptor_fun_body do %> + def descriptor do + # credo:disable-for-next-line + <%= @descriptor_fun_body %> + end + <% end %> + + <%= for {method_name, input, output, options} <- @methods do %> + rpc :<%= method_name %>, <%= input %>, <%= output %>, <%= options %> + <% end %> +end + +defmodule <%= @module %>.Stub do + <%= unless @module_doc? do %> + @moduledoc false + <% end %> + use GRPC.Stub, service: <%= @module %>.Service +end From 6b068f716b05bed02ed7734deb70b6fe42664263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 10:20:41 +0200 Subject: [PATCH 05/73] include api extensions --- lib/google/api/annotations.pb.ex | 8 ++++++ lib/google/api/http.pb.ex | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 lib/google/api/annotations.pb.ex create mode 100644 lib/google/api/http.pb.ex diff --git a/lib/google/api/annotations.pb.ex b/lib/google/api/annotations.pb.ex new file mode 100644 index 00000000..374877d3 --- /dev/null +++ b/lib/google/api/annotations.pb.ex @@ -0,0 +1,8 @@ +defmodule Google.Api.PbExtension do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + extend Google.Protobuf.MethodOptions, :http, 72_295_728, + optional: true, + type: Google.Api.HttpRule +end diff --git a/lib/google/api/http.pb.ex b/lib/google/api/http.pb.ex new file mode 100644 index 00000000..25dd83ad --- /dev/null +++ b/lib/google/api/http.pb.ex @@ -0,0 +1,43 @@ +defmodule Google.Api.Http do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :rules, 1, repeated: true, type: Google.Api.HttpRule + + field :fully_decode_reserved_expansion, 2, + type: :bool, + json_name: "fullyDecodeReservedExpansion" +end + +defmodule Google.Api.HttpRule do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + oneof :pattern, 0 + + field :selector, 1, type: :string + field :get, 2, type: :string, oneof: 0 + field :put, 3, type: :string, oneof: 0 + field :post, 4, type: :string, oneof: 0 + field :delete, 5, type: :string, oneof: 0 + field :patch, 6, type: :string, oneof: 0 + field :custom, 8, type: Google.Api.CustomHttpPattern, oneof: 0 + field :body, 7, type: :string + field :response_body, 12, type: :string, json_name: "responseBody" + + field :additional_bindings, 11, + repeated: true, + type: Google.Api.HttpRule, + json_name: "additionalBindings" +end + +defmodule Google.Api.CustomHttpPattern do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :kind, 1, type: :string + field :path, 2, type: :string +end From 4cab3b4d34959b74a0f805bc0afceca01b4d20ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 12:01:20 +0200 Subject: [PATCH 06/73] include http options in helloworld example --- examples/helloworld/README.md | 14 +- examples/helloworld/lib/helloworld.pb.ex | 25 +- examples/helloworld/lib/helloworld.svc.ex | 39 ++ examples/helloworld/lib/server.ex | 25 +- examples/helloworld/mix.exs | 3 +- examples/helloworld/mix.lock | 3 +- .../priv/protos/google/api/annotations.proto | 31 ++ .../priv/protos/google/api/http.proto | 375 ++++++++++++++++++ .../helloworld/priv/protos/helloworld.proto | 22 +- 9 files changed, 515 insertions(+), 22 deletions(-) create mode 100644 examples/helloworld/lib/helloworld.svc.ex create mode 100644 examples/helloworld/priv/protos/google/api/annotations.proto create mode 100644 examples/helloworld/priv/protos/google/api/http.proto diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index 924681fd..09da4801 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -17,6 +17,16 @@ $ mix run --no-halt $ mix run priv/client.exs ``` +## HTTP Transcoding + +``` shell +# Say hello +curl http://localhost:50051/v1/greeter/test + +# Say hello from +curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' --output - http://localhost:50051/v1/greeter +``` + ## Regenerate Elixir code from proto 1. Modify the proto `priv/protos/helloworld.proto` @@ -26,8 +36,10 @@ $ mix run priv/client.exs mix escript.install hex protobuf ``` 4. Generate the code: + ```shell -$ protoc -I priv/protos --elixir_out=plugins=grpc:./lib/ priv/protos/helloworld.proto +$ (cd ../../; mix build_protobuf_escript && mix escript.build) +$ protoc -I priv/protos --elixir_out=:./lib/ --grpc_elixir_out=./lib --plugin="../../deps/protobuf/protoc-gen-elixir" --plugin="../../protoc-gen-grpc_elixir" priv/protos/helloworld.proto ``` Refer to [protobuf-elixir](https://github.com/tony612/protobuf-elixir#usage) for more information. diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index a8ff6dfa..bd78735a 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -1,26 +1,25 @@ defmodule Helloworld.HelloRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string end -defmodule Helloworld.HelloReply do +defmodule Helloworld.HelloRequestFrom do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - field :message, 1, type: :string - field :today, 2, type: Google.Protobuf.Timestamp + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string + field :from, 2, type: :string end -defmodule Helloworld.Greeter.Service do +defmodule Helloworld.HelloReply do @moduledoc false - use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.10.0" - rpc :SayHello, Helloworld.HelloRequest, Helloworld.HelloReply -end + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 -defmodule Helloworld.Greeter.Stub do - @moduledoc false - use GRPC.Stub, service: Helloworld.Greeter.Service -end + field :message, 1, type: :string + field :today, 2, type: Google.Protobuf.Timestamp +end \ No newline at end of file diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex new file mode 100644 index 00000000..0afcba8b --- /dev/null +++ b/examples/helloworld/lib/helloworld.svc.ex @@ -0,0 +1,39 @@ +defmodule Helloworld.Greeter.Service do + @moduledoc false + + use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.11.0" + + rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/greeter/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:SayHelloFrom, Helloworld.HelloRequestFrom, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "*", + pattern: {:post, "/v1/greeter"}, + response_body: "", + selector: "" + } + } + }) +end + +defmodule Helloworld.Greeter.Stub do + @moduledoc false + + use GRPC.Stub, service: Helloworld.Greeter.Service +end \ No newline at end of file diff --git a/examples/helloworld/lib/server.ex b/examples/helloworld/lib/server.ex index b85241f8..68c72c10 100644 --- a/examples/helloworld/lib/server.ex +++ b/examples/helloworld/lib/server.ex @@ -1,16 +1,31 @@ defmodule Helloworld.Greeter.Server do - use GRPC.Server, service: Helloworld.Greeter.Service + use GRPC.Server, + service: Helloworld.Greeter.Service, + http_transcode: true @spec say_hello(Helloworld.HelloRequest.t(), GRPC.Server.Stream.t()) :: Helloworld.HelloReply.t() def say_hello(request, _stream) do + Helloworld.HelloReply.new( + message: "Hello #{request.name}", + today: today() + ) + end + + @spec say_hello_from(Helloworld.HelloFromRequest.t(), GRPC.Server.Stream.t()) :: + Helloworld.HelloReply.t() + def say_hello_from(request, _stream) do + Helloworld.HelloReply.new( + message: "Hello #{request.name}. From #{request.from}", + today: today() + ) + end + + defp today do nanos_epoch = System.system_time() |> System.convert_time_unit(:native, :nanosecond) seconds = div(nanos_epoch, 1_000_000_000) nanos = nanos_epoch - seconds * 1_000_000_000 - Helloworld.HelloReply.new( - message: "Hello #{request.name}", - today: %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} - ) + %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} end end diff --git a/examples/helloworld/mix.exs b/examples/helloworld/mix.exs index 9bf4cc3e..ecc3867e 100644 --- a/examples/helloworld/mix.exs +++ b/examples/helloworld/mix.exs @@ -19,7 +19,8 @@ defmodule Helloworld.Mixfile do defp deps do [ {:grpc, path: "../../"}, - {:protobuf, "~> 0.10"}, + {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", override: true}, + {:jason, "~> 1.3.0"}, {:google_protos, "~> 0.3.0"}, {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false} ] diff --git a/examples/helloworld/mix.lock b/examples/helloworld/mix.lock index f96a70d1..62dcb34f 100644 --- a/examples/helloworld/mix.lock +++ b/examples/helloworld/mix.lock @@ -5,6 +5,7 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "google_protos": {:hex, :google_protos, "0.3.0", "15faf44dce678ac028c289668ff56548806e313e4959a3aaf4f6e1ebe8db83f4", [:mix], [{:protobuf, "~> 0.10", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "1f6b7fb20371f72f418b98e5e48dae3e022a9a6de1858d4b254ac5a5d0b4035f"}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, - "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/examples/helloworld/priv/protos/google/api/annotations.proto b/examples/helloworld/priv/protos/google/api/annotations.proto new file mode 100644 index 00000000..efdab3db --- /dev/null +++ b/examples/helloworld/priv/protos/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/examples/helloworld/priv/protos/google/api/http.proto b/examples/helloworld/priv/protos/google/api/http.proto new file mode 100644 index 00000000..113fa936 --- /dev/null +++ b/examples/helloworld/priv/protos/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/examples/helloworld/priv/protos/helloworld.proto b/examples/helloworld/priv/protos/helloworld.proto index 12849981..55c41005 100644 --- a/examples/helloworld/priv/protos/helloworld.proto +++ b/examples/helloworld/priv/protos/helloworld.proto @@ -5,6 +5,7 @@ option java_package = "io.grpc.examples.helloworld"; option java_outer_classname = "HelloWorldProto"; option objc_class_prefix = "HLW"; +import "google/api/annotations.proto"; import "google/protobuf/timestamp.proto"; package helloworld; @@ -12,7 +13,18 @@ package helloworld; // The greeting service definition. service Greeter { // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} + rpc SayHello (HelloRequest) returns (HelloReply) { + option (google.api.http) = { + get: "/v1/greeter/{name}" + }; + } + + rpc SayHelloFrom (HelloRequestFrom) returns (HelloReply) { + option (google.api.http) = { + post: "/v1/greeter" + body: "*" + }; + } } // The request message containing the user's name. @@ -20,6 +32,14 @@ message HelloRequest { string name = 1; } +// HelloRequestFrom! +message HelloRequestFrom { + // Name! + string name = 1; + // From! + string from = 2; +} + // The response message containing the greetings message HelloReply { string message = 1; From dc5c6de0246517999f478048037fe57da0736f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 12:02:13 +0200 Subject: [PATCH 07/73] dont pack message if the method was called as a transcode req --- lib/grpc/server.ex | 3 ++- lib/grpc/server/adapters/cowboy.ex | 3 ++- lib/grpc/server/adapters/cowboy/handler.ex | 14 +++++++++++--- mix.exs | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index a699934a..42429c42 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -357,7 +357,8 @@ defmodule GRPC.Server do iex> GRPC.Server.send_reply(stream, reply) """ @spec send_reply(GRPC.Server.Stream.t(), struct()) :: GRPC.Server.Stream.t() - def send_reply(%{__interface__: interface} = stream, reply, opts \\ []) do + def send_reply(%{__interface__: interface, http_transcode: transcode} = stream, reply, opts \\ []) do + opts = Keyword.put(opts, :http_transcode, transcode) interface[:send_reply].(stream, reply, opts) end diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 4a80c332..e9c730bb 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -117,7 +117,8 @@ defmodule GRPC.Server.Adapters.Cowboy do @impl true def send_reply(%{pid: pid}, data, opts) do - Handler.stream_body(pid, data, opts, :nofin) + http_transcode = Keyword.get(opts, :http_transcode) + Handler.stream_body(pid, data, opts, :nofin, http_transcode) end @impl true diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index b6498b60..4c41bc1e 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -119,8 +119,8 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do sync_call(pid, :read_body) end - def stream_body(pid, data, opts, is_fin) do - send(pid, {:stream_body, data, opts, is_fin}) + def stream_body(pid, data, opts, is_fin, http_transcode \\ false) do + send(pid, {:stream_body, data, opts, is_fin, http_transcode}) end def stream_reply(pid, status, headers) do @@ -230,7 +230,15 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, req, state} end - def info({:stream_body, data, opts, is_fin}, req, state) do + # Handle http/json transcoded response + def info({:stream_body, data, _opts, is_fin, _http_transcode = true}, req, state) do + # TODO Compress + req = check_sent_resp(req) + :cowboy_req.stream_body(data, is_fin, req) + {:ok, req, state} + end + + def info({:stream_body, data, opts, is_fin, _}, req, state) do # If compressor exists, compress is true by default compressor = if opts[:compress] == false do diff --git a/mix.exs b/mix.exs index 4265275d..3da1b578 100644 --- a/mix.exs +++ b/mix.exs @@ -51,7 +51,7 @@ defmodule GRPC.Mixfile do {:gun, "~> 2.0.1", hex: :grpc_gun}, {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, - {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", only: [:dev, :test]}, + {:protobuf, github: "elixir-protobuf/protobuf", branch: "main"}, {:ex_doc, "~> 0.28.0", only: :dev}, {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false}, {:googleapis, From 1248ab20d4687bc8aaf7a8e1381add53ea106e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 12 Sep 2022 16:39:12 +0200 Subject: [PATCH 08/73] wip! build route from HttpRule --- lib/grpc/server/http_transcode.ex | 100 ++++++++++++++++++++ test/grpc/transcode_test.exs | 148 ++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 test/grpc/transcode_test.exs diff --git a/lib/grpc/server/http_transcode.ex b/lib/grpc/server/http_transcode.ex index 2b1813e7..a59cfc51 100644 --- a/lib/grpc/server/http_transcode.ex +++ b/lib/grpc/server/http_transcode.ex @@ -8,4 +8,104 @@ defmodule GRPC.Server.HTTPTranscode do def method(%{pattern: {method, _path}}) do method end + + @doc """ + https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule + + Template = "/" Segments [ Verb ] ; + Segments = Segment { "/" Segment } ; + Segment = "*" | "**" | LITERAL | Variable ; + Variable = "{" FieldPath [ "=" Segments ] "}" ; + FieldPath = IDENT { "." IDENT } ; + Verb = ":" LITERAL ; + """ + @spec build_route(term()) :: tuple() + def build_route(%Google.Api.HttpRule{pattern: {method, path}}) do + route = + path + |> tokenize([]) + |> parse([], []) + + {method, route} + end + + @spec tokenize(binary(), list()) :: list() + def tokenize(path, tokens \\ []) + + def tokenize(<<>>, tokens) do + Enum.reverse(tokens) + end + + def tokenize(segments, tokens) do + {token, rest} = do_tokenize(segments, <<>>) + tokenize(rest, [token | tokens]) + end + + @terminals [?/, ?{, ?}, ?=, ?*] + defp do_tokenize(<>, <<>>) when h in @terminals do + # parse(t, acc) + {{List.to_atom([h]), []}, t} + end + + defp do_tokenize(<> = rest, acc) when h in @terminals do + {{:literal, acc, []}, rest} + end + + defp do_tokenize(<>, acc) + when h in ?a..?z or h in ?A..?Z or h in ?0..?9 or h == ?_ or h == ?. do + do_tokenize(t, <>) + end + + defp do_tokenize(<<>>, acc) do + {{:literal, acc, []}, <<>>} + end + + @spec parse(list(tuple()), list(), list()) :: list() + def parse([], params, segments) do + {Enum.reverse(params), Enum.reverse(segments)} + end + + def parse([{:/, _} | rest], params, segments) do + parse(rest, params, segments) + end + + def parse([{:*, _} | rest], params, segments) do + parse(rest, params, [{:_, []} | segments]) + end + + def parse([{:literal, literal, _} | rest], params, segments) do + parse(rest, params, [literal | segments]) + end + + def parse([{:"{", _} | rest], params, segments) do + {params, segments, rest} = parse_binding(rest, params, segments) + parse(rest, params, segments) + end + + defp parse_binding([{:"}", []} | rest], params, segments) do + {params, segments, rest} + end + + defp parse_binding( + [{:literal, lit, _}, {:=, _}, {:literal, assign, _} = a | rest], + params, + segments + ) do + + {variable, _} = param = field_path(lit) + # assign = field_path(assign) + + parse_binding(rest, [param | params], [{variable, [assign]} | segments]) + end + + defp parse_binding([{:literal, lit, []} | rest], params, segments) do + {variable, _} = param = field_path(lit) + parse_binding(rest, [param | params], [{variable, []} | segments]) + end + + def field_path(identifier) do + [root | path] = String.split(identifier, ".") + {String.to_atom(root), path} + end + end diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs new file mode 100644 index 00000000..c4a08f51 --- /dev/null +++ b/test/grpc/transcode_test.exs @@ -0,0 +1,148 @@ +defmodule GRPC.TranscodeTest do + use ExUnit.Case, async: true + alias GRPC.Server.HTTPTranscode, as: Transcode + + describe "build_route/1" do + test "returns a route with {http_method, route} based on the http rule" do + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + assert {:get, {params, segments}} = Transcode.build_route(rule) + assert [message_id: []] == params + assert ["v1", "messages", {:message_id, []}] = segments + end + end + + describe "tokenize/2" do + test "can tokenize simple paths" do + assert [{:/, []}] = Transcode.tokenize("/") + + assert [{:/, []}, {:literal, "v1", []}, {:/, []}, {:literal, "messages", []}] = + Transcode.tokenize("/v1/messages") + end + + test "can tokenize simple paths with wildcards" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:literal, "messages", []}, + {:/, []}, + {:*, []} + ] == Transcode.tokenize("/v1/messages/*") + end + + test "can tokenize simple variables" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:literal, "messages", []}, + {:/, []}, + {:"{", []}, + {:literal, "message_id", []}, + {:"}", []} + ] == Transcode.tokenize("/v1/messages/{message_id}") + end + + test "can tokenize variable assignments in bindings" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:"{", []}, + {:literal, "name", []}, + {:=, []}, + {:literal, "messages", []}, + {:"}", []} + ] == Transcode.tokenize("/v1/{name=messages}") + end + + test "can tokenize field paths in bindings" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:literal, "messages", []}, + {:/, []}, + {:"{", []}, + {:literal, "message_id", []}, + {:"}", []}, + {:/, []}, + {:"{", []}, + {:literal, "sub.subfield", []}, + {:"}", []} + ] == Transcode.tokenize("/v1/messages/{message_id}/{sub.subfield}") + end + end + + describe "parse/3" do + test "can parse simple paths" do + assert {[], []} == + "/" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse paths with literals" do + assert {[], ["v1", "messages"]} == + "/v1/messages" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse paths with wildcards" do + assert {[], ["v1", "messages", {:_, []}]} == + "/v1/messages/*" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse simple bindings with variables" do + assert {[{:message_id, []}], ["v1", "messages", {:message_id, []}]} == + "/v1/messages/{message_id}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse bindings with variable assignment" do + assert {[{:name, []}], ["v1", {:name, ["messages"]}]} == + "/v1/{name=messages}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse multiple bindings with variable assignment" do + assert {[{:name, []}, {:message_id, []}], ["v1", {:name, ["messages"]}, {:message_id, []}]} == + "/v1/{name=messages}/{message_id}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse bindings with field paths " do + assert {[sub: ["subfield"]], ["v1", "messages", {:sub, []}]} == + "/v1/messages/{sub.subfield}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "supports deeper nested field path " do + assert {[sub: ["nested", "nested", "nested"]], ["v1", "messages", {:sub, []}]} == + "/v1/messages/{sub.nested.nested.nested}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse multiple-bindings with field paths " do + assert {[first: ["subfield"], second: ["subfield"]], + ["v1", "messages", {:first, []}, {:second, []}]} == + "/v1/messages/{first.subfield}/{second.subfield}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + end + + defp build_simple_rule(method, pattern) do + Google.Api.HttpRule.new(pattern: {method, pattern}) + end + + # rule = Google.Api.HttpRule.new(pattern: {:get, "/v1/{name=messages/*}"}) +end From fb1331667f201f768c88fd2511ccb3ba3d5211a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 12 Sep 2022 16:39:56 +0200 Subject: [PATCH 09/73] update helloworld example with HttpRule options --- examples/helloworld/lib/helloworld.pb.ex | 16 ++++++++++++ examples/helloworld/lib/helloworld.svc.ex | 26 +++++++++++++++++++ .../helloworld/priv/protos/helloworld.proto | 15 +++++++++++ 3 files changed, 57 insertions(+) diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index bd78735a..b849b575 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -22,4 +22,20 @@ defmodule Helloworld.HelloReply do field :message, 1, type: :string field :today, 2, type: Google.Protobuf.Timestamp +end + +defmodule Helloworld.GetMessageRequest do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string +end + +defmodule Helloworld.Message do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :text, 1, type: :string end \ No newline at end of file diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex index 0afcba8b..382e2f62 100644 --- a/examples/helloworld/lib/helloworld.svc.ex +++ b/examples/helloworld/lib/helloworld.svc.ex @@ -36,4 +36,30 @@ defmodule Helloworld.Greeter.Stub do @moduledoc false use GRPC.Stub, service: Helloworld.Greeter.Service +end + +defmodule Helloworld.Messaging.Service do + @moduledoc false + + use GRPC.Service, name: "helloworld.Messaging", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetMessage, Helloworld.GetMessageRequest, Helloworld.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/{name=messages/*}"}, + response_body: "", + selector: "" + } + } + }) +end + +defmodule Helloworld.Messaging.Stub do + @moduledoc false + + use GRPC.Stub, service: Helloworld.Messaging.Service end \ No newline at end of file diff --git a/examples/helloworld/priv/protos/helloworld.proto b/examples/helloworld/priv/protos/helloworld.proto index 55c41005..632519a0 100644 --- a/examples/helloworld/priv/protos/helloworld.proto +++ b/examples/helloworld/priv/protos/helloworld.proto @@ -45,3 +45,18 @@ message HelloReply { string message = 1; google.protobuf.Timestamp today = 2; } + +service Messaging { + rpc GetMessage(GetMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/{name=messages/*}" + }; + } +} + +message GetMessageRequest { + string name = 1; // Mapped to URL path. +} +message Message { + string text = 1; // The resource content. +} From a8e447347ac7b93e95708e297283e61f24c2e7d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 12 Sep 2022 16:45:00 +0200 Subject: [PATCH 10/73] rename mod to transcode --- lib/grpc/server.ex | 6 +++--- lib/grpc/server/{http_transcode.ex => transcode.ex} | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename lib/grpc/server/{http_transcode.ex => transcode.ex} (98%) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 42429c42..8bf33824 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -34,7 +34,7 @@ defmodule GRPC.Server do require Logger alias GRPC.RPCError - alias GRPC.Server.HTTPTranscode + alias GRPC.Server.Transcode @type rpc_req :: struct | Enumerable.t() @type rpc_return :: struct | any @@ -70,8 +70,8 @@ defmodule GRPC.Server do if http_transcode and Map.has_key?(options, :http) do %{value: http_opts} = Map.fetch!(options, :http) - http_path = HTTPTranscode.path(http_opts) - http_method = HTTPTranscode.method(http_opts) + http_path = Transcode.path(http_opts) + http_method = Transcode.method(http_opts) def __call_rpc__(unquote(http_path), stream) do GRPC.Server.call( diff --git a/lib/grpc/server/http_transcode.ex b/lib/grpc/server/transcode.ex similarity index 98% rename from lib/grpc/server/http_transcode.ex rename to lib/grpc/server/transcode.ex index a59cfc51..1f55f1ce 100644 --- a/lib/grpc/server/http_transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,4 +1,4 @@ -defmodule GRPC.Server.HTTPTranscode do +defmodule GRPC.Server.Transcode do @spec path(term()) :: String.t() def path(%{pattern: {_method, path}}) do path From 3b661c4571d35a51250e9ccc28acca0f7c858012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 08:59:57 +0200 Subject: [PATCH 11/73] fix! match rpc options in stub --- lib/grpc/server.ex | 1 - lib/grpc/stub.ex | 2 +- test/grpc/transcode_test.exs | 4 +--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 8bf33824..b1e6aab9 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -165,7 +165,6 @@ defmodule GRPC.Server do } = stream, func_name ) do - IO.inspect(res_stream, label: "do_handle_request") {:ok, data} = adapter.read_body(payload) request = codec.decode(data, req_mod) diff --git a/lib/grpc/stub.ex b/lib/grpc/stub.ex index 0ff942bf..c0dfa879 100644 --- a/lib/grpc/stub.ex +++ b/lib/grpc/stub.ex @@ -244,7 +244,7 @@ defmodule GRPC.Stub do """ @spec call(atom(), tuple(), GRPC.Client.Stream.t(), struct() | nil, keyword()) :: rpc_return def call(_service_mod, rpc, %{channel: channel} = stream, request, opts) do - {_, {req_mod, req_stream}, {res_mod, response_stream}} = rpc + {_, {req_mod, req_stream}, {res_mod, response_stream}, _rpc_options} = rpc stream = %{stream | request_mod: req_mod, response_mod: res_mod} diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs index c4a08f51..3b673103 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/transcode_test.exs @@ -1,6 +1,6 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true - alias GRPC.Server.HTTPTranscode, as: Transcode + alias GRPC.Server.Transcode describe "build_route/1" do test "returns a route with {http_method, route} based on the http rule" do @@ -143,6 +143,4 @@ defmodule GRPC.TranscodeTest do defp build_simple_rule(method, pattern) do Google.Api.HttpRule.new(pattern: {method, pattern}) end - - # rule = Google.Api.HttpRule.new(pattern: {:get, "/v1/{name=messages/*}"}) end From e0a3f6d3259f568c879c65d73d88da1df84b18f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 11:17:15 +0200 Subject: [PATCH 12/73] encode segments to path --- lib/grpc/server/transcode.ex | 15 +++++++++++++-- mix.exs | 3 ++- test/grpc/transcode_test.exs | 24 +++++++++++++++++------- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 1f55f1ce..8601086c 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -9,6 +9,19 @@ defmodule GRPC.Server.Transcode do method end + @spec to_path(term()) :: String.t() + def to_path({method, {_bindings, segments}} = _spec) do + match = + segments + |> Enum.map(&segment_to_string/1) + |> Enum.join("/") + + "/" <> match + end + + defp segment_to_string({binding, _}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" + defp segment_to_string(segment), do: segment + @doc """ https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule @@ -91,7 +104,6 @@ defmodule GRPC.Server.Transcode do params, segments ) do - {variable, _} = param = field_path(lit) # assign = field_path(assign) @@ -107,5 +119,4 @@ defmodule GRPC.Server.Transcode do [root | path] = String.split(identifier, ".") {String.to_atom(root), path} end - end diff --git a/mix.exs b/mix.exs index 3da1b578..0ae049f8 100644 --- a/mix.exs +++ b/mix.exs @@ -95,6 +95,7 @@ defmodule GRPC.Mixfile do end end) end + # https://github.com/elixir-protobuf/protobuf/blob/cdf3acc53f619866b4921b8216d2531da52ceba7/mix.exs#L140 defp gen_bootstrap_protos(_args) do proto_src = Mix.Project.deps_paths().googleapis @@ -117,7 +118,7 @@ defmodule GRPC.Mixfile do case Mix.shell().cmd(Enum.join(args, " ")) do 0 -> Mix.Task.rerun("format", [Path.join([elixir_out, "**", "*.pb.ex"])]) - other -> Mix.raise("'protoc' exited with non-zero status: #{other}") + other -> Mix.raise("'protoc' exited with non-zero status: #{other}") end end end diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs index 3b673103..8a711eea 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/transcode_test.exs @@ -2,13 +2,23 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true alias GRPC.Server.Transcode - describe "build_route/1" do - test "returns a route with {http_method, route} based on the http rule" do - rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert {:get, {params, segments}} = Transcode.build_route(rule) - assert [message_id: []] == params - assert ["v1", "messages", {:message_id, []}] = segments - end + test "build_route/1 returns a route with {http_method, route} based on the http rule" do + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + assert {:get, {params, segments}} = Transcode.build_route(rule) + assert [message_id: []] == params + assert ["v1", "messages", {:message_id, []}] = segments + end + + test "to_path/1 returns path segments as a string match" do + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + assert spec = Transcode.build_route(rule) + assert "/v1/messages/:message_id" = Transcode.to_path(spec) + end + + test "to_path/1 returns path segments as a string when there's multiple bindings" do + rule = build_simple_rule(:get, "/v1/users/{user_id}/messages/{message_id}") + assert spec = Transcode.build_route(rule) + assert "/v1/users/:user_id/messages/:message_id" = Transcode.to_path(spec) end describe "tokenize/2" do From 2a45dfcde6b22d582f0b2da7ae5928b052bd97df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 11:19:02 +0200 Subject: [PATCH 13/73] expose routes from server Routes are fetched from server and compiled into a dispatch conf. Instead of finding a matching server in the handler the path is matched and the handler is called with the correct server. --- lib/grpc/server.ex | 30 ++++++++++++++++++---- lib/grpc/server/adapters/cowboy.ex | 25 ++++++++++++++---- lib/grpc/server/adapters/cowboy/handler.ex | 22 +++------------- 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index b1e6aab9..c05c55cb 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -48,6 +48,22 @@ defmodule GRPC.Server do compressors = opts[:compressors] || [] http_transcode = opts[:http_transcode] || false + routes = + for {name, _, _, options} <- service_mod.__rpc_calls__, reduce: [] do + acc -> + path = "/#{service_name}/#{name}" + + http_paths = + if http_transcode and Map.has_key?(options, :http) do + %{value: http_opts} = Map.fetch!(options, :http) + [{:http_transcode, Transcode.build_route(http_opts)}] + else + [] + end + + http_paths ++ [{:grpc, path} | acc] + end + Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> func_name = name |> to_string |> Macro.underscore() |> String.to_atom() path = "/#{service_name}/#{name}" @@ -68,10 +84,9 @@ defmodule GRPC.Server do end if http_transcode and Map.has_key?(options, :http) do - %{value: http_opts} = Map.fetch!(options, :http) - - http_path = Transcode.path(http_opts) - http_method = Transcode.method(http_opts) + %{value: http_rule} = Map.fetch!(options, :http) + {http_method, _} = spec = Transcode.build_route(http_rule) + http_path = Transcode.to_path(spec) def __call_rpc__(unquote(http_path), stream) do GRPC.Server.call( @@ -110,6 +125,7 @@ defmodule GRPC.Server do def __meta__(:service), do: unquote(service_mod) def __meta__(:codecs), do: unquote(codecs) def __meta__(:compressors), do: unquote(compressors) + def __meta__(:routes), do: unquote(routes) end end @@ -356,7 +372,11 @@ defmodule GRPC.Server do iex> GRPC.Server.send_reply(stream, reply) """ @spec send_reply(GRPC.Server.Stream.t(), struct()) :: GRPC.Server.Stream.t() - def send_reply(%{__interface__: interface, http_transcode: transcode} = stream, reply, opts \\ []) do + def send_reply( + %{__interface__: interface, http_transcode: transcode} = stream, + reply, + opts \\ [] + ) do opts = Keyword.put(opts, :http_transcode, transcode) interface[:send_reply].(stream, reply, opts) end diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index e9c730bb..7f41408f 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -154,20 +154,35 @@ defmodule GRPC.Server.Adapters.Cowboy do Handler.set_compressor(pid, compressor) end + defp build_handlers(endpoint, servers, opts) do + Enum.flat_map(servers, fn {_name, server_mod} = server -> + routes = server_mod.__meta__(:routes) + Enum.map(routes, &build_route(&1, endpoint, server, opts)) + end) + end + + defp build_route({:grpc, path}, endpoint, server, opts) do + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + end + + defp build_route({:http_transcode, spec}, endpoint, server, opts) do + path = GRPC.Server.Transcode.to_path(spec) + + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + end + defp cowboy_start_args(endpoint, servers, port, opts) do # Custom handler to be able to listen in the same port, more info: # https://github.com/containous/traefik/issues/6211 {adapter_opts, opts} = Keyword.pop(opts, :adapter_opts, []) status_handler = Keyword.get(adapter_opts, :status_handler) + handlers = build_handlers(endpoint, servers, opts) handlers = if status_handler do - [ - status_handler, - {:_, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, servers, Enum.into(opts, %{})}} - ] + [status_handler | handlers] else - [{:_, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, servers, Enum.into(opts, %{})}}] + handlers end dispatch = :cowboy_router.compile([{:_, handlers}]) diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 4c41bc1e..f135ec82 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -13,17 +13,12 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do @spec init( map(), - state :: {endpoint :: atom(), servers :: %{String.t() => [module()]}, opts :: keyword()} + state :: {endpoint :: atom(), server :: {String.t(), module()}, route :: String.t(), opts :: keyword()} ) :: {:cowboy_loop, map(), map()} - def init(req, {endpoint, servers, opts} = state) do + def init(req, {endpoint, {_name, server}, route, opts} = state) do path = :cowboy_req.path(req) - Logger.info(fn -> - "path: #{path}" - end) - - with {:ok, server} <- find_server(servers, path), - {:ok, codec} <- find_codec(req, server), + with {:ok, codec} <- find_codec(req, server), # can be nil {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ @@ -36,7 +31,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do compressor: compressor } - pid = spawn_link(__MODULE__, :call_rpc, [server, path, stream]) + pid = spawn_link(__MODULE__, :call_rpc, [server, route, stream]) Process.flag(:trap_exit, true) req = :cowboy_req.set_resp_headers(HTTP2.server_headers(stream), req) @@ -62,15 +57,6 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end - # TODO compile routes instead of dynamic dispatch to find which server has - # which route - defp find_server(servers, path) do - case Enum.find(servers, fn {_name, server} -> server.service_name(path) != "" end) do - nil -> {:error, RPCError.exception(status: :unimplemented)} - {_, server} -> {:ok, server} - end - end - defp find_codec(req, server) do req_content_type = :cowboy_req.header("content-type", req) From 0cda91db9e0221ab8a032f6f7582b499bfa81c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 11:23:42 +0200 Subject: [PATCH 14/73] init http/json transcode integration tests --- examples/helloworld/lib/helloworld.pb.ex | 2 +- examples/helloworld/lib/helloworld.svc.ex | 2 +- test/grpc/integration/server_test.exs | 22 ++ test/support/google/api/annotations.proto | 31 ++ test/support/google/api/http.proto | 375 ++++++++++++++++++++++ test/support/route_guide_transocde.pb.ex | 46 +++ test/support/route_guide_transocde.proto | 51 +++ test/support/route_guide_transocde.svc.ex | 48 +++ 8 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 test/support/google/api/annotations.proto create mode 100644 test/support/google/api/http.proto create mode 100644 test/support/route_guide_transocde.pb.ex create mode 100644 test/support/route_guide_transocde.proto create mode 100644 test/support/route_guide_transocde.svc.ex diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index b849b575..82cce674 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -38,4 +38,4 @@ defmodule Helloworld.Message do use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :text, 1, type: :string -end \ No newline at end of file +end diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex index 382e2f62..dcf882ab 100644 --- a/examples/helloworld/lib/helloworld.svc.ex +++ b/examples/helloworld/lib/helloworld.svc.ex @@ -62,4 +62,4 @@ defmodule Helloworld.Messaging.Stub do @moduledoc false use GRPC.Stub, service: Helloworld.Messaging.Service -end \ No newline at end of file +end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index e2455719..093cf2c4 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -9,6 +9,16 @@ defmodule GRPC.Integration.ServerTest do end end + defmodule FeatureTranscodeServer do + use GRPC.Server, + service: RouteguideTranscode.RouteGuide.Service, + http_transcode: true + + def get_feature(point, _stream) do + Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") + end + end + defmodule HelloServer do use GRPC.Server, service: Helloworld.Greeter.Service @@ -241,4 +251,16 @@ defmodule GRPC.Integration.ServerTest do assert reply.message == "Hello, unauthenticated" end) end + + describe "http/json transcode" do + test "can transcode path params" do + run_server([FeatureTranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + {:ok, conn_pid} = :gun.open('localhost', port) + stream_ref = :gun.get(conn_pid, "/v1/feature/1/2") + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + end) + end + end end diff --git a/test/support/google/api/annotations.proto b/test/support/google/api/annotations.proto new file mode 100644 index 00000000..efdab3db --- /dev/null +++ b/test/support/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/test/support/google/api/http.proto b/test/support/google/api/http.proto new file mode 100644 index 00000000..113fa936 --- /dev/null +++ b/test/support/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/test/support/route_guide_transocde.pb.ex b/test/support/route_guide_transocde.pb.ex new file mode 100644 index 00000000..1079307f --- /dev/null +++ b/test/support/route_guide_transocde.pb.ex @@ -0,0 +1,46 @@ +defmodule RouteguideTranscode.Point do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :latitude, 1, type: :int32 + field :longitude, 2, type: :int32 +end + +defmodule RouteguideTranscode.Rectangle do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :lo, 1, type: RouteguideTranscode.Point + field :hi, 2, type: RouteguideTranscode.Point +end + +defmodule RouteguideTranscode.Feature do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string + field :location, 2, type: RouteguideTranscode.Point +end + +defmodule RouteguideTranscode.RouteNote do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :location, 1, type: RouteguideTranscode.Point + field :message, 2, type: :string +end + +defmodule RouteguideTranscode.RouteSummary do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :point_count, 1, type: :int32, json_name: "pointCount" + field :feature_count, 2, type: :int32, json_name: "featureCount" + field :distance, 3, type: :int32 + field :elapsed_time, 4, type: :int32, json_name: "elapsedTime" +end diff --git a/test/support/route_guide_transocde.proto b/test/support/route_guide_transocde.proto new file mode 100644 index 00000000..67bf7e9e --- /dev/null +++ b/test/support/route_guide_transocde.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.routeguide"; +option java_outer_classname = "RouteGuideProto"; + +import "google/api/annotations.proto"; + +package routeguide_transcode; + +service RouteGuide { + rpc GetFeature(Point) returns (Feature) { + option (google.api.http) = { + get: "/v1/feature/{latitude}/{longitude}" + }; + } + rpc ListFeatures(Rectangle) returns (stream Feature) { + option (google.api.http) = { + get: "/v1/feature" + }; + } + rpc RecordRoute(stream Point) returns (RouteSummary) {} + rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} +} + +message Point { + int32 latitude = 1; + int32 longitude = 2; +} + +message Rectangle { + Point lo = 1; + Point hi = 2; +} + +message Feature { + string name = 1; + Point location = 2; +} + +message RouteNote { + Point location = 1; + string message = 2; +} + +message RouteSummary { + int32 point_count = 1; + int32 feature_count = 2; + int32 distance = 3; + int32 elapsed_time = 4; +} diff --git a/test/support/route_guide_transocde.svc.ex b/test/support/route_guide_transocde.svc.ex new file mode 100644 index 00000000..adf27fdd --- /dev/null +++ b/test/support/route_guide_transocde.svc.ex @@ -0,0 +1,48 @@ +defmodule RouteguideTranscode.RouteGuide.Service do + @moduledoc false + + use GRPC.Service, name: "routeguide_transcode.RouteGuide", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetFeature, RouteguideTranscode.Point, RouteguideTranscode.Feature, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/feature/{latitude}/{longitude}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:ListFeatures, RouteguideTranscode.Rectangle, stream(RouteguideTranscode.Feature), %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/feature"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:RecordRoute, stream(RouteguideTranscode.Point), RouteguideTranscode.RouteSummary, %{}) + + rpc( + :RouteChat, + stream(RouteguideTranscode.RouteNote), + stream(RouteguideTranscode.RouteNote), + %{} + ) +end + +defmodule RouteguideTranscode.RouteGuide.Stub do + @moduledoc false + + use GRPC.Stub, service: RouteguideTranscode.RouteGuide.Service +end From 9e343e4f72e5ac9ba440ac1c41cc8e97a1a22415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 14:30:35 +0200 Subject: [PATCH 15/73] merge path bindings with body --- lib/grpc/codec/json.ex | 5 +++++ lib/grpc/server.ex | 10 +++++++++- lib/grpc/server/adapters/cowboy.ex | 8 ++++++++ lib/grpc/server/adapters/cowboy/handler.ex | 21 ++++++++++++++++++++- lib/grpc/server/transcode.ex | 10 ++++++++++ test/grpc/integration/server_test.exs | 18 ++++++++++++++++-- 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 078bb883..0a32bf53 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -9,6 +9,11 @@ defmodule GRPC.Codec.JSON do Protobuf.JSON.encode!(struct) end + + def decode(<<>>, module) do + struct(module, []) + end + def decode(binary, module) do Protobuf.JSON.decode!(binary, module) end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index c05c55cb..b51c9352 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -182,9 +182,17 @@ defmodule GRPC.Server do func_name ) do {:ok, data} = adapter.read_body(payload) + bindings = adapter.get_bindings(payload) + qs = adapter.get_qs(payload) request = codec.decode(data, req_mod) - call_with_interceptors(res_stream, func_name, stream, request) + case Transcode.map_request(request, bindings, qs, req_mod) do + {:ok, request} -> + call_with_interceptors(res_stream, func_name, stream, request) + + resp = {:error, _} -> + resp + end end defp do_handle_request( diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 7f41408f..122176c2 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -150,6 +150,14 @@ defmodule GRPC.Server.Adapters.Cowboy do Handler.get_cert(pid) end + def get_qs(%{pid: pid}) do + Handler.get_qs(pid) + end + + def get_bindings(%{pid: pid}) do + Handler.get_bindings(pid) + end + def set_compressor(%{pid: pid}, compressor) do Handler.set_compressor(pid, compressor) end diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index f135ec82..057b9774 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -19,7 +19,6 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do path = :cowboy_req.path(req) with {:ok, codec} <- find_codec(req, server), - # can be nil {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ server: server, @@ -141,6 +140,14 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do sync_call(pid, :get_cert) end + def get_qs(pid) do + sync_call(pid, :get_qs) + end + + def get_bindings(pid) do + sync_call(pid, :get_bindings) + end + defp sync_call(pid, key) do ref = make_ref() send(pid, {key, ref, self()}) @@ -216,6 +223,18 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, req, state} end + def info({:get_qs, ref, pid}, req, state) do + qs = :cowboy_req.qs(req) + send(pid, {ref, qs}) + {:ok, req, state} + end + + def info({:get_bindings, ref, pid}, req, state) do + bindings = :cowboy_req.bindings(req) + send(pid, {ref, bindings}) + {:ok, req, state} + end + # Handle http/json transcoded response def info({:stream_body, data, _opts, is_fin, _http_transcode = true}, req, state) do # TODO Compress diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 8601086c..d11cffe6 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,4 +1,14 @@ defmodule GRPC.Server.Transcode do + + @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} + def map_request(body_request, path_bindings, _query_string, req_mod) do + path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) + + with {:ok, path_request} <- Protobuf.JSON.from_decoded(path_bindings, req_mod) do + {:ok, Map.merge(body_request, path_request)} + end + end + @spec path(term()) :: String.t() def path(%{pattern: {_method, path}}) do path diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 093cf2c4..f91bf205 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -255,11 +255,25 @@ defmodule GRPC.Integration.ServerTest do describe "http/json transcode" do test "can transcode path params" do run_server([FeatureTranscodeServer], fn port -> + latitude = 10 + longitude = 20 + {:ok, conn_pid} = :gun.open('localhost', port) - {:ok, conn_pid} = :gun.open('localhost', port) - stream_ref = :gun.get(conn_pid, "/v1/feature/1/2") + + stream_ref = + :gun.get(conn_pid, "/v1/feature/#{latitude}/#{longitude}", [ + {"content-type", "application/json"} + ]) assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{ + "location" => %{"latitude" => ^latitude, "longitude" => ^longitude}, + "name" => name + } = Jason.decode!(body) + + assert name == "#{latitude},#{longitude}" end) end end From 1e18a038ac2eed1e1b3cb4240dc22d00d5434027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 16:34:40 +0200 Subject: [PATCH 16/73] http rule template lexer / parse rename literal -> identifier --- lib/grpc/server/transcode.ex | 46 +++++++++++++----------------------- test/grpc/transcode_test.exs | 28 +++++++++++----------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index d11cffe6..a58a034c 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,5 +1,4 @@ defmodule GRPC.Server.Transcode do - @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request(body_request, path_bindings, _query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) @@ -9,18 +8,8 @@ defmodule GRPC.Server.Transcode do end end - @spec path(term()) :: String.t() - def path(%{pattern: {_method, path}}) do - path - end - - @spec method(term()) :: String.t() - def method(%{pattern: {method, _path}}) do - method - end - @spec to_path(term()) :: String.t() - def to_path({method, {_bindings, segments}} = _spec) do + def to_path({_method, {_bindings, segments}} = _spec) do match = segments |> Enum.map(&segment_to_string/1) @@ -32,16 +21,15 @@ defmodule GRPC.Server.Transcode do defp segment_to_string({binding, _}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" defp segment_to_string(segment), do: segment - @doc """ - https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule + # https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule - Template = "/" Segments [ Verb ] ; - Segments = Segment { "/" Segment } ; - Segment = "*" | "**" | LITERAL | Variable ; - Variable = "{" FieldPath [ "=" Segments ] "}" ; - FieldPath = IDENT { "." IDENT } ; - Verb = ":" LITERAL ; - """ + # Template = "/" Segments [ Verb ] ; + # Segments = Segment { "/" Segment } ; + # Segment = "*" | "**" | LITERAL | Variable ; + # Variable = "{" FieldPath [ "=" Segments ] "}" ; + # FieldPath = IDENT { "." IDENT } ; + # Verb = ":" LITERAL ; + # @spec build_route(term()) :: tuple() def build_route(%Google.Api.HttpRule{pattern: {method, path}}) do route = @@ -71,7 +59,7 @@ defmodule GRPC.Server.Transcode do end defp do_tokenize(<> = rest, acc) when h in @terminals do - {{:literal, acc, []}, rest} + {{:identifier, acc, []}, rest} end defp do_tokenize(<>, acc) @@ -80,7 +68,7 @@ defmodule GRPC.Server.Transcode do end defp do_tokenize(<<>>, acc) do - {{:literal, acc, []}, <<>>} + {{:identifier, acc, []}, <<>>} end @spec parse(list(tuple()), list(), list()) :: list() @@ -96,8 +84,8 @@ defmodule GRPC.Server.Transcode do parse(rest, params, [{:_, []} | segments]) end - def parse([{:literal, literal, _} | rest], params, segments) do - parse(rest, params, [literal | segments]) + def parse([{:identifier, identifier, _} | rest], params, segments) do + parse(rest, params, [identifier | segments]) end def parse([{:"{", _} | rest], params, segments) do @@ -110,18 +98,18 @@ defmodule GRPC.Server.Transcode do end defp parse_binding( - [{:literal, lit, _}, {:=, _}, {:literal, assign, _} = a | rest], + [{:identifier, id, _}, {:=, _}, {:identifier, assign, _} | rest], params, segments ) do - {variable, _} = param = field_path(lit) + {variable, _} = param = field_path(id) # assign = field_path(assign) parse_binding(rest, [param | params], [{variable, [assign]} | segments]) end - defp parse_binding([{:literal, lit, []} | rest], params, segments) do - {variable, _} = param = field_path(lit) + defp parse_binding([{:identifier, id, []} | rest], params, segments) do + {variable, _} = param = field_path(id) parse_binding(rest, [param | params], [{variable, []} | segments]) end diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs index 8a711eea..bcb126a2 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/transcode_test.exs @@ -25,16 +25,16 @@ defmodule GRPC.TranscodeTest do test "can tokenize simple paths" do assert [{:/, []}] = Transcode.tokenize("/") - assert [{:/, []}, {:literal, "v1", []}, {:/, []}, {:literal, "messages", []}] = + assert [{:/, []}, {:identifier, "v1", []}, {:/, []}, {:identifier, "messages", []}] = Transcode.tokenize("/v1/messages") end test "can tokenize simple paths with wildcards" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:/, []}, {:*, []} ] == Transcode.tokenize("/v1/messages/*") @@ -43,12 +43,12 @@ defmodule GRPC.TranscodeTest do test "can tokenize simple variables" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:/, []}, {:"{", []}, - {:literal, "message_id", []}, + {:identifier, "message_id", []}, {:"}", []} ] == Transcode.tokenize("/v1/messages/{message_id}") end @@ -56,12 +56,12 @@ defmodule GRPC.TranscodeTest do test "can tokenize variable assignments in bindings" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, {:"{", []}, - {:literal, "name", []}, + {:identifier, "name", []}, {:=, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:"}", []} ] == Transcode.tokenize("/v1/{name=messages}") end @@ -69,16 +69,16 @@ defmodule GRPC.TranscodeTest do test "can tokenize field paths in bindings" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:/, []}, {:"{", []}, - {:literal, "message_id", []}, + {:identifier, "message_id", []}, {:"}", []}, {:/, []}, {:"{", []}, - {:literal, "sub.subfield", []}, + {:identifier, "sub.subfield", []}, {:"}", []} ] == Transcode.tokenize("/v1/messages/{message_id}/{sub.subfield}") end @@ -92,7 +92,7 @@ defmodule GRPC.TranscodeTest do |> Transcode.parse([], []) end - test "can parse paths with literals" do + test "can parse paths with identifiers" do assert {[], ["v1", "messages"]} == "/v1/messages" |> Transcode.tokenize() From adf26adb941fa6229875828db0ea0a50a732eda0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 16:35:11 +0200 Subject: [PATCH 17/73] fix warnings --- lib/grpc/codec/json.ex | 1 - lib/grpc/server.ex | 7 ------- lib/grpc/server/adapters/cowboy.ex | 7 +++++-- lib/grpc/server/adapters/cowboy/handler.ex | 6 +++--- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 0a32bf53..9f40c0c0 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -9,7 +9,6 @@ defmodule GRPC.Codec.JSON do Protobuf.JSON.encode!(struct) end - def decode(<<>>, module) do struct(module, []) end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index b51c9352..ab0f3ff8 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -436,13 +436,6 @@ defmodule GRPC.Server do stream end - @doc false - # @spec service_name(String.t()) :: String.t() - # def service_name(path) do - # ["", name | _] = String.split(path, "/") - # name - # end - @doc false @spec servers_to_map(module() | [module()]) :: %{String.t() => [module()]} def servers_to_map(servers) do diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 122176c2..359fba48 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -170,13 +170,15 @@ defmodule GRPC.Server.Adapters.Cowboy do end defp build_route({:grpc, path}, endpoint, server, opts) do - {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, + {endpoint, server, path, Enum.into(opts, %{})}} end defp build_route({:http_transcode, spec}, endpoint, server, opts) do path = GRPC.Server.Transcode.to_path(spec) - {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, + {endpoint, server, path, Enum.into(opts, %{})}} end defp cowboy_start_args(endpoint, servers, port, opts) do @@ -186,6 +188,7 @@ defmodule GRPC.Server.Adapters.Cowboy do status_handler = Keyword.get(adapter_opts, :status_handler) handlers = build_handlers(endpoint, servers, opts) + handlers = if status_handler do [status_handler | handlers] diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 057b9774..b02a82ab 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -13,11 +13,11 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do @spec init( map(), - state :: {endpoint :: atom(), server :: {String.t(), module()}, route :: String.t(), opts :: keyword()} + state :: + {endpoint :: atom(), server :: {String.t(), module()}, route :: String.t(), + opts :: keyword()} ) :: {:cowboy_loop, map(), map()} def init(req, {endpoint, {_name, server}, route, opts} = state) do - path = :cowboy_req.path(req) - with {:ok, codec} <- find_codec(req, server), {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ From de8d798053b03917991081a94ff8b50795558666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 14 Sep 2022 10:54:29 +0200 Subject: [PATCH 18/73] fix! merge request params before decoding into struct --- lib/grpc/codec/json.ex | 14 ++++++++++---- lib/grpc/server/transcode.ex | 8 ++++---- test/grpc/{ => server}/transcode_test.exs | 13 +++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) rename test/grpc/{ => server}/transcode_test.exs (89%) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 9f40c0c0..91c5c9f7 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -9,11 +9,17 @@ defmodule GRPC.Codec.JSON do Protobuf.JSON.encode!(struct) end - def decode(<<>>, module) do - struct(module, []) + def decode(<<>>, _module) do + %{} end - def decode(binary, module) do - Protobuf.JSON.decode!(binary, module) + def decode(binary, _module) do + if jason = load_jason() do + jason.decode!(binary) + else + raise "`:jason` library not loaded" + end end + + defp load_jason, do: Code.ensure_loaded?(Jason) and Jason end diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index a58a034c..d269c0e4 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,11 +1,11 @@ defmodule GRPC.Server.Transcode do + @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request(body_request, path_bindings, _query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) + request = Map.merge(path_bindings, body_request) - with {:ok, path_request} <- Protobuf.JSON.from_decoded(path_bindings, req_mod) do - {:ok, Map.merge(body_request, path_request)} - end + Protobuf.JSON.from_decoded(request, req_mod) end @spec to_path(term()) :: String.t() @@ -71,7 +71,7 @@ defmodule GRPC.Server.Transcode do {{:identifier, acc, []}, <<>>} end - @spec parse(list(tuple()), list(), list()) :: list() + @spec parse(list(tuple()), list(), list()) :: {list(), list()} def parse([], params, segments) do {Enum.reverse(params), Enum.reverse(segments)} end diff --git a/test/grpc/transcode_test.exs b/test/grpc/server/transcode_test.exs similarity index 89% rename from test/grpc/transcode_test.exs rename to test/grpc/server/transcode_test.exs index bcb126a2..8865b4b9 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -2,6 +2,19 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true alias GRPC.Server.Transcode + test "map_requests/3 can map request body to protobuf struct" do + body_request = %{"latitude" => 1, "longitude" => 2} + {:ok, request} = Transcode.map_request(body_request, %{}, "", Routeguide.Point) + assert Routeguide.Point.new(latitude: 1, longitude: 2) == request + end + + test "map_requests/3 can merge request body with path bindings to protobuf struct" do + body_request = %{"latitude" => 1} + bindings = %{"longitude" => 2} + {:ok, request} = Transcode.map_request(body_request, bindings, "", Routeguide.Point) + assert Routeguide.Point.new(latitude: 1, longitude: 2) == request + end + test "build_route/1 returns a route with {http_method, route} based on the http rule" do rule = build_simple_rule(:get, "/v1/messages/{message_id}") assert {:get, {params, segments}} = Transcode.build_route(rule) From f9ff4ab8c951590b7115a4a7a3543c31eef617c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 14 Sep 2022 10:55:00 +0200 Subject: [PATCH 19/73] update example curl requests --- examples/helloworld/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index 09da4801..e3c2b345 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -21,10 +21,10 @@ $ mix run priv/client.exs ``` shell # Say hello -curl http://localhost:50051/v1/greeter/test +curl -H 'Content-type: application/json' http://localhost:50051/v1/greeter/test # Say hello from -curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' --output - http://localhost:50051/v1/greeter +curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' http://localhost:50051/v1/greeter ``` ## Regenerate Elixir code from proto From e9747e9e0f661ddb3216df9fe841128e9ba6573a Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Wed, 14 Sep 2022 08:01:10 -0300 Subject: [PATCH 20/73] Fix typo in filenames --- .../{route_guide_transocde.pb.ex => route_guide_transcode.pb.ex} | 0 .../{route_guide_transocde.proto => route_guide_transcode.proto} | 0 ...{route_guide_transocde.svc.ex => route_guide_transcode.svc.ex} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename test/support/{route_guide_transocde.pb.ex => route_guide_transcode.pb.ex} (100%) rename test/support/{route_guide_transocde.proto => route_guide_transcode.proto} (100%) rename test/support/{route_guide_transocde.svc.ex => route_guide_transcode.svc.ex} (100%) diff --git a/test/support/route_guide_transocde.pb.ex b/test/support/route_guide_transcode.pb.ex similarity index 100% rename from test/support/route_guide_transocde.pb.ex rename to test/support/route_guide_transcode.pb.ex diff --git a/test/support/route_guide_transocde.proto b/test/support/route_guide_transcode.proto similarity index 100% rename from test/support/route_guide_transocde.proto rename to test/support/route_guide_transcode.proto diff --git a/test/support/route_guide_transocde.svc.ex b/test/support/route_guide_transcode.svc.ex similarity index 100% rename from test/support/route_guide_transocde.svc.ex rename to test/support/route_guide_transcode.svc.ex From 00588c6d1bcfefde0898d6978a28405605e59d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 8 Sep 2022 14:46:25 +0200 Subject: [PATCH 21/73] minimal working example --- lib/grpc/codec/json.ex | 15 +++++ lib/grpc/server.ex | 75 +++++++++++++++++++--- lib/grpc/server/adapters/cowboy/handler.ex | 38 ++++++----- lib/grpc/server/http_transcode.ex | 11 ++++ lib/grpc/server/stream.ex | 7 ++ lib/grpc/service.ex | 15 +++-- lib/grpc/stub.ex | 3 +- mix.exs | 1 + 8 files changed, 133 insertions(+), 32 deletions(-) create mode 100644 lib/grpc/codec/json.ex create mode 100644 lib/grpc/server/http_transcode.ex diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex new file mode 100644 index 00000000..078bb883 --- /dev/null +++ b/lib/grpc/codec/json.ex @@ -0,0 +1,15 @@ +defmodule GRPC.Codec.JSON do + @behaviour GRPC.Codec + + def name() do + "json" + end + + def encode(struct) do + Protobuf.JSON.encode!(struct) + end + + def decode(binary, module) do + Protobuf.JSON.decode!(binary, module) + end +end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 0cbed721..95d73850 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -34,6 +34,7 @@ defmodule GRPC.Server do require Logger alias GRPC.RPCError + alias GRPC.Server.HTTPTranscode @type rpc_req :: struct | Enumerable.t() @type rpc_return :: struct | any @@ -43,10 +44,12 @@ defmodule GRPC.Server do quote bind_quoted: [opts: opts], location: :keep do service_mod = opts[:service] service_name = service_mod.__meta__(:name) - codecs = opts[:codecs] || [GRPC.Codec.Proto, GRPC.Codec.WebText] + codecs = opts[:codecs] || [GRPC.Codec.Proto, GRPC.Codec.WebText, GRPC.Codec.JSON] compressors = opts[:compressors] || [] + http_transcode = opts[:http_transcode] || false - Enum.each(service_mod.__rpc_calls__, fn {name, _, _} = rpc -> + Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> + IO.inspect(options, pretty: true) func_name = name |> to_string |> Macro.underscore() |> String.to_atom() path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) @@ -64,12 +67,47 @@ defmodule GRPC.Server do unquote(func_name) ) end + + if http_transcode and Map.has_key?(options, :http) do + %{value: http_opts} = Map.fetch!(options, :http) + + http_path = HTTPTranscode.path(http_opts) + http_method = HTTPTranscode.method(http_opts) + + def __call_rpc__(unquote(http_path), stream) do + GRPC.Server.call( + unquote(service_mod), + %{ + stream + | service_name: unquote(service_name), + method_name: unquote(to_string(name)), + grpc_type: unquote(grpc_type), + http_method: unquote(http_method), + http_transcode: unquote(http_transcode) + }, + unquote(Macro.escape(put_elem(rpc, 0, func_name))), + unquote(func_name) + ) + end + + def service_name(unquote(http_path)) do + unquote(service_name) + end + end + + def service_name(unquote(path)) do + unquote(service_name) + end end) def __call_rpc__(_, stream) do raise GRPC.RPCError, status: :unimplemented end + def service_name(_) do + "" + end + def __meta__(:service), do: unquote(service_mod) def __meta__(:codecs), do: unquote(codecs) def __meta__(:compressors), do: unquote(compressors) @@ -82,7 +120,7 @@ defmodule GRPC.Server do def call( _service_mod, stream, - {_, {req_mod, req_stream}, {res_mod, res_stream}} = rpc, + {_, {req_mod, req_stream}, {res_mod, res_stream}, _options} = rpc, func_name ) do request_id = generate_request_id() @@ -116,6 +154,27 @@ defmodule GRPC.Server do end end + defp do_handle_request( + false, + res_stream, + %{ + request_mod: req_mod, + codec: codec, + adapter: adapter, + payload: payload, + http_transcode: true + } = stream, + func_name + ) do + {:ok, data} = adapter.read_body(payload) + request = codec.decode(data, req_mod) + Logger.debug(fn -> + "http transcode request #{inspect(request)}" + end) + + call_with_interceptors(res_stream, func_name, stream, request) + end + defp do_handle_request( false, res_stream, @@ -353,11 +412,11 @@ defmodule GRPC.Server do end @doc false - @spec service_name(String.t()) :: String.t() - def service_name(path) do - ["", name | _] = String.split(path, "/") - name - end + # @spec service_name(String.t()) :: String.t() + # def service_name(path) do + # ["", name | _] = String.split(path, "/") + # name + # end @doc false @spec servers_to_map(module() | [module()]) :: %{String.t() => [module()]} diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index f0f5641b..b6498b60 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -18,6 +18,10 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do def init(req, {endpoint, servers, opts} = state) do path = :cowboy_req.path(req) + Logger.info(fn -> + "path: #{path}" + end) + with {:ok, server} <- find_server(servers, path), {:ok, codec} <- find_codec(req, server), # can be nil @@ -51,37 +55,38 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:cowboy_loop, req, %{pid: pid, handling_timer: timer_ref, pending_reader: nil}} else {:error, error} -> + Logger.error(fn -> inspect(error) end) trailers = HTTP2.server_trailers(error.status, error.message) req = send_error_trailers(req, trailers) {:ok, req, state} end end + # TODO compile routes instead of dynamic dispatch to find which server has + # which route defp find_server(servers, path) do - case Map.fetch(servers, GRPC.Server.service_name(path)) do - s = {:ok, _} -> - s - - _ -> - {:error, RPCError.exception(status: :unimplemented)} + case Enum.find(servers, fn {_name, server} -> server.service_name(path) != "" end) do + nil -> {:error, RPCError.exception(status: :unimplemented)} + {_, server} -> {:ok, server} end end defp find_codec(req, server) do req_content_type = :cowboy_req.header("content-type", req) - {:ok, subtype} = extract_subtype(req_content_type) - codec = Enum.find(server.__meta__(:codecs), nil, fn c -> c.name() == subtype end) - - if codec do + with {:ok, subtype} <- extract_subtype(req_content_type), + codec when not is_nil(codec) <- + Enum.find(server.__meta__(:codecs), nil, fn c -> c.name() == subtype end) do {:ok, codec} else - # TODO: Send grpc-accept-encoding header - {:error, - RPCError.exception( - status: :unimplemented, - message: "No codec registered for content-type #{req_content_type}" - )} + err -> + Logger.error(fn -> inspect(err) end) + + {:error, + RPCError.exception( + status: :unimplemented, + message: "No codec registered for content-type #{req_content_type}" + )} end end @@ -444,6 +449,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end + defp extract_subtype("application/json"), do: {:ok, "json"} defp extract_subtype("application/grpc"), do: {:ok, "proto"} defp extract_subtype("application/grpc+"), do: {:ok, "proto"} defp extract_subtype("application/grpc;"), do: {:ok, "proto"} diff --git a/lib/grpc/server/http_transcode.ex b/lib/grpc/server/http_transcode.ex new file mode 100644 index 00000000..2b1813e7 --- /dev/null +++ b/lib/grpc/server/http_transcode.ex @@ -0,0 +1,11 @@ +defmodule GRPC.Server.HTTPTranscode do + @spec path(term()) :: String.t() + def path(%{pattern: {_method, path}}) do + path + end + + @spec method(term()) :: String.t() + def method(%{pattern: {method, _path}}) do + method + end +end diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index bd349722..d1f7ff4f 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -34,6 +34,9 @@ defmodule GRPC.Server.Stream do # compressor mainly is used in client decompressing, responses compressing should be set by # `GRPC.Server.set_compressor` compressor: module() | nil, + # For http transcoding + http_method: :get | :post | :put | :patch | :delete, + http_transcode: boolean(), __interface__: map() } @@ -51,11 +54,15 @@ defmodule GRPC.Server.Stream do adapter: nil, local: nil, compressor: nil, + http_method: :post, + http_transcode: false, __interface__: %{send_reply: &__MODULE__.send_reply/3} def send_reply(%{adapter: adapter, codec: codec} = stream, reply, opts) do # {:ok, data, _size} = reply |> codec.encode() |> GRPC.Message.to_data() data = codec.encode(reply) + IO.inspect(data, label: "#{__MODULE__}.send_reply data") + IO.inspect(stream.payload, label: "#{__MODULE__}.send_reply payload") adapter.send_reply(stream.payload, data, Keyword.put(opts, :codec, codec)) stream end diff --git a/lib/grpc/service.ex b/lib/grpc/service.ex index 5357b6c2..67ad1596 100644 --- a/lib/grpc/service.ex +++ b/lib/grpc/service.ex @@ -15,7 +15,7 @@ defmodule GRPC.Service do defmacro __using__(opts) do quote do - import GRPC.Service, only: [rpc: 3, stream: 1] + import GRPC.Service, only: [rpc: 4, stream: 1] Module.register_attribute(__MODULE__, :rpc_calls, accumulate: true) @before_compile GRPC.Service @@ -32,9 +32,10 @@ defmodule GRPC.Service do end end - defmacro rpc(name, request, reply) do + defmacro rpc(name, request, reply, options) do quote do - @rpc_calls {unquote(name), unquote(wrap_stream(request)), unquote(wrap_stream(reply))} + @rpc_calls {unquote(name), unquote(wrap_stream(request)), unquote(wrap_stream(reply)), + unquote(options)} end end @@ -54,8 +55,8 @@ defmodule GRPC.Service do quote do: {unquote(param), false} end - def grpc_type({_, {_, false}, {_, false}}), do: :unary - def grpc_type({_, {_, true}, {_, false}}), do: :client_stream - def grpc_type({_, {_, false}, {_, true}}), do: :server_stream - def grpc_type({_, {_, true}, {_, true}}), do: :bidi_stream + def grpc_type({_, {_, false}, {_, false}, _}), do: :unary + def grpc_type({_, {_, true}, {_, false}, _}), do: :client_stream + def grpc_type({_, {_, false}, {_, true}, _}), do: :server_stream + def grpc_type({_, {_, true}, {_, true}, _}), do: :bidi_stream end diff --git a/lib/grpc/stub.ex b/lib/grpc/stub.ex index db050c0a..07f48484 100644 --- a/lib/grpc/stub.ex +++ b/lib/grpc/stub.ex @@ -62,7 +62,8 @@ defmodule GRPC.Stub do service_mod = opts[:service] service_name = service_mod.__meta__(:name) - Enum.each(service_mod.__rpc_calls__, fn {name, {_, req_stream}, {_, res_stream}} = rpc -> + Enum.each(service_mod.__rpc_calls__, fn {name, {_, req_stream}, {_, res_stream}, _options} = + rpc -> func_name = name |> to_string |> Macro.underscore() path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) diff --git a/mix.exs b/mix.exs index af6eed0b..476ff26d 100644 --- a/mix.exs +++ b/mix.exs @@ -43,6 +43,7 @@ defmodule GRPC.Mixfile do # This is the same as :gun 2.0.0-rc.2, # but we can't depend on an RC for releases {:gun, "~> 2.0.1", hex: :grpc_gun}, + {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, {:protobuf, "~> 0.10", only: [:dev, :test]}, {:ex_doc, "~> 0.28.0", only: :dev}, From 028256386aeea048975bebeae80ad129ac654b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 8 Sep 2022 15:55:18 +0200 Subject: [PATCH 22/73] return correct content-type when codec is JSON --- lib/grpc/transport/http2.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/grpc/transport/http2.ex b/lib/grpc/transport/http2.ex index dd0eea40..0c41566c 100644 --- a/lib/grpc/transport/http2.ex +++ b/lib/grpc/transport/http2.ex @@ -12,6 +12,10 @@ defmodule GRPC.Transport.HTTP2 do %{"content-type" => "application/grpc-web-#{codec.name()}"} end + def server_headers(%{codec: GRPC.Codec.JSON = codec}) do + %{"content-type" => "application/#{codec.name()}"} + end + def server_headers(%{codec: codec}) do %{"content-type" => "application/grpc+#{codec.name()}"} end From 0a34f69215e44f8589152155e5377fb47cab9499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 8 Sep 2022 16:02:38 +0200 Subject: [PATCH 23/73] support default rpc options arg --- lib/grpc/server.ex | 5 +---- lib/grpc/server/stream.ex | 2 -- lib/grpc/service.ex | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 95d73850..a699934a 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -49,7 +49,6 @@ defmodule GRPC.Server do http_transcode = opts[:http_transcode] || false Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> - IO.inspect(options, pretty: true) func_name = name |> to_string |> Macro.underscore() |> String.to_atom() path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) @@ -166,11 +165,9 @@ defmodule GRPC.Server do } = stream, func_name ) do + IO.inspect(res_stream, label: "do_handle_request") {:ok, data} = adapter.read_body(payload) request = codec.decode(data, req_mod) - Logger.debug(fn -> - "http transcode request #{inspect(request)}" - end) call_with_interceptors(res_stream, func_name, stream, request) end diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index d1f7ff4f..6111e81d 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -61,8 +61,6 @@ defmodule GRPC.Server.Stream do def send_reply(%{adapter: adapter, codec: codec} = stream, reply, opts) do # {:ok, data, _size} = reply |> codec.encode() |> GRPC.Message.to_data() data = codec.encode(reply) - IO.inspect(data, label: "#{__MODULE__}.send_reply data") - IO.inspect(stream.payload, label: "#{__MODULE__}.send_reply payload") adapter.send_reply(stream.payload, data, Keyword.put(opts, :codec, codec)) stream end diff --git a/lib/grpc/service.ex b/lib/grpc/service.ex index 67ad1596..39b7edd9 100644 --- a/lib/grpc/service.ex +++ b/lib/grpc/service.ex @@ -15,7 +15,7 @@ defmodule GRPC.Service do defmacro __using__(opts) do quote do - import GRPC.Service, only: [rpc: 4, stream: 1] + import GRPC.Service, only: [rpc: 4, rpc: 3, stream: 1] Module.register_attribute(__MODULE__, :rpc_calls, accumulate: true) @before_compile GRPC.Service @@ -32,7 +32,7 @@ defmodule GRPC.Service do end end - defmacro rpc(name, request, reply, options) do + defmacro rpc(name, request, reply, options \\ quote(do: %{})) do quote do @rpc_calls {unquote(name), unquote(wrap_stream(request)), unquote(wrap_stream(reply)), unquote(options)} From d128b000ddb3b75b7956244aaab7c94581ea462a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 10:20:00 +0200 Subject: [PATCH 24/73] add custom protoc plugin --- lib/grpc/protoc/cli.ex | 226 +++++++++++++++++++++++++++ lib/grpc/protoc/generator.ex | 68 ++++++++ lib/grpc/protoc/generator/service.ex | 71 +++++++++ mix.exs | 62 +++++++- mix.lock | 3 +- priv/templates/service.ex.eex | 24 +++ 6 files changed, 451 insertions(+), 3 deletions(-) create mode 100644 lib/grpc/protoc/cli.ex create mode 100644 lib/grpc/protoc/generator.ex create mode 100644 lib/grpc/protoc/generator/service.ex create mode 100644 priv/templates/service.ex.eex diff --git a/lib/grpc/protoc/cli.ex b/lib/grpc/protoc/cli.ex new file mode 100644 index 00000000..84aa2215 --- /dev/null +++ b/lib/grpc/protoc/cli.ex @@ -0,0 +1,226 @@ +defmodule GRPC.Protoc.CLI do + @moduledoc """ + `protoc` plugin for generating Elixir code. + + `protoc-gen-elixir` (this name is important) **must** be in `$PATH`. You are not supposed + to call it directly, but only through `protoc`. + + ## Examples + + $ protoc --elixir_out=./lib your.proto + $ protoc --elixir_out=plugins=grpc:./lib/ *.proto + $ protoc -I protos --elixir_out=./lib protos/namespace/*.proto + + Options: + + * --version Print version of protobuf-elixir + * --help (-h) Print this help + + """ + + alias Protobuf.Protoc.Context + + # Entrypoint for the escript (protoc-gen-elixir). + @doc false + @spec main([String.t()]) :: :ok + def main(args) + + def main(["--version"]) do + {:ok, version} = :application.get_key(:protobuf, :vsn) + IO.puts(version) + end + + def main([opt]) when opt in ["--help", "-h"] do + IO.puts(@moduledoc) + end + + # When called through protoc, all input is passed through stdin. + def main([] = _args) do + Protobuf.load_extensions() + + # See https://groups.google.com/forum/#!topic/elixir-lang-talk/T5enez_BBTI. + :io.setopts(:standard_io, encoding: :latin1) + + # Read the standard input that protoc feeds us. + bin = binread_all!(:stdio) + + request = Protobuf.Decoder.decode(bin, Google.Protobuf.Compiler.CodeGeneratorRequest) + + ctx = + %Context{} + |> parse_params(request.parameter || "") + |> find_types(request.proto_file, request.file_to_generate) + + files = + Enum.flat_map(request.file_to_generate, fn file -> + desc = Enum.find(request.proto_file, &(&1.name == file)) + GRPC.Protoc.Generator.generate(ctx, desc) + end) + + Google.Protobuf.Compiler.CodeGeneratorResponse.new( + file: files, + supported_features: supported_features() + ) + |> Protobuf.encode_to_iodata() + |> IO.binwrite() + end + + def main(_args) do + raise "invalid arguments. See protoc-gen-elixir --help." + end + + def supported_features() do + # The only available feature is proto3 with optional fields. + # This is backwards compatible with proto2 optional fields. + Google.Protobuf.Compiler.CodeGeneratorResponse.Feature.value(:FEATURE_PROTO3_OPTIONAL) + end + + # Made public for testing. + @doc false + def parse_params(%Context{} = ctx, params_str) when is_binary(params_str) do + params_str + |> String.split(",") + |> Enum.reduce(ctx, &parse_param/2) + end + + defp parse_param("plugins=" <> plugins, ctx) do + %Context{ctx | plugins: String.split(plugins, "+")} + end + + defp parse_param("gen_descriptors=" <> value, ctx) do + case value do + "true" -> + %Context{ctx | gen_descriptors?: true} + + other -> + raise "invalid value for gen_descriptors option, expected \"true\", got: #{inspect(other)}" + end + end + + defp parse_param("package_prefix=" <> package, ctx) do + if package == "" do + raise "package_prefix can't be empty" + else + %Context{ctx | package_prefix: package} + end + end + + defp parse_param("transform_module=" <> module, ctx) do + %Context{ctx | transform_module: Module.concat([module])} + end + + defp parse_param("one_file_per_module=" <> value, ctx) do + case value do + "true" -> + %Context{ctx | one_file_per_module?: true} + + other -> + raise "invalid value for one_file_per_module option, expected \"true\", got: #{inspect(other)}" + end + end + + defp parse_param("include_docs=" <> value, ctx) do + case value do + "true" -> + %Context{ctx | include_docs?: true} + + other -> + raise "invalid value for include_docs option, expected \"true\", got: #{inspect(other)}" + end + end + + defp parse_param(_unknown, ctx) do + ctx + end + + # Made public for testing. + @doc false + @spec find_types(Context.t(), [Google.Protobuf.FileDescriptorProto.t()], [String.t()]) :: + Context.t() + def find_types(%Context{} = ctx, descs, files_to_generate) + when is_list(descs) and is_list(files_to_generate) do + global_type_mapping = + Map.new(descs, fn %Google.Protobuf.FileDescriptorProto{name: filename} = desc -> + {filename, find_types_in_proto(ctx, desc, files_to_generate)} + end) + + %Context{ctx | global_type_mapping: global_type_mapping} + end + + defp find_types_in_proto( + %Context{} = ctx, + %Google.Protobuf.FileDescriptorProto{} = desc, + files_to_generate + ) do + # Only take package_prefix into consideration for files that we're directly generating. + package_prefix = + if desc.name in files_to_generate do + ctx.package_prefix + else + nil + end + + ctx = + %Protobuf.Protoc.Context{ + namespace: [], + package_prefix: package_prefix, + package: desc.package + } + |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc) + + find_types_in_descriptor(_types = %{}, ctx, desc.message_type ++ desc.enum_type) + end + + defp find_types_in_descriptor(types_acc, ctx, descs) when is_list(descs) do + Enum.reduce(descs, types_acc, &find_types_in_descriptor(_acc = &2, ctx, _desc = &1)) + end + + defp find_types_in_descriptor( + types_acc, + ctx, + %Google.Protobuf.DescriptorProto{name: name} = desc + ) do + new_ctx = update_in(ctx.namespace, &(&1 ++ [name])) + + types_acc + |> update_types(ctx, name) + |> find_types_in_descriptor(new_ctx, desc.enum_type) + |> find_types_in_descriptor(new_ctx, desc.nested_type) + end + + defp find_types_in_descriptor( + types_acc, + ctx, + %Google.Protobuf.EnumDescriptorProto{name: name} + ) do + update_types(types_acc, ctx, name) + end + + defp update_types(types, %Context{namespace: ns, package: pkg} = ctx, name) do + type_name = Protobuf.Protoc.Generator.Util.mod_name(ctx, ns ++ [name]) + + mapping_name = + ([pkg] ++ ns ++ [name]) + |> Enum.reject(&is_nil/1) + |> Enum.join(".") + + Map.put(types, "." <> mapping_name, %{type_name: type_name}) + end + + if Version.match?(System.version(), "~> 1.13") do + defp binread_all!(device) do + case IO.binread(device, :eof) do + data when is_binary(data) -> data + :eof -> _previous_behavior = "" + other -> raise "reading from #{inspect(device)} failed: #{inspect(other)}" + end + end + else + defp binread_all!(device) do + case IO.binread(device, :all) do + data when is_binary(data) -> data + other -> raise "reading from #{inspect(device)} failed: #{inspect(other)}" + end + end + end +end diff --git a/lib/grpc/protoc/generator.ex b/lib/grpc/protoc/generator.ex new file mode 100644 index 00000000..e7a3085a --- /dev/null +++ b/lib/grpc/protoc/generator.ex @@ -0,0 +1,68 @@ +defmodule GRPC.Protoc.Generator do + @moduledoc false + + alias Protobuf.Protoc.Context + alias Protobuf.Protoc.Generator + + @spec generate(Context.t(), %Google.Protobuf.FileDescriptorProto{}) :: + [Google.Protobuf.Compiler.CodeGeneratorResponse.File.t()] + def generate(%Context{} = ctx, %Google.Protobuf.FileDescriptorProto{} = desc) do + module_definitions = + ctx + |> generate_module_definitions(desc) + |> Enum.reject(&is_nil/1) + + if ctx.one_file_per_module? do + Enum.map(module_definitions, fn {mod_name, content} -> + file_name = Macro.underscore(mod_name) <> ".svc.ex" + + Google.Protobuf.Compiler.CodeGeneratorResponse.File.new( + name: file_name, + content: content + ) + end) + else + # desc.name is the filename, ending in ".proto". + file_name = Path.rootname(desc.name) <> ".svc.ex" + + content = + module_definitions + |> Enum.map(fn {_mod_name, contents} -> [contents, ?\n] end) + |> IO.iodata_to_binary() + |> Generator.Util.format() + + [ + Google.Protobuf.Compiler.CodeGeneratorResponse.File.new( + name: file_name, + content: content + ) + ] + end + end + + defp generate_module_definitions(ctx, %Google.Protobuf.FileDescriptorProto{} = desc) do + ctx = + %Context{ + ctx + | syntax: syntax(desc.syntax), + package: desc.package, + dep_type_mapping: get_dep_type_mapping(ctx, desc.dependency, desc.name) + } + |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc) + + Enum.map(desc.service, &GRPC.Protoc.Generator.Service.generate(ctx, &1)) + end + + defp get_dep_type_mapping(%Context{global_type_mapping: global_mapping}, deps, file_name) do + mapping = + Enum.reduce(deps, %{}, fn dep, acc -> + Map.merge(acc, global_mapping[dep]) + end) + + Map.merge(mapping, global_mapping[file_name]) + end + + defp syntax("proto3"), do: :proto3 + defp syntax("proto2"), do: :proto2 + defp syntax(nil), do: :proto2 +end diff --git a/lib/grpc/protoc/generator/service.ex b/lib/grpc/protoc/generator/service.ex new file mode 100644 index 00000000..f7aeea10 --- /dev/null +++ b/lib/grpc/protoc/generator/service.ex @@ -0,0 +1,71 @@ +defmodule GRPC.Protoc.Generator.Service do + @moduledoc false + + alias Protobuf.Protoc.Context + alias Protobuf.Protoc.Generator.Util + + require EEx + + EEx.function_from_file( + :defp, + :service_template, + Path.expand("./templates/service.ex.eex", :code.priv_dir(:grpc)), + [:assigns] + ) + + @spec generate(Context.t(), Google.Protobuf.ServiceDescriptorProto.t()) :: + {String.t(), String.t()} + def generate(%Context{} = ctx, %Google.Protobuf.ServiceDescriptorProto{} = desc) do + # service can't be nested + mod_name = Util.mod_name(ctx, [Macro.camelize(desc.name)]) + name = Util.prepend_package_prefix(ctx.package, desc.name) + methods = Enum.map(desc.method, &generate_service_method(ctx, &1)) + + descriptor_fun_body = + if ctx.gen_descriptors? do + Util.descriptor_fun_body(desc) + else + nil + end + + {mod_name, + Util.format( + service_template( + module: mod_name, + service_name: name, + methods: methods, + descriptor_fun_body: descriptor_fun_body, + version: Util.version(), + module_doc?: ctx.include_docs? + ) + )} + end + + defp generate_service_method(ctx, method) do + input = service_arg(Util.type_from_type_name(ctx, method.input_type), method.client_streaming) + + output = + service_arg(Util.type_from_type_name(ctx, method.output_type), method.server_streaming) + + options = + method.options + |> opts() + |> inspect(limit: :infinity) + + {method.name, input, output, options} + end + + defp service_arg(type, _streaming? = true), do: "stream(#{type})" + defp service_arg(type, _streaming?), do: type + + defp opts(%Google.Protobuf.MethodOptions{__pb_extensions__: extensions}) + when extensions == %{} do + %{} + end + + defp opts(%Google.Protobuf.MethodOptions{__pb_extensions__: extensions}) do + for {{type, field}, value} <- extensions, into: %{} do + {field, %{type: type, value: value}} + end + end +end diff --git a/mix.exs b/mix.exs index 476ff26d..4265275d 100644 --- a/mix.exs +++ b/mix.exs @@ -13,6 +13,8 @@ defmodule GRPC.Mixfile do start_permanent: Mix.env() == :prod, deps: deps(), package: package(), + escript: escript(), + aliases: aliases(), description: "The Elixir implementation of gRPC", docs: [ extras: ["README.md"], @@ -37,6 +39,10 @@ defmodule GRPC.Mixfile do [extra_applications: [:logger]] end + def escript do + [main_module: GRPC.Protoc.CLI, name: "protoc-gen-grpc_elixir"] + end + defp deps do [ {:cowboy, "~> 2.9"}, @@ -45,9 +51,15 @@ defmodule GRPC.Mixfile do {:gun, "~> 2.0.1", hex: :grpc_gun}, {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, - {:protobuf, "~> 0.10", only: [:dev, :test]}, + {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", only: [:dev, :test]}, {:ex_doc, "~> 0.28.0", only: :dev}, - {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false} + {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false}, + {:googleapis, + github: "googleapis/googleapis", + branch: "master", + app: false, + compile: false, + only: [:dev, :test]} ] end @@ -60,6 +72,52 @@ defmodule GRPC.Mixfile do } end + defp aliases do + [ + gen_bootstrap_protos: [&build_protobuf_escript/1, &gen_bootstrap_protos/1], + build_protobuf_escript: &build_protobuf_escript/1 + ] + end + defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] + + defp build_protobuf_escript(_args) do + path = Mix.Project.deps_paths().protobuf + + File.cd!(path, fn -> + with 0 <- Mix.shell().cmd("mix deps.get"), + 0 <- Mix.shell().cmd("mix escript.build") do + :ok + else + other -> + Mix.raise("build_protobuf_escript/1 exited with non-zero status: #{other}") + end + end) + end + # https://github.com/elixir-protobuf/protobuf/blob/cdf3acc53f619866b4921b8216d2531da52ceba7/mix.exs#L140 + defp gen_bootstrap_protos(_args) do + proto_src = Mix.Project.deps_paths().googleapis + + protoc!("-I \"#{proto_src}\"", "./lib", [ + "google/api/http.proto", + "google/api/annotations.proto" + ]) + end + + defp protoc!(args, elixir_out, files_to_generate) + when is_binary(args) and is_binary(elixir_out) and is_list(files_to_generate) do + args = + [ + ~s(protoc), + ~s(--plugin=./deps/protobuf/protoc-gen-elixir), + ~s(--elixir_out="#{elixir_out}"), + args + ] ++ files_to_generate + + case Mix.shell().cmd(Enum.join(args, " ")) do + 0 -> Mix.Task.rerun("format", [Path.join([elixir_out, "**", "*.pb.ex"])]) + other -> Mix.raise("'protoc' exited with non-zero status: #{other}") + end + end end diff --git a/mix.lock b/mix.lock index 43cfde44..3ff0ba5b 100644 --- a/mix.lock +++ b/mix.lock @@ -6,12 +6,13 @@ "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, + "googleapis": {:git, "https://github.com/googleapis/googleapis.git", "f0e2be46a5602ad903800811c9583f9e4458de3c", [branch: "master"]}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, - "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, + "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/priv/templates/service.ex.eex b/priv/templates/service.ex.eex new file mode 100644 index 00000000..bbea50b6 --- /dev/null +++ b/priv/templates/service.ex.eex @@ -0,0 +1,24 @@ +defmodule <%= @module %>.Service do + <%= unless @module_doc? do %> + @moduledoc false + <% end %> + use GRPC.Service, name: <%= inspect(@service_name) %>, protoc_gen_elixir_version: "<%= @version %>" + + <%= if @descriptor_fun_body do %> + def descriptor do + # credo:disable-for-next-line + <%= @descriptor_fun_body %> + end + <% end %> + + <%= for {method_name, input, output, options} <- @methods do %> + rpc :<%= method_name %>, <%= input %>, <%= output %>, <%= options %> + <% end %> +end + +defmodule <%= @module %>.Stub do + <%= unless @module_doc? do %> + @moduledoc false + <% end %> + use GRPC.Stub, service: <%= @module %>.Service +end From d6efc8e2054f8de47cb5d58cef4b150e14728a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 10:20:41 +0200 Subject: [PATCH 25/73] include api extensions --- lib/google/api/annotations.pb.ex | 8 ++++++ lib/google/api/http.pb.ex | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 lib/google/api/annotations.pb.ex create mode 100644 lib/google/api/http.pb.ex diff --git a/lib/google/api/annotations.pb.ex b/lib/google/api/annotations.pb.ex new file mode 100644 index 00000000..374877d3 --- /dev/null +++ b/lib/google/api/annotations.pb.ex @@ -0,0 +1,8 @@ +defmodule Google.Api.PbExtension do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + extend Google.Protobuf.MethodOptions, :http, 72_295_728, + optional: true, + type: Google.Api.HttpRule +end diff --git a/lib/google/api/http.pb.ex b/lib/google/api/http.pb.ex new file mode 100644 index 00000000..25dd83ad --- /dev/null +++ b/lib/google/api/http.pb.ex @@ -0,0 +1,43 @@ +defmodule Google.Api.Http do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :rules, 1, repeated: true, type: Google.Api.HttpRule + + field :fully_decode_reserved_expansion, 2, + type: :bool, + json_name: "fullyDecodeReservedExpansion" +end + +defmodule Google.Api.HttpRule do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + oneof :pattern, 0 + + field :selector, 1, type: :string + field :get, 2, type: :string, oneof: 0 + field :put, 3, type: :string, oneof: 0 + field :post, 4, type: :string, oneof: 0 + field :delete, 5, type: :string, oneof: 0 + field :patch, 6, type: :string, oneof: 0 + field :custom, 8, type: Google.Api.CustomHttpPattern, oneof: 0 + field :body, 7, type: :string + field :response_body, 12, type: :string, json_name: "responseBody" + + field :additional_bindings, 11, + repeated: true, + type: Google.Api.HttpRule, + json_name: "additionalBindings" +end + +defmodule Google.Api.CustomHttpPattern do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :kind, 1, type: :string + field :path, 2, type: :string +end From 8f318cb2f1e2623a28000db1d51cd08076f0dd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 12:01:20 +0200 Subject: [PATCH 26/73] include http options in helloworld example --- examples/helloworld/README.md | 14 +- examples/helloworld/lib/helloworld.pb.ex | 25 +- examples/helloworld/lib/helloworld.svc.ex | 39 ++ examples/helloworld/lib/server.ex | 25 +- examples/helloworld/mix.exs | 3 +- examples/helloworld/mix.lock | 3 +- .../priv/protos/google/api/annotations.proto | 31 ++ .../priv/protos/google/api/http.proto | 375 ++++++++++++++++++ .../helloworld/priv/protos/helloworld.proto | 22 +- 9 files changed, 515 insertions(+), 22 deletions(-) create mode 100644 examples/helloworld/lib/helloworld.svc.ex create mode 100644 examples/helloworld/priv/protos/google/api/annotations.proto create mode 100644 examples/helloworld/priv/protos/google/api/http.proto diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index 924681fd..09da4801 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -17,6 +17,16 @@ $ mix run --no-halt $ mix run priv/client.exs ``` +## HTTP Transcoding + +``` shell +# Say hello +curl http://localhost:50051/v1/greeter/test + +# Say hello from +curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' --output - http://localhost:50051/v1/greeter +``` + ## Regenerate Elixir code from proto 1. Modify the proto `priv/protos/helloworld.proto` @@ -26,8 +36,10 @@ $ mix run priv/client.exs mix escript.install hex protobuf ``` 4. Generate the code: + ```shell -$ protoc -I priv/protos --elixir_out=plugins=grpc:./lib/ priv/protos/helloworld.proto +$ (cd ../../; mix build_protobuf_escript && mix escript.build) +$ protoc -I priv/protos --elixir_out=:./lib/ --grpc_elixir_out=./lib --plugin="../../deps/protobuf/protoc-gen-elixir" --plugin="../../protoc-gen-grpc_elixir" priv/protos/helloworld.proto ``` Refer to [protobuf-elixir](https://github.com/tony612/protobuf-elixir#usage) for more information. diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index a8ff6dfa..bd78735a 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -1,26 +1,25 @@ defmodule Helloworld.HelloRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string end -defmodule Helloworld.HelloReply do +defmodule Helloworld.HelloRequestFrom do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - field :message, 1, type: :string - field :today, 2, type: Google.Protobuf.Timestamp + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string + field :from, 2, type: :string end -defmodule Helloworld.Greeter.Service do +defmodule Helloworld.HelloReply do @moduledoc false - use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.10.0" - rpc :SayHello, Helloworld.HelloRequest, Helloworld.HelloReply -end + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 -defmodule Helloworld.Greeter.Stub do - @moduledoc false - use GRPC.Stub, service: Helloworld.Greeter.Service -end + field :message, 1, type: :string + field :today, 2, type: Google.Protobuf.Timestamp +end \ No newline at end of file diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex new file mode 100644 index 00000000..0afcba8b --- /dev/null +++ b/examples/helloworld/lib/helloworld.svc.ex @@ -0,0 +1,39 @@ +defmodule Helloworld.Greeter.Service do + @moduledoc false + + use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.11.0" + + rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/greeter/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:SayHelloFrom, Helloworld.HelloRequestFrom, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "*", + pattern: {:post, "/v1/greeter"}, + response_body: "", + selector: "" + } + } + }) +end + +defmodule Helloworld.Greeter.Stub do + @moduledoc false + + use GRPC.Stub, service: Helloworld.Greeter.Service +end \ No newline at end of file diff --git a/examples/helloworld/lib/server.ex b/examples/helloworld/lib/server.ex index b85241f8..68c72c10 100644 --- a/examples/helloworld/lib/server.ex +++ b/examples/helloworld/lib/server.ex @@ -1,16 +1,31 @@ defmodule Helloworld.Greeter.Server do - use GRPC.Server, service: Helloworld.Greeter.Service + use GRPC.Server, + service: Helloworld.Greeter.Service, + http_transcode: true @spec say_hello(Helloworld.HelloRequest.t(), GRPC.Server.Stream.t()) :: Helloworld.HelloReply.t() def say_hello(request, _stream) do + Helloworld.HelloReply.new( + message: "Hello #{request.name}", + today: today() + ) + end + + @spec say_hello_from(Helloworld.HelloFromRequest.t(), GRPC.Server.Stream.t()) :: + Helloworld.HelloReply.t() + def say_hello_from(request, _stream) do + Helloworld.HelloReply.new( + message: "Hello #{request.name}. From #{request.from}", + today: today() + ) + end + + defp today do nanos_epoch = System.system_time() |> System.convert_time_unit(:native, :nanosecond) seconds = div(nanos_epoch, 1_000_000_000) nanos = nanos_epoch - seconds * 1_000_000_000 - Helloworld.HelloReply.new( - message: "Hello #{request.name}", - today: %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} - ) + %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} end end diff --git a/examples/helloworld/mix.exs b/examples/helloworld/mix.exs index 9bf4cc3e..ecc3867e 100644 --- a/examples/helloworld/mix.exs +++ b/examples/helloworld/mix.exs @@ -19,7 +19,8 @@ defmodule Helloworld.Mixfile do defp deps do [ {:grpc, path: "../../"}, - {:protobuf, "~> 0.10"}, + {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", override: true}, + {:jason, "~> 1.3.0"}, {:google_protos, "~> 0.3.0"}, {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false} ] diff --git a/examples/helloworld/mix.lock b/examples/helloworld/mix.lock index f96a70d1..62dcb34f 100644 --- a/examples/helloworld/mix.lock +++ b/examples/helloworld/mix.lock @@ -5,6 +5,7 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "google_protos": {:hex, :google_protos, "0.3.0", "15faf44dce678ac028c289668ff56548806e313e4959a3aaf4f6e1ebe8db83f4", [:mix], [{:protobuf, "~> 0.10", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "1f6b7fb20371f72f418b98e5e48dae3e022a9a6de1858d4b254ac5a5d0b4035f"}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, - "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/examples/helloworld/priv/protos/google/api/annotations.proto b/examples/helloworld/priv/protos/google/api/annotations.proto new file mode 100644 index 00000000..efdab3db --- /dev/null +++ b/examples/helloworld/priv/protos/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/examples/helloworld/priv/protos/google/api/http.proto b/examples/helloworld/priv/protos/google/api/http.proto new file mode 100644 index 00000000..113fa936 --- /dev/null +++ b/examples/helloworld/priv/protos/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/examples/helloworld/priv/protos/helloworld.proto b/examples/helloworld/priv/protos/helloworld.proto index 12849981..55c41005 100644 --- a/examples/helloworld/priv/protos/helloworld.proto +++ b/examples/helloworld/priv/protos/helloworld.proto @@ -5,6 +5,7 @@ option java_package = "io.grpc.examples.helloworld"; option java_outer_classname = "HelloWorldProto"; option objc_class_prefix = "HLW"; +import "google/api/annotations.proto"; import "google/protobuf/timestamp.proto"; package helloworld; @@ -12,7 +13,18 @@ package helloworld; // The greeting service definition. service Greeter { // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} + rpc SayHello (HelloRequest) returns (HelloReply) { + option (google.api.http) = { + get: "/v1/greeter/{name}" + }; + } + + rpc SayHelloFrom (HelloRequestFrom) returns (HelloReply) { + option (google.api.http) = { + post: "/v1/greeter" + body: "*" + }; + } } // The request message containing the user's name. @@ -20,6 +32,14 @@ message HelloRequest { string name = 1; } +// HelloRequestFrom! +message HelloRequestFrom { + // Name! + string name = 1; + // From! + string from = 2; +} + // The response message containing the greetings message HelloReply { string message = 1; From dbf4f34b98efeb77d1d55731acbc1e9170cf69f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 9 Sep 2022 12:02:13 +0200 Subject: [PATCH 27/73] dont pack message if the method was called as a transcode req --- lib/grpc/server.ex | 3 ++- lib/grpc/server/adapters/cowboy.ex | 3 ++- lib/grpc/server/adapters/cowboy/handler.ex | 14 +++++++++++--- mix.exs | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index a699934a..42429c42 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -357,7 +357,8 @@ defmodule GRPC.Server do iex> GRPC.Server.send_reply(stream, reply) """ @spec send_reply(GRPC.Server.Stream.t(), struct()) :: GRPC.Server.Stream.t() - def send_reply(%{__interface__: interface} = stream, reply, opts \\ []) do + def send_reply(%{__interface__: interface, http_transcode: transcode} = stream, reply, opts \\ []) do + opts = Keyword.put(opts, :http_transcode, transcode) interface[:send_reply].(stream, reply, opts) end diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 51caa7f0..32348258 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -133,7 +133,8 @@ defmodule GRPC.Server.Adapters.Cowboy do @impl true def send_reply(%{pid: pid}, data, opts) do - Handler.stream_body(pid, data, opts, :nofin) + http_transcode = Keyword.get(opts, :http_transcode) + Handler.stream_body(pid, data, opts, :nofin, http_transcode) end @impl true diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index b6498b60..4c41bc1e 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -119,8 +119,8 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do sync_call(pid, :read_body) end - def stream_body(pid, data, opts, is_fin) do - send(pid, {:stream_body, data, opts, is_fin}) + def stream_body(pid, data, opts, is_fin, http_transcode \\ false) do + send(pid, {:stream_body, data, opts, is_fin, http_transcode}) end def stream_reply(pid, status, headers) do @@ -230,7 +230,15 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, req, state} end - def info({:stream_body, data, opts, is_fin}, req, state) do + # Handle http/json transcoded response + def info({:stream_body, data, _opts, is_fin, _http_transcode = true}, req, state) do + # TODO Compress + req = check_sent_resp(req) + :cowboy_req.stream_body(data, is_fin, req) + {:ok, req, state} + end + + def info({:stream_body, data, opts, is_fin, _}, req, state) do # If compressor exists, compress is true by default compressor = if opts[:compress] == false do diff --git a/mix.exs b/mix.exs index 4265275d..3da1b578 100644 --- a/mix.exs +++ b/mix.exs @@ -51,7 +51,7 @@ defmodule GRPC.Mixfile do {:gun, "~> 2.0.1", hex: :grpc_gun}, {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, - {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", only: [:dev, :test]}, + {:protobuf, github: "elixir-protobuf/protobuf", branch: "main"}, {:ex_doc, "~> 0.28.0", only: :dev}, {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false}, {:googleapis, From 089460f14286969636c5439dcee6fbbc89e138e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 12 Sep 2022 16:39:12 +0200 Subject: [PATCH 28/73] wip! build route from HttpRule --- lib/grpc/server/http_transcode.ex | 100 ++++++++++++++++++++ test/grpc/transcode_test.exs | 148 ++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 test/grpc/transcode_test.exs diff --git a/lib/grpc/server/http_transcode.ex b/lib/grpc/server/http_transcode.ex index 2b1813e7..a59cfc51 100644 --- a/lib/grpc/server/http_transcode.ex +++ b/lib/grpc/server/http_transcode.ex @@ -8,4 +8,104 @@ defmodule GRPC.Server.HTTPTranscode do def method(%{pattern: {method, _path}}) do method end + + @doc """ + https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule + + Template = "/" Segments [ Verb ] ; + Segments = Segment { "/" Segment } ; + Segment = "*" | "**" | LITERAL | Variable ; + Variable = "{" FieldPath [ "=" Segments ] "}" ; + FieldPath = IDENT { "." IDENT } ; + Verb = ":" LITERAL ; + """ + @spec build_route(term()) :: tuple() + def build_route(%Google.Api.HttpRule{pattern: {method, path}}) do + route = + path + |> tokenize([]) + |> parse([], []) + + {method, route} + end + + @spec tokenize(binary(), list()) :: list() + def tokenize(path, tokens \\ []) + + def tokenize(<<>>, tokens) do + Enum.reverse(tokens) + end + + def tokenize(segments, tokens) do + {token, rest} = do_tokenize(segments, <<>>) + tokenize(rest, [token | tokens]) + end + + @terminals [?/, ?{, ?}, ?=, ?*] + defp do_tokenize(<>, <<>>) when h in @terminals do + # parse(t, acc) + {{List.to_atom([h]), []}, t} + end + + defp do_tokenize(<> = rest, acc) when h in @terminals do + {{:literal, acc, []}, rest} + end + + defp do_tokenize(<>, acc) + when h in ?a..?z or h in ?A..?Z or h in ?0..?9 or h == ?_ or h == ?. do + do_tokenize(t, <>) + end + + defp do_tokenize(<<>>, acc) do + {{:literal, acc, []}, <<>>} + end + + @spec parse(list(tuple()), list(), list()) :: list() + def parse([], params, segments) do + {Enum.reverse(params), Enum.reverse(segments)} + end + + def parse([{:/, _} | rest], params, segments) do + parse(rest, params, segments) + end + + def parse([{:*, _} | rest], params, segments) do + parse(rest, params, [{:_, []} | segments]) + end + + def parse([{:literal, literal, _} | rest], params, segments) do + parse(rest, params, [literal | segments]) + end + + def parse([{:"{", _} | rest], params, segments) do + {params, segments, rest} = parse_binding(rest, params, segments) + parse(rest, params, segments) + end + + defp parse_binding([{:"}", []} | rest], params, segments) do + {params, segments, rest} + end + + defp parse_binding( + [{:literal, lit, _}, {:=, _}, {:literal, assign, _} = a | rest], + params, + segments + ) do + + {variable, _} = param = field_path(lit) + # assign = field_path(assign) + + parse_binding(rest, [param | params], [{variable, [assign]} | segments]) + end + + defp parse_binding([{:literal, lit, []} | rest], params, segments) do + {variable, _} = param = field_path(lit) + parse_binding(rest, [param | params], [{variable, []} | segments]) + end + + def field_path(identifier) do + [root | path] = String.split(identifier, ".") + {String.to_atom(root), path} + end + end diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs new file mode 100644 index 00000000..c4a08f51 --- /dev/null +++ b/test/grpc/transcode_test.exs @@ -0,0 +1,148 @@ +defmodule GRPC.TranscodeTest do + use ExUnit.Case, async: true + alias GRPC.Server.HTTPTranscode, as: Transcode + + describe "build_route/1" do + test "returns a route with {http_method, route} based on the http rule" do + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + assert {:get, {params, segments}} = Transcode.build_route(rule) + assert [message_id: []] == params + assert ["v1", "messages", {:message_id, []}] = segments + end + end + + describe "tokenize/2" do + test "can tokenize simple paths" do + assert [{:/, []}] = Transcode.tokenize("/") + + assert [{:/, []}, {:literal, "v1", []}, {:/, []}, {:literal, "messages", []}] = + Transcode.tokenize("/v1/messages") + end + + test "can tokenize simple paths with wildcards" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:literal, "messages", []}, + {:/, []}, + {:*, []} + ] == Transcode.tokenize("/v1/messages/*") + end + + test "can tokenize simple variables" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:literal, "messages", []}, + {:/, []}, + {:"{", []}, + {:literal, "message_id", []}, + {:"}", []} + ] == Transcode.tokenize("/v1/messages/{message_id}") + end + + test "can tokenize variable assignments in bindings" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:"{", []}, + {:literal, "name", []}, + {:=, []}, + {:literal, "messages", []}, + {:"}", []} + ] == Transcode.tokenize("/v1/{name=messages}") + end + + test "can tokenize field paths in bindings" do + assert [ + {:/, []}, + {:literal, "v1", []}, + {:/, []}, + {:literal, "messages", []}, + {:/, []}, + {:"{", []}, + {:literal, "message_id", []}, + {:"}", []}, + {:/, []}, + {:"{", []}, + {:literal, "sub.subfield", []}, + {:"}", []} + ] == Transcode.tokenize("/v1/messages/{message_id}/{sub.subfield}") + end + end + + describe "parse/3" do + test "can parse simple paths" do + assert {[], []} == + "/" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse paths with literals" do + assert {[], ["v1", "messages"]} == + "/v1/messages" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse paths with wildcards" do + assert {[], ["v1", "messages", {:_, []}]} == + "/v1/messages/*" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse simple bindings with variables" do + assert {[{:message_id, []}], ["v1", "messages", {:message_id, []}]} == + "/v1/messages/{message_id}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse bindings with variable assignment" do + assert {[{:name, []}], ["v1", {:name, ["messages"]}]} == + "/v1/{name=messages}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse multiple bindings with variable assignment" do + assert {[{:name, []}, {:message_id, []}], ["v1", {:name, ["messages"]}, {:message_id, []}]} == + "/v1/{name=messages}/{message_id}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse bindings with field paths " do + assert {[sub: ["subfield"]], ["v1", "messages", {:sub, []}]} == + "/v1/messages/{sub.subfield}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "supports deeper nested field path " do + assert {[sub: ["nested", "nested", "nested"]], ["v1", "messages", {:sub, []}]} == + "/v1/messages/{sub.nested.nested.nested}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + + test "can parse multiple-bindings with field paths " do + assert {[first: ["subfield"], second: ["subfield"]], + ["v1", "messages", {:first, []}, {:second, []}]} == + "/v1/messages/{first.subfield}/{second.subfield}" + |> Transcode.tokenize() + |> Transcode.parse([], []) + end + end + + defp build_simple_rule(method, pattern) do + Google.Api.HttpRule.new(pattern: {method, pattern}) + end + + # rule = Google.Api.HttpRule.new(pattern: {:get, "/v1/{name=messages/*}"}) +end From 3e3c052eaa697b556d774e5f758efb3c52226137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 12 Sep 2022 16:39:56 +0200 Subject: [PATCH 29/73] update helloworld example with HttpRule options --- examples/helloworld/lib/helloworld.pb.ex | 16 ++++++++++++ examples/helloworld/lib/helloworld.svc.ex | 26 +++++++++++++++++++ .../helloworld/priv/protos/helloworld.proto | 15 +++++++++++ 3 files changed, 57 insertions(+) diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index bd78735a..b849b575 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -22,4 +22,20 @@ defmodule Helloworld.HelloReply do field :message, 1, type: :string field :today, 2, type: Google.Protobuf.Timestamp +end + +defmodule Helloworld.GetMessageRequest do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string +end + +defmodule Helloworld.Message do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :text, 1, type: :string end \ No newline at end of file diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex index 0afcba8b..382e2f62 100644 --- a/examples/helloworld/lib/helloworld.svc.ex +++ b/examples/helloworld/lib/helloworld.svc.ex @@ -36,4 +36,30 @@ defmodule Helloworld.Greeter.Stub do @moduledoc false use GRPC.Stub, service: Helloworld.Greeter.Service +end + +defmodule Helloworld.Messaging.Service do + @moduledoc false + + use GRPC.Service, name: "helloworld.Messaging", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetMessage, Helloworld.GetMessageRequest, Helloworld.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/{name=messages/*}"}, + response_body: "", + selector: "" + } + } + }) +end + +defmodule Helloworld.Messaging.Stub do + @moduledoc false + + use GRPC.Stub, service: Helloworld.Messaging.Service end \ No newline at end of file diff --git a/examples/helloworld/priv/protos/helloworld.proto b/examples/helloworld/priv/protos/helloworld.proto index 55c41005..632519a0 100644 --- a/examples/helloworld/priv/protos/helloworld.proto +++ b/examples/helloworld/priv/protos/helloworld.proto @@ -45,3 +45,18 @@ message HelloReply { string message = 1; google.protobuf.Timestamp today = 2; } + +service Messaging { + rpc GetMessage(GetMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/{name=messages/*}" + }; + } +} + +message GetMessageRequest { + string name = 1; // Mapped to URL path. +} +message Message { + string text = 1; // The resource content. +} From 94c013c30809d8081adf3ffe8ced3a4cb88c4acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 12 Sep 2022 16:45:00 +0200 Subject: [PATCH 30/73] rename mod to transcode --- lib/grpc/server.ex | 6 +++--- lib/grpc/server/{http_transcode.ex => transcode.ex} | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename lib/grpc/server/{http_transcode.ex => transcode.ex} (98%) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 42429c42..8bf33824 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -34,7 +34,7 @@ defmodule GRPC.Server do require Logger alias GRPC.RPCError - alias GRPC.Server.HTTPTranscode + alias GRPC.Server.Transcode @type rpc_req :: struct | Enumerable.t() @type rpc_return :: struct | any @@ -70,8 +70,8 @@ defmodule GRPC.Server do if http_transcode and Map.has_key?(options, :http) do %{value: http_opts} = Map.fetch!(options, :http) - http_path = HTTPTranscode.path(http_opts) - http_method = HTTPTranscode.method(http_opts) + http_path = Transcode.path(http_opts) + http_method = Transcode.method(http_opts) def __call_rpc__(unquote(http_path), stream) do GRPC.Server.call( diff --git a/lib/grpc/server/http_transcode.ex b/lib/grpc/server/transcode.ex similarity index 98% rename from lib/grpc/server/http_transcode.ex rename to lib/grpc/server/transcode.ex index a59cfc51..1f55f1ce 100644 --- a/lib/grpc/server/http_transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,4 +1,4 @@ -defmodule GRPC.Server.HTTPTranscode do +defmodule GRPC.Server.Transcode do @spec path(term()) :: String.t() def path(%{pattern: {_method, path}}) do path From 1679d3e6418d3c32799e269664fffaf503cb8098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 08:59:57 +0200 Subject: [PATCH 31/73] fix! match rpc options in stub --- lib/grpc/server.ex | 1 - lib/grpc/stub.ex | 2 +- test/grpc/transcode_test.exs | 4 +--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 8bf33824..b1e6aab9 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -165,7 +165,6 @@ defmodule GRPC.Server do } = stream, func_name ) do - IO.inspect(res_stream, label: "do_handle_request") {:ok, data} = adapter.read_body(payload) request = codec.decode(data, req_mod) diff --git a/lib/grpc/stub.ex b/lib/grpc/stub.ex index 07f48484..f6f98224 100644 --- a/lib/grpc/stub.ex +++ b/lib/grpc/stub.ex @@ -247,7 +247,7 @@ defmodule GRPC.Stub do """ @spec call(atom(), tuple(), GRPC.Client.Stream.t(), struct() | nil, keyword()) :: rpc_return def call(_service_mod, rpc, %{channel: channel} = stream, request, opts) do - {_, {req_mod, req_stream}, {res_mod, response_stream}} = rpc + {_, {req_mod, req_stream}, {res_mod, response_stream}, _rpc_options} = rpc stream = %{stream | request_mod: req_mod, response_mod: res_mod} diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs index c4a08f51..3b673103 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/transcode_test.exs @@ -1,6 +1,6 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true - alias GRPC.Server.HTTPTranscode, as: Transcode + alias GRPC.Server.Transcode describe "build_route/1" do test "returns a route with {http_method, route} based on the http rule" do @@ -143,6 +143,4 @@ defmodule GRPC.TranscodeTest do defp build_simple_rule(method, pattern) do Google.Api.HttpRule.new(pattern: {method, pattern}) end - - # rule = Google.Api.HttpRule.new(pattern: {:get, "/v1/{name=messages/*}"}) end From bedd74e1358a40554a8b398ea4478f049e2fc153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 11:17:15 +0200 Subject: [PATCH 32/73] encode segments to path --- lib/grpc/server/transcode.ex | 15 +++++++++++++-- mix.exs | 3 ++- test/grpc/transcode_test.exs | 24 +++++++++++++++++------- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 1f55f1ce..8601086c 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -9,6 +9,19 @@ defmodule GRPC.Server.Transcode do method end + @spec to_path(term()) :: String.t() + def to_path({method, {_bindings, segments}} = _spec) do + match = + segments + |> Enum.map(&segment_to_string/1) + |> Enum.join("/") + + "/" <> match + end + + defp segment_to_string({binding, _}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" + defp segment_to_string(segment), do: segment + @doc """ https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule @@ -91,7 +104,6 @@ defmodule GRPC.Server.Transcode do params, segments ) do - {variable, _} = param = field_path(lit) # assign = field_path(assign) @@ -107,5 +119,4 @@ defmodule GRPC.Server.Transcode do [root | path] = String.split(identifier, ".") {String.to_atom(root), path} end - end diff --git a/mix.exs b/mix.exs index 3da1b578..0ae049f8 100644 --- a/mix.exs +++ b/mix.exs @@ -95,6 +95,7 @@ defmodule GRPC.Mixfile do end end) end + # https://github.com/elixir-protobuf/protobuf/blob/cdf3acc53f619866b4921b8216d2531da52ceba7/mix.exs#L140 defp gen_bootstrap_protos(_args) do proto_src = Mix.Project.deps_paths().googleapis @@ -117,7 +118,7 @@ defmodule GRPC.Mixfile do case Mix.shell().cmd(Enum.join(args, " ")) do 0 -> Mix.Task.rerun("format", [Path.join([elixir_out, "**", "*.pb.ex"])]) - other -> Mix.raise("'protoc' exited with non-zero status: #{other}") + other -> Mix.raise("'protoc' exited with non-zero status: #{other}") end end end diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs index 3b673103..8a711eea 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/transcode_test.exs @@ -2,13 +2,23 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true alias GRPC.Server.Transcode - describe "build_route/1" do - test "returns a route with {http_method, route} based on the http rule" do - rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert {:get, {params, segments}} = Transcode.build_route(rule) - assert [message_id: []] == params - assert ["v1", "messages", {:message_id, []}] = segments - end + test "build_route/1 returns a route with {http_method, route} based on the http rule" do + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + assert {:get, {params, segments}} = Transcode.build_route(rule) + assert [message_id: []] == params + assert ["v1", "messages", {:message_id, []}] = segments + end + + test "to_path/1 returns path segments as a string match" do + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + assert spec = Transcode.build_route(rule) + assert "/v1/messages/:message_id" = Transcode.to_path(spec) + end + + test "to_path/1 returns path segments as a string when there's multiple bindings" do + rule = build_simple_rule(:get, "/v1/users/{user_id}/messages/{message_id}") + assert spec = Transcode.build_route(rule) + assert "/v1/users/:user_id/messages/:message_id" = Transcode.to_path(spec) end describe "tokenize/2" do From 0abb32f7517a9f9b501fa13d5c96a5d0d72c6d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 11:19:02 +0200 Subject: [PATCH 33/73] expose routes from server Routes are fetched from server and compiled into a dispatch conf. Instead of finding a matching server in the handler the path is matched and the handler is called with the correct server. --- lib/grpc/server.ex | 30 ++++++++++++++++++---- lib/grpc/server/adapters/cowboy.ex | 25 ++++++++++++++---- lib/grpc/server/adapters/cowboy/handler.ex | 22 +++------------- 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index b1e6aab9..c05c55cb 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -48,6 +48,22 @@ defmodule GRPC.Server do compressors = opts[:compressors] || [] http_transcode = opts[:http_transcode] || false + routes = + for {name, _, _, options} <- service_mod.__rpc_calls__, reduce: [] do + acc -> + path = "/#{service_name}/#{name}" + + http_paths = + if http_transcode and Map.has_key?(options, :http) do + %{value: http_opts} = Map.fetch!(options, :http) + [{:http_transcode, Transcode.build_route(http_opts)}] + else + [] + end + + http_paths ++ [{:grpc, path} | acc] + end + Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> func_name = name |> to_string |> Macro.underscore() |> String.to_atom() path = "/#{service_name}/#{name}" @@ -68,10 +84,9 @@ defmodule GRPC.Server do end if http_transcode and Map.has_key?(options, :http) do - %{value: http_opts} = Map.fetch!(options, :http) - - http_path = Transcode.path(http_opts) - http_method = Transcode.method(http_opts) + %{value: http_rule} = Map.fetch!(options, :http) + {http_method, _} = spec = Transcode.build_route(http_rule) + http_path = Transcode.to_path(spec) def __call_rpc__(unquote(http_path), stream) do GRPC.Server.call( @@ -110,6 +125,7 @@ defmodule GRPC.Server do def __meta__(:service), do: unquote(service_mod) def __meta__(:codecs), do: unquote(codecs) def __meta__(:compressors), do: unquote(compressors) + def __meta__(:routes), do: unquote(routes) end end @@ -356,7 +372,11 @@ defmodule GRPC.Server do iex> GRPC.Server.send_reply(stream, reply) """ @spec send_reply(GRPC.Server.Stream.t(), struct()) :: GRPC.Server.Stream.t() - def send_reply(%{__interface__: interface, http_transcode: transcode} = stream, reply, opts \\ []) do + def send_reply( + %{__interface__: interface, http_transcode: transcode} = stream, + reply, + opts \\ [] + ) do opts = Keyword.put(opts, :http_transcode, transcode) interface[:send_reply].(stream, reply, opts) end diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 32348258..793ff48b 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -170,20 +170,35 @@ defmodule GRPC.Server.Adapters.Cowboy do Handler.set_compressor(pid, compressor) end + defp build_handlers(endpoint, servers, opts) do + Enum.flat_map(servers, fn {_name, server_mod} = server -> + routes = server_mod.__meta__(:routes) + Enum.map(routes, &build_route(&1, endpoint, server, opts)) + end) + end + + defp build_route({:grpc, path}, endpoint, server, opts) do + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + end + + defp build_route({:http_transcode, spec}, endpoint, server, opts) do + path = GRPC.Server.Transcode.to_path(spec) + + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + end + defp cowboy_start_args(endpoint, servers, port, opts) do # Custom handler to be able to listen in the same port, more info: # https://github.com/containous/traefik/issues/6211 {adapter_opts, opts} = Keyword.pop(opts, :adapter_opts, []) status_handler = Keyword.get(adapter_opts, :status_handler) + handlers = build_handlers(endpoint, servers, opts) handlers = if status_handler do - [ - status_handler, - {:_, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, servers, Enum.into(opts, %{})}} - ] + [status_handler | handlers] else - [{:_, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, servers, Enum.into(opts, %{})}}] + handlers end dispatch = :cowboy_router.compile([{:_, handlers}]) diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 4c41bc1e..f135ec82 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -13,17 +13,12 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do @spec init( map(), - state :: {endpoint :: atom(), servers :: %{String.t() => [module()]}, opts :: keyword()} + state :: {endpoint :: atom(), server :: {String.t(), module()}, route :: String.t(), opts :: keyword()} ) :: {:cowboy_loop, map(), map()} - def init(req, {endpoint, servers, opts} = state) do + def init(req, {endpoint, {_name, server}, route, opts} = state) do path = :cowboy_req.path(req) - Logger.info(fn -> - "path: #{path}" - end) - - with {:ok, server} <- find_server(servers, path), - {:ok, codec} <- find_codec(req, server), + with {:ok, codec} <- find_codec(req, server), # can be nil {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ @@ -36,7 +31,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do compressor: compressor } - pid = spawn_link(__MODULE__, :call_rpc, [server, path, stream]) + pid = spawn_link(__MODULE__, :call_rpc, [server, route, stream]) Process.flag(:trap_exit, true) req = :cowboy_req.set_resp_headers(HTTP2.server_headers(stream), req) @@ -62,15 +57,6 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end - # TODO compile routes instead of dynamic dispatch to find which server has - # which route - defp find_server(servers, path) do - case Enum.find(servers, fn {_name, server} -> server.service_name(path) != "" end) do - nil -> {:error, RPCError.exception(status: :unimplemented)} - {_, server} -> {:ok, server} - end - end - defp find_codec(req, server) do req_content_type = :cowboy_req.header("content-type", req) From 2056430674772c374f99837665e8717917497176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 11:23:42 +0200 Subject: [PATCH 34/73] init http/json transcode integration tests --- examples/helloworld/lib/helloworld.pb.ex | 2 +- examples/helloworld/lib/helloworld.svc.ex | 2 +- test/grpc/integration/server_test.exs | 22 ++ test/support/google/api/annotations.proto | 31 ++ test/support/google/api/http.proto | 375 ++++++++++++++++++++++ test/support/route_guide_transocde.pb.ex | 46 +++ test/support/route_guide_transocde.proto | 51 +++ test/support/route_guide_transocde.svc.ex | 48 +++ 8 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 test/support/google/api/annotations.proto create mode 100644 test/support/google/api/http.proto create mode 100644 test/support/route_guide_transocde.pb.ex create mode 100644 test/support/route_guide_transocde.proto create mode 100644 test/support/route_guide_transocde.svc.ex diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index b849b575..82cce674 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -38,4 +38,4 @@ defmodule Helloworld.Message do use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :text, 1, type: :string -end \ No newline at end of file +end diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex index 382e2f62..dcf882ab 100644 --- a/examples/helloworld/lib/helloworld.svc.ex +++ b/examples/helloworld/lib/helloworld.svc.ex @@ -62,4 +62,4 @@ defmodule Helloworld.Messaging.Stub do @moduledoc false use GRPC.Stub, service: Helloworld.Messaging.Service -end \ No newline at end of file +end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index e2455719..093cf2c4 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -9,6 +9,16 @@ defmodule GRPC.Integration.ServerTest do end end + defmodule FeatureTranscodeServer do + use GRPC.Server, + service: RouteguideTranscode.RouteGuide.Service, + http_transcode: true + + def get_feature(point, _stream) do + Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") + end + end + defmodule HelloServer do use GRPC.Server, service: Helloworld.Greeter.Service @@ -241,4 +251,16 @@ defmodule GRPC.Integration.ServerTest do assert reply.message == "Hello, unauthenticated" end) end + + describe "http/json transcode" do + test "can transcode path params" do + run_server([FeatureTranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + {:ok, conn_pid} = :gun.open('localhost', port) + stream_ref = :gun.get(conn_pid, "/v1/feature/1/2") + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + end) + end + end end diff --git a/test/support/google/api/annotations.proto b/test/support/google/api/annotations.proto new file mode 100644 index 00000000..efdab3db --- /dev/null +++ b/test/support/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/test/support/google/api/http.proto b/test/support/google/api/http.proto new file mode 100644 index 00000000..113fa936 --- /dev/null +++ b/test/support/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/test/support/route_guide_transocde.pb.ex b/test/support/route_guide_transocde.pb.ex new file mode 100644 index 00000000..1079307f --- /dev/null +++ b/test/support/route_guide_transocde.pb.ex @@ -0,0 +1,46 @@ +defmodule RouteguideTranscode.Point do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :latitude, 1, type: :int32 + field :longitude, 2, type: :int32 +end + +defmodule RouteguideTranscode.Rectangle do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :lo, 1, type: RouteguideTranscode.Point + field :hi, 2, type: RouteguideTranscode.Point +end + +defmodule RouteguideTranscode.Feature do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string + field :location, 2, type: RouteguideTranscode.Point +end + +defmodule RouteguideTranscode.RouteNote do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :location, 1, type: RouteguideTranscode.Point + field :message, 2, type: :string +end + +defmodule RouteguideTranscode.RouteSummary do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :point_count, 1, type: :int32, json_name: "pointCount" + field :feature_count, 2, type: :int32, json_name: "featureCount" + field :distance, 3, type: :int32 + field :elapsed_time, 4, type: :int32, json_name: "elapsedTime" +end diff --git a/test/support/route_guide_transocde.proto b/test/support/route_guide_transocde.proto new file mode 100644 index 00000000..67bf7e9e --- /dev/null +++ b/test/support/route_guide_transocde.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.routeguide"; +option java_outer_classname = "RouteGuideProto"; + +import "google/api/annotations.proto"; + +package routeguide_transcode; + +service RouteGuide { + rpc GetFeature(Point) returns (Feature) { + option (google.api.http) = { + get: "/v1/feature/{latitude}/{longitude}" + }; + } + rpc ListFeatures(Rectangle) returns (stream Feature) { + option (google.api.http) = { + get: "/v1/feature" + }; + } + rpc RecordRoute(stream Point) returns (RouteSummary) {} + rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} +} + +message Point { + int32 latitude = 1; + int32 longitude = 2; +} + +message Rectangle { + Point lo = 1; + Point hi = 2; +} + +message Feature { + string name = 1; + Point location = 2; +} + +message RouteNote { + Point location = 1; + string message = 2; +} + +message RouteSummary { + int32 point_count = 1; + int32 feature_count = 2; + int32 distance = 3; + int32 elapsed_time = 4; +} diff --git a/test/support/route_guide_transocde.svc.ex b/test/support/route_guide_transocde.svc.ex new file mode 100644 index 00000000..adf27fdd --- /dev/null +++ b/test/support/route_guide_transocde.svc.ex @@ -0,0 +1,48 @@ +defmodule RouteguideTranscode.RouteGuide.Service do + @moduledoc false + + use GRPC.Service, name: "routeguide_transcode.RouteGuide", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetFeature, RouteguideTranscode.Point, RouteguideTranscode.Feature, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/feature/{latitude}/{longitude}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:ListFeatures, RouteguideTranscode.Rectangle, stream(RouteguideTranscode.Feature), %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/feature"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:RecordRoute, stream(RouteguideTranscode.Point), RouteguideTranscode.RouteSummary, %{}) + + rpc( + :RouteChat, + stream(RouteguideTranscode.RouteNote), + stream(RouteguideTranscode.RouteNote), + %{} + ) +end + +defmodule RouteguideTranscode.RouteGuide.Stub do + @moduledoc false + + use GRPC.Stub, service: RouteguideTranscode.RouteGuide.Service +end From dd59b003125b8045ff84f04ce619756fa4cc5bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 14:30:35 +0200 Subject: [PATCH 35/73] merge path bindings with body --- lib/grpc/codec/json.ex | 5 +++++ lib/grpc/server.ex | 10 +++++++++- lib/grpc/server/adapters/cowboy.ex | 8 ++++++++ lib/grpc/server/adapters/cowboy/handler.ex | 21 ++++++++++++++++++++- lib/grpc/server/transcode.ex | 10 ++++++++++ test/grpc/integration/server_test.exs | 18 ++++++++++++++++-- 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 078bb883..0a32bf53 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -9,6 +9,11 @@ defmodule GRPC.Codec.JSON do Protobuf.JSON.encode!(struct) end + + def decode(<<>>, module) do + struct(module, []) + end + def decode(binary, module) do Protobuf.JSON.decode!(binary, module) end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index c05c55cb..b51c9352 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -182,9 +182,17 @@ defmodule GRPC.Server do func_name ) do {:ok, data} = adapter.read_body(payload) + bindings = adapter.get_bindings(payload) + qs = adapter.get_qs(payload) request = codec.decode(data, req_mod) - call_with_interceptors(res_stream, func_name, stream, request) + case Transcode.map_request(request, bindings, qs, req_mod) do + {:ok, request} -> + call_with_interceptors(res_stream, func_name, stream, request) + + resp = {:error, _} -> + resp + end end defp do_handle_request( diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 793ff48b..f01df58c 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -166,6 +166,14 @@ defmodule GRPC.Server.Adapters.Cowboy do Handler.get_cert(pid) end + def get_qs(%{pid: pid}) do + Handler.get_qs(pid) + end + + def get_bindings(%{pid: pid}) do + Handler.get_bindings(pid) + end + def set_compressor(%{pid: pid}, compressor) do Handler.set_compressor(pid, compressor) end diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index f135ec82..057b9774 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -19,7 +19,6 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do path = :cowboy_req.path(req) with {:ok, codec} <- find_codec(req, server), - # can be nil {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ server: server, @@ -141,6 +140,14 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do sync_call(pid, :get_cert) end + def get_qs(pid) do + sync_call(pid, :get_qs) + end + + def get_bindings(pid) do + sync_call(pid, :get_bindings) + end + defp sync_call(pid, key) do ref = make_ref() send(pid, {key, ref, self()}) @@ -216,6 +223,18 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, req, state} end + def info({:get_qs, ref, pid}, req, state) do + qs = :cowboy_req.qs(req) + send(pid, {ref, qs}) + {:ok, req, state} + end + + def info({:get_bindings, ref, pid}, req, state) do + bindings = :cowboy_req.bindings(req) + send(pid, {ref, bindings}) + {:ok, req, state} + end + # Handle http/json transcoded response def info({:stream_body, data, _opts, is_fin, _http_transcode = true}, req, state) do # TODO Compress diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 8601086c..d11cffe6 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,4 +1,14 @@ defmodule GRPC.Server.Transcode do + + @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} + def map_request(body_request, path_bindings, _query_string, req_mod) do + path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) + + with {:ok, path_request} <- Protobuf.JSON.from_decoded(path_bindings, req_mod) do + {:ok, Map.merge(body_request, path_request)} + end + end + @spec path(term()) :: String.t() def path(%{pattern: {_method, path}}) do path diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 093cf2c4..f91bf205 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -255,11 +255,25 @@ defmodule GRPC.Integration.ServerTest do describe "http/json transcode" do test "can transcode path params" do run_server([FeatureTranscodeServer], fn port -> + latitude = 10 + longitude = 20 + {:ok, conn_pid} = :gun.open('localhost', port) - {:ok, conn_pid} = :gun.open('localhost', port) - stream_ref = :gun.get(conn_pid, "/v1/feature/1/2") + + stream_ref = + :gun.get(conn_pid, "/v1/feature/#{latitude}/#{longitude}", [ + {"content-type", "application/json"} + ]) assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{ + "location" => %{"latitude" => ^latitude, "longitude" => ^longitude}, + "name" => name + } = Jason.decode!(body) + + assert name == "#{latitude},#{longitude}" end) end end From 5b12f343b79fa454851639e37f253b9c6ddf4451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 16:34:40 +0200 Subject: [PATCH 36/73] http rule template lexer / parse rename literal -> identifier --- lib/grpc/server/transcode.ex | 46 +++++++++++++----------------------- test/grpc/transcode_test.exs | 28 +++++++++++----------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index d11cffe6..a58a034c 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,5 +1,4 @@ defmodule GRPC.Server.Transcode do - @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request(body_request, path_bindings, _query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) @@ -9,18 +8,8 @@ defmodule GRPC.Server.Transcode do end end - @spec path(term()) :: String.t() - def path(%{pattern: {_method, path}}) do - path - end - - @spec method(term()) :: String.t() - def method(%{pattern: {method, _path}}) do - method - end - @spec to_path(term()) :: String.t() - def to_path({method, {_bindings, segments}} = _spec) do + def to_path({_method, {_bindings, segments}} = _spec) do match = segments |> Enum.map(&segment_to_string/1) @@ -32,16 +21,15 @@ defmodule GRPC.Server.Transcode do defp segment_to_string({binding, _}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" defp segment_to_string(segment), do: segment - @doc """ - https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule + # https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule - Template = "/" Segments [ Verb ] ; - Segments = Segment { "/" Segment } ; - Segment = "*" | "**" | LITERAL | Variable ; - Variable = "{" FieldPath [ "=" Segments ] "}" ; - FieldPath = IDENT { "." IDENT } ; - Verb = ":" LITERAL ; - """ + # Template = "/" Segments [ Verb ] ; + # Segments = Segment { "/" Segment } ; + # Segment = "*" | "**" | LITERAL | Variable ; + # Variable = "{" FieldPath [ "=" Segments ] "}" ; + # FieldPath = IDENT { "." IDENT } ; + # Verb = ":" LITERAL ; + # @spec build_route(term()) :: tuple() def build_route(%Google.Api.HttpRule{pattern: {method, path}}) do route = @@ -71,7 +59,7 @@ defmodule GRPC.Server.Transcode do end defp do_tokenize(<> = rest, acc) when h in @terminals do - {{:literal, acc, []}, rest} + {{:identifier, acc, []}, rest} end defp do_tokenize(<>, acc) @@ -80,7 +68,7 @@ defmodule GRPC.Server.Transcode do end defp do_tokenize(<<>>, acc) do - {{:literal, acc, []}, <<>>} + {{:identifier, acc, []}, <<>>} end @spec parse(list(tuple()), list(), list()) :: list() @@ -96,8 +84,8 @@ defmodule GRPC.Server.Transcode do parse(rest, params, [{:_, []} | segments]) end - def parse([{:literal, literal, _} | rest], params, segments) do - parse(rest, params, [literal | segments]) + def parse([{:identifier, identifier, _} | rest], params, segments) do + parse(rest, params, [identifier | segments]) end def parse([{:"{", _} | rest], params, segments) do @@ -110,18 +98,18 @@ defmodule GRPC.Server.Transcode do end defp parse_binding( - [{:literal, lit, _}, {:=, _}, {:literal, assign, _} = a | rest], + [{:identifier, id, _}, {:=, _}, {:identifier, assign, _} | rest], params, segments ) do - {variable, _} = param = field_path(lit) + {variable, _} = param = field_path(id) # assign = field_path(assign) parse_binding(rest, [param | params], [{variable, [assign]} | segments]) end - defp parse_binding([{:literal, lit, []} | rest], params, segments) do - {variable, _} = param = field_path(lit) + defp parse_binding([{:identifier, id, []} | rest], params, segments) do + {variable, _} = param = field_path(id) parse_binding(rest, [param | params], [{variable, []} | segments]) end diff --git a/test/grpc/transcode_test.exs b/test/grpc/transcode_test.exs index 8a711eea..bcb126a2 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/transcode_test.exs @@ -25,16 +25,16 @@ defmodule GRPC.TranscodeTest do test "can tokenize simple paths" do assert [{:/, []}] = Transcode.tokenize("/") - assert [{:/, []}, {:literal, "v1", []}, {:/, []}, {:literal, "messages", []}] = + assert [{:/, []}, {:identifier, "v1", []}, {:/, []}, {:identifier, "messages", []}] = Transcode.tokenize("/v1/messages") end test "can tokenize simple paths with wildcards" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:/, []}, {:*, []} ] == Transcode.tokenize("/v1/messages/*") @@ -43,12 +43,12 @@ defmodule GRPC.TranscodeTest do test "can tokenize simple variables" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:/, []}, {:"{", []}, - {:literal, "message_id", []}, + {:identifier, "message_id", []}, {:"}", []} ] == Transcode.tokenize("/v1/messages/{message_id}") end @@ -56,12 +56,12 @@ defmodule GRPC.TranscodeTest do test "can tokenize variable assignments in bindings" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, {:"{", []}, - {:literal, "name", []}, + {:identifier, "name", []}, {:=, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:"}", []} ] == Transcode.tokenize("/v1/{name=messages}") end @@ -69,16 +69,16 @@ defmodule GRPC.TranscodeTest do test "can tokenize field paths in bindings" do assert [ {:/, []}, - {:literal, "v1", []}, + {:identifier, "v1", []}, {:/, []}, - {:literal, "messages", []}, + {:identifier, "messages", []}, {:/, []}, {:"{", []}, - {:literal, "message_id", []}, + {:identifier, "message_id", []}, {:"}", []}, {:/, []}, {:"{", []}, - {:literal, "sub.subfield", []}, + {:identifier, "sub.subfield", []}, {:"}", []} ] == Transcode.tokenize("/v1/messages/{message_id}/{sub.subfield}") end @@ -92,7 +92,7 @@ defmodule GRPC.TranscodeTest do |> Transcode.parse([], []) end - test "can parse paths with literals" do + test "can parse paths with identifiers" do assert {[], ["v1", "messages"]} == "/v1/messages" |> Transcode.tokenize() From f7eba63fced772ea2a96417aa3ef300b4a2cec80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 13 Sep 2022 16:35:11 +0200 Subject: [PATCH 37/73] fix warnings --- lib/grpc/codec/json.ex | 1 - lib/grpc/server.ex | 7 ------- lib/grpc/server/adapters/cowboy.ex | 7 +++++-- lib/grpc/server/adapters/cowboy/handler.ex | 6 +++--- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 0a32bf53..9f40c0c0 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -9,7 +9,6 @@ defmodule GRPC.Codec.JSON do Protobuf.JSON.encode!(struct) end - def decode(<<>>, module) do struct(module, []) end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index b51c9352..ab0f3ff8 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -436,13 +436,6 @@ defmodule GRPC.Server do stream end - @doc false - # @spec service_name(String.t()) :: String.t() - # def service_name(path) do - # ["", name | _] = String.split(path, "/") - # name - # end - @doc false @spec servers_to_map(module() | [module()]) :: %{String.t() => [module()]} def servers_to_map(servers) do diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index f01df58c..e914621c 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -186,13 +186,15 @@ defmodule GRPC.Server.Adapters.Cowboy do end defp build_route({:grpc, path}, endpoint, server, opts) do - {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, + {endpoint, server, path, Enum.into(opts, %{})}} end defp build_route({:http_transcode, spec}, endpoint, server, opts) do path = GRPC.Server.Transcode.to_path(spec) - {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, + {endpoint, server, path, Enum.into(opts, %{})}} end defp cowboy_start_args(endpoint, servers, port, opts) do @@ -202,6 +204,7 @@ defmodule GRPC.Server.Adapters.Cowboy do status_handler = Keyword.get(adapter_opts, :status_handler) handlers = build_handlers(endpoint, servers, opts) + handlers = if status_handler do [status_handler | handlers] diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 057b9774..b02a82ab 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -13,11 +13,11 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do @spec init( map(), - state :: {endpoint :: atom(), server :: {String.t(), module()}, route :: String.t(), opts :: keyword()} + state :: + {endpoint :: atom(), server :: {String.t(), module()}, route :: String.t(), + opts :: keyword()} ) :: {:cowboy_loop, map(), map()} def init(req, {endpoint, {_name, server}, route, opts} = state) do - path = :cowboy_req.path(req) - with {:ok, codec} <- find_codec(req, server), {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ From d2c47386edf867d535d8c2b9657d8a4b05e1c2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 14 Sep 2022 10:54:29 +0200 Subject: [PATCH 38/73] fix! merge request params before decoding into struct --- lib/grpc/codec/json.ex | 14 ++++++++++---- lib/grpc/server/transcode.ex | 8 ++++---- test/grpc/{ => server}/transcode_test.exs | 13 +++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) rename test/grpc/{ => server}/transcode_test.exs (89%) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 9f40c0c0..91c5c9f7 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -9,11 +9,17 @@ defmodule GRPC.Codec.JSON do Protobuf.JSON.encode!(struct) end - def decode(<<>>, module) do - struct(module, []) + def decode(<<>>, _module) do + %{} end - def decode(binary, module) do - Protobuf.JSON.decode!(binary, module) + def decode(binary, _module) do + if jason = load_jason() do + jason.decode!(binary) + else + raise "`:jason` library not loaded" + end end + + defp load_jason, do: Code.ensure_loaded?(Jason) and Jason end diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index a58a034c..d269c0e4 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,11 +1,11 @@ defmodule GRPC.Server.Transcode do + @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request(body_request, path_bindings, _query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) + request = Map.merge(path_bindings, body_request) - with {:ok, path_request} <- Protobuf.JSON.from_decoded(path_bindings, req_mod) do - {:ok, Map.merge(body_request, path_request)} - end + Protobuf.JSON.from_decoded(request, req_mod) end @spec to_path(term()) :: String.t() @@ -71,7 +71,7 @@ defmodule GRPC.Server.Transcode do {{:identifier, acc, []}, <<>>} end - @spec parse(list(tuple()), list(), list()) :: list() + @spec parse(list(tuple()), list(), list()) :: {list(), list()} def parse([], params, segments) do {Enum.reverse(params), Enum.reverse(segments)} end diff --git a/test/grpc/transcode_test.exs b/test/grpc/server/transcode_test.exs similarity index 89% rename from test/grpc/transcode_test.exs rename to test/grpc/server/transcode_test.exs index bcb126a2..8865b4b9 100644 --- a/test/grpc/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -2,6 +2,19 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true alias GRPC.Server.Transcode + test "map_requests/3 can map request body to protobuf struct" do + body_request = %{"latitude" => 1, "longitude" => 2} + {:ok, request} = Transcode.map_request(body_request, %{}, "", Routeguide.Point) + assert Routeguide.Point.new(latitude: 1, longitude: 2) == request + end + + test "map_requests/3 can merge request body with path bindings to protobuf struct" do + body_request = %{"latitude" => 1} + bindings = %{"longitude" => 2} + {:ok, request} = Transcode.map_request(body_request, bindings, "", Routeguide.Point) + assert Routeguide.Point.new(latitude: 1, longitude: 2) == request + end + test "build_route/1 returns a route with {http_method, route} based on the http rule" do rule = build_simple_rule(:get, "/v1/messages/{message_id}") assert {:get, {params, segments}} = Transcode.build_route(rule) From 1a016a77498bb086398f4336a3e48831b35a4c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 14 Sep 2022 10:55:00 +0200 Subject: [PATCH 39/73] update example curl requests --- examples/helloworld/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index 09da4801..e3c2b345 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -21,10 +21,10 @@ $ mix run priv/client.exs ``` shell # Say hello -curl http://localhost:50051/v1/greeter/test +curl -H 'Content-type: application/json' http://localhost:50051/v1/greeter/test # Say hello from -curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' --output - http://localhost:50051/v1/greeter +curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' http://localhost:50051/v1/greeter ``` ## Regenerate Elixir code from proto From 898c1fe87605a18e27f86fb087f0363701f6251e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 14 Sep 2022 14:33:26 +0200 Subject: [PATCH 40/73] include http method in __call_rpc__ This enables to differentiate to transcoded calls which has the same URL path but different request methods --- lib/grpc/server.ex | 16 +------- lib/grpc/server/adapters/cowboy/handler.ex | 11 +++++- lib/grpc/server/transcode.ex | 1 - test/grpc/integration/server_test.exs | 37 ++++++++++++++++++- ...ocde.pb.ex => route_guide_transcode.pb.ex} | 0 ...ocde.proto => route_guide_transcode.proto} | 12 +++++- ...de.svc.ex => route_guide_transcode.svc.ex} | 18 ++++++++- 7 files changed, 73 insertions(+), 22 deletions(-) rename test/support/{route_guide_transocde.pb.ex => route_guide_transcode.pb.ex} (100%) rename test/support/{route_guide_transocde.proto => route_guide_transcode.proto} (82%) rename test/support/{route_guide_transocde.svc.ex => route_guide_transcode.svc.ex} (71%) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index ab0f3ff8..fff752d5 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -69,7 +69,7 @@ defmodule GRPC.Server do path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) - def __call_rpc__(unquote(path), stream) do + def __call_rpc__(unquote(path), unquote(:post), stream) do GRPC.Server.call( unquote(service_mod), %{ @@ -88,7 +88,7 @@ defmodule GRPC.Server do {http_method, _} = spec = Transcode.build_route(http_rule) http_path = Transcode.to_path(spec) - def __call_rpc__(unquote(http_path), stream) do + def __call_rpc__(unquote(http_path), unquote(http_method), stream) do GRPC.Server.call( unquote(service_mod), %{ @@ -103,14 +103,6 @@ defmodule GRPC.Server do unquote(func_name) ) end - - def service_name(unquote(http_path)) do - unquote(service_name) - end - end - - def service_name(unquote(path)) do - unquote(service_name) end end) @@ -118,10 +110,6 @@ defmodule GRPC.Server do raise GRPC.RPCError, status: :unimplemented end - def service_name(_) do - "" - end - def __meta__(:service), do: unquote(service_mod) def __meta__(:codecs), do: unquote(codecs) def __meta__(:compressors), do: unquote(compressors) diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index b02a82ab..83be83ee 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -20,6 +20,12 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do def init(req, {endpoint, {_name, server}, route, opts} = state) do with {:ok, codec} <- find_codec(req, server), {:ok, compressor} <- find_compressor(req, server) do + http_method = + req + |> :cowboy_req.method() + |> String.downcase() + |> String.to_existing_atom() + stream = %GRPC.Server.Stream{ server: server, endpoint: endpoint, @@ -27,6 +33,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do payload: %{pid: self()}, local: opts[:local], codec: codec, + http_method: http_method, compressor: compressor } @@ -390,8 +397,8 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end - defp do_call_rpc(server, path, stream) do - result = server.__call_rpc__(path, stream) + defp do_call_rpc(server, path, %{http_method: http_method} = stream) do + result = server.__call_rpc__(path, http_method, stream) case result do {:ok, stream, response} -> diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index d269c0e4..c5410734 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,5 +1,4 @@ defmodule GRPC.Server.Transcode do - @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request(body_request, path_bindings, _query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index f91bf205..1d97b376 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -17,6 +17,10 @@ defmodule GRPC.Integration.ServerTest do def get_feature(point, _stream) do Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") end + + def create_feature(point, _stream) do + Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") + end end defmodule HelloServer do @@ -261,7 +265,7 @@ defmodule GRPC.Integration.ServerTest do {:ok, conn_pid} = :gun.open('localhost', port) stream_ref = - :gun.get(conn_pid, "/v1/feature/#{latitude}/#{longitude}", [ + :gun.get(conn_pid, "/v1/features/#{latitude}/#{longitude}", [ {"content-type", "application/json"} ]) @@ -276,5 +280,36 @@ defmodule GRPC.Integration.ServerTest do assert name == "#{latitude},#{longitude}" end) end + + test "service methods can have the same path but different methods in http rule option" do + run_server([FeatureTranscodeServer], fn port -> + latitude = 10 + longitude = 20 + + {:ok, conn_pid} = :gun.open('localhost', port) + + body = %{"latitude" => latitude, "longitude" => 20} + + stream_ref = + :gun.post( + conn_pid, + "/v1/features", + [ + {"content-type", "application/json"} + ], + Jason.encode!(body) + ) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{ + "location" => %{"latitude" => ^latitude, "longitude" => ^longitude}, + "name" => name + } = Jason.decode!(body) + + assert name == "#{latitude},#{longitude}" + end) + end end end diff --git a/test/support/route_guide_transocde.pb.ex b/test/support/route_guide_transcode.pb.ex similarity index 100% rename from test/support/route_guide_transocde.pb.ex rename to test/support/route_guide_transcode.pb.ex diff --git a/test/support/route_guide_transocde.proto b/test/support/route_guide_transcode.proto similarity index 82% rename from test/support/route_guide_transocde.proto rename to test/support/route_guide_transcode.proto index 67bf7e9e..146ead13 100644 --- a/test/support/route_guide_transocde.proto +++ b/test/support/route_guide_transcode.proto @@ -11,14 +11,22 @@ package routeguide_transcode; service RouteGuide { rpc GetFeature(Point) returns (Feature) { option (google.api.http) = { - get: "/v1/feature/{latitude}/{longitude}" + get: "/v1/features/{latitude}/{longitude}" }; } + rpc ListFeatures(Rectangle) returns (stream Feature) { option (google.api.http) = { - get: "/v1/feature" + get: "/v1/features" + }; + } + + rpc CreateFeature(Point) returns (Feature) { + option (google.api.http) = { + post: "/v1/features" }; } + rpc RecordRoute(stream Point) returns (RouteSummary) {} rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} } diff --git a/test/support/route_guide_transocde.svc.ex b/test/support/route_guide_transcode.svc.ex similarity index 71% rename from test/support/route_guide_transocde.svc.ex rename to test/support/route_guide_transcode.svc.ex index adf27fdd..79884efd 100644 --- a/test/support/route_guide_transocde.svc.ex +++ b/test/support/route_guide_transcode.svc.ex @@ -10,7 +10,7 @@ defmodule RouteguideTranscode.RouteGuide.Service do __unknown_fields__: [], additional_bindings: [], body: "", - pattern: {:get, "/v1/feature/{latitude}/{longitude}"}, + pattern: {:get, "/v1/features/{latitude}/{longitude}"}, response_body: "", selector: "" } @@ -24,7 +24,21 @@ defmodule RouteguideTranscode.RouteGuide.Service do __unknown_fields__: [], additional_bindings: [], body: "", - pattern: {:get, "/v1/feature"}, + pattern: {:get, "/v1/features"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:CreateFeature, RouteguideTranscode.Point, RouteguideTranscode.Feature, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:post, "/v1/features"}, response_body: "", selector: "" } From 2c4c5541718a7aa44947dfecd372ec23260a5f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 15 Sep 2022 10:20:09 +0200 Subject: [PATCH 41/73] add query decoder --- lib/grpc/server/transcode/query.ex | 89 +++++++++++++++++++++++ test/grpc/server/transcode/query_test.exs | 30 ++++++++ 2 files changed, 119 insertions(+) create mode 100644 lib/grpc/server/transcode/query.ex create mode 100644 test/grpc/server/transcode/query_test.exs diff --git a/lib/grpc/server/transcode/query.ex b/lib/grpc/server/transcode/query.ex new file mode 100644 index 00000000..4036e1b1 --- /dev/null +++ b/lib/grpc/server/transcode/query.ex @@ -0,0 +1,89 @@ +defmodule GRPC.Server.Transcode.Query do + # This module is based on https://github.com/elixir-plug/plug/blob/main/lib/plug/conn/query.ex + + @moduledoc """ + Decoding of URL-encoded queries as per the rules outlined in the documentation for [`google.api.HttpRule`](https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule) + + It provides similar functionality to `URI.decode_query/3` or `Plug.Conn.Query.decode/4` with the following differences: + + 1. A repeated key is treated as a list of values + 1. Sub-paths on the form `path.subpath` are decoded as nested maps + 1. Sub-paths with the same leaf key are decoded as a list + """ + + @doc """ + Decodes the given `query`. + + The `query` is assumed to be encoded in the "x-www-form-urlencoded" format. + + `acc` is the initial "accumulator" where decoded values will be added. + + ## Examples + + iex> decode("a=A&b=B") + %{"a" => "A", "b" => "B"} + + iex> decode("param=A¶m=B") + %{"param" => ["A", "B"]} + + iex> decode("root.a=A&root.b=B") + %{"root" => %{"a" => "A", "b" => "B"}} + + iex> decode("root.a=A&root.a=B") + %{"root" => ["A", "B"]} + + """ + @spec decode(String.t(), map()) :: %{optional(String.t()) => term()} + def decode(query, acc \\ %{}) + + def decode("", acc) do + acc + end + + def decode(query, acc) when is_binary(query) do + parts = :binary.split(query, "&", [:global]) + + Enum.reduce( + Enum.reverse(parts), + acc, + &decode_www_pair(&1, &2) + ) + end + + defp decode_www_pair("", acc) do + acc + end + + defp decode_www_pair(binary, acc) do + current = + case :binary.split(binary, "=") do + [key, value] -> + {decode_www_form(key), decode_www_form(value)} + + [key] -> + {decode_www_form(key), ""} + end + + decode_pair(current, acc) + end + + defp decode_www_form(value) do + URI.decode_www_form(value) + end + + defp decode_pair({key, value}, acc) do + parts = :binary.split(key, ".", [:global]) + assign_map(parts, value, acc) + end + + defp assign_map(parts, value, acc) do + {_, acc} = + get_and_update_in(acc, Enum.map(parts, &Access.key(&1, %{})), fn + prev when prev == %{} -> {prev, value} + prev when is_list(prev) -> {prev, [value | prev]} + prev -> {prev, [value, prev]} + end) + + acc + end +end diff --git a/test/grpc/server/transcode/query_test.exs b/test/grpc/server/transcode/query_test.exs new file mode 100644 index 00000000..ac208a74 --- /dev/null +++ b/test/grpc/server/transcode/query_test.exs @@ -0,0 +1,30 @@ +defmodule GRPC.Transcode.QueryTest do + use ExUnit.Case, async: true + alias GRPC.Server.Transcode.Query + + test "`a=b&c=d` should be decoded as a map" do + assert %{"a" => "b", "c" => "d"} == Query.decode("a=b&c=d") + end + + test "`param=A¶m=B` should be decoded as a list" do + assert %{"param" => ["A", "B"]} == Query.decode("param=A¶m=B") + end + + test "`root.a=A&root.b=B` should be decoded as a nested map" do + assert %{"root" => %{"a" => "A", "b" => "B"}} == Query.decode("root.a=A&root.b=B") + end + + test "`root.a=A&root.a=B` should be decoded as a nested map with a list" do + assert %{"root" => %{"a" => ["A", "B"]}} == Query.decode("root.a=A&root.a=B") + end + + test "deeply nested map should be decoded" do + assert %{"root" => %{"a" => %{"b" => %{"c" => %{"d" => "A"}}}, "b" => "B"}, "c" => "C"} == + Query.decode("root.a.b.c.d=A&root.b=B&c=C") + end + + test "pairs without value are decoded as `\"\"`" do + assert %{"param" => "", "a" => "A"} == + Query.decode("param=&a=A") + end +end From 6118f5cf0b921f9ba22e379a1e217d09e8219e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 15 Sep 2022 11:16:36 +0200 Subject: [PATCH 42/73] include query string in request mapping --- lib/grpc/server/transcode.ex | 7 +- test/grpc/integration/server_test.exs | 80 ++++++++++++++++------- test/support/route_guide_transcode.pb.ex | 46 ------------- test/support/route_guide_transcode.proto | 59 ----------------- test/support/route_guide_transcode.svc.ex | 62 ------------------ test/support/transcode_messages.pb.ex | 24 +++++++ test/support/transcode_messages.proto | 44 +++++++++++++ test/support/transcode_messages.svc.ex | 67 +++++++++++++++++++ 8 files changed, 196 insertions(+), 193 deletions(-) delete mode 100644 test/support/route_guide_transcode.pb.ex delete mode 100644 test/support/route_guide_transcode.proto delete mode 100644 test/support/route_guide_transcode.svc.ex create mode 100644 test/support/transcode_messages.pb.ex create mode 100644 test/support/transcode_messages.proto create mode 100644 test/support/transcode_messages.svc.ex diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index c5410734..07c87650 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,8 +1,11 @@ defmodule GRPC.Server.Transcode do + alias __MODULE__.Query + @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} - def map_request(body_request, path_bindings, _query_string, req_mod) do + def map_request(body_request, path_bindings, query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) - request = Map.merge(path_bindings, body_request) + query = Query.decode(query_string) + request = Enum.reduce([query, body_request], path_bindings, &Map.merge(&2, &1)) Protobuf.JSON.from_decoded(request, req_mod) end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 1d97b376..5c1ebdf3 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -9,17 +9,28 @@ defmodule GRPC.Integration.ServerTest do end end - defmodule FeatureTranscodeServer do + defmodule TranscodeServer do use GRPC.Server, - service: RouteguideTranscode.RouteGuide.Service, + service: Transcode.Messaging.Service, http_transcode: true - def get_feature(point, _stream) do - Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") + def get_message(msg_request, _stream) do + Transcode.Message.new(name: msg_request.name, text: "get_message") end - def create_feature(point, _stream) do - Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") + def create_message(msg, _stream) do + msg + end + + def get_message_with_query(msg_request, _stream) do + Transcode.Message.new(name: msg_request.name, text: "get_message_with_query") + end + + def get_message_with_subpath_query(msg_request, _stream) do + Transcode.Message.new( + name: msg_request.message.name, + text: "get_message_with_subpath_query" + ) end end @@ -258,14 +269,13 @@ defmodule GRPC.Integration.ServerTest do describe "http/json transcode" do test "can transcode path params" do - run_server([FeatureTranscodeServer], fn port -> - latitude = 10 - longitude = 20 + run_server([TranscodeServer], fn port -> + name = "foo" {:ok, conn_pid} = :gun.open('localhost', port) stream_ref = - :gun.get(conn_pid, "/v1/features/#{latitude}/#{longitude}", [ + :gun.get(conn_pid, "/v1/messages/#{name}", [ {"content-type", "application/json"} ]) @@ -273,42 +283,64 @@ defmodule GRPC.Integration.ServerTest do assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) assert %{ - "location" => %{"latitude" => ^latitude, "longitude" => ^longitude}, - "name" => name + "name" => ^name, + "text" => _name } = Jason.decode!(body) + end) + end + + test "can transcode query params" do + run_server([TranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + + stream_ref = + :gun.get(conn_pid, "/v1/messages?name=some_name", [ + {"content-type", "application/json"} + ]) - assert name == "#{latitude},#{longitude}" + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{ + "name" => "some_name", + "text" => "get_message_with_query" + } = Jason.decode!(body) end) end test "service methods can have the same path but different methods in http rule option" do - run_server([FeatureTranscodeServer], fn port -> - latitude = 10 - longitude = 20 - + run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) - body = %{"latitude" => latitude, "longitude" => 20} + payload = %{"name" => "foo", "text" => "bar"} stream_ref = :gun.post( conn_pid, - "/v1/features", + "/v1/messages", [ {"content-type", "application/json"} ], - Jason.encode!(body) + Jason.encode!(payload) ) assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + assert ^payload = Jason.decode!(body) + + stream_ref = + :gun.get(conn_pid, "/v1/messages?name=another_name", [ + {"content-type", "application/json"} + ]) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + assert %{ - "location" => %{"latitude" => ^latitude, "longitude" => ^longitude}, - "name" => name + "name" => "another_name", + "text" => "get_message_with_query" } = Jason.decode!(body) - - assert name == "#{latitude},#{longitude}" end) end end diff --git a/test/support/route_guide_transcode.pb.ex b/test/support/route_guide_transcode.pb.ex deleted file mode 100644 index 1079307f..00000000 --- a/test/support/route_guide_transcode.pb.ex +++ /dev/null @@ -1,46 +0,0 @@ -defmodule RouteguideTranscode.Point do - @moduledoc false - - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :latitude, 1, type: :int32 - field :longitude, 2, type: :int32 -end - -defmodule RouteguideTranscode.Rectangle do - @moduledoc false - - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :lo, 1, type: RouteguideTranscode.Point - field :hi, 2, type: RouteguideTranscode.Point -end - -defmodule RouteguideTranscode.Feature do - @moduledoc false - - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :name, 1, type: :string - field :location, 2, type: RouteguideTranscode.Point -end - -defmodule RouteguideTranscode.RouteNote do - @moduledoc false - - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :location, 1, type: RouteguideTranscode.Point - field :message, 2, type: :string -end - -defmodule RouteguideTranscode.RouteSummary do - @moduledoc false - - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :point_count, 1, type: :int32, json_name: "pointCount" - field :feature_count, 2, type: :int32, json_name: "featureCount" - field :distance, 3, type: :int32 - field :elapsed_time, 4, type: :int32, json_name: "elapsedTime" -end diff --git a/test/support/route_guide_transcode.proto b/test/support/route_guide_transcode.proto deleted file mode 100644 index 146ead13..00000000 --- a/test/support/route_guide_transcode.proto +++ /dev/null @@ -1,59 +0,0 @@ -syntax = "proto3"; - -option java_multiple_files = true; -option java_package = "io.grpc.examples.routeguide"; -option java_outer_classname = "RouteGuideProto"; - -import "google/api/annotations.proto"; - -package routeguide_transcode; - -service RouteGuide { - rpc GetFeature(Point) returns (Feature) { - option (google.api.http) = { - get: "/v1/features/{latitude}/{longitude}" - }; - } - - rpc ListFeatures(Rectangle) returns (stream Feature) { - option (google.api.http) = { - get: "/v1/features" - }; - } - - rpc CreateFeature(Point) returns (Feature) { - option (google.api.http) = { - post: "/v1/features" - }; - } - - rpc RecordRoute(stream Point) returns (RouteSummary) {} - rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} -} - -message Point { - int32 latitude = 1; - int32 longitude = 2; -} - -message Rectangle { - Point lo = 1; - Point hi = 2; -} - -message Feature { - string name = 1; - Point location = 2; -} - -message RouteNote { - Point location = 1; - string message = 2; -} - -message RouteSummary { - int32 point_count = 1; - int32 feature_count = 2; - int32 distance = 3; - int32 elapsed_time = 4; -} diff --git a/test/support/route_guide_transcode.svc.ex b/test/support/route_guide_transcode.svc.ex deleted file mode 100644 index 79884efd..00000000 --- a/test/support/route_guide_transcode.svc.ex +++ /dev/null @@ -1,62 +0,0 @@ -defmodule RouteguideTranscode.RouteGuide.Service do - @moduledoc false - - use GRPC.Service, name: "routeguide_transcode.RouteGuide", protoc_gen_elixir_version: "0.11.0" - - rpc(:GetFeature, RouteguideTranscode.Point, RouteguideTranscode.Feature, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/features/{latitude}/{longitude}"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:ListFeatures, RouteguideTranscode.Rectangle, stream(RouteguideTranscode.Feature), %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/features"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:CreateFeature, RouteguideTranscode.Point, RouteguideTranscode.Feature, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:post, "/v1/features"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:RecordRoute, stream(RouteguideTranscode.Point), RouteguideTranscode.RouteSummary, %{}) - - rpc( - :RouteChat, - stream(RouteguideTranscode.RouteNote), - stream(RouteguideTranscode.RouteNote), - %{} - ) -end - -defmodule RouteguideTranscode.RouteGuide.Stub do - @moduledoc false - - use GRPC.Stub, service: RouteguideTranscode.RouteGuide.Service -end diff --git a/test/support/transcode_messages.pb.ex b/test/support/transcode_messages.pb.ex new file mode 100644 index 00000000..98c49e8f --- /dev/null +++ b/test/support/transcode_messages.pb.ex @@ -0,0 +1,24 @@ +defmodule Transcode.GetMessageRequest do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string +end + +defmodule Transcode.Message do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string + field :text, 2, type: :string +end + +defmodule Transcode.NestedGetMessageRequest do + @moduledoc false + + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :message, 1, type: Transcode.GetMessageRequest +end \ No newline at end of file diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto new file mode 100644 index 00000000..ee724ed8 --- /dev/null +++ b/test/support/transcode_messages.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +import "google/api/annotations.proto"; + +package transcode; + +service Messaging { + rpc GetMessage(GetMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/messages/{name}" + }; + } + + rpc GetMessageWithQuery(GetMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/messages" + }; + } + + rpc CreateMessage(Message) returns (Message) { + option (google.api.http) = { + post: "/v1/messages" + }; + } + + rpc GetMessageWithSubpathQuery(NestedGetMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/messages/nested" + }; + } +} + +message GetMessageRequest { + string name = 1; // Mapped to URL path. +} + +message Message { + string name = 1; + string text = 2; +} + +message NestedGetMessageRequest { + GetMessageRequest message = 1; +} diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex new file mode 100644 index 00000000..20d5ba37 --- /dev/null +++ b/test/support/transcode_messages.svc.ex @@ -0,0 +1,67 @@ +defmodule Transcode.Messaging.Service do + @moduledoc false + + use GRPC.Service, name: "transcode.Messaging", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetMessage, Transcode.GetMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithQuery, Transcode.GetMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:CreateMessage, Transcode.Message, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:post, "/v1/messages"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithSubpathQuery, Transcode.NestedGetMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/nested"}, + response_body: "", + selector: "" + } + } + }) +end + +defmodule Transcode.Messaging.Stub do + @moduledoc false + + use GRPC.Stub, service: Transcode.Messaging.Service +end \ No newline at end of file From c36293b0ccc4020fc3972a8c105ac261c559115d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 15 Sep 2022 16:06:07 +0200 Subject: [PATCH 43/73] map request using HttpRule.body --- lib/grpc/server.ex | 21 +++++++++++----- lib/grpc/server/transcode.ex | 24 ++++++++++++++++++ lib/grpc/service.ex | 4 +++ test/grpc/integration/server_test.exs | 26 +++++++++++++++++++ test/grpc/server/transcode_test.exs | 35 ++++++++++++++++++++++++++ test/support/transcode_messages.pb.ex | 2 +- test/support/transcode_messages.proto | 11 ++++++-- test/support/transcode_messages.svc.ex | 16 +++++++++++- 8 files changed, 129 insertions(+), 10 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index fff752d5..936dd4e4 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -49,14 +49,14 @@ defmodule GRPC.Server do http_transcode = opts[:http_transcode] || false routes = - for {name, _, _, options} <- service_mod.__rpc_calls__, reduce: [] do + for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, reduce: [] do acc -> path = "/#{service_name}/#{name}" http_paths = if http_transcode and Map.has_key?(options, :http) do - %{value: http_opts} = Map.fetch!(options, :http) - [{:http_transcode, Transcode.build_route(http_opts)}] + %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) + [{:http_transcode, Transcode.build_route(http_rule)}] else [] end @@ -84,7 +84,7 @@ defmodule GRPC.Server do end if http_transcode and Map.has_key?(options, :http) do - %{value: http_rule} = Map.fetch!(options, :http) + %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) {http_method, _} = spec = Transcode.build_route(http_rule) http_path = Transcode.to_path(spec) @@ -161,6 +161,7 @@ defmodule GRPC.Server do false, res_stream, %{ + rpc: rpc, request_mod: req_mod, codec: codec, adapter: adapter, @@ -170,11 +171,19 @@ defmodule GRPC.Server do func_name ) do {:ok, data} = adapter.read_body(payload) + request_body = codec.decode(data, req_mod) + + request_body = + if rule = GRPC.Service.rpc_options(rpc, :http) do + Transcode.map_request_body(rule.value, request_body) + else + request_body + end + bindings = adapter.get_bindings(payload) qs = adapter.get_qs(payload) - request = codec.decode(data, req_mod) - case Transcode.map_request(request, bindings, qs, req_mod) do + case Transcode.map_request(request_body, bindings, qs, req_mod) do {:ok, request} -> call_with_interceptors(res_stream, func_name, stream, request) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 07c87650..cd15e668 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,6 +1,15 @@ defmodule GRPC.Server.Transcode do alias __MODULE__.Query + # Leaf request fields (recursive expansion nested messages in the request message) are classified into three categories: + # + # 1. Fields referred by the path template. They are passed via the URL path. + # 2. Fields referred by the HttpRule.body. They are passed via the HTTP request body. + # 3. All other fields are passed via the URL query parameters, and the parameter name is the field path in the request message. A repeated field can be represented as multiple query parameters under the same name. + # + # If HttpRule.body is "*", there is no URL query parameter, all fields are passed via URL path and HTTP request body. + # + # If HttpRule.body is omitted, there is no HTTP request body, all fields are passed via URL path and URL query parameters. @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request(body_request, path_bindings, query_string, req_mod) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) @@ -10,6 +19,21 @@ defmodule GRPC.Server.Transcode do Protobuf.JSON.from_decoded(request, req_mod) end + @spec map_request_body(term(), term()) :: term() + def map_request_body(%Google.Api.HttpRule{body: "*"}, request_body), do: request_body + def map_request_body(%Google.Api.HttpRule{body: ""}, request_body), do: request_body + + # TODO The field is required to be present on the toplevel request message + def map_request_body(%Google.Api.HttpRule{body: field}, request_body), + do: %{field => request_body} + + @spec map_response_body(Google.Api.HttpRule.t(), map()) :: map() + def map_response_body(%Google.Api.HttpRule{response_body: ""}, response_body), do: response_body + + # TODO The field is required to be present on the toplevel response message + def map_response_body(%Google.Api.HttpRule{response_body: field}, response_body), + do: Map.get(response_body, field) + @spec to_path(term()) :: String.t() def to_path({_method, {_bindings, segments}} = _spec) do match = diff --git a/lib/grpc/service.ex b/lib/grpc/service.ex index 39b7edd9..1c75c97f 100644 --- a/lib/grpc/service.ex +++ b/lib/grpc/service.ex @@ -59,4 +59,8 @@ defmodule GRPC.Service do def grpc_type({_, {_, true}, {_, false}, _}), do: :client_stream def grpc_type({_, {_, false}, {_, true}, _}), do: :server_stream def grpc_type({_, {_, true}, {_, true}, _}), do: :bidi_stream + + def rpc_options({_, _, _, options}), do: options + + def rpc_options({_, _, _, options}, type), do: Map.get(options, type) end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 5c1ebdf3..7d0ff315 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -22,6 +22,13 @@ defmodule GRPC.Integration.ServerTest do msg end + def create_message_with_nested_body(msg_request, _stream) do + Transcode.Message.new( + name: msg_request.message.name, + text: "create_message_with_nested_body" + ) + end + def get_message_with_query(msg_request, _stream) do Transcode.Message.new(name: msg_request.name, text: "get_message_with_query") end @@ -308,6 +315,25 @@ defmodule GRPC.Integration.ServerTest do end) end + test "can map request body using HttpRule.body" do + run_server([TranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + + body = %{"name" => "name"} + + stream_ref = + :gun.post(conn_pid, "/v1/messages/nested", [ + {"content-type", "application/json"} + ], Jason.encode!(body)) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{"name" => "name", "text" => "create_message_with_nested_body"} = + Jason.decode!(body) + end) + end + test "service methods can have the same path but different methods in http rule option" do run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index 8865b4b9..f1207ba3 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -15,6 +15,41 @@ defmodule GRPC.TranscodeTest do assert Routeguide.Point.new(latitude: 1, longitude: 2) == request end + test "map_request_body/2 with HttpRule.body: '*'" do + rule = Google.Api.HttpRule.new(body: "*") + request_body = %{"a" => "b"} + + assert request_body == Transcode.map_request_body(rule, request_body) + end + + test "map_request_body/2 with empty HttpRule.body" do + rule = Google.Api.HttpRule.new(body: "") + request_body = %{"a" => "b"} + + assert request_body == Transcode.map_request_body(rule, request_body) + end + + test "map_request_body/2 with HttpRule.body: " do + rule = Google.Api.HttpRule.new(body: "message") + request_body = %{"a" => "b"} + + assert %{"message" => %{"a" => "b"}} == Transcode.map_request_body(rule, request_body) + end + + test "map_response_body/2 with empty HttpRule.response_body" do + rule = Google.Api.HttpRule.new(response_body: "") + request_body = %{"a" => "b"} + + assert request_body == Transcode.map_response_body(rule, request_body) + end + + test "map_response_body/2 with HttpRule.response_body: " do + rule = Google.Api.HttpRule.new(response_body: "message") + request_body = %{"message" => %{"a" => "b"}} + + assert %{"a" => "b"} == Transcode.map_response_body(rule, request_body) + end + test "build_route/1 returns a route with {http_method, route} based on the http rule" do rule = build_simple_rule(:get, "/v1/messages/{message_id}") assert {:get, {params, segments}} = Transcode.build_route(rule) diff --git a/test/support/transcode_messages.pb.ex b/test/support/transcode_messages.pb.ex index 98c49e8f..0145db10 100644 --- a/test/support/transcode_messages.pb.ex +++ b/test/support/transcode_messages.pb.ex @@ -15,7 +15,7 @@ defmodule Transcode.Message do field :text, 2, type: :string end -defmodule Transcode.NestedGetMessageRequest do +defmodule Transcode.NestedMessageRequest do @moduledoc false use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto index ee724ed8..197838e1 100644 --- a/test/support/transcode_messages.proto +++ b/test/support/transcode_messages.proto @@ -23,7 +23,14 @@ service Messaging { }; } - rpc GetMessageWithSubpathQuery(NestedGetMessageRequest) returns (Message) { + rpc CreateMessageWithNestedBody(NestedMessageRequest) returns (Message) { + option (google.api.http) = { + post: "/v1/messages/nested", + body: "message" + }; + } + + rpc GetMessageWithSubpathQuery(NestedMessageRequest) returns (Message) { option (google.api.http) = { get: "/v1/messages/nested" }; @@ -39,6 +46,6 @@ message Message { string text = 2; } -message NestedGetMessageRequest { +message NestedMessageRequest { GetMessageRequest message = 1; } diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex index 20d5ba37..283bb307 100644 --- a/test/support/transcode_messages.svc.ex +++ b/test/support/transcode_messages.svc.ex @@ -45,7 +45,21 @@ defmodule Transcode.Messaging.Service do } }) - rpc(:GetMessageWithSubpathQuery, Transcode.NestedGetMessageRequest, Transcode.Message, %{ + rpc(:CreateMessageWithNestedBody, Transcode.NestedMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "message", + pattern: {:post, "/v1/messages/nested"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithSubpathQuery, Transcode.NestedMessageRequest, Transcode.Message, %{ http: %{ type: Google.Api.PbExtension, value: %Google.Api.HttpRule{ From 152a2bf26b8f3f214d7ae89343128cf0cc3374e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 15 Sep 2022 16:32:54 +0200 Subject: [PATCH 44/73] comply to request mapping rules by checking HttpRule.body param --- lib/grpc/server.ex | 11 +----- lib/grpc/server/transcode.ex | 54 ++++++++++++++++++-------- test/grpc/integration/server_test.exs | 11 ++++-- test/grpc/server/transcode_test.exs | 44 ++++++++++----------- test/support/transcode_messages.pb.ex | 2 +- test/support/transcode_messages.proto | 1 + test/support/transcode_messages.svc.ex | 4 +- 7 files changed, 72 insertions(+), 55 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 936dd4e4..1b6f8e70 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -172,18 +172,11 @@ defmodule GRPC.Server do ) do {:ok, data} = adapter.read_body(payload) request_body = codec.decode(data, req_mod) - - request_body = - if rule = GRPC.Service.rpc_options(rpc, :http) do - Transcode.map_request_body(rule.value, request_body) - else - request_body - end - + rule = GRPC.Service.rpc_options(rpc, :http) || %{value: %{}} bindings = adapter.get_bindings(payload) qs = adapter.get_qs(payload) - case Transcode.map_request(request_body, bindings, qs, req_mod) do + case Transcode.map_request(rule.value, request_body, bindings, qs, req_mod) do {:ok, request} -> call_with_interceptors(res_stream, func_name, stream, request) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index cd15e668..448c0e73 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,30 +1,50 @@ defmodule GRPC.Server.Transcode do alias __MODULE__.Query - # Leaf request fields (recursive expansion nested messages in the request message) are classified into three categories: - # - # 1. Fields referred by the path template. They are passed via the URL path. - # 2. Fields referred by the HttpRule.body. They are passed via the HTTP request body. - # 3. All other fields are passed via the URL query parameters, and the parameter name is the field path in the request message. A repeated field can be represented as multiple query parameters under the same name. - # - # If HttpRule.body is "*", there is no URL query parameter, all fields are passed via URL path and HTTP request body. - # - # If HttpRule.body is omitted, there is no HTTP request body, all fields are passed via URL path and URL query parameters. - @spec map_request(map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} - def map_request(body_request, path_bindings, query_string, req_mod) do + @doc """ + Leaf request fields (recursive expansion nested messages in the request message) are classified into three categories: + + 1. Fields referred by the path template. They are passed via the URL path. + 2. Fields referred by the HttpRule.body. They are passed via the HTTP request body. + 3. All other fields are passed via the URL query parameters, and the parameter name is the field path in the request message. A repeated field can be represented as multiple query parameters under the same name. + + If HttpRule.body is "*", there is no URL query parameter, all fields are passed via URL path and HTTP request body. + + If HttpRule.body is omitted, there is no HTTP request body, all fields are passed via URL path and URL query parameters. + """ + @spec map_request(Google.Api.HttpRule.t(), map(), map(), String.t(), module()) :: + {:ok, struct()} | {:error, term()} + def map_request( + %Google.Api.HttpRule{body: ""}, + _body_request, + path_bindings, + query_string, + req_mod + ) do path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) query = Query.decode(query_string) - request = Enum.reduce([query, body_request], path_bindings, &Map.merge(&2, &1)) + request = Map.merge(path_bindings, query) + + Protobuf.JSON.from_decoded(request, req_mod) + end + + def map_request( + %Google.Api.HttpRule{} = rule, + body_request, + path_bindings, + _query_string, + req_mod + ) do + path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) + request = Map.merge(path_bindings, map_request_body(rule, body_request)) Protobuf.JSON.from_decoded(request, req_mod) end - @spec map_request_body(term(), term()) :: term() - def map_request_body(%Google.Api.HttpRule{body: "*"}, request_body), do: request_body - def map_request_body(%Google.Api.HttpRule{body: ""}, request_body), do: request_body + defp map_request_body(%Google.Api.HttpRule{body: "*"}, request_body), do: request_body + defp map_request_body(%Google.Api.HttpRule{body: ""}, request_body), do: request_body - # TODO The field is required to be present on the toplevel request message - def map_request_body(%Google.Api.HttpRule{body: field}, request_body), + defp map_request_body(%Google.Api.HttpRule{body: field}, request_body), do: %{field => request_body} @spec map_response_body(Google.Api.HttpRule.t(), map()) :: map() diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 7d0ff315..10adc3d9 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -322,9 +322,14 @@ defmodule GRPC.Integration.ServerTest do body = %{"name" => "name"} stream_ref = - :gun.post(conn_pid, "/v1/messages/nested", [ - {"content-type", "application/json"} - ], Jason.encode!(body)) + :gun.post( + conn_pid, + "/v1/messages/nested", + [ + {"content-type", "application/json"} + ], + Jason.encode!(body) + ) assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index f1207ba3..476c386f 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -2,38 +2,36 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true alias GRPC.Server.Transcode - test "map_requests/3 can map request body to protobuf struct" do - body_request = %{"latitude" => 1, "longitude" => 2} - {:ok, request} = Transcode.map_request(body_request, %{}, "", Routeguide.Point) - assert Routeguide.Point.new(latitude: 1, longitude: 2) == request - end - - test "map_requests/3 can merge request body with path bindings to protobuf struct" do - body_request = %{"latitude" => 1} - bindings = %{"longitude" => 2} - {:ok, request} = Transcode.map_request(body_request, bindings, "", Routeguide.Point) - assert Routeguide.Point.new(latitude: 1, longitude: 2) == request - end - - test "map_request_body/2 with HttpRule.body: '*'" do + test "map_request/5 with HttpRule.body: '*'" do rule = Google.Api.HttpRule.new(body: "*") - request_body = %{"a" => "b"} + request_body = %{"latitude" => 1, "longitude" => 2} + bindings = %{} + qs = "latitude=10&longitude=20" - assert request_body == Transcode.map_request_body(rule, request_body) + assert {:ok, %Routeguide.Point{latitude: 1, longitude: 2}} = + Transcode.map_request(rule, request_body, bindings, qs, Routeguide.Point) end - test "map_request_body/2 with empty HttpRule.body" do + test "map_request/5 with empty HttpRule.body" do rule = Google.Api.HttpRule.new(body: "") - request_body = %{"a" => "b"} + request_body = %{"latitude" => 10, "longitude" => 20} + bindings = %{"latitude" => 5} + qs = "longitude=6" - assert request_body == Transcode.map_request_body(rule, request_body) + assert {:ok, %Routeguide.Point{latitude: 5, longitude: 6}} = + Transcode.map_request(rule, request_body, bindings, qs, Routeguide.Point) end - test "map_request_body/2 with HttpRule.body: " do - rule = Google.Api.HttpRule.new(body: "message") - request_body = %{"a" => "b"} + test "map_request/2 with HttpRule.body: " do + rule = Google.Api.HttpRule.new(body: "location") + request_body = %{"latitude" => 1, "longitude" => 2} + bindings = %{"name" => "test"} + + assert {:ok, %Routeguide.Feature{name: "test", location: point}} = + Transcode.map_request(rule, request_body, bindings, "name=Foo", Routeguide.Feature) - assert %{"message" => %{"a" => "b"}} == Transcode.map_request_body(rule, request_body) + assert point.latitude == 1 + assert point.longitude == 2 end test "map_response_body/2 with empty HttpRule.response_body" do diff --git a/test/support/transcode_messages.pb.ex b/test/support/transcode_messages.pb.ex index 0145db10..6c8b2da0 100644 --- a/test/support/transcode_messages.pb.ex +++ b/test/support/transcode_messages.pb.ex @@ -21,4 +21,4 @@ defmodule Transcode.NestedMessageRequest do use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :message, 1, type: Transcode.GetMessageRequest -end \ No newline at end of file +end diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto index 197838e1..42518486 100644 --- a/test/support/transcode_messages.proto +++ b/test/support/transcode_messages.proto @@ -20,6 +20,7 @@ service Messaging { rpc CreateMessage(Message) returns (Message) { option (google.api.http) = { post: "/v1/messages" + body: "*" }; } diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex index 283bb307..0a205dec 100644 --- a/test/support/transcode_messages.svc.ex +++ b/test/support/transcode_messages.svc.ex @@ -37,7 +37,7 @@ defmodule Transcode.Messaging.Service do value: %Google.Api.HttpRule{ __unknown_fields__: [], additional_bindings: [], - body: "", + body: "*", pattern: {:post, "/v1/messages"}, response_body: "", selector: "" @@ -78,4 +78,4 @@ defmodule Transcode.Messaging.Stub do @moduledoc false use GRPC.Stub, service: Transcode.Messaging.Service -end \ No newline at end of file +end From f97122b1ab7e7169d50ce6a627ed2677aa7c94c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 16 Sep 2022 16:24:43 +0200 Subject: [PATCH 45/73] support newline delimeted json for transcoded server streams --- lib/grpc/server.ex | 3 +-- lib/grpc/server/stream.ex | 28 +++++++++++++++++++--- test/grpc/integration/server_test.exs | 32 ++++++++++++++++++++++++++ test/support/transcode_messages.proto | 6 +++++ test/support/transcode_messages.svc.ex | 14 +++++++++++ 5 files changed, 78 insertions(+), 5 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 1b6f8e70..590fb5d2 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -371,11 +371,10 @@ defmodule GRPC.Server do """ @spec send_reply(GRPC.Server.Stream.t(), struct()) :: GRPC.Server.Stream.t() def send_reply( - %{__interface__: interface, http_transcode: transcode} = stream, + %{__interface__: interface} = stream, reply, opts \\ [] ) do - opts = Keyword.put(opts, :http_transcode, transcode) interface[:send_reply].(stream, reply, opts) end diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index 6111e81d..b8158510 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -58,10 +58,32 @@ defmodule GRPC.Server.Stream do http_transcode: false, __interface__: %{send_reply: &__MODULE__.send_reply/3} - def send_reply(%{adapter: adapter, codec: codec} = stream, reply, opts) do - # {:ok, data, _size} = reply |> codec.encode() |> GRPC.Message.to_data() + def send_reply( + %{grpc_type: :server_stream, codec: codec, http_transcode: true} = stream, + reply, + opts + ) do data = codec.encode(reply) - adapter.send_reply(stream.payload, data, Keyword.put(opts, :codec, codec)) + + do_send_reply(stream, [data, "\n"], opts) + end + + def send_reply(%{codec: codec} = stream, reply, opts) do + do_send_reply(stream, codec.encode(reply), opts) + end + + defp do_send_reply( + %{adapter: adapter, codec: codec, http_transcode: http_transcode} = stream, + data, + opts + ) do + opts = + opts + |> Keyword.put(:codec, codec) + |> Keyword.put(:http_transcode, http_transcode) + + adapter.send_reply(stream.payload, data, opts) + stream end end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 10adc3d9..645e338d 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -18,6 +18,18 @@ defmodule GRPC.Integration.ServerTest do Transcode.Message.new(name: msg_request.name, text: "get_message") end + def stream_messages(msg_request, stream) do + Enum.each(1..5, fn i -> + msg = + Transcode.Message.new( + name: msg_request.name, + text: "#{i}" + ) + + GRPC.Server.send_reply(stream, msg) + end) + end + def create_message(msg, _stream) do msg end @@ -339,6 +351,26 @@ defmodule GRPC.Integration.ServerTest do end) end + test "can send streaming responses" do + run_server([TranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + + stream_ref = + :gun.get( + conn_pid, + "/v1/messages/stream/stream_test", + [ + {"content-type", "application/json"} + ] + ) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + msgs = String.split(body, "\n", trim: true) + assert length(msgs) == 5 + end) + end + test "service methods can have the same path but different methods in http rule option" do run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto index 42518486..116b0a79 100644 --- a/test/support/transcode_messages.proto +++ b/test/support/transcode_messages.proto @@ -11,6 +11,12 @@ service Messaging { }; } + rpc StreamMessages(GetMessageRequest) returns (stream Message) { + option (google.api.http) = { + get: "/v1/messages/stream/{name}" + }; + } + rpc GetMessageWithQuery(GetMessageRequest) returns (Message) { option (google.api.http) = { get: "/v1/messages" diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex index 0a205dec..f9c20f80 100644 --- a/test/support/transcode_messages.svc.ex +++ b/test/support/transcode_messages.svc.ex @@ -17,6 +17,20 @@ defmodule Transcode.Messaging.Service do } }) + rpc(:StreamMessages, Transcode.GetMessageRequest, stream(Transcode.Message), %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/stream/{name}"}, + response_body: "", + selector: "" + } + } + }) + rpc(:GetMessageWithQuery, Transcode.GetMessageRequest, Transcode.Message, %{ http: %{ type: Google.Api.PbExtension, From 96fb7639c07d1d933298c83260e13ae6ea14d9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 19 Sep 2022 10:29:18 +0200 Subject: [PATCH 46/73] base generator on protobuf 0.11.0 --- lib/grpc/protoc/cli.ex | 144 +-------------------------- lib/grpc/protoc/generator/service.ex | 3 +- mix.exs | 2 +- mix.lock | 2 +- priv/templates/service.ex.eex | 2 - 5 files changed, 6 insertions(+), 147 deletions(-) diff --git a/lib/grpc/protoc/cli.ex b/lib/grpc/protoc/cli.ex index 84aa2215..19bc6537 100644 --- a/lib/grpc/protoc/cli.ex +++ b/lib/grpc/protoc/cli.ex @@ -48,8 +48,8 @@ defmodule GRPC.Protoc.CLI do ctx = %Context{} - |> parse_params(request.parameter || "") - |> find_types(request.proto_file, request.file_to_generate) + |> Protobuf.Protoc.CLI.parse_params(request.parameter || "") + |> Protobuf.Protoc.CLI.find_types(request.proto_file, request.file_to_generate) files = Enum.flat_map(request.file_to_generate, fn file -> @@ -59,7 +59,7 @@ defmodule GRPC.Protoc.CLI do Google.Protobuf.Compiler.CodeGeneratorResponse.new( file: files, - supported_features: supported_features() + supported_features: Protobuf.Protoc.CLI.supported_features() ) |> Protobuf.encode_to_iodata() |> IO.binwrite() @@ -69,144 +69,6 @@ defmodule GRPC.Protoc.CLI do raise "invalid arguments. See protoc-gen-elixir --help." end - def supported_features() do - # The only available feature is proto3 with optional fields. - # This is backwards compatible with proto2 optional fields. - Google.Protobuf.Compiler.CodeGeneratorResponse.Feature.value(:FEATURE_PROTO3_OPTIONAL) - end - - # Made public for testing. - @doc false - def parse_params(%Context{} = ctx, params_str) when is_binary(params_str) do - params_str - |> String.split(",") - |> Enum.reduce(ctx, &parse_param/2) - end - - defp parse_param("plugins=" <> plugins, ctx) do - %Context{ctx | plugins: String.split(plugins, "+")} - end - - defp parse_param("gen_descriptors=" <> value, ctx) do - case value do - "true" -> - %Context{ctx | gen_descriptors?: true} - - other -> - raise "invalid value for gen_descriptors option, expected \"true\", got: #{inspect(other)}" - end - end - - defp parse_param("package_prefix=" <> package, ctx) do - if package == "" do - raise "package_prefix can't be empty" - else - %Context{ctx | package_prefix: package} - end - end - - defp parse_param("transform_module=" <> module, ctx) do - %Context{ctx | transform_module: Module.concat([module])} - end - - defp parse_param("one_file_per_module=" <> value, ctx) do - case value do - "true" -> - %Context{ctx | one_file_per_module?: true} - - other -> - raise "invalid value for one_file_per_module option, expected \"true\", got: #{inspect(other)}" - end - end - - defp parse_param("include_docs=" <> value, ctx) do - case value do - "true" -> - %Context{ctx | include_docs?: true} - - other -> - raise "invalid value for include_docs option, expected \"true\", got: #{inspect(other)}" - end - end - - defp parse_param(_unknown, ctx) do - ctx - end - - # Made public for testing. - @doc false - @spec find_types(Context.t(), [Google.Protobuf.FileDescriptorProto.t()], [String.t()]) :: - Context.t() - def find_types(%Context{} = ctx, descs, files_to_generate) - when is_list(descs) and is_list(files_to_generate) do - global_type_mapping = - Map.new(descs, fn %Google.Protobuf.FileDescriptorProto{name: filename} = desc -> - {filename, find_types_in_proto(ctx, desc, files_to_generate)} - end) - - %Context{ctx | global_type_mapping: global_type_mapping} - end - - defp find_types_in_proto( - %Context{} = ctx, - %Google.Protobuf.FileDescriptorProto{} = desc, - files_to_generate - ) do - # Only take package_prefix into consideration for files that we're directly generating. - package_prefix = - if desc.name in files_to_generate do - ctx.package_prefix - else - nil - end - - ctx = - %Protobuf.Protoc.Context{ - namespace: [], - package_prefix: package_prefix, - package: desc.package - } - |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc) - - find_types_in_descriptor(_types = %{}, ctx, desc.message_type ++ desc.enum_type) - end - - defp find_types_in_descriptor(types_acc, ctx, descs) when is_list(descs) do - Enum.reduce(descs, types_acc, &find_types_in_descriptor(_acc = &2, ctx, _desc = &1)) - end - - defp find_types_in_descriptor( - types_acc, - ctx, - %Google.Protobuf.DescriptorProto{name: name} = desc - ) do - new_ctx = update_in(ctx.namespace, &(&1 ++ [name])) - - types_acc - |> update_types(ctx, name) - |> find_types_in_descriptor(new_ctx, desc.enum_type) - |> find_types_in_descriptor(new_ctx, desc.nested_type) - end - - defp find_types_in_descriptor( - types_acc, - ctx, - %Google.Protobuf.EnumDescriptorProto{name: name} - ) do - update_types(types_acc, ctx, name) - end - - defp update_types(types, %Context{namespace: ns, package: pkg} = ctx, name) do - type_name = Protobuf.Protoc.Generator.Util.mod_name(ctx, ns ++ [name]) - - mapping_name = - ([pkg] ++ ns ++ [name]) - |> Enum.reject(&is_nil/1) - |> Enum.join(".") - - Map.put(types, "." <> mapping_name, %{type_name: type_name}) - end - if Version.match?(System.version(), "~> 1.13") do defp binread_all!(device) do case IO.binread(device, :eof) do diff --git a/lib/grpc/protoc/generator/service.ex b/lib/grpc/protoc/generator/service.ex index f7aeea10..8af7eb20 100644 --- a/lib/grpc/protoc/generator/service.ex +++ b/lib/grpc/protoc/generator/service.ex @@ -35,8 +35,7 @@ defmodule GRPC.Protoc.Generator.Service do service_name: name, methods: methods, descriptor_fun_body: descriptor_fun_body, - version: Util.version(), - module_doc?: ctx.include_docs? + version: Util.version() ) )} end diff --git a/mix.exs b/mix.exs index 0ae049f8..e4cf4f9f 100644 --- a/mix.exs +++ b/mix.exs @@ -51,7 +51,7 @@ defmodule GRPC.Mixfile do {:gun, "~> 2.0.1", hex: :grpc_gun}, {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, - {:protobuf, github: "elixir-protobuf/protobuf", branch: "main"}, + {:protobuf, "~> 0.11"}, {:ex_doc, "~> 0.28.0", only: :dev}, {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false}, {:googleapis, diff --git a/mix.lock b/mix.lock index 3ff0ba5b..c6c5a74f 100644 --- a/mix.lock +++ b/mix.lock @@ -13,6 +13,6 @@ "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, - "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, + "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/priv/templates/service.ex.eex b/priv/templates/service.ex.eex index bbea50b6..0d1b1cfd 100644 --- a/priv/templates/service.ex.eex +++ b/priv/templates/service.ex.eex @@ -1,7 +1,5 @@ defmodule <%= @module %>.Service do - <%= unless @module_doc? do %> @moduledoc false - <% end %> use GRPC.Service, name: <%= inspect(@service_name) %>, protoc_gen_elixir_version: "<%= @version %>" <%= if @descriptor_fun_body do %> From fcca6efa188118ef894acfeb39eeb5a2efb50849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 19 Sep 2022 10:29:58 +0200 Subject: [PATCH 47/73] map transcoded response using HttpRule.response_body --- lib/grpc/server/stream.ex | 15 +++++++++--- lib/grpc/server/transcode.ex | 10 +++++--- test/grpc/integration/server_test.exs | 33 +++++++++++++++++++++++++- test/grpc/server/transcode_test.exs | 6 ++--- test/support/transcode_messages.pb.ex | 12 ++++++---- test/support/transcode_messages.proto | 11 +++++++++ test/support/transcode_messages.svc.ex | 16 ++++++++++++- 7 files changed, 88 insertions(+), 15 deletions(-) diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index b8158510..013008fc 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -59,13 +59,22 @@ defmodule GRPC.Server.Stream do __interface__: %{send_reply: &__MODULE__.send_reply/3} def send_reply( - %{grpc_type: :server_stream, codec: codec, http_transcode: true} = stream, + %{grpc_type: :server_stream, codec: codec, http_transcode: true, rpc: rpc} = stream, reply, opts ) do - data = codec.encode(reply) - do_send_reply(stream, [data, "\n"], opts) + rule = GRPC.Service.rpc_options(rpc, :http) || %{value: %{}} + response = GRPC.Server.Transcode.map_response_body(rule.value, reply) + + do_send_reply(stream, [codec.encode(response), "\n"], opts) + end + + def send_reply(%{codec: codec, http_transcode: true, rpc: rpc} = stream, reply, opts) do + rule = GRPC.Service.rpc_options(rpc, :http) || %{value: %{}} + response = GRPC.Server.Transcode.map_response_body(rule.value, reply) + + do_send_reply(stream, codec.encode(response), opts) end def send_reply(%{codec: codec} = stream, reply, opts) do diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 448c0e73..e32247be 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -47,12 +47,16 @@ defmodule GRPC.Server.Transcode do defp map_request_body(%Google.Api.HttpRule{body: field}, request_body), do: %{field => request_body} - @spec map_response_body(Google.Api.HttpRule.t(), map()) :: map() + @spec map_response_body(Google.Api.HttpRule.t() | map(), map()) :: map() def map_response_body(%Google.Api.HttpRule{response_body: ""}, response_body), do: response_body # TODO The field is required to be present on the toplevel response message - def map_response_body(%Google.Api.HttpRule{response_body: field}, response_body), - do: Map.get(response_body, field) + def map_response_body(%Google.Api.HttpRule{response_body: field}, response_body) do + key = String.to_existing_atom(field) + Map.get(response_body, key) + end + + def map_response_body(%{}, response_body), do: response_body @spec to_path(term()) :: String.t() def to_path({_method, {_bindings, segments}} = _spec) do diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 645e338d..7df9a137 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -41,6 +41,16 @@ defmodule GRPC.Integration.ServerTest do ) end + def get_message_with_response_body(msg_request, _) do + Transcode.MessageOut.new( + response: + Transcode.Message.new( + name: msg_request.name, + text: "get_message_with_response_body" + ) + ) + end + def get_message_with_query(msg_request, _stream) do Transcode.Message.new(name: msg_request.name, text: "get_message_with_query") end @@ -327,7 +337,7 @@ defmodule GRPC.Integration.ServerTest do end) end - test "can map request body using HttpRule.body" do + test "can map request body using HttpRule.body and response using HttpRule.response_body" do run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) @@ -351,6 +361,27 @@ defmodule GRPC.Integration.ServerTest do end) end + test "can map response body using HttpRule.response_body" do + run_server([TranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + name = "response_body_mapper" + stream_ref = + :gun.get( + conn_pid, + "/v1/messages/response_body/#{name}", + [ + {"content-type", "application/json"} + ] + ) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{"name" => ^name, "text" => "get_message_with_response_body"} = + Jason.decode!(body) + end) + end + test "can send streaming responses" do run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index 476c386f..e331de28 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -36,16 +36,16 @@ defmodule GRPC.TranscodeTest do test "map_response_body/2 with empty HttpRule.response_body" do rule = Google.Api.HttpRule.new(response_body: "") - request_body = %{"a" => "b"} + request_body = %{message: %{a: "b"}} assert request_body == Transcode.map_response_body(rule, request_body) end test "map_response_body/2 with HttpRule.response_body: " do rule = Google.Api.HttpRule.new(response_body: "message") - request_body = %{"message" => %{"a" => "b"}} + request_body = %{message: %{a: "b"}} - assert %{"a" => "b"} == Transcode.map_response_body(rule, request_body) + assert %{a: "b"} == Transcode.map_response_body(rule, request_body) end test "build_route/1 returns a route with {http_method, route} based on the http rule" do diff --git a/test/support/transcode_messages.pb.ex b/test/support/transcode_messages.pb.ex index 6c8b2da0..7714cef1 100644 --- a/test/support/transcode_messages.pb.ex +++ b/test/support/transcode_messages.pb.ex @@ -1,6 +1,12 @@ -defmodule Transcode.GetMessageRequest do +defmodule Transcode.MessageOut do @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + field :response, 1, type: Transcode.Message +end + +defmodule Transcode.GetMessageRequest do + @moduledoc false use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string @@ -8,7 +14,6 @@ end defmodule Transcode.Message do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string @@ -17,8 +22,7 @@ end defmodule Transcode.NestedMessageRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :message, 1, type: Transcode.GetMessageRequest -end +end \ No newline at end of file diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto index 116b0a79..05975279 100644 --- a/test/support/transcode_messages.proto +++ b/test/support/transcode_messages.proto @@ -30,6 +30,13 @@ service Messaging { }; } + rpc GetMessageWithResponseBody(GetMessageRequest) returns (MessageOut) { + option (google.api.http) = { + get: "/v1/messages/response_body/{name}", + response_body: "response" + }; + } + rpc CreateMessageWithNestedBody(NestedMessageRequest) returns (Message) { option (google.api.http) = { post: "/v1/messages/nested", @@ -44,6 +51,10 @@ service Messaging { } } +message MessageOut { + Message response = 1; +} + message GetMessageRequest { string name = 1; // Mapped to URL path. } diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex index f9c20f80..b90b6779 100644 --- a/test/support/transcode_messages.svc.ex +++ b/test/support/transcode_messages.svc.ex @@ -59,6 +59,20 @@ defmodule Transcode.Messaging.Service do } }) + rpc(:GetMessageWithResponseBody, Transcode.GetMessageRequest, Transcode.MessageOut, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/response_body/{name}"}, + response_body: "response", + selector: "" + } + } + }) + rpc(:CreateMessageWithNestedBody, Transcode.NestedMessageRequest, Transcode.Message, %{ http: %{ type: Google.Api.PbExtension, @@ -92,4 +106,4 @@ defmodule Transcode.Messaging.Stub do @moduledoc false use GRPC.Stub, service: Transcode.Messaging.Service -end +end \ No newline at end of file From 82cc13e6c7aef5329eec08b3630fc7f20cdfa509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Mon, 19 Sep 2022 13:41:55 +0200 Subject: [PATCH 48/73] support nested field paths in http paths --- lib/grpc/server/stream.ex | 1 - lib/grpc/server/transcode.ex | 107 ++------------- lib/grpc/server/transcode/field_path.ex | 20 +++ lib/grpc/server/transcode/query.ex | 21 +-- lib/grpc/server/transcode/template.ex | 95 +++++++++++++ test/grpc/integration/server_test.exs | 25 ++++ test/grpc/server/transcode/template_test.exs | 137 +++++++++++++++++++ test/grpc/server/transcode_test.exs | 134 ++---------------- test/support/transcode_messages.pb.ex | 2 +- test/support/transcode_messages.proto | 6 + test/support/transcode_messages.svc.ex | 16 ++- 11 files changed, 326 insertions(+), 238 deletions(-) create mode 100644 lib/grpc/server/transcode/field_path.ex create mode 100644 lib/grpc/server/transcode/template.ex create mode 100644 test/grpc/server/transcode/template_test.exs diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index 013008fc..476e3d06 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -63,7 +63,6 @@ defmodule GRPC.Server.Stream do reply, opts ) do - rule = GRPC.Service.rpc_options(rpc, :http) || %{value: %{}} response = GRPC.Server.Transcode.map_response_body(rule.value, reply) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index e32247be..7c652828 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,5 +1,5 @@ defmodule GRPC.Server.Transcode do - alias __MODULE__.Query + alias __MODULE__.{Query, Template, FieldPath} @doc """ Leaf request fields (recursive expansion nested messages in the request message) are classified into three categories: @@ -21,7 +21,7 @@ defmodule GRPC.Server.Transcode do query_string, req_mod ) do - path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) + path_bindings = map_path_bindings(path_bindings) query = Query.decode(query_string) request = Map.merge(path_bindings, query) @@ -35,8 +35,9 @@ defmodule GRPC.Server.Transcode do _query_string, req_mod ) do - path_bindings = Map.new(path_bindings, fn {k, v} -> {to_string(k), v} end) - request = Map.merge(path_bindings, map_request_body(rule, body_request)) + path_bindings = map_path_bindings(path_bindings) + body_request = map_request_body(rule, body_request) + request = Map.merge(path_bindings, body_request) Protobuf.JSON.from_decoded(request, req_mod) end @@ -58,7 +59,7 @@ defmodule GRPC.Server.Transcode do def map_response_body(%{}, response_body), do: response_body - @spec to_path(term()) :: String.t() + @spec to_path({atom(), Template.route()}) :: String.t() def to_path({_method, {_bindings, segments}} = _spec) do match = segments @@ -71,100 +72,22 @@ defmodule GRPC.Server.Transcode do defp segment_to_string({binding, _}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" defp segment_to_string(segment), do: segment - # https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule - - # Template = "/" Segments [ Verb ] ; - # Segments = Segment { "/" Segment } ; - # Segment = "*" | "**" | LITERAL | Variable ; - # Variable = "{" FieldPath [ "=" Segments ] "}" ; - # FieldPath = IDENT { "." IDENT } ; - # Verb = ":" LITERAL ; - # - @spec build_route(term()) :: tuple() + @spec build_route(Google.Api.HttpRule.t()) :: {atom(), Template.route()} def build_route(%Google.Api.HttpRule{pattern: {method, path}}) do route = path - |> tokenize([]) - |> parse([], []) + |> Template.tokenize([]) + |> Template.parse([], []) {method, route} end - @spec tokenize(binary(), list()) :: list() - def tokenize(path, tokens \\ []) - - def tokenize(<<>>, tokens) do - Enum.reverse(tokens) - end - - def tokenize(segments, tokens) do - {token, rest} = do_tokenize(segments, <<>>) - tokenize(rest, [token | tokens]) - end - - @terminals [?/, ?{, ?}, ?=, ?*] - defp do_tokenize(<>, <<>>) when h in @terminals do - # parse(t, acc) - {{List.to_atom([h]), []}, t} - end - - defp do_tokenize(<> = rest, acc) when h in @terminals do - {{:identifier, acc, []}, rest} - end - - defp do_tokenize(<>, acc) - when h in ?a..?z or h in ?A..?Z or h in ?0..?9 or h == ?_ or h == ?. do - do_tokenize(t, <>) - end - - defp do_tokenize(<<>>, acc) do - {{:identifier, acc, []}, <<>>} - end - - @spec parse(list(tuple()), list(), list()) :: {list(), list()} - def parse([], params, segments) do - {Enum.reverse(params), Enum.reverse(segments)} - end - - def parse([{:/, _} | rest], params, segments) do - parse(rest, params, segments) - end - - def parse([{:*, _} | rest], params, segments) do - parse(rest, params, [{:_, []} | segments]) - end - - def parse([{:identifier, identifier, _} | rest], params, segments) do - parse(rest, params, [identifier | segments]) - end - - def parse([{:"{", _} | rest], params, segments) do - {params, segments, rest} = parse_binding(rest, params, segments) - parse(rest, params, segments) - end - - defp parse_binding([{:"}", []} | rest], params, segments) do - {params, segments, rest} - end - - defp parse_binding( - [{:identifier, id, _}, {:=, _}, {:identifier, assign, _} | rest], - params, - segments - ) do - {variable, _} = param = field_path(id) - # assign = field_path(assign) - - parse_binding(rest, [param | params], [{variable, [assign]} | segments]) - end - - defp parse_binding([{:identifier, id, []} | rest], params, segments) do - {variable, _} = param = field_path(id) - parse_binding(rest, [param | params], [{variable, []} | segments]) - end + @spec map_path_bindings(map()) :: map() + def map_path_bindings(bindings) when bindings == %{}, do: bindings - def field_path(identifier) do - [root | path] = String.split(identifier, ".") - {String.to_atom(root), path} + def map_path_bindings(bindings) do + for {k, v} <- bindings, reduce: %{} do + acc -> FieldPath.decode_pair({to_string(k), v}, acc) + end end end diff --git a/lib/grpc/server/transcode/field_path.ex b/lib/grpc/server/transcode/field_path.ex new file mode 100644 index 00000000..2f584900 --- /dev/null +++ b/lib/grpc/server/transcode/field_path.ex @@ -0,0 +1,20 @@ +defmodule GRPC.Server.Transcode.FieldPath do + @moduledoc false + + @spec decode_pair({binary(), term()}, map()) :: map() + def decode_pair({key, value}, acc) do + parts = :binary.split(key, ".", [:global]) + assign_map(parts, value, acc) + end + + defp assign_map(parts, value, acc) do + {_, acc} = + get_and_update_in(acc, Enum.map(parts, &Access.key(&1, %{})), fn + prev when prev == %{} -> {prev, value} + prev when is_list(prev) -> {prev, [value | prev]} + prev -> {prev, [value, prev]} + end) + + acc + end +end diff --git a/lib/grpc/server/transcode/query.ex b/lib/grpc/server/transcode/query.ex index 4036e1b1..dde93bdc 100644 --- a/lib/grpc/server/transcode/query.ex +++ b/lib/grpc/server/transcode/query.ex @@ -1,6 +1,5 @@ defmodule GRPC.Server.Transcode.Query do # This module is based on https://github.com/elixir-plug/plug/blob/main/lib/plug/conn/query.ex - @moduledoc """ Decoding of URL-encoded queries as per the rules outlined in the documentation for [`google.api.HttpRule`](https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule) @@ -11,6 +10,8 @@ defmodule GRPC.Server.Transcode.Query do 1. Sub-paths with the same leaf key are decoded as a list """ + alias GRPC.Server.Transcode.FieldPath + @doc """ Decodes the given `query`. @@ -64,26 +65,10 @@ defmodule GRPC.Server.Transcode.Query do {decode_www_form(key), ""} end - decode_pair(current, acc) + FieldPath.decode_pair(current, acc) end defp decode_www_form(value) do URI.decode_www_form(value) end - - defp decode_pair({key, value}, acc) do - parts = :binary.split(key, ".", [:global]) - assign_map(parts, value, acc) - end - - defp assign_map(parts, value, acc) do - {_, acc} = - get_and_update_in(acc, Enum.map(parts, &Access.key(&1, %{})), fn - prev when prev == %{} -> {prev, value} - prev when is_list(prev) -> {prev, [value | prev]} - prev -> {prev, [value, prev]} - end) - - acc - end end diff --git a/lib/grpc/server/transcode/template.ex b/lib/grpc/server/transcode/template.ex new file mode 100644 index 00000000..abae524d --- /dev/null +++ b/lib/grpc/server/transcode/template.ex @@ -0,0 +1,95 @@ +defmodule GRPC.Server.Transcode.Template do + @moduledoc false + # https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule + # Template = "/" Segments [ Verb ] ; + # Segments = Segment { "/" Segment } ; + # Segment = "*" | "**" | LITERAL | Variable ; + # Variable = "{" FieldPath [ "=" Segments ] "}" ; + # FieldPath = IDENT { "." IDENT } ; + # Verb = ":" LITERAL ; + @type segments :: list(atom | String.t()) + @type bindings :: list(atom) + @type route :: {bindings(), segments()} + + @spec tokenize(binary(), list()) :: list() + def tokenize(path, tokens \\ []) + + def tokenize(<<>>, tokens) do + Enum.reverse(tokens) + end + + def tokenize(segments, tokens) do + {token, rest} = do_tokenize(segments, <<>>) + tokenize(rest, [token | tokens]) + end + + @terminals [?/, ?{, ?}, ?=, ?*] + defp do_tokenize(<>, <<>>) when h in @terminals do + # parse(t, acc) + {{List.to_atom([h]), []}, t} + end + + defp do_tokenize(<> = rest, acc) when h in @terminals do + {{:identifier, acc, []}, rest} + end + + defp do_tokenize(<>, acc) + when h in ?a..?z or h in ?A..?Z or h in ?0..?9 or h == ?_ or h == ?. do + do_tokenize(t, <>) + end + + defp do_tokenize(<<>>, acc) do + {{:identifier, acc, []}, <<>>} + end + + @spec parse(list(tuple()), list(), list()) :: route() + def parse([], params, segments) do + {Enum.reverse(params), Enum.reverse(segments)} + end + + def parse([{:/, _} | rest], params, segments) do + parse(rest, params, segments) + end + + def parse([{:*, _} | rest], params, segments) do + parse(rest, params, [{:_, []} | segments]) + end + + def parse([{:identifier, identifier, _} | rest], params, segments) do + parse(rest, params, [identifier | segments]) + end + + def parse([{:"{", _} | rest], params, segments) do + {params, segments, rest} = parse_binding(rest, params, segments) + parse(rest, params, segments) + end + + defp parse_binding([{:"}", []} | rest], params, segments) do + {params, segments, rest} + end + + defp parse_binding( + [{:identifier, id, _}, {:=, _}, {:identifier, assign, _} | rest], + params, + segments + ) do + {variable, _} = param = field_path(id) + # assign = field_path(assign) + + parse_binding(rest, [param | params], [{variable, [assign]} | segments]) + end + + defp parse_binding([{:identifier, id, []} | rest], params, segments) do + {variable, _} = param = field_path(id) + parse_binding(rest, [param | params], [{variable, []} | segments]) + end + + defp field_path(identifier) do + id_key = String.to_atom(identifier) + + case String.split(identifier, ".") do + [_root] -> {id_key, []} + [_root | _] = path -> {id_key, path} + end + end +end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 7df9a137..d70d47e1 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -41,6 +41,10 @@ defmodule GRPC.Integration.ServerTest do ) end + def get_message_with_field_path(msg_request, _) do + msg_request.message + end + def get_message_with_response_body(msg_request, _) do Transcode.MessageOut.new( response: @@ -365,6 +369,7 @@ defmodule GRPC.Integration.ServerTest do run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) name = "response_body_mapper" + stream_ref = :gun.get( conn_pid, @@ -402,6 +407,26 @@ defmodule GRPC.Integration.ServerTest do end) end + test "can use field paths in requests" do + run_server([TranscodeServer], fn port -> + {:ok, conn_pid} = :gun.open('localhost', port) + name = "fieldpath" + + stream_ref = + :gun.get( + conn_pid, + "/v1/messages/fieldpath/#{name}", + [ + {"content-type", "application/json"} + ] + ) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + assert %{"name" => ^name} = Jason.decode!(body) + end) + end + test "service methods can have the same path but different methods in http rule option" do run_server([TranscodeServer], fn port -> {:ok, conn_pid} = :gun.open('localhost', port) diff --git a/test/grpc/server/transcode/template_test.exs b/test/grpc/server/transcode/template_test.exs new file mode 100644 index 00000000..14c7e695 --- /dev/null +++ b/test/grpc/server/transcode/template_test.exs @@ -0,0 +1,137 @@ +defmodule GRPC.Transcode.TemplateTest do + use ExUnit.Case, async: true + alias GRPC.Server.Transcode.Template + + describe "tokenize/2" do + test "can tokenize simple paths" do + assert [{:/, []}] = Template.tokenize("/") + + assert [{:/, []}, {:identifier, "v1", []}, {:/, []}, {:identifier, "messages", []}] = + Template.tokenize("/v1/messages") + end + + test "can tokenize simple paths with wildcards" do + assert [ + {:/, []}, + {:identifier, "v1", []}, + {:/, []}, + {:identifier, "messages", []}, + {:/, []}, + {:*, []} + ] == Template.tokenize("/v1/messages/*") + end + + test "can tokenize simple variables" do + assert [ + {:/, []}, + {:identifier, "v1", []}, + {:/, []}, + {:identifier, "messages", []}, + {:/, []}, + {:"{", []}, + {:identifier, "message_id", []}, + {:"}", []} + ] == Template.tokenize("/v1/messages/{message_id}") + end + + test "can tokenize variable assignments in bindings" do + assert [ + {:/, []}, + {:identifier, "v1", []}, + {:/, []}, + {:"{", []}, + {:identifier, "name", []}, + {:=, []}, + {:identifier, "messages", []}, + {:"}", []} + ] == Template.tokenize("/v1/{name=messages}") + end + + test "can tokenize field paths in bindings" do + assert [ + {:/, []}, + {:identifier, "v1", []}, + {:/, []}, + {:identifier, "messages", []}, + {:/, []}, + {:"{", []}, + {:identifier, "message_id", []}, + {:"}", []}, + {:/, []}, + {:"{", []}, + {:identifier, "sub.subfield", []}, + {:"}", []} + ] == Template.tokenize("/v1/messages/{message_id}/{sub.subfield}") + end + end + + describe "parse/3" do + test "can parse simple paths" do + assert {[], []} == + "/" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse paths with identifiers" do + assert {[], ["v1", "messages"]} == + "/v1/messages" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse paths with wildcards" do + assert {[], ["v1", "messages", {:_, []}]} == + "/v1/messages/*" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse simple bindings with variables" do + assert {[{:message_id, []}], ["v1", "messages", {:message_id, []}]} == + "/v1/messages/{message_id}" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse bindings with variable assignment" do + assert {[{:name, []}], ["v1", {:name, ["messages"]}]} == + "/v1/{name=messages}" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse multiple bindings with variable assignment" do + assert {[{:name, []}, {:message_id, []}], ["v1", {:name, ["messages"]}, {:message_id, []}]} == + "/v1/{name=messages}/{message_id}" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse bindings with field paths" do + assert {["sub.subfield": ["sub", "subfield"]], ["v1", "messages", {:"sub.subfield", []}]} == + "/v1/messages/{sub.subfield}" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "supports deeper nested field path " do + assert {["sub.nested.nested.nested": ["sub", "nested", "nested", "nested"]], + ["v1", "messages", {:"sub.nested.nested.nested", []}]} == + "/v1/messages/{sub.nested.nested.nested}" + |> Template.tokenize() + |> Template.parse([], []) + end + + test "can parse multiple-bindings with field paths " do + assert {[ + "first.subfield": ["first", "subfield"], + "second.subfield": ["second", "subfield"] + ], + ["v1", "messages", {:"first.subfield", []}, {:"second.subfield", []}]} == + "/v1/messages/{first.subfield}/{second.subfield}" + |> Template.tokenize() + |> Template.parse([], []) + end + end +end diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index e331de28..7424d2e7 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -67,133 +67,17 @@ defmodule GRPC.TranscodeTest do assert "/v1/users/:user_id/messages/:message_id" = Transcode.to_path(spec) end - describe "tokenize/2" do - test "can tokenize simple paths" do - assert [{:/, []}] = Transcode.tokenize("/") - - assert [{:/, []}, {:identifier, "v1", []}, {:/, []}, {:identifier, "messages", []}] = - Transcode.tokenize("/v1/messages") - end - - test "can tokenize simple paths with wildcards" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:identifier, "messages", []}, - {:/, []}, - {:*, []} - ] == Transcode.tokenize("/v1/messages/*") - end - - test "can tokenize simple variables" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:identifier, "messages", []}, - {:/, []}, - {:"{", []}, - {:identifier, "message_id", []}, - {:"}", []} - ] == Transcode.tokenize("/v1/messages/{message_id}") - end - - test "can tokenize variable assignments in bindings" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:"{", []}, - {:identifier, "name", []}, - {:=, []}, - {:identifier, "messages", []}, - {:"}", []} - ] == Transcode.tokenize("/v1/{name=messages}") - end - - test "can tokenize field paths in bindings" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:identifier, "messages", []}, - {:/, []}, - {:"{", []}, - {:identifier, "message_id", []}, - {:"}", []}, - {:/, []}, - {:"{", []}, - {:identifier, "sub.subfield", []}, - {:"}", []} - ] == Transcode.tokenize("/v1/messages/{message_id}/{sub.subfield}") - end + test "map_route_bindings/2 should stringify the keys" do + path_binding_atom = %{foo: "bar"} + path_binding_string = %{foo: "bar"} + + assert %{"foo" => "bar"} == Transcode.map_path_bindings(path_binding_atom) + assert %{"foo" => "bar"} == Transcode.map_path_bindings(path_binding_string) end - describe "parse/3" do - test "can parse simple paths" do - assert {[], []} == - "/" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse paths with identifiers" do - assert {[], ["v1", "messages"]} == - "/v1/messages" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse paths with wildcards" do - assert {[], ["v1", "messages", {:_, []}]} == - "/v1/messages/*" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse simple bindings with variables" do - assert {[{:message_id, []}], ["v1", "messages", {:message_id, []}]} == - "/v1/messages/{message_id}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse bindings with variable assignment" do - assert {[{:name, []}], ["v1", {:name, ["messages"]}]} == - "/v1/{name=messages}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse multiple bindings with variable assignment" do - assert {[{:name, []}, {:message_id, []}], ["v1", {:name, ["messages"]}, {:message_id, []}]} == - "/v1/{name=messages}/{message_id}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse bindings with field paths " do - assert {[sub: ["subfield"]], ["v1", "messages", {:sub, []}]} == - "/v1/messages/{sub.subfield}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "supports deeper nested field path " do - assert {[sub: ["nested", "nested", "nested"]], ["v1", "messages", {:sub, []}]} == - "/v1/messages/{sub.nested.nested.nested}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse multiple-bindings with field paths " do - assert {[first: ["subfield"], second: ["subfield"]], - ["v1", "messages", {:first, []}, {:second, []}]} == - "/v1/messages/{first.subfield}/{second.subfield}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end + test "map_route_bindings/2 with '.' delimited identifiers should create a nested map" do + path_binding = %{"foo.bar.baz" => "biz"} + assert %{"foo" => %{"bar" => %{"baz" => "biz"}}} == Transcode.map_path_bindings(path_binding) end defp build_simple_rule(method, pattern) do diff --git a/test/support/transcode_messages.pb.ex b/test/support/transcode_messages.pb.ex index 7714cef1..59015514 100644 --- a/test/support/transcode_messages.pb.ex +++ b/test/support/transcode_messages.pb.ex @@ -25,4 +25,4 @@ defmodule Transcode.NestedMessageRequest do use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :message, 1, type: Transcode.GetMessageRequest -end \ No newline at end of file +end diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto index 05975279..d0f6cc4a 100644 --- a/test/support/transcode_messages.proto +++ b/test/support/transcode_messages.proto @@ -23,6 +23,12 @@ service Messaging { }; } + rpc GetMessageWithFieldPath(NestedMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/messages/fieldpath/{message.name}" + }; + } + rpc CreateMessage(Message) returns (Message) { option (google.api.http) = { post: "/v1/messages" diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex index b90b6779..edb768c0 100644 --- a/test/support/transcode_messages.svc.ex +++ b/test/support/transcode_messages.svc.ex @@ -45,6 +45,20 @@ defmodule Transcode.Messaging.Service do } }) + rpc(:GetMessageWithFieldPath, Transcode.NestedMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/fieldpath/{message.name}"}, + response_body: "", + selector: "" + } + } + }) + rpc(:CreateMessage, Transcode.Message, Transcode.Message, %{ http: %{ type: Google.Api.PbExtension, @@ -106,4 +120,4 @@ defmodule Transcode.Messaging.Stub do @moduledoc false use GRPC.Stub, service: Transcode.Messaging.Service -end \ No newline at end of file +end From d8ffd4289208dedebf282d20132eb7482a3a10d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 20 Sep 2022 16:58:11 +0200 Subject: [PATCH 49/73] Fix! parse sub-paths for variable assignment --- examples/helloworld/mix.exs | 2 +- examples/helloworld/mix.lock | 2 +- interop/mix.exs | 2 +- interop/mix.lock | 2 +- lib/grpc/server/transcode.ex | 30 ++++++++++++++++++-- lib/grpc/server/transcode/template.ex | 16 ++++++++--- priv/templates/service.ex.eex | 2 -- test/grpc/server/transcode/template_test.exs | 19 +++++++++++-- test/grpc/server/transcode_test.exs | 2 +- 9 files changed, 62 insertions(+), 15 deletions(-) diff --git a/examples/helloworld/mix.exs b/examples/helloworld/mix.exs index ecc3867e..a1265354 100644 --- a/examples/helloworld/mix.exs +++ b/examples/helloworld/mix.exs @@ -19,7 +19,7 @@ defmodule Helloworld.Mixfile do defp deps do [ {:grpc, path: "../../"}, - {:protobuf, github: "elixir-protobuf/protobuf", branch: "main", override: true}, + {:protobuf, "~> 0.11.0"}, {:jason, "~> 1.3.0"}, {:google_protos, "~> 0.3.0"}, {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false} diff --git a/examples/helloworld/mix.lock b/examples/helloworld/mix.lock index 62dcb34f..df73eb18 100644 --- a/examples/helloworld/mix.lock +++ b/examples/helloworld/mix.lock @@ -6,6 +6,6 @@ "google_protos": {:hex, :google_protos, "0.3.0", "15faf44dce678ac028c289668ff56548806e313e4959a3aaf4f6e1ebe8db83f4", [:mix], [{:protobuf, "~> 0.10", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "1f6b7fb20371f72f418b98e5e48dae3e022a9a6de1858d4b254ac5a5d0b4035f"}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, - "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, + "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/interop/mix.exs b/interop/mix.exs index 2ce8d69c..caee13fc 100644 --- a/interop/mix.exs +++ b/interop/mix.exs @@ -23,7 +23,7 @@ defmodule Interop.MixProject do defp deps do [ {:grpc, path: "..", override: true}, - {:protobuf, "~> 0.10"}, + {:protobuf, "~> 0.11.0"}, {:grpc_prometheus, ">= 0.1.0"}, {:grpc_statsd, "~> 0.1.0"}, {:statix, ">= 1.2.1"}, diff --git a/interop/mix.lock b/interop/mix.lock index 11f40443..43470da5 100644 --- a/interop/mix.lock +++ b/interop/mix.lock @@ -10,7 +10,7 @@ "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm", "b479a33d4aa4ba7909186e29bb6c1240254e0047a8e2a9f88463f50c0089370e"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "9fd13404a48437e044b288b41f76e64acd9735fb8b0e3809f494811dfa66d0fb"}, "prometheus_httpd": {:hex, :prometheus_httpd, "2.1.11", "f616ed9b85b536b195d94104063025a91f904a4cfc20255363f49a197d96c896", [:rebar3], [{:accept, "~> 0.3", [hex: :accept, repo: "hexpm", optional: false]}, {:prometheus, "~> 4.2", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "0bbe831452cfdf9588538eb2f570b26f30c348adae5e95a7d87f35a5910bcf92"}, - "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, + "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "recon": {:hex, :recon, "2.5.0", "2f7fcbec2c35034bade2f9717f77059dc54eb4e929a3049ca7ba6775c0bd66cd", [:mix, :rebar3], [], "hexpm", "72f3840fedd94f06315c523f6cecf5b4827233bed7ae3fe135b2a0ebeab5e196"}, "statix": {:hex, :statix, "1.2.1", "4f23c8cc2477ea0de89fed5e34f08c54b0d28b838f7b8f26613155f2221bb31e", [:mix], [], "hexpm", "7f988988fddcce19ae376bb8e47aa5ea5dabf8d4ba78d34d1ae61eb537daf72e"}, diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 7c652828..31429ded 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -29,7 +29,7 @@ defmodule GRPC.Server.Transcode do end def map_request( - %Google.Api.HttpRule{} = rule, + %Google.Api.HttpRule{body: "*"} = rule, body_request, path_bindings, _query_string, @@ -42,6 +42,21 @@ defmodule GRPC.Server.Transcode do Protobuf.JSON.from_decoded(request, req_mod) end + def map_request( + %Google.Api.HttpRule{} = rule, + body_request, + path_bindings, + query_string, + req_mod + ) do + path_bindings = map_path_bindings(path_bindings) + query = Query.decode(query_string) + body_request = map_request_body(rule, body_request) + request = Enum.reduce([query, body_request], path_bindings, &Map.merge(&2, &1)) + + Protobuf.JSON.from_decoded(request, req_mod) + end + defp map_request_body(%Google.Api.HttpRule{body: "*"}, request_body), do: request_body defp map_request_body(%Google.Api.HttpRule{body: ""}, request_body), do: request_body @@ -69,7 +84,18 @@ defmodule GRPC.Server.Transcode do "/" <> match end - defp segment_to_string({binding, _}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" + defp segment_to_string({:_, []}), do: "[...]" + defp segment_to_string({binding, []}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" + + defp segment_to_string({binding, [_ | rest]}) when is_atom(binding) do + sub_path = + rest + |> Enum.map(&segment_to_string/1) + |> Enum.join("/") + + ":#{Atom.to_string(binding)}" <> "/" <> sub_path + end + defp segment_to_string(segment), do: segment @spec build_route(Google.Api.HttpRule.t()) :: {atom(), Template.route()} diff --git a/lib/grpc/server/transcode/template.ex b/lib/grpc/server/transcode/template.ex index abae524d..21592c52 100644 --- a/lib/grpc/server/transcode/template.ex +++ b/lib/grpc/server/transcode/template.ex @@ -42,7 +42,7 @@ defmodule GRPC.Server.Transcode.Template do {{:identifier, acc, []}, <<>>} end - @spec parse(list(tuple()), list(), list()) :: route() + @spec parse(list(tuple()), list(), list()) :: route() | {list(), list(), list()} def parse([], params, segments) do {Enum.reverse(params), Enum.reverse(segments)} end @@ -64,19 +64,27 @@ defmodule GRPC.Server.Transcode.Template do parse(rest, params, segments) end + def parse([{:"}", _} | _rest] = acc, params, segments) do + {params, segments, acc} + end + + defp parse_binding([], params, segments) do + {params, segments, []} + end + defp parse_binding([{:"}", []} | rest], params, segments) do {params, segments, rest} end defp parse_binding( - [{:identifier, id, _}, {:=, _}, {:identifier, assign, _} | rest], + [{:identifier, id, _}, {:=, _} | rest], params, segments ) do {variable, _} = param = field_path(id) - # assign = field_path(assign) + {_, assign, rest} = parse(rest, [], []) - parse_binding(rest, [param | params], [{variable, [assign]} | segments]) + parse_binding(rest, [param | params], [{variable, Enum.reverse(assign)} | segments]) end defp parse_binding([{:identifier, id, []} | rest], params, segments) do diff --git a/priv/templates/service.ex.eex b/priv/templates/service.ex.eex index 0d1b1cfd..087b106d 100644 --- a/priv/templates/service.ex.eex +++ b/priv/templates/service.ex.eex @@ -15,8 +15,6 @@ defmodule <%= @module %>.Service do end defmodule <%= @module %>.Stub do - <%= unless @module_doc? do %> @moduledoc false - <% end %> use GRPC.Stub, service: <%= @module %>.Service end diff --git a/test/grpc/server/transcode/template_test.exs b/test/grpc/server/transcode/template_test.exs index 14c7e695..5a7cc06d 100644 --- a/test/grpc/server/transcode/template_test.exs +++ b/test/grpc/server/transcode/template_test.exs @@ -47,6 +47,21 @@ defmodule GRPC.Transcode.TemplateTest do ] == Template.tokenize("/v1/{name=messages}") end + test "can tokenize variable sub-paths in bindings" do + assert [ + {:/, []}, + {:identifier, "v1", []}, + {:/, []}, + {:"{", []}, + {:identifier, "name", []}, + {:=, []}, + {:identifier, "messages", []}, + {:/, []}, + {:*, []}, + {:"}", []} + ] == Template.tokenize("/v1/{name=messages/*}") + end + test "can tokenize field paths in bindings" do assert [ {:/, []}, @@ -95,8 +110,8 @@ defmodule GRPC.Transcode.TemplateTest do end test "can parse bindings with variable assignment" do - assert {[{:name, []}], ["v1", {:name, ["messages"]}]} == - "/v1/{name=messages}" + assert {[{:name, []}], ["v1", {:name, ["messages", {:_, []}]}]} == + "/v1/{name=messages/*}" |> Template.tokenize() |> Template.parse([], []) end diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index 7424d2e7..8be89c89 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -28,7 +28,7 @@ defmodule GRPC.TranscodeTest do bindings = %{"name" => "test"} assert {:ok, %Routeguide.Feature{name: "test", location: point}} = - Transcode.map_request(rule, request_body, bindings, "name=Foo", Routeguide.Feature) + Transcode.map_request(rule, request_body, bindings, "", Routeguide.Feature) assert point.latitude == 1 assert point.longitude == 2 From 690cbef0b9e1144a795fa31f5adf6435b33071c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 21 Sep 2022 10:06:38 +0200 Subject: [PATCH 50/73] support calling grpc method with json if http_transcode == true --- lib/grpc/server.ex | 2 ++ lib/grpc/server/adapters/cowboy/handler.ex | 34 ++++++++++++---------- test/grpc/integration/server_test.exs | 23 +++++++++++++++ 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 590fb5d2..7f21b62c 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -48,6 +48,8 @@ defmodule GRPC.Server do compressors = opts[:compressors] || [] http_transcode = opts[:http_transcode] || false + codecs = if http_transcode, do: [GRPC.Codec.JSON | codecs], else: codecs + routes = for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, reduce: [] do acc -> diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 83be83ee..3c95dff4 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -18,7 +18,8 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do opts :: keyword()} ) :: {:cowboy_loop, map(), map()} def init(req, {endpoint, {_name, server}, route, opts} = state) do - with {:ok, codec} <- find_codec(req, server), + with {:ok, sub_type, content_type} <- find_content_type_subtype(req), + {:ok, codec} <- find_codec(sub_type, content_type, server), {:ok, compressor} <- find_compressor(req, server) do http_method = req @@ -34,7 +35,8 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do local: opts[:local], codec: codec, http_method: http_method, - compressor: compressor + compressor: compressor, + http_transcode: sub_type == "json" } pid = spawn_link(__MODULE__, :call_rpc, [server, route, stream]) @@ -63,25 +65,25 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end - defp find_codec(req, server) do - req_content_type = :cowboy_req.header("content-type", req) - - with {:ok, subtype} <- extract_subtype(req_content_type), - codec when not is_nil(codec) <- - Enum.find(server.__meta__(:codecs), nil, fn c -> c.name() == subtype end) do + defp find_codec(subtype, content_type, server) do + if codec = Enum.find(server.__meta__(:codecs), nil, fn c -> c.name() == subtype end) do {:ok, codec} else - err -> - Logger.error(fn -> inspect(err) end) - - {:error, - RPCError.exception( - status: :unimplemented, - message: "No codec registered for content-type #{req_content_type}" - )} + {:error, + RPCError.exception( + status: :unimplemented, + message: "No codec registered for content-type #{content_type}" + )} end end + defp find_content_type_subtype(req) do + req_content_type = :cowboy_req.header("content-type", req) + + {:ok, subtype} = extract_subtype(req_content_type) + {:ok, subtype, req_content_type} + end + defp find_compressor(req, server) do encoding = :cowboy_req.header("grpc-encoding", req) diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index d70d47e1..6f197fa1 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -301,6 +301,29 @@ defmodule GRPC.Integration.ServerTest do end describe "http/json transcode" do + test "grpc method can be called using json when http_transcode == true" do + run_server([TranscodeServer], fn port -> + name = "direct_call" + + {:ok, conn_pid} = :gun.open('localhost', port) + + stream_ref = + :gun.post( + conn_pid, + "/transcode.Messaging/GetMessage", + [ + {"content-type", "application/json"} + ], + Jason.encode!(%{"name" => name}) + ) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{"text" => "get_message"} = Jason.decode!(body) + end) + end + test "can transcode path params" do run_server([TranscodeServer], fn port -> name = "foo" From 69ef24b926735f44b6025108c962762fc525ddf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 21 Sep 2022 14:47:51 +0200 Subject: [PATCH 51/73] remove params from template parsing --- lib/grpc/server/transcode.ex | 4 +- lib/grpc/server/transcode/template.ex | 59 +++++++++----------- test/grpc/server/transcode/template_test.exs | 41 ++++++-------- test/grpc/server/transcode_test.exs | 3 +- 4 files changed, 47 insertions(+), 60 deletions(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 31429ded..a9ec252d 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -75,7 +75,7 @@ defmodule GRPC.Server.Transcode do def map_response_body(%{}, response_body), do: response_body @spec to_path({atom(), Template.route()}) :: String.t() - def to_path({_method, {_bindings, segments}} = _spec) do + def to_path({_method, segments} = _spec) do match = segments |> Enum.map(&segment_to_string/1) @@ -103,7 +103,7 @@ defmodule GRPC.Server.Transcode do route = path |> Template.tokenize([]) - |> Template.parse([], []) + |> Template.parse([]) {method, route} end diff --git a/lib/grpc/server/transcode/template.ex b/lib/grpc/server/transcode/template.ex index 21592c52..02441a71 100644 --- a/lib/grpc/server/transcode/template.ex +++ b/lib/grpc/server/transcode/template.ex @@ -8,8 +8,7 @@ defmodule GRPC.Server.Transcode.Template do # FieldPath = IDENT { "." IDENT } ; # Verb = ":" LITERAL ; @type segments :: list(atom | String.t()) - @type bindings :: list(atom) - @type route :: {bindings(), segments()} + @type route :: {atom(), segments()} @spec tokenize(binary(), list()) :: list() def tokenize(path, tokens \\ []) @@ -42,62 +41,56 @@ defmodule GRPC.Server.Transcode.Template do {{:identifier, acc, []}, <<>>} end - @spec parse(list(tuple()), list(), list()) :: route() | {list(), list(), list()} - def parse([], params, segments) do - {Enum.reverse(params), Enum.reverse(segments)} + @spec parse(list(tuple()), list()) :: route() | {list(), list()} + def parse([], segments) do + Enum.reverse(segments) end - def parse([{:/, _} | rest], params, segments) do - parse(rest, params, segments) + def parse([{:/, _} | rest], segments) do + parse(rest, segments) end - def parse([{:*, _} | rest], params, segments) do - parse(rest, params, [{:_, []} | segments]) + def parse([{:*, _} | rest], segments) do + parse(rest, [{:_, []} | segments]) end - def parse([{:identifier, identifier, _} | rest], params, segments) do - parse(rest, params, [identifier | segments]) + def parse([{:identifier, identifier, _} | rest], segments) do + parse(rest, [identifier | segments]) end - def parse([{:"{", _} | rest], params, segments) do - {params, segments, rest} = parse_binding(rest, params, segments) - parse(rest, params, segments) + def parse([{:"{", _} | rest], segments) do + {segments, rest} = parse_binding(rest, segments) + parse(rest, segments) end - def parse([{:"}", _} | _rest] = acc, params, segments) do - {params, segments, acc} + def parse([{:"}", _} | _rest] = acc, segments) do + {segments, acc} end - defp parse_binding([], params, segments) do - {params, segments, []} + defp parse_binding([], segments) do + {segments, []} end - defp parse_binding([{:"}", []} | rest], params, segments) do - {params, segments, rest} + defp parse_binding([{:"}", []} | rest], segments) do + {segments, rest} end defp parse_binding( [{:identifier, id, _}, {:=, _} | rest], - params, segments ) do - {variable, _} = param = field_path(id) - {_, assign, rest} = parse(rest, [], []) + variable = field_path(id) + {assign, rest} = parse(rest, []) - parse_binding(rest, [param | params], [{variable, Enum.reverse(assign)} | segments]) + parse_binding(rest, [{variable, Enum.reverse(assign)} | segments]) end - defp parse_binding([{:identifier, id, []} | rest], params, segments) do - {variable, _} = param = field_path(id) - parse_binding(rest, [param | params], [{variable, []} | segments]) + defp parse_binding([{:identifier, id, []} | rest], segments) do + variable = field_path(id) + parse_binding(rest, [{variable, []} | segments]) end defp field_path(identifier) do - id_key = String.to_atom(identifier) - - case String.split(identifier, ".") do - [_root] -> {id_key, []} - [_root | _] = path -> {id_key, path} - end + String.to_atom(identifier) end end diff --git a/test/grpc/server/transcode/template_test.exs b/test/grpc/server/transcode/template_test.exs index 5a7cc06d..a91ef77f 100644 --- a/test/grpc/server/transcode/template_test.exs +++ b/test/grpc/server/transcode/template_test.exs @@ -82,71 +82,66 @@ defmodule GRPC.Transcode.TemplateTest do describe "parse/3" do test "can parse simple paths" do - assert {[], []} == + assert [] == "/" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse paths with identifiers" do - assert {[], ["v1", "messages"]} == + assert ["v1", "messages"] == "/v1/messages" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse paths with wildcards" do - assert {[], ["v1", "messages", {:_, []}]} == + assert ["v1", "messages", {:_, []}] == "/v1/messages/*" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse simple bindings with variables" do - assert {[{:message_id, []}], ["v1", "messages", {:message_id, []}]} == + assert ["v1", "messages", {:message_id, []}] == "/v1/messages/{message_id}" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse bindings with variable assignment" do - assert {[{:name, []}], ["v1", {:name, ["messages", {:_, []}]}]} == + assert ["v1", {:name, ["messages", {:_, []}]}] == "/v1/{name=messages/*}" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse multiple bindings with variable assignment" do - assert {[{:name, []}, {:message_id, []}], ["v1", {:name, ["messages"]}, {:message_id, []}]} == + assert ["v1", {:name, ["messages"]}, {:message_id, []}] == "/v1/{name=messages}/{message_id}" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse bindings with field paths" do - assert {["sub.subfield": ["sub", "subfield"]], ["v1", "messages", {:"sub.subfield", []}]} == + assert ["v1", "messages", {:"sub.subfield", []}] == "/v1/messages/{sub.subfield}" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "supports deeper nested field path " do - assert {["sub.nested.nested.nested": ["sub", "nested", "nested", "nested"]], - ["v1", "messages", {:"sub.nested.nested.nested", []}]} == + assert ["v1", "messages", {:"sub.nested.nested.nested", []}] == "/v1/messages/{sub.nested.nested.nested}" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end test "can parse multiple-bindings with field paths " do - assert {[ - "first.subfield": ["first", "subfield"], - "second.subfield": ["second", "subfield"] - ], - ["v1", "messages", {:"first.subfield", []}, {:"second.subfield", []}]} == + assert ["v1", "messages", {:"first.subfield", []}, {:"second.subfield", []}] == "/v1/messages/{first.subfield}/{second.subfield}" |> Template.tokenize() - |> Template.parse([], []) + |> Template.parse([]) end end end diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index 8be89c89..472f92cc 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -50,8 +50,7 @@ defmodule GRPC.TranscodeTest do test "build_route/1 returns a route with {http_method, route} based on the http rule" do rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert {:get, {params, segments}} = Transcode.build_route(rule) - assert [message_id: []] == params + assert {:get, segments} = Transcode.build_route(rule) assert ["v1", "messages", {:message_id, []}] = segments end From d170bb8fdca8d0875b2e59accf9ce70e0590b3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 22 Sep 2022 10:34:58 +0200 Subject: [PATCH 52/73] add GRPC.Server transcode example --- lib/grpc/server.ex | 79 ++++++++++++++++++++++++++++++ lib/grpc/server/transcode.ex | 19 +++---- lib/grpc/server/transcode/query.ex | 36 +++----------- lib/grpc/service.ex | 23 ++++++++- 4 files changed, 115 insertions(+), 42 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 7f21b62c..905535d2 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -29,6 +29,85 @@ defmodule GRPC.Server do The request will be a `Enumerable.t`(created by Elixir's `Stream`) of requests if it's streaming. If a reply is streaming, you need to call `send_reply/2` to send replies one by one instead of returning reply in the end. + + ## gRPC http/json transcoding + + Transcoding can be enabled by using the option `http_transcode: true`: + + defmodule Greeter.Service do + use GRPC.Service, name: "ping" + + rpc :SayHello, Request, Reply + rpc :SayGoodbye, stream(Request), stream(Reply) + end + + defmodule Greeter.Server do + use GRPC.Server, service: Greeter.Service, http_transcode: true + + def say_hello(request, _stream) do + Reply.new(message: "Hello" <> request.name) + end + + def say_goodbye(request_enum, stream) do + requests = Enum.map request_enum, &(&1) + GRPC.Server.send_reply(stream, reply1) + GRPC.Server.send_reply(stream, reply2) + end + end + + With transcoding enabled gRPC methods can be used over HTTP/1 with JSON i.e + + POST localhost/helloworld.Greeter/SayHello` + Content-Type: application/json + { + "message": "gRPC" + } + + HTTP/1.1 200 OK + Content-Type: application/json + { + "message": "Hello gRPC" + } + + By using `option (google.api.http)` annotations in the `.proto` file the mapping between + HTTP/JSON to gRPC methods and parameters can be customized: + + syntax = "proto3"; + + import "google/api/annotations.proto"; + import "google/protobuf/timestamp.proto"; + + package helloworld; + + service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) { + option (google.api.http) = { + get: "/v1/greeter/{name}" + }; + } + } + + message HelloRequest { + string name = 1; + } + + message HelloReply { + string message = 1; + } + + In addition to the `POST localhost/helloworld.Greeter/SayHello` route in the previous examples + this creates an additional route: `GET localhost/v1/greeter/:name` + + GET localhost/v1/greeter/gRPC + Accept: application/json + + HTTP/1.1 200 OK + Content-Type: application/json + { + "message": "Hello gRPC" + } + + For more comprehensive documentation on annotation usage in `.proto` files [see](https://cloud.google.com/endpoints/docs/grpc/transcoding) """ require Logger diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index a9ec252d..43b59908 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,17 +1,14 @@ defmodule GRPC.Server.Transcode do + @moduledoc false alias __MODULE__.{Query, Template, FieldPath} - @doc """ - Leaf request fields (recursive expansion nested messages in the request message) are classified into three categories: - - 1. Fields referred by the path template. They are passed via the URL path. - 2. Fields referred by the HttpRule.body. They are passed via the HTTP request body. - 3. All other fields are passed via the URL query parameters, and the parameter name is the field path in the request message. A repeated field can be represented as multiple query parameters under the same name. - - If HttpRule.body is "*", there is no URL query parameter, all fields are passed via URL path and HTTP request body. - - If HttpRule.body is omitted, there is no HTTP request body, all fields are passed via URL path and URL query parameters. - """ + # The request mapping follow the following rules: + # + # 1. Fields referred by the path template. They are passed via the URL path. + # 2. Fields referred by the HttpRule.body. They are passed via the HTTP request body. + # 3. All other fields are passed via the URL query parameters, and the parameter name is the field path in the request message. A repeated field can be represented as multiple query parameters under the same name. + # If HttpRule.body is "*", there is no URL query parameter, all fields are passed via URL path and HTTP request body. + # If HttpRule.body is omitted, there is no HTTP request body, all fields are passed via URL path and URL query parameters. @spec map_request(Google.Api.HttpRule.t(), map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request( diff --git a/lib/grpc/server/transcode/query.ex b/lib/grpc/server/transcode/query.ex index dde93bdc..c6592e20 100644 --- a/lib/grpc/server/transcode/query.ex +++ b/lib/grpc/server/transcode/query.ex @@ -1,39 +1,15 @@ defmodule GRPC.Server.Transcode.Query do + @moduledoc false # This module is based on https://github.com/elixir-plug/plug/blob/main/lib/plug/conn/query.ex - @moduledoc """ - Decoding of URL-encoded queries as per the rules outlined in the documentation for [`google.api.HttpRule`](https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule) + # Decoding of URL-encoded queries as per the rules outlined in the documentation for [`google.api.HttpRule`](https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule) + # It provides similar functionality to `URI.decode_query/3` or `Plug.Conn.Query.decode/4` with the following differences: - It provides similar functionality to `URI.decode_query/3` or `Plug.Conn.Query.decode/4` with the following differences: - - 1. A repeated key is treated as a list of values - 1. Sub-paths on the form `path.subpath` are decoded as nested maps - 1. Sub-paths with the same leaf key are decoded as a list - """ + # 1. A repeated key is treated as a list of values + # 1. Sub-paths on the form `path.subpath` are decoded as nested maps + # 1. Sub-paths with the same leaf key are decoded as a list alias GRPC.Server.Transcode.FieldPath - @doc """ - Decodes the given `query`. - - The `query` is assumed to be encoded in the "x-www-form-urlencoded" format. - - `acc` is the initial "accumulator" where decoded values will be added. - - ## Examples - - iex> decode("a=A&b=B") - %{"a" => "A", "b" => "B"} - - iex> decode("param=A¶m=B") - %{"param" => ["A", "B"]} - - iex> decode("root.a=A&root.b=B") - %{"root" => %{"a" => "A", "b" => "B"}} - - iex> decode("root.a=A&root.a=B") - %{"root" => ["A", "B"]} - - """ @spec decode(String.t(), map()) :: %{optional(String.t()) => term()} def decode(query, acc \\ %{}) diff --git a/lib/grpc/service.ex b/lib/grpc/service.ex index 1c75c97f..45e0e89c 100644 --- a/lib/grpc/service.ex +++ b/lib/grpc/service.ex @@ -3,7 +3,7 @@ defmodule GRPC.Service do Define gRPC service used by Stub and Server. You should use `Protobuf` to to generate code instead of using this module directly. - It imports DSL functions like `rpc/3` and `stream/1` for defining the RPC + It imports DSL functions like `rpc/4` and `stream/1` for defining the RPC functions easily: defmodule Greeter.Service do @@ -11,6 +11,27 @@ defmodule GRPC.Service do rpc :SayHello, HelloRequest, stream(HelloReply) end + + `option (google.api.http)` annotations are supported for gRPC http/json transcoding. Once generated the 4th argument to `rpc/4` contains + the `Google.Api.HttpRule` option. + + defmodule Greeter.Service do + use GRPC.Service, name: "helloworld.Greeter" + + rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/greeter/{name}"}, + response_body: "", + selector: "" + } + } + }) + end """ defmacro __using__(opts) do From 51b7c33ea7ae5015bc185d6f977e6a88c0e37c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Fri, 23 Sep 2022 09:37:20 +0200 Subject: [PATCH 53/73] get requests can use accept header to find codec --- lib/grpc/server/adapters/cowboy/handler.ex | 37 ++++++++++++++++------ test/grpc/integration/server_test.exs | 18 +++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 3c95dff4..9eda27a1 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -18,15 +18,15 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do opts :: keyword()} ) :: {:cowboy_loop, map(), map()} def init(req, {endpoint, {_name, server}, route, opts} = state) do - with {:ok, sub_type, content_type} <- find_content_type_subtype(req), + http_method = + req + |> :cowboy_req.method() + |> String.downcase() + |> String.to_existing_atom() + + with {:ok, sub_type, content_type} <- find_content_type_subtype(http_method, req), {:ok, codec} <- find_codec(sub_type, content_type, server), {:ok, compressor} <- find_compressor(req, server) do - http_method = - req - |> :cowboy_req.method() - |> String.downcase() - |> String.to_existing_atom() - stream = %GRPC.Server.Stream{ server: server, endpoint: endpoint, @@ -77,11 +77,28 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end - defp find_content_type_subtype(req) do + defp find_content_type_subtype(:get, req) do + content_type = + case :cowboy_req.header("accept", req) do + :undefined -> + :cowboy_req.header("content-type", req) + + content_type -> + content_type + end + + find_subtype(content_type) + end + + defp find_content_type_subtype(_, req) do req_content_type = :cowboy_req.header("content-type", req) - {:ok, subtype} = extract_subtype(req_content_type) - {:ok, subtype, req_content_type} + find_subtype(req_content_type) + end + + defp find_subtype(content_type) do + {:ok, subtype} = extract_subtype(content_type) + {:ok, subtype, content_type} end defp find_compressor(req, server) do diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 6f197fa1..d21479bc 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -324,6 +324,24 @@ defmodule GRPC.Integration.ServerTest do end) end + test "accept: application/json can be used with get requests" do + run_server([TranscodeServer], fn port -> + name = "direct_call" + + {:ok, conn_pid} = :gun.open('localhost', port) + + stream_ref = + :gun.get(conn_pid, "/v1/messages/#{name}", [ + {"accept", "application/json"} + ]) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) + + assert %{"text" => "get_message"} = Jason.decode!(body) + end) + end + test "can transcode path params" do run_server([TranscodeServer], fn port -> name = "foo" From 431aedf01f3a022eac021e028cc9d4735e61bc67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 20 Oct 2022 09:39:09 +0200 Subject: [PATCH 54/73] remove reference to HttpRule type --- lib/grpc/server/transcode.ex | 25 +++++++++++++------------ test/support/transcode_messages.proto | 6 ++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 43b59908..9672c083 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -2,6 +2,7 @@ defmodule GRPC.Server.Transcode do @moduledoc false alias __MODULE__.{Query, Template, FieldPath} + @type t :: map() # The request mapping follow the following rules: # # 1. Fields referred by the path template. They are passed via the URL path. @@ -9,10 +10,10 @@ defmodule GRPC.Server.Transcode do # 3. All other fields are passed via the URL query parameters, and the parameter name is the field path in the request message. A repeated field can be represented as multiple query parameters under the same name. # If HttpRule.body is "*", there is no URL query parameter, all fields are passed via URL path and HTTP request body. # If HttpRule.body is omitted, there is no HTTP request body, all fields are passed via URL path and URL query parameters. - @spec map_request(Google.Api.HttpRule.t(), map(), map(), String.t(), module()) :: + @spec map_request(t(), map(), map(), String.t(), module()) :: {:ok, struct()} | {:error, term()} def map_request( - %Google.Api.HttpRule{body: ""}, + %{body: ""}, _body_request, path_bindings, query_string, @@ -26,7 +27,7 @@ defmodule GRPC.Server.Transcode do end def map_request( - %Google.Api.HttpRule{body: "*"} = rule, + %{body: "*"} = rule, body_request, path_bindings, _query_string, @@ -40,7 +41,7 @@ defmodule GRPC.Server.Transcode do end def map_request( - %Google.Api.HttpRule{} = rule, + %{} = rule, body_request, path_bindings, query_string, @@ -54,17 +55,17 @@ defmodule GRPC.Server.Transcode do Protobuf.JSON.from_decoded(request, req_mod) end - defp map_request_body(%Google.Api.HttpRule{body: "*"}, request_body), do: request_body - defp map_request_body(%Google.Api.HttpRule{body: ""}, request_body), do: request_body + defp map_request_body(%{body: "*"}, request_body), do: request_body + defp map_request_body(%{body: ""}, request_body), do: request_body - defp map_request_body(%Google.Api.HttpRule{body: field}, request_body), + defp map_request_body(%{body: field}, request_body), do: %{field => request_body} - @spec map_response_body(Google.Api.HttpRule.t() | map(), map()) :: map() - def map_response_body(%Google.Api.HttpRule{response_body: ""}, response_body), do: response_body + @spec map_response_body(t() | map(), map()) :: map() + def map_response_body(%{response_body: ""}, response_body), do: response_body # TODO The field is required to be present on the toplevel response message - def map_response_body(%Google.Api.HttpRule{response_body: field}, response_body) do + def map_response_body(%{response_body: field}, response_body) do key = String.to_existing_atom(field) Map.get(response_body, key) end @@ -95,8 +96,8 @@ defmodule GRPC.Server.Transcode do defp segment_to_string(segment), do: segment - @spec build_route(Google.Api.HttpRule.t()) :: {atom(), Template.route()} - def build_route(%Google.Api.HttpRule{pattern: {method, path}}) do + @spec build_route(t()) :: {atom(), Template.route()} + def build_route(%{pattern: {method, path}}) do route = path |> Template.tokenize([]) diff --git a/test/support/transcode_messages.proto b/test/support/transcode_messages.proto index d0f6cc4a..273e3127 100644 --- a/test/support/transcode_messages.proto +++ b/test/support/transcode_messages.proto @@ -17,6 +17,12 @@ service Messaging { }; } + rpc GetMessageWithSubPath(GetMessageRequest) returns (Message) { + option (google.api.http) = { + get: "/v1/{name=}" + }; + } + rpc GetMessageWithQuery(GetMessageRequest) returns (Message) { option (google.api.http) = { get: "/v1/messages" From 65e5a114c6b5eef7ae4a60971dce68c38e71a5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 26 Oct 2022 08:59:34 +0200 Subject: [PATCH 55/73] rm protoc plugin in favour of protobuf_generate --- examples/helloworld/README.md | 8 +- examples/helloworld/lib/helloworld.pb.ex | 57 +++++++++- examples/helloworld/lib/helloworld.svc.ex | 65 ----------- examples/helloworld/mix.exs | 1 + examples/helloworld/mix.lock | 1 + lib/google/api/http.pb.ex | 3 - lib/grpc/protoc/cli.ex | 88 --------------- lib/grpc/protoc/generator.ex | 68 ----------- lib/grpc/protoc/generator/service.ex | 70 ------------ mix.exs | 47 ++++---- mix.lock | 3 +- priv/templates/service.ex.eex | 20 ---- test/support/transcode_messages.pb.ex | 131 ++++++++++++++++++++++ test/support/transcode_messages.svc.ex | 123 -------------------- 14 files changed, 216 insertions(+), 469 deletions(-) delete mode 100644 examples/helloworld/lib/helloworld.svc.ex delete mode 100644 lib/grpc/protoc/cli.ex delete mode 100644 lib/grpc/protoc/generator.ex delete mode 100644 lib/grpc/protoc/generator/service.ex delete mode 100644 priv/templates/service.ex.eex delete mode 100644 test/support/transcode_messages.svc.ex diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index e3c2b345..5d672b8c 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -31,15 +31,15 @@ curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "an 1. Modify the proto `priv/protos/helloworld.proto` 2. Install `protoc` [here](https://developers.google.com/protocol-buffers/docs/downloads) -3. Install `protoc-gen-elixir` + ``` -mix escript.install hex protobuf +mix deps.get ``` + 4. Generate the code: ```shell -$ (cd ../../; mix build_protobuf_escript && mix escript.build) -$ protoc -I priv/protos --elixir_out=:./lib/ --grpc_elixir_out=./lib --plugin="../../deps/protobuf/protoc-gen-elixir" --plugin="../../protoc-gen-grpc_elixir" priv/protos/helloworld.proto +$ mix protobuf.generate --include-path=priv/protos --plugins=ProtobufGenerate.Plugins.GRPCWithOptions --output-path=./lib priv/protos/helloworld.proto ``` Refer to [protobuf-elixir](https://github.com/tony612/protobuf-elixir#usage) for more information. diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index 82cce674..e1a444b7 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -1,6 +1,5 @@ defmodule Helloworld.HelloRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string @@ -8,7 +7,6 @@ end defmodule Helloworld.HelloRequestFrom do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string @@ -17,7 +15,6 @@ end defmodule Helloworld.HelloReply do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :message, 1, type: :string @@ -26,7 +23,6 @@ end defmodule Helloworld.GetMessageRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :name, 1, type: :string @@ -34,8 +30,59 @@ end defmodule Helloworld.Message do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :text, 1, type: :string end + +defmodule Helloworld.Greeter.Service do + @moduledoc false + use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.11.0" + + rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/greeter/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:SayHelloFrom, Helloworld.HelloRequestFrom, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "*", + pattern: {:post, "/v1/greeter"}, + response_body: "", + selector: "" + } + } + }) +end + +defmodule Helloworld.Messaging.Service do + @moduledoc false + use GRPC.Service, name: "helloworld.Messaging", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetMessage, Helloworld.GetMessageRequest, Helloworld.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/{name=messages/*}"}, + response_body: "", + selector: "" + } + } + }) +end \ No newline at end of file diff --git a/examples/helloworld/lib/helloworld.svc.ex b/examples/helloworld/lib/helloworld.svc.ex deleted file mode 100644 index dcf882ab..00000000 --- a/examples/helloworld/lib/helloworld.svc.ex +++ /dev/null @@ -1,65 +0,0 @@ -defmodule Helloworld.Greeter.Service do - @moduledoc false - - use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.11.0" - - rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/greeter/{name}"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:SayHelloFrom, Helloworld.HelloRequestFrom, Helloworld.HelloReply, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "*", - pattern: {:post, "/v1/greeter"}, - response_body: "", - selector: "" - } - } - }) -end - -defmodule Helloworld.Greeter.Stub do - @moduledoc false - - use GRPC.Stub, service: Helloworld.Greeter.Service -end - -defmodule Helloworld.Messaging.Service do - @moduledoc false - - use GRPC.Service, name: "helloworld.Messaging", protoc_gen_elixir_version: "0.11.0" - - rpc(:GetMessage, Helloworld.GetMessageRequest, Helloworld.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/{name=messages/*}"}, - response_body: "", - selector: "" - } - } - }) -end - -defmodule Helloworld.Messaging.Stub do - @moduledoc false - - use GRPC.Stub, service: Helloworld.Messaging.Service -end diff --git a/examples/helloworld/mix.exs b/examples/helloworld/mix.exs index a1265354..f1b13088 100644 --- a/examples/helloworld/mix.exs +++ b/examples/helloworld/mix.exs @@ -20,6 +20,7 @@ defmodule Helloworld.Mixfile do [ {:grpc, path: "../../"}, {:protobuf, "~> 0.11.0"}, + {:protobuf_generate, "~> 0.1.1", only: [:dev, :test]}, {:jason, "~> 1.3.0"}, {:google_protos, "~> 0.3.0"}, {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false} diff --git a/examples/helloworld/mix.lock b/examples/helloworld/mix.lock index df73eb18..afdfde5e 100644 --- a/examples/helloworld/mix.lock +++ b/examples/helloworld/mix.lock @@ -7,5 +7,6 @@ "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, + "protobuf_generate": {:hex, :protobuf_generate, "0.1.1", "f6098b85161dcfd48a4f6f1abee4ee5e057981dfc50aafb1aa4bd5b0529aa89b", [:mix], [{:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "93a38c8e2aba2a17e293e9ef1359122741f717103984aa6d1ebdca0efb17ab9d"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/lib/google/api/http.pb.ex b/lib/google/api/http.pb.ex index 25dd83ad..ebd043a6 100644 --- a/lib/google/api/http.pb.ex +++ b/lib/google/api/http.pb.ex @@ -1,6 +1,5 @@ defmodule Google.Api.Http do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :rules, 1, repeated: true, type: Google.Api.HttpRule @@ -12,7 +11,6 @@ end defmodule Google.Api.HttpRule do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 oneof :pattern, 0 @@ -35,7 +33,6 @@ end defmodule Google.Api.CustomHttpPattern do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 field :kind, 1, type: :string diff --git a/lib/grpc/protoc/cli.ex b/lib/grpc/protoc/cli.ex deleted file mode 100644 index 19bc6537..00000000 --- a/lib/grpc/protoc/cli.ex +++ /dev/null @@ -1,88 +0,0 @@ -defmodule GRPC.Protoc.CLI do - @moduledoc """ - `protoc` plugin for generating Elixir code. - - `protoc-gen-elixir` (this name is important) **must** be in `$PATH`. You are not supposed - to call it directly, but only through `protoc`. - - ## Examples - - $ protoc --elixir_out=./lib your.proto - $ protoc --elixir_out=plugins=grpc:./lib/ *.proto - $ protoc -I protos --elixir_out=./lib protos/namespace/*.proto - - Options: - - * --version Print version of protobuf-elixir - * --help (-h) Print this help - - """ - - alias Protobuf.Protoc.Context - - # Entrypoint for the escript (protoc-gen-elixir). - @doc false - @spec main([String.t()]) :: :ok - def main(args) - - def main(["--version"]) do - {:ok, version} = :application.get_key(:protobuf, :vsn) - IO.puts(version) - end - - def main([opt]) when opt in ["--help", "-h"] do - IO.puts(@moduledoc) - end - - # When called through protoc, all input is passed through stdin. - def main([] = _args) do - Protobuf.load_extensions() - - # See https://groups.google.com/forum/#!topic/elixir-lang-talk/T5enez_BBTI. - :io.setopts(:standard_io, encoding: :latin1) - - # Read the standard input that protoc feeds us. - bin = binread_all!(:stdio) - - request = Protobuf.Decoder.decode(bin, Google.Protobuf.Compiler.CodeGeneratorRequest) - - ctx = - %Context{} - |> Protobuf.Protoc.CLI.parse_params(request.parameter || "") - |> Protobuf.Protoc.CLI.find_types(request.proto_file, request.file_to_generate) - - files = - Enum.flat_map(request.file_to_generate, fn file -> - desc = Enum.find(request.proto_file, &(&1.name == file)) - GRPC.Protoc.Generator.generate(ctx, desc) - end) - - Google.Protobuf.Compiler.CodeGeneratorResponse.new( - file: files, - supported_features: Protobuf.Protoc.CLI.supported_features() - ) - |> Protobuf.encode_to_iodata() - |> IO.binwrite() - end - - def main(_args) do - raise "invalid arguments. See protoc-gen-elixir --help." - end - - if Version.match?(System.version(), "~> 1.13") do - defp binread_all!(device) do - case IO.binread(device, :eof) do - data when is_binary(data) -> data - :eof -> _previous_behavior = "" - other -> raise "reading from #{inspect(device)} failed: #{inspect(other)}" - end - end - else - defp binread_all!(device) do - case IO.binread(device, :all) do - data when is_binary(data) -> data - other -> raise "reading from #{inspect(device)} failed: #{inspect(other)}" - end - end - end -end diff --git a/lib/grpc/protoc/generator.ex b/lib/grpc/protoc/generator.ex deleted file mode 100644 index e7a3085a..00000000 --- a/lib/grpc/protoc/generator.ex +++ /dev/null @@ -1,68 +0,0 @@ -defmodule GRPC.Protoc.Generator do - @moduledoc false - - alias Protobuf.Protoc.Context - alias Protobuf.Protoc.Generator - - @spec generate(Context.t(), %Google.Protobuf.FileDescriptorProto{}) :: - [Google.Protobuf.Compiler.CodeGeneratorResponse.File.t()] - def generate(%Context{} = ctx, %Google.Protobuf.FileDescriptorProto{} = desc) do - module_definitions = - ctx - |> generate_module_definitions(desc) - |> Enum.reject(&is_nil/1) - - if ctx.one_file_per_module? do - Enum.map(module_definitions, fn {mod_name, content} -> - file_name = Macro.underscore(mod_name) <> ".svc.ex" - - Google.Protobuf.Compiler.CodeGeneratorResponse.File.new( - name: file_name, - content: content - ) - end) - else - # desc.name is the filename, ending in ".proto". - file_name = Path.rootname(desc.name) <> ".svc.ex" - - content = - module_definitions - |> Enum.map(fn {_mod_name, contents} -> [contents, ?\n] end) - |> IO.iodata_to_binary() - |> Generator.Util.format() - - [ - Google.Protobuf.Compiler.CodeGeneratorResponse.File.new( - name: file_name, - content: content - ) - ] - end - end - - defp generate_module_definitions(ctx, %Google.Protobuf.FileDescriptorProto{} = desc) do - ctx = - %Context{ - ctx - | syntax: syntax(desc.syntax), - package: desc.package, - dep_type_mapping: get_dep_type_mapping(ctx, desc.dependency, desc.name) - } - |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc) - - Enum.map(desc.service, &GRPC.Protoc.Generator.Service.generate(ctx, &1)) - end - - defp get_dep_type_mapping(%Context{global_type_mapping: global_mapping}, deps, file_name) do - mapping = - Enum.reduce(deps, %{}, fn dep, acc -> - Map.merge(acc, global_mapping[dep]) - end) - - Map.merge(mapping, global_mapping[file_name]) - end - - defp syntax("proto3"), do: :proto3 - defp syntax("proto2"), do: :proto2 - defp syntax(nil), do: :proto2 -end diff --git a/lib/grpc/protoc/generator/service.ex b/lib/grpc/protoc/generator/service.ex deleted file mode 100644 index 8af7eb20..00000000 --- a/lib/grpc/protoc/generator/service.ex +++ /dev/null @@ -1,70 +0,0 @@ -defmodule GRPC.Protoc.Generator.Service do - @moduledoc false - - alias Protobuf.Protoc.Context - alias Protobuf.Protoc.Generator.Util - - require EEx - - EEx.function_from_file( - :defp, - :service_template, - Path.expand("./templates/service.ex.eex", :code.priv_dir(:grpc)), - [:assigns] - ) - - @spec generate(Context.t(), Google.Protobuf.ServiceDescriptorProto.t()) :: - {String.t(), String.t()} - def generate(%Context{} = ctx, %Google.Protobuf.ServiceDescriptorProto{} = desc) do - # service can't be nested - mod_name = Util.mod_name(ctx, [Macro.camelize(desc.name)]) - name = Util.prepend_package_prefix(ctx.package, desc.name) - methods = Enum.map(desc.method, &generate_service_method(ctx, &1)) - - descriptor_fun_body = - if ctx.gen_descriptors? do - Util.descriptor_fun_body(desc) - else - nil - end - - {mod_name, - Util.format( - service_template( - module: mod_name, - service_name: name, - methods: methods, - descriptor_fun_body: descriptor_fun_body, - version: Util.version() - ) - )} - end - - defp generate_service_method(ctx, method) do - input = service_arg(Util.type_from_type_name(ctx, method.input_type), method.client_streaming) - - output = - service_arg(Util.type_from_type_name(ctx, method.output_type), method.server_streaming) - - options = - method.options - |> opts() - |> inspect(limit: :infinity) - - {method.name, input, output, options} - end - - defp service_arg(type, _streaming? = true), do: "stream(#{type})" - defp service_arg(type, _streaming?), do: type - - defp opts(%Google.Protobuf.MethodOptions{__pb_extensions__: extensions}) - when extensions == %{} do - %{} - end - - defp opts(%Google.Protobuf.MethodOptions{__pb_extensions__: extensions}) do - for {{type, field}, value} <- extensions, into: %{} do - {field, %{type: type, value: value}} - end - end -end diff --git a/mix.exs b/mix.exs index e4cf4f9f..645970fc 100644 --- a/mix.exs +++ b/mix.exs @@ -13,7 +13,6 @@ defmodule GRPC.Mixfile do start_permanent: Mix.env() == :prod, deps: deps(), package: package(), - escript: escript(), aliases: aliases(), description: "The Elixir implementation of gRPC", docs: [ @@ -39,10 +38,6 @@ defmodule GRPC.Mixfile do [extra_applications: [:logger]] end - def escript do - [main_module: GRPC.Protoc.CLI, name: "protoc-gen-grpc_elixir"] - end - defp deps do [ {:cowboy, "~> 2.9"}, @@ -52,6 +47,7 @@ defmodule GRPC.Mixfile do {:jason, "~> 1.0", optional: true}, {:cowlib, "~> 2.11"}, {:protobuf, "~> 0.11"}, + {:protobuf_generate, "~> 0.1.1", only: [:dev, :test]}, {:ex_doc, "~> 0.28.0", only: :dev}, {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false}, {:googleapis, @@ -74,45 +70,52 @@ defmodule GRPC.Mixfile do defp aliases do [ - gen_bootstrap_protos: [&build_protobuf_escript/1, &gen_bootstrap_protos/1], - build_protobuf_escript: &build_protobuf_escript/1 + test: [ + &gen_test_protos/1, + "test" + ], + gen_bootstrap_protos: &gen_bootstrap_protos/1 ] end defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] - defp build_protobuf_escript(_args) do - path = Mix.Project.deps_paths().protobuf + defp gen_test_protos(_args) do + api_src = Mix.Project.deps_paths().googleapis + transcode_src = "test/support" - File.cd!(path, fn -> - with 0 <- Mix.shell().cmd("mix deps.get"), - 0 <- Mix.shell().cmd("mix escript.build") do - :ok - else - other -> - Mix.raise("build_protobuf_escript/1 exited with non-zero status: #{other}") - end - end) + protoc!( + [ + "--include-path=#{api_src}", + "--include-path=#{transcode_src}", + "--plugins=ProtobufGenerate.Plugins.GRPCWithOptions" + ], + "./#{transcode_src}", + ["test/support/transcode_messages.proto"] + ) end # https://github.com/elixir-protobuf/protobuf/blob/cdf3acc53f619866b4921b8216d2531da52ceba7/mix.exs#L140 defp gen_bootstrap_protos(_args) do proto_src = Mix.Project.deps_paths().googleapis - protoc!("-I \"#{proto_src}\"", "./lib", [ + protoc!("--include-path=#{proto_src}", "./lib", [ "google/api/http.proto", "google/api/annotations.proto" ]) end + defp protoc!(args, elixir_out, files_to_generate) when is_list(args) do + protoc!(Enum.join(args, " "), elixir_out, files_to_generate) + end + defp protoc!(args, elixir_out, files_to_generate) when is_binary(args) and is_binary(elixir_out) and is_list(files_to_generate) do args = [ - ~s(protoc), - ~s(--plugin=./deps/protobuf/protoc-gen-elixir), - ~s(--elixir_out="#{elixir_out}"), + ~s(mix protobuf.generate), + ~s(--output-path="#{elixir_out}"), args ] ++ files_to_generate diff --git a/mix.lock b/mix.lock index c6c5a74f..d3936c63 100644 --- a/mix.lock +++ b/mix.lock @@ -8,11 +8,12 @@ "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, "googleapis": {:git, "https://github.com/googleapis/googleapis.git", "f0e2be46a5602ad903800811c9583f9e4458de3c", [branch: "master"]}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, - "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, + "protobuf_generate": {:hex, :protobuf_generate, "0.1.1", "f6098b85161dcfd48a4f6f1abee4ee5e057981dfc50aafb1aa4bd5b0529aa89b", [:mix], [{:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "93a38c8e2aba2a17e293e9ef1359122741f717103984aa6d1ebdca0efb17ab9d"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/priv/templates/service.ex.eex b/priv/templates/service.ex.eex deleted file mode 100644 index 087b106d..00000000 --- a/priv/templates/service.ex.eex +++ /dev/null @@ -1,20 +0,0 @@ -defmodule <%= @module %>.Service do - @moduledoc false - use GRPC.Service, name: <%= inspect(@service_name) %>, protoc_gen_elixir_version: "<%= @version %>" - - <%= if @descriptor_fun_body do %> - def descriptor do - # credo:disable-for-next-line - <%= @descriptor_fun_body %> - end - <% end %> - - <%= for {method_name, input, output, options} <- @methods do %> - rpc :<%= method_name %>, <%= input %>, <%= output %>, <%= options %> - <% end %> -end - -defmodule <%= @module %>.Stub do - @moduledoc false - use GRPC.Stub, service: <%= @module %>.Service -end diff --git a/test/support/transcode_messages.pb.ex b/test/support/transcode_messages.pb.ex index 59015514..bfc042d5 100644 --- a/test/support/transcode_messages.pb.ex +++ b/test/support/transcode_messages.pb.ex @@ -26,3 +26,134 @@ defmodule Transcode.NestedMessageRequest do field :message, 1, type: Transcode.GetMessageRequest end + +defmodule Transcode.Messaging.Service do + @moduledoc false + use GRPC.Service, name: "transcode.Messaging", protoc_gen_elixir_version: "0.11.0" + + rpc(:GetMessage, Transcode.GetMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:StreamMessages, Transcode.GetMessageRequest, stream(Transcode.Message), %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/stream/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithSubPath, Transcode.GetMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/{name=}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithQuery, Transcode.GetMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithFieldPath, Transcode.NestedMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/fieldpath/{message.name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:CreateMessage, Transcode.Message, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "*", + pattern: {:post, "/v1/messages"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithResponseBody, Transcode.GetMessageRequest, Transcode.MessageOut, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/response_body/{name}"}, + response_body: "response", + selector: "" + } + } + }) + + rpc(:CreateMessageWithNestedBody, Transcode.NestedMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "message", + pattern: {:post, "/v1/messages/nested"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:GetMessageWithSubpathQuery, Transcode.NestedMessageRequest, Transcode.Message, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/messages/nested"}, + response_body: "", + selector: "" + } + } + }) +end diff --git a/test/support/transcode_messages.svc.ex b/test/support/transcode_messages.svc.ex deleted file mode 100644 index edb768c0..00000000 --- a/test/support/transcode_messages.svc.ex +++ /dev/null @@ -1,123 +0,0 @@ -defmodule Transcode.Messaging.Service do - @moduledoc false - - use GRPC.Service, name: "transcode.Messaging", protoc_gen_elixir_version: "0.11.0" - - rpc(:GetMessage, Transcode.GetMessageRequest, Transcode.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/messages/{name}"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:StreamMessages, Transcode.GetMessageRequest, stream(Transcode.Message), %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/messages/stream/{name}"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:GetMessageWithQuery, Transcode.GetMessageRequest, Transcode.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/messages"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:GetMessageWithFieldPath, Transcode.NestedMessageRequest, Transcode.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/messages/fieldpath/{message.name}"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:CreateMessage, Transcode.Message, Transcode.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "*", - pattern: {:post, "/v1/messages"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:GetMessageWithResponseBody, Transcode.GetMessageRequest, Transcode.MessageOut, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/messages/response_body/{name}"}, - response_body: "response", - selector: "" - } - } - }) - - rpc(:CreateMessageWithNestedBody, Transcode.NestedMessageRequest, Transcode.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "message", - pattern: {:post, "/v1/messages/nested"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:GetMessageWithSubpathQuery, Transcode.NestedMessageRequest, Transcode.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/messages/nested"}, - response_body: "", - selector: "" - } - } - }) -end - -defmodule Transcode.Messaging.Stub do - @moduledoc false - - use GRPC.Stub, service: Transcode.Messaging.Service -end From ee9b94d96a2ec7c50fa760e027ee8eaab523f62d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 26 Oct 2022 09:01:06 +0200 Subject: [PATCH 56/73] add new cowboy router middleware The default routing in cowboy is not capable enough to express 'complex' routes on the form `/v1/{a=shelves/*}/books/**`. The added router is derived from `:cowboy_router` but implements new matching to allow to match on the above statement. --- lib/grpc/server/adapters/cowboy.ex | 12 +- lib/grpc/server/adapters/cowboy/router.ex | 367 ++++++++++++++++++ lib/grpc/server/transcode/template.ex | 4 + src/grpc_stream_h.erl | 2 +- .../server/adapter/cowboy/router_test.exs | 169 ++++++++ test/grpc/server/transcode/template_test.exs | 30 +- 6 files changed, 575 insertions(+), 9 deletions(-) create mode 100644 lib/grpc/server/adapters/cowboy/router.ex create mode 100644 test/grpc/server/adapter/cowboy/router_test.exs diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index e914621c..8a236f49 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -186,15 +186,12 @@ defmodule GRPC.Server.Adapters.Cowboy do end defp build_route({:grpc, path}, endpoint, server, opts) do - {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, - {endpoint, server, path, Enum.into(opts, %{})}} + {path, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} end - defp build_route({:http_transcode, spec}, endpoint, server, opts) do + defp build_route({:http_transcode, {_method, route} = spec}, endpoint, server, opts) do path = GRPC.Server.Transcode.to_path(spec) - - {String.to_charlist(path), GRPC.Server.Adapters.Cowboy.Handler, - {endpoint, server, path, Enum.into(opts, %{})}} + {route, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} end defp cowboy_start_args(endpoint, servers, port, opts) do @@ -212,7 +209,8 @@ defmodule GRPC.Server.Adapters.Cowboy do handlers end - dispatch = :cowboy_router.compile([{:_, handlers}]) + dispatch = GRPC.Server.Adapters.Cowboy.Router.compile([{:_, handlers}]) + idle_timeout = Keyword.get(opts, :idle_timeout) || :infinity num_acceptors = Keyword.get(opts, :num_acceptors) || @default_num_acceptors max_connections = Keyword.get(opts, :max_connections) || @default_max_connections diff --git a/lib/grpc/server/adapters/cowboy/router.ex b/lib/grpc/server/adapters/cowboy/router.ex new file mode 100644 index 00000000..c847cc51 --- /dev/null +++ b/lib/grpc/server/adapters/cowboy/router.ex @@ -0,0 +1,367 @@ +defmodule GRPC.Server.Adapters.Cowboy.Router do + use Bitwise + @behaviour :cowboy_middleware + + @dialyzer {:nowarn_function, compile: 1} + + def compile(routes) do + for {host, paths} <- routes do + [{host_match, _, _}] = :cowboy_router.compile([{host, []}]) + compiled_paths = compile_paths(paths, []) + + {host_match, [], compiled_paths} + end + end + + def compile_paths([], acc) do + Enum.reverse(acc) + end + + def compile_paths([{route, handler, opts} | paths], acc) when is_binary(route) do + {_, route} = GRPC.Server.Transcode.build_route(%{pattern: {:post, route}}) + + compile_paths(paths, [{route, [], handler, opts} | acc]) + end + + def compile_paths([{route, handler, opts} | paths], acc) do + compile_paths(paths, [{route, [], handler, opts} | acc]) + end + + @impl :cowboy_middleware + def execute( + req = %{host: host, path: path}, + env = %{dispatch: dispatch} + ) do + dispatch = + case dispatch do + {:persistent_term, key} -> + :persistent_term.get(key) + + _ -> + dispatch + end + + case match(dispatch, host, path) do + {:ok, handler, handler_opts, bindings, host_info, path_info} -> + {:ok, Map.merge(req, %{host_info: host_info, path_info: path_info, bindings: bindings}), + Map.merge(env, %{handler: handler, handler_opts: handler_opts})} + + {:error, :notfound, :host} -> + {:stop, :cowboy_req.reply(400, req)} + + {:error, :badrequest, :path} -> + {:stop, :cowboy_req.reply(400, req)} + + {:error, :notfound, :path} -> + {:stop, :cowboy_req.reply(404, req)} + end + end + + def match([], _, _) do + {:error, :notfound, :host} + end + + def match([{:_, [], path_matchs} | _Tail], _, path) do + match_path(path_matchs, :undefined, path, %{}) + end + + def match([{host_match, fields, path_matchs} | tail], tokens, path) + when is_list(tokens) do + case list_match(tokens, host_match, %{}) do + false -> + match(tail, tokens, path) + + {true, bindings, host_info} -> + host_info = + case host_info do + :undefined -> + :undefined + + _ -> + Enum.reverse(host_info) + end + + case check_constraints(fields, bindings) do + {:ok, bindings} -> + match_path(path_matchs, host_info, path, bindings) + + :nomatch -> + match(tail, tokens, path) + end + end + end + + def match(dispatch, host, path) do + match(dispatch, split_host(host), path) + end + + defp match_path([], _, _, _) do + {:error, :notfound, :path} + end + + defp match_path([{:_, [], handler, opts} | _Tail], host_info, _, bindings) do + {:ok, handler, opts, bindings, host_info, :undefined} + end + + defp match_path([{"*", _, handler, opts} | _Tail], host_info, "*", bindings) do + {:ok, handler, opts, bindings, host_info, :undefined} + end + + defp match_path([_ | tail], host_info, "*", bindings) do + match_path(tail, host_info, "*", bindings) + end + + defp match_path([{path_match, fields, handler, opts} | tail], host_info, tokens, bindings) + when is_list(tokens) do + case list_match(tokens, path_match, bindings) do + false -> + match_path(tail, host_info, tokens, bindings) + + {true, path_binds, path_info} -> + case check_constraints(fields, path_binds) do + {:ok, path_binds} -> + {:ok, handler, opts, path_binds, host_info, path_info} + + :nomatch -> + match_path(tail, host_info, tokens, bindings) + end + end + end + + defp match_path(_Dispatch, _HostInfo, :badrequest, _Bindings) do + {:error, :badrequest, :path} + end + + defp match_path(dispatch, host_info, path, bindings) do + match_path(dispatch, host_info, split_path(path), bindings) + end + + defp check_constraints([], bindings) do + {:ok, bindings} + end + + defp check_constraints([field | tail], bindings) when is_atom(field) do + check_constraints(tail, bindings) + end + + defp check_constraints([field | tail], bindings) do + name = :erlang.element(1, field) + + case bindings do + %{^name => value} -> + constraints = :erlang.element(2, field) + + case :cowboy_constraints.validate( + value, + constraints + ) do + {:ok, value} -> + check_constraints(tail, Map.put(bindings, name, value)) + + {:error, _} -> + :nomatch + end + + _ -> + check_constraints(tail, bindings) + end + end + + defp split_host(host) do + split_host(host, []) + end + + defp split_host(host, acc) do + case :binary.match(host, ".") do + :nomatch when host === <<>> -> + acc + + :nomatch -> + [host | acc] + + {pos, _} -> + <> = host + false = byte_size(segment) == 0 + split_host(rest, [segment | acc]) + end + end + + defp split_path(<>) do + split_path(path, []) + end + + defp split_path(_) do + :badrequest + end + + defp split_path(path, acc) do + try do + case :binary.match(path, "/") do + :nomatch when path === <<>> -> + acc + |> Enum.map(&:cow_uri.urldecode/1) + |> Enum.reverse() + |> remove_dot_segments([]) + + :nomatch -> + [path | acc] + |> Enum.map(&:cow_uri.urldecode/1) + |> Enum.reverse() + |> remove_dot_segments([]) + + {pos, _} -> + <> = path + split_path(rest, [segment | acc]) + end + catch + :error, _ -> + :badrequest + end + end + + defp remove_dot_segments([], acc) do + Enum.reverse(acc) + end + + defp remove_dot_segments(["." | segments], acc) do + remove_dot_segments(segments, acc) + end + + defp remove_dot_segments([".." | segments], acc = []) do + remove_dot_segments(segments, acc) + end + + defp remove_dot_segments([".." | segments], [_ | acc]) do + remove_dot_segments(segments, acc) + end + + defp remove_dot_segments([s | segments], acc) do + remove_dot_segments(segments, [s | acc]) + end + + def list_match(list, [{:__, []}], binds) do + {true, binds, list} + end + + def list_match([_s | tail], [{:_, _} | tail_match], binds) do + list_match(tail, tail_match, binds) + end + + def list_match([s | tail], [s | tail_match], binds) do + list_match(tail, tail_match, binds) + end + + def list_match([segment | tail], [{binding, [{:_, _}]} | matchers], bindings) do + put_binding(bindings, binding, segment, tail, matchers) + end + + def list_match([segment | tail], [{binding, [segment]} | matchers], bindings) + when is_atom(binding) do + put_binding(bindings, binding, segment, tail, matchers) + end + + def list_match(rest, [{binding, [{any, _}]}], bindings) + when is_atom(binding) and any in [:_, :__] do + value = Enum.join(rest, "/") + + list_match([], [], Map.put(bindings, binding, value)) + end + + def list_match([segment | _] = rest, [{binding, [segment, {any, _}]}], bindings) + when is_atom(binding) and any in [:_, :__] do + value = Enum.join(rest, "/") + + list_match([], [], Map.put(bindings, binding, value)) + end + + def list_match( + [segment | tail], + [{binding, [segment | sub_matches]} | matches], + bindings + ) + when is_atom(binding) do + end_condition = + case matches do + [next | _] -> next + [] -> :undefined + end + + with {matched_segments, tail} <- match_until(tail, end_condition, sub_matches, []) do + value = Enum.join([segment | matched_segments], "/") + bindings = Map.put(bindings, binding, value) + + list_match(tail, matches, bindings) + end + end + + def list_match([segment | tail], [{binding, []} | matchers], bindings) when is_atom(binding) do + put_binding(bindings, binding, segment, tail, matchers) + end + + def list_match([], [], binds) do + {true, binds, :undefined} + end + + def list_match(_list, _match, _binds) do + false + end + + # End recursion, since there's no "outside" matches we should iterate to end of segments + def match_until([], :undefined, [], acc) do + {Enum.reverse(acc), []} + end + + # End recursion, end condition is a binding with a matching complex start segment + def match_until( + [segment | _] = segments, + _end_condition = {binding, [segment | _]}, + [], + acc + ) + when is_atom(binding) do + {Enum.reverse(acc), segments} + end + + # End recursion since the submatch contains a trailing wildcard but we have more matches "outside" this sub-segment + def match_until([segment | _] = segments, _end_condition = segment, [], acc) do + {Enum.reverse(acc), segments} + end + + # Reached the "end" of this wildcard, so we proceed with the next match + def match_until([_segment | _] = segments, end_condition, [{:__, []}, match | matches], acc) do + match_until(segments, end_condition, [match | matches], acc) + end + + # Segment is matching the wildcard and have not reached "end" of wildcard + def match_until([segment | segments], end_condition, [{:__, []} | _] = matches, acc) do + match_until(segments, end_condition, matches, [segment | acc]) + end + + # Current match is matching segment, add to accumulator and set next match as the current one + def match_until([segment | segments], end_condition, [segment | matches], acc) do + match_until(segments, end_condition, matches, [segment | acc]) + end + + # 'Any' match is matching first segment, add to accumulator and set next match as the current one + def match_until([segment | segments], end_condition, [{:_, []} | matches], acc) do + match_until(segments, end_condition, matches, [segment | acc]) + end + + # No match + def match_until(_segments, _end_condition, _matches, _acc) do + false + end + + defp put_binding(bindings, binding, value, tail, matchers) do + case bindings do + %{^binding => ^value} -> + list_match(tail, matchers, bindings) + + %{^binding => _} -> + false + + _ -> + list_match(tail, matchers, Map.put(bindings, binding, value)) + end + end +end diff --git a/lib/grpc/server/transcode/template.ex b/lib/grpc/server/transcode/template.ex index 02441a71..c5211446 100644 --- a/lib/grpc/server/transcode/template.ex +++ b/lib/grpc/server/transcode/template.ex @@ -50,6 +50,10 @@ defmodule GRPC.Server.Transcode.Template do parse(rest, segments) end + def parse([{:*, _}, {:*, _} | rest], segments) do + parse(rest, [{:__, []} | segments]) + end + def parse([{:*, _} | rest], segments) do parse(rest, [{:_, []} | segments]) end diff --git a/src/grpc_stream_h.erl b/src/grpc_stream_h.erl index 4e2652e7..42e2c052 100644 --- a/src/grpc_stream_h.erl +++ b/src/grpc_stream_h.erl @@ -33,7 +33,7 @@ -> {[{spawn, pid(), timeout()}], #state{}}. init(StreamID, Req=#{ref := Ref}, Opts) -> Env = maps:get(env, Opts, #{}), - Middlewares = maps:get(middlewares, Opts, [cowboy_router, cowboy_handler]), + Middlewares = maps:get(middlewares, Opts, ['Elixir.GRPC.Server.Adapters.Cowboy.Router', cowboy_handler]), Shutdown = maps:get(shutdown_timeout, Opts, 5000), Pid = proc_lib:spawn_link(?MODULE, request_process, [Req, Env, Middlewares]), Expect = expect(Req), diff --git a/test/grpc/server/adapter/cowboy/router_test.exs b/test/grpc/server/adapter/cowboy/router_test.exs new file mode 100644 index 00000000..cebec6f7 --- /dev/null +++ b/test/grpc/server/adapter/cowboy/router_test.exs @@ -0,0 +1,169 @@ +defmodule GRPC.Server.Adapters.Cowboy.RouterTest do + use ExUnit.Case, async: true + alias GRPC.Server.Adapters.Cowboy.Router + + describe "match/3" do + test "with no_host" do + assert {:error, :notfound, :host} = Router.match([], [], []) + end + + test "with no bindings" do + dispatch = make_dispatch("/transcode.Messaging/GetMessage") + + assert {:ok, Handler, [], %{}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/transcode.Messaging/GetMessage") + + assert {:error, :notfound, :path} == Router.match(dispatch, "localhost", "/unknown/path") + end + + test "with simple bindings" do + dispatch = make_dispatch("/v1/{name}") + + assert {:ok, Handler, [], %{name: "messages"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/messages") + end + + test "with nested bindings" do + dispatch = make_dispatch("/v1/{message.name}") + + assert {:ok, Handler, [], %{"message.name": "messages"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/messages") + end + + test "with multiple bindings" do + dispatch = make_dispatch("/v1/users/{user_id}/messages/{message.message_id}") + + assert {:ok, Handler, [], %{user_id: "1", "message.message_id": "2"}, :undefined, + :undefined} == + Router.match(dispatch, "localhost", "/v1/users/1/messages/2") + end + + test "with multiple sequential bindings" do + dispatch = make_dispatch("/v1/{a}/{b}/{c}") + + assert {:ok, Handler, [], %{a: "a", b: "b", c: "c"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/a/b/c") + end + + test "with any " do + dispatch = make_dispatch("/*") + + assert {:ok, Handler, [], %{}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1") + end + + test "with 'any' assignment" do + dispatch = make_dispatch("/{a=*}") + + assert {:ok, Handler, [], %{a: "v1"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1") + end + + test "with 'catch all' assignment" do + dispatch = make_dispatch("/{a=**}") + + assert {:ok, Handler, [], %{a: "v1/messages"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/messages") + end + + test "with 'any' and 'catch all'" do + dispatch = make_dispatch("/*/**") + + assert {:ok, Handler, [], %{}, :undefined, ["foo", "bar", "baz"]} == + Router.match(dispatch, "localhost", "/v1/foo/bar/baz") + end + + test "with 'any' and 'catch all' assignment" do + dispatch = make_dispatch("/*/a/{b=c/*}/d/{e=**}") + + assert {:ok, Handler, [], %{b: "c/foo", e: "bar/baz/biz"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/a/c/foo/d/bar/baz/biz") + end + + test "with complex binding" do + dispatch = make_dispatch("/v1/{name=messages}") + + assert {:ok, Handler, [], %{name: "messages"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/messages") + + assert {:error, :notfound, :path} == + Router.match(dispatch, "localhost", "/v1/should_not_match") + end + + test "with complex binding and 'any'" do + dispatch = make_dispatch("/v1/{name=messages/*}") + + assert {:ok, Handler, [], %{name: "messages/12345"}, :undefined, :undefined} == + Router.match(dispatch, "localhost", "/v1/messages/12345") + + assert {:error, :notfound, :path} == + Router.match(dispatch, "localhost", "/v1/should_not_match/12345") + end + + test "with complex binding, wildcards and trailing route" do + dispatch = make_dispatch("/v1/{name=shelves/*/books/*}") + + assert {:ok, Handler, [], %{name: "shelves/example-shelf/books/example-book"}, :undefined, + :undefined} == + Router.match(dispatch, "localhost", "/v1/shelves/example-shelf/books/example-book") + + assert {:error, :notfound, :path} == + Router.match(dispatch, "localhost", "/v1/shelves/example-shelf/not_books") + end + + test "with complex binding, wildcards and suffix" do + dispatch = make_dispatch("/v1/{name=shelves/*/books/*}/suffix") + + assert {:ok, Handler, [], %{name: "shelves/example-shelf/books/example-book"}, :undefined, + :undefined} == + Router.match( + dispatch, + "localhost", + "/v1/shelves/example-shelf/books/example-book/suffix" + ) + + assert {:error, :notfound, :path} == + Router.match( + dispatch, + "localhost", + "/v1/shelves/example-shelf/books/example-book/another_suffix" + ) + end + + test "with mixed complex binding" do + dispatch = make_dispatch("/v1/{a=users/*}/messages/{message_id}/{c=books/*}") + + assert {:ok, Handler, [], %{a: "users/foobar", message_id: "1", c: "books/barbaz"}, + :undefined, + :undefined} == + Router.match(dispatch, "localhost", "/v1/users/foobar/messages/1/books/barbaz") + + assert {:error, :notfound, :path} == + Router.match(dispatch, "localhost", "/v1/users/1/books/barbaz") + end + + test "with mixed sequential complex binding" do + dispatch = make_dispatch("/v1/{a=users/*}/{b=messages}/{c=books/*}") + + assert {:ok, Handler, [], %{a: "users/foobar", b: "messages", c: "books/barbaz"}, + :undefined, + :undefined} == + Router.match(dispatch, "localhost", "/v1/users/foobar/messages/books/barbaz") + + assert {:error, :notfound, :path} == + Router.match(dispatch, "localhost", "/v1/users/foobar/messages/book/books/barbaz") + end + end + + defp make_dispatch(path) do + rule = %{pattern: {:get, path}} + {_method, route} = GRPC.Server.Transcode.build_route(rule) + + [ + {:_, [], + [ + {route, [], Handler, []} + ]} + ] + end +end diff --git a/test/grpc/server/transcode/template_test.exs b/test/grpc/server/transcode/template_test.exs index a91ef77f..c124cfb9 100644 --- a/test/grpc/server/transcode/template_test.exs +++ b/test/grpc/server/transcode/template_test.exs @@ -78,6 +78,20 @@ defmodule GRPC.Transcode.TemplateTest do {:"}", []} ] == Template.tokenize("/v1/messages/{message_id}/{sub.subfield}") end + + test "can tokenize single wildcard" do + assert [{:/, []}, {:*, []}] == Template.tokenize("/*") + end + + test "can tokenize multiple wildcards" do + assert [ + {:/, []}, + {:*, []}, + {:/, []}, + {:*, []}, + {:*, []} + ] == Template.tokenize("/*/**") + end end describe "parse/3" do @@ -95,13 +109,20 @@ defmodule GRPC.Transcode.TemplateTest do |> Template.parse([]) end - test "can parse paths with wildcards" do + test "can parse paths with 'any'" do assert ["v1", "messages", {:_, []}] == "/v1/messages/*" |> Template.tokenize() |> Template.parse([]) end + test "can parse paths with 'catch all'" do + assert ["v1", "messages", {:__, []}] == + "/v1/messages/**" + |> Template.tokenize() + |> Template.parse([]) + end + test "can parse simple bindings with variables" do assert ["v1", "messages", {:message_id, []}] == "/v1/messages/{message_id}" @@ -116,6 +137,13 @@ defmodule GRPC.Transcode.TemplateTest do |> Template.parse([]) end + test "can parse bindings with variable assignment to any" do + assert ["v1", {:name, [{:_, []}]}] == + "/v1/{name=*}" + |> Template.tokenize() + |> Template.parse([]) + end + test "can parse multiple bindings with variable assignment" do assert ["v1", {:name, ["messages"]}, {:message_id, []}] == "/v1/{name=messages}/{message_id}" From 95a4540f7adf3dc7b120a73eebc84e4a45916b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 26 Oct 2022 16:56:57 +0200 Subject: [PATCH 57/73] move routing specific func. to GRPC.Server.Router --- examples/helloworld/lib/helloworld.pb.ex | 2 +- lib/grpc/server.ex | 16 +- lib/grpc/server/adapters/cowboy.ex | 5 +- lib/grpc/server/adapters/cowboy/router.ex | 254 ++---------------- lib/grpc/server/router.ex | 214 +++++++++++++++ .../{transcode => router}/field_path.ex | 2 +- .../server/{transcode => router}/query.ex | 4 +- .../server/{transcode => router}/template.ex | 57 ++-- lib/grpc/server/stream.ex | 2 +- lib/grpc/server/transcode.ex | 39 +-- .../server/adapter/cowboy/router_test.exs | 7 +- .../{transcode => router}/query_test.exs | 4 +- .../{transcode => router}/template_test.exs | 4 +- test/grpc/server/router_test.exs | 161 +++++++++++ test/grpc/server/transcode_test.exs | 22 -- 15 files changed, 442 insertions(+), 351 deletions(-) create mode 100644 lib/grpc/server/router.ex rename lib/grpc/server/{transcode => router}/field_path.ex (91%) rename lib/grpc/server/{transcode => router}/query.ex (94%) rename lib/grpc/server/{transcode => router}/template.ex (53%) rename test/grpc/server/{transcode => router}/query_test.exs (92%) rename test/grpc/server/{transcode => router}/template_test.exs (98%) create mode 100644 test/grpc/server/router_test.exs diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index e1a444b7..0fa96786 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -85,4 +85,4 @@ defmodule Helloworld.Messaging.Service do } } }) -end \ No newline at end of file +end diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 905535d2..bc916578 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -113,7 +113,7 @@ defmodule GRPC.Server do require Logger alias GRPC.RPCError - alias GRPC.Server.Transcode + alias GRPC.Server.{Router, Transcode} @type rpc_req :: struct | Enumerable.t() @type rpc_return :: struct | any @@ -123,7 +123,7 @@ defmodule GRPC.Server do quote bind_quoted: [opts: opts], location: :keep do service_mod = opts[:service] service_name = service_mod.__meta__(:name) - codecs = opts[:codecs] || [GRPC.Codec.Proto, GRPC.Codec.WebText, GRPC.Codec.JSON] + codecs = opts[:codecs] || [GRPC.Codec.Proto, GRPC.Codec.WebText] compressors = opts[:compressors] || [] http_transcode = opts[:http_transcode] || false @@ -134,15 +134,16 @@ defmodule GRPC.Server do acc -> path = "/#{service_name}/#{name}" - http_paths = + acc = if http_transcode and Map.has_key?(options, :http) do %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) - [{:http_transcode, Transcode.build_route(http_rule)}] + route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) + [route | acc] else - [] + acc end - http_paths ++ [{:grpc, path} | acc] + [{:grpc, path} | acc] end Enum.each(service_mod.__rpc_calls__, fn {name, _, _, options} = rpc -> @@ -166,8 +167,7 @@ defmodule GRPC.Server do if http_transcode and Map.has_key?(options, :http) do %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) - {http_method, _} = spec = Transcode.build_route(http_rule) - http_path = Transcode.to_path(spec) + {http_method, http_path, _matches} = Router.build_route(http_rule) def __call_rpc__(unquote(http_path), unquote(http_method), stream) do GRPC.Server.call( diff --git a/lib/grpc/server/adapters/cowboy.ex b/lib/grpc/server/adapters/cowboy.ex index 8a236f49..ae7c1e69 100644 --- a/lib/grpc/server/adapters/cowboy.ex +++ b/lib/grpc/server/adapters/cowboy.ex @@ -189,9 +189,8 @@ defmodule GRPC.Server.Adapters.Cowboy do {path, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} end - defp build_route({:http_transcode, {_method, route} = spec}, endpoint, server, opts) do - path = GRPC.Server.Transcode.to_path(spec) - {route, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} + defp build_route({:http_transcode, {_method, path, match}}, endpoint, server, opts) do + {match, GRPC.Server.Adapters.Cowboy.Handler, {endpoint, server, path, Enum.into(opts, %{})}} end defp cowboy_start_args(endpoint, servers, port, opts) do diff --git a/lib/grpc/server/adapters/cowboy/router.ex b/lib/grpc/server/adapters/cowboy/router.ex index c847cc51..7180cd7a 100644 --- a/lib/grpc/server/adapters/cowboy/router.ex +++ b/lib/grpc/server/adapters/cowboy/router.ex @@ -1,7 +1,13 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do - use Bitwise + # Most of the functionality in this module is lifted from :cowboy_router, with the unused parts + # removed. Since the template language for Google.Api.HttpRule is quite rich, it cannot be expressed + # in terms of the default routing offered by cowboy. + # This module is configured to be used as middleware in `src/grpc_stream_h.erl` instead of :cowoby_router + @moduledoc false @behaviour :cowboy_middleware + alias GRPC.Server.Router + @dialyzer {:nowarn_function, compile: 1} def compile(routes) do @@ -17,10 +23,10 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do Enum.reverse(acc) end - def compile_paths([{route, handler, opts} | paths], acc) when is_binary(route) do - {_, route} = GRPC.Server.Transcode.build_route(%{pattern: {:post, route}}) + def compile_paths([{path, handler, opts} | paths], acc) when is_binary(path) do + {_, _, matches} = Router.build_route(path) - compile_paths(paths, [{route, [], handler, opts} | acc]) + compile_paths(paths, [{matches, [], handler, opts} | acc]) end def compile_paths([{route, handler, opts} | paths], acc) do @@ -65,36 +71,6 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do match_path(path_matchs, :undefined, path, %{}) end - def match([{host_match, fields, path_matchs} | tail], tokens, path) - when is_list(tokens) do - case list_match(tokens, host_match, %{}) do - false -> - match(tail, tokens, path) - - {true, bindings, host_info} -> - host_info = - case host_info do - :undefined -> - :undefined - - _ -> - Enum.reverse(host_info) - end - - case check_constraints(fields, bindings) do - {:ok, bindings} -> - match_path(path_matchs, host_info, path, bindings) - - :nomatch -> - match(tail, tokens, path) - end - end - end - - def match(dispatch, host, path) do - match(dispatch, split_host(host), path) - end - defp match_path([], _, _, _) do {:error, :notfound, :path} end @@ -113,14 +89,14 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do defp match_path([{path_match, fields, handler, opts} | tail], host_info, tokens, bindings) when is_list(tokens) do - case list_match(tokens, path_match, bindings) do + case Router.match(tokens, path_match, bindings) do false -> match_path(tail, host_info, tokens, bindings) - {true, path_binds, path_info} -> + {true, path_binds} -> case check_constraints(fields, path_binds) do {:ok, path_binds} -> - {:ok, handler, opts, path_binds, host_info, path_info} + {:ok, handler, opts, path_binds, host_info, :undefined} :nomatch -> match_path(tail, host_info, tokens, bindings) @@ -133,7 +109,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do end defp match_path(dispatch, host_info, path, bindings) do - match_path(dispatch, host_info, split_path(path), bindings) + match_path(dispatch, host_info, Router.split_path(path), bindings) end defp check_constraints([], bindings) do @@ -145,11 +121,11 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do end defp check_constraints([field | tail], bindings) do - name = :erlang.element(1, field) + name = elem(field, 0) case bindings do %{^name => value} -> - constraints = :erlang.element(2, field) + constraints = elem(field, 1) case :cowboy_constraints.validate( value, @@ -166,202 +142,4 @@ defmodule GRPC.Server.Adapters.Cowboy.Router do check_constraints(tail, bindings) end end - - defp split_host(host) do - split_host(host, []) - end - - defp split_host(host, acc) do - case :binary.match(host, ".") do - :nomatch when host === <<>> -> - acc - - :nomatch -> - [host | acc] - - {pos, _} -> - <> = host - false = byte_size(segment) == 0 - split_host(rest, [segment | acc]) - end - end - - defp split_path(<>) do - split_path(path, []) - end - - defp split_path(_) do - :badrequest - end - - defp split_path(path, acc) do - try do - case :binary.match(path, "/") do - :nomatch when path === <<>> -> - acc - |> Enum.map(&:cow_uri.urldecode/1) - |> Enum.reverse() - |> remove_dot_segments([]) - - :nomatch -> - [path | acc] - |> Enum.map(&:cow_uri.urldecode/1) - |> Enum.reverse() - |> remove_dot_segments([]) - - {pos, _} -> - <> = path - split_path(rest, [segment | acc]) - end - catch - :error, _ -> - :badrequest - end - end - - defp remove_dot_segments([], acc) do - Enum.reverse(acc) - end - - defp remove_dot_segments(["." | segments], acc) do - remove_dot_segments(segments, acc) - end - - defp remove_dot_segments([".." | segments], acc = []) do - remove_dot_segments(segments, acc) - end - - defp remove_dot_segments([".." | segments], [_ | acc]) do - remove_dot_segments(segments, acc) - end - - defp remove_dot_segments([s | segments], acc) do - remove_dot_segments(segments, [s | acc]) - end - - def list_match(list, [{:__, []}], binds) do - {true, binds, list} - end - - def list_match([_s | tail], [{:_, _} | tail_match], binds) do - list_match(tail, tail_match, binds) - end - - def list_match([s | tail], [s | tail_match], binds) do - list_match(tail, tail_match, binds) - end - - def list_match([segment | tail], [{binding, [{:_, _}]} | matchers], bindings) do - put_binding(bindings, binding, segment, tail, matchers) - end - - def list_match([segment | tail], [{binding, [segment]} | matchers], bindings) - when is_atom(binding) do - put_binding(bindings, binding, segment, tail, matchers) - end - - def list_match(rest, [{binding, [{any, _}]}], bindings) - when is_atom(binding) and any in [:_, :__] do - value = Enum.join(rest, "/") - - list_match([], [], Map.put(bindings, binding, value)) - end - - def list_match([segment | _] = rest, [{binding, [segment, {any, _}]}], bindings) - when is_atom(binding) and any in [:_, :__] do - value = Enum.join(rest, "/") - - list_match([], [], Map.put(bindings, binding, value)) - end - - def list_match( - [segment | tail], - [{binding, [segment | sub_matches]} | matches], - bindings - ) - when is_atom(binding) do - end_condition = - case matches do - [next | _] -> next - [] -> :undefined - end - - with {matched_segments, tail} <- match_until(tail, end_condition, sub_matches, []) do - value = Enum.join([segment | matched_segments], "/") - bindings = Map.put(bindings, binding, value) - - list_match(tail, matches, bindings) - end - end - - def list_match([segment | tail], [{binding, []} | matchers], bindings) when is_atom(binding) do - put_binding(bindings, binding, segment, tail, matchers) - end - - def list_match([], [], binds) do - {true, binds, :undefined} - end - - def list_match(_list, _match, _binds) do - false - end - - # End recursion, since there's no "outside" matches we should iterate to end of segments - def match_until([], :undefined, [], acc) do - {Enum.reverse(acc), []} - end - - # End recursion, end condition is a binding with a matching complex start segment - def match_until( - [segment | _] = segments, - _end_condition = {binding, [segment | _]}, - [], - acc - ) - when is_atom(binding) do - {Enum.reverse(acc), segments} - end - - # End recursion since the submatch contains a trailing wildcard but we have more matches "outside" this sub-segment - def match_until([segment | _] = segments, _end_condition = segment, [], acc) do - {Enum.reverse(acc), segments} - end - - # Reached the "end" of this wildcard, so we proceed with the next match - def match_until([_segment | _] = segments, end_condition, [{:__, []}, match | matches], acc) do - match_until(segments, end_condition, [match | matches], acc) - end - - # Segment is matching the wildcard and have not reached "end" of wildcard - def match_until([segment | segments], end_condition, [{:__, []} | _] = matches, acc) do - match_until(segments, end_condition, matches, [segment | acc]) - end - - # Current match is matching segment, add to accumulator and set next match as the current one - def match_until([segment | segments], end_condition, [segment | matches], acc) do - match_until(segments, end_condition, matches, [segment | acc]) - end - - # 'Any' match is matching first segment, add to accumulator and set next match as the current one - def match_until([segment | segments], end_condition, [{:_, []} | matches], acc) do - match_until(segments, end_condition, matches, [segment | acc]) - end - - # No match - def match_until(_segments, _end_condition, _matches, _acc) do - false - end - - defp put_binding(bindings, binding, value, tail, matchers) do - case bindings do - %{^binding => ^value} -> - list_match(tail, matchers, bindings) - - %{^binding => _} -> - false - - _ -> - list_match(tail, matchers, Map.put(bindings, binding, value)) - end - end end diff --git a/lib/grpc/server/router.ex b/lib/grpc/server/router.ex new file mode 100644 index 00000000..38b00311 --- /dev/null +++ b/lib/grpc/server/router.ex @@ -0,0 +1,214 @@ +defmodule GRPC.Server.Router do + @moduledoc """ + """ + alias __MODULE__.Template + + @type http_method :: :get | :put | :post | :patch | :delete + @type route :: {http_method(), String.t(), Template.matchers()} + + @wildcards [:_, :__] + + @spec build_route(binary() | map()) :: route() + def build_route(path) when is_binary(path), do: build_route(:post, path) + def build_route(%{pattern: {method, path}}), do: build_route(method, path) + + @doc """ + Builds a t:route/0 from a URL path or `t:Google.Api.Http.t/0`. + + The matcher part in the route can be used in `match/3` to match on a URL path or a list of segments. + + ## Examples + + {:get, "/v1/messages/{message_id}", match} = GRPC.Server.Router.build_route(:get, "/v1/messages/{message_id}") + + {:get, path, match} = GRPC.Server.Router.build_route(:get, "/v1/{book.location=shelves/*}/books/{book.name=*}") + {true, %{"book.location": "shelves/example-shelf", "book.name": "example-book"}} = GRPC.Server.Router.match("/v1/shelves/example-shelf/books/example-book", match, []) + """ + @spec build_route(atom(), binary()) :: route() + def build_route(method, path) when is_binary(path) do + match = + path + |> Template.tokenize([]) + |> Template.parse([]) + + {method, path, match} + end + + @doc """ + Split URL path into segments, removing the leading and trailing slash. + + ## Examples + + ["v1", "messages"] = GRPC.Server.Router.split_path("/v1/messages") + """ + @spec split_path(String.t()) :: iolist() + def split_path(bin) do + for segment <- String.split(bin, "/"), segment != "", do: segment + end + + @doc """ + Matches a URL path or URL segements against a compiled route matcher. Matched bindings from the segments are extracted + into a map. If the same variable name is used in multiple bindings, the value must match otherwise the route is not considered a match. + + ## Examples + + {_, _, match} = GRPC.Server.Router.build_route(:get, "/v1/{name=messages}") + {true, %{name: "messages"}} = GRPC.Server.Router.match("/v1/messages", match) + false = GRPC.Server.Router.match("/v1/messages/foobar", match) + + + {_, _, match} = GRPC.Server.Router.build_route(:get, "/v1/{name=shelves/*/books/*) + {true, %{name: "shelves/books/book"}} = GRPC.Server.Router.match("/v1/shelves/example-shelf/books/book", match) + + false = GRPC.Server.Router.match("/v1/shelves/example-shelf/something-els/books/book", match) + + """ + @spec match(String.t() | [String.t()], Template.matchers()) :: {true, map()} | false + def match(path, match) do + match(path, match, %{}) + end + + @spec match(String.t() | [String.t()], Template.matchers(), map()) :: {true, map()} | false + def match(path, match, bindings) when is_binary(path) do + path + |> split_path() + |> match(match, bindings) + end + + # The last matcher is a 'catch all' matcher, so the rest of segments are matching. + def match(_segments, [{:__, []}], bindings) do + {true, bindings} + end + + # 'Any' matcher matches a single segment, cont. recursion. + def match([_s | segments], [{:_, _} | matchers], bindings) do + match(segments, matchers, bindings) + end + + # Matching against a 'literal' match, cont. recursion + def match([segment | segments], [_literal = segment | matchers], bindings) do + match(segments, matchers, bindings) + end + + # /v1/{a=*} is the same as /v1/{a}. Matching and binding the segment to `binding` + def match([segment | tail], [{binding, [{:_, _}]} | matchers], bindings) do + put_binding(bindings, binding, segment, tail, matchers) + end + + # /v1/{a=messages} binding a matching literal + def match([segment | segments], [{binding, [segment]} | matchers], bindings) do + put_binding(bindings, binding, segment, segments, matchers) + end + + # /v1/{a=*} /v1/{a=**} theres no more matchers after the wildcard, bind + # the rest of the segments to `binding` + def match(rest, [{binding, [{any, _}]}], bindings) when any in @wildcards do + value = Enum.join(rest, "/") + + match([], [], Map.put(bindings, binding, value)) + end + + # /v1/{a=messages/*} /v1/{a=messages/**} theres no more matchers after the wildcard, bind + # the rest of the segments including the current segment to `binding` + def match([segment | _] = segments, [{binding, [segment, {any, _}]}], bindings) + when any in @wildcards do + value = Enum.join(segments, "/") + + match([], [], Map.put(bindings, binding, value)) + end + + # /v1/{a=users/*/messages/*}/suffix. There are sub-matches inside the capture + # so the segments are matched with match submatches until an end-condition + # is reached + def match( + [segment | tail], + [{binding, [segment | sub_matches]} | matches], + bindings + ) do + end_condition = + case matches do + [next | _] -> next + [] -> :undefined + end + + with {matched_segments, tail} <- match_until(tail, end_condition, sub_matches, []) do + value = Enum.join([segment | matched_segments], "/") + bindings = Map.put(bindings, binding, value) + + match(tail, matches, bindings) + end + end + + # /v1/messages/{message_id} simple binding + def match([segment | segments], [{binding, []} | matchers], bindings) when is_atom(binding) do + put_binding(bindings, binding, segment, segments, matchers) + end + + def match([], [], bindings) do + {true, bindings} + end + + # no match + def match(_segments, _matches, _bindings) do + false + end + + # End recursion, since there's no "outside" matches we should iterate to end of segments + defp match_until([], :undefined, [], acc) do + {Enum.reverse(acc), []} + end + + # End recursion, end condition is a binding with a matching complex start segment + defp match_until( + [segment | _] = segments, + _end_condition = {binding, [segment | _]}, + [], + acc + ) + when is_atom(binding) do + {Enum.reverse(acc), segments} + end + + # End recursion since the submatch contains a trailing wildcard but we have more matches "outside" this sub-segment + defp match_until([segment | _] = segments, _end_condition = segment, [], acc) do + {Enum.reverse(acc), segments} + end + + # Reached the "end" of this wildcard, so we proceed with the next match + defp match_until([_segment | _] = segments, end_condition, [{:__, []}, match | matches], acc) do + match_until(segments, end_condition, [match | matches], acc) + end + + # Segment is matching the wildcard and have not reached "end" of wildcard + defp match_until([segment | segments], end_condition, [{:__, []} | _] = matches, acc) do + match_until(segments, end_condition, matches, [segment | acc]) + end + + # Current match is matching segment, add to accumulator and set next match as the current one + defp match_until([segment | segments], end_condition, [segment | matches], acc) do + match_until(segments, end_condition, matches, [segment | acc]) + end + + # 'Any' match is matching first segment, add to accumulator and set next match as the current one + defp match_until([segment | segments], end_condition, [{:_, []} | matches], acc) do + match_until(segments, end_condition, matches, [segment | acc]) + end + + # No match + defp match_until(_segments, _end_condition, _matches, _acc) do + false + end + + defp put_binding(bindings, binding, value, segments, matchers) do + case bindings do + %{^binding => ^value} -> + match(segments, matchers, bindings) + + %{^binding => _} -> + false + + _ -> + match(segments, matchers, Map.put(bindings, binding, value)) + end + end +end diff --git a/lib/grpc/server/transcode/field_path.ex b/lib/grpc/server/router/field_path.ex similarity index 91% rename from lib/grpc/server/transcode/field_path.ex rename to lib/grpc/server/router/field_path.ex index 2f584900..d82038ef 100644 --- a/lib/grpc/server/transcode/field_path.ex +++ b/lib/grpc/server/router/field_path.ex @@ -1,4 +1,4 @@ -defmodule GRPC.Server.Transcode.FieldPath do +defmodule GRPC.Server.Router.FieldPath do @moduledoc false @spec decode_pair({binary(), term()}, map()) :: map() diff --git a/lib/grpc/server/transcode/query.ex b/lib/grpc/server/router/query.ex similarity index 94% rename from lib/grpc/server/transcode/query.ex rename to lib/grpc/server/router/query.ex index c6592e20..409ba269 100644 --- a/lib/grpc/server/transcode/query.ex +++ b/lib/grpc/server/router/query.ex @@ -1,4 +1,4 @@ -defmodule GRPC.Server.Transcode.Query do +defmodule GRPC.Server.Router.Query do @moduledoc false # This module is based on https://github.com/elixir-plug/plug/blob/main/lib/plug/conn/query.ex # Decoding of URL-encoded queries as per the rules outlined in the documentation for [`google.api.HttpRule`](https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule) @@ -8,7 +8,7 @@ defmodule GRPC.Server.Transcode.Query do # 1. Sub-paths on the form `path.subpath` are decoded as nested maps # 1. Sub-paths with the same leaf key are decoded as a list - alias GRPC.Server.Transcode.FieldPath + alias GRPC.Server.Router.FieldPath @spec decode(String.t(), map()) :: %{optional(String.t()) => term()} def decode(query, acc \\ %{}) diff --git a/lib/grpc/server/transcode/template.ex b/lib/grpc/server/router/template.ex similarity index 53% rename from lib/grpc/server/transcode/template.ex rename to lib/grpc/server/router/template.ex index c5211446..b5fc6168 100644 --- a/lib/grpc/server/transcode/template.ex +++ b/lib/grpc/server/router/template.ex @@ -1,4 +1,4 @@ -defmodule GRPC.Server.Transcode.Template do +defmodule GRPC.Server.Router.Template do @moduledoc false # https://cloud.google.com/endpoints/docs/grpc-service-config/reference/rpc/google.api#google.api.HttpRule # Template = "/" Segments [ Verb ] ; @@ -7,10 +7,10 @@ defmodule GRPC.Server.Transcode.Template do # Variable = "{" FieldPath [ "=" Segments ] "}" ; # FieldPath = IDENT { "." IDENT } ; # Verb = ":" LITERAL ; - @type segments :: list(atom | String.t()) - @type route :: {atom(), segments()} + @type segment_match :: String.t() | {atom(), [segment_match]} + @type matchers :: [segment_match] - @spec tokenize(binary(), list()) :: list() + @spec tokenize(binary(), [tuple()]) :: [tuple()] def tokenize(path, tokens \\ []) def tokenize(<<>>, tokens) do @@ -24,7 +24,6 @@ defmodule GRPC.Server.Transcode.Template do @terminals [?/, ?{, ?}, ?=, ?*] defp do_tokenize(<>, <<>>) when h in @terminals do - # parse(t, acc) {{List.to_atom([h]), []}, t} end @@ -41,57 +40,57 @@ defmodule GRPC.Server.Transcode.Template do {{:identifier, acc, []}, <<>>} end - @spec parse(list(tuple()), list()) :: route() | {list(), list()} - def parse([], segments) do - Enum.reverse(segments) + @spec parse(tokens :: [tuple()], matchers()) :: matchers() | {matchers, tokens :: [tuple()]} + def parse([], matchers) do + Enum.reverse(matchers) end - def parse([{:/, _} | rest], segments) do - parse(rest, segments) + def parse([{:/, _} | rest], matchers) do + parse(rest, matchers) end - def parse([{:*, _}, {:*, _} | rest], segments) do - parse(rest, [{:__, []} | segments]) + def parse([{:*, _}, {:*, _} | rest], matchers) do + parse(rest, [{:__, []} | matchers]) end - def parse([{:*, _} | rest], segments) do - parse(rest, [{:_, []} | segments]) + def parse([{:*, _} | rest], matchers) do + parse(rest, [{:_, []} | matchers]) end - def parse([{:identifier, identifier, _} | rest], segments) do - parse(rest, [identifier | segments]) + def parse([{:identifier, identifier, _} | rest], matchers) do + parse(rest, [identifier | matchers]) end - def parse([{:"{", _} | rest], segments) do - {segments, rest} = parse_binding(rest, segments) - parse(rest, segments) + def parse([{:"{", _} | rest], matchers) do + {matchers, rest} = parse_binding(rest, matchers) + parse(rest, matchers) end - def parse([{:"}", _} | _rest] = acc, segments) do - {segments, acc} + def parse([{:"}", _} | _rest] = acc, matchers) do + {matchers, acc} end - defp parse_binding([], segments) do - {segments, []} + defp parse_binding([], matchers) do + {matchers, []} end - defp parse_binding([{:"}", []} | rest], segments) do - {segments, rest} + defp parse_binding([{:"}", []} | rest], matchers) do + {matchers, rest} end defp parse_binding( [{:identifier, id, _}, {:=, _} | rest], - segments + matchers ) do variable = field_path(id) {assign, rest} = parse(rest, []) - parse_binding(rest, [{variable, Enum.reverse(assign)} | segments]) + parse_binding(rest, [{variable, Enum.reverse(assign)} | matchers]) end - defp parse_binding([{:identifier, id, []} | rest], segments) do + defp parse_binding([{:identifier, id, []} | rest], matchers) do variable = field_path(id) - parse_binding(rest, [{variable, []} | segments]) + parse_binding(rest, [{variable, []} | matchers]) end defp field_path(identifier) do diff --git a/lib/grpc/server/stream.ex b/lib/grpc/server/stream.ex index 476e3d06..1c0df8c9 100644 --- a/lib/grpc/server/stream.ex +++ b/lib/grpc/server/stream.ex @@ -35,7 +35,7 @@ defmodule GRPC.Server.Stream do # `GRPC.Server.set_compressor` compressor: module() | nil, # For http transcoding - http_method: :get | :post | :put | :patch | :delete, + http_method: GRPC.Server.Router.http_method(), http_transcode: boolean(), __interface__: map() } diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 9672c083..4ef236ae 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,6 +1,6 @@ defmodule GRPC.Server.Transcode do @moduledoc false - alias __MODULE__.{Query, Template, FieldPath} + alias GRPC.Server.Router.{Query, FieldPath} @type t :: map() # The request mapping follow the following rules: @@ -64,48 +64,11 @@ defmodule GRPC.Server.Transcode do @spec map_response_body(t() | map(), map()) :: map() def map_response_body(%{response_body: ""}, response_body), do: response_body - # TODO The field is required to be present on the toplevel response message def map_response_body(%{response_body: field}, response_body) do key = String.to_existing_atom(field) Map.get(response_body, key) end - def map_response_body(%{}, response_body), do: response_body - - @spec to_path({atom(), Template.route()}) :: String.t() - def to_path({_method, segments} = _spec) do - match = - segments - |> Enum.map(&segment_to_string/1) - |> Enum.join("/") - - "/" <> match - end - - defp segment_to_string({:_, []}), do: "[...]" - defp segment_to_string({binding, []}) when is_atom(binding), do: ":#{Atom.to_string(binding)}" - - defp segment_to_string({binding, [_ | rest]}) when is_atom(binding) do - sub_path = - rest - |> Enum.map(&segment_to_string/1) - |> Enum.join("/") - - ":#{Atom.to_string(binding)}" <> "/" <> sub_path - end - - defp segment_to_string(segment), do: segment - - @spec build_route(t()) :: {atom(), Template.route()} - def build_route(%{pattern: {method, path}}) do - route = - path - |> Template.tokenize([]) - |> Template.parse([]) - - {method, route} - end - @spec map_path_bindings(map()) :: map() def map_path_bindings(bindings) when bindings == %{}, do: bindings diff --git a/test/grpc/server/adapter/cowboy/router_test.exs b/test/grpc/server/adapter/cowboy/router_test.exs index cebec6f7..9b3343a5 100644 --- a/test/grpc/server/adapter/cowboy/router_test.exs +++ b/test/grpc/server/adapter/cowboy/router_test.exs @@ -69,7 +69,7 @@ defmodule GRPC.Server.Adapters.Cowboy.RouterTest do test "with 'any' and 'catch all'" do dispatch = make_dispatch("/*/**") - assert {:ok, Handler, [], %{}, :undefined, ["foo", "bar", "baz"]} == + assert {:ok, Handler, [], %{}, :undefined, :undefined} == Router.match(dispatch, "localhost", "/v1/foo/bar/baz") end @@ -156,13 +156,12 @@ defmodule GRPC.Server.Adapters.Cowboy.RouterTest do end defp make_dispatch(path) do - rule = %{pattern: {:get, path}} - {_method, route} = GRPC.Server.Transcode.build_route(rule) + {_method, _, match} = GRPC.Server.Router.build_route(path) [ {:_, [], [ - {route, [], Handler, []} + {match, [], Handler, []} ]} ] end diff --git a/test/grpc/server/transcode/query_test.exs b/test/grpc/server/router/query_test.exs similarity index 92% rename from test/grpc/server/transcode/query_test.exs rename to test/grpc/server/router/query_test.exs index ac208a74..ffa4d0ca 100644 --- a/test/grpc/server/transcode/query_test.exs +++ b/test/grpc/server/router/query_test.exs @@ -1,6 +1,6 @@ -defmodule GRPC.Transcode.QueryTest do +defmodule GRPC.Server.Router.QueryTest do use ExUnit.Case, async: true - alias GRPC.Server.Transcode.Query + alias GRPC.Server.Router.Query test "`a=b&c=d` should be decoded as a map" do assert %{"a" => "b", "c" => "d"} == Query.decode("a=b&c=d") diff --git a/test/grpc/server/transcode/template_test.exs b/test/grpc/server/router/template_test.exs similarity index 98% rename from test/grpc/server/transcode/template_test.exs rename to test/grpc/server/router/template_test.exs index c124cfb9..f19b9cd2 100644 --- a/test/grpc/server/transcode/template_test.exs +++ b/test/grpc/server/router/template_test.exs @@ -1,6 +1,6 @@ -defmodule GRPC.Transcode.TemplateTest do +defmodule GRPC.Server.Router.TemplateTest do use ExUnit.Case, async: true - alias GRPC.Server.Transcode.Template + alias GRPC.Server.Router.Template describe "tokenize/2" do test "can tokenize simple paths" do diff --git a/test/grpc/server/router_test.exs b/test/grpc/server/router_test.exs new file mode 100644 index 00000000..f0d1e3f2 --- /dev/null +++ b/test/grpc/server/router_test.exs @@ -0,0 +1,161 @@ +defmodule GRPC.Server.RouterTest do + use ExUnit.Case, async: true + alias GRPC.Server.Router + + describe "build_route/1" do + test "returns a route with {http_method, path, match} based on the template string" do + path = "/v1/messages/{message_id}" + + assert {:get, ^path, match} = Router.build_route(:get, "/v1/messages/{message_id}") + assert ["v1", "messages", {:message_id, []}] = match + end + + test "defaults to setting method to `:post` if no method was provided" do + path = "/transcode.Messaging/GetMessage" + + assert {:post, ^path, match} = Router.build_route(path) + assert ["transcode.Messaging", "GetMessage"] = match + end + + test "returns a route with {http_method, path, match} based HttRule" do + path = "/v1/messages/{message_id}" + rule = build_simple_rule(:get, "/v1/messages/{message_id}") + + assert {:get, ^path, match} = Router.build_route(rule) + assert ["v1", "messages", {:message_id, []}] = match + end + end + + describe "match/3" do + test "with no segments" do + assert {true, %{}} = Router.match("/", []) + end + + test "with segments and no matchers" do + refute Router.match("/foo", []) + end + + test "with no bindings" do + {_, _, match} = Router.build_route("/transcode.Messaging/GetMessage") + + assert {true, %{}} == Router.match("/transcode.Messaging/GetMessage", match) + assert false == Router.match("/transcode.Messaging/GetMessages", match) + end + + test "with simple bindings" do + {_, _, match} = Router.build_route(:get, "/v1/{name}") + + assert {true, %{name: "messages"}} == Router.match("/v1/messages", match) + end + + test "with nested bindings" do + {_, _, match} = Router.build_route(:get, "/v1/{message.name}") + + assert {true, %{"message.name": "messages"}} == Router.match("/v1/messages", match) + end + + test "with multiple bindings" do + {_, _, match} = + Router.build_route(:get, "/v1/users/{user_id}/messages/{message.message_id}") + + assert {true, %{user_id: "1", "message.message_id": "2"}} == + Router.match("/v1/users/1/messages/2", match) + end + + test "with multiple sequential bindings" do + {_, _, match} = Router.build_route("/v1/{a}/{b}/{c}") + + assert {true, %{a: "a", b: "b", c: "c"}} == Router.match("/v1/a/b/c", match) + end + + test "with 'any'" do + {_, _, match} = Router.build_route("/*") + + assert {true, %{}} == Router.match("/v1", match) + end + + test "with 'any' assignment" do + {_, _, match} = Router.build_route("/{a=*}") + + assert {true, %{a: "v1"}} == Router.match("/v1", match) + end + + test "with 'catch all' assignment" do + {_, _, match} = Router.build_route("/{a=**}") + + assert {true, %{a: "v1/messages"}} == Router.match("/v1/messages", match) + end + + test "with 'any' and 'catch all'" do + {_, _, match} = Router.build_route("/*/**") + assert {true, %{}} == Router.match("/v1/foo/bar/baz", match) + end + + test "with 'any' and 'catch all' assignment" do + {_, _, match} = Router.build_route("/*/a/{b=c/*}/d/{e=**}") + + assert {true, %{b: "c/foo", e: "bar/baz/biz"}} == + Router.match("/v1/a/c/foo/d/bar/baz/biz", match) + end + + test "with complex binding" do + {_, _, match} = Router.build_route("/v1/{name=messages}") + + assert {true, %{name: "messages"}} == Router.match("/v1/messages", match) + refute Router.match("/v1/should_not_match", match) + end + + test "with complex binding and 'any'" do + {_, _, match} = Router.build_route("/v1/{name=messages/*}") + + assert {true, %{name: "messages/12345"}} == Router.match("/v1/messages/12345", match) + refute Router.match("/v1/should_not_match/12345", match) + end + + test "with complex binding, wildcards and trailing route" do + {_, _, match} = Router.build_route("/v1/{name=shelves/*/books/*}") + + assert {true, %{name: "shelves/example-shelf/books/example-book"}} == + Router.match("/v1/shelves/example-shelf/books/example-book", match) + + refute Router.match("/v1/shelves/example-shelf/not_books", match) + end + + test "with complex binding, wildcards and suffix" do + {_, _, match} = Router.build_route("/v1/{name=shelves/*/books/*}/suffix") + + assert {true, %{name: "shelves/example-shelf/books/example-book"}} == + Router.match( + "/v1/shelves/example-shelf/books/example-book/suffix", + match + ) + + refute Router.match( + "/v1/shelves/example-shelf/books/example-book/another_suffix", + match + ) + end + + test "with mixed complex binding" do + {_, _, match} = Router.build_route("/v1/{a=users/*}/messages/{message_id}/{c=books/*}") + + assert {true, %{a: "users/foobar", message_id: "1", c: "books/barbaz"}} == + Router.match("/v1/users/foobar/messages/1/books/barbaz", match) + + assert false == Router.match("/v1/users/1/books/barbaz", match) + end + + test "with mixed sequential complex binding" do + {_, _, match} = Router.build_route("/v1/{a=users/*}/{b=messages}/{c=books/*}") + + assert {true, %{a: "users/foobar", b: "messages", c: "books/barbaz"}} == + Router.match("/v1/users/foobar/messages/books/barbaz", match) + + refute Router.match("/v1/users/foobar/messages/book/books/barbaz", match) + end + end + + defp build_simple_rule(method, pattern) do + Google.Api.HttpRule.new(pattern: {method, pattern}) + end +end diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index 472f92cc..751041b9 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -48,24 +48,6 @@ defmodule GRPC.TranscodeTest do assert %{a: "b"} == Transcode.map_response_body(rule, request_body) end - test "build_route/1 returns a route with {http_method, route} based on the http rule" do - rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert {:get, segments} = Transcode.build_route(rule) - assert ["v1", "messages", {:message_id, []}] = segments - end - - test "to_path/1 returns path segments as a string match" do - rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert spec = Transcode.build_route(rule) - assert "/v1/messages/:message_id" = Transcode.to_path(spec) - end - - test "to_path/1 returns path segments as a string when there's multiple bindings" do - rule = build_simple_rule(:get, "/v1/users/{user_id}/messages/{message_id}") - assert spec = Transcode.build_route(rule) - assert "/v1/users/:user_id/messages/:message_id" = Transcode.to_path(spec) - end - test "map_route_bindings/2 should stringify the keys" do path_binding_atom = %{foo: "bar"} path_binding_string = %{foo: "bar"} @@ -78,8 +60,4 @@ defmodule GRPC.TranscodeTest do path_binding = %{"foo.bar.baz" => "biz"} assert %{"foo" => %{"bar" => %{"baz" => "biz"}}} == Transcode.map_path_bindings(path_binding) end - - defp build_simple_rule(method, pattern) do - Google.Api.HttpRule.new(pattern: {method, pattern}) - end end From 271768c23ba4b301aa423270a9b0d3069e67ab02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Wed, 26 Oct 2022 17:19:16 +0200 Subject: [PATCH 58/73] don't generate protos before tests --- mix.exs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mix.exs b/mix.exs index 645970fc..e3fd57fb 100644 --- a/mix.exs +++ b/mix.exs @@ -70,10 +70,6 @@ defmodule GRPC.Mixfile do defp aliases do [ - test: [ - &gen_test_protos/1, - "test" - ], gen_bootstrap_protos: &gen_bootstrap_protos/1 ] end From 3957bf92cdfefae6634e409dbabf5e885a01baf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 27 Oct 2022 13:08:25 +0200 Subject: [PATCH 59/73] reset helloworld example --- examples/helloworld/README.md | 18 +---- examples/helloworld/lib/helloworld.pb.ex | 74 ++----------------- examples/helloworld/lib/server.ex | 25 ++----- examples/helloworld/mix.exs | 4 +- examples/helloworld/mix.lock | 4 +- .../helloworld/priv/protos/helloworld.proto | 37 +--------- 6 files changed, 17 insertions(+), 145 deletions(-) diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index 5d672b8c..924681fd 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -17,29 +17,17 @@ $ mix run --no-halt $ mix run priv/client.exs ``` -## HTTP Transcoding - -``` shell -# Say hello -curl -H 'Content-type: application/json' http://localhost:50051/v1/greeter/test - -# Say hello from -curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' http://localhost:50051/v1/greeter -``` - ## Regenerate Elixir code from proto 1. Modify the proto `priv/protos/helloworld.proto` 2. Install `protoc` [here](https://developers.google.com/protocol-buffers/docs/downloads) - +3. Install `protoc-gen-elixir` ``` -mix deps.get +mix escript.install hex protobuf ``` - 4. Generate the code: - ```shell -$ mix protobuf.generate --include-path=priv/protos --plugins=ProtobufGenerate.Plugins.GRPCWithOptions --output-path=./lib priv/protos/helloworld.proto +$ protoc -I priv/protos --elixir_out=plugins=grpc:./lib/ priv/protos/helloworld.proto ``` Refer to [protobuf-elixir](https://github.com/tony612/protobuf-elixir#usage) for more information. diff --git a/examples/helloworld/lib/helloworld.pb.ex b/examples/helloworld/lib/helloworld.pb.ex index 0fa96786..a8ff6dfa 100644 --- a/examples/helloworld/lib/helloworld.pb.ex +++ b/examples/helloworld/lib/helloworld.pb.ex @@ -1,88 +1,26 @@ defmodule Helloworld.HelloRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 field :name, 1, type: :string end -defmodule Helloworld.HelloRequestFrom do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :name, 1, type: :string - field :from, 2, type: :string -end - defmodule Helloworld.HelloReply do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 field :message, 1, type: :string field :today, 2, type: Google.Protobuf.Timestamp end -defmodule Helloworld.GetMessageRequest do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :name, 1, type: :string -end - -defmodule Helloworld.Message do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 - - field :text, 1, type: :string -end - defmodule Helloworld.Greeter.Service do @moduledoc false - use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.11.0" + use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.10.0" - rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/greeter/{name}"}, - response_body: "", - selector: "" - } - } - }) - - rpc(:SayHelloFrom, Helloworld.HelloRequestFrom, Helloworld.HelloReply, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "*", - pattern: {:post, "/v1/greeter"}, - response_body: "", - selector: "" - } - } - }) + rpc :SayHello, Helloworld.HelloRequest, Helloworld.HelloReply end -defmodule Helloworld.Messaging.Service do +defmodule Helloworld.Greeter.Stub do @moduledoc false - use GRPC.Service, name: "helloworld.Messaging", protoc_gen_elixir_version: "0.11.0" - - rpc(:GetMessage, Helloworld.GetMessageRequest, Helloworld.Message, %{ - http: %{ - type: Google.Api.PbExtension, - value: %Google.Api.HttpRule{ - __unknown_fields__: [], - additional_bindings: [], - body: "", - pattern: {:get, "/v1/{name=messages/*}"}, - response_body: "", - selector: "" - } - } - }) + use GRPC.Stub, service: Helloworld.Greeter.Service end diff --git a/examples/helloworld/lib/server.ex b/examples/helloworld/lib/server.ex index 68c72c10..b85241f8 100644 --- a/examples/helloworld/lib/server.ex +++ b/examples/helloworld/lib/server.ex @@ -1,31 +1,16 @@ defmodule Helloworld.Greeter.Server do - use GRPC.Server, - service: Helloworld.Greeter.Service, - http_transcode: true + use GRPC.Server, service: Helloworld.Greeter.Service @spec say_hello(Helloworld.HelloRequest.t(), GRPC.Server.Stream.t()) :: Helloworld.HelloReply.t() def say_hello(request, _stream) do - Helloworld.HelloReply.new( - message: "Hello #{request.name}", - today: today() - ) - end - - @spec say_hello_from(Helloworld.HelloFromRequest.t(), GRPC.Server.Stream.t()) :: - Helloworld.HelloReply.t() - def say_hello_from(request, _stream) do - Helloworld.HelloReply.new( - message: "Hello #{request.name}. From #{request.from}", - today: today() - ) - end - - defp today do nanos_epoch = System.system_time() |> System.convert_time_unit(:native, :nanosecond) seconds = div(nanos_epoch, 1_000_000_000) nanos = nanos_epoch - seconds * 1_000_000_000 - %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} + Helloworld.HelloReply.new( + message: "Hello #{request.name}", + today: %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} + ) end end diff --git a/examples/helloworld/mix.exs b/examples/helloworld/mix.exs index f1b13088..9bf4cc3e 100644 --- a/examples/helloworld/mix.exs +++ b/examples/helloworld/mix.exs @@ -19,9 +19,7 @@ defmodule Helloworld.Mixfile do defp deps do [ {:grpc, path: "../../"}, - {:protobuf, "~> 0.11.0"}, - {:protobuf_generate, "~> 0.1.1", only: [:dev, :test]}, - {:jason, "~> 1.3.0"}, + {:protobuf, "~> 0.10"}, {:google_protos, "~> 0.3.0"}, {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false} ] diff --git a/examples/helloworld/mix.lock b/examples/helloworld/mix.lock index afdfde5e..f96a70d1 100644 --- a/examples/helloworld/mix.lock +++ b/examples/helloworld/mix.lock @@ -5,8 +5,6 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "google_protos": {:hex, :google_protos, "0.3.0", "15faf44dce678ac028c289668ff56548806e313e4959a3aaf4f6e1ebe8db83f4", [:mix], [{:protobuf, "~> 0.10", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "1f6b7fb20371f72f418b98e5e48dae3e022a9a6de1858d4b254ac5a5d0b4035f"}, "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, - "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, - "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, - "protobuf_generate": {:hex, :protobuf_generate, "0.1.1", "f6098b85161dcfd48a4f6f1abee4ee5e057981dfc50aafb1aa4bd5b0529aa89b", [:mix], [{:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "93a38c8e2aba2a17e293e9ef1359122741f717103984aa6d1ebdca0efb17ab9d"}, + "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/examples/helloworld/priv/protos/helloworld.proto b/examples/helloworld/priv/protos/helloworld.proto index 632519a0..12849981 100644 --- a/examples/helloworld/priv/protos/helloworld.proto +++ b/examples/helloworld/priv/protos/helloworld.proto @@ -5,7 +5,6 @@ option java_package = "io.grpc.examples.helloworld"; option java_outer_classname = "HelloWorldProto"; option objc_class_prefix = "HLW"; -import "google/api/annotations.proto"; import "google/protobuf/timestamp.proto"; package helloworld; @@ -13,18 +12,7 @@ package helloworld; // The greeting service definition. service Greeter { // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) { - option (google.api.http) = { - get: "/v1/greeter/{name}" - }; - } - - rpc SayHelloFrom (HelloRequestFrom) returns (HelloReply) { - option (google.api.http) = { - post: "/v1/greeter" - body: "*" - }; - } + rpc SayHello (HelloRequest) returns (HelloReply) {} } // The request message containing the user's name. @@ -32,31 +20,8 @@ message HelloRequest { string name = 1; } -// HelloRequestFrom! -message HelloRequestFrom { - // Name! - string name = 1; - // From! - string from = 2; -} - // The response message containing the greetings message HelloReply { string message = 1; google.protobuf.Timestamp today = 2; } - -service Messaging { - rpc GetMessage(GetMessageRequest) returns (Message) { - option (google.api.http) = { - get: "/v1/{name=messages/*}" - }; - } -} - -message GetMessageRequest { - string name = 1; // Mapped to URL path. -} -message Message { - string text = 1; // The resource content. -} From 9acc0ab6ba8b9b495a304ab3bf5102ab14278cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 27 Oct 2022 13:08:42 +0200 Subject: [PATCH 60/73] add helloworld_transcode example --- examples/helloworld_transcoding/.gitignore | 23 ++ examples/helloworld_transcoding/README.md | 83 ++++ .../helloworld_transcoding/config/config.exs | 3 + .../helloworld_transcoding/config/dev.exs | 1 + .../helloworld_transcoding/config/prod.exs | 4 + .../helloworld_transcoding/config/test.exs | 1 + .../helloworld_transcoding/lib/endpoint.ex | 6 + .../lib/google/api/annotations.pb.ex | 8 + .../lib/google/api/http.pb.ex | 40 ++ .../lib/helloworld.pb.ex | 55 +++ .../lib/helloworld_app.ex | 12 + examples/helloworld_transcoding/lib/server.ex | 31 ++ examples/helloworld_transcoding/mix.exs | 29 ++ examples/helloworld_transcoding/mix.lock | 12 + .../helloworld_transcoding/priv/client.exs | 9 + .../priv/protos/google/api/annotations.proto | 31 ++ .../priv/protos/google/api/http.proto | 375 ++++++++++++++++++ .../priv/protos/helloworld.proto | 47 +++ .../test/hello_world_test.exs | 16 + .../test/test_helper.exs | 1 + 20 files changed, 787 insertions(+) create mode 100644 examples/helloworld_transcoding/.gitignore create mode 100644 examples/helloworld_transcoding/README.md create mode 100644 examples/helloworld_transcoding/config/config.exs create mode 100644 examples/helloworld_transcoding/config/dev.exs create mode 100644 examples/helloworld_transcoding/config/prod.exs create mode 100644 examples/helloworld_transcoding/config/test.exs create mode 100644 examples/helloworld_transcoding/lib/endpoint.ex create mode 100644 examples/helloworld_transcoding/lib/google/api/annotations.pb.ex create mode 100644 examples/helloworld_transcoding/lib/google/api/http.pb.ex create mode 100644 examples/helloworld_transcoding/lib/helloworld.pb.ex create mode 100644 examples/helloworld_transcoding/lib/helloworld_app.ex create mode 100644 examples/helloworld_transcoding/lib/server.ex create mode 100644 examples/helloworld_transcoding/mix.exs create mode 100644 examples/helloworld_transcoding/mix.lock create mode 100644 examples/helloworld_transcoding/priv/client.exs create mode 100644 examples/helloworld_transcoding/priv/protos/google/api/annotations.proto create mode 100644 examples/helloworld_transcoding/priv/protos/google/api/http.proto create mode 100644 examples/helloworld_transcoding/priv/protos/helloworld.proto create mode 100644 examples/helloworld_transcoding/test/hello_world_test.exs create mode 100644 examples/helloworld_transcoding/test/test_helper.exs diff --git a/examples/helloworld_transcoding/.gitignore b/examples/helloworld_transcoding/.gitignore new file mode 100644 index 00000000..06dbcb6f --- /dev/null +++ b/examples/helloworld_transcoding/.gitignore @@ -0,0 +1,23 @@ +# The directory Mix will write compiled artifacts to. +/_build + +# If you run "mix test --cover", coverage assets end up here. +/cover + +# The directory Mix downloads your dependencies sources to. +/deps + +# Where 3rd-party dependencies like ExDoc output generated docs. +/doc + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +/priv/grpc_c.so* +/src/grpc_c +/tmp + +/log \ No newline at end of file diff --git a/examples/helloworld_transcoding/README.md b/examples/helloworld_transcoding/README.md new file mode 100644 index 00000000..66624ce0 --- /dev/null +++ b/examples/helloworld_transcoding/README.md @@ -0,0 +1,83 @@ +# Helloworld with HTTP/json transcoding in grpc-elixir + +## Usage + +1. Install deps and compile +```shell +$ mix do deps.get, compile +``` + +2. Run the server +```shell +$ mix run --no-halt +``` + +3. Run the client script +```shell +$ mix run priv/client.exs +``` + +## HTTP Transcoding + +``` shell +# Say hello +$ curl -H 'accept: application/json' http://localhost:50051/v1/greeter/test + +# Say hello from +$ curl -XPOST -H 'Content-type: application/json' -d '{"name": "test", "from": "anon"}' http://localhost:50051/v1/greeter +``` + +## Regenerate Elixir code from proto + +1. Modify the proto `priv/protos/helloworld.proto` + +2. Install `protoc` [here](https://developers.google.com/protocol-buffers/docs/downloads) + +``` +mix deps.get +``` + +4. Generate `google.api.http` extensions: + +``` shell +$ mix protobuf.generate --include-path=priv/protos --output-path=./lib priv/protos/google/api/annotations.proto priv/protos/google/api/http.proto +``` + +4. Generate the code: + +```shell +$ mix protobuf.generate --include-path=priv/protos --plugins=ProtobufGenerate.Plugins.GRPCWithOptions --output-path=./lib priv/protos/helloworld.proto +``` + +Refer to [protobuf-elixir](https://github.com/tony612/protobuf-elixir#usage) for more information. + +## How to start server when starting your application? + +Pass `start_server: true` as an option for the `GRPC.Server.Supervisor` in your supervision tree. + +## Benchmark + +Using [ghz](https://ghz.sh/) + +``` +$ MIX_ENV=prod iex -S mix +# Now cowboy doesn't work well with concurrency in a connection, like --concurrency 6 --connections 1 +$ ghz --insecure --proto priv/protos/helloworld.proto --call helloworld.Greeter.SayHello -d '{"name":"Joe"}' -z 10s --concurrency 6 --connections 6 127.0.0.1:50051 +# The result is for branch improve-perf +Summary: + Count: 124239 + Total: 10.00 s + Slowest: 18.85 ms + Fastest: 0.18 ms + Average: 0.44 ms + Requests/sec: 12423.71 + +# Go +Summary: + Count: 258727 + Total: 10.00 s + Slowest: 5.39 ms + Fastest: 0.09 ms + Average: 0.19 ms + Requests/sec: 25861.68 +``` diff --git a/examples/helloworld_transcoding/config/config.exs b/examples/helloworld_transcoding/config/config.exs new file mode 100644 index 00000000..9def7c2c --- /dev/null +++ b/examples/helloworld_transcoding/config/config.exs @@ -0,0 +1,3 @@ +import Config + +import_config "#{Mix.env}.exs" diff --git a/examples/helloworld_transcoding/config/dev.exs b/examples/helloworld_transcoding/config/dev.exs new file mode 100644 index 00000000..becde769 --- /dev/null +++ b/examples/helloworld_transcoding/config/dev.exs @@ -0,0 +1 @@ +import Config diff --git a/examples/helloworld_transcoding/config/prod.exs b/examples/helloworld_transcoding/config/prod.exs new file mode 100644 index 00000000..2dd33c31 --- /dev/null +++ b/examples/helloworld_transcoding/config/prod.exs @@ -0,0 +1,4 @@ +import Config + +config :logger, + level: :warn diff --git a/examples/helloworld_transcoding/config/test.exs b/examples/helloworld_transcoding/config/test.exs new file mode 100644 index 00000000..becde769 --- /dev/null +++ b/examples/helloworld_transcoding/config/test.exs @@ -0,0 +1 @@ +import Config diff --git a/examples/helloworld_transcoding/lib/endpoint.ex b/examples/helloworld_transcoding/lib/endpoint.ex new file mode 100644 index 00000000..70533a48 --- /dev/null +++ b/examples/helloworld_transcoding/lib/endpoint.ex @@ -0,0 +1,6 @@ +defmodule Helloworld.Endpoint do + use GRPC.Endpoint + + intercept GRPC.Logger.Server + run Helloworld.Greeter.Server +end diff --git a/examples/helloworld_transcoding/lib/google/api/annotations.pb.ex b/examples/helloworld_transcoding/lib/google/api/annotations.pb.ex new file mode 100644 index 00000000..48d40932 --- /dev/null +++ b/examples/helloworld_transcoding/lib/google/api/annotations.pb.ex @@ -0,0 +1,8 @@ +defmodule Google.Api.PbExtension do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + extend Google.Protobuf.MethodOptions, :http, 72_295_728, + optional: true, + type: Google.Api.HttpRule +end \ No newline at end of file diff --git a/examples/helloworld_transcoding/lib/google/api/http.pb.ex b/examples/helloworld_transcoding/lib/google/api/http.pb.ex new file mode 100644 index 00000000..524a0598 --- /dev/null +++ b/examples/helloworld_transcoding/lib/google/api/http.pb.ex @@ -0,0 +1,40 @@ +defmodule Google.Api.Http do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :rules, 1, repeated: true, type: Google.Api.HttpRule + + field :fully_decode_reserved_expansion, 2, + type: :bool, + json_name: "fullyDecodeReservedExpansion" +end + +defmodule Google.Api.HttpRule do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + oneof :pattern, 0 + + field :selector, 1, type: :string + field :get, 2, type: :string, oneof: 0 + field :put, 3, type: :string, oneof: 0 + field :post, 4, type: :string, oneof: 0 + field :delete, 5, type: :string, oneof: 0 + field :patch, 6, type: :string, oneof: 0 + field :custom, 8, type: Google.Api.CustomHttpPattern, oneof: 0 + field :body, 7, type: :string + field :response_body, 12, type: :string, json_name: "responseBody" + + field :additional_bindings, 11, + repeated: true, + type: Google.Api.HttpRule, + json_name: "additionalBindings" +end + +defmodule Google.Api.CustomHttpPattern do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :kind, 1, type: :string + field :path, 2, type: :string +end \ No newline at end of file diff --git a/examples/helloworld_transcoding/lib/helloworld.pb.ex b/examples/helloworld_transcoding/lib/helloworld.pb.ex new file mode 100644 index 00000000..a49adbf6 --- /dev/null +++ b/examples/helloworld_transcoding/lib/helloworld.pb.ex @@ -0,0 +1,55 @@ +defmodule Helloworld.HelloRequest do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string +end + +defmodule Helloworld.HelloRequestFrom do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :name, 1, type: :string + field :from, 2, type: :string +end + +defmodule Helloworld.HelloReply do + @moduledoc false + use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3 + + field :message, 1, type: :string + field :today, 2, type: Google.Protobuf.Timestamp +end + +defmodule Helloworld.Greeter.Service do + @moduledoc false + use GRPC.Service, name: "helloworld.Greeter", protoc_gen_elixir_version: "0.11.0" + + rpc(:SayHello, Helloworld.HelloRequest, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "", + pattern: {:get, "/v1/greeter/{name}"}, + response_body: "", + selector: "" + } + } + }) + + rpc(:SayHelloFrom, Helloworld.HelloRequestFrom, Helloworld.HelloReply, %{ + http: %{ + type: Google.Api.PbExtension, + value: %Google.Api.HttpRule{ + __unknown_fields__: [], + additional_bindings: [], + body: "*", + pattern: {:post, "/v1/greeter"}, + response_body: "", + selector: "" + } + } + }) +end diff --git a/examples/helloworld_transcoding/lib/helloworld_app.ex b/examples/helloworld_transcoding/lib/helloworld_app.ex new file mode 100644 index 00000000..d84d62a5 --- /dev/null +++ b/examples/helloworld_transcoding/lib/helloworld_app.ex @@ -0,0 +1,12 @@ +defmodule HelloworldApp do + use Application + + def start(_type, _args) do + children = [ + {GRPC.Server.Supervisor, endpoint: Helloworld.Endpoint, port: 50051, start_server: true} + ] + + opts = [strategy: :one_for_one, name: HelloworldApp] + Supervisor.start_link(children, opts) + end +end diff --git a/examples/helloworld_transcoding/lib/server.ex b/examples/helloworld_transcoding/lib/server.ex new file mode 100644 index 00000000..68c72c10 --- /dev/null +++ b/examples/helloworld_transcoding/lib/server.ex @@ -0,0 +1,31 @@ +defmodule Helloworld.Greeter.Server do + use GRPC.Server, + service: Helloworld.Greeter.Service, + http_transcode: true + + @spec say_hello(Helloworld.HelloRequest.t(), GRPC.Server.Stream.t()) :: + Helloworld.HelloReply.t() + def say_hello(request, _stream) do + Helloworld.HelloReply.new( + message: "Hello #{request.name}", + today: today() + ) + end + + @spec say_hello_from(Helloworld.HelloFromRequest.t(), GRPC.Server.Stream.t()) :: + Helloworld.HelloReply.t() + def say_hello_from(request, _stream) do + Helloworld.HelloReply.new( + message: "Hello #{request.name}. From #{request.from}", + today: today() + ) + end + + defp today do + nanos_epoch = System.system_time() |> System.convert_time_unit(:native, :nanosecond) + seconds = div(nanos_epoch, 1_000_000_000) + nanos = nanos_epoch - seconds * 1_000_000_000 + + %Google.Protobuf.Timestamp{seconds: seconds, nanos: nanos} + end +end diff --git a/examples/helloworld_transcoding/mix.exs b/examples/helloworld_transcoding/mix.exs new file mode 100644 index 00000000..f1b13088 --- /dev/null +++ b/examples/helloworld_transcoding/mix.exs @@ -0,0 +1,29 @@ +defmodule Helloworld.Mixfile do + use Mix.Project + + def project do + [ + app: :helloworld, + version: "0.1.0", + elixir: "~> 1.4", + build_embedded: Mix.env() == :prod, + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + def application do + [mod: {HelloworldApp, []}, applications: [:logger, :grpc]] + end + + defp deps do + [ + {:grpc, path: "../../"}, + {:protobuf, "~> 0.11.0"}, + {:protobuf_generate, "~> 0.1.1", only: [:dev, :test]}, + {:jason, "~> 1.3.0"}, + {:google_protos, "~> 0.3.0"}, + {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false} + ] + end +end diff --git a/examples/helloworld_transcoding/mix.lock b/examples/helloworld_transcoding/mix.lock new file mode 100644 index 00000000..afdfde5e --- /dev/null +++ b/examples/helloworld_transcoding/mix.lock @@ -0,0 +1,12 @@ +%{ + "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, + "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, + "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, + "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, + "google_protos": {:hex, :google_protos, "0.3.0", "15faf44dce678ac028c289668ff56548806e313e4959a3aaf4f6e1ebe8db83f4", [:mix], [{:protobuf, "~> 0.10", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "1f6b7fb20371f72f418b98e5e48dae3e022a9a6de1858d4b254ac5a5d0b4035f"}, + "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, + "protobuf_generate": {:hex, :protobuf_generate, "0.1.1", "f6098b85161dcfd48a4f6f1abee4ee5e057981dfc50aafb1aa4bd5b0529aa89b", [:mix], [{:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "93a38c8e2aba2a17e293e9ef1359122741f717103984aa6d1ebdca0efb17ab9d"}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, +} diff --git a/examples/helloworld_transcoding/priv/client.exs b/examples/helloworld_transcoding/priv/client.exs new file mode 100644 index 00000000..dc6bea5d --- /dev/null +++ b/examples/helloworld_transcoding/priv/client.exs @@ -0,0 +1,9 @@ +{:ok, channel} = GRPC.Stub.connect("localhost:50051", interceptors: [GRPC.Logger.Client]) + +{:ok, reply} = + channel + |> Helloworld.Greeter.Stub.say_hello(Helloworld.HelloRequest.new(name: "grpc-elixir")) + +# pass tuple `timeout: :infinity` as a second arg to stay in IEx debugging + +IO.inspect(reply) diff --git a/examples/helloworld_transcoding/priv/protos/google/api/annotations.proto b/examples/helloworld_transcoding/priv/protos/google/api/annotations.proto new file mode 100644 index 00000000..efdab3db --- /dev/null +++ b/examples/helloworld_transcoding/priv/protos/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/examples/helloworld_transcoding/priv/protos/google/api/http.proto b/examples/helloworld_transcoding/priv/protos/google/api/http.proto new file mode 100644 index 00000000..113fa936 --- /dev/null +++ b/examples/helloworld_transcoding/priv/protos/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/examples/helloworld_transcoding/priv/protos/helloworld.proto b/examples/helloworld_transcoding/priv/protos/helloworld.proto new file mode 100644 index 00000000..55c41005 --- /dev/null +++ b/examples/helloworld_transcoding/priv/protos/helloworld.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.helloworld"; +option java_outer_classname = "HelloWorldProto"; +option objc_class_prefix = "HLW"; + +import "google/api/annotations.proto"; +import "google/protobuf/timestamp.proto"; + +package helloworld; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) { + option (google.api.http) = { + get: "/v1/greeter/{name}" + }; + } + + rpc SayHelloFrom (HelloRequestFrom) returns (HelloReply) { + option (google.api.http) = { + post: "/v1/greeter" + body: "*" + }; + } +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +// HelloRequestFrom! +message HelloRequestFrom { + // Name! + string name = 1; + // From! + string from = 2; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; + google.protobuf.Timestamp today = 2; +} diff --git a/examples/helloworld_transcoding/test/hello_world_test.exs b/examples/helloworld_transcoding/test/hello_world_test.exs new file mode 100644 index 00000000..962d07ac --- /dev/null +++ b/examples/helloworld_transcoding/test/hello_world_test.exs @@ -0,0 +1,16 @@ +defmodule HelloworldTest do + @moduledoc false + + use ExUnit.Case + + setup_all do + {:ok, channel} = GRPC.Stub.connect("localhost:50051", interceptors: [GRPC.Logger.Client]) + [channel: channel] + end + + test "helloworld should be successful", %{channel: channel} do + req = Helloworld.HelloRequest.new(name: "grpc-elixir") + assert {:ok, %{message: msg, today: _}} = Helloworld.Greeter.Stub.say_hello(channel, req) + assert msg == "Hello grpc-elixir" + end +end diff --git a/examples/helloworld_transcoding/test/test_helper.exs b/examples/helloworld_transcoding/test/test_helper.exs new file mode 100644 index 00000000..869559e7 --- /dev/null +++ b/examples/helloworld_transcoding/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() From 6a21790d021a17f3852eb8c381ce28ef26bd6ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Thu, 27 Oct 2022 13:09:08 +0200 Subject: [PATCH 61/73] include google api protos in tests only --- mix.exs | 5 +++-- {lib => test/support}/google/api/annotations.pb.ex | 0 {lib => test/support}/google/api/http.pb.ex | 0 3 files changed, 3 insertions(+), 2 deletions(-) rename {lib => test/support}/google/api/annotations.pb.ex (100%) rename {lib => test/support}/google/api/http.pb.ex (100%) diff --git a/mix.exs b/mix.exs index e3fd57fb..70303571 100644 --- a/mix.exs +++ b/mix.exs @@ -70,7 +70,8 @@ defmodule GRPC.Mixfile do defp aliases do [ - gen_bootstrap_protos: &gen_bootstrap_protos/1 + gen_bootstrap_protos: &gen_bootstrap_protos/1, + gen_test_protos: [&gen_bootstrap_protos/1, &gen_test_protos/1] ] end @@ -96,7 +97,7 @@ defmodule GRPC.Mixfile do defp gen_bootstrap_protos(_args) do proto_src = Mix.Project.deps_paths().googleapis - protoc!("--include-path=#{proto_src}", "./lib", [ + protoc!("--include-path=#{proto_src}", "./test/support", [ "google/api/http.proto", "google/api/annotations.proto" ]) diff --git a/lib/google/api/annotations.pb.ex b/test/support/google/api/annotations.pb.ex similarity index 100% rename from lib/google/api/annotations.pb.ex rename to test/support/google/api/annotations.pb.ex diff --git a/lib/google/api/http.pb.ex b/test/support/google/api/http.pb.ex similarity index 100% rename from lib/google/api/http.pb.ex rename to test/support/google/api/http.pb.ex From cb6d617e5924d1d2ad1a271246dc99c4a6116ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Th=C3=B6rnqvist?= Date: Tue, 15 Nov 2022 13:40:22 +0100 Subject: [PATCH 62/73] map gRPC code -> http status --- lib/grpc/server/adapters/cowboy/handler.ex | 78 ++++++++++++---------- lib/grpc/status.ex | 19 ++++++ test/grpc/integration/server_test.exs | 48 +++++++++++++ 3 files changed, 110 insertions(+), 35 deletions(-) diff --git a/lib/grpc/server/adapters/cowboy/handler.ex b/lib/grpc/server/adapters/cowboy/handler.ex index 9eda27a1..4dbd9f75 100644 --- a/lib/grpc/server/adapters/cowboy/handler.ex +++ b/lib/grpc/server/adapters/cowboy/handler.ex @@ -24,7 +24,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do |> String.downcase() |> String.to_existing_atom() - with {:ok, sub_type, content_type} <- find_content_type_subtype(http_method, req), + with {:ok, sub_type, content_type} <- find_content_type_subtype(req), {:ok, codec} <- find_codec(sub_type, content_type, server), {:ok, compressor} <- find_compressor(req, server) do stream = %GRPC.Server.Stream{ @@ -36,7 +36,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do codec: codec, http_method: http_method, compressor: compressor, - http_transcode: sub_type == "json" + http_transcode: transcode?(req) } pid = spawn_link(__MODULE__, :call_rpc, [server, route, stream]) @@ -60,7 +60,7 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:error, error} -> Logger.error(fn -> inspect(error) end) trailers = HTTP2.server_trailers(error.status, error.message) - req = send_error_trailers(req, trailers) + req = send_error_trailers(req, 200, trailers) {:ok, req, state} end end @@ -77,11 +77,11 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do end end - defp find_content_type_subtype(:get, req) do + defp find_content_type_subtype(req) do content_type = - case :cowboy_req.header("accept", req) do + case :cowboy_req.header("content-type", req) do :undefined -> - :cowboy_req.header("content-type", req) + :cowboy_req.header("accept", req) content_type -> content_type @@ -90,12 +90,6 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do find_subtype(content_type) end - defp find_content_type_subtype(_, req) do - req_content_type = :cowboy_req.header("content-type", req) - - find_subtype(req_content_type) - end - defp find_subtype(content_type) do {:ok, subtype} = extract_subtype(content_type) {:ok, subtype, content_type} @@ -291,7 +285,9 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do msg = "A unaccepted encoding #{compressor.name()} is set, valid are: #{:cowboy_req.header("grpc-accept-encoding", req)}" - req = send_error(req, state, msg) + error = RPCError.exception(status: :internal, message: msg) + req = send_error(req, error, state, :rpc_error) + {:stop, req, state} else case GRPC.Message.to_data(data, compressor: compressor, codec: opts[:codec]) do @@ -301,7 +297,8 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, req, state} {:error, msg} -> - req = send_error(req, state, msg) + error = RPCError.exception(status: :internal, message: msg) + req = send_error(req, error, state, :rpc_error) {:stop, req, state} end end @@ -328,11 +325,10 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, req, state} end - def info({:handling_timeout, _}, req, state = %{pid: pid}) do + def info({:handling_timeout, _}, req, state) do error = %RPCError{status: GRPC.Status.deadline_exceeded(), message: "Deadline expired"} - trailers = HTTP2.server_trailers(error.status, error.message) - exit_handler(pid, :timeout) - req = send_error_trailers(req, trailers) + req = send_error(req, error, state, :timeout) + {:stop, req, state} end @@ -354,27 +350,26 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do # expected error raised from user to return error immediately def info({:EXIT, pid, {%RPCError{} = error, _stacktrace}}, req, state = %{pid: pid}) do - trailers = HTTP2.server_trailers(error.status, error.message) - exit_handler(pid, :rpc_error) - req = send_error_trailers(req, trailers) + req = send_error(req, error, state, :rpc_error) {:stop, req, state} end # unknown error raised from rpc - def info({:EXIT, pid, {:handle_error, _kind}}, req, state = %{pid: pid}) do + def info({:EXIT, pid, {:handle_error, _kind}} = err, req, state = %{pid: pid}) do + Logger.warn("3. #{inspect(state)} #{inspect(err)}") + error = %RPCError{status: GRPC.Status.unknown(), message: "Internal Server Error"} - trailers = HTTP2.server_trailers(error.status, error.message) - exit_handler(pid, :error) - req = send_error_trailers(req, trailers) + req = send_error(req, error, state, :error) + {:stop, req, state} end def info({:EXIT, pid, {reason, stacktrace}}, req, state = %{pid: pid}) do Logger.error(Exception.format(:error, reason, stacktrace)) + error = %RPCError{status: GRPC.Status.unknown(), message: "Internal Server Error"} - trailers = HTTP2.server_trailers(error.status, error.message) - exit_handler(pid, reason) - req = send_error_trailers(req, trailers) + req = send_error(req, error, state, reason) + {:stop, req, state} end @@ -458,12 +453,12 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do :cowboy_req.stream_reply(200, req) end - defp send_error_trailers(%{has_sent_resp: _} = req, trailers) do + defp send_error_trailers(%{has_sent_resp: _} = req, _, trailers) do :cowboy_req.stream_trailers(trailers, req) end - defp send_error_trailers(req, trailers) do - :cowboy_req.reply(200, trailers, req) + defp send_error_trailers(req, status, trailers) do + :cowboy_req.reply(status, trailers, req) end def exit_handler(pid, reason) do @@ -507,12 +502,25 @@ defmodule GRPC.Server.Adapters.Cowboy.Handler do {:ok, "proto"} end - defp send_error(req, %{pid: pid}, msg) do - error = RPCError.exception(status: :internal, message: msg) + defp transcode?(%{version: "HTTP/1.1"}), do: true + + defp transcode?(req) do + case find_content_type_subtype(req) do + {:ok, "json", _} -> true + _ -> false + end + end + + defp send_error(req, error, state, reason) do trailers = HTTP2.server_trailers(error.status, error.message) - exit_handler(pid, :rpc_error) - send_error_trailers(req, trailers) + status = if transcode?(req), do: GRPC.Status.http_code(error.status), else: 200 + + if pid = Map.get(state, :pid) do + exit_handler(pid, reason) + end + + send_error_trailers(req, status, trailers) end # Similar with cowboy's read_body, but we need to receive the message diff --git a/lib/grpc/status.ex b/lib/grpc/status.ex index 877d5030..2ad90752 100644 --- a/lib/grpc/status.ex +++ b/lib/grpc/status.ex @@ -165,4 +165,23 @@ defmodule GRPC.Status do def code_name(14), do: "Unavailable" def code_name(15), do: "DataLoss" def code_name(16), do: "Unauthenticated" + + @spec http_code(t()) :: t() + def http_code(0), do: 200 + def http_code(1), do: 400 + def http_code(2), do: 500 + def http_code(3), do: 400 + def http_code(4), do: 504 + def http_code(5), do: 404 + def http_code(6), do: 409 + def http_code(7), do: 403 + def http_code(8), do: 429 + def http_code(9), do: 412 + def http_code(10), do: 409 + def http_code(11), do: 400 + def http_code(12), do: 501 + def http_code(13), do: 500 + def http_code(14), do: 503 + def http_code(15), do: 500 + def http_code(16), do: 401 end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index d21479bc..a247b378 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -9,6 +9,18 @@ defmodule GRPC.Integration.ServerTest do end end + defmodule TranscodeErrorServer do + use GRPC.Server, + service: Transcode.Messaging.Service, + http_transcode: true + + def get_message(req, _stream) do + status = String.to_existing_atom(req.name) + + raise GRPC.RPCError, status: status + end + end + defmodule TranscodeServer do use GRPC.Server, service: Transcode.Messaging.Service, @@ -324,6 +336,42 @@ defmodule GRPC.Integration.ServerTest do end) end + test "should map grpc error codes to http status" do + run_server([TranscodeErrorServer], fn port -> + for {code_name, status} <- [ + {"cancelled", 400}, + {"unknown", 500}, + {"invalid_argument", 400}, + {"deadline_exceeded", 504}, + {"not_found", 404}, + {"already_exists", 409}, + {"permission_denied", 403}, + {"resource_exhausted", 429}, + {"failed_precondition", 412}, + {"aborted", 409}, + {"out_of_range", 400}, + {"unimplemented", 501}, + {"internal", 500}, + {"unavailable", 503}, + {"data_loss", 500}, + {"unauthenticated", 401} + ] do + {:ok, conn_pid} = :gun.open('localhost', port) + + stream_ref = + :gun.get( + conn_pid, + "/v1/messages/#{code_name}", + [ + {"accept", "application/json"} + ] + ) + + assert_receive {:gun_response, ^conn_pid, ^stream_ref, :fin, ^status, _headers} + end + end) + end + test "accept: application/json can be used with get requests" do run_server([TranscodeServer], fn port -> name = "direct_call" From ead4eaf06eb4461c1ac87a15d8f053bf99151906 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 15 Feb 2024 18:50:49 -0300 Subject: [PATCH 63/73] Minor adjusts --- lib/grpc/codec/json.ex | 54 +++++++++++++++++++++++++++ mix.exs | 5 +-- test/grpc/integration/server_test.exs | 4 +- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 91c5c9f7..47a05747 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -1,14 +1,68 @@ defmodule GRPC.Codec.JSON do + @moduledoc """ + JSON Codec for gRPC communication. + + This module implements the `GRPC.Codec` behaviour, providing encoding and decoding functions + for JSON serialization in the context of gRPC communication. + + ## Behavior Functions + + - `name/0`: Returns the name of the codec, which is "json". + - `encode/1`: Encodes a struct using the Protobuf.JSON.encode!/1 function. + - `decode/2`: Decodes binary data into a map using the Jason library. + + This module requires the Jason dependency. + + """ @behaviour GRPC.Codec def name() do "json" end + @doc """ + Encodes a struct using the Protobuf.JSON.encode!/1 function. + + ### Parameters: + + - `struct` - The struct to be encoded. + + ### Returns: + + The encoded binary data. + + ### Example: + + ```elixir + %MyStruct{id: 1, name: "John"} |> GRPC.Codec.JSON.encode() + ``` + + """ def encode(struct) do Protobuf.JSON.encode!(struct) end + @doc """ + Decodes binary data into a map using the Jason library. + Parameters: + + binary - The binary data to be decoded. + module - Module to be created. + + Returns: + + A map representing the decoded data. + + Raises: + + Raises an error if the :jason library is not loaded. + + Example: + + ```elixir + binary_data |> GRPC.Codec.JSON.decode(__MODULE__) + ``` + """ def decode(<<>>, _module) do %{} end diff --git a/mix.exs b/mix.exs index 0ae049f8..a590a187 100644 --- a/mix.exs +++ b/mix.exs @@ -32,9 +32,6 @@ defmodule GRPC.Mixfile do ] end - # Configuration for the OTP application - # - # Type "mix help compile.app" for more information def application do [extra_applications: [:logger]] end @@ -49,7 +46,7 @@ defmodule GRPC.Mixfile do # This is the same as :gun 2.0.0-rc.2, # but we can't depend on an RC for releases {:gun, "~> 2.0.1", hex: :grpc_gun}, - {:jason, "~> 1.0", optional: true}, + {:jason, ">= 0.0.0", optional: true}, {:cowlib, "~> 2.11"}, {:protobuf, github: "elixir-protobuf/protobuf", branch: "main"}, {:ex_doc, "~> 0.28.0", only: :dev}, diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index f91bf205..af700b47 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -141,7 +141,7 @@ defmodule GRPC.Integration.ServerTest do {:ok, conn_pid} = :gun.open('localhost', port) stream_ref = :gun.get(conn_pid, "/status") - assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert_received {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} end, 0, adapter_opts: [status_handler: status_handler] @@ -265,7 +265,7 @@ defmodule GRPC.Integration.ServerTest do {"content-type", "application/json"} ]) - assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert_received {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) assert %{ From 64768a3fd7609495773ab2dd7d3af20f6481b101 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 15 Feb 2024 19:43:07 -0300 Subject: [PATCH 64/73] Adjusts --- lib/grpc/codec/json.ex | 18 +-- lib/grpc/protoc/cli.ex | 18 +-- lib/grpc/server.ex | 18 ++- lib/grpc/server/router/template.ex | 5 +- mix.lock | 4 - test/grpc/integration/server_test.exs | 33 ----- test/grpc/server/transcode_test.exs | 166 -------------------------- 7 files changed, 23 insertions(+), 239 deletions(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 44e2c389..8f6da912 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -16,9 +16,7 @@ defmodule GRPC.Codec.JSON do """ @behaviour GRPC.Codec - def name() do - "json" - end + def name(), do: "json" @doc """ Encodes a struct using the Protobuf.JSON.encode!/1 function. @@ -64,17 +62,7 @@ defmodule GRPC.Codec.JSON do binary_data |> GRPC.Codec.JSON.decode(__MODULE__) ``` """ - def decode(<<>>, _module) do - %{} - end - - def decode(binary, _module) do - if jason = load_jason() do - jason.decode!(binary) - else - raise "`:jason` library not loaded" - end - end + def decode(<<>>, _module), do: %{} - defp load_jason, do: Code.ensure_loaded?(Jason) and Jason + def decode(binary, _module), do: Jason.decode!(binary) end diff --git a/lib/grpc/protoc/cli.ex b/lib/grpc/protoc/cli.ex index 84aa2215..bfcaa94d 100644 --- a/lib/grpc/protoc/cli.ex +++ b/lib/grpc/protoc/cli.ex @@ -119,15 +119,15 @@ defmodule GRPC.Protoc.CLI do end end - defp parse_param("include_docs=" <> value, ctx) do - case value do - "true" -> - %Context{ctx | include_docs?: true} - - other -> - raise "invalid value for include_docs option, expected \"true\", got: #{inspect(other)}" - end - end + # defp parse_param("include_docs=" <> value, ctx) do + # case value do + # "true" -> + # %Context{ctx | include_docs?: true} + + # other -> + # raise "invalid value for include_docs option, expected \"true\", got: #{inspect(other)}" + # end + # end defp parse_param(_unknown, ctx) do ctx diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index e4e4c6b1..3515e530 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -30,7 +30,7 @@ defmodule GRPC.Server do if it's streaming. If a reply is streaming, you need to call `send_reply/2` to send replies one by one instead of returning reply in the end. - ## gRPC http/json transcoding + ## gRPC HTTP/JSON transcoding Transcoding can be enabled by using the option `http_transcode: true`: @@ -131,18 +131,14 @@ defmodule GRPC.Server do codecs = if http_transcode, do: [GRPC.Codec.JSON | codecs], else: codecs routes = - for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, reduce: [] do + for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, + http_transcode and Map.has_key?(options, :http), + reduce: [] do acc -> path = "/#{service_name}/#{name}" - - acc = - if http_transcode and Map.has_key?(options, :http) do - %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) - route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) - [route | acc] - else - acc - end + %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) + route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) + acc = [route | acc] [{:grpc, path} | acc] end diff --git a/lib/grpc/server/router/template.ex b/lib/grpc/server/router/template.ex index b5fc6168..15cd7459 100644 --- a/lib/grpc/server/router/template.ex +++ b/lib/grpc/server/router/template.ex @@ -94,6 +94,9 @@ defmodule GRPC.Server.Router.Template do end defp field_path(identifier) do - String.to_atom(identifier) + String.to_existing_atom(identifier) + rescue + _e -> + String.to_atom(identifier) end end diff --git a/mix.lock b/mix.lock index c9c43224..d3936c63 100644 --- a/mix.lock +++ b/mix.lock @@ -13,11 +13,7 @@ "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, -<<<<<<< HEAD - "protobuf": {:git, "https://github.com/elixir-protobuf/protobuf.git", "cdf3acc53f619866b4921b8216d2531da52ceba7", [branch: "main"]}, -======= "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, "protobuf_generate": {:hex, :protobuf_generate, "0.1.1", "f6098b85161dcfd48a4f6f1abee4ee5e057981dfc50aafb1aa4bd5b0529aa89b", [:mix], [{:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "93a38c8e2aba2a17e293e9ef1359122741f717103984aa6d1ebdca0efb17ab9d"}, ->>>>>>> drowz/grpc_transcoding "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, } diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 3dcad30c..dbf5e2bc 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -9,15 +9,6 @@ defmodule GRPC.Integration.ServerTest do end end -<<<<<<< HEAD - defmodule FeatureTranscodeServer do - use GRPC.Server, - service: RouteguideTranscode.RouteGuide.Service, - http_transcode: true - - def get_feature(point, _stream) do - Routeguide.Feature.new(location: point, name: "#{point.latitude},#{point.longitude}") -======= defmodule TranscodeErrorServer do use GRPC.Server, service: Transcode.Messaging.Service, @@ -85,7 +76,6 @@ defmodule GRPC.Integration.ServerTest do name: msg_request.message.name, text: "get_message_with_subpath_query" ) ->>>>>>> drowz/grpc_transcoding end end @@ -323,12 +313,6 @@ defmodule GRPC.Integration.ServerTest do end describe "http/json transcode" do -<<<<<<< HEAD - test "can transcode path params" do - run_server([FeatureTranscodeServer], fn port -> - latitude = 10 - longitude = 20 -======= test "grpc method can be called using json when http_transcode == true" do run_server([TranscodeServer], fn port -> name = "direct_call" @@ -409,26 +393,10 @@ defmodule GRPC.Integration.ServerTest do test "can transcode path params" do run_server([TranscodeServer], fn port -> name = "foo" ->>>>>>> drowz/grpc_transcoding {:ok, conn_pid} = :gun.open('localhost', port) stream_ref = -<<<<<<< HEAD - :gun.get(conn_pid, "/v1/feature/#{latitude}/#{longitude}", [ - {"content-type", "application/json"} - ]) - - assert_received {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} - assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) - - assert %{ - "location" => %{"latitude" => ^latitude, "longitude" => ^longitude}, - "name" => name - } = Jason.decode!(body) - - assert name == "#{latitude},#{longitude}" -======= :gun.get(conn_pid, "/v1/messages/#{name}", [ {"content-type", "application/json"} ]) @@ -581,7 +549,6 @@ defmodule GRPC.Integration.ServerTest do "name" => "another_name", "text" => "get_message_with_query" } = Jason.decode!(body) ->>>>>>> drowz/grpc_transcoding end) end end diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index e4c0e5ac..751041b9 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -2,171 +2,6 @@ defmodule GRPC.TranscodeTest do use ExUnit.Case, async: true alias GRPC.Server.Transcode -<<<<<<< HEAD - test "map_requests/3 can map request body to protobuf struct" do - body_request = %{"latitude" => 1, "longitude" => 2} - {:ok, request} = Transcode.map_request(body_request, %{}, "", Routeguide.Point) - assert Routeguide.Point.new(latitude: 1, longitude: 2) == request - end - - test "map_requests/3 can merge request body with path bindings to protobuf struct" do - body_request = %{"latitude" => 1} - bindings = %{"longitude" => 2} - {:ok, request} = Transcode.map_request(body_request, bindings, "", Routeguide.Point) - assert Routeguide.Point.new(latitude: 1, longitude: 2) == request - end - - test "build_route/1 returns a route with {http_method, route} based on the http rule" do - rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert {:get, {params, segments}} = Transcode.build_route(rule) - assert [message_id: []] == params - assert ["v1", "messages", {:message_id, []}] = segments - end - - test "to_path/1 returns path segments as a string match" do - rule = build_simple_rule(:get, "/v1/messages/{message_id}") - assert spec = Transcode.build_route(rule) - assert "/v1/messages/:message_id" = Transcode.to_path(spec) - end - - test "to_path/1 returns path segments as a string when there's multiple bindings" do - rule = build_simple_rule(:get, "/v1/users/{user_id}/messages/{message_id}") - assert spec = Transcode.build_route(rule) - assert "/v1/users/:user_id/messages/:message_id" = Transcode.to_path(spec) - end - - describe "tokenize/2" do - test "can tokenize simple paths" do - assert [{:/, []}] = Transcode.tokenize("/") - - assert [{:/, []}, {:identifier, "v1", []}, {:/, []}, {:identifier, "messages", []}] = - Transcode.tokenize("/v1/messages") - end - - test "can tokenize simple paths with wildcards" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:identifier, "messages", []}, - {:/, []}, - {:*, []} - ] == Transcode.tokenize("/v1/messages/*") - end - - test "can tokenize simple variables" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:identifier, "messages", []}, - {:/, []}, - {:"{", []}, - {:identifier, "message_id", []}, - {:"}", []} - ] == Transcode.tokenize("/v1/messages/{message_id}") - end - - test "can tokenize variable assignments in bindings" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:"{", []}, - {:identifier, "name", []}, - {:=, []}, - {:identifier, "messages", []}, - {:"}", []} - ] == Transcode.tokenize("/v1/{name=messages}") - end - - test "can tokenize field paths in bindings" do - assert [ - {:/, []}, - {:identifier, "v1", []}, - {:/, []}, - {:identifier, "messages", []}, - {:/, []}, - {:"{", []}, - {:identifier, "message_id", []}, - {:"}", []}, - {:/, []}, - {:"{", []}, - {:identifier, "sub.subfield", []}, - {:"}", []} - ] == Transcode.tokenize("/v1/messages/{message_id}/{sub.subfield}") - end - end - - describe "parse/3" do - test "can parse simple paths" do - assert {[], []} == - "/" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse paths with identifiers" do - assert {[], ["v1", "messages"]} == - "/v1/messages" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse paths with wildcards" do - assert {[], ["v1", "messages", {:_, []}]} == - "/v1/messages/*" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse simple bindings with variables" do - assert {[{:message_id, []}], ["v1", "messages", {:message_id, []}]} == - "/v1/messages/{message_id}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse bindings with variable assignment" do - assert {[{:name, []}], ["v1", {:name, ["messages"]}]} == - "/v1/{name=messages}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse multiple bindings with variable assignment" do - assert {[{:name, []}, {:message_id, []}], ["v1", {:name, ["messages"]}, {:message_id, []}]} == - "/v1/{name=messages}/{message_id}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse bindings with field paths " do - assert {[sub: ["subfield"]], ["v1", "messages", {:sub, []}]} == - "/v1/messages/{sub.subfield}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "supports deeper nested field path " do - assert {[sub: ["nested", "nested", "nested"]], ["v1", "messages", {:sub, []}]} == - "/v1/messages/{sub.nested.nested.nested}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - - test "can parse multiple-bindings with field paths " do - assert {[first: ["subfield"], second: ["subfield"]], - ["v1", "messages", {:first, []}, {:second, []}]} == - "/v1/messages/{first.subfield}/{second.subfield}" - |> Transcode.tokenize() - |> Transcode.parse([], []) - end - end - - defp build_simple_rule(method, pattern) do - Google.Api.HttpRule.new(pattern: {method, pattern}) -======= test "map_request/5 with HttpRule.body: '*'" do rule = Google.Api.HttpRule.new(body: "*") request_body = %{"latitude" => 1, "longitude" => 2} @@ -224,6 +59,5 @@ defmodule GRPC.TranscodeTest do test "map_route_bindings/2 with '.' delimited identifiers should create a nested map" do path_binding = %{"foo.bar.baz" => "biz"} assert %{"foo" => %{"bar" => %{"baz" => "biz"}}} == Transcode.map_path_bindings(path_binding) ->>>>>>> drowz/grpc_transcoding end end From 24bbf51940afbee78952ac12ff6969b030648387 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 15 Feb 2024 22:00:48 -0300 Subject: [PATCH 65/73] Fix. Assert payload --- lib/grpc/server.ex | 29 +++++++++++++++++++++------ test/grpc/integration/server_test.exs | 14 ++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 9af10f84..0a00b8d9 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -130,15 +130,32 @@ defmodule GRPC.Server do codecs = if http_transcode, do: [GRPC.Codec.JSON | codecs], else: codecs + # routes = + # for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, + # http_transcode and Map.has_key?(options, :http), + # reduce: [] do + # acc -> + # path = "/#{service_name}/#{name}" + # %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) + # route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) + # acc = [route | acc] + + # [{:grpc, path} | acc] + # end + routes = - for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, - http_transcode and Map.has_key?(options, :http), - reduce: [] do + for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, reduce: [] do acc -> path = "/#{service_name}/#{name}" - %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) - route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) - acc = [route | acc] + + acc = + if http_transcode and Map.has_key?(options, :http) do + %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) + route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) + [route | acc] + else + acc + end [{:grpc, path} | acc] end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index d76a2cb4..0b8b2201 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -585,7 +585,7 @@ defmodule GRPC.Integration.ServerTest do run_server([HelloServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "delay", duration: 1000) + req = %Helloworld.HelloRequest{name: "delay", duration: 1000} assert {:ok, _} = Helloworld.Greeter.Stub.say_hello(channel, req) end) @@ -619,7 +619,8 @@ defmodule GRPC.Integration.ServerTest do assert %{ stream: %GRPC.Client.Stream{ rpc: - {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}} + {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}, + %{}} } } = metadata @@ -630,7 +631,8 @@ defmodule GRPC.Integration.ServerTest do assert %{ stream: %GRPC.Client.Stream{ rpc: - {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}} + {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}, + %{}} } } = metadata @@ -707,7 +709,8 @@ defmodule GRPC.Integration.ServerTest do assert %{ stream: %GRPC.Client.Stream{ rpc: - {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}} + {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}, + %{}} } } = metadata @@ -718,7 +721,8 @@ defmodule GRPC.Integration.ServerTest do assert %{ stream: %GRPC.Client.Stream{ rpc: - {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}} + {"say_hello", {Helloworld.HelloRequest, false}, {Helloworld.HelloReply, false}, + %{}} } } = metadata From 8613a46fd9a4738dcf9d30b0f3d08512708048e6 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 15 Feb 2024 22:20:22 -0300 Subject: [PATCH 66/73] Remove deprecated and fix some tests --- lib/grpc/server.ex | 13 ----- .../integration/client_interceptor_test.exs | 2 +- test/grpc/integration/connection_test.exs | 2 +- test/grpc/integration/endpoint_test.exs | 2 +- test/grpc/integration/namespace_test.exs | 4 +- test/grpc/integration/server_test.exs | 47 +++++++++---------- test/grpc/integration/service_test.exs | 16 +++---- test/grpc/server/router_test.exs | 2 +- test/grpc/server/transcode_test.exs | 10 ++-- 9 files changed, 42 insertions(+), 56 deletions(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 0a00b8d9..7b5d0b6c 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -130,19 +130,6 @@ defmodule GRPC.Server do codecs = if http_transcode, do: [GRPC.Codec.JSON | codecs], else: codecs - # routes = - # for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, - # http_transcode and Map.has_key?(options, :http), - # reduce: [] do - # acc -> - # path = "/#{service_name}/#{name}" - # %{value: http_rule} = GRPC.Service.rpc_options(rpc, :http) - # route = Macro.escape({:http_transcode, Router.build_route(http_rule)}) - # acc = [route | acc] - - # [{:grpc, path} | acc] - # end - routes = for {name, _, _, options} = rpc <- service_mod.__rpc_calls__, reduce: [] do acc -> diff --git a/test/grpc/integration/client_interceptor_test.exs b/test/grpc/integration/client_interceptor_test.exs index 3e2109e6..a699a0dc 100644 --- a/test/grpc/integration/client_interceptor_test.exs +++ b/test/grpc/integration/client_interceptor_test.exs @@ -7,7 +7,7 @@ defmodule GRPC.Integration.ClientInterceptorTest do def say_hello(req, stream) do headers = GRPC.Stream.get_headers(stream) label = headers["x-test-label"] - Helloworld.HelloReply.new(message: "Hello, #{req.name} #{label}") + %Helloworld.HelloReply{message: "Hello, #{req.name} #{label}"} end end diff --git a/test/grpc/integration/connection_test.exs b/test/grpc/integration/connection_test.exs index f4d70aad..53eb9b52 100644 --- a/test/grpc/integration/connection_test.exs +++ b/test/grpc/integration/connection_test.exs @@ -34,7 +34,7 @@ defmodule GRPC.Integration.ConnectionTest do {:ok, _, port} = GRPC.Server.start(server, 0, cred: cred) try do - point = Routeguide.Point.new(latitude: 409_146_138, longitude: -746_188_906) + point = %Routeguide.Point{latitude: 409_146_138, longitude: -746_188_906} {:ok, channel} = GRPC.Stub.connect("localhost:#{port}", cred: cred) assert {:ok, _} = Routeguide.RouteGuide.Stub.get_feature(channel, point) diff --git a/test/grpc/integration/endpoint_test.exs b/test/grpc/integration/endpoint_test.exs index 638b829d..a4dd0c2d 100644 --- a/test/grpc/integration/endpoint_test.exs +++ b/test/grpc/integration/endpoint_test.exs @@ -6,7 +6,7 @@ defmodule GRPC.Integration.EndpointTest do use GRPC.Server, service: Helloworld.Greeter.Service def say_hello(req, _stream) do - Helloworld.HelloReply.new(message: "Hello, #{req.name}") + %Helloworld.HelloReply{message: "Hello, #{req.name}"} end end diff --git a/test/grpc/integration/namespace_test.exs b/test/grpc/integration/namespace_test.exs index c572534c..d187ce80 100644 --- a/test/grpc/integration/namespace_test.exs +++ b/test/grpc/integration/namespace_test.exs @@ -12,9 +12,9 @@ defmodule GRPC.Integration.NamespaceTest do test "it works when outer namespace is same with inner's" do run_server(FeatureServer, fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - point = Routeguide.Point.new(latitude: 409_146_138, longitude: -746_188_906) + point = %Routeguide.Point{latitude: 409_146_138, longitude: -746_188_906} {:ok, feature} = channel |> Routeguide.RouteGuide.Stub.get_feature(point) - assert feature == Routeguide.Feature.new(location: point, name: "409146138,-746188906") + assert feature == %Routeguide.Feature{location: point, name: "409146138,-746188906"} end) end end diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 0b8b2201..2e137403 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -87,32 +87,32 @@ defmodule GRPC.Integration.ServerTest do def say_hello(%{name: "delay", duration: duration}, _stream) do Process.sleep(duration) - Helloworld.HelloReply.new(message: "Hello") + %Helloworld.HelloReply{message: "Hello"} end def say_hello(%{name: "large response"}, _stream) do name = String.duplicate("a", round(:math.pow(2, 14))) - Helloworld.HelloReply.new(message: "Hello, #{name}") + %Helloworld.HelloReply{message: "Hello, #{name}"} end def say_hello(%{name: "get peer"}, stream) do {ip, _port} = stream.adapter.get_peer(stream.payload) name = to_string(:inet_parse.ntoa(ip)) - Helloworld.HelloReply.new(message: "Hello, #{name}") + %Helloworld.HelloReply{message: "Hello, #{name}"} end def say_hello(%{name: "get cert"}, stream) do case stream.adapter.get_cert(stream.payload) do :undefined -> - Helloworld.HelloReply.new(message: "Hello, unauthenticated") + %Helloworld.HelloReply{message: "Hello, unauthenticated"} _ -> - Helloworld.HelloReply.new(message: "Hello, authenticated") + %Helloworld.HelloReply{message: "Hello, authenticated"} end end def say_hello(req, _stream) do - Helloworld.HelloReply.new(message: "Hello, #{req.name}") + %Helloworld.HelloReply{message: "Hello, #{req.name}"} end def check_headers(_req, stream) do @@ -185,11 +185,11 @@ defmodule GRPC.Integration.ServerTest do test "multiple servers works" do run_server([FeatureServer, HelloServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - point = Routeguide.Point.new(latitude: 409_146_138, longitude: -746_188_906) + point = %Routeguide.Point{latitude: 409_146_138, longitude: -746_188_906} {:ok, feature} = channel |> Routeguide.RouteGuide.Stub.get_feature(point) - assert feature == Routeguide.Feature.new(location: point, name: "409146138,-746188906") + assert feature == %Routeguide.Feature{location: point, name: "409146138,-746188906"} - req = Helloworld.HelloRequest.new(name: "Elixir") + req = %Helloworld.HelloRequest{name: "Elixir"} {:ok, reply} = channel |> Helloworld.Greeter.Stub.say_hello(req) assert reply.message == "Hello, Elixir" end) @@ -202,7 +202,7 @@ defmodule GRPC.Integration.ServerTest do [HelloServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "Elixir") + req = %Helloworld.HelloRequest{name: "Elixir"} {:ok, reply} = channel |> Helloworld.Greeter.Stub.say_hello(req) assert reply.message == "Hello, Elixir" @@ -219,7 +219,7 @@ defmodule GRPC.Integration.ServerTest do test "returns appropriate error for unary requests" do run_server([HelloErrorServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "Elixir") + req = %Helloworld.HelloRequest{name: "Elixir"} {:error, reply} = channel |> Helloworld.Greeter.Stub.say_hello(req) assert %GRPC.RPCError{ @@ -232,7 +232,7 @@ defmodule GRPC.Integration.ServerTest do test "return errors for unknown errors" do run_server([HelloErrorServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "unknown error") + req = %Helloworld.HelloRequest{name: "unknown error"} assert {:error, %GRPC.RPCError{message: "Internal Server Error", status: GRPC.Status.unknown()}} == @@ -243,7 +243,7 @@ defmodule GRPC.Integration.ServerTest do test "returns appropriate error for stream requests" do run_server([FeatureErrorServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - rect = Routeguide.Rectangle.new() + rect = %Routeguide.Rectangle{} error = %GRPC.RPCError{message: "Please authenticate", status: 16} assert {:error, ^error} = channel |> Routeguide.RouteGuide.Stub.list_features(rect) end) @@ -252,7 +252,7 @@ defmodule GRPC.Integration.ServerTest do test "return large response(more than MAX_FRAME_SIZE 16384)" do run_server([HelloServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "large response") + req = %Helloworld.HelloRequest{name: "large response"} {:ok, reply} = channel |> Helloworld.Greeter.Stub.say_hello(req) name = String.duplicate("a", round(:math.pow(2, 14))) assert "Hello, #{name}" == reply.message @@ -262,7 +262,7 @@ defmodule GRPC.Integration.ServerTest do test "return deadline error for slow server" do run_server([TimeoutServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - rect = Routeguide.Rectangle.new() + rect = %Routeguide.Rectangle{} error = %GRPC.RPCError{message: "Deadline expired", status: 4} assert {:error, ^error} = @@ -273,9 +273,9 @@ defmodule GRPC.Integration.ServerTest do test "return normally for a little slow server" do run_server([SlowServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - low = Routeguide.Point.new(latitude: 400_000_000, longitude: -750_000_000) - high = Routeguide.Point.new(latitude: 420_000_000, longitude: -730_000_000) - rect = Routeguide.Rectangle.new(lo: low, hi: high) + low = %Routeguide.Point{latitude: 400_000_000, longitude: -750_000_000} + high = %Routeguide.Point{latitude: 420_000_000, longitude: -730_000_000} + rect = %Routeguide.Rectangle{lo: low, hi: high} {:ok, stream} = channel |> Routeguide.RouteGuide.Stub.list_features(rect, timeout: 500) Enum.each(stream, fn {:ok, feature} -> @@ -293,8 +293,7 @@ defmodule GRPC.Integration.ServerTest do headers: [{"authorization", token}] ) - {:ok, reply} = - channel |> Helloworld.Greeter.Stub.check_headers(Helloworld.HeaderRequest.new()) + {:ok, reply} = channel |> Helloworld.Greeter.Stub.check_headers(%Helloworld.HeaderRequest{}) assert reply.authorization == token end) @@ -304,7 +303,7 @@ defmodule GRPC.Integration.ServerTest do run_server([HelloServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "get peer") + req = %Helloworld.HelloRequest{name: "get peer"} {:ok, reply} = channel |> Helloworld.Greeter.Stub.say_hello(req) assert reply.message == "Hello, 127.0.0.1" end) @@ -314,7 +313,7 @@ defmodule GRPC.Integration.ServerTest do run_server([HelloServer], fn port -> assert {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "get cert") + req = %Helloworld.HelloRequest{name: "get cert"} assert {:ok, reply} = channel |> Helloworld.Greeter.Stub.say_hello(req) assert reply.message == "Hello, unauthenticated" end) @@ -476,7 +475,7 @@ defmodule GRPC.Integration.ServerTest do ] ) - assert_receive {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} + assert_receive {:gun_up, ^conn_pid, :http} assert {:ok, body} = :gun.await_body(conn_pid, stream_ref) assert %{"name" => ^name, "text" => "get_message_with_response_body"} = @@ -662,7 +661,7 @@ defmodule GRPC.Integration.ServerTest do run_server([HelloServer], fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - req = Helloworld.HelloRequest.new(name: "raise", duration: 1100) + req = %Helloworld.HelloRequest{name: "raise", duration: 1100} assert {:error, %GRPC.RPCError{status: 2}} = Helloworld.Greeter.Stub.say_hello(channel, req) diff --git a/test/grpc/integration/service_test.exs b/test/grpc/integration/service_test.exs index 48b5ffdd..ba321e2b 100644 --- a/test/grpc/integration/service_test.exs +++ b/test/grpc/integration/service_test.exs @@ -87,8 +87,8 @@ defmodule GRPC.Integration.ServiceTest do test "client streaming RPC works" do run_server(FeatureServer, fn port -> {:ok, channel} = GRPC.Stub.connect("localhost:#{port}") - point1 = Routeguide.Point.new(latitude: 400_000_000, longitude: -750_000_000) - point2 = Routeguide.Point.new(latitude: 420_000_000, longitude: -730_000_000) + point1 = %Routeguide.Point{latitude: 400_000_000, longitude: -750_000_000} + point2 = %Routeguide.Point{latitude: 420_000_000, longitude: -730_000_000} stream = channel |> Routeguide.RouteGuide.Stub.record_route() GRPC.Stub.send_request(stream, point1) GRPC.Stub.send_request(stream, point2, end_stream: true) @@ -105,8 +105,8 @@ defmodule GRPC.Integration.ServiceTest do task = Task.async(fn -> Enum.each(1..6, fn i -> - point = Routeguide.Point.new(latitude: 0, longitude: rem(i, 3) + 1) - note = Routeguide.RouteNote.new(location: point, message: "Message #{i}") + point = %Routeguide.Point{latitude: 0, longitude: rem(i, 3) + 1} + note = %Routeguide.RouteNote{location: point, message: "Message #{i}"} opts = if i == 6, do: [end_stream: true], else: [] GRPC.Stub.send_request(stream, note, opts) end) @@ -137,8 +137,8 @@ defmodule GRPC.Integration.ServiceTest do task = Task.async(fn -> Enum.each(1..5, fn i -> - point = Routeguide.Point.new(latitude: 0, longitude: rem(i, 3) + 1) - note = Routeguide.RouteNote.new(location: point, message: "Message #{i}") + point = %Routeguide.Point{latitude: 0, longitude: rem(i, 3) + 1} + note = %Routeguide.RouteNote{location: point, message: "Message #{i}"} # note that we don't send end of stream yet here GRPC.Stub.send_request(stream, note, []) end) @@ -155,8 +155,8 @@ defmodule GRPC.Integration.ServiceTest do assert "Reply: " <> _msg = note.message if note.message == "Reply: Message 5" do - point = Routeguide.Point.new(latitude: 0, longitude: rem(6, 3) + 1) - note = Routeguide.RouteNote.new(location: point, message: "Message #{6}") + point = %Routeguide.Point{latitude: 0, longitude: rem(6, 3) + 1} + note = %Routeguide.RouteNote{location: point, message: "Message #{6}"} GRPC.Stub.send_request(stream, note, end_stream: true) end diff --git a/test/grpc/server/router_test.exs b/test/grpc/server/router_test.exs index f0d1e3f2..f6718309 100644 --- a/test/grpc/server/router_test.exs +++ b/test/grpc/server/router_test.exs @@ -156,6 +156,6 @@ defmodule GRPC.Server.RouterTest do end defp build_simple_rule(method, pattern) do - Google.Api.HttpRule.new(pattern: {method, pattern}) + %Google.Api.HttpRule{pattern: {method, pattern}} end end diff --git a/test/grpc/server/transcode_test.exs b/test/grpc/server/transcode_test.exs index 751041b9..61995a85 100644 --- a/test/grpc/server/transcode_test.exs +++ b/test/grpc/server/transcode_test.exs @@ -3,7 +3,7 @@ defmodule GRPC.TranscodeTest do alias GRPC.Server.Transcode test "map_request/5 with HttpRule.body: '*'" do - rule = Google.Api.HttpRule.new(body: "*") + rule = %Google.Api.HttpRule{body: "*"} request_body = %{"latitude" => 1, "longitude" => 2} bindings = %{} qs = "latitude=10&longitude=20" @@ -13,7 +13,7 @@ defmodule GRPC.TranscodeTest do end test "map_request/5 with empty HttpRule.body" do - rule = Google.Api.HttpRule.new(body: "") + rule = %Google.Api.HttpRule{body: ""} request_body = %{"latitude" => 10, "longitude" => 20} bindings = %{"latitude" => 5} qs = "longitude=6" @@ -23,7 +23,7 @@ defmodule GRPC.TranscodeTest do end test "map_request/2 with HttpRule.body: " do - rule = Google.Api.HttpRule.new(body: "location") + rule = %Google.Api.HttpRule{body: "location"} request_body = %{"latitude" => 1, "longitude" => 2} bindings = %{"name" => "test"} @@ -35,14 +35,14 @@ defmodule GRPC.TranscodeTest do end test "map_response_body/2 with empty HttpRule.response_body" do - rule = Google.Api.HttpRule.new(response_body: "") + rule = %Google.Api.HttpRule{response_body: ""} request_body = %{message: %{a: "b"}} assert request_body == Transcode.map_response_body(rule, request_body) end test "map_response_body/2 with HttpRule.response_body: " do - rule = Google.Api.HttpRule.new(response_body: "message") + rule = %Google.Api.HttpRule{response_body: "message"} request_body = %{message: %{a: "b"}} assert %{a: "b"} == Transcode.map_response_body(rule, request_body) From 9b8a4ca32bace2cf3066d549bab9472fd2160c98 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 15 Feb 2024 22:24:25 -0300 Subject: [PATCH 67/73] Fix. status handler test --- test/grpc/integration/server_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/grpc/integration/server_test.exs b/test/grpc/integration/server_test.exs index 2e137403..7a6ae8cd 100644 --- a/test/grpc/integration/server_test.exs +++ b/test/grpc/integration/server_test.exs @@ -208,6 +208,7 @@ defmodule GRPC.Integration.ServerTest do {:ok, conn_pid} = :gun.open(~c"localhost", port) stream_ref = :gun.get(conn_pid, "/status") + Process.sleep(100) assert_received {:gun_response, ^conn_pid, ^stream_ref, :nofin, 200, _headers} end, From d9b0a3e82194f4e66960b632ca6747f9453cdac6 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:20:48 -0300 Subject: [PATCH 68/73] Update lib/grpc/codec/json.ex --- lib/grpc/codec/json.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 8f6da912..5a89b189 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -12,7 +12,6 @@ defmodule GRPC.Codec.JSON do - `decode/2`: Decodes binary data into a map using the Jason library. This module requires the Jason dependency. - """ @behaviour GRPC.Codec From cf86ec0209961fa4cab9d255d504abc0389bd358 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:21:27 -0300 Subject: [PATCH 69/73] Update lib/grpc/codec/json.ex --- lib/grpc/codec/json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/grpc/codec/json.ex b/lib/grpc/codec/json.ex index 5a89b189..d84a75c1 100644 --- a/lib/grpc/codec/json.ex +++ b/lib/grpc/codec/json.ex @@ -53,7 +53,7 @@ defmodule GRPC.Codec.JSON do Raises: - Raises an error if the :jason library is not loaded. + Raises an error if the Jason library is not loaded. Example: From 4bb2d7152d8e6dd46d91358004fc8f9c63e4b500 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:22:40 -0300 Subject: [PATCH 70/73] Update lib/grpc/server.ex --- lib/grpc/server.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/grpc/server.ex b/lib/grpc/server.ex index 7b5d0b6c..cdd51bc3 100644 --- a/lib/grpc/server.ex +++ b/lib/grpc/server.ex @@ -152,7 +152,7 @@ defmodule GRPC.Server do path = "/#{service_name}/#{name}" grpc_type = GRPC.Service.grpc_type(rpc) - def __call_rpc__(unquote(path), unquote(:post), stream) do + def __call_rpc__(unquote(path), :post, stream) do GRPC.Server.call( unquote(service_mod), %{ From e28265485bcc35f197a9ecd81dd39f56af67d937 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:26:51 -0300 Subject: [PATCH 71/73] Update lib/grpc/transport/http2.ex --- lib/grpc/transport/http2.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/grpc/transport/http2.ex b/lib/grpc/transport/http2.ex index aea7625d..0d47631f 100644 --- a/lib/grpc/transport/http2.ex +++ b/lib/grpc/transport/http2.ex @@ -12,8 +12,9 @@ defmodule GRPC.Transport.HTTP2 do %{"content-type" => "application/grpc-web-#{codec.name()}"} end - def server_headers(%{codec: GRPC.Codec.JSON = codec}) do - %{"content-type" => "application/#{codec.name()}"} + # TO-DO: refactor when we add a GRPC.Codec.content_type callback + def server_headers(%{codec: GRPC.Codec.JSON}) do + %{"content-type" => "application/json"} end def server_headers(%{codec: codec}) do From c4861b6f9f64f6ed6de070ab135ac9f2d0e7d9ca Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:31:37 -0300 Subject: [PATCH 72/73] Update lib/grpc/server/transcode.ex --- lib/grpc/server/transcode.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/grpc/server/transcode.ex b/lib/grpc/server/transcode.ex index 4ef236ae..1d2562c5 100644 --- a/lib/grpc/server/transcode.ex +++ b/lib/grpc/server/transcode.ex @@ -1,6 +1,7 @@ defmodule GRPC.Server.Transcode do @moduledoc false - alias GRPC.Server.Router.{Query, FieldPath} + alias GRPC.Server.Router.Query + alias GRPC.Server.Router.FieldPath @type t :: map() # The request mapping follow the following rules: From 23842424fce45dc35469a40a653dfa35f2f7f871 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:38:19 -0300 Subject: [PATCH 73/73] chore: remove prometheus stray deps --- interop/mix.exs | 2 -- interop/mix.lock | 9 ++++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/interop/mix.exs b/interop/mix.exs index caee13fc..05f61478 100644 --- a/interop/mix.exs +++ b/interop/mix.exs @@ -24,11 +24,9 @@ defmodule Interop.MixProject do [ {:grpc, path: "..", override: true}, {:protobuf, "~> 0.11.0"}, - {:grpc_prometheus, ">= 0.1.0"}, {:grpc_statsd, "~> 0.1.0"}, {:statix, ">= 1.2.1"}, {:extrace, "~> 0.2"}, - {:prometheus, "~> 4.0", override: true} ] end end diff --git a/interop/mix.lock b/interop/mix.lock index 4bf86bef..0533eb90 100644 --- a/interop/mix.lock +++ b/interop/mix.lock @@ -1,16 +1,15 @@ %{ "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, + "extrace": {:hex, :extrace, "0.5.0", "4ee5419fbc3820c4592daebe0f8527001aa623578d9a725d8ae521315fce0277", [:mix], [{:recon, "~> 2.5", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "2a3ab7fa0701949efee1034293fa0b0e65926ffe256ccd6d0e10dd8a9406cd02"}, "grpc": {:git, "https://github.com/elixir-grpc/grpc.git", "21422839798e49bf6d29327fab0a7add51becedd", []}, - "grpc_prometheus": {:hex, :grpc_prometheus, "0.1.0", "a2f45ca83018c4ae59e4c293b7455634ac09e38c36cba7cc1fb8affdf462a6d5", [:mix], [{:grpc, ">= 0.0.0", [hex: :grpc, repo: "hexpm", optional: true]}, {:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8b9ab3098657e7daec0b3edc78e1d02418bc0871618d8ca89b51b74a8086bb71"}, "grpc_statsd": {:hex, :grpc_statsd, "0.1.0", "a95ae388188486043f92a3c5091c143f5a646d6af80c9da5ee616546c4d8f5ff", [:mix], [{:grpc, ">= 0.0.0", [hex: :grpc, repo: "hexpm", optional: true]}, {:statix, ">= 0.0.0", [hex: :statix, repo: "hexpm", optional: true]}], "hexpm", "de0c05db313c7b3ffeff345855d173fd82fec3de16591a126b673f7f698d9e74"}, - "gun": {:hex, :grpc_gun, "2.0.1", "221b792df3a93e8fead96f697cbaf920120deacced85c6cd3329d2e67f0871f8", [:rebar3], [{:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "795a65eb9d0ba16697e6b0e1886009ce024799e43bb42753f0c59b029f592831"}, - "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm", "b479a33d4aa4ba7909186e29bb6c1240254e0047a8e2a9f88463f50c0089370e"}, - "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "9fd13404a48437e044b288b41f76e64acd9735fb8b0e3809f494811dfa66d0fb"}, - "prometheus_httpd": {:hex, :prometheus_httpd, "2.1.11", "f616ed9b85b536b195d94104063025a91f904a4cfc20255363f49a197d96c896", [:rebar3], [{:accept, "~> 0.3", [hex: :accept, repo: "hexpm", optional: false]}, {:prometheus, "~> 4.2", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "0bbe831452cfdf9588538eb2f570b26f30c348adae5e95a7d87f35a5910bcf92"}, + "gun": {:hex, :gun, "2.0.1", "160a9a5394800fcba41bc7e6d421295cf9a7894c2252c0678244948e3336ad73", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "a10bc8d6096b9502205022334f719cc9a08d9adcfbfc0dbee9ef31b56274a20b"}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, "protobuf": {:hex, :protobuf, "0.11.0", "58d5531abadea3f71135e97bd214da53b21adcdb5b1420aee63f4be8173ec927", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "30ad9a867a5c5a0616cac9765c4d2c2b7b0030fa81ea6d0c14c2eb5affb6ac52"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"}, + "statix": {:hex, :statix, "1.4.0", "c822abd1e60e62828e8460e932515d0717aa3c089b44cc3f795d43b94570b3a8", [:mix], [], "hexpm", "507373cc80925a9b6856cb14ba17f6125552434314f6613c907d295a09d1a375"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, }