- Bypass grpc upstream?
- Read grpc headers and body from the filter?
- Write grpc headers and body from the filter?
- grpc-transcode plugin essence
- grpc-web plugin essence
- proxy-mirror
For grpc errors,
e.g. Unauthenticated
, it's able to return
grpc-status=?
in http 200 response with content-length=0
.
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/helloworld.Greeter/SayHello",
"plugins": {
"serverless-pre-function": {
"phase": "access",
"functions" : ["return function(_, ctx)
local core = require(\"apisix.core\")
core.response.set_header(\"grpc-status\", 16)
core.response.set_header(\"grpc-message\", \"some error\")
core.response.set_header(\"content-length\", 0)
core.response.set_header(\"content-type\", ctx.var.http_content_type)
return 200
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
grpc-go client log:
2023/01/18 19:40:58 could not greet: rpc error: code = Unauthenticated desc = some error
Note that we must return 200
, otherwise the error_page
would inject html body, which
is considered as error by the grpc client.
grpc-go client log:
2023/01/18 16:04:46 could not greet: rpc error: code = Unauthenticated
desc = unexpected HTTP status code received from server: 401 (Unauthorized);
transport: received unexpected content-type "text/html; charset=utf-8"
If you really need to return code other than 200
, then you need to call ngx.exit(ngx.HTTP_OK)
to bypass error_page
:
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/helloworld.Greeter/SayHello",
"plugins": {
"serverless-pre-function": {
"phase": "access",
"functions" : ["return function(_, ctx)
local core = require(\"apisix.core\")
core.response.set_header(\"grpc-status\", 11)
core.response.set_header(\"grpc-message\", \"some error\")
core.response.set_header(\"content-type\", ctx.var.http_content_type)
ngx.status = 403
ngx.exit(ngx.HTTP_OK)
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
grpcurl -import-path /opt/grpc-go/examples/helloworld/helloworld/ -proto helloworld.proto -plaintext -d '{"name":"foo"}' 'localhost:9081' helloworld.Greeter.SayHello
ERROR:
Code: PermissionDenied
Message: unexpected HTTP status code received from server: 403 (Forbidden)
Note that for code other than 200
, e.g. 403
, the client will use code instead of grpc-status
.
You could see that from tcpdump, code 403
and grpc-status=11
returns, but 403
takes precedent.
error_page
also works for APISIX:
# config.yaml
nginx_config:
http_server_configuration_snippet: |
error_page 404 = @grpc_unimplemented;
location @grpc_unimplemented {
add_header grpc-status 12;
add_header grpc-message unimplemented;
add_header content-type application/grpc;
return 200;
}
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/helloworld.Greeter/SayHello",
"plugins": {
"serverless-pre-function": {
"phase": "access",
"functions" : ["return function(_, ctx)
return 404
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
grpcurl -import-path /opt/grpc-go/examples/helloworld/helloworld/ -proto helloworld.proto -plaintext -d '{"name":"foo"}' 'localhost:9081' helloworld.Greeter.SayHello
ERROR:
Code: Unimplemented
Message: unimplemented
For grpc_status == 0
, trailer headers must exist, but openresty has no API here,
so it's impossible to return normal response from APISIX directly.
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/helloworld.Greeter/SayHello",
"plugins": {
"serverless-pre-function": {
"phase": "access",
"functions" : ["return function(_, ctx)
local core = require(\"apisix.core\")
core.response.set_header(\"errorCode\", 0)
core.response.set_header(\"Trailer\", \"Grpc-Status, Grpc-Message\")
core.response.set_header(\"Grpc-Status\", 2)
core.response.set_header(\"Grpc-Message\", \"no error\")
core.response.set_header(\"Content-length\", 0)
core.response.set_header(\"Content-type\", ctx.var.http_content_type)
local data = \"000000000d0a0b48656c6c6f20776f726c64\"
local data = (data:gsub(\"..\", function (cc)
return string.char(tonumber(cc, 16))
end))
core.response.set_header(\"Content-length\", #data)
ngx.exit(200, data)
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
grpc-go client log:
2023/01/18 21:35:27 could not greet: rpc error: code = Internal
desc = server closed the stream without sending trailers
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/routeguide.RouteGuide/*",
"plugins": {
"serverless-pre-function": {
"phase": "access",
"functions" : ["return function(_, ctx)
local core = require(\"apisix.core\")
ctx.tag = 666
core.log.warn(\"req: ctx=\", tostring(ctx))
core.log.warn(\"req: headers: \", core.json.encode(ngx.req.get_headers()))
core.log.warn(\"req: len=\", #core.request.get_body())
end"]
},
"serverless-post-function": {
"phase": "body_filter",
"functions" : ["return function(_, ctx)
local core = require(\"apisix.core\")
core.log.warn(\"rsp: ctx=\", tostring(ctx), \", ctx.tag=\", ctx.tag)
core.log.warn(\"rsp: headers: \", core.json.encode(ngx.resp.get_headers()))
core.log.warn(\"rsp: body: len=\", #ngx.arg[1], \", eof=\", ngx.arg[2])
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
HEADERS
frame triggerbody_fitler
once- each
DATA
frame triggersbody_filter
once, witheof=false
- trailer
HEADERS
frame triggersbody_fitler
once, witheof=true
, body is nil, you could read trailer headers, e.g.grpc-status
andgrpc-message
, via$send_trailer_*
variables - all messages of one grpc stream correspond to the same request context, i.e
ctx
points to the same object
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/helloworld.Greeter/*",
"plugins": {
"serverless-post-function": {
"phase": "body_filter",
"functions" : ["return function(_, ctx)
local eof = ngx.arg[2]
if eof == true then
local inspect = require(\"inspect\")
ngx.log(ngx.WARN, \"rsp: grpc_status: \", inspect(ctx.var.sent_trailer_grpc_status))
ngx.log(ngx.WARN, \"rsp: grpc_message: \", inspect(ctx.var.sent_trailer_grpc_message))
end
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
grpcurl -vv -import-path /opt/grpc-go/examples/helloworld/helloworld/ \
-proto helloworld.proto -plaintext -d '{"name":"foo"}' \
'localhost:9081' helloworld.Greeter.SayHello
logs:
2023/03/03 11:10:39 [warn] 2433381#2433381: *49507 [lua] [string "return function(_, ctx)..."]:5: func(): rsp: grpc_status: "0" while sending to client, client: ::1, server: _, request: "POST /helloworld.Greeter/SayHello HTTP/2.0", upstream: "grpc://127.0.0.1:50051", host: "localhost:9081"
2023/03/03 11:10:39 [warn] 2433381#2433381: *49507 [lua] [string "return function(_, ctx)..."]:6: func(): rsp: grpc_message: "" while sending to client, client: ::1, server: _, request: "POST /helloworld.Greeter/SayHello HTTP/2.0", upstream: "grpc://127.0.0.1:50051", host: "localhost:9081"
For error response, no trailer headers, grpc-status
and grpc-message
are in the first HEADERS
frame,
so you have to check them in header_filter
.
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/helloworld.Greeter/*",
"plugins": {
"serverless-post-function": {
"phase": "header_filter",
"functions" : ["return function(_, ctx)
ngx.log(ngx.WARN, \"rsp: headers: \", require(\"inspect\")(ngx.resp.get_headers()))
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
logs:
2023/03/03 11:04:31 [warn] 2433381#2433381: *39727 [lua] [string "return function(_, ctx)..."]:2: func(): rsp: headers: {
connection = "close",
["content-length"] = "0",
["content-type"] = "application/grpc",
["grpc-message"] = "unknown service helloworld.Greeter",
["grpc-status"] = "12",
server = "APISIX/3.1.0",
<metatable> = {
__index = <function 1>
}
} while reading response header from upstream, client: ::1, server: _, request: "POST /helloworld.Greeter/SayHello HTTP/2.0", upstream: "grpc://127.0.0.1:50051", host: "localhost:9081"
HEADERS
and allDATA
frames together triggeraccess
phase once, because openresty would collect all client messages into one whole request viangx.req.read_body()
- all others are identical to server-side streaming
- identical to client-side streaming and server-side streaming
For error response (grpc_status != 0
), grpc-status: *
is shown in both
the first and the last HEADER
frame.
- in header filter, you could read
grpc-status
header. - openresty triggers two times of body filter, both
len=0
, witheof=true
in the last one.
curl -v --raw 'http://127.0.0.1:9080/grpctest?latitude=409146138&longitude=-746188906'
* Trying 127.0.0.1:9080...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 9080 (#0)
> GET /grpctest?latitude=409146138&longitude=-746188906 HTTP/1.1
> Host: 127.0.0.1:9080
> User-Agent: curl/7.68.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 501 Not Implemented
< Date: Fri, 20 Jan 2023 11:23:35 GMT
< Content-Type: application/json
< Transfer-Encoding: chunked
< Connection: keep-alive
< grpc-status: 12
< grpc-message: unknown service routeguide.RouteGuide
< Server: APISIX/3.1.0
<
0
* Connection #0 to host 127.0.0.1 left intact
curl -v --raw http://127.0.0.1:9080/grpctest?name=hello
* Trying 127.0.0.1:9080...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 9080 (#0)
> GET /grpctest?name=hello HTTP/1.1
> Host: 127.0.0.1:9080
> User-Agent: curl/7.68.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Date: Fri, 20 Jan 2023 10:01:38 GMT
< Content-Type: application/json
< Transfer-Encoding: chunked
< Connection: keep-alive
< Server: APISIX/3.1.0
< Trailer: grpc-status
< Trailer: grpc-message
<
19
{"message":"Hello hello"}
0
grpc-status: 0
grpc-message:
logs:
2023/01/20 17:57:39 [warn] 1993523#1993523: *308901 [lua] [string "return function(_, ctx)..."]:3: func(): rsp: ctx=table: 0x7f9a8f03d198, ctx.tag=nil while reading response header from upstream, client: 127.0.0.1, server: _, request: "GET /grpctest?name=hello HTTP/1.1", upstream: "grpc://127.0.0.1:50051", host: "127.0.0.1:9080"
2023/01/20 17:57:39 [warn] 1993523#1993523: *308901 [lua] [string "return function(_, ctx)..."]:4: func(): rsp: headers: {"content-type":"application\/json","connection":"keep-alive","transfer-encoding":"chunked","grpc-status":"12","grpc-message":"unknown service helloworld.Greeter","server":"APISIX\/3.1.0"} while reading response header from upstream, client: 127.0.0.1, server: _, request: "GET /grpctest?name=hello HTTP/1.1", upstream: "grpc://127.0.0.1:50051", host: "127.0.0.1:9080"
2023/01/20 17:57:39 [warn] 1993523#1993523: *308901 [lua] [string "return function(_, ctx)..."]:5: func(): rsp: body: len=0, eof=false while reading response header from upstream, client: 127.0.0.1, server: _, request: "GET /grpctest?name=hello HTTP/1.1", upstream: "grpc://127.0.0.1:50051", host: "127.0.0.1:9080"
2023/01/20 17:57:39 [warn] 1993523#1993523: *308901 [lua] [string "return function(_, ctx)..."]:3: func(): rsp: ctx=table: 0x7f9a8f03d198, ctx.tag=nil while sending to client, client: 127.0.0.1, server: _, request: "GET /grpctest?name=hello HTTP/1.1", upstream: "grpc://127.0.0.1:50051", host: "127.0.0.1:9080"
2023/01/20 17:57:39 [warn] 1993523#1993523: *308901 [lua] [string "return function(_, ctx)..."]:4: func(): rsp: headers: {"content-type":"application\/json","connection":"keep-alive","transfer-encoding":"chunked","grpc-status":"12","grpc-message":"unknown service helloworld.Greeter","server":"APISIX\/3.1.0"} while sending to client, client: 127.0.0.1, server: _, request: "GET /grpctest?name=hello HTTP/1.1", upstream: "grpc://127.0.0.1:50051", host: "127.0.0.1:9080"
2023/01/20 17:57:39 [warn] 1993523#1993523: *308901 [lua] [string "return function(_, ctx)..."]:5: func(): rsp: body: len=0, eof=true while sending to client, client: 127.0.0.1, server: _, request: "GET /grpctest?name=hello HTTP/1.1", upstream: "grpc://127.0.0.1:50051", host: "127.0.0.1:9080"
The first strange len=0
is triggered by empty body flush:
Thread 1 "openresty" hit Breakpoint 1, ngx_http_lua_body_filter (in=0x5570db8de1d8, r=0x5570db8e5bb0) at ../ngx_lua-0.10.21/src/ngx_http_lua_bodyfilterby.c:343
343 rc = llcf->body_filter_handler(r, in);
(gdb) bt
#0 ngx_http_lua_body_filter (in=0x5570db8de1d8, r=0x5570db8e5bb0) at ../ngx_lua-0.10.21/src/ngx_http_lua_bodyfilterby.c:343
#1 ngx_http_lua_body_filter (r=0x5570db8e5bb0, in=<optimized out>) at ../ngx_lua-0.10.21/src/ngx_http_lua_bodyfilterby.c:233
#2 0x00005570da9e77e3 in ngx_output_chain (ctx=ctx@entry=0x5570db8eaf80, in=in@entry=0x7fffbb358410) at src/core/ngx_output_chain.c:74
#3 0x00005570daa62dd8 in ngx_http_copy_filter (r=0x5570db8e5bb0, in=0x7fffbb358410) at src/http/ngx_http_copy_filter_module.c:152
#4 0x00005570daa2852b in ngx_http_output_filter (r=r@entry=0x5570db8e5bb0, in=in@entry=0x7fffbb358410) at src/http/ngx_http_core_module.c:1881
#5 0x00005570daa2c664 in ngx_http_send_special (r=r@entry=0x5570db8e5bb0, flags=flags@entry=2) at src/http/ngx_http_request.c:3585
#6 0x00005570daa472a1 in ngx_http_upstream_send_response (u=0x5570db8dd7b0, r=0x5570db8e5bb0) at src/http/ngx_http_upstream.c:3176
#7 ngx_http_upstream_process_header (r=0x5570db8e5bb0, u=0x5570db8dd7b0) at src/http/ngx_http_upstream.c:2601
#8 0x00005570daa3fbe4 in ngx_http_upstream_handler (ev=0x7f2f3e551eb0) at src/http/ngx_http_upstream.c:1310
#9 0x00005570daa11c9b in ngx_epoll_process_events (cycle=0x5570db796150, timer=<optimized out>, flags=<optimized out>) at src/event/modules/ngx_epoll_module.c:901
#10 0x00005570daa05e29 in ngx_process_events_and_timers (cycle=cycle@entry=0x5570db796150) at src/event/ngx_event.c:257
#11 0x00005570daa0f575 in ngx_worker_process_cycle (cycle=cycle@entry=0x5570db796150, data=data@entry=0x0) at src/os/unix/ngx_process_cycle.c:806
#12 0x00005570daa0ddd6 in ngx_spawn_process (cycle=cycle@entry=0x5570db796150, proc=proc@entry=0x5570daa0f520 <ngx_worker_process_cycle>, data=data@entry=0x0, name=name
@entry=0x5570dab62f25 "worker process", respawn=respawn@entry=-3) at src/os/unix/ngx_process.c:199
#13 0x00005570daa0fc24 in ngx_start_worker_processes (cycle=cycle@entry=0x5570db796150, n=1, type=type@entry=-3) at src/os/unix/ngx_process_cycle.c:392
#14 0x00005570daa1077e in ngx_master_process_cycle (cycle=0x5570db796150) at src/os/unix/ngx_process_cycle.c:138
#15 0x00005570da9e1fa6 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:386
- set request/response headers: ok
- overwrite request body: ok
- no need to set
content-length
- But you can only send one message to server, i.e. client-streaming takes no effect.
- no need to set
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/routeguide.RouteGuide/*",
"plugins": {
"serverless-pre-function": {
"phase": "access",
"functions" : ["return function(_, ctx)
local core = require(\"apisix.core\")
core.request.get_body()
local data = \"00000000110880ae99b5011080d6e8c7faffffffff01\"
local data = (data:gsub(\"..\", function (cc)
return string.char(tonumber(cc, 16))
end))
ngx.req.set_body_data(data)
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
- overwrite response body per server message: ok
- no need to set
content-length
- should ignore last body, i.e. trailer
HEADERS
frame
- no need to set
curl http://127.0.0.1:9180/apisix/admin/routes/test_grpc \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods": ["POST", "GET"],
"uri": "/routeguide.RouteGuide/*",
"plugins": {
"serverless-post-function": {
"phase": "body_filter",
"functions" : ["return function(_, ctx)
if #ngx.arg[1] > 0 then
local data = \"000000000c0a0210011206666f6f626172\"
local data = (data:gsub(\"..\", function (cc)
return string.char(tonumber(cc, 16))
end))
ngx.arg[1] = data
end
end"]
}
},
"upstream": {
"scheme":"grpc",
"type": "roundrobin",
"nodes": {
"localhost:50051": 1
}
}
}'
- only support unary, because:
- all messages from client-side streaming are merged via
ngx.req.read_body()
- it collects all response bodies via
core.response.hold_body_chunk(ctx)
- all messages from client-side streaming are merged via
unary call:
protoc --descriptor_set_out=/tmp/route_guide.pb --include_imports \
--proto_path=/opt/grpc-go/examples/route_guide/routeguide/ route_guide.proto
curl http://127.0.0.1:9180/apisix/admin/protos/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"content" : "'"$(base64 -w0 /tmp/route_guide.pb)"'"
}'
curl http://127.0.0.1:9180/apisix/admin/routes/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods":[
"GET"
],
"uri":"/grpctest",
"plugins":{
"grpc-transcode":{
"proto_id":"1",
"service":"routeguide.RouteGuide",
"method":"GetFeature"
}
},
"upstream":{
"scheme":"grpc",
"type":"roundrobin",
"nodes":{
"127.0.0.1:50051":1
}
}
}'
curl -v --raw 'http://127.0.0.1:9080/grpctest?latitude=409146138&longitude=-746188906'
* Trying 127.0.0.1:9080...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 9080 (#0)
> GET /grpctest?latitude=409146138&longitude=-746188906 HTTP/1.1
> Host: 127.0.0.1:9080
> User-Agent: curl/7.68.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Date: Fri, 20 Jan 2023 11:22:43 GMT
< Content-Type: application/json
< Transfer-Encoding: chunked
< Connection: keep-alive
< Server: APISIX/3.1.0
< Trailer: grpc-status
< Trailer: grpc-message
<
7e
{"name":"Berkshire Valley Management Area Trail, Jefferson, NJ, USA","location":{"latitude":409146138,"longitude":-746188906}}
0
grpc-status: 0
grpc-message:
* Connection #0 to host 127.0.0.1 left intact
text escape method:
https://appdevtools.com/json-escape-unescape
curl http://127.0.0.1:9180/apisix/admin/protos/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d @- <<"EOF"
{
"content" : "syntax = \"proto3\";\n\noption go_package = \"google.golang.org/grpc/examples/helloworld/helloworld\";\noption java_multiple_files = true;\noption java_package = \"io.grpc.examples.helloworld\";\noption java_outer_classname = \"HelloWorldProto\";\n\npackage helloworld;\n\n// The greeting service definition.\nservice Greeter {\n // Sends a greeting\n rpc SayHello (HelloRequest) returns (HelloReply) {}\n}\n\n// The request message containing the user's name.\nmessage HelloRequest {\n string name = 1;\n}\n\n// The response message containing the greetings\nmessage HelloReply {\n string message = 1;\n}\n"
}
EOF
# or use jq to escape proto file
curl http://127.0.0.1:9180/apisix/admin/protos/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"content" : '"$(jq -R -s '.' < helloworld.proto)"'
}'
curl http://127.0.0.1:9180/apisix/admin/routes/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"methods":[
"GET"
],
"uri":"/grpctest",
"plugins":{
"grpc-transcode":{
"proto_id":"1",
"service":"helloworld.Greeter",
"method":"SayHello"
}
},
"upstream":{
"scheme":"grpc",
"type":"roundrobin",
"nodes":{
"127.0.0.1:50051":1
}
}
}'
server-streaming:
curl http://127.0.0.1:9180/apisix/admin/routes/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"uri":"/grpctest",
"plugins":{
"grpc-transcode":{
"proto_id":"1",
"service":"routeguide.RouteGuide",
"method":"ListFeatures"
}
},
"upstream":{
"scheme":"grpc",
"type":"roundrobin",
"nodes":{
"127.0.0.1:50051":1
}
}
}'
curl -H 'content-type: application/json' -v --raw http://127.0.0.1:9080/grpctest -d '
{
"lo":{"latitude": 400000000, "longitude": -750000000},
"hi":{"latitude": 420000000, "longitude": -730000000}
}'
* Trying 127.0.0.1:9080...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 9080 (#0)
> POST /grpctest HTTP/1.1
> Host: 127.0.0.1:9080
> User-Agent: curl/7.68.0
> Accept: */*
> content-type: application/json
> Content-Length: 121
>
* upload completely sent off: 121 out of 121 bytes
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Date: Fri, 20 Jan 2023 14:20:59 GMT
< Content-Type: application/json
< Transfer-Encoding: chunked
< Connection: keep-alive
< Server: APISIX/3.1.0
< Trailer: grpc-status
< Trailer: grpc-message
<
6e
{"name":"101 New Jersey 10, Whippany, NJ 07981, USA","location":{"longitude":-743999179,"latitude":408122808}}
0
grpc-status: 0
grpc-message:
* Connection #0 to host 127.0.0.1 left intact
go run client.go -addr localhost:50051
2023/01/20 19:31:04 Looking for features within lo:{latitude:400000000 longitude:-750000000} hi:{latitude:420000000 longitude:-730000000}
2023/01/20 19:31:04 Feature: name: "Patriots Path, Mendham, NJ 07945, USA", point:(407838351, -746143763)
2023/01/20 19:31:07 Feature: name: "101 New Jersey 10, Whippany, NJ 07981, USA", point:(408122808, -743999179)
Set an inspect breakpint to check:
dbg.set_hook("apisix/plugins/grpc-transcode/response.lua", 131, nil, function(info)
for k,v in pairs(info.vals.decoded) do
core.log.warn("k=",k,",v=",type(v))
end
return false
end)
breakpoint output:
#buffer=131
k=name,v=string
k=location,v=table
You could see that although two messages are collected by hold_body_chunk
(size=131=63+68
),
buf pb.decode
could only decode the last message.
That's why the server-streaming is broken in grpc transcode plugin.
google.protobuf.Struct bugfix:
https://gist.github.com/kingluo/2a6af0b600cc9804870985458c472350
-
this plugin only handles
content-type
andbase64
encode/decode. -
confrom to grpc-web spec, support unary and server-side streaming
- if use HTTP 1.1, then each chunk denotes one stream message from server
- the trailer
HEADERS
is intactly put in the trailer part following the last chunk (remind that lua could not touch them)
- for server-streaming
- if the upstream
DATA
frames arrives very close in time, nginx will merge them into one chunk to the downstream - if the time interval between two chunks is long enough, e.g. 3 seconds, then each
DATA
frame corresponds to one separate chunk
- if the upstream
APISIX does not support proxy-mirror for grpc yet, although nginx does.
- current proxy_mirror location is filled with proxy_pass directives, i.e. http1 only
- APISIX enables mirror dynamically, set a flag in the request ctx, but for grpc upstream, it invokes
ngx.exec
, which clears the ctx, so mirror is disabled - we need to set up correct parameters like URL before sending it out to the grpc mirror target, otherwise, it will be an error due to the wrong URL
/proxy_mirror
:
After some fix, APISIX can mirror grpc traffic.
proxy-mirror-grpc.patch
note that it's demo only, which hardcodes the parameters.
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 142d9229..df5650d5 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -760,6 +760,8 @@ http {
access_by_lua_block {
apisix.grpc_access_phase()
+ require("resty.apisix.client").enable_mirror()
+ ngx.req.set_uri("/helloworld.Greeter/SayHello")
}
{% if use_apisix_base then %}
@@ -773,6 +775,7 @@ http {
grpc_set_header Content-Type application/grpc;
grpc_socket_keepalive on;
grpc_pass $upstream_scheme://apisix_backend;
+ mirror /proxy_mirror;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
@@ -815,27 +818,11 @@ http {
location = /proxy_mirror {
internal;
- {% if not use_apisix_base then %}
- if ($upstream_mirror_uri = "") {
- return 200;
+ rewrite_by_lua_block {
+ ngx.req.set_uri("/helloworld.Greeter/SayHello")
}
- {% end %}
-
- {% if proxy_mirror_timeouts then %}
- {% if proxy_mirror_timeouts.connect then %}
- proxy_connect_timeout {* proxy_mirror_timeouts.connect *};
- {% end %}
- {% if proxy_mirror_timeouts.read then %}
- proxy_read_timeout {* proxy_mirror_timeouts.read *};
- {% end %}
- {% if proxy_mirror_timeouts.send then %}
- proxy_send_timeout {* proxy_mirror_timeouts.send *};
- {% end %}
- {% end %}
- proxy_http_version 1.1;
- proxy_set_header Host $upstream_host;
- proxy_pass $upstream_mirror_uri;
+ grpc_pass 127.0.0.1:50052;
}
{% end %}
}
diff --git a/apisix/plugins/proxy-mirror.lua b/apisix/plugins/proxy-mirror.lua
index 312d3ec3..38f4afdc 100644
--- a/apisix/plugins/proxy-mirror.lua
+++ b/apisix/plugins/proxy-mirror.lua
@@ -27,7 +27,6 @@ local schema = {
properties = {
host = {
type = "string",
- pattern = [=[^http(s)?:\/\/([\da-zA-Z.-]+|\[[\da-fA-F:]+\])(:\d+)?$]=],
},
path = {
type = "string",
test:
curl http://127.0.0.1:9180/apisix/admin/routes/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"plugins": {
"proxy-mirror": {
"host": "grpc://127.0.0.1:50052",
"sample_ratio": 1
}
},
"upstream": {
"scheme": "grpc",
"nodes": {
"127.0.0.1:50051": 1
}
},
"uri": "/helloworld.Greeter/SayHello"
}'
foo@bar:/opt/grpc-go/examples/helloworld/greeter_client# go run main.go -addr 127.0.0.1:9081
foo@bar:/opt/grpc-go/examples/helloworld/greeter_server# go run main.go -port 50052
2023/04/08 22:06:20 server listening at 127.0.0.1:50052
2023/04/09 15:56:51 Received: tonic
grpc->dubbo是可以透传的,但是有三点需要注意: