InputStream myInputStream = new ByteArrayInputStream(pemBytes); // <<<<<< pemBytes here is the CA ROOT cert. This is all we use for server side auth only. SslContext sslContext = GrpcSslContexts.forClient().trustManager(myInputStream) .sslProvider(sslprovider).keyManager(clientKey, clientCert).build(); /// <<<<<<<<<<<<< This is how we set the client key (java) and the client x509 cert. this.channelBuilder = NettyChannelBuilder.forAddress(addr, port).sslContext(sslContext) .negotiationType(ntype); if (cn != null) { channelBuilder.overrideAuthority(cn); } addNettyBuilderProps(channelBuilder, properties); } catch (SSLException sslex) { throw new RuntimeException(sslex); }
Generated on the fly so nothing lost in exposing this :)
2018-02-28 22:44:13,006 grpc-default-executor-1 ERROR OrdererClient:157 - Received error on channel foo, orderer orderer.example.com, url grpcs://localhost:7050, UNAVAILABLE: io exception io.grpc.StatusRuntimeException: UNAVAILABLE: io exception at io.grpc.Status.asRuntimeException(Status.java:526) at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:419) at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:41) at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:684) at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:41) at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:391) at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:475) at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63) at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:557) at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:478) at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:590) at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.lang.Thread.run(Thread.java:795) Caused by: javax.net.ssl.SSLHandshakeException: error:10000412:SSL routines:OPENSSL_internal:SSLV3_ALERT_BAD_CERTIFICATE at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.shutdownWithError(ReferenceCountedOpenSslEngine.java:876) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.sslReadErrorResult(ReferenceCountedOpenSslEngine.java:1124) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1080) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1146) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1189) at io.netty.handler.ssl.SslHandler$SslEngineType$1.unwrap(SslHandler.java:216) at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1247) at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1158) at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1193) at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:138) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) ... 1 more 201
Carl Mastrangelo
unread,
Mar 1, 2018, 2:16:14 PM3/1/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
Not tried that yet but I removed the keyManger setting and got the very same error. So I'm really now thinking it's nothing actually wrong with the client keys/certs being set there.
cr2...@gmail.com
unread,
Mar 1, 2018, 3:58:11 PM3/1/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
Does this just wrapper the local native oppenssl implementation ?
On Thursday, March 1, 2018 at 2:16:14 PM UTC-5, Carl Mastrangelo wrote:
Carl Mastrangelo
unread,
Mar 1, 2018, 4:08:05 PM3/1/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
Yes, basically. If you can use the openssl tool (like s_client) to use the certs, then we would know this is a problem on the java side.
cr2...@gmail.com
unread,
Mar 1, 2018, 7:41:20 PM3/1/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
I captured our generated cert and the private key in files .. tlsCertPem tlsKeyPem Then did a`openssl s_client -connect localhost:7050 -cert tlsCertPem -key tlsKeyPem -debug` I got what was a telnet like session .. typed a few lines hit enter : and ..
```
orderer.example.com | 2018-03-0200:24:45.019 UTC [grpc] Printf -> DEBU 0ba transport: http2Server.HandleStreams received bogus greeting from client:"hitherieh\njfsfjskfjs\njfk"
So I think this rules out invalid client certs / keys.
cr2...@gmail.com
unread,
Mar 1, 2018, 7:46:24 PM3/1/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
This has been reproduced on Fedora 27 with IBM Jdk / Ubuntu OpenJdk and Mac not sure of what JDK.
On Wednesday, February 28, 2018 at 5:47:41 PM UTC-5, cr2...@gmail.com wrote:
cr2...@gmail.com
unread,
Mar 1, 2018, 8:51:31 PM3/1/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
It's possible to give multiple client certificates. Is there some selection criteria when connecting to a specific server that determines which if any client certs are acceptable ? Will it simpie try all ? (not done much with mutual auth :)
On Wednesday, February 28, 2018 at 5:47:41 PM UTC-5, cr2...@gmail.com wrote:
cr2...@gmail.com
unread,
Mar 2, 2018, 9:43:16 AM3/2/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
Hi I see the certificate in the builder. Have not found a reference to private key .. should I? Some guidance where to look ? >>> https://pasteboard.co/Ha2YMA2.jpg
On Wednesday, February 28, 2018 at 5:47:41 PM UTC-5, cr2...@gmail.com wrote:
Carl Mastrangelo
unread,
Mar 2, 2018, 5:06:20 PM3/2/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
You don't happen to have the server requesting a client side cert do you?
Otherwise, I don't have much else to guess. Personally I would fire up gdb and step through the authentication, but that may be a lot of effort.
cr2...@gmail.com
unread,
Mar 3, 2018, 1:16:08 PM3/3/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
I think we made some progress. We've got a case where it worked. There were several issue at play making it difficult to nail down and leading to false theories.. It's a bit preliminary but one issue with a particular JDK not seeming to work at all, we think. The other is the client certificates I think have to match server organization. Which they did'nt and a bit of lacking of understanding on my part. However the error SSLV3_ALERT_BAD_CERTIFICATE is not very helpful .. like how about NO_MATCH_CLIENT_CERTIFICATE?
On Wednesday, February 28, 2018 at 5:47:41 PM UTC-5, cr2...@gmail.com wrote:
Carl Mastrangelo
unread,
Mar 3, 2018, 8:02:08 PM3/3/18
Reply to author
Sign in to reply to author
Forward
Sign in to forward
Delete
You do not have permission to delete messages in this group
Copy link
Report message
Show original message
Either email addresses are anonymous for this group or you need the view member email addresses permission to view the original message
to grpc.io
One other thing you can try is using Jetty ALPN (See the SECURITY.md file in the grpc-java repo). That would tell you if it is Netty's integration with OpenSSL that is broken.