Solr certificate mismatch

Mark

unread,

Jul 15, 2015, 11:16:24 AM7/15/15

to ddf-...@googlegroups.com

I have gone through the steps to update my certificates in DDF. I went through the instructions in the 2.7.0 Management PDF and tried to restart DDF. I verified that my new serverKeystore.jks file does not reference "localhost" at all.

Upon restarting DDF, I see the following stack trace in the DDF logfile. Is there another setting that either I missed or is not in section "Configuring DDF with New Certificates" of the Managing PDF file.

Here's the stack trace:

org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://localhost:8993/solr

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507)[225:persistence-core-impl:2.7.1]

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)[225:persistence-core-impl:2.7.1]

at org.apache.solr.client.solrj.request.CoreAdminRequest.process(CoreAdminRequest.java:493)[225:persistence-core-impl:2.7.1]

at org.apache.solr.client.solrj.request.CoreAdminRequest.createCore(CoreAdminRequest.java:570)[225:persistence-core-impl:2.7.1]

at org.apache.solr.client.solrj.request.CoreAdminRequest.createCore(CoreAdminRequest.java:550)[225:persistence-core-impl:2.7.1]

at org.codice.solr.factory.SolrServerFactory.createSolrCore(SolrServerFactory.java:330)[225:persistence-core-impl:2.7.1]

at org.codice.solr.factory.SolrServerFactory.getHttpSolrServer(SolrServerFactory.java:140)[225:persistence-core-impl:2.7.1]

at org.codice.solr.factory.SolrServerFactory.getHttpSolrServer(SolrServerFactory.java:118)[225:persistence-core-impl:2.7.1]

at org.codice.ddf.persistence.internal.PersistentStoreImpl.getSolrCore(PersistentStoreImpl.java:288)[225:persistence-core-impl:2.7.1]

at org.codice.ddf.persistence.internal.PersistentStoreImpl.get(PersistentStoreImpl.java:178)[225:persistence-core-impl:2.7.1]

at Proxyf9512e91_caf5_4a97_93b7_d0458748b6cc.get(Unknown Source)

at org.codice.ddf.persistence.events.ActivityListener.<init>(ActivityListener.java:39)

at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)[:1.7.0_79]

at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)[:1.7.0_79]

at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)[:1.7.0_79]

at java.lang.reflect.Constructor.newInstance(Constructor.java:526)[:1.7.0_79]

at org.apache.aries.blueprint.utils.ReflectionUtils.newInstance(ReflectionUtils.java:329)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BeanRecipe.newInstance(BeanRecipe.java:962)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BeanRecipe.getInstance(BeanRecipe.java:331)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BeanRecipe.internalCreate2(BeanRecipe.java:806)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BeanRecipe.internalCreate(BeanRecipe.java:787)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.di.AbstractRecipe$1.call(AbstractRecipe.java:79)[14:org.apache.aries.blueprint.core:1.4.2]

at java.util.concurrent.FutureTask.run(FutureTask.java:262)[:1.7.0_79]

at org.apache.aries.blueprint.di.AbstractRecipe.create(AbstractRecipe.java:88)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.di.RefRecipe.internalCreate(RefRecipe.java:62)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.di.AbstractRecipe.create(AbstractRecipe.java:106)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.ServiceRecipe.createService(ServiceRecipe.java:284)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.ServiceRecipe.internalGetService(ServiceRecipe.java:251)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.ServiceRecipe.internalCreate(ServiceRecipe.java:148)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.di.AbstractRecipe$1.call(AbstractRecipe.java:79)[14:org.apache.aries.blueprint.core:1.4.2]

at java.util.concurrent.FutureTask.run(FutureTask.java:262)[:1.7.0_79]

at org.apache.aries.blueprint.di.AbstractRecipe.create(AbstractRecipe.java:88)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintRepository.createInstances(BlueprintRepository.java:245)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintRepository.createAll(BlueprintRepository.java:183)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintContainerImpl.instantiateEagerComponents(BlueprintContainerImpl.java:682)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintContainerImpl.doRun(BlueprintContainerImpl.java:377)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintContainerImpl.run(BlueprintContainerImpl.java:269)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintExtender.createContainer(BlueprintExtender.java:294)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintExtender.createContainer(BlueprintExtender.java:263)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.blueprint.container.BlueprintExtender.modifiedBundle(BlueprintExtender.java:253)[14:org.apache.aries.blueprint.core:1.4.2]

at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$Tracked.customizerModified(BundleHookBundleTracker.java:500)[9:org.apache.aries.util:1.1.0]

at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$Tracked.customizerModified(BundleHookBundleTracker.java:433)[9:org.apache.aries.util:1.1.0]

at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$AbstractTracked.track(BundleHookBundleTracker.java:725)[9:org.apache.aries.util:1.1.0]

at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$Tracked.bundleChanged(BundleHookBundleTracker.java:463)[9:org.apache.aries.util:1.1.0]

at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$BundleEventHook.event(BundleHookBundleTracker.java:422)[9:org.apache.aries.util:1.1.0]

at org.eclipse.osgi.framework.internal.core.Framework$10.call(Framework.java:1605)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.internal.serviceregistry.ServiceRegistry.notifyHookPrivileged(ServiceRegistry.java:1239)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.internal.serviceregistry.ServiceRegistry.notifyHooksPrivileged(ServiceRegistry.java:1222)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.Framework.notifyEventHooksPrivileged(Framework.java:1602)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.Framework.publishBundleEventPrivileged(Framework.java:1557)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.Framework.publishBundleEvent(Framework.java:1504)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.Framework.publishBundleEvent(Framework.java:1499)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.BundleHost.startWorker(BundleHost.java:391)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.AbstractBundle.resume(AbstractBundle.java:390)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.Framework.resumeBundle(Framework.java:1176)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.StartLevelManager.resumeBundles(StartLevelManager.java:559)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.StartLevelManager.resumeBundles(StartLevelManager.java:544)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.StartLevelManager.incFWSL(StartLevelManager.java:457)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.StartLevelManager.doSetStartLevel(StartLevelManager.java:243)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.StartLevelManager.dispatchEvent(StartLevelManager.java:438)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.internal.core.StartLevelManager.dispatchEvent(StartLevelManager.java:1)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.eventmgr.EventManager.dispatchEvent(EventManager.java:230)[osgi-3.9.1-v20140110-1610.jar:]

at org.eclipse.osgi.framework.eventmgr.EventManager$EventThread.run(EventManager.java:340)[osgi-3.9.1-v20140110-1610.jar:]

Caused by: javax.net.ssl.SSLException: hostname in certificate didn't match: <localhost> != <My FQDN>

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:238)[225:persistence-core-impl:2.7.1]

at org.apache.http.conn.ssl.BrowserCompatHostnameVerifier.verify(BrowserCompatHostnameVerifier.java:54)[225:persistence-core-impl:2.7.1]

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:159)[225:persistence-core-impl:2.7.1]

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:140)[225:persistence-core-impl:2.7.1]

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.verifyHostname(SSLConnectionSocketFactory.java:286)[225:persistence-core-impl:2.7.1]

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:276)[225:persistence-core-impl:2.7.1]

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:254)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.conn.HttpClientConnectionOperator.connect(HttpClientConnectionOperator.java:123)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:318)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:363)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:219)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)[225:persistence-core-impl:2.7.1]

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)[225:persistence-core-impl:2.7.1]

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)[225:persistence-core-impl:2.7.1]

Scott Tustison

unread,

Jul 15, 2015, 11:49:52 AM7/15/15

to ddf-...@googlegroups.com

It sounds like you did not change the Solr URL to use your new hostname. There are a number of configs that will need to be updated as well as the users.properties file.

Mark

unread,

Jul 15, 2015, 11:51:53 AM7/15/15

to ddf-...@googlegroups.com

Is it possible to perform this change since I cannot log in to DDF from a browser?

Keith Wire

unread,

Jul 15, 2015, 12:33:44 PM7/15/15

to Mark, ddf-...@googlegroups.com

It is possible although not trivial if you cannot login to the admin console. I would recommend starting over and be sure to update all the configuration in Step 4 of https://codice.atlassian.net/wiki/display/DDF/Configure+DDF+with+New+Certificates.

You could also configure the "Whitelist contexts" to include "/admin,/system/console" while you are going through these steps. This should allow you to access the admin web consoles even if something is not configured correctly.

--Keith

--
You received this message because you are subscribed to the Google Groups "ddf-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to ddf-users+...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

--

Keith C Wire

303-335-5025

keith....@gmail.com

Mark

unread,

Jul 15, 2015, 5:27:26 PM7/15/15

to ddf-...@googlegroups.com, elihu...@gmail.com

Thanks for the response. I've gone through the steps documented in the Maintenance PDF and the link you provided on 2 separate installs (windows, RHEL) and have experienced the same exception in the logs. I used the Java keytool to verify that the new serverKeystore.jks file does not list "localhost" in the DN. I even ran some find/grep scripts in linux to verify that there are no references to "localhost" in any configuration file.

Could something be cached in the data directory that may cause this? Could I stop DDF, move the data directory to the side and restart DDF to see if that gets rid of the problem?

Thanks,

Mark

unread,

Jul 16, 2015, 12:45:37 PM7/16/15

to ddf-...@googlegroups.com

I removed "PKI" from the Authentication Types on the /solr context in the Web Context Policy Manager. Since I'm using the defaults for all other contexts, PKI will not be used for the entire system. When I restart DDF, I still observe the same exception in the ddf.log file.

Mark

unread,

Jul 16, 2015, 3:44:07 PM7/16/15

to ddf-...@googlegroups.com

I found some information that may help in debugging this problem I'm experiencing. In ddf.log, I see the following exception:

org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://localhost:8993/solr

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507)[353:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)[353:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.request.CoreAdminRequest.process(CoreAdminRequest.java:493)[353:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.request.CoreAdminRequest.getStatus(CoreAdminRequest.java:541)[353:catalog-solr-external-provider:2.7.3]

at org.codice.solr.factory.SolrServerFactory.solrCoreExists(SolrServerFactory.java:351)[353:catalog-solr-external-provider:2.7.3]

at org.codice.solr.factory.SolrServerFactory.createSolrCore(SolrServerFactory.java:323)[353:catalog-solr-external-provider:2.7.3]

at org.codice.solr.factory.SolrServerFactory.getHttpSolrServer(SolrServerFactory.java:140)[353:catalog-solr-external-provider:2.7.3]

at ddf.catalog.solr.external.SolrHttpCatalogProvider.<init>(SolrHttpCatalogProvider.java:113)

at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)[:1.7.0_79]

at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)[:1.7.0_79]

at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)[:1.7.0_79]

at java.lang.reflect.Constructor.newInstance(Constructor.java:526)[:1.7.0_79]

Caused by: javax.net.ssl.SSLException: hostname in certificate didn't match: <localhost> != <MY FQDN>

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:238)[353:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.BrowserCompatHostnameVerifier.verify(BrowserCompatHostnameVerifier.java:54)[353:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:159)[353:catalog-solr-external-provider:2.7.3]

Starting at SolrHttpCatalogProvider line 113, the following constructor is called:

public SolrHttpCatalogProvider(FilterAdapter filterAdapter, SolrFilterDelegateFactory

solrFilterDelegateFactory) {

this(filterAdapter, null, solrFilterDelegateFactory, null);

server = SolrServerFactory

.getHttpSolrServer(url,

SOLR_CATALOG_CORE_NAME,

SOLR_CATALOG_CONFIG_FILE);

}

The variable "url" is a global variable with a value of "SolrServerFactory.DEFAULT_HTTPS_ADDRESS" which is set to "https://localhost:8993/solr". I dug down into the SolrServerFactory.getHttpSolrServer(String,String,String) method. Here's the method:

public static SolrServer getHttpSolrServer(String url, String coreName, String configFile) {

if (StringUtils.isBlank(url)) {

url = DEFAULT_HTTPS_ADDRESS;

}

if (System.getProperty("host") != null && System.getProperty("jetty.port") != null && System

.getProperty("hostContext") != null) {

url = "http://" + System.getProperty("host") + ":" + System.getProperty("jetty.port") +

"/" + StringUtils.stripStart(System.getProperty("hostContext"), "/");

}

SolrServer server;

if (StringUtils.startsWith(url, "https")) {

CloseableHttpClient client = getHttpClient();

createSolrCore(url, coreName, configFile, client);

server = new HttpSolrServer(url + "/" + coreName, client);

} else {

createSolrCore(url, coreName, configFile, null);

server = new HttpSolrServer(url + "/" + coreName);

}

return server;

}

So we know that url is not blank so the first "if" statement does not get assign a default value of "url". While there's a "host" value set in ddf.platform.config.cfg, I could not find where jetty.port or hostContext are set so I don't believe that the second "if" statement will set the url value. If it did, I believe it is still incorrect for use cases where the connection for Solr should be HTTPS because the value is hard-coded to "http". In the interest of brevity, I'll leave out the remainder of the getHttpSolrServer method, as it does not change the url value.

I hope I'm on the right track with trying to track down this problem. Let me know if this all sounds correct or not and if it could be leading to the certificate hostname mismatch in DDF 2.7.0.

Thanks,

Mark

unread,

Jul 17, 2015, 9:26:42 AM7/17/15

to ddf-...@googlegroups.com

To verify that the configuration change for the Catalog Solr External Provider is getting set properly, I ran the command:

config:list "(service.pid=ddf.catalog.solr.external.SolrHttpCatalogProvider)"

and I get back my FQHN in the url. So it is getting set properly and does match my certificate.

Jason Smith

unread,

Jul 17, 2015, 10:17:16 AM7/17/15

to ddf-...@googlegroups.com

Mark -

I just ran through a DDF 2.7.0 install with new certs. I followed the instructions per https://codice.atlassian.net/wiki/display/DDF/Configure+DDF+with+New+Certificates. I was able to leave the web context policy alone. I did notice that after all my configurations were complete, upon startup, you'll see an error about the hostname in the certificate not matching. This appears to occur only at initialization, as everything appears to be working for me once start up is complete (ie, able to ingest, search etc). Let me know if that is not the case for you. I took some notes during the install that I can send, but for the most part the configuration was the same as on the wiki page. If you are changing the alias of the private key or changing the keystore / certificate password,there are some additional steps necessary as documented here: https://codice.atlassian.net/wiki/display/DDF/Configuring+WSS.

- Jason

Mark Webb

unread,

Jul 17, 2015, 10:28:19 AM7/17/15

to Jason Smith, ddf-...@googlegroups.com

Jason,

Thanks for getting back to me. My certs list FQDN and different aliases in them. I'm following the steps on the links:

https://codice.atlassian.net/wiki/display/DDF/Configure+DDF+with+New+Certificates

https://codice.atlassian.net/wiki/display/DDF/Configuring+WSS

I will go through and set up DDF in order to check that I can ingest/search...etc and get back to you.

Cheers,

Mark

--
You received this message because you are subscribed to a topic in the Google Groups "ddf-users" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/ddf-users/Mqlg5hqT8zQ/unsubscribe.
To unsubscribe from this group and all its topics, send an email to ddf-users+...@googlegroups.com.

Jason Smith

unread,

Jul 17, 2015, 10:41:22 AM7/17/15

to ddf-...@googlegroups.com, jason.l...@gmail.com

Here are my detailed notes. Hope this helps:

First, it is important to know the values of the private key entry in the NEW keystore. For my example, I will use the following:
keystore private key alias: ddf
keystore private key FQDN/CN: ddf
Start by unzip and configuring DDF before installing your new keystore:
- unzip distro
- run ddf.sh
- Walk through https://localhost:8993/admin
- update the configurations per https://codice.atlassian.net/wiki/display/DDF/Configure+DDF+with+New+Certificates. Make sure that the hostname matches the FQDN in the new certificate you will be using
- note that if you are changing the keystore alias from "localhost" (to "ddf" in my example), you'll need to change the signature and username properties in the STS Server config. See screenshot:

- shutodown ddf
- Create a new self signed cert
You can reference https://codice.atlassian.net/wiki/display/DDF/Configure+DDF+with+New+Certificates, however I tweaked the scripts to use a non-localhost alias (ddf)
FQDN/CN: ddf
alias: ddf
- change the "localhost" line in <DDF_HOME/etc/user.properties:
## from
localhost=localhost,group,admin,manager,viewer,webconsole
## To
<FQDN>=<FQDN>,group,admin,manager,viewer,webconsole
- copy your new keystore to <DDF_HOME>/etc/serverKeystore.jks
- Additional step - if you changed the alias like I did (or keystore password), you'll need to update the following files:
<DDF_HOME>/etc/ws-security/issuer/signature.properties
<DDF_HOME>/etc/ws-security/issuer/encryption.properties
<DDF_HOME>/etc/ws-security/server/encryption.properties
<DDF_HOME>/etc/ws-security/server/signature.properties
This is covered in more detail here https://codice.atlassian.net/wiki/display/DDF/Configuring+WSS

- start DDF
- I noticed an error about the hostname not matching the one in the certificate. This appeared only at startup, so I think it is a non-issue.

Mark Webb

unread,

Jul 17, 2015, 11:47:03 AM7/17/15

to Jason Smith, ddf-...@googlegroups.com

Jason,

I tried to post data to the catalog using the command (Note that I needed to add the "-k" option for certificate checking):

curl -k -H "Content-type: application/json;id=geojson" -i -X POST -d @geojson_valid.json http://<FQDN>:8993/services/catalog

And received a 500 error message. I've attached the ddf.log file.

ddf.log

Jason Smith

unread,

Jul 17, 2015, 12:05:20 PM7/17/15

to ddf-...@googlegroups.com, jason.l...@gmail.com

Hmm... this worked for me. Note that I think you had a typo below - should be https://<FQDN>:8993/services/catalog.

Based on the logs, I think there must be an issue in the STS server configuration. Can you double check that following is correct in the STS server config?

signature username must match the private key alias
encryption username must match the private key alias
token issuer must match the FQDN

Also, can you verify your keystore? Here is mine:

keytool -list -keystore etc/keystores/serverKeystore.jks -v -alias ddf

Alias name: ddf

Creation date: Jul 17, 2015

Entry type: PrivateKeyEntry

Owner: EMAILADDRESS=d...@example.org, CN=ddf, OU=Dev, O=DDF, L=Goodyear, ST=AZ, C=US

Note that the CN must equal your FQDN.

- Jason

Jason Smith

unread,

Jul 17, 2015, 12:26:51 PM7/17/15

to ddf-...@googlegroups.com

For some reason I read Solr Server and was thinking STS Server. I am guessing you have checked the following configurations already and verified the FQDN is correct?

- Catalog External Solr Catalog Provider

- Platform Global Config

- Persistent Store

- Catalog Federation Strategy

- Jason

Mark

unread,

Jul 17, 2015, 2:36:26 PM7/17/15

to ddf-...@googlegroups.com

I was using https, the host that I ran the test on is different than the host I am posting to this group from. Sorry for the confusion.

I checked the serverKeystore.jks file, the CN is set to my FQDN. I checked the STS Server name and it is matched to the key alias.

...

Mark

unread,

Jul 17, 2015, 2:37:41 PM7/17/15

to ddf-...@googlegroups.com

I verified that all the settings you listed below are set to the FQDN.

Mark

unread,

Jul 20, 2015, 1:23:20 PM7/20/15

to ddf-...@googlegroups.com

I went through the steps on a clean install and am able to connect to /admin, but when I try and go to /search I see the following in the logs:

org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://localhost:8993/solr/catalog

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507)[345:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)[345:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.request.SolrPing.process(SolrPing.java:70)[345:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.SolrServer.ping(SolrServer.java:293)[345:catalog-solr-external-provider:2.7.3]

at ddf.catalog.solr.external.SolrHttpCatalogProvider.isServerUp(SolrHttpCatalogProvider.java:281)[345:catalog-solr-external-provider:2.7.3]

at ddf.catalog.solr.external.SolrHttpCatalogProvider.getProvider(SolrHttpCatalogProvider.java:256)[345:catalog-solr-external-provider:2.7.3]

at ddf.catalog.solr.external.SolrHttpCatalogProvider.isAvailable(SolrHttpCatalogProvider.java:147)[345:catalog-solr-external-provider:2.7.3]

at Proxyf31fe548_4587_4259_ada4_7faa93d22d98.isAvailable(Unknown Source)[:]

at ddf.catalog.util.impl.CachedSource.checkStatus(CachedSource.java:159)[302:catalog-core-standardframework:2.7.3]

at ddf.catalog.util.impl.SourcePollerRunner$1.run(SourcePollerRunner.java:109)[302:catalog-core-standardframework:2.7.3]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)[:1.7.0_79]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)[:1.7.0_79]

at java.lang.Thread.run(Thread.java:745)[:1.7.0_79]

Caused by: javax.net.ssl.SSLException: hostname in certificate didn't match: <localhost> != <FQDN>

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:238)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.BrowserCompatHostnameVerifier.verify(BrowserCompatHostnameVerifier.java:54)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:159)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:140)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.verifyHostname(SSLConnectionSocketFactory.java:286)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:276)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:254)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.conn.HttpClientConnectionOperator.connect(HttpClientConnectionOperator.java:123)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:318)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:363)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:219)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)[345:catalog-solr-external-provider:2.7.3]

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)[345:catalog-solr-external-provider:2.7.3]

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)[345:catalog-solr-external-provider:2.7.3]

... 12 more

On Wednesday, July 15, 2015 at 11:16:24 AM UTC-4, Mark wrote:

Mark

unread,

Jul 20, 2015, 1:56:23 PM7/20/15

to ddf-...@googlegroups.com

I restarted DDF and was able to connect to /search (not sure what caused DDF to start working yet). I then ran the command to test out the ingest so I'd have something to search against:

curl -k -H "Content-type: application/json;id=geojson" -i -X POST -d @geojson_valid.json https://<FQDN>:8993/services/catalog

and received this stack trace:

org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://localhost:8993/solr/catalog

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507)

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)

at org.apache.solr.client.solrj.request.SolrPing.process(SolrPing.java:70)

at org.apache.solr.client.solrj.SolrServer.ping(SolrServer.java:293)

at ddf.catalog.solr.external.SolrHttpCatalogProvider.isServerUp(SolrHttpCatalogProvider.java:281)

at ddf.catalog.solr.external.SolrHttpCatalogProvider.getProvider(SolrHttpCatalogProvider.java:256)

at ddf.catalog.solr.external.SolrHttpCatalogProvider.create(SolrHttpCatalogProvider.java:186)

at ddf.catalog.impl.CatalogFrameworkImpl.create(CatalogFrameworkImpl.java:828)

at Proxyc270e5ec_a4da_461d_bfcc_cb451826ca14.create(Unknown Source)

at org.codice.ddf.endpoints.rest.RESTEndpoint.addDocument(RESTEndpoint.java:646)[310:catalog-rest-endpoint:2.7.3]

at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.7.0_79]

at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)[:1.7.0_79]

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[:1.7.0_79]

at java.lang.reflect.Method.invoke(Method.java:606)[:1.7.0_79]

at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:181)[130:org.apache.cxf.cxf-core:3.0.4]

at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:97)[130:org.apache.cxf.cxf-core:3.0.4]

at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:200)[145:org.apache.cxf.cxf-rt-frontend-jaxrs:3.0.4]

at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:99)[145:org.apache.cxf.cxf-rt-frontend-jaxrs:3.0.4]

at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)[130:org.apache.cxf.cxf-core:3.0.4]

at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96)[130:org.apache.cxf.cxf-core:3.0.4]

at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:307)[130:org.apache.cxf.cxf-core:3.0.4]

at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)[130:org.apache.cxf.cxf-core:3.0.4]

at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:251)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:171)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:293)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:212)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at javax.servlet.http.HttpServlet.service(HttpServlet.java:595)[54:org.apache.geronimo.specs.geronimo-servlet_3.0_spec:1.0.0]

at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:268)[138:org.apache.cxf.cxf-rt-transports-http:3.0.4]

at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)

at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.codice.ddf.platform.filter.delegate.ProxyFilterChain.doFilter(ProxyFilterChain.java:110)[70:platform-filter-delegate:2.7.1]

at org.codice.ddf.security.filter.authorization.AuthorizationFilter.doFilter(AuthorizationFilter.java:125)

at org.codice.ddf.platform.filter.delegate.ProxyFilterChain.doFilter(ProxyFilterChain.java:106)[70:platform-filter-delegate:2.7.1]

at org.codice.ddf.security.filter.login.LoginFilter$2$1.run(LoginFilter.java:259)[259:security-filter-login:2.7.1]

at org.codice.ddf.security.filter.login.LoginFilter$2$1.run(LoginFilter.java:256)[259:security-filter-login:2.7.1]

at java.security.AccessController.doPrivileged(Native Method)[:1.7.0_79]

at javax.security.auth.Subject.doAs(Subject.java:415)[:1.7.0_79]

at org.codice.ddf.security.filter.login.LoginFilter$2.call(LoginFilter.java:269)[259:security-filter-login:2.7.1]

at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)[209:org.apache.shiro.core:1.2.3]

at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)[209:org.apache.shiro.core:1.2.3]

at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)[209:org.apache.shiro.core:1.2.3]

at org.codice.ddf.security.filter.login.LoginFilter.doFilter(LoginFilter.java:252)[259:security-filter-login:2.7.1]

at org.codice.ddf.platform.filter.delegate.ProxyFilterChain.doFilter(ProxyFilterChain.java:106)[70:platform-filter-delegate:2.7.1]

at org.codice.ddf.security.filter.websso.WebSSOFilter.handleRequest(WebSSOFilter.java:222)[256:security-filter-web-sso:2.7.1]

at org.codice.ddf.security.filter.websso.WebSSOFilter.doFilter(WebSSOFilter.java:132)[256:security-filter-web-sso:2.7.1]

at org.codice.ddf.platform.filter.delegate.ProxyFilterChain.doFilter(ProxyFilterChain.java:106)[70:platform-filter-delegate:2.7.1]

at org.codice.ddf.platform.filter.delegate.DelegateServletFilter.doFilter(DelegateServletFilter.java:102)[70:platform-filter-delegate:2.7.1]

at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1476)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.ops4j.pax.web.service.jetty.internal.HttpServiceServletHandler.doHandle(HttpServiceServletHandler.java:69)[69:org.ops4j.pax.web.pax-web-jetty:3.1.2]

at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.ops4j.pax.web.service.jetty.internal.HttpServiceContext.doHandle(HttpServiceContext.java:240)[69:org.ops4j.pax.web.pax-web-jetty:3.1.2]

at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.ops4j.pax.web.service.jetty.internal.JettyServerHandlerCollection.handle(JettyServerHandlerCollection.java:77)[69:org.ops4j.pax.web.pax-web-jetty:3.1.2]

at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.Server.handle(Server.java:370)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:982)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1043)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.io.nio.SslConnection.handle(SslConnection.java:196)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)[60:org.eclipse.jetty.aggregate.jetty-all-server:8.1.15.v20140411]

at java.lang.Thread.run(Thread.java:745)[:1.7.0_79]

Caused by: javax.net.ssl.SSLException: hostname in certificate didn't match: <localhost> != <FQDN>

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:238)

at org.apache.http.conn.ssl.BrowserCompatHostnameVerifier.verify(BrowserCompatHostnameVerifier.java:54)

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:159)

at org.apache.http.conn.ssl.AbstractVerifier.verify(AbstractVerifier.java:140)

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.verifyHostname(SSLConnectionSocketFactory.java:286)

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:276)

at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:254)

at org.apache.http.impl.conn.HttpClientConnectionOperator.connect(HttpClientConnectionOperator.java:123)

at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:318)

at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:363)

at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:219)

at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)

at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)

at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)

at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)

at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)

... 76 more

java.lang.IllegalArgumentException: Solr Server is not connected. Please check the Solr Server status or url, and then retry.

at ddf.catalog.solr.external.SolrHttpCatalogProvider$UnconfiguredCatalogProvider.create(SolrHttpCatalogProvider.java:356)