Puppetmaster setup with separate CA server configuration help

[linuxbsdfreak]

Hi All,

I am setting up puppetmaster with nginx and passenger and separating
the Puppetmaster primary CA server. I have 3 host

loadbalancer01 - Nginx doing LB on IP address and also running
puppetmaster with passenger under 127.0.0.1 (port 8140).
primaryca - Puppetmaster Primary CA
pclient - Puppet Client

The did the following steps:

On Primary CA server:
----------------------------

cd /var/lib/puppet/

puppetca generate
puppet cert -g loadbalancer01 --
certdnsnames=puppetmaster:loadbalancer01
puppet cert -g primaryca --certdnsnames=puppetmaster:primaryca

scp private_keys/loadbalancer01.pem root@loadbalancer01:/var/lib/
puppet/ssl/private_keys/
scp ca/signed/loadbalancer01.pem root@loadbalancer01:/var/lib/puppet/
ssl/certs/
scp ca/ca_crt.pem root@loadbalancer01:/var/lib/puppet/ssl/certs/ca.pem

On loadbalancer01
----------------------
cd /var/lib/puppet
mkdir -p ssl/private_keys ssl/certs

Following are the nginx/puppet.conf configs of loadbalancer01
----------------------------------------------------------------------------------

[main]
logdir = /var/log/puppet
rundir = /var/run/puppet
ssldir = $vardir/ssl

[agent]
classfile = $vardir/classes.txt
localconfig = $vardir/localconfig
server = loadbalancer01
listen = true

[master]
ssl_client_header = SSL_CLIENT_S_DN
ssl_client_verify_header = SSL_CLIENT_VERIFY
ca = false

-------------
Nginx.conf
---------------
user nginx;
worker_processes 10;
worker_rlimit_nofile 100000;

error_log logs/error.log info;
pid logs/nginx.pid;

events {
worker_connections 1024;
use epoll;
}

http {

passenger_root /usr/lib/ruby/gems/1.8/gems/passenger-3.0.7;
passenger_ruby /usr/bin/ruby;

include mime.types;
default_type application/octet-stream;

## Common options ##
include options.conf;

## Proxy settings ##
include proxy.conf;

## lb domains ##
include puppetmasterlb.conf;

include lb01.conf;
}

--------- lb01.conf -------------

server {
listen 127.0.0.1:8140;
# server_name loadbalancer01;
root /etc/puppet/puppetmaster/public;
passenger_enabled on;
passenger_use_global_queue on;
rails_env production;

ssl off;

access_log /var/log/puppet/lb01_access.log;
error_log /var/log/puppet/lb01_error.log;
passenger_set_cgi_param SSL_CLIENT_S_DN
$ssl_client_s_dn;
passenger_set_cgi_param SSL_CLIENT_VERIFY
$ssl_client_verify;

# make sure we serve everything as raw
types { }
default_type application/x-raw;

# serve static file for the [files] mountpoint
location /production/file_content/files/ {
allow 192.168.122.0/24;
allow all;

alias /etc/puppet/files/;
}

# serve modules files sections
location ~ /production/file_content/[^/]+/files/ {
allow 192.168.122.0/24;
allow all;

root /etc/puppet/modules;

# rewrite /production/file_content/module/files/file.txt to /
module/file.text
rewrite ^/production/file_content/([^/]+)/files/(.+)$
$1/$2 break;
}
}

--------------- puppetmasterlb.conf ----------

# Backend puppetmasters and ca servers

upstream puppetmasters {
server 127.0.0.1:8140;
# server pserver02:8140;
}

upstream puppetca {
server primaryca:8140;
server secondaryca:8140 backup;
}

server {
listen 192.168.122.14:8140;
server_name loadbalancer01;

ssl on;
ssl_session_timeout 5m;
ssl_certificate /var/lib/puppet/ssl/certs/loadbalancer01.pem;
ssl_certificate_key /var/lib/puppet/ssl/private_keys/
loadbalancer01.pem;
ssl_client_certificate /var/lib/puppet/ssl/certs/ca.pem;
ssl_protocols SSLv3 TLSv1;
ssl_ciphers ALL:!ADH:RC4+RSA:+HIGH:+MEDIUM:-LOW:-SSLv2:-EXP;
ssl_prefer_server_ciphers on;
ssl_verify_client optional;
ssl_verify_depth 1;

access_log logs/loadbalancer01_access.log main;
error_log logs/loadbalancer01_error.log;

proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Client-Verify $ssl_client_verify;
proxy_set_header X-Client-DN $ssl_client_s_dn;
proxy_set_header X-SSL-Subject $ssl_client_s_dn;
proxy_set_header X-SSL-Issuer $ssl_client_i_dn;
proxy_buffer_size 16k;
proxy_buffers 8 32k;
proxy_busy_buffers_size 64k;
proxy_temp_file_write_size 64k;
proxy_read_timeout 65;

location ^~ /production/certificate/ca {
proxy_pass https://puppetca;
}
location ^~ /production/certificate {
proxy_pass https://puppetca;
}

location ^~ /production/certificate_revocation_list/ca {
proxy_pass https://puppetca;
}

location ^~ / {
proxy_pass http://puppetmasters;

}

}

--- nginx.conf of Primary CA ---------

user nginx;
worker_processes 10;
worker_rlimit_nofile 100000;

error_log logs/error.log info;
pid logs/nginx.pid;

events {
worker_connections 1024;
use epoll;
}

http {
passenger_root /usr/lib/ruby/gems/1.8/gems/passenger-3.0.7;
passenger_ruby /usr/bin/ruby;

include mime.types;
default_type application/octet-stream;

log_format main '$remote_addr - $remote_user [$time_local]
"$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';

access_log logs/access.log main;

sendfile on;

# These are good default values.
tcp_nopush on;
tcp_nodelay off;

server_tokens off;

keepalive_timeout 65;
server {
listen 192.168.122.17:8140;
server_name primaryca;
root /etc/puppet/puppetmaster/public;
passenger_enabled on;
passenger_use_global_queue on;
rails_env production;

ssl on;
ssl_session_timeout 5m;
ssl_certificate /var/lib/puppet/ssl/certs/primaryca.pem;
ssl_certificate_key /var/lib/puppet/ssl/private_keys/
primaryca.pem;
ssl_client_certificate /var/lib/puppet/ssl/ca/ca_crt.pem;
ssl_crl /var/lib/puppet/ssl/ca/ca_crl.pem;
ssl_protocols SSLv3 TLSv1;
ssl_ciphers ALL:!ADH:RC4+RSA:+HIGH:+MEDIUM:-LOW:-SSLv2:-EXP;
ssl_prefer_server_ciphers on;
ssl_verify_depth 1;

access_log /var/log/puppet/primaryca_access.log;
error_log /var/log/puppet/primaryca_error.log;

passenger_set_cgi_param SSL_CLIENT_S_DN
$ssl_client_s_dn;
passenger_set_cgi_param SSL_CLIENT_VERIFY
$ssl_client_verify;

}
}

--- puppet.conf Primary CA ----

[main]
logdir = /var/log/puppet
rundir = /var/run/puppet
ssldir = $vardir/ssl


[agent]
classfile = $vardir/classes.txt
localconfig = $vardir/localconfig
server = primaryca
listen = true

[master]
autosign = true
ssl_client_header = SSL_CLIENT_S_DN
ssl_client_verify_header = SSL_CLIENT_VERIFY
certname = puppetmaster
ca = true


=========

When i sent a request from the puppet client machine(pclient). The CSR
gets signed by the primary ca server by checking

puppet cert --list --all

+ loadbalancer01 (A2:86:20:5A:47:EE:4F:6A:1D:66:94:CB:33:0E:F4:BE)
+ pclient (7E:D2:31:DC:5F:DB:B1:E1:90:82:46:69:2E:2C:28:BF)
+ primaryca (A0:B5:7F:6B:CC:0A:F7:A6:1E:7D:B4:E4:09:8D:47:17)
+ puppetmaster (56:62:43:FA:9C:67:F0:E7:DF:EB:9A:A1:71:1D:57:BB)

However i get the following error while trying to receive the catalog:

notice: Ignoring --listen on onetime run
err: Could not retrieve catalog from remote server: Error 403 on
SERVER: Forbidden request: pclient(192.168.122.12) access to /
certificate_revocation_list/ca [find] at line 93
warning: Not using cache on failed catalog

- I think this may be related to the auth.conf

--- Auth.conf on loadbalancer01

# allow nodes to retrieve their own catalog (ie their configuration)
path ~ ^/catalog/([^/]+)$
method find
allow $1

# allow all nodes to access the certificates services
path /certificate_revocation_list/ca
method find
allow *

# allow all nodes to store their reports
path /report
method save
allow *

# inconditionnally allow access to all files services
# which means in practice that fileserver.conf will
# still be used
path /file
allow *

### Unauthenticated ACL, for clients for which the current master
doesn't
### have a valid certificate

# allow access to the master CA
path /certificate/ca
auth no
method find
allow *

path /certificate/
auth no
method find
allow *

path /certificate_request
auth any
method find, save
allow *

# this one is not stricly necessary, but it has the merit
# to show the default policy which is deny everything else
path /
auth any

--- auth.conf on primaryca

# allow nodes to retrieve their own catalog (ie their configuration)
path ~ ^/catalog/([^/]+)$
method find
allow $1

# allow all nodes to access the certificates services
path /certificate_revocation_list/ca
method find
allow *

# allow all nodes to store their reports
path /report
method save
allow *

# inconditionnally allow access to all files services
# which means in practice that fileserver.conf will
# still be used
path /file
allow *

### Unauthenticated ACL, for clients for which the current master
doesn't
### have a valid certificate

# allow access to the master CA
path /certificate/ca
auth no
method find
allow *

path /certificate/
auth no
method find
allow *

path /certificate_request
auth any
method find, save
allow *

# this one is not stricly necessary, but it has the merit
# to show the default policy which is deny everything else
path /
auth any

- Have i configured anythng wrong.

---- I am trying to split the CA requests to the primaryCA server and
the non CA requests to the puppetmaster running on the localhost.

I had read in the Pro puppet book that this can be done in Apache with
the following directives.

# Ordering of ProxyPass directives is important
# Direct all Puppet agent CA requests to a specific set of workers.
ProxyPassMatch ^(/.*?)/(certificate.*?)/(.*)$ balancer://puppetmasterca
ProxyPassReverse ^(/.*?)/(certificate.*?)/(.*)$ balancer://puppetmasterca
# Direct all other Puppet agent requests to the default set of
workers.
ProxyPass / balancer://puppetmaster/
ProxyPassReverse / balancer://puppetmaster/
ProxyPreserveHost On

However i would like to accomplish with nginx. Could any one help me
with this and also with the above error.

Regards,
Kevin

[linuxbsdfreak]

Hi All

Has anyone able to sucessfully split puppet CA requests from the
normal catalog requests without specifying ca_server option in
puppet.conf using the configuration mentioned below.

Regards,
Kevin

>                 proxy_passhttp://puppetmasters;

[Marc Fournier]

Kevin,

I'm not sure I perfectly understand your setup but I reckon your problem
is that ...

Excerpts from linuxbsdfreak's message of Fri Jul 08 14:48:19 +0200 2011:
> [...]

> Following are the nginx/puppet.conf configs of loadbalancer01
> ----------------------------------------------------------------------------------
>
> [main]
> logdir = /var/log/puppet
> rundir = /var/run/puppet
> ssldir = $vardir/ssl
>
> [agent]
> classfile = $vardir/classes.txt
> localconfig = $vardir/localconfig
> server = loadbalancer01
> listen = true
>
> [master]
> ssl_client_header = SSL_CLIENT_S_DN
> ssl_client_verify_header = SSL_CLIENT_VERIFY
> ca = false

... SSL_CLIENT_S_DN and SSL_CLIENT_VERIFY don't exist because nginx and
puppetmaster don't speak SSL anymore at this point, only plain HTTP.

Referring to your nginx configuration below:

> --------------- puppetmasterlb.conf ----------
> [...]

> proxy_redirect off;
> proxy_set_header Host $host;
> proxy_set_header X-Real-IP $remote_addr;
> proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
> proxy_set_header X-Client-Verify $ssl_client_verify;
> proxy_set_header X-Client-DN $ssl_client_s_dn;
> proxy_set_header X-SSL-Subject $ssl_client_s_dn;
> proxy_set_header X-SSL-Issuer $ssl_client_i_dn;

... you are setting the values the puppetmaster needs in the
X-Client-Verify and X-Client-DN HTTP headers. Therefore, you could try
changing your puppetmaster settings to read:

[master]
ssl_client_header = HTTP_X_CLIENT_DN
ssl_client_verify_header = HTTP_X_CLIENT_VERIFY

> However i get the following error while trying to receive the catalog:
>
> notice: Ignoring --listen on onetime run
> err: Could not retrieve catalog from remote server: Error 403 on
> SERVER: Forbidden request: pclient(192.168.122.12) access to /
> certificate_revocation_list/ca [find] at line 93
> warning: Not using cache on failed catalog
>
> - I think this may be related to the auth.conf

I would suspect that this error is only a side effect. Your client is
not considered authenticated by the master and therefore gets denied
access to whatever resource it is asking for.

> --- Auth.conf on loadbalancer01
> [...]

> # allow all nodes to access the certificates services
> path /certificate_revocation_list/ca
> method find
> allow *
>

No "auth no" found here, which would explain...

Hope this helps !

Cheers,
Marc