Salt stastes are not reading my /etc/hosts

ronna mijares

unread,

Apr 12, 2016, 4:26:13 PM4/12/16

to Salt-users

Hi folks

I am having a problem running states of salt.

I have my /etc/hosts file with the name of my internal server and my DNS using google:

/etc/hosts:

10.0.1.2 server1

/etc/resolv.conf:

nameserver 8.8.8.8

Apparently if a run a command to one of my server, salt should look for it in my /etc/hosts, but not It looks for it in my resolv.conf taking more than 25 seconds with just one consulting.

I am checking this with the command strace.

strace salt -N 'server-group' cmd.run "grep server1 /etc/hosts" -l debug

.

poll([{fd=3, events=POLLIN}], 1, 4999) = 1 ([{fd=3, revents=POLLIN}])

ioctl(3, FIONREAD, [38]) = 0

recvfrom(3, "&=\201\202\0\1\0\0\0\0\0\0\10salt-STG\4yapo\3int\2c"..., 2048, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("8.8.8.8")}, [16]) = 38

poll([{fd=3, events=POLLIN}], 1, 3314) = 1 ([{fd=3, revents=POLLIN}])

ioctl(3, FIONREAD, [38]) = 0

recvfrom(3, "\206\205\201\202\0\1\0\0\0\0\0\0\10salt-STG\4yapo\3int\2c"..., 65536, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("8.8.8.8")}, [16]) = 38

poll([{fd=3, events=POLLOUT}], 1, 0) = 1 ([{fd=3, revents=POLLOUT}])

sendto(3, "&=\1\0\0\1\0\0\0\0\0\0\10salt-STG\4yapo\3int\2c"..., 38, MSG_NOSIGNAL, NULL, 0) = 38

poll([{fd=3, events=POLLIN|POLLOUT}], 1, 5000) = 1 ([{fd=3, revents=POLLOUT}])

sendto(3, "\206\205\1\0\0\1\0\0\0\0\0\0\10salt-STG\4yapo\3int\2c"..., 38, MSG_NOSIGNAL, NULL, 0) = 38

poll([{fd=3, events=POLLIN}], 1, 4999) = 1 ([{fd=3, revents=POLLIN}])

ioctl(3, FIONREAD, [38]) = 0

recvfrom(3, "\206\205\201\202\0\1\0\0\0\0\0\0\10salt-STG\4yapo\3int\2c"..., 2048, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("8.8.8.8")}, [16]) = 38

poll([{fd=3, events=POLLIN}], 1, 3308) = 1 ([{fd=3, revents=POLLIN}])

recvfrom(3, "&=\201\202\0\1\0\0\0\0\0\0\10salt-STG\4yapo\3int\2c"..., 65536, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("8.8.8.8")}, [16]) = 38

close(3) = 0

.

Bit if I ping to any of this server, it use my /etc/hosts file:

# strace ping server1

::::

.

open("/etc/resolv.conf", O_RDONLY) = 4

fstat(4, {st_mode=S_IFREG|0644, st_size=68, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0e87e7e000

read(4, "# Generated by NetworkManager\nse"..., 4096) = 68

read(4, "", 4096) = 0

close(4) = 0

munmap(0x7f0e87e7e000, 4096) = 0

socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 4

connect(4, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)

close(4) = 0

socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 4

connect(4, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)

close(4) = 0

open("/etc/nsswitch.conf", O_RDONLY) = 4

fstat(4, {st_mode=S_IFREG|0644, st_size=1688, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0e87e7e000

read(4, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1688

read(4, "", 4096) = 0

close(4) = 0

munmap(0x7f0e87e7e000, 4096) = 0

open("/etc/ld.so.cache", O_RDONLY) = 4

fstat(4, {st_mode=S_IFREG|0644, st_size=25590, ...}) = 0

mmap(NULL, 25590, PROT_READ, MAP_PRIVATE, 4, 0) = 0x7f0e87e78000

close(4) = 0

open("/lib64/libnss_files.so.2", O_RDONLY) = 4

read(4, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360!\0\0\0\0\0\0"..., 832) = 832

fstat(4, {st_mode=S_IFREG|0755, st_size=65928, ...}) = 0

mmap(NULL, 2151824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 4, 0) = 0x7f0e815fd000

mprotect(0x7f0e81609000, 2097152, PROT_NONE) = 0

mmap(0x7f0e81809000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 4, 0xc000) = 0x7f0e81809000

close(4) = 0

mprotect(0x7f0e81809000, 4096, PROT_READ) = 0

munmap(0x7f0e87e78000, 25590) = 0

open("/etc/host.conf", O_RDONLY) = 4

fstat(4, {st_mode=S_IFREG|0644, st_size=9, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0e87e7e000

read(4, "multi on\n", 4096) = 9

read(4, "", 4096) = 0

close(4) = 0

munmap(0x7f0e87e7e000, 4096) = 0

open("/etc/hosts", O_RDONLY|O_CLOEXEC) = 4

fcntl(4, F_GETFD) = 0x1 (flags FD_CLOEXEC)

fstat(4, {st_mode=S_IFREG|0644, st_size=1278, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0e87e7e000

read(4, "127.0.0.1 localhost localhost."..., 4096) = 1278

read(4, "", 4096) = 0

close(4) = 0

munmap(0x7f0e87e7e000, 4096) = 0

socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4

connect(4, {sa_family=AF_INET, sin_port=htons(1025), sin_addr=inet_addr("10.0.1.2")}, 16) = 0

getsockname(4, {sa_family=AF_INET, sin_port=htons(39525), sin_addr=inet_addr("10.0.1.110")}, [16]) = 0

close(4) = 0

setsockopt(3, SOL_IP, IP_RECVERR, [1], 4) = 0

setsockopt(3, SOL_SOCKET, SO_SNDBUF, [324], 4) = 0

setsockopt(3, SOL_SOCKET, SO_RCVBUF, [65536], 4) = 0

getsockopt(3, SOL_SOCKET, SO_RCVBUF, [131072], [4]) = 0

fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0e87e7e000

write(1, "PING ch4stg.yapo.int.cl (10.45.1"..., 59PING ch4stg.yapo.int.cl (10.45.1.20) 56(84) bytes of data.

) = 59

setsockopt(3, SOL_SOCKET, SO_TIMESTAMP, [1], 4) = 0

setsockopt(3, SOL_SOCKET, SO_SNDTIMEO, "\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 0

setsockopt(3, SOL_SOCKET, SO_RCVTIMEO, "\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 0

rt_sigaction(SIGINT, {0x7f0e87e878e0, [], SA_RESTORER|SA_INTERRUPT, 0x7f0e876cd6a0}, NULL, 8) = 0

rt_sigaction(SIGALRM, {0x7f0e87e878e0, [], SA_RESTORER|SA_INTERRUPT, 0x7f0e876cd6a0}, NULL, 8) = 0

rt_sigaction(SIGQUIT, {0x7f0e87e878f0, [], SA_RESTORER|SA_INTERRUPT, 0x7f0e876cd6a0}, NULL, 8) = 0

ioctl(1, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, {B9600 opost isig icanon echo ...}) = 0

ioctl(1, TIOCGWINSZ, {ws_row=71, ws_col=270, ws_xpixel=1890, ws_ypixel=994}) = 0

sendmsg(3, {msg_name(16)={sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("10.0.1.2")}, msg_iov(1)=[{"\10\0\240e\256\25\0\1\21Y\rW\0\0\0\0\275\0\17\0\0\0\0\0\20\21\22\23\24\25\26\27"..., 64}], msg_controllen=0, msg_flags=0}, 0) = 64

recvmsg(3, {msg_name(16)={sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("10.0.1.2)}, msg_iov(1)=[{"E\0\0T\ve\0\0@\1Xi\n-\1\24\n-\1n\0\0\250e\256\25\0\1\21Y\rW"..., 192}], msg_controllen=32, {cmsg_len=32, cmsg_level=SOL_SOCKET, cmsg_type=0x1d /* SCM_??? */, ...}, msg_flags=0}, 0) = 84

open("/etc/hosts", O_RDONLY|O_CLOEXEC) = 4

fstat(4, {st_mode=S_IFREG|0644, st_size=1278, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0e87e7d000

read(4, "127.0.0.1 localhost localhost."..., 4096) = 1278

close(4) = 0

munmap(0x7f0e87e7d000, 4096) = 0

write(1, "64 bytes from server1"..., 7964 bytes from server1 (10.0.1.2): icmp_seq=1 ttl=64 time=0.246 ms

) = 79

recvmsg(3, 0x7ffec2651160, 0) = -1 EAGAIN (Resource temporarily unavailable)

sendmsg(3, {msg_name(16)={sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("10.45.1.20")}, msg_iov(1)=[{"\10\0\3b\256\25\0\2\22Y\rW\0\0\0\0Y\3\17\0\0\0\0\0\20\21\22\23\24\25\26\27"..., 64}], msg_controllen=0, msg_flags=0}, MSG_CONFIRM) = 64

recvmsg(3, {msg_name(16)={sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("10.45.1.20")}, msg_iov(1)=[{"E\0\0T\vf\0\0@\1Xh\n-\1\24\n-\1n\0\0\vb\256\25\0\2\22Y\rW"..., 192}], msg_controllen=32, {cmsg_len=32, cmsg_level=SOL_SOCKET, cmsg_type=0x1d /* SCM_??? */, ...}, msg_flags=0}, 0) = 84

write(1, "64 bytes from server1..., 7964 bytes from server1 (10.45.1.20): icmp_seq=2 ttl=64 time=0.285 ms

Thanks

David Boucha

unread,

Apr 13, 2016, 1:28:26 PM4/13/16

to salt users list

Salt doesn't use the hostname of your server. By default Salt uses the minion id for targeting. In fact, the minions connect to the master in a pub/sub architecture and listen for published commands. The minion then reviews the target and executes the command if the minion matches the target.

So in your example salt -N 'server-group' cmd.run "grep server1 /etc/hosts" -l debug all minions will see that command, then the minions that you've listed in the nodegroup "server-group" will see that they match and then execute the command.

--
You received this message because you are subscribed to the Google Groups "Salt-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to salt-users+...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

Message has been deleted

ronna mijares

unread,

Apr 14, 2016, 3:13:20 PM4/14/16

to Salt-users

Hi David

Thanks a lot for your answer. I think I miss something in my question.

I have in my salt master configuration, the hostname of my servers:

nodegroups:

re: 'srv9.cl'

db: 'srv7.cl,srv6.cl'

bc: 'srv40.cl'

/etc/host

10.0.1.3 srv9.cl

10.0.1.4 srv6.cl

10.0.1.5 srv7.cl

10.0.1.6 srv40.cl

So, when I executed a salt command to any of my minions, the strace command shows me that it is looking for this name in my DNS, but I have this hostnames in my /etc/hosts.

So, when this happened any salt command can take more than 20 second for each server.

# time salt -N 'bc' test.ping

srv40stg.cl:

True

real 0m22.423s

Can you help me?

Thanks

Dimitri Maziuk

unread,

Apr 14, 2016, 3:40:16 PM4/14/16

to salt-...@googlegroups.com

On 04/14/2016 02:13 PM, 'ronna mijares' via Salt-users wrote:
> Hi David
>
> Thanks a lot for your answer. I think I miss something in my question.

I suspect test.ping doesn't work quite the way David described. Or, for
that matter, the way manual says it should (strace shows it using ping,
TFM says it doesn't).

First, on 10.0.1.6 check /etc/salt/minion_id and make sure it is the
minion you're after. Then check /etc/nsswitch.conf (on both sides, to be
sure): you want "hosts: files dns [...]", not the other way around.

--
Dimitri Maziuk
Programmer/sysadmin
BioMagResBank, UW-Madison -- http://www.bmrb.wisc.edu

signature.asc

ronna mijares

unread,

Apr 14, 2016, 4:05:23 PM4/14/16

to Salt-users

Hi Dimitri

The information in one of my minions is the same as its hostname:

$ cat /etc/salt/minion_id

srv40stg.cl

$ hostname

srv40stg.cl

And I already had checked my /etc/nsswitch.conf and I have the correct order in my salt master, first my host table and after DNS

hosts: files dns

The problem is when I executed a salt operation, it takes more than 20 seconds what I think is toooooo much for each server.

Thanks Dimitri

David Boucha

unread,

Apr 15, 2016, 11:58:58 AM4/15/16

to salt users list

How does it behave when you just run this:

salt 'srv40stg.cl' test.ping

ronna mijares

unread,

Apr 18, 2016, 8:38:58 AM4/18/16

to Salt-users

Hi David

It took 24 second in receive an answer:

[root@salt-STG ~]# time salt srv40stg.cl test.ping

srv40stg.cll:

True

real 0m24.330s

user 0m0.507s

sys 0m0.079s

[salt-STG ~]# time salt srv40stg.cll cmd.run "hostname"

srv40stg.cl:

srv40stg.cl

real 0m25.483s

user 0m0.514s

sys 0m0.068s

David Boucha

unread,

Apr 18, 2016, 1:04:19 PM4/18/16

to salt users list

Salt is just using the standard python dns libraries. So I'm not sure exactly what's going on here.

Can you provide more information about how your system is set up?

What options have you configured in your master config and your minion configs?

Are you using any custom grains?

Dimitri Maziuk

unread,

Apr 18, 2016, 1:44:23 PM4/18/16

to salt-...@googlegroups.com

On 04/18/2016 12:04 PM, David Boucha wrote:
> Salt is just using the standard python dns libraries. So I'm not sure
> exactly what's going on here.

*If* test.ping actually pings the minion by hostname and *if* the
problem is in DNS, ping from command line should also run in 20+
seconds. Or, assuming it's gethostbyname:

# python
Python 2.7.5 (default, Nov 20 2015, 02:00:19)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-4)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import socket
>>> socket.gethostbyname( "herring" )
'144.92.217.33'

also 20+ seconds.
(does not happen here.)

HTH

signature.asc

ronna mijares

unread,

Apr 19, 2016, 10:50:42 AM4/19/16

to Salt-users

Hi guys

I did what you asked, and the answer using python and the answer was immediate

[root@salt-STG ~]# python

Python 2.6.6 (r266:84292, Jul 23 2015, 15:22:56)

[GCC 4.4.7 20120313 (Red Hat 4.4.7-11)] on linux2

Type "help", "copyright", "credits" or "license" for more information.

>>> import socket

>>> socket.gethostbyname( "srv40stg.cl" )

'10.0.1.40'

Testing, we remove "dns" from our /etc/nsswitch.conf, and salt answer immediately.

hosts: files

[salt-STG ~]# time salt srv40stg.cl cmd.run "hostname"

srv40stg.cl:

srv40stg.cl

real 0m0.762s

user 0m0.494s

sys 0m0.073s

But, if I add dns in the file, it took again more than 20 seconds:

hosts: files dns

[salt-STG ~]# time salt srv40stg.cl cmd.run "hostname"

srv40stg.cl:

srv40stg.cl

real 0m32.416s

user 0m0.500s

sys 0m0.069s

Thanks a lot for your help

ronna mijares

unread,

Apr 19, 2016, 11:09:26 AM4/19/16

to Salt-users

David

The answer for your questions are:

Can you provide more information about how your system is set up?

Master

CentOS release 6.7 (Final)

salt-2015.8.7-1.el6.noarch

salt-ssh-2015.8.7-1.el6.noarch

salt-master-2015.8.7-1.el6.noarch

Minion:

CentOS release 6.6 (Final)

salt-minion-2015.8.7-1.el6.noarch

salt-2015.8.7-1.el6.noarch

What options have you configured in your master config and your minion configs?

interface: 10.0.1.110

worker_threads: 5

state_top: top.sls

file_roots:

base:

- /srv/salt/

log_file: /var/log/salt/master

nodegroups:

re: 'srv9'
db: 'L@srv6,srv7'
bc: 'srv40'
tr: 'L@srv4,srv10'
se: 'L@srv2,srv3'
pa: 'srv32'
mo: 'L@srv24,srv25'
da: 'L@srv8,srv12'
mix: 'srv29
red: 'srv36'
nex: 'L@srv30,srv31'
ngi: 'srv21'

Are you using any custom grains?

No, I am not. I am using

Dimitri Maziuk

unread,

Apr 19, 2016, 3:09:16 PM4/19/16

to salt-...@googlegroups.com

Uhm... that strace that was showing ping, where did that come from? --
here's the code from
/usr/lib/python2.7/site-packages/salt/modules/test.py#ping() (you made
me look):

if not salt.utils.is_proxy():
return True
else:
ping_cmd = __opts__['proxy']['proxytype'] + '.ping'
if __opts__.get('add_proxymodule_to_opts', False):
return __opts__['proxymodule'][ping_cmd]()
else:
return __proxy__[ping_cmd]()

So unless this is a wrong test.py (?), it seems TFM is 1/3rd right and I
was 1/3rd wrong, and *if* you don't have a "proxy" defined someplace,
the whole test.ping should be a noop.

The question would be what TH test.ping is *actually running*.

signature.asc

ronna mijares

unread,

Apr 22, 2016, 11:34:25 AM4/22/16

to Salt-users

Hi this ping came from the salt server where I run salt states.

Regards

Reply all

Reply to author

Forward