[root@dc5-linux-test04 msvoboda]# /var/cfengine/bin/cf-agent -I -K -f failsafe.cf2014-02-23T06:17:41-0800 notice: R: cf3: The md5sum hash maps on dc5-cfe-test.corp.cfengine.com:/var/cfengine/masterfiles/md5_digest are identical to dc5-linux-test04.corp:/var/cfengine/md5_digest. There is no need to perform any further network transfers at Sun Feb 23 06:17:40 20142014-02-23T06:17:41-0800 info: /scale_cfengine_data_transfers/methods/'dump_discovered_file_classes'/dump_discovered_file_classes/files/'/var/cfengine/file_classes.txt': Edit file '/var/cfengine/file_classes.txt'[root@dc5-linux-test04 msvoboda]# cat -n /var/cfengine/inputs/failsafe.cf 1 # filesource \$HeadURL: svn+ssh://cs...@dc5-sysops-svn.corp.cfengine.com/export/content/sysops-svn/cfengine/branches/dc5-cfe-test.corp/generic_cf-agent_policies/failsafe.cf $ 2 # version \$Revision: 99317 $ 3 # modifiedby \$LastChangedBy: msvoboda $ 4 # lastmodified \$Date: 2014-02-21 13:22:14 -0800 (Fri, 21 Feb 2014) $ 5 6 # Dont break this, or any file included in @inputs below up, otherwise, automated Cfengine file transfers break everywhere. 7 # You will spend the rest of the week recovering machines. Before making a modification to any of these files, you must test 8 # extensively. Make sure the guppies work. Copy the policy files out to machines in production and test. 9 # Recovering Cfengine when file transfers are broken is extremely difficult to recover from. You will have to SSH to every machine 10 # and copy out fixed configuration files. Dont fuck it up. 11 12 body common control 13 { 14 bundlesequence => {"range_sync_execution_status", "scale_cfengine_data_transfers", }; 15 inputs => {"shared_global_environment.cf", "cf-execd.cf", 17 } 18 ################################################################################################################################### 19 bundle common global 20 { 21 # Define the environment and site as classes. We will use this in update.cf 22 # to determine what point in the SVN tree / filesystem directory hirarchy we should pull site specific policies. 23 # failsafe_global.cf is one of them. That means failsafe_global.cf is dependant upon itself for a properly 24 # functioning CFEngine infrastructure. Thats how critical this is. Dont break it. 25 vars: 26 any:: 27 "global_nic" string => execresult("/usr/bin/getent hosts `/bin/hostname` | awk '{print $1}'","useshell"); 28 29 classes: 30 # Never ever disable this module. Cfengine will fall apart if you do. 31 !define_range_classes:: 32 "define_range_classes" expression => usemodule("module_define_range_classes.sh",""); 33 } 34 body agent control 35 { 36 # Increase the editfilesize to 1mb 37 editfilesize => "1M"; 38 39 # Allow us to use DNS 40 skipidentify => "false"; 41 42 # if default runtime is 5 mins we need this for long jobs 43 ifelapsed => "1"; 44 45 # Allow us to use DNS 46 skipidentify => "false"; 47 48 # Bind to the global NIC on all inbound / outbound network i/o 49 bindtointerface => "$(global.global_nic)"; 50 51 # We hit this class if we didn't find our SITE/ENV from shared_global_environment.cf. We can't perform network transfers 52 # if we dont know where we are and who we should contact. Someone probably forgot to update shared_global_environment with 53 # the new VLAN information. 54 abortclasses => { "stop_cfengine_execution" }; 55 }
Cfengine executes every 5 minutes. There are 12x 5m intervals in an hour. There are 24 hours in a day. This means there are 1440 Cfengine executions in a day on a single machine.
6 bundle agent scale_cfengine_data_transfers 7 { 8 classes: 9 # If we dont have a SITE/ENV defined in shared_global_environment, then we must abort execution. We dont know what to transfer10 # into the client's /var/cfengine/inputs_site_specific, so bail. promises.cf would miss a bunch of stuff too. we probably added11 # a VLAN, but we didn't update shared_global_environment.cf with that mapping.12 any::13 "stop_cfengine_execution" or => {"ABORT_CFENGINE_EXECUTION", };14 "daily_splayclass" expression => splayclass("$(sys.host)$(sys.ipv4)","daily");[msvoboda@dc1-app9103 ~]$ grep 'The full Cfengine file transfer policy is active' /var/log/messagesFeb 23 03:43:01 dc1-app9103 cf3[11212]: R: cf3: The full Cfengine file transfer policy is active on dc1-app9103.prod. Performing a network transfer from dc1-core17-mps03.prod.cfengine.com at Sun Feb 23 03:42:57 2014 [msvoboda@dc1-app9103 ~]$ dateSun Feb 23 14:39:15 UTC 2014
16 !master_policy_servers::17 "bootstrap_required" not => fileexists("/var/cfengine/inputs/promises.cf");22 !master_policy_servers.!super_cfengine_client_file_transfers::23 "standard_cfengine_client_file_transfers" expression => "any";65 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **66 # Full Cfengine MD5 Comparision67 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **68 69 # If promises.cf is missing on a client, then perform a full file transfer to bootstrap.70 # Otherwise, perform a full transfer once a day for file deletions.71 standard_cfengine_client_file_transfers::72 "perform_daily_full_file_transfer" usebundle => perform_daily_full_file_transfer,73 handle => perform_daily_full_file_transfer;242 bundle agent perform_daily_full_file_transfer243 {244 vars:245 "client[/var/cfengine/inputs]" string => "/var/cfengine/masterfiles/generic_cf-agent_policies";246 "client[/var/cfengine/modules]" string => "/var/cfengine/masterfiles/cf-agent_modules";247 "client[/var/cfengine/torrents]" string => "/var/cfengine/masterfiles/mps_generated_torrents";248 "client[/var/cfengine/inputs_site_specific]" string => "/var/cfengine/masterfiles/$(shared_global_environment.environment)/$(shared_global_environment.site)/cf-agent";249 250 "client_dirs" slist => getindices("client");251 files:252 # Remove data under the four directory structures if they still exist on the client, but have been removed from the MPS.253 # This is the daily full Cfengine file transfer performed by clients. If the scaling policy screws up, this should allow254 # for automated recovery. This bundle is activated by a splayclass, meaning, clients hash their hostname and IP address to determine255 # when this policy will execute. (It only picks one 5m execution over the 24h period)256 # It should cause clients to execute this policy throughout the day instead of all at once.257 # There are 60m / 5m schedule = 12 executions. 12 executions an hour * 24 hours = 1440 executions a day.258 # This means, theoritically, if a MPS has 1440 clients, only one client performs a full file transfer during a given 5m execution.259 # All other clients should be executing in the scaling policy.260 any::261 "$(client_dirs)"262 handle => "daily_full_filesystem_copy",263 copy_from => quad_remote_copy("$(client[$(client_dirs)])",264 "$(shared_global_environment.mps1)",265 "$(shared_global_environment.mps2)",266 "$(shared_global_environment.mps3)",267 "$(shared_global_environment.mps4)"),268 depth_search => recurse_ignore_dot_svn("inf"),269 action => immediate;270 reports:271 !sysops_svn_servers::272 "cf3: The full Cfengine file transfer policy is active on $(sys.host). Performing a network transfer from $(shared_global_environment.primary_mps) at $(sys.date)";273 }
|
19 !master_policy_servers.!bootstrap_required.!daily_splayclass.!sunos_5_10::20 "super_cfengine_client_file_transfers" expression => "any";
41 methods:42 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **43 # Scale Cfengine Client Transfers44 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **45 46 super_cfengine_client_file_transfers::47 "copy_md5sum_generated_data" usebundle => copy_md5sum_generated_data,48 handle => copy_md5sum_generated_data;49 50 md5_hash_maps_updated.super_cfengine_client_file_transfers::51 "define_digest_data" usebundle => define_digest_data,52 depends_on => {"copy_md5sum_generated_data", },53 handle => define_digest_data,54 classes => if_repaired("scale_cfengine_client_transfers");55 56 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **57 # Scale Cfengine Client Transfers58 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **
107 bundle agent copy_md5sum_generated_data108 {109 vars:110 any::111 "digest_directory" string => "/var/cfengine/md5_digest";112 "mps_digest_directory" string => "/var/cfengine/masterfiles/md5_digest";113 114 files:115 # Copy all MD5SUM files from the remote cf-serverd instance to local disk.116 any::117 "$(digest_directory)"118 handle => "copy_digests_from_mps_to_client",119 copy_from => quad_remote_copy("$(mps_digest_directory)",120 "$(shared_global_environment.mps1)",121 "$(shared_global_environment.mps2)",122 "$(shared_global_environment.mps3)",123 "$(shared_global_environment.mps4)"),124 depth_search => recurse_ignore_dot_svn("1"),125 action => immediate,126 classes => if_repaired_namespace_scope("md5_hash_maps_updated");127 128 reports:129 !md5_hash_maps_updated.!sysops_svn_servers::130 "cf3: The md5sum hash maps on $(shared_global_environment.primary_mps):$(mps_digest_directory) are identical to $(sys.host):$(digest_directory). There is no need to perform any further network transfers at $(sys.date)";131 }
133 bundle agent define_digest_data 134 { 135 vars: 136 any:: 137 "digest_directory" string => "$(copy_md5sum_generated_data.digest_directory)"; 138 "inputs" string => "$(digest_directory)/generic_cf-agent_policies.md5"; 139 "modules" string => "$(digest_directory)/cf-agent_modules.md5"; 140 "torrents" string => "$(digest_directory)/mps_generated_torrents.md5"; 141 "inputs_site_specific" string => "$(digest_directory)/$(shared_global_environment.environment)-$(shared_global_environment.site).md5"; 142 143 classes: 144 any:: 145 "inputs_exists" expression => fileexists("$(inputs)"); 146 "modules_exists" expression => fileexists("$(modules)"); 147 "torrents_exists" expression => fileexists("$(torrents)"); 148 "inputs_site_specific_exists" expression => fileexists("$(inputs_site_specific)"); 149 "data_exists" and => {"inputs_exists", "modules_exists", "torrents_exists", "inputs_site_specific_exists", }; 150 151 methods: 152 data_exists:: 153 "inputs_execution" usebundle => define_classes_of_files_to_be_requested("$(inputs)"); 154 "modules_execution" usebundle => define_classes_of_files_to_be_requested("$(modules)"); 155 "torrents_execution" usebundle => define_classes_of_files_to_be_requested("$(torrents)"); 156 "inputs_site_specific_execution" usebundle => define_classes_of_files_to_be_requested("$(inputs_site_specific)"); 157 }
159 bundle agent define_classes_of_files_to_be_requested(digest)160 {161 vars:162 any::163 "number_of_entries_in_data" int => readstringarrayidx("data",164 "$(digest)",165 "\s*#[^\n]*",166 ":::",167 99999999999,168 99999999999);169 170 "data_line" slist => getindices("data");171 172 classes:173 any::174 "$(data[$(data_line)][0])_exists" expression => fileexists("$(data[$(data_line)][0])"),175 comment => "compute exists for $(data[$(data_line)][0])",176 scope => "namespace";177 178 "$(data[$(data_line)][0])_missing" not => fileexists("$(data[$(data_line)][0])"),179 comment => "compute missing for $(data[$(data_line)][0])",180 scope => "namespace";181 182 183 any::184 # This becomes the class that determines if we transfer the file. If we find a "differs" class, we perform the file transfer.185 "$(data[$(data_line)][0])_differs" not => hashmatch("$(data[$(data_line)][0])", "md5", "$(data[$(data_line)][1])"),186 scope => "namespace",187 comment => "compute differs for $(data[$(data_line)][0])",188 ifvarclass => canonify("$(data[$(data_line)][0])_exists");189 190 methods:191 any::192 "perform_individual_file_transfer" usebundle => perform_individual_file_transfer,193 comment => "execute transfer for $(digest)",194 handle => perform_individual_file_transfer;195 196 reports:197 "cf3: Found mismatched file $(inputs[$(inputs_line)][0]). Performing individual file transfer now"198 ifvarclass => canonify("$(inputs[$(inputs_line)][0])_differs");199 }
201 bundle agent perform_individual_file_transfer202 {203 vars:204 # Copy data into our workspace.205 any::206 "data_line" slist => getindices("define_classes_of_files_to_be_requested.data");207 208 # The name of the file as it will appear on local disk209 "data[$(data_line)][0]" string => "$(define_classes_of_files_to_be_requested.data[$(data_line)][0])";210 211 # The name of the file as it appears on the MPS212 "data[$(data_line)][1]" string => "$(define_classes_of_files_to_be_requested.data[$(data_line)][2])";213 214 files:215 # Perform the individual file transfers on differences or files that exist on the MPS but not on the client.216 any::217 "$(data[$(data_line)][0])"218 handle => "copy_single_file_differs_from_mps_to_client",219 copy_from => quad_remote_copy("$(data[$(data_line)][1])",220 "$(shared_global_environment.mps1)",221 "$(shared_global_environment.mps2)",222 "$(shared_global_environment.mps3)",223 "$(shared_global_environment.mps4)"),224 comment => "copy $(data[$(data_line)][0])",225 action => immediate,226 ifvarclass => or(canonify("$(data[$(data_line)][0])_differs"), canonify("$(data[$(data_line)][0])_missing")),227 classes => if_repaired_namespace_scope("$(data[$(data_line)][0])_copied");228 229 any::230 "/etc/cfe.d/scaling_file_transfers.txt"231 handle => "insert_when_dynamic_file_transfers_happened",232 create => "true",233 perms => mog("0644","root","root"),234 edit_line => append_if_no_line("$(sys.date)#$(data[$(data_line)][0])"),235 ifvarclass => canonify("$(data[$(data_line)][0])_copied");236 237 reports:238 "cf3: Cfengine file transfer performed. $(shared_global_environment.primary_mps):$(data[$(data_line)][1]) pulled to $(sys.host):$(data[$(data_line)][0])"239 ifvarclass => canonify("$(data[$(data_line)][0])_copied");240 }75 master_policy_servers::76 "mps_perform_local_full_copy" usebundle => mps_perform_local_full_copy,77 handle => mps_perform_local_full_copy;78 79 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **80 # Full Cfengine MD5 Comparision81 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **
|
275 bundle agent mps_perform_local_full_copy276 {277 vars:278 "mps[/var/cfengine/inputs]" string => "/var/cfengine/masterfiles/generic_cf-agent_policies";279 "mps[/var/cfengine/modules]" string => "/var/cfengine/masterfiles/cf-agent_modules";280 "mps[/var/cfengine/torrents]" string => "/var/cfengine/masterfiles/mps_generated_torrents";281 "mps[/var/cfengine/inputs_site_specific]" string => "/var/cfengine/masterfiles/$(shared_global_environment.environment)/$(shared_global_environment.site)/cf-agent";282 283 "mps_dirs" slist => getindices("mps");284 files:285 # The MPS execute this file transfer every time since we do not perform a network transfer and this is a straight local file comparison.286 master_policy_servers::287 "$(mps_dirs)"288 handle => "mps_full_local_filesystem_copy",289 copy_from => mycopy("$(mps[$(mps_dirs)])"),290 depth_search => recurse_ignore_dot_svn("inf"),291 action => immediate;292 }
93 # Every machine should maintain /etc/cm.conf, no matter what file transfer policy is in effect. 94 any:: 95 "manage_etc_cm_conf" usebundle => manage_etc_cm_conf, 96 handle => manage_etc_cm_conf; 97 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** 98 # Update Files On Disk 99 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **100
310 bundle agent manage_etc_cm_conf311 {312 vars:313 mps1_primary::314 "mps_order" slist => {"PRIMARY_MPS:$(shared_global_environment.mps1)",315 "SECONDARY_MPS:$(shared_global_environment.mps2)",316 "THIRD_MPS:$(shared_global_environment.mps3)",317 "FORTH_MPS:$(shared_global_environment.mps4)",};318 319 mps2_primary::320 "mps_order" slist => {"PRIMARY_MPS:$(shared_global_environment.mps2)",321 "SECONDARY_MPS:$(shared_global_environment.mps3)",322 "THIRD_MPS:$(shared_global_environment.mps4)",323 "FORTH_MPS:$(shared_global_environment.mps1)",};324 325 mps3_primary::326 "mps_order" slist => {"PRIMARY_MPS:$(shared_global_environment.mps3)",327 "SECONDARY_MPS:$(shared_global_environment.mps4)",328 "THIRD_MPS:$(shared_global_environment.mps1)",329 "FORTH_MPS:$(shared_global_environment.mps2)",};330 331 mps4_primary::332 "mps_order" slist => {"PRIMARY_MPS:$(shared_global_environment.mps4)",333 "SECONDARY_MPS:$(shared_global_environment.mps1)",334 "THIRD_MPS:$(shared_global_environment.mps2)",335 "FORTH_MPS:$(shared_global_environment.mps3)",};336 337 files:338 # Monday morning at 9am EST, remove /etc/cm.conf and rebuild.339 Monday.GMT_Hr14::340 "/etc/cm.conf"341 handle => "scaled_weekly_purge_of_cm_conf",342 edit_defaults => empty,343 create => "true",344 edit_line => append_if_no_line("/etc/cm.conf regnereated at $(sys.date)"),345 perms => mog("0644","root","root"),346 classes => if_repaired("cm_conf_modified");347 348 any::349 "/etc/cm.conf"350 handle => "scaled_add_multi_mps_to_cm_conf",351 create => "true",352 perms => mog("0644","root","root"),353 edit_line => append_if_no_line("$(mps_order)"),354 classes => if_repaired("cm_conf_modified");355 356 master_policy_servers::357 "/etc/cm.conf"358 handle => "scaled_add_localhost_to_cm_conf",359 create => "true",360 perms => mog("0644","root","root"),361 edit_line => append_if_no_line("MASTER_POLICY_SERVER:$(sys.host)"),362 classes => if_repaired("cm_conf_modified");363 364 # Insert our site/env into cm.conf since we remove module_site_env and rely on shared_global_environment.cf now.365 linux|sunos_5_10::366 "/etc/cm.conf"367 handle => "scaled_add_env_site_to_cm_conf",368 edit_line => append_if_no_line("ENV_SITE:$(shared_global_environment.environment)@$(shared_global_environment.site)"),369 classes => if_repaired("cm_conf_modified");370 371 reports:372 cm_conf_modified::373 "cf3: /etc/cm.conf was modified on $(sys.host)";374 }86 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **87 # Update Files On Disk88 ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## ** ## **89 any::90 "dump_discovered_file_classes" usebundle => dump_discovered_file_classes,91 handle => dump_discovered_file_classes;294 bundle agent dump_discovered_file_classes295 {296 vars:297 any::298 "discovered_classes" slist => classesmatching(".*");299 "sorted_classes" slist => sort("discovered_classes", "lex");300 301 files:302 any::303 "/var/cfengine/file_classes.txt"304 handle => "dump_all_discovered_classses_for_file_transfers",305 edit_defaults => empty, 306 create => "true",307 edit_line => append_if_no_line("@(sorted_classes)");308 }