Ehr, the BMC failover-function seems to be really messed up. If it cannot get an IP-address through DHCP on it’s dedicated BMC-port, it tries to get it through eth0/1. However, once this failover has been trigged, it doesn’t seem to go back to trying on the dedicated BMC-port anymore, which is really annoying (as the dedicated BMC-port should have higher priority than the other ports, in my opinion).
When I switched the management-network to use the new IP-layout today, it had some “downtime”, causing even more BMC-cards to failover. So, I’ll do some more digging, but I guess I’ll have to completely shut those nodes down, remove the power, and power it on again.
root@ns0:~# cat /var/log/syslog|grep -i "no free leases"|perl -wple 's|\t+| |g'|grep -i "feb 10"|grep -i "10.162.32.1"|cut -d' ' -f8,10|sort|uniq|perl -wple 's|:$||g,s| |#|g' 00:25:90:10:74:b6#10.162.32.1 00:25:90:10:75:8a#10.162.32.1 00:25:90:11:21:42#10.162.32.1 00:25:90:11:ce:1a#10.162.32.1 00:25:90:12:30:28#10.162.32.1 00:25:90:12:30:40#10.162.32.1 00:25:90:12:30:44#10.162.32.1 00:25:90:12:30:70#10.162.32.1 00:25:90:12:30:7c#10.162.32.1 00:25:90:12:31:98#10.162.32.1 00:25:90:12:31:a8#10.162.32.1 00:25:90:12:31:b4#10.162.32.1 00:25:90:12:31:c4#10.162.32.1 00:25:90:12:31:dc#10.162.32.1 00:25:90:12:31:f4#10.162.32.1 00:25:90:12:32:00#10.162.32.1 00:25:90:12:32:20#10.162.32.1 00:25:90:12:32:24#10.162.32.1 00:25:90:12:32:28#10.162.32.1 00:25:90:12:32:5c#10.162.32.1 00:25:90:12:32:6c#10.162.32.1 00:25:90:12:32:9c#10.162.32.1 00:25:90:12:32:f4#10.162.32.1 00:25:90:12:33:30#10.162.32.1 00:25:90:12:33:40#10.162.32.1 00:25:90:12:33:5c#10.162.32.1 00:25:90:12:33:64#10.162.32.1 00:25:90:12:33:78#10.162.32.1 00:25:90:12:33:8c#10.162.32.1 00:25:90:12:33:98#10.162.32.1 00:25:90:12:33:b0#10.162.32.1 00:25:90:12:33:dc#10.162.32.1 00:25:90:12:34:04#10.162.32.1 00:25:90:12:6d:ce#10.162.32.1 00:25:90:12:70:16#10.162.32.1 00:25:90:12:70:2e#10.162.32.1 00:25:90:12:cd:38#10.162.32.1 00:25:90:12:cd:d0#10.162.32.1 00:25:90:12:ce:28#10.162.32.1 00:25:90:12:ce:40#10.162.32.1 00:25:90:12:ce:44#10.162.32.1 00:25:90:12:ce:50#10.162.32.1 00:25:90:12:ce:64#10.162.32.1 00:25:90:12:ce:a8#10.162.32.1 00:25:90:12:ce:c4#10.162.32.1 00:25:90:12:d0:d8#10.162.32.1 00:25:90:13:fa:d8#10.162.32.1 00:25:90:14:4a:6a#10.162.32.1 00:25:90:14:4a:a6#10.162.32.1 00:e0:81:4d:2d:c8#10.162.32.1 00:e0:81:c0:70:57#10.162.32.1 00:e0:81:c0:70:58#10.162.32.1 00:e0:81:c0:70:95#10.162.32.1 00:e0:81:c0:70:d7#10.162.32.1 00:e0:81:c0:70:e4#10.162.32.1 00:e0:81:c0:71:91#10.162.32.1 00:e0:81:c0:71:eb#10.162.32.1 00:e0:81:c0:75:4d#10.162.32.1 00:e0:81:c0:75:50#10.162.32.1 00:e0:81:c0:75:6f#10.162.32.1 |
And with hostnames;
joachim@keklolwtf: ~/Documents/CERN/Temp/Hostname-stuff $ for host in $(cat no-free-leases_10feb2); do mac=`echo $host|cut -d'#' -f1`; network=`echo $host|cut -d'#' -f2`; old_hostname=`cat ldap.ldif|grep -i -A1 "$mac"|tail -1|perl -wple 's|dhcpOption: host-name||gi,s|"||gi,s| ||gi'`; new_hostname=`cat dhcpd-10.162.*|grep -i "$mac"|cut -d' ' -f2`; echo -e "$old_hostname -> $new_hostname\t$network\t$mac"; done cngpu07-bmc -> cngpu006-bmc 10.162.32.1 00:25:90:10:74:b6 cngpu06-bmc -> cngpu005-bmc 10.162.32.1 00:25:90:10:75:8a cngpu10-bmc -> cngpu009-bmc 10.162.32.1 00:25:90:11:21:42 cngpu02-bmc -> cngpu001-bmc 10.162.32.1 00:25:90:11:ce:1a cngpu14-bmc -> cngpu013-bmc 10.162.32.1 00:25:90:12:30:28 cngpu17-bmc -> cngpu016-bmc 10.162.32.1 00:25:90:12:30:40 cngpu24-bmc -> cngpu023-bmc 10.162.32.1 00:25:90:12:30:44 cngpu32-bmc -> cngpu031-bmc 10.162.32.1 00:25:90:12:30:70 cngpu04-bmc -> cngpu003-bmc 10.162.32.1 00:25:90:12:30:7c cngpu30-bmc -> cngpu029-bmc 10.162.32.1 00:25:90:12:31:98 cnhltout27-bmc -> cn027-bmc 10.162.32.1 00:25:90:12:31:a8 cngpu09-bmc -> cngpu008-bmc 10.162.32.1 00:25:90:12:31:b4 cnhltout26-bmc -> cn026-bmc 10.162.32.1 00:25:90:12:31:c4 cngpu20-bmc -> cngpu019-bmc 10.162.32.1 00:25:90:12:31:dc cngpu23-bmc -> cngpu022-bmc 10.162.32.1 00:25:90:12:31:f4 cngpu31-bmc -> cngpu030-bmc 10.162.32.1 00:25:90:12:32:00 cngpu08-bmc -> cngpu007-bmc 10.162.32.1 00:25:90:12:32:20 cngpu26-bmc -> cngpu025-bmc 10.162.32.1 00:25:90:12:32:24 cngpu25-bmc -> cngpu024-bmc 10.162.32.1 00:25:90:12:32:28 cngpu01-bmc -> cngpu000-bmc 10.162.32.1 00:25:90:12:32:5c cngpu03-bmc -> cngpu002-bmc 10.162.32.1 00:25:90:12:32:6c cngpu33-bmc -> cngpu032-bmc 10.162.32.1 00:25:90:12:32:9c cngpu21-bmc -> cngpu020-bmc 10.162.32.1 00:25:90:12:32:f4 cngpu13-bmc -> cngpu012-bmc 10.162.32.1 00:25:90:12:33:30 cngpu34-bmc -> cngpu033-bmc 10.162.32.1 00:25:90:12:33:40 cngpu15-bmc -> cngpu014-bmc 10.162.32.1 00:25:90:12:33:5c cngpu12-bmc -> cngpu011-bmc 10.162.32.1 00:25:90:12:33:64 cngpu29-bmc -> cngpu028-bmc 10.162.32.1 00:25:90:12:33:78 cngpu22-bmc -> cngpu021-bmc 10.162.32.1 00:25:90:12:33:8c cngpu19-bmc -> cngpu018-bmc 10.162.32.1 00:25:90:12:33:98 cngpu16-bmc -> cngpu015-bmc 10.162.32.1 00:25:90:12:33:b0 cngpu18-bmc -> cngpu017-bmc 10.162.32.1 00:25:90:12:33:dc cngpu11-bmc -> cngpu010-bmc 10.162.32.1 00:25:90:12:34:04 cngpu27-bmc -> cngpu026-bmc 10.162.32.1 00:25:90:12:6d:ce cngpu28-bmc -> cngpu027-bmc 10.162.32.1 00:25:90:12:70:16 cngpu05-bmc -> cngpu004-bmc 10.162.32.1 00:25:90:12:70:2e cnhltout09-bmc -> cn009-bmc 10.162.32.1 00:25:90:12:cd:38 cnhltout25-bmc -> cn025-bmc 10.162.32.1 00:25:90:12:cd:d0 cnhltout20-bmc -> cn020-bmc 10.162.32.1 00:25:90:12:ce:28 cnhltout11-bmc -> cn011-bmc 10.162.32.1 00:25:90:12:ce:40 cnhltout13-bmc -> cn013-bmc 10.162.32.1 00:25:90:12:ce:44 cnhltout08-bmc -> cn008-bmc 10.162.32.1 00:25:90:12:ce:50 cnhltout12-bmc -> cn012-bmc 10.162.32.1 00:25:90:12:ce:64 cnhltout07-bmc -> cn007-bmc 10.162.32.1 00:25:90:12:ce:a8 cnhltout06-bmc -> cn006-bmc 10.162.32.1 00:25:90:12:ce:c4 cnhltout24-bmc -> cn024-bmc 10.162.32.1 00:25:90:12:d0:d8 cnhltout23-bmc -> cn023-bmc 10.162.32.1 00:25:90:13:fa:d8 cnhltout22-bmc -> cn022-bmc 10.162.32.1 00:25:90:14:4a:6a cnhltout21-bmc -> cn021-bmc 10.162.32.1 00:25:90:14:4a:a6 feptofc10-bmc -> feptofc10-bmc 10.162.32.1 00:e0:81:4d:2d:c8 feptrd14-bmc -> feptrd14-bmc 10.162.32.1 00:e0:81:c0:70:57 feptofa04-bmc -> feptofa04-bmc 10.162.32.1 00:e0:81:c0:70:58 fepemcal2-bmc -> fepemcal2-bmc 10.162.32.1 00:e0:81:c0:70:95 feptofa16-bmc -> feptofa16-bmc 10.162.32.1 00:e0:81:c0:70:d7 feptofa00-bmc -> feptofa00-bmc 10.162.32.1 00:e0:81:c0:70:e4 fepsdd0-bmc -> fepsdd0-bmc 10.162.32.1 00:e0:81:c0:71:91 feptofc16-bmc -> feptofc16-bmc 10.162.32.1 00:e0:81:c0:71:eb fepemcal3-bmc -> fepemcal3-bmc 10.162.32.1 00:e0:81:c0:75:4d feptofa06-bmc -> feptofa06-bmc 10.162.32.1 00:e0:81:c0:75:50 fepsdd1-bmc -> fepsdd1-bmc 10.162.32.1 00:e0:81:c0:75:6f |