Как headhunter удалось безопасно нарушить rfc 793 (tcp) и обойти...
TRANSCRIPT
Как HeadHunter удалосьбезопасно нарушить RFC 793 (TCP) и обойти сетевые ловушки сервисной архитектуры
Андрей Шоринзамдиректора эксплуатацииhh.ru
# 0 - no source validation
net.ipv4.conf.all.rp_filter=0
# 2 - always use the best local address for this target
net.ipv4.conf.all.arp_announce=2
# 1 - reply only if the target IP address is local
# address configured on the incoming interface
net.ipv4.conf.all.arp_ignore=1
ip sla 1
tcp-connect 192.168.1.11 80 control disable
timeout 1000
threshold 1000
frequency 1
tag balancer1
ip sla schedule 1 life forever start-time now
ip route 10.1.1.1 255.255.255.255 192.168.1.11 track 1
track 1 ip sla 1 reachability
ip route 10.1.1.1 255.255.255.255 192.168.1.12 track 2
track 2 ip sla 2 reachability
ip cef load-sharing algorithm
include-ports source destination
routing-options {
static {
route 10.1.1.1/32 {
next-hop [ 192.168.1.11 192.168.1.12 ];
}
}
}
routing-options {
static {
route 10.1.1.1/32 {
next-hop [ 192.168.1.11 192.168.1.12 ];
bfd-liveness-detection {
minimum-interval 300;
multiplier 2;
no-adaptation;
}
}
}
routing-options {
ppm;
forwarding-table {
export per-flow-lb;
}
}
policy-options {
policy-statement per-flow-lb {
then {
load-balance per-packet;
}
}
}
https://github.com/ashorin/OpenBFDD/tree/debianize
/etc/default/openbfdd:
OPENBFDD_CONTROL=\
"$CONTROL_COMMAND load /etc/openbfdd.conf"
/etc/openbfdd.conf:
connect local 192.168.1.11 remote 192.168.1.1
session all set mintx 100 ms
session all set minrx 100 ms
session all set multi 2
Monit:
check program gw_ready with path /usr/local/bin/bfdd.sh
if status != 0 then alert
check process openbfdd with pidfile /var/run/openbfdd.pid
start program = "/sbin/start openbfdd"
stop program = "/sbin/stop openbfdd"
mode active
/usr/local/bin/bfdd.sh:
check_nginx_up() {
wget -qO /dev/null --timeout=1 —tries=1 http://127.0.0.1:80/nginx-satus
}
retval=0
if ! check_nginx_up; then
echo nginx is down >&2
retval=3
fi
if [ $retval -eq 0 ]; then
/usr/bin/pgrep bfdd-beacon || /usr/bin/monit -c /etc/monit/monitrc start openbfdd
else
/usr/bin/pgrep bfdd-beacon && /usr/bin/monit -c /etc/monit/monitrc stop openbfdd
fi
103 k pkt/sec
10 k pkt/sec
# Name: UDP/TCP source port
# mode: streaming
# port flows octets packets
1006 468 4201193 2703
1007 466 4165639 2692
1008 416 3491145 2377
1009 411 3412711 2352
39748 2 27240572 523730
39751 1 66627663 1281258
# Name: UDP/TCP source port
# mode: streaming
# port flows octets packets
1006 468 4201193 2703
1007 466 4165639 2692
1008 416 3491145 2377
1009 411 3412711 2352
39748 2 27240572 523730
39751 1 66627663 1281258
# Name: UDP/TCP source port
# mode: streaming
# port flows octets packets
1006 468 4201193 2703
1007 466 4165639 2692
1008 416 3491145 2377
1009 411 3412711 2352
39748 2 27240572 523730
39751 1 66627663 1281258
`
# Name: UDP/TCP source port
# mode: streaming
# port flows octets packets
1006 468 4201193 2703
1007 466 4165639 2692
1008 416 3491145 2377
1009 411 3412711 2352
39748 2 27240572 523730
39751 1 66627663 1281258
`
# Name: UDP/TCP source port
# mode: streaming
# port flows octets packets
1006 468 4201193 2703
1007 466 4165639 2692
1008 416 3491145 2377
1009 411 3412711 2352
39748 2 27240572 523730
39751 1 66627663 1281258
`÷
3491145 23773412711 2352
27240572 52373066627663 1281258
÷
÷
≈ 1460
= 52
sudo tcpdump -ni eth0 host ( 10.1.1.1 or 192.168.1.11 ) and len == 66 -w 52b-%Y-%m-%d_%H-%M.pcap -G 60
sudo tcpdump -ni eth0 host ( 10.1.1.1 or 192.168.1.11 ) and len == 66 -w 52b-%Y-%m-%d_%H-%M.pcap -G 60
# 8 часов
* * * * * root find /var/flow/intbal-pcap -maxdepth 1 -type f -mmin +480 -delete
-rw-r--r-- 1 root root 401381030 Sep 10 11:39 52b-2014-09-10_11-38.pcap
-rw-r--r-- 1 root root 400369068 Sep 10 11:40 52b-2014-09-10_11-39.pcap
-rw-r--r-- 1 root root 517185620 Sep 10 11:41 52b-2014-09-10_11-40.pcap
-rw-r--r-- 1 root root 1803984614 Sep 10 11:42 52b-2014-09-10_11-41.pcap
-rw-r--r-- 1 root root 1982921976 Sep 10 11:43 52b-2014-09-10_11-42.pcap
-rw-r--r-- 1 root root 461025642 Sep 10 11:44 52b-2014-09-10_11-43.pcap
-rw-r--r-- 1 root root 401152356 Sep 10 11:45 52b-2014-09-10_11-44.pcap
-rw-r--r-- 1 root root 402100506 Sep 10 11:46 52b-2014-09-10_11-45.pcap
`
tshark -r 52b-2014-09-10_11-42.pcap
-w 52b-2014-09-10_11-42-logic3-33813.pcap
-R "ip.addr==192.168.2.39 and tcp.port==33813"
Sep 10 13:41:01 switch 1y43w: %TRACKING-5-STATE:
3 ip sla 3 reachability Up->Down
Sep 10 11:42:11 switch 1y43w: %TRACKING-5-STATE:
3 ip sla 3 reachability Down->Up
exp-12295-nginx1 IN A 192.168.2.182
exp-12295-nginx2 IN A 192.168.2.183
exp-12295-client1 IN A 192.168.2.184
exp-12295-shared IN A 192.168.2.186
ip ro l 192.168.2.186
192.168.2.186 via 192.168.2.183 dev eth0 metric 10
192.168.2.186 via 192.168.2.182 dev eth0 metric 20` `
mkfifo backpipe
nc -l 8082 0<backpipe |
nc -p 19999 192.168.2.186 80 1>backpipe
wget -O/dev/null http://127.0.0.1:8082/long-file
Nginx:
listen 80 default so_keepalive=1s:1s:1;
Nginx:
listen 80 default so_keepalive=1s:1s:1;
Cisco:
track 1 ip sla 1 reachability
delay up 3
Nginx:
listen 80 default so_keepalive=1s:1s:1;
Cisco:
track 1 ip sla 1 reachability
delay up 3
Juniper:
holddown-interval 2500;
Андрей Шорин
замдиректора эксплуатации hh.ru
linkedin.com/in/andshorin
Балансировка балансировщиков
на коммутаторах:
bit.ly/switch-balancing