# HG changeset patch # User igor # Date 1190908589 -10800 # Node ID b6dc3480caae8d5bd5540137b3d56eb20535d903 xen-drbd merge diff -r 000000000000 -r b6dc3480caae INSTALL --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/INSTALL Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,27 @@ +#!/bin/sh + +DEST_DIR=/usr/bin + +cp init.d-xen-drbd /etc/init.d/xen-drbd +chmod +x /etc/init.d/xen-drbd + +cp xen-scripts-network-xen-drbd /etc/xen/scripts/network-xen-drbd +chmod +x /etc/xen/scripts/network-xen-drbd + +cp default-xen-drbd /etc/default/xen-drbd +cp xen-drbd-start.py /etc/xen/xen-drbd-start +cp xen-drbd.py ${DEST_DIR} + +for i in 2 3 4 5 +do + ln -sf ../init.d/xen-drbd /etc/rc$i.d/S95xen-drbd + #rm /etc/rc$i.d/S22xen-drbd +done + +for i in 0 1 6 +do + ln -sf ../init.d/xen-drbd /etc/rc$i.d/K05xen-drbd + #rm /etc/rc$i.d/K21xen-drbd +done + + diff -r 000000000000 -r b6dc3480caae config-example.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/config-example.py Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,86 @@ + +node1='dom0' +node2='dom0m' +i_am=node1 + +ip_address = { + node1: '192.168.1.190', + node2: '192.168.1.191' +} + +node1_ip=ip_address[node1] +node2_ip=ip_address[node2] + +domains=['log', 'dns', 'gw', 'igw', 'ldap', 'mail', 'proxy', 'samba', 'vpn', 'intraweb', 'jabber'] + +domain_home = { + node1 : ['log', 'dns', 'gw', 'igw', 'ldap', 'mail', 'proxy', 'vpn', 'intraweb', 'jabber'], + node2 : ['samba'], + } + +kernel = "/boot/vmlinuz-2.6.18-5-xen-686" +ramdisk = "/boot/initrd.img-2.6.18-5-xen-686" + +mem_table={ + 'log' :256, + 'dns' :64, + 'gw' :64, + 'igw' :128, + 'ldap' :64, + 'mail' :256, + 'proxy' :256, + 'samba' :256, + 'vpn' :64, + 'web' :128, + 'intraweb':128, + 'jabber':64, +} + +lvm_vg_name="XEN" +disk_table={ + 'log' : ['10G'], + 'dns' : ['2G'], + 'gw' : ['2G'], + 'igw' : ['2G'], + 'ldap' : ['2G'], + 'mail' : ['2G','maildir:2G'], + 'proxy' : ['10G'], + 'samba' : ['2G','samba-share:2G'], + 'vpn' : ['2G'], + 'web' : ['10G'], + 'intraweb' : ['10G'], + 'jabber' : ['2G'], +} + +bridges=['br0','br1'] +vlans=[4094,4093] +management_vlan=4094 # vlan 2 +trunk='eth1' +management_ip=ip_address[i_am] +management_gw='192.168.1.197' + +vbridges_table={ + 'log' : ['br1'], + 'dns' : ['br1'], + 'gw' : ['br0', 'br1'], + 'igw' : ['br1'], + 'ldap' : ['br1'], + 'mail' : ['br1'], + 'proxy' : ['br1'], + 'samba' : ['br1'], + 'vpn' : ['br1'], + 'web' : ['br1'], + 'intraweb' : ['br1'], + 'jabber' : ['br1'], +} + +ip_network="192.168.3" +domain_name="example.com" +ip_nameserver="192.168.3.2" +ip_gateway="192.168.3.3" + +debian_release="etch" +debian_mirror="http://debian.org.ua/debian" +apt_get_install="less tcpdump libpam-ldap libnss-ldap snmpd libc6-xen openssh-server" + + diff -r 000000000000 -r b6dc3480caae default-xen-drbd --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/default-xen-drbd Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,4 @@ +XEN_DRBD_PATH='/usr/local/bin' +TOPOLOGY_NAME='eb' +START_ACTION=nothing #migrate-and-start-all + diff -r 000000000000 -r b6dc3480caae init.d-xen-drbd --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/init.d-xen-drbd Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,25 @@ +#!/bin/sh + +XEN_DRBD_PATH='/usr/bin' +TOPOLOGY_NAME='xen-drbd' +START_ACTION='nothing' +[ -e /etc/default/xen-drbd ] && . /etc/default/xen-drbd + +case $1 in + start) + $XEN_DRBD_PATH/xen-drbd-install.py ${TOPOLOGY_NAME} make-links | sh -s + if [ -z "$START_ACTION" ] || echo $START_ACTION | grep -qix nothing + then + true + else + $XEN_DRBD_PATH/xen-drbd.py $START_ACTION + fi + ;; + stop) + $XEN_DRBD_PATH/xen-drbd.py migrate-all-out + ;; + *) + echo $0 'start|stop' + ;; +esac + diff -r 000000000000 -r b6dc3480caae xen-drbd-install.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xen-drbd-install.py Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,273 @@ +#!/usr/bin/python + +import sys + +network=sys.argv[1] +try: + exec 'from %s import domains,disk_table,lvm_vg_name,node1,node1_ip,node2,node2_ip, \ + ip_network, domain_name, ip_nameserver, ip_gateway, apt_get_install, \ + debian_release, debian_mirror, \ + bridges, vlans, management_vlan, trunk, management_ip, management_gw ' % (network) +except: + print "Can't find or interpret module %s with topology description" %(network) + sys.exit(1) + +drbd_base_port=7790 +meta_disk='/dev/%s/meta'%lvm_vg_name +drbd_devices=[] + +def make_lvm(do_print=1): + if do_print: + print "lvcreate -L %s -n %s /dev/%s" % ("10G", "meta", lvm_vg_name) + for dom in domains: + for disk in disk_table[dom]: + if disk.find(":") == -1: + if do_print: + print "lvcreate -L %s -n %s /dev/%s" % (disk, dom, lvm_vg_name) + drbd_devices.append([dom, "/dev/%s/%s" % (lvm_vg_name, dom)]) + else: + if do_print: + print "lvcreate -L %s -n %s /dev/%s" % (disk.split(':')[1], disk.split(':')[0], lvm_vg_name) + drbd_devices.append([disk.split(':')[0], "/dev/%s/%s" % (lvm_vg_name, disk.split(':')[0])]) + + +def make_drbdconf(): + print "cat < /etc/drbd.conf" + i=0 + for drbd in drbd_devices: + resource=drbd[0] + disk=drbd[1] + drbd_number=i + i+=1 + print """ +resource %s { + protocol C; + net { + allow-two-primaries; + after-sb-0pri discard-least-changes; + after-sb-1pri call-pri-lost-after-sb; + after-sb-2pri call-pri-lost-after-sb; + } + syncer { + rate 5M; + } + on %s + { + device /dev/drbd%s; + disk %s; + address %s:%s; + meta-disk %s[%s]; + } + on %s + { + device /dev/drbd%s; + disk %s; + address %s:%s; + meta-disk %s[%s]; + } +} + +""" % (resource, node1, drbd_number, disk, node1_ip, i+drbd_base_port, meta_disk, drbd_number, node2, drbd_number, disk, node2_ip, i+drbd_base_port, meta_disk, drbd_number ) + print "DRBD" + + +def make_drbd_links(): + make_lvm(0) + print "mkdir -p /dev/drbd" + i=0 + for drbd in drbd_devices: + drbd_number=i + i+=1 + resource=drbd[0] + disk=drbd[1] + print "ln -sf /dev/drbd%s /dev/drbd/%s" %(drbd_number, resource) + +def make_fs(): + for dom in domains: + print "mkfs.ext3 /dev/drbd/"+dom + + +def make_mount(): + print "mkdir -p /domU" + for dom in domains: + print "mkdir -p /domU/%s" % (dom) + print "mount /dev/drbd/%s /domU/%s" % (dom,dom) + +def make_umount(): + for dom in domains: + print "umount /domU/%s" % (dom) + +def make_dns(): + i=1 + for dom in domains: + print "%s A %s.%s" % (dom, ip_network, i) + i+=1 + +def make_dns_reverse(): + i=1 + for dom in domains: + print "%s A %s.%s." % (i, dom, domain_name) + i+=1 + +def make_install_debian(): + print """ +debootstrap %s /domU/%s %s +chroot /domU/%s apt-get install -y --force-yes %s + """ % (debian_release,domains[0],debian_mirror,domains[0],apt_get_install) + +def make_rsync(): + for dom in domains[1:]: + print """ +rsync -a /domU/%s/ /domU/%s/ + """ % (domains[0], dom) + +def make_etcfstab(): + i=0 + for dom in domains: + i+=1 + print """ +cat < /domU/%s/etc/fstab +proc /proc proc defaults 0 0 +/dev/hda1 / ext3 defaults,errors=remount-ro 0 1 +FSTAB + """ % (dom) + +def make_etcnetworkinterfaces(): + i=0 + for dom in domains: + i+=1 + print """ +cat < /domU/%s/etc/network/interfaces +auto lo eth0 +iface lo inet loopback + +iface eth0 inet static + address %s.%s + netmask 255.255.255.0 + network %s.0 + broadcast %s.255 + gateway %s + dns-nameservers %s + dns-search %s +INTERFACES + """ % (dom,ip_network,i,ip_network, ip_network, ip_gateway, domain_name, ip_nameserver) + +def make_etcresolvconf(): + for dom in domains: + print """ +cat < /domU/%s/etc/resolv.conf +search %s +nameserver %s +RESOLVCONF + """ % (dom,domain_name,ip_nameserver) + +def make_etchostname(): + for dom in domains: + print """ +cat < /domU/%s/etc/hostname +%s +HOSTNAME + """ % (dom,dom) + + +def make_etchosts(): + i=0 + for dom in domains: + i+=1 + print """ +cat < /domU/%s/etc/hosts +127.0.0.1 localhost +192.168.1.%s %s.%s %s + +::1 ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +ff02::3 ip6-allhosts +HOSTS + """ % (dom,i,dom,domain_name,dom) + +def make_rootssh(): + for dom in domains: + print """ +mkdir -p /domU/%s/root/.ssh/ +cp ~/.ssh/id_dsa.pub /domU/%s/root/.ssh/authorized_keys +chmod 600 /domU/%s/root/.ssh/authorized_keys + """ % (dom,dom,dom) + +def make_l3rc(): + for dom in domains: + print """ + cat < /domU/%s/root/.l3rc +l3cd=/users/igor/xen-drbd-dom0-setup/%s.%s/root +EOF + """ % (dom,dom,domain_name) + + +def make_etc(): + make_etcfstab() + make_etcnetworkinterfaces() + make_etcresolvconf() + make_etchosts() + make_etchostname() + make_rootssh() + make_l3rc() + +def make_all_system(): + make_drbd_links() + make_fs() + make_mount() + make_install_debian() + make_rsync() + make_etc() + make_umount() + +def get_management_ip(): + return management_ip + +def get_default_gateway(): + return management_gw + +def make_bridges(): + ip=get_management_ip() + gw=get_default_gateway() + print "ifconfig %s 0.0.0.0" % (trunk) + for bridge in bridges: + vlan=vlans[bridges.index(bridge)] + print "vconfig add %s %s" % (trunk,vlan) + print "/etc/xen/scripts/network-bridge start vifnum=%s bridge=%s netdev=%s.%s" % (bridges.index(bridge)+2,bridge,trunk,vlan) + print "ifconfig %s.%s %s" % (trunk,management_vlan,management_ip) + if management_gw: + print "route add default gw %s" % (management_gw) + + +def show_usage(): + print """ +Usage: + xen-drbd-install (to view) + xen-drbd-install | sh -s (to run) + + is a name of the file, which contains network description + +Commands: + make-all + make-bridges + make-links +""" + +#make_all_system() + +if len(sys.argv) > 1: + command = sys.argv[2] + if command == 'make-all': + make_all_system() + if command == 'make-bridges': + make_bridges() + elif command == 'make-links': + make_drbd_links() + else: + show_usage() +else: + show_usage() + diff -r 000000000000 -r b6dc3480caae xen-drbd-start.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xen-drbd-start.py Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,51 @@ +#!/usr/bin/python + + +# External variables: +# * network +# * domain + +import sys + +sys.path.append('/root') +try: + exec 'from %s import *' % (network) +except: + print "Can't find or interpret module %s with topology description" %(network) + sys.exit(1) + +name=domain +N = domains.index(domain) + + +memory=mem_table[domain] + +disk=[] +i=0 +for disk_description in disk_table[domain]: + i+=1 + if disk_description.find(":") == -1: + disk.append('phy:/dev/drbd/%s,hda%s,w' % (domain,i) ) + else: + disk.append('phy:/dev/drbd/%s,hda%s,w' % ((disk_description.split(':'))[0],i) ) + +vif=[] +vbridges = vbridges_table[domain] +x=1 +for i in vbridges: + vif.append('bridge='+i+',mac=00:16:3e:01:'+hex(int(N))[2:]+':'+hex(int('c0',16)+x)[2:]) + x+=1 + +root="/dev/hda1 ro" +extra="" + +def print_config(): + print "name =", name + print "kernel =", kernel + print "ramdisk =", ramdisk + print "memory =", memory + print "disk =", disk + print "vif =", vif + print "root =", root + print "extra =", extra + diff -r 000000000000 -r b6dc3480caae xen-drbd.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xen-drbd.py Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,238 @@ +#!/usr/bin/python + +network='eb' + +import sys,os +from commands import mkarg + +sys.path.append('/etc/xen') +try: + exec 'from %s import *' % (network) +except: + print "Can't find or interpret module %s with topology description" %(network) + sys.exit(1) + +domain_create_line="xm create xen-drbd-start network="+network+" domain=%s" + +def the_peer_of(node): + if node == node1: + another_node=node2 + else: + another_node=node1 + return another_node + +he_is=the_peer_of(i_am) + +def log_error(error): + print error + +def run_now(command,node=i_am): + if node == i_am: + line=command+" > /dev/stderr" + else: + line="ssh %s %s < /dev/null > /dev/stderr" % (node,mkarg(command)) + (p, child_stdout, child_stderr) = os.popen3(line) + output = child_stderr.read() + #p = os.popen(line) + #output = p.read() + p.close() + return output + +def run(command,node=i_am): + run_now(command,node) +# if node == i_am: +# print command +# else: +# print "ssh %s %s < /dev/null > /dev/stderr" % (node,mkarg(command)) + + +def get_drbd_resources(domain): + disk=[] + for disk_description in disk_table[domain]: + if disk_description.find(":") == -1: + disk.append(domain) + else: + disk.append((disk_description.split(':'))[0]) + return disk + + +def set_drbd_primary(domain, node=i_am): + drbd_resources=get_drbd_resources(domain) + for drbd in drbd_resources: + run("drbdadm primary %s"%(drbd),node) + +def set_drbd_secondary(domain, node=i_am): + drbd_resources=get_drbd_resources(domain) + for drbd in drbd_resources: + run("drbdadm secondary %s"%(drbd),node) + +def get_domain_id(domain,node=i_am): + """ + Returns domain id of the or -1 if the is not running on the + """ + res=run_now("xm list | awk '{if ($1 == \"'%s'\") print $2}'" % domain, node).rstrip("\n") + if not res: + res = -1 + else: + res = int(res) + return res + +def get_drbd_id(resource,node=i_am): + res=run_now("ls -l /dev/drbd/%s 2> /dev/null | awk '{print $10}' | sed s@/dev/drbd@@" %resource, node).rstrip("\n") + if not res: + res = -1 + else: + res = int(res) + return res + +def get_drbd_state(resource,node=i_am): + res=run_now("drbdadm state %s | sed s@/.*@@" %resource, node).rstrip("\n") + if not res: + res = -1 + return res + +def get_drbd_cstate(resource,node=i_am): + res=run_now("drbdadm cstate %s " %resource, node).rstrip("\n") + if not res: + res = -1 + return res + + +def start_domain(domain,node=i_am): + if (get_domain_id(domain,i_am) != -1): + log_error("Domain %s is running already on the node %s" % (domain,i_am)) + return -1 + if (get_domain_id(domain,he_is) != -1): + log_error("Domain %s is running already on the node %s" % (domain,he_is)) + return -1 + another_node=the_peer_of(node) + set_drbd_secondary(domain,another_node) + set_drbd_primary(domain,node) + run(domain_create_line % domain) + +def migrate_domain_out(domain,node=i_am): + if (get_domain_id(domain,node) == -1): + log_error("Domain %s is not running on the node %s" % (domain,node)) + return -1 + another_node=the_peer_of(node) + print "Migrating the domain <%s> from the node <%s> to the node <%s>" % (domain, node, another_node) + set_drbd_primary(domain,another_node) + run("xm migrate %s %s --live" % (domain,another_node),node) + run("sleep 2") + set_drbd_secondary(domain,node) + print "+ Done" + +def migrate_domain_in(domain,node=i_am): + migrate_domain_out(domain,the_peer_of(node)) + +def migrate_domain(domain,node): + migrate_domain_in(domain,node) + +def running_domains(node=i_am): + xm_domains=run_now("xm list | awk '{print $1}'", node).split("\n") + return filter(lambda x: x in xm_domains, domains) + +def migrate_all_out(node=i_am): + for domain in running_domains(node): + migrate_domain_out(domain,node) + +def migrate_all_in(node=i_am): + for domain in running_domains(the_peer_of(node)): + migrate_domain_in(domain,node) + +def start_all(node=i_am): + for domain in domains: + if not domain in running_domains(node) and not domain in running_domains(the_peer_of(node)): + start_domain(domain,node) + +def start_my_domains(node=i_am): + for domain in domain_home[node]: + if not domain in running_domains(node) and not domain in running_domains(the_peer_of(node)): + start_domain(domain,node) + +def migrate_my_domains_home(node=i_am): + for domain in domain_home[node]: + if not domain in running_domains(node) and domain in running_domains(the_peer_of(node)): + migrate_domain_in(domain,node) + +def migrate_and_start_all(node=i_am): + migrate_my_domains_home(node) + start_my_domains(node) + +def show_usage(): + print """ +Usage: + xen-drbd command [domain] + +Commands: + start domain + start-all + start-my-domains + + migrate-out domain + migrate-in domain + migrate-all-out + migrate-all-in + migrate-my-domains-home + + migrate-and-start-all + """ + +def test(): + print "get_drbd_resources(samba)=",get_drbd_resources("samba") + print "get_domain_id(Domain-0)=",get_domain_id("Domain-0") + print "get_domain_id(samba)=",get_domain_id("samba") + print "get_drbd_id(samba)=",get_drbd_id("samba") + print "get_drbd_id(samba, he_is)=",get_drbd_id("samba",he_is) + print "get_drbd_id(unknown_resource, he_is)=",get_drbd_id("unknown_resource",he_is) + print "get_drbd_state(samba, he_is)=",get_drbd_state("samba") + print "get_drbd_state(samba, he_is)=",get_drbd_state("samba",he_is) + print "get_drbd_cstate(samba)=",get_drbd_cstate("samba") + print "get_drbd_cstate(samba, he_is)=",get_drbd_cstate("samba",he_is) + print "start_domain(samba)" + start_domain("samba") + print "migrate_domain_out(samba)" + migrate_domain_in("samba") + + +#print "get_domain_id(samba,node2)=",get_domain_id("samba",node2) +#sys.exit(0) + +if len(sys.argv) == 1: + show_usage() + sys.exit(0) + +command=sys.argv[1] +if len(sys.argv) == 3: + domain=sys.argv[2] + if command == 'start': + start_domain(domain) + elif command == 'migrate-out': + migrate_domain_out(domain) + elif command == 'migrate-in': + migrate_domain_in(domain) + else: + show_usage() + sys.exit(0) +elif len(sys.argv) == 2: + if command == 'start-all': + start_all() + elif command == 'start-my-domains': + start_my_domains() + elif command == 'migrate-all-out': + migrate_all_out() + elif command == 'migrate-all-in': + migrate_all_in() + elif command == 'migrate-my-domains-home': + migrate_my_domains_home() + elif command == 'migrate-and-start-all': + migrate_and_start_all() + elif command == 'list': + print running_domains() + else: + show_usage() + sys.exit(0) + + + + diff -r 000000000000 -r b6dc3480caae xen-scripts-network-xen-drbd --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xen-scripts-network-xen-drbd Thu Sep 27 18:56:29 2007 +0300 @@ -0,0 +1,17 @@ +#!/bin/sh + +XEN_DRBD_PATH='/usr/bin' +TOPOLOGY_NAME='xen-drbd' +[ -e /etc/default/xen-drbd ] && . /etc/default/xen-drbd + +case $1 in + start) + $XEN_DRBD_PATH/xen-drbd-install.py ${TOPOLOGY_NAME} make-bridges | sh -s + ;; + stop) + ;; + *) + echo $0 'start|stop' + ;; +esac +