| # Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ |
| # Copyright (c) 2010 Chris Moyer http://coredumped.org/ |
| # |
| # Permission is hereby granted, free of charge, to any person obtaining a |
| # copy of this software and associated documentation files (the |
| # "Software"), to deal in the Software without restriction, including |
| # without limitation the rights to use, copy, modify, merge, publish, dis- |
| # tribute, sublicense, and/or sell copies of the Software, and to permit |
| # persons to whom the Software is furnished to do so, subject to the fol- |
| # lowing conditions: |
| # |
| # The above copyright notice and this permission notice shall be included |
| # in all copies or substantial portions of the Software. |
| # |
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
| # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- |
| # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT |
| # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
| # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| # IN THE SOFTWARE. |
| |
| """ |
| High-level abstraction of an EC2 server |
| """ |
| from __future__ import with_statement |
| import boto.ec2 |
| from boto.mashups.iobject import IObject |
| from boto.pyami.config import BotoConfigPath, Config |
| from boto.sdb.db.model import Model |
| from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty, CalculatedProperty |
| from boto.manage import propget |
| from boto.ec2.zone import Zone |
| from boto.ec2.keypair import KeyPair |
| import os, time, StringIO |
| from contextlib import closing |
| from boto.exception import EC2ResponseError |
| |
| InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', |
| 'c1.medium', 'c1.xlarge', |
| 'm2.2xlarge', 'm2.4xlarge'] |
| |
| class Bundler(object): |
| |
| def __init__(self, server, uname='root'): |
| from boto.manage.cmdshell import SSHClient |
| self.server = server |
| self.uname = uname |
| self.ssh_client = SSHClient(server, uname=uname) |
| |
| def copy_x509(self, key_file, cert_file): |
| print '\tcopying cert and pk over to /mnt directory on server' |
| self.ssh_client.open_sftp() |
| path, name = os.path.split(key_file) |
| self.remote_key_file = '/mnt/%s' % name |
| self.ssh_client.put_file(key_file, self.remote_key_file) |
| path, name = os.path.split(cert_file) |
| self.remote_cert_file = '/mnt/%s' % name |
| self.ssh_client.put_file(cert_file, self.remote_cert_file) |
| print '...complete!' |
| |
| def bundle_image(self, prefix, size, ssh_key): |
| command = "" |
| if self.uname != 'root': |
| command = "sudo " |
| command += 'ec2-bundle-vol ' |
| command += '-c %s -k %s ' % (self.remote_cert_file, self.remote_key_file) |
| command += '-u %s ' % self.server._reservation.owner_id |
| command += '-p %s ' % prefix |
| command += '-s %d ' % size |
| command += '-d /mnt ' |
| if self.server.instance_type == 'm1.small' or self.server.instance_type == 'c1.medium': |
| command += '-r i386' |
| else: |
| command += '-r x86_64' |
| return command |
| |
| def upload_bundle(self, bucket, prefix, ssh_key): |
| command = "" |
| if self.uname != 'root': |
| command = "sudo " |
| command += 'ec2-upload-bundle ' |
| command += '-m /mnt/%s.manifest.xml ' % prefix |
| command += '-b %s ' % bucket |
| command += '-a %s ' % self.server.ec2.aws_access_key_id |
| command += '-s %s ' % self.server.ec2.aws_secret_access_key |
| return command |
| |
| def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, |
| size=None, ssh_key=None, fp=None, clear_history=True): |
| iobject = IObject() |
| if not bucket: |
| bucket = iobject.get_string('Name of S3 bucket') |
| if not prefix: |
| prefix = iobject.get_string('Prefix for AMI file') |
| if not key_file: |
| key_file = iobject.get_filename('Path to RSA private key file') |
| if not cert_file: |
| cert_file = iobject.get_filename('Path to RSA public cert file') |
| if not size: |
| size = iobject.get_int('Size (in MB) of bundled image') |
| if not ssh_key: |
| ssh_key = self.server.get_ssh_key_file() |
| self.copy_x509(key_file, cert_file) |
| if not fp: |
| fp = StringIO.StringIO() |
| fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath) |
| fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ') |
| if clear_history: |
| fp.write('history -c; ') |
| fp.write(self.bundle_image(prefix, size, ssh_key)) |
| fp.write('; ') |
| fp.write(self.upload_bundle(bucket, prefix, ssh_key)) |
| fp.write('; ') |
| fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath) |
| fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys') |
| command = fp.getvalue() |
| print 'running the following command on the remote server:' |
| print command |
| t = self.ssh_client.run(command) |
| print '\t%s' % t[0] |
| print '\t%s' % t[1] |
| print '...complete!' |
| print 'registering image...' |
| self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix)) |
| return self.image_id |
| |
| class CommandLineGetter(object): |
| |
| def get_ami_list(self): |
| my_amis = [] |
| for ami in self.ec2.get_all_images(): |
| # hack alert, need a better way to do this! |
| if ami.location.find('pyami') >= 0: |
| my_amis.append((ami.location, ami)) |
| return my_amis |
| |
| def get_region(self, params): |
| region = params.get('region', None) |
| if isinstance(region, str) or isinstance(region, unicode): |
| region = boto.ec2.get_region(region) |
| params['region'] = region |
| if not region: |
| prop = self.cls.find_property('region_name') |
| params['region'] = propget.get(prop, choices=boto.ec2.regions) |
| self.ec2 = params['region'].connect() |
| |
| def get_name(self, params): |
| if not params.get('name', None): |
| prop = self.cls.find_property('name') |
| params['name'] = propget.get(prop) |
| |
| def get_description(self, params): |
| if not params.get('description', None): |
| prop = self.cls.find_property('description') |
| params['description'] = propget.get(prop) |
| |
| def get_instance_type(self, params): |
| if not params.get('instance_type', None): |
| prop = StringProperty(name='instance_type', verbose_name='Instance Type', |
| choices=InstanceTypes) |
| params['instance_type'] = propget.get(prop) |
| |
| def get_quantity(self, params): |
| if not params.get('quantity', None): |
| prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') |
| params['quantity'] = propget.get(prop) |
| |
| def get_zone(self, params): |
| if not params.get('zone', None): |
| prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', |
| choices=self.ec2.get_all_zones) |
| params['zone'] = propget.get(prop) |
| |
| def get_ami_id(self, params): |
| valid = False |
| while not valid: |
| ami = params.get('ami', None) |
| if not ami: |
| prop = StringProperty(name='ami', verbose_name='AMI') |
| ami = propget.get(prop) |
| try: |
| rs = self.ec2.get_all_images([ami]) |
| if len(rs) == 1: |
| valid = True |
| params['ami'] = rs[0] |
| except EC2ResponseError: |
| pass |
| |
| def get_group(self, params): |
| group = params.get('group', None) |
| if isinstance(group, str) or isinstance(group, unicode): |
| group_list = self.ec2.get_all_security_groups() |
| for g in group_list: |
| if g.name == group: |
| group = g |
| params['group'] = g |
| if not group: |
| prop = StringProperty(name='group', verbose_name='EC2 Security Group', |
| choices=self.ec2.get_all_security_groups) |
| params['group'] = propget.get(prop) |
| |
| def get_key(self, params): |
| keypair = params.get('keypair', None) |
| if isinstance(keypair, str) or isinstance(keypair, unicode): |
| key_list = self.ec2.get_all_key_pairs() |
| for k in key_list: |
| if k.name == keypair: |
| keypair = k.name |
| params['keypair'] = k.name |
| if not keypair: |
| prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair', |
| choices=self.ec2.get_all_key_pairs) |
| params['keypair'] = propget.get(prop).name |
| |
| def get(self, cls, params): |
| self.cls = cls |
| self.get_region(params) |
| self.ec2 = params['region'].connect() |
| self.get_name(params) |
| self.get_description(params) |
| self.get_instance_type(params) |
| self.get_zone(params) |
| self.get_quantity(params) |
| self.get_ami_id(params) |
| self.get_group(params) |
| self.get_key(params) |
| |
| class Server(Model): |
| |
| # |
| # The properties of this object consists of real properties for data that |
| # is not already stored in EC2 somewhere (e.g. name, description) plus |
| # calculated properties for all of the properties that are already in |
| # EC2 (e.g. hostname, security groups, etc.) |
| # |
| name = StringProperty(unique=True, verbose_name="Name") |
| description = StringProperty(verbose_name="Description") |
| region_name = StringProperty(verbose_name="EC2 Region Name") |
| instance_id = StringProperty(verbose_name="EC2 Instance ID") |
| elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address") |
| production = BooleanProperty(verbose_name="Is This Server Production", default=False) |
| ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True) |
| zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True) |
| hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True) |
| private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True) |
| groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True) |
| security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True) |
| key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True) |
| instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True) |
| status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True) |
| launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True) |
| console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True) |
| |
| packages = [] |
| plugins = [] |
| |
| @classmethod |
| def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key): |
| if not cfg.has_section('Credentials'): |
| cfg.add_section('Credentials') |
| cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id) |
| cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key) |
| if not cfg.has_section('DB_Server'): |
| cfg.add_section('DB_Server') |
| cfg.set('DB_Server', 'db_type', 'SimpleDB') |
| cfg.set('DB_Server', 'db_name', cls._manager.domain.name) |
| |
| @classmethod |
| def create(cls, config_file=None, logical_volume = None, cfg = None, **params): |
| """ |
| Create a new instance based on the specified configuration file or the specified |
| configuration and the passed in parameters. |
| |
| If the config_file argument is not None, the configuration is read from there. |
| Otherwise, the cfg argument is used. |
| |
| The config file may include other config files with a #import reference. The included |
| config files must reside in the same directory as the specified file. |
| |
| The logical_volume argument, if supplied, will be used to get the current physical |
| volume ID and use that as an override of the value specified in the config file. This |
| may be useful for debugging purposes when you want to debug with a production config |
| file but a test Volume. |
| |
| The dictionary argument may be used to override any EC2 configuration values in the |
| config file. |
| """ |
| if config_file: |
| cfg = Config(path=config_file) |
| if cfg.has_section('EC2'): |
| # include any EC2 configuration values that aren't specified in params: |
| for option in cfg.options('EC2'): |
| if option not in params: |
| params[option] = cfg.get('EC2', option) |
| getter = CommandLineGetter() |
| getter.get(cls, params) |
| region = params.get('region') |
| ec2 = region.connect() |
| cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key) |
| ami = params.get('ami') |
| kp = params.get('keypair') |
| group = params.get('group') |
| zone = params.get('zone') |
| # deal with possibly passed in logical volume: |
| if logical_volume != None: |
| cfg.set('EBS', 'logical_volume_name', logical_volume.name) |
| cfg_fp = StringIO.StringIO() |
| cfg.write(cfg_fp) |
| # deal with the possibility that zone and/or keypair are strings read from the config file: |
| if isinstance(zone, Zone): |
| zone = zone.name |
| if isinstance(kp, KeyPair): |
| kp = kp.name |
| reservation = ami.run(min_count=1, |
| max_count=params.get('quantity', 1), |
| key_name=kp, |
| security_groups=[group], |
| instance_type=params.get('instance_type'), |
| placement = zone, |
| user_data = cfg_fp.getvalue()) |
| l = [] |
| i = 0 |
| elastic_ip = params.get('elastic_ip') |
| instances = reservation.instances |
| if elastic_ip != None and instances.__len__() > 0: |
| instance = instances[0] |
| print 'Waiting for instance to start so we can set its elastic IP address...' |
| # Sometimes we get a message from ec2 that says that the instance does not exist. |
| # Hopefully the following delay will giv eec2 enough time to get to a stable state: |
| time.sleep(5) |
| while instance.update() != 'running': |
| time.sleep(1) |
| instance.use_ip(elastic_ip) |
| print 'set the elastic IP of the first instance to %s' % elastic_ip |
| for instance in instances: |
| s = cls() |
| s.ec2 = ec2 |
| s.name = params.get('name') + '' if i==0 else str(i) |
| s.description = params.get('description') |
| s.region_name = region.name |
| s.instance_id = instance.id |
| if elastic_ip and i == 0: |
| s.elastic_ip = elastic_ip |
| s.put() |
| l.append(s) |
| i += 1 |
| return l |
| |
| @classmethod |
| def create_from_instance_id(cls, instance_id, name, description=''): |
| regions = boto.ec2.regions() |
| for region in regions: |
| ec2 = region.connect() |
| try: |
| rs = ec2.get_all_instances([instance_id]) |
| except: |
| rs = [] |
| if len(rs) == 1: |
| s = cls() |
| s.ec2 = ec2 |
| s.name = name |
| s.description = description |
| s.region_name = region.name |
| s.instance_id = instance_id |
| s._reservation = rs[0] |
| for instance in s._reservation.instances: |
| if instance.id == instance_id: |
| s._instance = instance |
| s.put() |
| return s |
| return None |
| |
| @classmethod |
| def create_from_current_instances(cls): |
| servers = [] |
| regions = boto.ec2.regions() |
| for region in regions: |
| ec2 = region.connect() |
| rs = ec2.get_all_instances() |
| for reservation in rs: |
| for instance in reservation.instances: |
| try: |
| Server.find(instance_id=instance.id).next() |
| boto.log.info('Server for %s already exists' % instance.id) |
| except StopIteration: |
| s = cls() |
| s.ec2 = ec2 |
| s.name = instance.id |
| s.region_name = region.name |
| s.instance_id = instance.id |
| s._reservation = reservation |
| s.put() |
| servers.append(s) |
| return servers |
| |
| def __init__(self, id=None, **kw): |
| Model.__init__(self, id, **kw) |
| self.ssh_key_file = None |
| self.ec2 = None |
| self._cmdshell = None |
| self._reservation = None |
| self._instance = None |
| self._setup_ec2() |
| |
| def _setup_ec2(self): |
| if self.ec2 and self._instance and self._reservation: |
| return |
| if self.id: |
| if self.region_name: |
| for region in boto.ec2.regions(): |
| if region.name == self.region_name: |
| self.ec2 = region.connect() |
| if self.instance_id and not self._instance: |
| try: |
| rs = self.ec2.get_all_instances([self.instance_id]) |
| if len(rs) >= 1: |
| for instance in rs[0].instances: |
| if instance.id == self.instance_id: |
| self._reservation = rs[0] |
| self._instance = instance |
| except EC2ResponseError: |
| pass |
| |
| def _status(self): |
| status = '' |
| if self._instance: |
| self._instance.update() |
| status = self._instance.state |
| return status |
| |
| def _hostname(self): |
| hostname = '' |
| if self._instance: |
| hostname = self._instance.public_dns_name |
| return hostname |
| |
| def _private_hostname(self): |
| hostname = '' |
| if self._instance: |
| hostname = self._instance.private_dns_name |
| return hostname |
| |
| def _instance_type(self): |
| it = '' |
| if self._instance: |
| it = self._instance.instance_type |
| return it |
| |
| def _launch_time(self): |
| lt = '' |
| if self._instance: |
| lt = self._instance.launch_time |
| return lt |
| |
| def _console_output(self): |
| co = '' |
| if self._instance: |
| co = self._instance.get_console_output() |
| return co |
| |
| def _groups(self): |
| gn = [] |
| if self._reservation: |
| gn = self._reservation.groups |
| return gn |
| |
| def _security_group(self): |
| groups = self._groups() |
| if len(groups) >= 1: |
| return groups[0].id |
| return "" |
| |
| def _zone(self): |
| zone = None |
| if self._instance: |
| zone = self._instance.placement |
| return zone |
| |
| def _key_name(self): |
| kn = None |
| if self._instance: |
| kn = self._instance.key_name |
| return kn |
| |
| def put(self): |
| Model.put(self) |
| self._setup_ec2() |
| |
| def delete(self): |
| if self.production: |
| raise ValueError("Can't delete a production server") |
| #self.stop() |
| Model.delete(self) |
| |
| def stop(self): |
| if self.production: |
| raise ValueError("Can't delete a production server") |
| if self._instance: |
| self._instance.stop() |
| |
| def terminate(self): |
| if self.production: |
| raise ValueError("Can't delete a production server") |
| if self._instance: |
| self._instance.terminate() |
| |
| def reboot(self): |
| if self._instance: |
| self._instance.reboot() |
| |
| def wait(self): |
| while self.status != 'running': |
| time.sleep(5) |
| |
| def get_ssh_key_file(self): |
| if not self.ssh_key_file: |
| ssh_dir = os.path.expanduser('~/.ssh') |
| if os.path.isdir(ssh_dir): |
| ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name) |
| if os.path.isfile(ssh_file): |
| self.ssh_key_file = ssh_file |
| if not self.ssh_key_file: |
| iobject = IObject() |
| self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file') |
| return self.ssh_key_file |
| |
| def get_cmdshell(self): |
| if not self._cmdshell: |
| import cmdshell |
| self.get_ssh_key_file() |
| self._cmdshell = cmdshell.start(self) |
| return self._cmdshell |
| |
| def reset_cmdshell(self): |
| self._cmdshell = None |
| |
| def run(self, command): |
| with closing(self.get_cmdshell()) as cmd: |
| status = cmd.run(command) |
| return status |
| |
| def get_bundler(self, uname='root'): |
| self.get_ssh_key_file() |
| return Bundler(self, uname) |
| |
| def get_ssh_client(self, uname='root', ssh_pwd=None): |
| from boto.manage.cmdshell import SSHClient |
| self.get_ssh_key_file() |
| return SSHClient(self, uname=uname, ssh_pwd=ssh_pwd) |
| |
| def install(self, pkg): |
| return self.run('apt-get -y install %s' % pkg) |
| |
| |
| |
