python-boto 2.2.2-0ubuntu2 / usr / share / pyshared / boto / manage / volume.py

# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

from __future__ import with_statement
from boto.sdb.db.model import Model
from boto.sdb.db.property import StringProperty, IntegerProperty, ListProperty, ReferenceProperty, CalculatedProperty
from boto.manage.server import Server
from boto.manage import propget
import boto.utils
import boto.ec2
import time
import traceback
from contextlib import closing
import datetime


class CommandLineGetter(object):
    
    def get_region(self, params):
        if not params.get('region', None):
            prop = self.cls.find_property('region_name')
            params['region'] = propget.get(prop, choices=boto.ec2.regions)

    def get_zone(self, params):
        if not params.get('zone', None):
            prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone',
                                  choices=self.ec2.get_all_zones)
            params['zone'] = propget.get(prop)
            
    def get_name(self, params):
        if not params.get('name', None):
            prop = self.cls.find_property('name')
            params['name'] = propget.get(prop)

    def get_size(self, params):
        if not params.get('size', None):
            prop = IntegerProperty(name='size', verbose_name='Size (GB)')
            params['size'] = propget.get(prop)

    def get_mount_point(self, params):
        if not params.get('mount_point', None):
            prop = self.cls.find_property('mount_point')
            params['mount_point'] = propget.get(prop)

    def get_device(self, params):
        if not params.get('device', None):
            prop = self.cls.find_property('device')
            params['device'] = propget.get(prop)

    def get(self, cls, params):
        self.cls = cls
        self.get_region(params)
        self.ec2 = params['region'].connect()
        self.get_zone(params)
        self.get_name(params)
        self.get_size(params)
        self.get_mount_point(params)
        self.get_device(params)

class Volume(Model):

    name = StringProperty(required=True, unique=True, verbose_name='Name')
    region_name = StringProperty(required=True, verbose_name='EC2 Region')
    zone_name = StringProperty(required=True, verbose_name='EC2 Zone')
    mount_point = StringProperty(verbose_name='Mount Point')
    device = StringProperty(verbose_name="Device Name", default='/dev/sdp')
    volume_id = StringProperty(required=True)
    past_volume_ids = ListProperty(item_type=str)
    server = ReferenceProperty(Server, collection_name='volumes',
                               verbose_name='Server Attached To')
    volume_state = CalculatedProperty(verbose_name="Volume State",
                                      calculated_type=str, use_method=True)
    attachment_state = CalculatedProperty(verbose_name="Attachment State",
                                          calculated_type=str, use_method=True)
    size = CalculatedProperty(verbose_name="Size (GB)",
                              calculated_type=int, use_method=True)

    @classmethod
    def create(cls, **params):
        getter = CommandLineGetter()
        getter.get(cls, params)
        region = params.get('region')
        ec2 = region.connect()
        zone = params.get('zone')
        size = params.get('size')
        ebs_volume = ec2.create_volume(size, zone.name)
        v = cls()
        v.ec2 = ec2
        v.volume_id = ebs_volume.id
        v.name = params.get('name')
        v.mount_point = params.get('mount_point')
        v.device = params.get('device')
        v.region_name = region.name
        v.zone_name = zone.name
        v.put()
        return v

    @classmethod
    def create_from_volume_id(cls, region_name, volume_id, name):
        vol = None
        ec2 = boto.ec2.connect_to_region(region_name)
        rs = ec2.get_all_volumes([volume_id])
        if len(rs) == 1:
            v = rs[0]
            vol = cls()
            vol.volume_id = v.id
            vol.name = name
            vol.region_name = v.region.name
            vol.zone_name = v.zone
            vol.put()
        return vol

    def create_from_latest_snapshot(self, name, size=None):
        snapshot = self.get_snapshots()[-1]
        return self.create_from_snapshot(name, snapshot, size)

    def create_from_snapshot(self, name, snapshot, size=None):
        if size < self.size:
            size = self.size
        ec2 = self.get_ec2_connection()
        if self.zone_name == None or self.zone_name == '':
            # deal with the migration case where the zone is not set in the logical volume:
            current_volume = ec2.get_all_volumes([self.volume_id])[0]
            self.zone_name = current_volume.zone
        ebs_volume = ec2.create_volume(size, self.zone_name, snapshot)
        v = Volume()
        v.ec2 = self.ec2
        v.volume_id = ebs_volume.id
        v.name = name
        v.mount_point = self.mount_point
        v.device = self.device
        v.region_name = self.region_name
        v.zone_name = self.zone_name
        v.put()
        return v
    
    def get_ec2_connection(self):
        if self.server:
            return self.server.ec2
        if not hasattr(self, 'ec2') or self.ec2 == None:
            self.ec2 = boto.ec2.connect_to_region(self.region_name)
        return self.ec2

    def _volume_state(self):
        ec2 = self.get_ec2_connection()
        rs = ec2.get_all_volumes([self.volume_id])
        return rs[0].volume_state()

    def _attachment_state(self):
        ec2 = self.get_ec2_connection()
        rs = ec2.get_all_volumes([self.volume_id])
        return rs[0].attachment_state()

    def _size(self):
        if not hasattr(self, '__size'):
            ec2 = self.get_ec2_connection()
            rs = ec2.get_all_volumes([self.volume_id])
            self.__size = rs[0].size
        return self.__size

    def install_xfs(self):
        if self.server:
            self.server.install('xfsprogs xfsdump')

    def get_snapshots(self):
        """
        Returns a list of all completed snapshots for this volume ID.
        """
        ec2 = self.get_ec2_connection()
        rs = ec2.get_all_snapshots()
        all_vols = [self.volume_id] + self.past_volume_ids
        snaps = []
        for snapshot in rs:
            if snapshot.volume_id in all_vols:
                if snapshot.progress == '100%':
                    snapshot.date = boto.utils.parse_ts(snapshot.start_time)
                    snapshot.keep = True
                    snaps.append(snapshot)
        snaps.sort(cmp=lambda x,y: cmp(x.date, y.date))
        return snaps

    def attach(self, server=None):
        if self.attachment_state == 'attached':
            print 'already attached'
            return None
        if server:
            self.server = server
            self.put()
        ec2 = self.get_ec2_connection()
        ec2.attach_volume(self.volume_id, self.server.instance_id, self.device)

    def detach(self, force=False):
        state = self.attachment_state
        if state == 'available' or state == None or state == 'detaching':
            print 'already detached'
            return None
        ec2 = self.get_ec2_connection()
        ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force)
        self.server = None
        self.put()

    def checkfs(self, use_cmd=None):
        if self.server == None:
            raise ValueError, 'server attribute must be set to run this command'
        # detemine state of file system on volume, only works if attached
        if use_cmd:
            cmd = use_cmd
        else:
            cmd = self.server.get_cmdshell()
        status = cmd.run('xfs_check %s' % self.device)
        if not use_cmd:
            cmd.close()
        if status[1].startswith('bad superblock magic number 0'):
            return False
        return True

    def wait(self):
        if self.server == None:
            raise ValueError, 'server attribute must be set to run this command'
        with closing(self.server.get_cmdshell()) as cmd:
            # wait for the volume device to appear
            cmd = self.server.get_cmdshell()
            while not cmd.exists(self.device):
                boto.log.info('%s still does not exist, waiting 10 seconds' % self.device)
                time.sleep(10)

    def format(self):
        if self.server == None:
            raise ValueError, 'server attribute must be set to run this command'
        status = None
        with closing(self.server.get_cmdshell()) as cmd:
            if not self.checkfs(cmd):
                boto.log.info('make_fs...')
                status = cmd.run('mkfs -t xfs %s' % self.device)
        return status

    def mount(self):
        if self.server == None:
            raise ValueError, 'server attribute must be set to run this command'
        boto.log.info('handle_mount_point')
        with closing(self.server.get_cmdshell()) as cmd:
            cmd = self.server.get_cmdshell()
            if not cmd.isdir(self.mount_point):
                boto.log.info('making directory')
                # mount directory doesn't exist so create it
                cmd.run("mkdir %s" % self.mount_point)
            else:
                boto.log.info('directory exists already')
                status = cmd.run('mount -l')
                lines = status[1].split('\n')
                for line in lines:
                    t = line.split()
                    if t and t[2] == self.mount_point:
                        # something is already mounted at the mount point
                        # unmount that and mount it as /tmp
                        if t[0] != self.device:
                            cmd.run('umount %s' % self.mount_point)
                            cmd.run('mount %s /tmp' % t[0])
                            cmd.run('chmod 777 /tmp')
                            break
            # Mount up our new EBS volume onto mount_point
            cmd.run("mount %s %s" % (self.device, self.mount_point))
            cmd.run('xfs_growfs %s' % self.mount_point)

    def make_ready(self, server):
        self.server = server
        self.put()
        self.install_xfs()
        self.attach()
        self.wait()
        self.format()
        self.mount()

    def freeze(self):
        if self.server:
            return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point)

    def unfreeze(self):
        if self.server:
            return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point)

    def snapshot(self):
        # if this volume is attached to a server
        # we need to freeze the XFS file system
        try:
            self.freeze()
            if self.server == None:
                snapshot = self.get_ec2_connection().create_snapshot(self.volume_id)
            else:
                snapshot = self.server.ec2.create_snapshot(self.volume_id)
            boto.log.info('Snapshot of Volume %s created: %s' %  (self.name, snapshot))
        except Exception:
            boto.log.info('Snapshot error')
            boto.log.info(traceback.format_exc())
        finally:
            status = self.unfreeze()
            return status

    def get_snapshot_range(self, snaps, start_date=None, end_date=None):
        l = []
        for snap in snaps:
            if start_date and end_date:
                if snap.date >= start_date and snap.date <= end_date:
                    l.append(snap)
            elif start_date:
                if snap.date >= start_date:
                    l.append(snap)
            elif end_date:
                if snap.date <= end_date:
                    l.append(snap)
            else:
                l.append(snap)
        return l

    def trim_snapshots(self, delete=False):
        """
        Trim the number of snapshots for this volume.  This method always
        keeps the oldest snapshot.  It then uses the parameters passed in
        to determine how many others should be kept.

        The algorithm is to keep all snapshots from the current day.  Then
        it will keep the first snapshot of the day for the previous seven days.
        Then, it will keep the first snapshot of the week for the previous
        four weeks.  After than, it will keep the first snapshot of the month
        for as many months as there are.

        """
        snaps = self.get_snapshots()
        # Always keep the oldest and the newest
        if len(snaps) <= 2:
            return snaps
        snaps = snaps[1:-1]
        now = datetime.datetime.now(snaps[0].date.tzinfo)
        midnight = datetime.datetime(year=now.year, month=now.month,
                                     day=now.day, tzinfo=now.tzinfo)
        # Keep the first snapshot from each day of the previous week
        one_week = datetime.timedelta(days=7, seconds=60*60)
        print midnight-one_week, midnight
        previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight)
        print previous_week
        if not previous_week:
            return snaps
        current_day = None
        for snap in previous_week:
            if current_day and current_day == snap.date.day:
                snap.keep = False
            else:
                current_day = snap.date.day
        # Get ourselves onto the next full week boundary
        if previous_week:
            week_boundary = previous_week[0].date
            if week_boundary.weekday() != 0:
                delta = datetime.timedelta(days=week_boundary.weekday())
                week_boundary = week_boundary - delta
        # Keep one within this partial week
        partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date)
        if len(partial_week) > 1:
            for snap in partial_week[1:]:
                snap.keep = False
        # Keep the first snapshot of each week for the previous 4 weeks
        for i in range(0,4):
            weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary)
            if len(weeks_worth) > 1:
                for snap in weeks_worth[1:]:
                    snap.keep = False
            week_boundary = week_boundary - one_week
        # Now look through all remaining snaps and keep one per month
        remainder = self.get_snapshot_range(snaps, end_date=week_boundary)
        current_month = None
        for snap in remainder:
            if current_month and current_month == snap.date.month:
                snap.keep = False
            else:
                current_month = snap.date.month
        if delete:
            for snap in snaps:
                if not snap.keep:
                    boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name))
                    snap.delete()
        return snaps
                
    def grow(self, size):
        pass

    def copy(self, snapshot):
        pass

    def get_snapshot_from_date(self, date):
        pass

    def delete(self, delete_ebs_volume=False):
        if delete_ebs_volume:
            self.detach()
            ec2 = self.get_ec2_connection()
            ec2.delete_volume(self.volume_id)
        Model.delete(self)

    def archive(self):
        # snapshot volume, trim snaps, delete volume-id
        pass