/lib/galaxy/webapps/reports/controllers/system.py

  1import os
  2import logging
  3from datetime import datetime, timedelta
  4from galaxy.web.base.controller import BaseUIController, web
  5from decimal import Decimal
  6from galaxy import model, util
  7from galaxy.model.orm import and_, desc, eagerload
  8log = logging.getLogger( __name__ )
  9
 10class System( BaseUIController ):
 11    @web.expose
 12    def index( self, trans, **kwd ):
 13        params = util.Params( kwd )
 14        message = ''
 15        if params.userless_histories_days:
 16            userless_histories_days = params.userless_histories_days
 17        else:
 18            userless_histories_days = '60'
 19        if params.deleted_histories_days:
 20            deleted_histories_days = params.deleted_histories_days
 21        else:
 22            deleted_histories_days = '60'
 23        if params.deleted_datasets_days:
 24            deleted_datasets_days = params.deleted_datasets_days
 25        else:
 26            deleted_datasets_days = '60'
 27        file_path, disk_usage, datasets, file_size_str = self.disk_usage( trans, **kwd )
 28        if 'action' in kwd:
 29            if kwd['action'] == "userless_histories":
 30                userless_histories_days, message = self.userless_histories( trans, **kwd )
 31            elif kwd['action'] == "deleted_histories":
 32                deleted_histories_days, message = self.deleted_histories( trans, **kwd )
 33            elif kwd['action'] == "deleted_datasets":
 34                deleted_datasets_days, message = self.deleted_datasets( trans, **kwd )
 35        return trans.fill_template( '/webapps/reports/system.mako',
 36                                    file_path=file_path,
 37                                    disk_usage=disk_usage,
 38                                    datasets=datasets,
 39                                    file_size_str=file_size_str,
 40                                    userless_histories_days=userless_histories_days,
 41                                    deleted_histories_days=deleted_histories_days,
 42                                    deleted_datasets_days=deleted_datasets_days,
 43                                    message=message,
 44                                    nice_size=nice_size )
 45    def userless_histories( self, trans, **kwd ):
 46        """The number of userless histories and associated datasets that have not been updated for the specified number of days."""
 47        params = util.Params( kwd )
 48        message = ''
 49        if params.userless_histories_days:
 50            userless_histories_days = int( params.userless_histories_days )
 51            cutoff_time = datetime.utcnow() - timedelta( days=userless_histories_days )
 52            history_count = 0
 53            dataset_count = 0
 54            for history in trans.sa_session.query( model.History ) \
 55                                           .filter( and_( model.History.table.c.user_id == None,
 56                                                          model.History.table.c.deleted == True,
 57                                                          model.History.table.c.update_time < cutoff_time ) ):
 58                for dataset in history.datasets:
 59                    if not dataset.deleted:
 60                        dataset_count += 1
 61                history_count += 1
 62            message = "%d userless histories ( including a total of %d datasets ) have not been updated for at least %d days." %( history_count, dataset_count, userless_histories_days )
 63        else:
 64            message = "Enter the number of days."
 65        return str( userless_histories_days ), message
 66    def deleted_histories( self, trans, **kwd ):
 67        """
 68        The number of histories that were deleted more than the specified number of days ago, but have not yet been purged.
 69        Also included is the number of datasets associated with the histories.
 70        """
 71        params = util.Params( kwd )
 72        message = ''
 73        if params.deleted_histories_days:
 74            deleted_histories_days = int( params.deleted_histories_days )
 75            cutoff_time = datetime.utcnow() - timedelta( days=deleted_histories_days )
 76            history_count = 0
 77            dataset_count = 0
 78            disk_space = 0
 79            histories = trans.sa_session.query( model.History ) \
 80                                        .filter( and_( model.History.table.c.deleted == True,
 81                                                       model.History.table.c.purged == False,
 82                                                       model.History.table.c.update_time < cutoff_time ) ) \
 83                                        .options( eagerload( 'datasets' ) )
 84
 85            for history in histories:
 86                for hda in history.datasets:
 87                    if not hda.dataset.purged:
 88                        dataset_count += 1
 89                        try:
 90                            disk_space += hda.dataset.file_size
 91                        except:
 92                            pass
 93                history_count += 1
 94            message = "%d histories ( including a total of %d datasets ) were deleted more than %d days ago, but have not yet been purged, " \
 95                    "disk space: %s." % ( history_count, dataset_count, deleted_histories_days, nice_size( disk_space, True ) )
 96        else:
 97            message = "Enter the number of days."
 98        return str( deleted_histories_days ), message
 99    def deleted_datasets( self, trans, **kwd ):
100        """The number of datasets that were deleted more than the specified number of days ago, but have not yet been purged."""
101        params = util.Params( kwd )
102        message = ''
103        if params.deleted_datasets_days:
104            deleted_datasets_days = int( params.deleted_datasets_days )
105            cutoff_time = datetime.utcnow() - timedelta( days=deleted_datasets_days )
106            dataset_count = 0
107            disk_space = 0
108            for dataset in trans.sa_session.query( model.Dataset ) \
109                                           .filter( and_( model.Dataset.table.c.deleted == True,
110                                                          model.Dataset.table.c.purged == False,
111                                                          model.Dataset.table.c.update_time < cutoff_time ) ):
112                dataset_count += 1
113                try:
114                    disk_space += dataset.file_size
115                except:
116                    pass
117            message =  "%d datasets were deleted more than %d days ago, but have not yet been purged," \
118                " disk space: %s." % ( dataset_count, deleted_datasets_days, nice_size( disk_space, True ))
119        else:
120            message = "Enter the number of days."
121        return str( deleted_datasets_days ), message
122    @web.expose
123    def dataset_info( self, trans, **kwd ):
124        params = util.Params( kwd )
125        message = ''
126        dataset = trans.sa_session.query( model.Dataset ).get( trans.security.decode_id( kwd.get( 'id', '' ) ) )
127        # Get all associated hdas and lddas that use the same disk file.
128        associated_hdas = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \
129                                          .filter( and_( trans.model.HistoryDatasetAssociation.deleted == False,
130                                                         trans.model.HistoryDatasetAssociation.dataset_id == dataset.id ) ) \
131                                          .all()
132        associated_lddas = trans.sa_session.query( trans.model.LibraryDatasetDatasetAssociation ) \
133                                           .filter( and_( trans.model.LibraryDatasetDatasetAssociation.deleted == False,
134                                                          trans.model.LibraryDatasetDatasetAssociation.dataset_id == dataset.id ) ) \
135                                           .all()
136        return trans.fill_template( '/webapps/reports/dataset_info.mako',
137                                    dataset=dataset,
138                                    associated_hdas=associated_hdas,
139                                    associated_lddas=associated_lddas,
140                                    message=message )
141    def get_disk_usage( self, file_path ):
142        df_cmd = 'df -h ' + file_path
143        is_sym_link = os.path.islink( file_path )
144        file_system = disk_size = disk_used = disk_avail = disk_cap_pct = mount = None
145        df_file = os.popen( df_cmd )
146        while True:
147            df_line = df_file.readline()
148            df_line = df_line.strip()
149            if df_line:
150                df_line = df_line.lower()
151                if 'filesystem' in df_line or 'proc' in df_line:
152                    continue
153                elif is_sym_link:
154                    if ':' in df_line and '/' in df_line:
155                        mount = df_line
156                    else:
157                        try:
158                            disk_size, disk_used, disk_avail, disk_cap_pct, file_system = df_line.split()
159                            break
160                        except:
161                            pass
162                else:
163                    try:
164                        file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount = df_line.split()
165                        break
166                    except:
167                        pass
168            else:
169                break # EOF
170        df_file.close()
171        return ( file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount  )
172    @web.expose
173    def disk_usage( self, trans, **kwd ):
174        file_path = trans.app.config.file_path
175        disk_usage = self.get_disk_usage( file_path )
176        min_file_size = 2**32 # 4 Gb
177        file_size_str = nice_size( min_file_size )
178        datasets = trans.sa_session.query( model.Dataset ) \
179                                   .filter( and_( model.Dataset.table.c.purged == False,
180                                                  model.Dataset.table.c.file_size > min_file_size ) ) \
181                                   .order_by( desc( model.Dataset.table.c.file_size ) )
182        return file_path, disk_usage, datasets, file_size_str
183
184def nice_size(size, include_bytes=False):
185    """Returns a readably formatted string with the size"""
186    niced = False
187    nice_string = "%s bytes" % size
188    try:
189        nsize = Decimal(size)
190        for x in ['bytes','KB','MB','GB']:
191            if nsize.compare(Decimal("1024.0")) == Decimal("-1"):
192                nice_string = "%3.1f %s" % (nsize, x)
193                niced = True
194                break
195            nsize /= Decimal("1024.0")
196        if not niced:
197            nice_string = "%3.1f %s" % (nsize, 'TB')
198            niced = True
199        if include_bytes and x != 'bytes':
200            nice_string = "%s (%s bytes)" % (nice_string, size)
201    except:
202        pass
203    return nice_string