Skip to content
This repository has been archived by the owner on Feb 23, 2023. It is now read-only.

Commit

Permalink
Merge pull request #69 from pigay/fix-sigpipe
Browse files Browse the repository at this point in the history 
SIGPIPE default behaviour
  • Loading branch information
Pierre Gay authored May 28, 2018
2 parents 30a1164 + bbc2f64 commit c9b43f2
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 62 deletions.
122 changes: 63 additions & 59 deletions Interfaces/scripts/dls.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import os
import getopt
from signal import signal, SIGPIPE, SIG_DFL

from DIRAC import S_OK
from COMDIRAC.Interfaces import DSession
Expand All @@ -17,6 +18,9 @@
from COMDIRAC.Interfaces import ConfigCache
from DIRAC.Core.Base import Script

# broken pipe default behaviour
signal( SIGPIPE, SIG_DFL )

class Params:
def __init__ ( self ):
self.long = False
Expand Down Expand Up @@ -76,7 +80,7 @@ def getSize( self ):
def getHuman( self ):
return self.human

params = Params( )
params = Params()

Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
Expand All @@ -95,7 +99,7 @@ def getHuman( self ):
Script.registerSwitch( "r", "reverse", "reverse sort order", params.setReverse )
Script.registerSwitch( "n", "numericid", "numeric UID and GID", params.setNumericID )
Script.registerSwitch( "S", "size", "size based order", params.setSize )
Script.registerSwitch( "H", "human-readable","size human readable", params.setHuman )
Script.registerSwitch( "H", "human-readable", "size human readable", params.setHuman )

configCache = ConfigCache()
Script.parseCommandLine( ignoreErrors = True )
Expand All @@ -107,66 +111,66 @@ def getHuman( self ):
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI

class ReplicaDirectoryListing( DirectoryListing ):
def addFileWithReplicas( self,name,fileDict,numericid, replicas ):
def addFileWithReplicas( self, name, fileDict, numericid, replicas ):
""" Pretty print of the file ls output with replica info
"""
self.addFile( name, fileDict, replicas, numericid )

self.entries[ -1 ] += tuple( replicas )

def humanReadableSize(self,num,suffix='B'):
def humanReadableSize( self, num, suffix = 'B' ):
""" Translate file size in bytes to human readable
Powers of 2 are used (1Mi = 2^20 = 1048576 bytes).
"""
num = int(num)
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num = int( num )
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs( num ) < 1024.0:
return "%3.1f%s%s" % ( num, unit, suffix )
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
return "%.1f%s%s" % ( num, 'Yi', suffix )

def printListing( self,reverse,timeorder,sizeorder,humanread):
def printListing( self, reverse, timeorder, sizeorder, humanread ):
"""
"""
if timeorder:
if reverse:
self.entries.sort( key=lambda x: x[ 5 ] )
self.entries.sort( key = lambda x: x[ 5 ] )
else:
self.entries.sort( key=lambda x: x[ 5 ],reverse=True )
self.entries.sort( key = lambda x: x[ 5 ], reverse = True )
elif sizeorder:
if reverse:
self.entries.sort(key=lambda x: x[4])
self.entries.sort( key = lambda x: x[4] )
else:
self.entries.sort(key=lambda x: x[4],reverse=True)
self.entries.sort( key = lambda x: x[4], reverse = True )
else:
if reverse:
self.entries.sort( key=lambda x: x[ 6 ],reverse=True )
self.entries.sort( key = lambda x: x[ 6 ], reverse = True )
else:
self.entries.sort( key=lambda x: x[ 6 ] )
self.entries.sort( key = lambda x: x[ 6 ] )

# Determine the field widths
wList = [0] * 7
for d in self.entries:
for i in range( 7 ):
if humanread and i == 4:
humanread_len = len( str( self.humanReadableSize( d[ 4 ] )) )
humanread_len = len( str( self.humanReadableSize( d[ 4 ] ) ) )
if humanread_len > wList[ 4 ]:
wList[ 4 ] = humanread_len
else:
if len( str( d[ i ] )) > wList[ i ]:
wList[ i ] = len( str( d[ i ] ))
if len( str( d[ i ] ) ) > wList[ i ]:
wList[ i ] = len( str( d[ i ] ) )

for e in self.entries:
size = e[ 4 ]
if humanread:
size = self.humanReadableSize(e[ 4 ])
size = self.humanReadableSize( e[ 4 ] )

print str( e[ 0 ] ),
print str( e[ 1 ] ).rjust( wList[ 1 ] ),
print str( e[ 2 ] ).ljust( wList[ 2 ] ),
print str( e[ 3 ] ).ljust( wList[ 3 ] ),
print str( size ).rjust( wList[ 4 ] ),
print str( size ).rjust( wList[ 4 ] ),
print str( e[ 5 ] ).rjust( wList[ 5 ] ),
print str( e[ 6 ] )

Expand All @@ -182,12 +186,12 @@ def getReplicas( self, path ):
result = self.fc.getReplicas( path )
if result[ 'OK' ]:
if result[ 'Value' ][ 'Successful' ]:
for se,entry in result[ 'Value' ][ 'Successful' ][ path ].items( ):
for se, entry in result[ 'Value' ][ 'Successful' ][ path ].items():
replicas.append( se.ljust( 15 ) + " " + entry )
else:
print "Replicas: ", result#[ 'Message' ]
print "Replicas: ", result # [ 'Message' ]
except Exception, x:
replicas.append( "replicas failed:" + str( x ))
replicas.append( "replicas failed:" + str( x ) )
return tuple( replicas )

def do_ls( self, args ):
Expand All @@ -204,7 +208,7 @@ def do_ls( self, args ):
powers of 2 are used (1Mi = 2^20 B).
"""

argss = args.split( )
argss = args.split()
# Get switches
long = False
reverse = False
Expand All @@ -213,19 +217,19 @@ def do_ls( self, args ):
sizeorder = False
humanread = False
short_opts = 'ltrnSH'
long_opts = ['long','timeorder','reverse','numericid','sizeorder','human-readable']
long_opts = ['long', 'timeorder', 'reverse', 'numericid', 'sizeorder', 'human-readable']
path = self.cwd
if len(argss) > 0:
if len( argss ) > 0:
try:
optlist, arguments = getopt.getopt(argss,short_opts,long_opts)
optlist, arguments = getopt.getopt( argss, short_opts, long_opts )
except getopt.GetoptError, e:
print str(e)
print str( e )
print self.do_ls.__doc__
return
# Duplicated options are allowed: later options have precedence, e.g.,
# '-ltSt' will be order by time
# '-ltStS' will be order by size
options = [ opt for (opt, arg) in optlist]
options = [ opt for ( opt, arg ) in optlist]
for opt in options:
if opt in ['-l', '--long']:
long = True
Expand All @@ -241,12 +245,12 @@ def do_ls( self, args ):
humanread = True

if timeorder and sizeorder:
options = [w.replace('--sizeorder','-S') for w in options]
options = [w.replace('--human-readable','-H') for w in options]
options = [w.replace( '--sizeorder', '-S' ) for w in options]
options = [w.replace( '--human-readable', '-H' ) for w in options]
options.reverse()
# The last ['-S','-t'] provided is the one we use: reverse order
# means that the last provided has the smallest index.
if options.index('-S') < options.index('-t'):
if options.index( '-S' ) < options.index( '-t' ):
timeorder = False
else:
sizeorder = False
Expand All @@ -261,27 +265,27 @@ def do_ls( self, args ):
path = tmparg
input_path = True
if path[0] != '/':
path = self.cwd+'/'+path
path = path.replace( r'//','/' )
path = self.cwd + '/' + path
path = path.replace( r'//', '/' )

# remove last character if it is "/"
if path[ -1 ] == '/' and path != '/':
path = path[ :-1 ]

# Check if the target path is a file
result = self.fc.isFile( path )
result = self.fc.isFile( path )
if not result[ 'OK' ]:
print "Error: can not verify path"
return
elif path in result[ 'Value' ][ 'Successful' ] and result[ 'Value' ][ 'Successful' ][ path ]:
result = self.fc.getFileMetadata( path )
dList = ReplicaDirectoryListing( )
dList = ReplicaDirectoryListing()
fileDict = result[ 'Value' ][ 'Successful' ][ path ]

replicas = self.getReplicas( path )

dList.addFileWithReplicas( os.path.basename( path ),fileDict,numericid, replicas )
dList.printListing( reverse,timeorder,sizeorder,humanread)
dList.addFileWithReplicas( os.path.basename( path ), fileDict, numericid, replicas )
dList.printListing( reverse, timeorder, sizeorder, humanread )
return

result = self.fc.isDirectory( path )
Expand All @@ -294,8 +298,8 @@ def do_ls( self, args ):

# Get directory contents now
try:
result = self.fc.listDirectory( path,long )
dList = ReplicaDirectoryListing( )
result = self.fc.listDirectory( path, long )
dList = ReplicaDirectoryListing()
if result[ 'OK' ]:
if result[ 'Value' ][ 'Successful' ]:
for entry in result[ 'Value' ][ 'Successful' ][ path ][ 'Files' ]:
Expand All @@ -305,8 +309,8 @@ def do_ls( self, args ):
if long:
fileDict = result[ 'Value' ][ 'Successful' ][ path ][ 'Files' ][ entry ][ 'MetaData' ]
if fileDict:
replicas = self.getReplicas( os.path.join( path, fname ))
dList.addFileWithReplicas( fname,fileDict,numericid, replicas )
replicas = self.getReplicas( os.path.join( path, fname ) )
dList.addFileWithReplicas( fname, fileDict, numericid, replicas )
else:
dList.addSimpleFile( fname )
for entry in result[ 'Value' ][ 'Successful' ][ path ][ 'SubDirs' ]:
Expand All @@ -316,30 +320,30 @@ def do_ls( self, args ):
if long:
dirDict = result[ 'Value' ][ 'Successful' ][ path ][ 'SubDirs' ][ entry ]
if dirDict:
dList.addDirectory( dname,dirDict,numericid )
dList.addDirectory( dname, dirDict, numericid )
else:
dList.addSimpleFile( dname )
for entry in result[ 'Value' ][ 'Successful' ][ path ][ 'Links' ]:
pass

if long:
dList.printListing( reverse,timeorder,sizeorder,humanread)
dList.printListing( reverse, timeorder, sizeorder, humanread )
else:
dList.printOrdered( )
dList.printOrdered()
else:
print "Error:",result[ 'Message' ]
print "Error:", result[ 'Message' ]
except Exception, x:
print "Error:", str( x )

session = DSession( )
session = DSession()

fccli = None

if params.getReplicas( ):
fccli = ReplicaFileCatalogClientCLI( createCatalog( ) )
if params.getReplicas():
fccli = ReplicaFileCatalogClientCLI( createCatalog() )
params.setLong( None )
else:
fccli = FileCatalogClientCLI( createCatalog( ) )
fccli = FileCatalogClientCLI( createCatalog() )

optstr = ""
if params.long: optstr += "l"
Expand All @@ -354,28 +358,28 @@ def do_ls( self, args ):
# This would introduce some duplication of code :-(
if params.long and params.time and params.size:
short_opts = 'ltrnSH'
long_opts = ['long','timeorder','reverse','numericid','sizeorder','human-readable']
long_opts = ['long', 'timeorder', 'reverse', 'numericid', 'sizeorder', 'human-readable']
try:
optlist, arguments = getopt.getopt( sys.argv[1:],short_opts,long_opts )
options = [ opt for (opt, arg) in optlist ]
options = [ w.replace('--size','-S') for w in options ]
options = [ w.replace('--human-readable','-H') for w in options ]
optlist, arguments = getopt.getopt( sys.argv[1:], short_opts, long_opts )
options = [ opt for ( opt, arg ) in optlist ]
options = [ w.replace( '--size', '-S' ) for w in options ]
options = [ w.replace( '--human-readable', '-H' ) for w in options ]
options.reverse()
# The last ['-S','-t'] provided is the one we use: reverse order
# means that the last provided has the smallest index.
# Indeed, setTime/setSize dont has impact: the important thing is
# to add '-S'/'-t' at the end, then do_ls takes care of the rest.
if options.index('-S') < options.index('-t'):
if options.index( '-S' ) < options.index( '-t' ):
params.setTime( False )
optstr = optstr + '-S '
else:
params.setSize( False )
optstr = optstr + '-t '

except getopt.GetoptError, e:
print str(e)
print str( e )
print fccli.do_ls.__doc__
exit(1)
exit( 1 )

for p in pathFromArguments( session, args ):
print "%s:" % p
Expand Down
11 changes: 8 additions & 3 deletions Interfaces/scripts/dstat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"""
__RCSID__ = "$Id$"

from signal import signal, SIGPIPE, SIG_DFL

from COMDIRAC.Interfaces import ConfigCache
from DIRAC.Core.Base import Script
Expand All @@ -19,6 +20,10 @@
'Running', 'Stalled', 'Done', 'Completed', 'Failed']
JOB_FINAL_STATES = ['Done', 'Completed', 'Failed']

# broken pipe default behaviour
signal( SIGPIPE, SIG_DFL )


def selectJobs( owner, date, jobGroup, jobName ):
conditions = {'Owner' : owner}
if jobGroup: conditions["JobGroup"] = jobGroup
Expand Down Expand Up @@ -129,20 +134,20 @@ def setInputFile( self, arg = None ):
def getInputFile( self ):
return self.inputFile

params = Params( )
params = Params()

Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ' % Script.scriptName,
'Arguments:', ] ) )
Script.registerSwitch( "u:", "User=", "job owner", params.setUser )
Script.registerSwitch( "S:", "Status=","select job by status (comma separated list of statuses in: %s)" % ','.join(JOB_STATES), params.setStatus )
Script.registerSwitch( "S:", "Status=", "select job by status (comma separated list of statuses in: %s)" % ','.join( JOB_STATES ), params.setStatus )
Script.registerSwitch( "a", "StatusAll", "display jobs of any status", params.setStatusAll )
Script.registerSwitch( "g:", "JobGroup=", "select job by job group", params.setJobGroup )
Script.registerSwitch( "n:", "JobName=", "select job by job name", params.setJobName )
Script.registerSwitch( "f:", "Fmt=", "display format (pretty, csv, json)", params.setFmt )
Script.registerSwitch( "D:", "JobDate=", "age of jobs to display (in days)", params.setJobDate )
Script.registerSwitch( "F:", "Fields=", "display list of job fields (comma separated list of fields. e.g. %s)" % ','.join(DEFAULT_DISPLAY_COLUMNS + EXTRA_DISPLAY_COLUMNS), params.setFields )
Script.registerSwitch( "F:", "Fields=", "display list of job fields (comma separated list of fields. e.g. %s)" % ','.join( DEFAULT_DISPLAY_COLUMNS + EXTRA_DISPLAY_COLUMNS ), params.setFields )
Script.registerSwitch( "i:", "input-file=", "read JobIDs from file", params.setInputFile )

configCache = ConfigCache()
Expand Down

0 comments on commit c9b43f2

Please sign in to comment.