Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_flagcmd.py: 49%
1068 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-11-01 07:19 +0000
1import os
2import copy
3import numpy as np
4from collections import defaultdict
6from casatasks import casalog
7from casatools import ms, quanta, table, agentflagger
8from .mstools import write_history
9from . import flaghelper as fh
11qalocal = quanta( )
12tblocal = table( )
14def flagcmd(
15 vis=None,
16 inpmode=None,
17 inpfile=None,
18 tablerows=None,
19 reason=None,
20 useapplied=None,
21 tbuff=None,
22 ants=None,
23 action=None,
24 flagbackup=None,
25 clearall=None,
26 rowlist=None,
27 plotfile=None,
28 savepars=None,
29 outfile=None,
30 overwrite=None
31 ):
33 #
34 # Task flagcmd
35 # Reads flag commands from file or string and applies to MS
37 try:
38 from xml.dom import minidom
39 except Exception as exc:
40 raise ImportError('Failed to load xml.dom.minidom into python: {}'.format(exc))
42 casalog.origin('flagcmd')
44 aflocal = agentflagger()
45 mslocal = ms()
46 mslocal2 = ms()
48 try:
49 # Use a default ntime to open the MS. The user-set ntime will be
50 # used in the tool later
51 ntime = 0.0
53 # Open the MS and attach it to the tool
54 if (type(vis) == str) & os.path.exists(vis):
55 aflocal.open(vis, ntime)
56 else:
57 raise ValueError( 'Visibility data set not found - please verify the name' )
59 # Check if vis is a cal table:
60 # typevis = 1 --> cal table
61 # typevis = 0 --> MS
62 # typevis = 2 --> MMS
63 iscal = False
64 typevis = fh.isCalTable(vis)
65 if typevis == 1:
66 iscal = True
68 if action != 'apply' and action != 'list':
69 raise ValueError( 'Unsupported action for cal tables. Only apply and list are supported.' )
71 if inpmode == 'table' and isinstance(inpfile, str) and inpfile == '':
72 raise ValueError( 'inpmode=\'table\' needs an MS as inpfile' )
74 flagcmds = {}
75 if inpmode == 'table' and fh.isCalTable(inpfile) == 0:
76 # Read flag cmds from the MS
77 flagcmds = readCalCmds(vis, inpfile, [], tablerows, reason, useapplied)
78 listmode = 'cmd'
80 elif inpmode == 'list':
81 # Read flag cmds from a list
82 flagcmds = readCalCmds(vis, '', inpfile, [], reason, True)
83 listmode = ''
84 else:
85 raise ValueError( 'Unsupported inpmode for cal tables' )
87 # Apply flag cmds
88 if len(flagcmds.keys( )) == 0:
89 raise RuntimeError( 'There are no unapplied flags in the input. '\
90 'Set useapplied=True to also use the previously-applied flags.' )
92 # List flags on the screen/logger
93 if action == 'list':
94 casalog.post('Executing action = list')
95 listFlagCmd(myflags=flagcmds, myoutfile='', listmode=listmode)
97 elif action == 'apply':
98 casalog.post('Executing action = apply')
99 applyCalCmds(aflocal, vis, flagcmds, tablerows, flagbackup, outfile)
101 # Save the flag cmds to an output file
102 if savepars:
103 if not overwrite and os.path.exists(outfile):
104 raise ValueError( 'You have set overwrite to False. Remove %s before saving the flag commands'%outfile )
106 fh.writeFlagCommands(vis, flagcmds, False, '', outfile, False)
108 else:
109 # Input vis is an MS
111 # Get overall MS time range for later use (if needed)
112 mslocal2.open(vis)
113 timd = mslocal2.range(['time'])
114 mslocal2.close()
115 if len(timd) != 0:
116 ms_startmjds = timd['time'][0]
117 ms_endmjds = timd['time'][1]
118 t = qalocal.quantity(ms_startmjds, 's')
119 t1sdata = t['value']
120 ms_starttime = qalocal.time(t, form='ymd', prec=9)[0][0]
121 t = qalocal.quantity(ms_endmjds, 's')
122 t2sdata = t['value']
123 ms_endtime = qalocal.time(t, form='ymd', prec=9)[0]
124 # NOTE: could also use values from OBSERVATION table col TIME_RANGE
125 casalog.post('MS spans timerange ' + ms_starttime + ' to '
126 + ms_endtime)
128 myflagcmd = {}
130 if action == 'clear':
131 casalog.post('Action "clear" will disregard inpmode (no reading)')
132 # Clear flag commands from FLAG_CMD in vis
133 msfile = vis
135 if clearall:
136 casalog.post('Deleting all rows from FLAG_CMD in MS '
137 + msfile)
138 clearFlagCmd(msfile, myrowlist=rowlist)
139 else:
140 casalog.post('Safety Mode: you chose not to set clearall=True, no action'
141 )
142 return
144 elif inpmode == 'table':
146 casalog.post('Reading from FLAG_CMD table')
147 # Read from FLAG_CMD table into command list
148 if inpfile == '':
149 msfile = vis
150 else:
151 msfile = inpfile
153 myflagcmd = readFromTable( msfile, myflagrows=tablerows,
154 useapplied=useapplied, myreason=reason)
156 listmode = 'table'
157 elif inpmode == 'list':
158 if action == 'unapply':
159 casalog.post("The unapply action can only be used with inpmode='table'",'WARN')
160 casalog.post("Save the commands to the FLAG_CMD table before using unapply",'WARN')
161 raise ValueError( "Unsupported action='unapply' for inpmode='list'" )
163 # ##### TO DO: take time ranges calculation into account ??????
164 # Parse the input file
165 try:
166 # Input commands are given in a list
167 if isinstance(inpfile, list):
169 # It is a list of input files
170 if os.path.isfile(inpfile[0]):
171 flaglist = []
172 for ifile in inpfile:
173 casalog.post('Will read commands from the file '+ifile)
174 flaglist = flaglist + fh.readFile(ifile)
176 myflagcmd = fh.parseDictionary(flaglist, reason)
178 # It is a list of strings with flag commands
179 else:
180 casalog.post('Will read commands from a Python list')
181 myflagcmd = fh.parseDictionary(inpfile, reason)
182 listmode = ''
184 # Input commands are given in a file
185 elif isinstance(inpfile, str):
187 if inpfile == '':
188 casalog.post('Input file is empty', 'ERROR')
190 casalog.post('Will read commands from the file '+inpfile)
191 flaglist = fh.readFile(inpfile)
192 casalog.post('%s'%flaglist,'DEBUG')
194 myflagcmd = fh.parseDictionary(flaglist, reason)
195 listmode = 'file'
197 else:
198 casalog.post('Input type is not supported', 'ERROR')
200 listmode = 'list'
201 casalog.post('%s'%myflagcmd,'DEBUG1')
203 except Exception as instance:
204 raise Exception( 'Error reading the input list: {} '.format(instance))
207 elif inpmode == 'xml':
209 casalog.post('Reading from Flag.xml')
210 if action == 'unapply':
211 casalog.post("The unapply action can only be used with inpmode='table' or 'list';'",'WARN')
212 casalog.post("save the commands to the FLAG_CMD table before using unapply.",'WARN')
213 raise ValueError( "Unsupported action='unapply' for inpmode='xml'" )
215 # Read from Flag.xml (also needs Antenna.xml)
216 if inpfile == '':
217 flagtable = vis
218 else:
219 flagtable = inpfile
221 # Actually parse table. Fail if Flag.xml or Antenna.xml is not found
222 try:
223 myflags = fh.parseXML(flagtable, mytbuff=tbuff)
224 except Exception as exc:
225 raise RuntimeError('Error while parsing XML: {}'.format(exc))
227 casalog.post('%s' % myflags, 'DEBUG')
229 # Construct flags per antenna, selecting by reason if desired
230 if ants != '' or reason != 'any':
231 myflagcmd = selectXMLFlags(myflags, myantenna=ants,myreason=reason)
232 else:
233 myflagcmd = myflags
235# listmode = 'online'
236 listmode = 'xml'
238 else:
239 raise ValueError( 'Input type is not supported' )
241 # Before performing any action on the flag cmds, check them!
242 vrows = list(myflagcmd.keys())
243 if len(vrows) == 0:
244 raise RuntimeError( 'There are no unapplied flags in the input. Set useapplied=True to also use the previously-applied flags.' )
245 else:
246 casalog.post('Read ' + str(len(vrows))
247 + ' lines from input')
249 casalog.post('Flagcmd dictionary is: %s'%myflagcmd, 'DEBUG1')
251 #
252 # ACTION to perform on input file
253 #
254 casalog.post('Executing action = ' + action)
256 # List the flag commands from inpfile on the screen
257 # and save them or not to outfile
258 if action == 'list':
260 # List the flag cmds on the screen
261 listFlagCommands(myflagcmd, listmode=listmode)
263 # Save the flag cmds to the outfile
264 if savepars:
265 # These cmds came from the internal FLAG_CMD, only list on the screen
266 if outfile == '':
267 if inpmode == 'table' and inpfile == '':
268 pass
269 else:
270 casalog.post('Saving commands to FLAG_CMD')
271 fh.writeFlagCommands(vis, myflagcmd, False,
272 '', '', True)
273 elif not overwrite and os.path.exists(outfile):
274 raise RuntimeError( 'You have set overwrite to False. Remove %s before saving the flag commands'%outfile )
276 else:
277 casalog.post('Saving commands to ' + outfile)
278 fh.writeFlagCommands(vis, myflagcmd, False, '', outfile, False)
280 elif action == 'apply' or action == 'unapply':
282 # Apply/Unapply the flag commands to the data
283 apply = True
285 # Select the data
287 # Select a loose union of the data selection from the list.
288 # The loose union will be calculated for field and spw only.
289 # Antenna, correlation and timerange should be handled by the agent
290 unionpars = {}
291# if len(vrows) > 1:
292# unionpars = fh.parseUnion(vis, myflagcmd)
293# if len(unionpars.keys()) > 0:
294# casalog.post('Pre-selecting a subset of the MS: ')
295# casalog.post('%s' % unionpars)
296# else:
297# casalog.post('Iterating through the entire MS')
299# elif len(vrows) == 1:
300 if len(vrows) == 1:
302 # Get all the selection parameters, but set correlation to ''
303 # if the table was selected by row, we need to
304 # know what is the key number in the dictionary
305 cmd0 = myflagcmd[vrows[0]]['command']
306 unionpars = fh.parseSelectionPars(cmd0)
307 casalog.post('The selected subset of the MS will be: ')
308 casalog.post('%s' % unionpars)
310 aflocal.selectdata(unionpars)
312 # Parse the agents parameters
313 if action == 'unapply':
314 apply = False
316 casalog.post('Parse the parameters for the agents')
317 fh.parseAgents(aflocal, myflagcmd, [], apply, True, '')
319 # Initialize the Agents
320 aflocal.init()
322 # Backup the flags before running
323 if flagbackup:
324 fh.backupFlags(aflocal, msfile='', prename='flagcmd')
326 # Run the tool
327 aflocal.run(True)
329 aflocal.done()
331 # Update the APPLIED column
332 if savepars:
333 # These flags came from internal FLAG_CMD. Always update APPLIED
334 if outfile == '':
335 if inpmode == 'table' and inpfile == '':
336 updateTable(vis, mycol='APPLIED',
337 myval=apply, myrowlist=vrows)
338 else:
339 # save to FLAG_CMD
340 casalog.post('Saving commands to FLAG_CMD')
341 fh.writeFlagCommands(vis, myflagcmd, apply, '',
342 '', True)
343 # Save to a file
344 else:
345 # Still need to update APPLIED column
346 if inpmode == 'table' and inpfile == '':
347 updateTable(vis, mycol='APPLIED',
348 myval=apply, myrowlist=vrows)
350 if not overwrite and os.path.exists(outfile):
351 raise RuntimeError( 'You have set overwrite to False. Remove %s before saving the flag commands'%outfile )
353 else:
354 casalog.post('Saving commands to file '+ outfile)
355 fh.writeFlagCommands(vis, myflagcmd, apply, '', outfile, False)
357 # Do not save cmds but maybe update APPLIED
358 else:
359 if inpmode == 'table' and inpfile == '':
360 updateTable(vis, mycol='APPLIED', myval=apply,
361 myrowlist=vrows)
363 elif action == 'plot':
365 keylist = myflagcmd.keys()
366 if len(keylist) > 0:
367 # Plot flag commands from FLAG_CMD or xml
368 casalog.post('Warning: will only reliably plot individual per-antenna flags'
369 )
370 fns = newplotflags(myflagcmd, plotfile, t1sdata, t2sdata)
371 return {'plotfiles': fns}
373 else:
374 casalog.post('Warning: empty flag dictionary, nothing to plot'
375 )
376 elif action == 'extract':
377 # Make the command dictionary a string again
378 outdict = copy.deepcopy(myflagcmd)
379 for key in myflagcmd.keys():
380 cmddict = myflagcmd[key]['command']
381 cmdline = ""
382 for k,v in cmddict.items():
383 cmdline = cmdline + k + '=' + str(v) + ' '
384 cmdline.rstrip()
385 outdict[key]['command'] = cmdline
387 casalog.post('Returning extracted dictionary')
388 return outdict
391 finally:
392 aflocal.done()
394 # Write history only to action='apply' or 'unapply'
395 # write history
396 if not iscal and (action == 'apply' or action == 'unapply'):
397 try:
398 param_names = flagcmd.__code__.co_varnames[:flagcmd.__code__.co_argcount]
399 vars = locals( )
400 param_vals = [vars[p] for p in param_names]
402 write_history(mslocal, vis, 'flagcmd', param_names,
403 param_vals, casalog)
405 except Exception as instance:
406 casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
407 'WARN')
410# ************************************************************************
411# Helper Functions
412# ************************************************************************
415def readFromTable(
416 msfile,
417 myflagrows=[],
418 useapplied=True,
419 myreason='any',
420 ):
421 '''Read flag commands from rows of the FLAG_CMD table of msfile
422 If useapplied=False then include only rows with APPLIED=False
423 If myreason is anything other than '', then select on that'''
425 #
426 # Return flagcmd structure:
427 #
428 # The flagcmd key is the integer row number from FLAG_CMD
429 #
430 # Dictionary structure:
431 # key : 'id' (string)
432 # 'mode' (string) flag mode '','clip','shadow','quack'
433 # 'antenna' (string)
434 # 'timerange' (string)
435 # 'reason' (string)
436 # 'time' (float) in mjd seconds
437 # 'interval' (float) in mjd seconds
438 # 'cmd' (string) string (for COMMAND col in FLAG_CMD)
439 # 'type' (string) 'FLAG' / 'UNFLAG'
440 # 'applied' (bool) set to True here on read-in
441 # 'level' (int) set to 0 here on read-in
442 # 'severity' (int) set to 0 here on read-in
444 # Open and read columns from FLAG_CMD
445 mstable = os.path.join(msfile,'FLAG_CMD')
447 # Note, tb.getcol doesn't allow random row access, read all
449 try:
450 tblocal.open(mstable)
451 f_time = tblocal.getcol('TIME')
452 f_interval = tblocal.getcol('INTERVAL')
453 f_type = tblocal.getcol('TYPE')
454 f_reas = tblocal.getcol('REASON')
455 f_level = tblocal.getcol('LEVEL')
456 f_severity = tblocal.getcol('SEVERITY')
457 f_applied = tblocal.getcol('APPLIED')
458 f_cmd = tblocal.getcol('COMMAND')
459 tblocal.close()
460 except:
461 casalog.post('Error reading table ' + mstable, 'ERROR')
462 raise Exception
464 nrows = len(f_time)
466 myreaslist = []
468 # Parse myreason
469 if type(myreason) == str:
470 if myreason != 'any':
471 myreaslist.append(myreason)
472 elif type(myreason) == list:
473 myreaslist = myreason
474 else:
475 casalog.post('Cannot read reason; it contains unknown variable types'
476 , 'ERROR')
477 return
479 myflagcmd = {}
481 if nrows > 0:
482 nflagd = 0
483 if len(myflagrows) > 0:
484 rowlist = myflagrows
485 else:
486 rowlist = list(range(nrows))
487 # Prune rows if needed
488 if not useapplied:
489 rowl = []
490 for i in rowlist:
491 if not f_applied[i]:
492 rowl.append(i)
493 rowlist = rowl
494 if len(myreaslist) > 0:
495 rowl = []
496 for i in rowlist:
497 if myreaslist.count(f_reas[i]) > 0:
498 rowl.append(i)
499 rowlist = rowl
501 # Define the way to parse the strings
502 myParser = fh.Parser(' ', '=')
504 for i in rowlist:
505 flagd = {}
506 cmd = f_cmd[i]
507 if cmd == '':
508 casalog.post('Ignoring empty COMMAND string', 'WARN')
509 continue
511 # Extract antenna and timerange strings from cmd
512 flagd['id'] = str(i)
513 flagd['antenna'] = ''
514 flagd['mode'] = ''
515 flagd['timerange'] = ''
516 flagd['time'] = f_time[i]
517 flagd['interval'] = f_interval[i]
518 flagd['type'] = f_type[i]
519 flagd['reason'] = f_reas[i]
520 flagd['level'] = f_level[i]
521 flagd['severity'] = f_severity[i]
522 flagd['applied'] = f_applied[i]
524 # If shadow, remove the addantenna dictionary
525 if cmd.__contains__('shadow') \
526 and cmd.__contains__('addantenna'):
527 i0 = cmd.rfind('addantenna')
528 if cmd[i0 + 11] == '{':
529 # It is a dictionary. Remove it from line
530 i1 = cmd.rfind('}')
531 antpar = cmd[i0 + 11:i1 + 1]
532 temp = cmd[i0:i1 + 1]
533 newcmd = cmd.replace(temp, '')
534 antpardict = fh.convertStringToDict(antpar)
535 flagd['addantenna'] = antpardict
536 cmd = newcmd
538 # Get a dictionary without type evaluation
539 preparsing = myParser.parseNoEval(cmd)
541 # Evaluate the types into a new dictionary
542 parsed = fh.evaluateParameters(preparsing)
544 flagd['command'] = parsed
546 if 'mode' in parsed:
547 flagd['mode'] = parsed['mode']
548 if 'timerange' in parsed:
549 flagd['timerange'] = parsed['timerange']
550 if 'antenna' in parsed:
551 flagd['antenna'] = parsed['antenna']
552 if 'id' in parsed:
553 flagd['id'] = parsed['id']
555 # Keep original key index, might need this later
556 myflagcmd[i] = flagd
557 nflagd += 1
559 else:
560 casalog.post('FLAG_CMD table in %s is empty, no flags extracted'
561 % msfile, 'WARN')
563 return myflagcmd
565def readFromCmd(cmdlist, ms_startmjds, ms_endmjds):
566 '''Read the parameters from a list of commands'''
568 # Read a list of strings and return a dictionary of parameters
569 myflagd = {}
570 nrows = len(cmdlist)
571 if nrows == 0:
572 casalog.post('WARNING: empty flag command list', 'WARN')
573 return myflagd
575 t = qalocal.quantity(ms_startmjds, 's')
576 ms_startdate = qalocal.time(t, form=['ymd', 'no_time'])[0]
577 t0 = qalocal.totime(ms_startdate + '/00:00:00.0')
578 # t0d = qalocal.convert(t0,'d')
579 t0s = qalocal.convert(t0, 's')
581 ncmds = 0
582 for i in range(nrows):
583 cmdstr = cmdlist[i]
584 # break string into key=val sets
585 keyvlist = cmdstr.split()
586 if len(keyvlist) > 0:
587 ant = ''
588 timstr = ''
589 tim = 0.5 * (ms_startmjds + ms_endmjds)
590 intvl = ms_endmjds - ms_startmjds
591 reas = ''
592 cmd = ''
593 fid = str(i)
594 typ = 'FLAG'
595 appl = False
596 levl = 0
597 sevr = 0
598 fmode = ''
599 for keyv in keyvlist:
600 # check for comment character #
601 if keyv.count('#') > 0:
602 # break out of loop parsing keyvals
603 break
604 try:
605 (xkey, val) = keyv.split('=')
606 except:
607 casalog.post('Not a key=val pair: ' + keyv, 'WARN')
608 break
609 xval = val
610 # Use eval to deal with conversion from string
611 # xval = eval(val)
612 # strip quotes from value (if still a string)
613 if type(xval) == str:
614 if xval.count("'") > 0:
615 xval = xval.strip("'")
616 if xval.count('"') > 0:
617 xval = xval.strip('"')
619 # Strip these out of command string
620 if xkey == 'reason':
621 reas = xval
622 elif xkey == 'applied':
623 appl = False
624 if xval == 'True':
625 appl = True
626 elif xkey == 'level':
627 levl = int(xval)
628 elif xkey == 'severity':
629 sevr = int(xval)
630 elif xkey == 'time':
631 tim = xval
632 elif xkey == 'interval':
633 intvl = xval
634 else:
635 # Extract (but keep in string)
636 if xkey == 'timerange':
637 timstr = xval
638 # Extract TIME,INTERVAL
639 try:
640 (startstr, endstr) = timstr.split('~')
641 except:
642 if timstr.count('~') == 0:
643 # casalog.post('Assuming a single start time ')
644 startstr = timstr
645 endstr = timstr
646 else:
647 raise Exception("too may ~'s. Not a start~end range. Error "
648 "parsing " + timstr )
649 t = qalocal.totime(startstr)
650 starts = qalocal.convert(t, 's')
651 if starts['value'] < 1.E6:
652 # assume a time offset from ref
653 starts = qalocal.add(t0s, starts)
654 startmjds = starts['value']
655 if endstr == '':
656 endstr = startstr
657 t = qalocal.totime(endstr)
658 ends = qalocal.convert(t, 's')
659 if ends['value'] < 1.E6:
660 # assume a time offset from ref
661 ends = qalocal.add(t0s, ends)
662 endmjds = ends['value']
663 tim = 0.5 * (startmjds + endmjds)
664 intvl = endmjds - startmjds
665 elif xkey == 'antenna':
667 ant = xval
668 elif xkey == 'id':
669 fid = xval
670 elif xkey == 'unflag':
671 if xval == 'True':
672 typ = 'UNFLAG'
673 elif xkey == 'mode':
674 fmode = xval
675 cmd = cmd + ' ' + keyv
676 # Done parsing keyvals
677 # Make sure there is a non-blank command string after reason/id extraction
678 if cmd != '':
679 flagd = {}
680 flagd['id'] = fid
681 flagd['mode'] = fmode
682 flagd['antenna'] = ant
683 flagd['timerange'] = timstr
684 flagd['reason'] = reas
685 flagd['command'] = cmd
686 flagd['time'] = tim
687 flagd['interval'] = intvl
688 flagd['type'] = typ
689 flagd['level'] = levl
690 flagd['severity'] = sevr
691 flagd['applied'] = appl
692 # Insert into main dictionary
693 myflagd[ncmds] = flagd
694 ncmds += 1
696 casalog.post('Parsed ' + str(ncmds) + ' flag command strings')
698 return myflagd
701def readFromFile(
702 cmdlist,
703 ms_startmjds,
704 ms_endmjds,
705 myreason='',
706 ):
707 '''Parse list of flag command strings and return dictionary of flagcmds
708 Inputs:
709 cmdlist (list,string) list of command strings (default for TIME,INTERVAL)
710 ms_startmjds (float) starting mjd (sec) of MS (default for TIME,INTERVAL)
711 ms_endmjds (float) ending mjd (sec) of MS'''
713#
714# Usage: myflagcmd = getflags(cmdlist)
715#
716# Dictionary structure:
717# fid : 'id' (string)
718# 'mode' (string) flag mode '','clip','shadow','quack'
719# 'antenna' (string)
720# 'timerange' (string)
721# 'reason' (string)
722# 'time' (float) in mjd seconds
723# 'interval' (float) in mjd seconds
724# 'cmd' (string) string (for COMMAND col in FLAG_CMD)
725# 'type' (string) 'FLAG' / 'UNFLAG'
726# 'applied' (bool) set to True here on read-in
727# 'level' (int) set to 0 here on read-in
728# 'severity' (int) set to 0 here on read-in
729#
730# v3.2 Updated STM 2010-12-03 (3.2.0) handle comments # again
731# v3.2 Updated STM 2010-12-08 (3.2.0) bug fixes in flagsort use, parsing
732# v3.3 Updated STM 2010-12-20 (3.2.0) bug fixes parsing errors
733#
735 myflagd = {}
736 nrows = len(cmdlist)
737 if nrows == 0:
738 casalog.post('WARNING: empty flag command list', 'WARN')
739 return myflagd
741 # Parse the reason
742 reasonlist = []
743 if type(myreason) == str:
744 if myreason != '':
745 reasonlist.append(myreason)
746 elif type(myreason) == list:
747 reasonlist = myreason
748 else:
749 casalog.post('Cannot read reason; it contains unknown variable types'
750 , 'ERROR')
751 return
753 t = qalocal.quantity(ms_startmjds, 's')
754 ms_startdate = qalocal.time(t, form=['ymd', 'no_time'])[0]
755 t0 = qalocal.totime(ms_startdate + '/00:00:00.0')
756 # t0d = qalocal.convert(t0,'d')
757 t0s = qalocal.convert(t0, 's')
759 rowlist = []
761 # IMPLEMENT THIS LATER
762 # First select by reason. Simple selection...
763# if len(reasonlist) > 0:
764# for i in range(nrows):
765# cmdstr = cmdlist[i]
766# keyvlist = cmdstr.split()
767# if len(keyvlist) > 0:
768# for keyv in keyvlist:
769# (xkey, xval) = keyv.split('=')
770#
771# if type(xval) == str:
772# if xval.count("'") > 0:
773# xval = xval.strip("'")
774# if xval.count('"') > 0:
775# xval = xval.strip('"')
776#
777# if xkey == 'reason':
778# if reasonlist.count(xval) > 0
779# else:
780 rowlist = list(range(nrows))
782 # Now read the only the commands from the file that satisfies the reason selection
784 ncmds = 0
785# for i in range(nrows):
786 for i in rowlist:
787 cmdstr = cmdlist[i]
789 # break string into key=val sets
790 keyvlist = cmdstr.split()
791 if len(keyvlist) > 0:
792 ant = ''
793 timstr = ''
794 tim = 0.5 * (ms_startmjds + ms_endmjds)
795 intvl = ms_endmjds - ms_startmjds
796 reas = ''
797 cmd = ''
798 fid = str(i)
799 typ = 'FLAG'
800 appl = False
801 levl = 0
802 sevr = 0
803 fmode = ''
804 for keyv in keyvlist:
805 # check for comment character #
806 if keyv.count('#') > 0:
807 # break out of loop parsing keyvals
808 break
809 try:
810 (xkey, val) = keyv.split('=')
811 except:
812 casalog.post('Not a key=val pair: ' + keyv, "WARN")
813 break
814 xval = val
815 # Use eval to deal with conversion from string
816 # xval = eval(val)
817 # strip quotes from value (if still a string)
818 if type(xval) == str:
819 if xval.count("'") > 0:
820 xval = xval.strip("'")
821 if xval.count('"') > 0:
822 xval = xval.strip('"')
824 # Strip these out of command string
825 if xkey == 'reason':
826 reas = xval
827 elif xkey == 'applied':
829 appl = False
830 if xval == 'True':
831 appl = True
832 elif xkey == 'level':
833 levl = int(xval)
834 elif xkey == 'severity':
835 sevr = int(xval)
836 elif xkey == 'time':
837 tim = xval
838 elif xkey == 'interval':
839 intvl = xval
840 else:
841 # Extract (but keep in string)
842 if xkey == 'timerange':
843 timstr = xval
844 # Extract TIME,INTERVAL
845 try:
846 (startstr, endstr) = timstr.split('~')
847 except:
848 if timstr.count('~') == 0:
849 # casalog.post('Assuming a single start time ')
850 startstr = timstr
851 endstr = timstr
852 else:
853 raise Exception( "too may ~'s. Not a start~end range. Error "
854 "parsing " + timstr )
855 t = qalocal.totime(startstr)
856 starts = qalocal.convert(t, 's')
857 if starts['value'] < 1.E6:
858 # assume a time offset from ref
859 starts = qalocal.add(t0s, starts)
860 startmjds = starts['value']
861 if endstr == '':
862 endstr = startstr
863 t = qalocal.totime(endstr)
864 ends = qalocal.convert(t, 's')
865 if ends['value'] < 1.E6:
866 # assume a time offset from ref
867 ends = qalocal.add(t0s, ends)
868 endmjds = ends['value']
869 tim = 0.5 * (startmjds + endmjds)
870 intvl = endmjds - startmjds
871 elif xkey == 'antenna':
873 ant = xval
874 elif xkey == 'id':
875 fid = xval
876 elif xkey == 'unflag':
877 if xval == 'True':
878 typ = 'UNFLAG'
879 elif xkey == 'mode':
880 fmode = xval
881 cmd = cmd + ' ' + keyv
882 # Done parsing keyvals
883 # Make sure there is a non-blank command string after reason/id extraction
884 if cmd != '':
885 flagd = {}
886 flagd['id'] = fid
887 flagd['mode'] = fmode
888 flagd['antenna'] = ant
889 flagd['timerange'] = timstr
890 flagd['reason'] = reas
891 flagd['command'] = cmd
892 flagd['time'] = tim
893 flagd['interval'] = intvl
894 flagd['type'] = typ
895 flagd['level'] = levl
896 flagd['severity'] = sevr
897 flagd['applied'] = appl
898 # Insert into main dictionary
899 myflagd[ncmds] = flagd
900 ncmds += 1
902 casalog.post('Parsed ' + str(ncmds) + ' flag command strings')
904 return myflagd
907def updateTable(
908 msfile,
909 mycol='',
910 myval=None,
911 myrowlist=[],
912 ):
913 '''Update commands in myrowlist of the FLAG_CMD table of msfile
914 Usage: updateflagcmd(msfile,myrow,mycol,myval)'''
916 # Example:
917 #
918 # updateflagcmd(msfile,mycol='APPLIED',myval=True)
919 # Mark all rows as APPLIED=True
920 #
921 # updateflagcmd(msfile,mycol='APPLIED',myval=True,myrowlist=[0,1,2])
922 # Mark rows 0,1,2 as APPLIED=True
923 #
925 if mycol == '':
926 casalog.post('WARNING: No column to was specified to update; doing nothing'
927 , 'WARN')
928 return
930 # Open and read columns from FLAG_CMD
931 mstable = os.path.join(msfile,'FLAG_CMD')
932 try:
933 tblocal.open(mstable, nomodify=False)
934 except:
935 raise Exception( 'Error opening table ' + mstable )
937 nrows = int(tblocal.nrows())
939 # Check against allowed colnames
940 colnames = tblocal.colnames()
941 if colnames.count(mycol) < 1:
942 casalog.post('Error: column mycol=' + mycol + ' not one of: '
943 + str(colnames))
944 return
946 nlist = len(myrowlist)
948 if nlist > 0:
949 rowlist = myrowlist
950 else:
952 rowlist = list(range(nrows))
953 nlist = nrows
955 if nlist > 0:
956 try:
957 tblocal.putcell(mycol, rowlist, myval)
958 except:
959 raise Exception( 'Error updating FLAG_CMD column ' + mycol \
960 + ' to value ' + str(myval) )
962 casalog.post('Updated ' + str(nlist)
963 + ' rows of FLAG_CMD table in MS')
964 tblocal.close()
966def listFlagCommands(myflags=None, listmode=''):
967 '''List flags from MS or a file. The flags are read from
968 a dictionary created by fh.parseDictionary()
969 Format according to listmode:
970 ='' do nothing
971 ='list' Format for flag command strings
972 ='table' Format for FLAG_CMD flags
973 ='xml' Format for online flags'''
975 #
976 # Dictionary structure:
977 # fid : 'id' (string)
978 # 'mode' (string) flag mode '','clip','shadow','quack'
979 # 'antenna' (string)
980 # 'timerange' (string)
981 # 'reason' (string)
982 # 'time' (float) in mjd seconds
983 # 'interval' (float) in mjd seconds
984 # 'cmd' (string) string (for COMMAND col in FLAG_CMD)
985 # 'type' (string) 'FLAG' / 'UNFLAG'
986 # 'applied' (bool)
987 # 'level' (int)
988 # 'severity' (int)
989 #
992 # list to logger and screen
993 if listmode == 'table':
994 phdr = '%-8s %-32s %-7s %s' % (
995 'Row',
996 'Reason',
997 'Applied',
998 'Command'
999 )
1000 casalog.post(phdr)
1001 mydash=80*'-'
1002 casalog.post('%-80s'%mydash)
1003 for k in myflags.keys():
1004# time = myflags[k]['TIME']
1005 row = myflags[k]['id']
1006 reason = myflags[k]['reason']
1007 applied = myflags[k]['applied']
1009 cmddict = myflags[k]['command']
1010 cmdline = ""
1011 for key,val in cmddict.items():
1012 cmdstr = ""
1013 if isinstance(val, str):
1014 # Add quotes to string values
1015 cmdstr = "'"+val+"'"
1016 val = cmdstr
1017 cmdline = cmdline + key + '=' + str(val) + ' '
1019 # Print to logger
1020 pstr = '%-8s %-32s %-7s %s' % (
1021 row, reason,applied,cmdline)
1022 casalog.post(pstr)
1024 elif listmode == 'list':
1025 phdr = '%-8s %-32s %s' % ('Key', 'Reason', 'Command')
1026 casalog.post(phdr)
1027 mydash=80*'-'
1028 casalog.post('%-80s'%mydash)
1029 for k in myflags.keys():
1030 cmddict = myflags[k]['command']
1031 reason = myflags[k]['reason']
1032 if 'reason' in cmddict:
1033 cmddict.pop('reason')
1035 cmdline = ""
1036 for key,val in cmddict.items():
1037 cmdstr = ""
1038 if isinstance(val, str):
1039 # Add quotes to string values
1040 cmdstr = "'"+val+"'"
1041 val = cmdstr
1042 cmdline = cmdline + key + '=' + str(val) + ' '
1044 # Print to logger
1045 pstr = '%-8s %-32s %s' % (k, reason, cmdline)
1046 casalog.post(pstr)
1048 elif listmode == 'xml':
1049 phdr = '%-8s %-8s %-48s %-32s' % ('Key', 'Antenna',
1050 'Timerange', 'Reason')
1051 casalog.post(phdr)
1052 mydash=80*'-'
1053 casalog.post('%-80s'%mydash)
1054 for k in myflags.keys():
1055 reason = ''
1056 antenna = ''
1057 timerange = ''
1058 cmddict = myflags[k]
1059 if 'reason' in cmddict:
1060 reason = cmddict['reason']
1061 if 'antenna' in cmddict:
1062 antenna = cmddict['antenna']
1063 if 'timerange' in cmddict:
1064 timerange = cmddict['timerange']
1066 pstr = '%-8s %-8s %-48s %-32s' % (k, antenna,timerange,reason)
1067 casalog.post(pstr)
1069 return
1071def listFlagCmd(
1072 myflags=None,
1073 myantenna='',
1074 myreason='',
1075 myoutfile='',
1076 listmode='',
1077 ):
1078 '''List flags in myflags dictionary
1080 Format according to listmode:
1081 ='' do nothing
1082 ='file' Format for flag command strings
1083 ='cmd' Format for FLAG_CMD flags
1084 ='online' Format for online flags'''
1086 #
1087 # Dictionary structure:
1088 # fid : 'id' (string)
1089 # 'mode' (string) flag mode '','clip','shadow','quack'
1090 # 'antenna' (string)
1091 # 'timerange' (string)
1092 # 'reason' (string)
1093 # 'time' (float) in mjd seconds
1094 # 'interval' (float) in mjd seconds
1095 # 'cmd' (string) string (for COMMAND col in FLAG_CMD)
1096 # 'type' (string) 'FLAG' / 'UNFLAG'
1097 # 'applied' (bool)
1098 # 'level' (int)
1099 # 'severity' (int)
1100 #
1102 useid = False
1104 if myoutfile != '':
1105 try:
1106 lfout = open(myoutfile, 'w')
1107 except:
1108 raise Exception( 'Error opening list output file ' \
1109 + myoutfile )
1111 keylist = myflags.keys()
1112 if len(keylist) == 0:
1113 casalog.post('There are no flags to list', 'WARN')
1114 return
1115 # Sort keys
1116# keylist.sort
1118 # Set up any selection
1119 if myantenna != '':
1120 casalog.post('Selecting flags by antenna="' + str(myantenna)
1121 + '"')
1122 myantlist = myantenna.split(',')
1124 if myreason != '':
1125 casalog.post('Selecting flags by reason="' + str(myreason) + '"'
1126 )
1127 myreaslist = myreason.split(',')
1129 if listmode == 'online':
1130 phdr = '%8s %12s %8s %32s %48s' % ('Key', 'FlagID', 'Antenna',
1131 'Reason', 'Timerange')
1132 elif listmode == 'cmd':
1133 phdr = '%8s %45s %32s %6s %7s %3s %3s %s' % (
1134 'Row',
1135 'Timerange',
1136 'Reason',
1137 'Type',
1138 'Applied',
1139 'Level',
1140 'Severity',
1141 'Command',
1142 )
1143 elif listmode == 'file':
1144 phdr = '%8s %32s %s' % ('Key', 'Reason', 'Command')
1145 else:
1146 return
1148 if myoutfile != '':
1149 # list to output file
1150 lfout.write(phdr + '\n')
1151 else:
1152 # list to logger
1153 casalog.post(phdr)
1155 # Loop over flags
1156 for key in keylist:
1157 fld = myflags[key]
1158 # Get fields
1159 skey = str(key)
1160 if 'id' in fld and useid:
1161 fid = fld['id']
1162 else:
1163 fid = str(key)
1164 if 'antenna' in fld:
1165 ant = fld['antenna']
1166 else:
1167 ant = 'Unset'
1168 if 'timerange' in fld:
1169 timr = fld['timerange']
1170 else:
1171 timr = 'Unset'
1172 if 'reason' in fld:
1173 reas = fld['reason']
1174 else:
1175 reas = 'Unset'
1176 if 'command' in fld:
1177 cmd = fld['command']
1178 # To be verified
1179# if 'addantenna' in fld:
1180# addantenna = fld['addantenna']
1181# cmd = cmd + ' addantenna=' + str(addantenna)
1182# else:
1184# cmd = 'Unset'
1185 if 'type' in fld:
1186 typ = fld['type']
1187 else:
1188 typ = 'FLAG'
1189 if 'level' in fld:
1190 levl = str(fld['level'])
1191 else:
1192 levl = '0'
1193 if 'severity' in fld:
1194 sevr = str(fld['severity'])
1195 else:
1196 sevr = '0'
1197 if 'applied' in fld:
1198 appl = str(fld['applied'])
1199 else:
1200 appl = 'Unset'
1202 # Print out listing
1203 if myantenna == '' or myantlist.count(ant) > 0:
1204 if myreason == '' or myreaslist.count(reas) > 0:
1205 if listmode == 'online':
1206 pstr = '%8s %12s %8s %32s %48s' % (skey, fid, ant,
1207 reas, timr)
1208 elif listmode == 'cmd':
1209 # Loop over dictionary with commands
1210 cmdline = ""
1211 for k,v in cmd.items():
1212 cmdline = cmdline + k + '=' + str(v) + ' '
1214 cmdline = cmdline.rstrip()
1215 pstr = '%8s %45s %32s %6s %7s %3s %3s %s' % (
1216 skey,
1217 timr,
1218 reas,
1219 typ,
1220 appl,
1221 levl,
1222 sevr,
1223 cmdline,
1224 )
1225 else:
1226 cmdline = ""
1227 for k,v in cmd.items():
1228 cmdline = cmdline + k + '=' + str(v) + ' '
1230 cmdline = cmdline.rstrip()
1231 pstr = '%8s %45s %32s %6s %7s %3s %3s %s' % (
1232 skey,
1233 timr,
1234 reas,
1235 typ,
1236 appl,
1237 levl,
1238 sevr,
1239 cmdline,
1240 )
1241 pstr = '%8s %32s %s' % (skey, reas, cmdline)
1242 if myoutfile != '':
1243 # list to output file
1244 lfout.write(pstr + '\n')
1245 else:
1246 # list to logger
1247 casalog.post(pstr)
1248 if myoutfile != '':
1249 lfout.close()
1252def selectXMLFlags(
1253 myflags=None,
1254 myantenna='',
1255 myreason='any',
1256 ):
1258 #
1259 # Return dictionary of input flags using selection by antenna/reason
1260 # and grouped/sorted by flagsort.
1261 #
1262 # selectFlags: Return dictionary of flags using selection by antenna/reason
1263 # and grouped/sorted by flagsort.
1264 # myflags (dictionary) input flag dictionary (e.g. from readflagxml
1265 # myantenna (string) selection by antenna(s)
1266 # myreason (string) selection by reason(s)
1267 #
1268 # Usage: myflagd = selectFlags(myflags,antenna,reason)
1269 #
1270 # Dictionary structure:
1271 # fid : 'id' (string)
1272 # 'mode' (string) flag mode '','clip','shadow','quack','online'
1273 # 'antenna' (string)
1274 # 'timerange' (string)
1275 # 'reason' (string)
1276 # 'time' (float) in mjd seconds
1277 # 'interval' (float) in mjd seconds
1278 # 'cmd' (string) command string (for COMMAND col in FLAG_CMD)
1279 # 'type' (string) 'FLAG' / 'UNFLAG'
1280 # 'applied' (bool) set to True here on read-in
1281 # 'level' (int) set to 0 here on read-in
1282 # 'severity' (int) set to 0 here on read-in
1283 #
1284 #
1285 #
1286 # Check if any operation is needed
1287 if myantenna == '' and myreason == '':
1288 casalog.post('No selection or sorting needed - sortflags returning input dictionary')
1289 flagd = myflags
1290 return flagd
1291 #
1292 flagd = {}
1293 nflagd = 0
1294 keylist = myflags.keys()
1295 casalog.post('Selecting from ' + str(len(keylist)) \
1296 + ' flagging commands')
1298 if len(keylist) == 0:
1299 casalog.post('No flags found in input dictionary')
1300 return myflags
1302 #
1303 # Sort by key
1304 #
1305 keylist.sort()
1306 #
1307 # Construct flag command list for selected ant,reason
1308 #
1309 casalog.post('Selecting flags by antenna="' + str(myantenna)
1310 + '"')
1311 myantlist = myantenna.split(',')
1313 casalog.post('Selecting flags by reason="' + str(myreason) + '"'
1314 )
1315 myreaslist = []
1316# Parse myreason
1317 if type(myreason) == str:
1318 if myreason == '':
1319 casalog.post('WARNING: reason= is treated as selection on a blank REASON!'
1320 , 'WARN')
1321 if myreason != 'any':
1322 myreaslist.append(myreason)
1323 elif type(myreason) == list:
1324 myreaslist = myreason
1325 else:
1326 casalog.post('ERROR: reason contains unknown variable type'
1327 , 'SEVERE')
1328 return
1329 if len(myreaslist) > 0:
1330 casalog.post('Selecting for reasons: ' + str(myreaslist))
1332# Note antenna and reason selection checks for inclusion not exclusivity
1333 doselect = myantenna != '' or len(myreaslist) > 0
1335# Now loop over flags, break into sorted and unsorted groups
1336 nunsortd = 0
1337 unsortd = {}
1338 unsortdlist = []
1340# All flags are in unsorted list
1341 unsortd = myflags.copy()
1342 unsortdlist = unsortd.keys()
1343 nunsortd = len(unsortdlist)
1345# selection on unsorted flags
1346 if doselect and nunsortd > 0:
1347 keylist = unsortd.keys()
1348 for key in keylist:
1349 myd = unsortd[key]
1350 ant = myd['antenna']
1351 antlist = ant.split(',')
1352 reas = myd['reason']
1353 reaslist = reas.split(',')
1354 # break this flag by antenna
1355 antstr = ''
1356 reastr = ''
1357 addf = False
1359 for a in antlist:
1360 if myantenna == '' or myantlist.count(a) > 0:
1361 addr = False
1362 if len(myreaslist) > 0:
1363 for r in myreaslist:
1364 if reas == r or reaslist.count(r) > 0:
1365 addr = True
1366 # check if this is a new reason
1367 rlist = reastr.split(',')
1368 if reastr != '':
1369 rlist = reastr.split(',')
1370 if rlist.count(r) == 0:
1371 reastr += ',' + r
1372 else:
1373 reastr = r
1374 else:
1375 addr = True
1376 reastr = reas
1377 if addr:
1378 addf = True
1379 if antstr != '':
1380 # check if this is a new antenna
1381 alist = antstr.split(',')
1382 if alist.count(a) == 0:
1383 antstr += ',' + a
1384 else:
1385 antstr = a
1386 if addf:
1387 flagd[nflagd] = myd
1388 flagd[nflagd]['antenna'] = antstr
1389 flagd[nflagd]['reason'] = reastr
1390 nflagd += 1
1391 flagdlist = flagd.keys()
1392 elif nunsortd > 0:
1393 # just copy to flagd w/o selection
1394 flagd = unsortd.copy()
1395 flagdlist = flagd.keys()
1396 nflagd = len(flagdlist)
1398 if nflagd > 0:
1399 casalog.post('Found total of ' + str(nflagd)
1400 + ' flags meeting selection criteria')
1401 else:
1402 casalog.post('No flagging commands found meeting criteria')
1404 return flagd
1407# Done
1410def clearFlagCmd(msfile, myrowlist=[]):
1411 #
1412 # Delete flag commands (rows) from the FLAG_CMD table of msfile
1413 #
1414 # Open and read columns from FLAG_CMD
1416 mstable = os.path.join(msfile,'FLAG_CMD')
1417 try:
1418 tblocal.open(mstable, nomodify=False)
1419 except:
1420 raise Exception( 'Error opening table ' + mstable )
1422 nrows = int(tblocal.nrows())
1423 casalog.post('There were ' + str(nrows) + ' rows in FLAG_CMD')
1424 if nrows > 0:
1425 if len(myrowlist) > 0:
1426 rowlist = myrowlist
1427 else:
1428 rowlist = list(range(nrows))
1429 try:
1430 tblocal.removerows(rowlist)
1431 casalog.post('Deleted ' + str(len(rowlist))
1432 + ' from FLAG_CMD table in MS')
1433 except:
1434 tblocal.close()
1435 raise Exception( 'Error removing rows ' + str(rowlist) \
1436 + ' from table ' + mstable )
1438 else:
1439 casalog.post('No rows to clear')
1441 tblocal.close()
1443def newplotflags(
1444 myflags,
1445 plotname,
1446 t1sdata,
1447 t2sdata,
1448 ):
1449 #
1450 # Function to plot flagging dictionary
1451 # Adapted from J.Marvil
1452 # Updated STM v4.1 2011-11-02 to handle ALMA flags
1453 # Updated STM v4.2 2012-02-16 trim flag times to data times
1454 # Updated STM v4.2 2012-04-10 bug fix in trim flag times to data times
1455 # Updated STM v4.2 2012-04-10 messages to logger
1457 # After the swig converstion, it seems that the following
1458 # line is not needed anymore
1459 # qa = casac.qa = qatool = casac.quanta()
1461 try:
1462 import pylab as pl
1463 except ImportError as exc:
1464 raise ImportError('Failed to load pylab, required by flagcmd: {}'.format(exc))
1466 # list of supported colors (in order)
1467 colorlist = [
1468 'red',
1469 'blue',
1470 'green',
1471 'black',
1472 'cyan',
1473 'magenta',
1474 'yellow',
1475 'orange',
1476 ]
1477 ncolors = len(colorlist)
1479 # get list of flag keys
1480 keylist = myflags.keys()
1482 # get lists of antennas and reasons
1483 # make plotting dictionary
1484 myants = []
1485 myreas = []
1486 plotflag = {}
1487 ipf = 0
1488 for key in keylist:
1489 antstr = myflags[key]['antenna']
1490 reastr = myflags[key]['reason']
1491 timstr = myflags[key]['timerange']
1492 if antstr != '':
1493 # flags that have antenna specified
1494 antlist = antstr.split(',')
1495 nantlist = len(antlist)
1496 else:
1497 # Special
1498 antlist = ['All']
1499 nantlist = 1
1500 #
1501 realist = reastr.split(',')
1502 nrealist = len(realist)
1503 #
1504 timlist = timstr.split(',')
1505 ntimlist = len(timlist)
1506 #
1507 # Break these into nants x ntimes flags
1508 # Trick is assigning multiple reasons
1509 # Normal cases:
1510 # A. One reason, single/multiple antennas x times
1511 # B. Multiple reasons=times, single/multiple antenna(s)
1512 # C. Single time, multiple antennas/reasons
1513 # D. Multiple reasons, no way to correspond with times
1514 #
1515 timmin = 1.0E11
1516 timmax = 0.0
1517 if nrealist == 1:
1518 # simplest case, single reason
1519 reas = realist[0]
1520 if reas == '':
1521 reas = 'Unknown'
1522 if myreas.count(reas) == 0:
1523 myreas.append(reas)
1524 for ia in range(nantlist):
1525 ant = antlist[ia]
1526 if myants.count(ant) == 0:
1527 myants.append(ant)
1528 for it in range(ntimlist):
1529 times = timlist[it]
1530 plotflag[ipf] = {}
1531 plotflag[ipf]['antenna'] = ant
1532 plotflag[ipf]['reason'] = reas
1533 plotflag[ipf]['timerange'] = times
1534 plotflag[ipf]['show'] = True
1535 ipf += 1
1536 elif nrealist == ntimlist:
1537 # corresponding reasons and times
1538 for ia in range(nantlist):
1539 ant = antlist[ia]
1540 if myants.count(ant) == 0:
1541 myants.append(ant)
1542 for it in range(ntimlist):
1543 times = timlist[it]
1544 reas = realist[it]
1545 if reas == '':
1546 reas = 'Unknown'
1547 if myreas.count(reas) == 0:
1548 myreas.append(reas)
1549 plotflag[ipf] = {}
1550 plotflag[ipf]['antenna'] = ant
1551 plotflag[ipf]['reason'] = reas
1552 plotflag[ipf]['timerange'] = times
1553 plotflag[ipf]['show'] = True
1554 ipf += 1
1555 else:
1556 # no correspondence between multiple reasons and ants/times
1557 # assign reason 'Miscellaneous'
1558 reas = 'Miscellaneous'
1559 if myreas.count(reas) == 0:
1560 myreas.append(reas)
1561 for ia in range(nantlist):
1562 ant = antlist[ia]
1563 if myants.count(ant) == 0:
1564 myants.append(ant)
1565 for it in range(ntimlist):
1566 times = timlist[it]
1567 plotflag[ipf] = {}
1568 plotflag[ipf]['antenna'] = ant
1569 plotflag[ipf]['reason'] = reas
1570 plotflag[ipf]['timerange'] = times
1571 plotflag[ipf]['show'] = True
1572 ipf += 1
1574 myants.sort()
1575 nants = len(myants)
1576 nreas = len(myreas)
1577 casalog.post('Found ' + str(nreas) + ' reasons to plot for '
1578 + str(nants) + ' antennas')
1579 npf = ipf
1580 casalog.post('Found ' + str(npf) + ' total flag ranges to plot')
1582 # sort out times
1583 for ipf in range(npf):
1584 times = plotflag[ipf]['timerange']
1585 if times != '':
1586 if times.count('~') > 0:
1587 t1 = times[:times.find('~')]
1588 t2 = times[times.find('~') + 1:]
1589 else:
1590 t1 = times
1591 t2 = t1
1592 (t1s, t2s) = (qalocal.convert(t1, 's')['value'], qalocal.convert(t2,
1593 's')['value'])
1594 plotflag[ipf]['t1s'] = t1s
1595 plotflag[ipf]['t2s'] = t2s
1596 if t1s < timmin:
1597 timmin = t1s
1598 if t2s > timmax:
1599 timmax = t2s
1600 # min,max times
1601 q1 = qalocal.quantity(timmin, 's')
1602 time1 = qalocal.time(q1, form='ymd', prec=9)[0]
1603 q2 = qalocal.quantity(timmax, 's')
1604 time2 = qalocal.time(q2, form='ymd', prec=9)[0]
1605 casalog.post('Found flag times from ' + time1 + ' to ' + time2)
1607 # sort out blank times
1608 for ipf in range(npf):
1609 times = plotflag[ipf]['timerange']
1610 if times == '':
1611 if t2sdata >= t1sdata > 0:
1612 plotflag[ipf]['t1s'] = t1sdata
1613 plotflag[ipf]['t2s'] = t2sdata
1614 else:
1615 plotflag[ipf]['t1s'] = timmin
1616 plotflag[ipf]['t2s'] = timmax
1618 # if flag times are beyond range of data, trim them
1619 # Added STM 2012-02-16, fixed STM 2012-04-10
1620 ndropped = 0
1621 if t2sdata >= t1sdata > 0 and (timmin < t1sdata or timmax
1622 > t2sdata):
1623 # min,max data times
1624 q1 = qalocal.quantity(t1sdata, 's')
1625 tdata1 = qalocal.time(q1, form='ymd', prec=9)[0]
1626 q2 = qalocal.quantity(t2sdata, 's')
1627 tdata2 = qalocal.time(q2, form='ymd', prec=9)[0]
1628 casalog.post('WARNING: Trimming flag times to data limits '
1629 + tdata1 + ' to ' + tdata2)
1631 for ipf in range(npf):
1632 t1s = plotflag[ipf]['t1s']
1633 t2s = plotflag[ipf]['t2s']
1634 if t1s < t1sdata:
1635 if t2s >= t1sdata:
1636 # truncate to t1sdata
1637 plotflag[ipf]['t1s'] = t1sdata
1638 else:
1639 # entirely outside data range, do not plot
1640 plotflag[ipf]['show'] = False
1641 ndropped += 1
1643 if t2s > t2sdata:
1644 if t1s <= t2sdata:
1645 # truncate to t2sdata
1646 plotflag[ipf]['t2s'] = t2sdata
1647 else:
1648 # entirely outside data range, do not plot
1649 plotflag[ipf]['show'] = False
1650 ndropped += 1
1652 if ndropped > 0:
1653 casalog.post('WARNING: Trimming dropped ' + str(ndropped)
1654 + ' flags entirely')
1656 # make reason dictionary with mapping of colors and offsets (-0.3 to 0.3)
1657 readict = {}
1658 reakeys = []
1659 if nreas > ncolors:
1660 for i in range(nreas):
1661 reas = myreas[i]
1662 readict[reas] = {}
1663 if i < ncolors - 1:
1664 colr = colorlist[i]
1665 readict[reas]['color'] = colr
1666 readict[reas]['index'] = i
1667 offs = 0.3 - float(i) * 0.6 / float(ncolors - 1)
1668 readict[reas]['offset'] = offs
1669 reakeys.append(reas)
1670 else:
1671 colr = colorlist[ncolors - 1]
1672 readict[reas]['color'] = colr
1673 readict[reas]['index'] = ncolors - 1
1674 readict[reas]['offset'] = -0.3
1675 reakeys.append('Other')
1676 readict['Other'] = {}
1677 readict['Other']['color'] = colorlist[ncolors - 1]
1678 readict['Other']['index'] = ncolors - 1
1679 readict['Other']['offset'] = -0.3
1680 else:
1681 for i in range(nreas):
1682 reas = myreas[i]
1683 reakeys.append(reas)
1684 colr = colorlist[i]
1685 offs = 0.3 - float(i) * 0.6 / float(ncolors - 1)
1686 readict[reas] = {}
1687 readict[reas]['color'] = colr
1688 readict[reas]['index'] = i
1689 readict[reas]['offset'] = offs
1690 nlegend = len(reakeys)
1691 casalog.post('Will plot ' + str(nlegend) + ' reasons in legend')
1693 if plotname == '':
1694 pl.ion()
1695 else:
1696 pl.ioff()
1698 plotflagperant = defaultdict(list)
1699 for ipf, flag in plotflag.items():
1700 if not flag['show']:
1701 continue
1702 nflag = flag.copy()
1703 nflag['color'] = readict[flag['reason']]['color']
1704 nflag['offset'] = readict[flag['reason']]['offset']
1705 plotflagperant[flag['antenna']].append(nflag)
1707 nplotted = sum(len(x) for x in plotflagperant.values())
1708 casalog.post('Plotted %d flags' % nplotted)
1710 figs = []
1711 figsize = (8, 6)
1712 # maximum number of antennas per plot page (CAS-5187)
1713 antlimit = 28
1714 if len(myants) <= antlimit:
1715 figs.append(pl.figure(figsize=figsize))
1716 _plotants(figs[0], plotflagperant, myants, readict)
1717 else:
1718 # prefer DA on first page
1719 da = [x for x in myants if ('DA' in x) or ('CM' in x)]
1720 no_da = [x for x in myants if x not in da]
1721 # but limit to 28 antennas per figure
1722 if len(da) > antlimit:
1723 no_da.extend(da[antlimit:])
1724 da = da[:antlimit]
1726 if da:
1727 figs.append(pl.figure(figsize=figsize))
1728 _plotants(figs[-1], plotflagperant, da, readict)
1730 # stuff the rest on other figures
1731 while no_da:
1732 figs.append(pl.figure(figsize=figsize))
1733 _plotants(figs[-1], plotflagperant, no_da[:antlimit], readict)
1734 no_da = no_da[antlimit:]
1736 filenames = []
1737 if plotname == '':
1738 pl.draw()
1739 else:
1740 if len(figs) == 1:
1741 figs[0].savefig(plotname, dpi=150)
1742 filenames.append(plotname)
1743 else:
1744 fdirname = os.path.dirname(plotname)
1745 filename, ext = os.path.splitext(os.path.basename(plotname))
1746 for i, f in enumerate(figs):
1747 fn = '%s-%03d%s' % (os.path.join(fdirname, filename), i + 1, ext)
1748 filenames.append(fn)
1749 f.savefig(fn, dpi=150)
1750 return filenames
1753def _plotants(figure, plotflagperant, antlist, readict_inp):
1754 """
1755 plot flags of antennas
1757 Parameters
1758 ----------
1759 figure : matplotlib figure
1760 plotflagperant: dict
1761 dictionary of antennas containing list of flag plot dictionaries
1762 antlist: dict
1763 list of antenna names to plot
1764 readict:
1765 list of reasons for the legend
1766 """
1768 ax1 = figure.add_axes([.15, .1, .75, .85])
1769 nants = len(antlist)
1770 readict = dict()
1771 used_reasons = set()
1772 # These style params can be critical to produce meaningful (or not too
1773 # misleading) plots (CAS-13100)
1774 style_params = {'alpha': .7, 'marker': '.', 'markersize': 1, 'linewidth': 1}
1775 for antind, thisant in enumerate(antlist):
1776 for flag in plotflagperant[thisant]:
1777 thisoffset = flag['offset'] + antind + 1
1778 ax1.plot([flag['t1s'], flag['t2s']], [thisoffset] * 2,
1779 color=flag['color'], **style_params)
1780 used_reasons.add(flag['reason'])
1782 # remove reasons that are not needed
1783 for k, v in readict_inp.items():
1784 if k in used_reasons:
1785 readict[k] = v
1787 myXlim = ax1.get_xlim()
1788 myXrange = myXlim[1] - myXlim[0]
1789 # Pad the time axis?
1790 PadTime = 0.050000000000000003
1791 if PadTime > 0:
1792 xPad = PadTime * myXrange
1793 x0 = myXlim[0] - xPad
1794 x1 = myXlim[1] + xPad
1795 ax1.set_xlim(x0, x1)
1796 myXrange = x1 - x0
1797 else:
1798 # casalog.post(' Rescaled x axis')
1799 x0 = myXlim[0]
1800 x1 = myXlim[1]
1802 legendFontSize = 12
1803 myYupper = nants + len(readict) + 1.5
1804 # place legend text
1805 x = x0 + 0.050000000000000003 * myXrange
1806 for i, reas in enumerate(readict.keys()):
1807 colr = readict[reas]['color']
1808 ax1.text(x, i + nants + 1, reas, color=colr, size=legendFontSize)
1809 ax1.set_ylim([0, myYupper])
1811 ax1.set_yticks(range(1, len(antlist) + 1))
1812 ax1.set_yticklabels(antlist)
1813 # casalog.post(' Relabled y axis')
1815 nxticks = 3
1816 ax1.set_xticks(np.linspace(myXlim[0], myXlim[1], nxticks))
1818 mytime = []
1819 myTimestr = []
1820 for itim in range(nxticks):
1821 time = myXlim[0] + (myXlim[1] - myXlim[0]) * float(itim) \
1822 / float(nxticks - 1)
1823 mytime.append(time)
1824 q1 = qalocal.quantity(time, 's')
1825 time1 = qalocal.time(q1, form='ymd', prec=9)[0]
1826 if itim > 0:
1827 time1s = time1[11:]
1828 else:
1829 time1s = time1
1830 myTimestr.append(time1s)
1832 ax1.set_xticklabels(myTimestr)
1834# def isModeValid(line):
1835# '''Check if mode is valid based on a line
1836# molinede --> line with strings
1837# Returns True if mode is either one of the following:
1838# '',manual,clip,quack,shadow,elevation '''
1839#
1840# if line.__contains__('mode'):
1841# if line.__contains__('manual') or line.__contains__('clip') \
1842# or line.__contains__('quack') or line.__contains__('shadow'
1843# ) or line.__contains__('elevation'):
1844# return True
1845# else:
1846# return False
1847#
1848# # No mode means manual
1849# return True
1852#
1853#******************** CAL TABLE FUNCTIONS **************************
1854#
1855def readCalCmds(caltable, msfile, flaglist, rows, reason, useapplied):
1856 '''Flag a cal table
1858 caltable cal table name
1859 msfile optional MS with flag cmds
1860 flagcmds list with flag cmds or [] when msfile is given
1861 reason select only flag cmds with this reason(s)
1862 useapplied select APPLIED true or false
1863 '''
1865 myflagcmd = {}
1866 if msfile != '':
1867 casalog.post('Reading flag cmds from FLAG_CMD table of MS')
1868 # Read only the selected rows for action = apply and
1869 myflagcmd = readFromTable(msfile, myflagrows=rows, useapplied=useapplied, myreason=reason)
1871 elif flaglist != []:
1872 # Parse the input file
1873 if isinstance(flaglist, list):
1874 casalog.post('Reading from input list')
1875 cmdlist = flaglist
1877 casalog.post('Input ' + str(len(cmdlist))
1878 + ' lines from input list')
1879 # Make a FLAG_CMD compatible dictionary and select by reason
1880 myflagcmd = fh.parseDictionary(cmdlist, reason, False)
1882 elif isinstance(flaglist, str):
1884 casalog.post('Reading from input file '+flaglist)
1885 cmdlist = fh.readFile(flaglist)
1887 # Make a FLAG_CMD compatible dictionary and select by reason
1888 myflagcmd = fh.parseDictionary(cmdlist, reason, False)
1890 else:
1891 casalog.post('Unsupported inpfile type', 'ERROR')
1893 return myflagcmd
1895def applyCalCmds(aflocal, caltable, myflagcmd, tablerows, flagbackup, outfile):
1897 # Get the list of parameters
1898 cmdkeys = list(myflagcmd.keys())
1900 # Select the data
1901 selpars = {}
1902 if len(cmdkeys) == 1:
1903 # Get all the selection parameters, but set correlation to ''
1904 cmd0 = myflagcmd[cmdkeys[0]]['command']
1905 selpars = fh.parseSelectionPars(cmd0)
1906 casalog.post('The selected subset of the MS will be: ')
1907 casalog.post('%s' % selpars)
1909 aflocal.selectdata(selpars)
1911 fh.parseAgents(aflocal, myflagcmd, [], True, True, '')
1913 # Initialize the Agents
1914 aflocal.init()
1916 # Backup the flags before running
1917 if flagbackup:
1918 fh.backupFlags(aflocal, msfile='', prename='flagcmd')
1920 # Run the tool
1921 aflocal.run(True, True)
1923 aflocal.done()