Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/flaghelper.py: 3%
2349 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-10-31 18:48 +0000
1import os
2import time
3import ast
4import copy
5import numpy
6import inspect
8from collections import deque,defaultdict
9from collections import OrderedDict
11from casatasks import casalog, flagdata
12from casatools import table,quanta,ms,agentflagger
13from .parallel.parallel_task_helper import ParallelTaskHelper
15###some helper tools
16tblocal = table()
17mslocal = ms()
18qalocal = quanta()
20'''
21A set of helper functions for the tasks flagdata and flagcmd.
22Class Parser: to parse flag commands
24I/O functions:
25 get_flag_cmd_list
26 readFile
27 readFiles
28 readAndParse
29 parseDictionary
30 parseXML
31 readAntennaList
32 writeAntennaList
33 writeFlagCommands
34 writeRflagThresholdFile
36Parameter handling
37 compressSelectionList
38 evalParams
39 selectReason
40 parseSelectionPars
41 parseUnion
42 purgeEmptyPars
43 purgeParameter
44 parseAgents
46Others
47 backupFlags
48 convertDictToString
49 convertStringToDict
50 extractAntennaInfo
51 save_rflag_consolidated_files
52 parseRflagOutputFromSummary
54'''
56debug = False
58def get_task_arg_default( func, arg ):
59 spec = inspect.getfullargspec(func.__call__)
61 if arg not in spec.args:
62 raise Exception("cannot find '%s' among the function arguments" % arg)
63 return spec.defaults[spec.args.index(arg)-1]
65class Parser():
66 ''' Parser for input files.
67 primarydivider --> first split the string by this character
68 secondarydivider --> next split the string by this character
70 The constructor takes two separators.
71 It first splits the string by the 'primarydivider'. It
72 then loops through each entry after the first split and
73 verifies if the 'secondarydivider' is in the string. If yes,
74 the string is added to a list. If not, it removes the
75 whitespace of the initial split, by bringing back two
76 strings together. This will allow the primarydivider to be
77 part of the string itself. Last thing, it returns an ordered
78 dictionary using the imported class OrderedDict.
79 '''
80 def __init__(self,primarydivider,secondarydivider):
81 self.prime = primarydivider
82 self.second = secondarydivider
84 def parse2Dictionary(self,string):
85 res = self.initialsplit(string)
86 new = []
87 for entry in res:
88 if self.second not in entry:
89 new[-1] += ' ' + entry
90 else:
91 new.append(entry)
92# return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new])
93 return OrderedDict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new])
95 def parseNoEval(self,string):
96 res = self.initialsplit(string)
97 new = []
98 for entry in res:
99 if self.second not in entry:
100 new[-1] += ' ' + entry
101 else:
102 new.append(entry)
103# return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new])
104 return OrderedDict((entry[0],entry[1]) for entry in [entry.split(self.second,1) for entry in new])
106 def parse2List(self,string):
107 res = self.initialsplit(string)
108 new = []
109 for entry in res:
110 if self.second not in entry:
111 new[-1] += ' ' + entry
112 else:
113 new.append(entry)
114 return new
115# return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new])
116# return OrderedDict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new])
118 def initialsplit(self,string):
119 nstring = string.strip()
120 return nstring.split(self.prime)
122#######################################################
123#
124# Reading functions
125#
126#######################################################
127def isCalTable(msname):
128 '''Check if a file is a cal table
129 msname --> filename
130 Return 1 for cal table, 0 for MS and 2 for MMS'''
132 try:
133 tblocal.open(msname)
134 except:
135 raise ValueError("Unable to open MS %s" % msname)
137 tbinfo = tblocal.info()
138 tblocal.close()
139 retval = 0
141 if tbinfo['type'] == 'Calibration':
142 retval = 1
144 elif tbinfo['type'] == 'Measurement Set':
145 # MMS type
146 if tbinfo['subType'] == 'CONCATENATED' and ParallelTaskHelper.isParallelMS(msname):
147 retval = 2
148 else:
149 # MS type
150 retval = 0
151 else:
152 retval = 0
155 return retval
157def addAbsolutePath(input_file):
158 '''Read in the lines from an input file
159 input_file --> file in disk with a list of strinZeeuwgs per line
161 Return a new file saved to disk with relative file names
162 changed to absolute file names. The new file will have the name
163 of the input_file + '.tmp'
164 '''
166 # Get a list of the flag commands from the file
167 cmdlist = readFile(input_file)
169 # Make a dictionary of the flag commands
170 flagdict = parseDictionary(cmdlist, 'any', False)
172 for key in flagdict:
173 cmddict = flagdict[key]['command']
174 if 'addantenna' in cmddict and isinstance(cmddict['addantenna'],str) and \
175 cmddict['addantenna'] != '':
176 cmddict['addantenna'] = os.path.abspath(cmddict['addantenna'])
177 if 'timedev' in cmddict and isinstance(cmddict['timedev'],str) and \
178 cmddict['timedev'] != '':
179 cmddict['timedev'] = os.path.abspath(cmddict['timedev'])
180 if 'freqdev' in cmddict and isinstance(cmddict['freqdev'],str) and \
181 cmddict['freqdev'] != '':
182 cmddict['freqdev'] = os.path.abspath(cmddict['freqdev'])
185 # Convert the dictionary back to a list and save it
186 output_file = os.path.abspath(input_file + ".tmp")
187 writeFlagCommands('', flagdict, False, '', output_file, append=False)
189 # Return the temporary file
190 return output_file
192def get_flag_cmd_list(inpfile):
193 """
194 For flagdata list mode, get the list of commands from whatever has been given in the
195 inpfile input parameter (a file, a list of files, or a Python string with commands).
196 The output list has one item (command) per line in the input file(s) or string.
198 :param inpfile: inpfile parameter as passed to task flagdata
200 :returns: list of commands found in file(s) or string
201 """
202 # inpfile is a file
203 if isinstance(inpfile, str) and os.path.isfile(inpfile):
204 flaglist = readFile(inpfile)
205 nlines = len(flaglist)
206 casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile))
208 # inpfile is a list of files
209 elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]):
210 flaglist = readFiles(inpfile)
212 # Python list of strings
213 elif isinstance(inpfile, list):
214 flaglist = inpfile
216 else:
217 raise ValueError('Unsupported input list of flag commands or input file does not '
218 'exist')
220 return flaglist
222def readFile(inputfile):
223 '''Read in the lines from an input file
224 inputfile --> file in disk with a list of strings per line
226 Returns a list. Blank lines are skipped. Boolean values will
227 be capitalized to avoid errors in the parser later.
228 '''
230 flagfile = inputfile
232 if (type(flagfile) == str) & os.path.exists(flagfile):
233 try:
234 ff = open(flagfile, 'r')
235 except:
236 casalog.post('Error opening file ' + flagfile,'ERROR')
237 raise
238 else:
239 casalog.post('ASCII file not found - please verify the name','ERROR')
240 raise Exception('ASCII file not found')
242 # Parse file
243 try:
244 cmdlist = []
245 for line in ff:
246 cmd = line.rstrip()
247 if cmd == '':
248 continue
249 if cmd.startswith('#'):
250 continue
251 uppercmd = cmd.replace('true','True')
252 cmd = uppercmd.replace('false','False')
254 cmdlist.append(cmd)
256 except:
257 casalog.post('Error reading lines from file '+ff.name, 'SEVERE')
258 ff.close()
259 raise
261 ff.close()
263 return cmdlist
265def readFiles(inputfiles):
266 '''Read in a list of files with flag commands
267 inputfiles --> list of files in disk
269 Returns all files concatenated into one single list. Blank lines are skipped.
270 Boolean values will be capitalized to avoid errors in the parser later.
271 '''
273 if not isinstance(inputfiles, list):
274 casalog.post('Error opening list of flag commands ' + inputfiles,'ERROR')
275 raise
277 cmdlist = []
279 # Read files
280 for flagfile in inputfiles:
281 cmd = readFile(flagfile)
282 nlines = len(cmd)
283 casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile))
284 cmdlist = cmdlist + cmd
286 return cmdlist
288def readAndParse(inputlist, tbuff=None):
289 '''Read in a list of flag commands and parse them into a dictionary.
290 The flag commands can be from a list of files or from a list of flag commands.
291 If tbuff is given, it will be applied to the timerange parameters.
293 Note: when tbuff is None, it is best to use the individual functions
294 readFile(s) and parseDictionary to avoid parsing the file twice!
296 inputlist --> list of files in disk containing the flag commands or
297 a list of Python strings with flag commands.
298 tbuff --> list of time buffers to apply to each timerange. If tbuff
299 is a list, it should be the same size of the list of inputfiles.
300 If it is a Double value, it will be applied only to the first
301 input file.
303 Returns a list of dictionaries. Blank lines are skipped. Boolean values will
304 be capitalized to avoid errors in the parser.
306 Each parameter=value will become a key=value in the dictionary. Example:
307 inputlist = ["mode='manual' spw='0' autocorr=true",
308 "mode='shadow'"]
310 Returns:
311 [{'mode':'manual','spw':'0','autocorr':True},
312 {'mode':'shadow'}]
314 '''
315 if not isinstance(inputlist, list):
316 casalog.post('Error opening list of flag commands ' + inputlist,'ERROR')
317 raise
319 # List of files
320 if os.path.isfile(inputlist[0]):
321 isFile = True
323 # List of strings
324 else:
325 isFile = False
327 if tbuff is None:
328 doPadding = False
329 else:
330 doPadding = True
332 # Make the list of tbuff a deque
333# dtbuff = deque()
334#
335# if isinstance(tbuff, float):
336# dtbuff.append(tbuff)
337# elif isinstance(tbuff, list):
338# dtbuff = deque(i for i in tbuff)
340 # List of dictionaries to return
341 listofdict = []
343 # Initialize the parser
344 myParser = Parser(' ', '=')
346 # Read files
347 if isFile:
348 for flagfile in inputlist:
349 cmdlist = readFile(flagfile)
350 nlines = len(cmdlist)
351 casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile))
353 parsedlist = []
354 for cmd in cmdlist:
355 #Get a dictionary without type evaluation
356 preparsing = myParser.parseNoEval(cmd)
358 # Evaluate the types
359 parsed = evaluateParameters(preparsing)
360 parsedlist.append(parsed)
362 # Apply time buffer to file
363 if doPadding:
364# mytbuff = dtbuff.popleft()
365 applyTimeBufferList(parsedlist, tbuff)
366# if dtbuff.__len__() == 0:
367# doPadding = False
369 listofdict = listofdict + parsedlist
371 # It is a list of strings
372 else:
374 cmdlist = inputlist
375 nlines = len(cmdlist)
376 casalog.post('Read %s command(s) from a Python list of strings'%nlines)
378 parsedlist = []
379 for cmd in cmdlist:
380 #Get a dictionary without type evaluation
381 preparsing = myParser.parseNoEval(cmd)
383 # Evaluate the types
384 parsed = evaluateParameters(preparsing)
385 parsedlist.append(parsed)
387 # Apply time buffer to list
388 if doPadding:
389# mytbuff = dtbuff.popleft()
390 applyTimeBufferList(parsedlist, tbuff)
392 listofdict = listofdict + parsedlist
394 return listofdict
396def applyTimeBufferList(alist, tbuff=None):
397 ''' Apply in-place a time buffer to ALL timerange parameters of a
398 list of dictionaries with several flag commands. It will do the following:
400 alist --> list of dictionaries with flag commands.
401 Ex: [{'antenna':'DV01', 'timerange':'2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'},
402 {'antenna':'DV02', 'timerange':'2013/10/15/10:25:30.110~2013/10/15/10:25:32.230'},
403 ...]
404 tbuff --> float value or list of 2 values of time buffer to apply to all timerange parameters.
405 When tbuff is a list of 2 values, the first value is applied to the lower time,
406 the second to the upper time.
408 * it assumes that timerange has syntax t0~t1
409 * split timerange in '~' to get t0 and t1
410 * convert value to time in days using qalocal.totime
411 * convert days to seconds
412 * subtract tbuff0 from t0 and add tbuff1 to t1
413 * convert back to time string with the form 'ymd' using qalocal.time
414 * write new values back to dictionary
416 '''
417# if not isinstance(tbuff, float):
418# casalog.post('Time buffer (tbuff) is not of type float', 'WARN')
419# return
421 casalog.post('Apply time buffer padding to list of dictionaries', 'DEBUG1')
423 # When tbuff is float, the range is regular, otherwise it's irregular
424 if isinstance(tbuff, list) and len(tbuff) == 2:
425 tbuff0 = tbuff[0]
426 tbuff1 = tbuff[1]
427 elif isinstance(tbuff, list) and len(tbuff) == 1:
428 tbuff0 = tbuff1 = tbuff[0]
429 elif isinstance(tbuff, float):
430 tbuff0 = tbuff1 = tbuff
431 else:
432 casalog.post('Time buffer (tbuff) is not of type float or list', 'WARN')
433 return
435 def get_date_format(timestamp):
436 '''Guess date-time string format to use in quanta tool time conversion,
437 depending on whether the date seems to be present (as in
438 2013/11/15/10:35:05)
439 '''
440 if timestamp.count('/') == 3:
441 return 'ymd'
442 else:
443 return ''
445 for cmddict in alist:
446 if 'timerange' in cmddict:
447 timerange = cmddict['timerange']
448 if timerange.find('~') != -1:
449 t0, t1 = timerange.split('~',1)
450 date_time_format_t0 = get_date_format(t0)
451 date_time_format_t1 = get_date_format(t1)
453 # start time
454 startTime = qalocal.totime(t0)['value']
455 startTimeSec = (startTime * 24 * 3600) - tbuff0
456 startTimeSec = qalocal.quantity(startTimeSec, 's')
457 paddedT0 = qalocal.time(startTimeSec, form=date_time_format_t0, prec=9)[0]
458 # end time
459 endTime = qalocal.totime(t1)['value']
460 endTimeSec = (endTime * 24 * 3600) + tbuff1
461 endTimeSec = qalocal.quantity(endTimeSec, 's')
462 paddedT1 = qalocal.time(endTimeSec, form=date_time_format_t1, prec=9)[0]
464 # update the original dictionary
465 cmddict['timerange'] = paddedT0+'~'+paddedT1
467 return
469def parseDictionary(cmdlist, reason='any', shadow=True):
470 '''Create a dictionary after parsing a list of flag commands.
471 If reason is different than 'any', only the selected
472 rows will be parsed to the final dictionary.
474 cmdlist --> list of flag commands OR list of dictionaries with flag commands as key:value
475 reason --> reason or list of reasons to select from
476 shadow --> True will make a dictionary of the addantenna parameter
477 and add it to the flag command
479 Returns a dictionary with the the selected rows. Each parameter
480 will become a key in the dictionary.
481 If a mode='shadow' is present in the dictionary and the parameter
482 addantenna is a filename (string), it will create a dictionary
483 of the filename and add it to the parsed dictionary.
485 '''
486 if cmdlist.__len__() == 0:
487 raise Exception('Empty list of commands')
489 # Gather all requested reasons
490 myreaslist = []
491 if type(reason) == str:
492 if reason != 'any':
493 myreaslist.append(reason)
494 elif type(reason) == list:
495 myreaslist = reason
496 else:
497 casalog.post('Cannot read reason; it contains unknown variable types', 'ERROR')
498 return
500 # Separate per ' ', then per '='
501 myParser = Parser(' ', '=')
503 flagdict = {}
504 row = 0
505 for cmd in cmdlist:
506 cmddict = {}
508 # Simple list of strings ([key1='value1' key2='value2']
509 if isinstance(cmd, str):
510 # Skip comment and empty lines
511 if cmd.startswith('#'):
512 continue
513 if cmd == '':
514 continue
516 uppercmd = cmd.replace('true','True')
517 cmd = uppercmd.replace('false','False')
519 # Get a dictionary without type evaluation
520 preparsing = myParser.parseNoEval(cmd)
522 # Evaluate the types
523 parsed = evaluateParameters(preparsing)
525 # List of dictionaries: [{'key1':'value1', 'key2':'value2'}]
526 elif isinstance(cmd, dict):
527 parsed = cmd
529 # Parse the flag commands into a dictionary
530 mode = ''
531 antenna = ''
532 timerange = ''
533 cmddict['row'] = str(row)
534 cmddict['id'] = str(row)
535 cmddict['command'] = parsed
536 if 'mode' in parsed:
537 mode = parsed['mode']
539 input_reason = ''
540 if 'reason' in parsed:
541 input_reason = parsed['reason']
543 cmddict['reason'] = input_reason
545 if 'timerange' in parsed:
546 timerange = parsed['timerange']
547 if 'antenna' in parsed:
548 antenna = parsed['antenna']
550 cmddict['applied'] = False
551 cmddict['time'] = 0.0
552 cmddict['interval'] = 0
553 cmddict['level'] = 0
554 cmddict['severity'] = 0
555 cmddict['type'] = ''
556 cmddict['mode'] = mode
557 cmddict['timerange'] = timerange
558 cmddict['antenna'] = antenna
560 flagdict[row] = cmddict
561 row += 1
563 # Select the input cmds based on the requested reasons
564 selected_dict = {}
565 cmddict = {}
566 row = 0
567 if myreaslist.__len__() > 0:
568 for key in flagdict:
569 cmddict = flagdict[key]['command']
570 if 'reason' in cmddict:
571 input_reason = cmddict['reason']
572 if selectReason(myreaslist, input_reason):
573 selected_dict[row] = flagdict[key]
574 row += 1
576 if selected_dict.__len__() == 0:
577 raise Exception('No input lines matching requested reason(s)')
578 else:
579 selected_dict = flagdict
581 # Specific for the shadow mode
582 if shadow:
583 cmddict = {}
584 for key in selected_dict:
585 cmddict = selected_dict[key]['command']
586 if (
587 'addantenna' in cmddict
588 and isinstance(cmddict['addantenna'], str)
589 and cmddict['addantenna'] != ''
590 ):
591 # Create a dictionary and replace the parameter
592 casalog.post('The addantenna parameter will be parsed as a dictionary', 'DEBUG1')
593 antdict = readAntennaList(cmddict['addantenna'])
594 selected_dict[key]['command']['addantenna'] = antdict
596 return selected_dict
598def selectReason(reasonlist, pattern):
599 '''Return True if pattern is in the reasonlist'''
601 if isinstance(reasonlist, str):
602 if pattern == reasonlist:
603 return True
604 elif isinstance(reasonlist, list):
605 for r in reasonlist:
606 if pattern == r:
607 return True
609 return False
611def parseSelectionPars(cmddict):
612 '''Return only the selection parameters into a dictionary:
613 cmddict --> one row-dictionary with parameters
614 The correlation parameter will not be considered.
615 '''
617 # Only these parameters will be included in dictionary
618 # correlation is handled by the agents
619 parlist = ['observation','array','feed','scan','field','spw',
620 'timerange','uvrange','intent','antenna']
622 return {par: cmddict[par] for par in parlist if par in cmddict}
624def parseUnion(vis, flagdict):
625 '''Get a dictionary of a union of all selection parameters from a dictionary
626 of flag commands.
627 vis --> MS
628 flagdict --> dictionary of parameters and values (par=val) such as the one
629 returned by parseDictionary()
631 Returns a new dictionary with the union of the selection parameters.
632 '''
633 # Union dictionary to return
634 dictpars = {}
636 # Strings for each parameter
637 scans = ''
638 fields = ''
639 ants = ''
640 times = ''
641 corrs = ''
642 ints = ''
643 feeds = ''
644 arrays = ''
645 uvs = ''
646 spws = ''
647 obs = ''
649 # Counters for each parameter
650 si = 0 # count the number of lines with scan
651 fi = 0 # count the number of lines with field
652 ai = 0 # count the number of lines with antenna
653 ti = 0 # count the number of lines with timerange
654 coi = 0 # count the number of lines with correlation
655 ii = 0 # count the number of lines with intent
656 fei = 0 # count the number of lines with feed
657 ari = 0 # count the number of lines with array
658 ui = 0 # count the number of lines with uvrange
659 pi = 0 # count the number of lines with spw
660 oi = 0 # count the number of lines with observation
662 for k in flagdict.keys():
663 # Each key is a dictionary of one flag command
664 cmddict = flagdict[k]['command']
666 for xkey,xval in cmddict.items():
667 # Check which parameter
668 if xkey == "scan":
669 scans += xval + ','
670 si += 1
671 elif xkey == "field":
672 fields += xval + ','
673 fi += 1
674 elif xkey == "antenna":
675 ants += xval + ';'
676 ai += 1
677 elif xkey == "timerange":
678 times += xval + ','
679 ti += 1
680 elif xkey == "correlation":
681 corrs += xval + ','
682 coi += 1
683 elif xkey == "intent":
684 ints += xval + ','
685 ii += 1
686 elif xkey == "feed":
687 feeds += xval + ','
688 fei += 1
689 elif xkey == "array":
690 arrays += xval + ','
691 ari += 1
692 elif xkey == "uvrange":
693 uvs += xval + ','
694 ui += 1
695 elif xkey == "spw":
696 spws += xval + ','
697 pi += 1
698 elif xkey == "observation":
699 obs += xval + ','
700 oi += 1
702 # Dictionary with number of occurrences of each parameter
703 npars = {
704 'field':fi,
705 'scan':si,
706 'antenna':ai,
707 'spw':pi,
708 'timerange':ti,
709 'correlation':coi,
710 'intent':ii,
711 'feed':fei,
712 'array':ari,
713 'uvrange':ui,
714 'observation':oi
715 }
717 # Strip out the extra comma at the end
718 scans = scans.rstrip(',')
719 fields = fields.rstrip(',')
720 ants = ants.rstrip(';')
721 times = times.rstrip(',')
722 corrs = corrs.rstrip(',')
723 ints = ints.rstrip(',')
724 feeds = feeds.rstrip(',')
725 arrays = arrays.rstrip(',')
726 uvs = uvs.rstrip(',')
727 spws = spws.rstrip(',')
728 obs = obs.rstrip(',')
730 dictpars['scan'] = scans
731 dictpars['field'] = fields
732 # Antennas are handled better within the framework.
733 dictpars['antenna'] = ''
734 # Times are handled better within the framework.
735 dictpars['timerange'] = ''
736 # Correlations should be handled only by the agents
737 dictpars['correlation'] = ''
738 # CAS-4682, do not create union for intents
739 dictpars['intent'] = ''
740 dictpars['feed'] = feeds
741 dictpars['array'] = arrays
742 dictpars['uvrange'] = uvs
743 dictpars['spw'] = spws
744 dictpars['observation'] = obs
746 # Compress the selection list to reduce MSSelection parsing time.
747 # 'field','spw','antenna' strings in dicpars will be modified in-place.
748 compressSelectionList(vis,dictpars);
750 # The number of keys in flagdict is the number of flag commands
751 ncmds = flagdict.__len__()
753 # Make the union. Only leave non-empty parameters in dictionary
754 for k,v in npars.items():
755 if v < ncmds:
756 dictpars.pop(k)
758 return dictpars
760def _merge_timerange(commands):
761 ''' merge manual commands that only differ in timerange and agentname
762 this speeds up manual flagging using large lists
764 cmd -> list of flagging commands
766 returns list of commands with unique key
767 '''
768 merged = dict()
769 lunique = []
770 for cmd in commands:
771 try:
772 # only merge manual commands with timerange
773 if (cmd.get('mode') != 'manual') or ('timerange' not in cmd):
774 raise ValueError
775 # create sorted list of command keys excluding agentname which
776 # changes for each manual flag
777 compound_key = sorted(x for x in cmd.keys() if x not in ('timerange', 'agentname'))
778 # create compound key of all command keys and their values (e.g. antenna:1)
779 compound = tuple((x, cmd[x]) for x in compound_key)
781 # skip invalid timeranges so they don't remove the whole agent group
782 if '~' in cmd['timerange']:
783 t0,t1 = cmd['timerange'].split('~', 1)
784 startTime = qalocal.totime(t0)['value']
785 endTime = qalocal.totime(t1)['value']
786 if endTime <= startTime:
787 raise ValueError
789 # merge timerange duplicate compound keys
790 try:
791 merged[compound]['timerange'] += ',' + cmd['timerange']
792 except KeyError:
793 merged[compound] = copy.deepcopy(cmd)
794 except:
795 # add merged keys to non-mergeable keys, also on errors like
796 # non-hashable keys
797 lunique.extend(merged.values())
798 # append non mergeable key
799 lunique.append(copy.deepcopy(cmd))
800 # reset merge to preserve ordering of manual and other flags
801 # e.g. summary,manual,manual,manual,summary,manual
802 # to summary,merged-manual,summary,manual
803 merged = dict()
805 # add remaining merged keys to non-mergeable keys
806 lunique.extend(merged.values())
807 return lunique
809def parseAgents(aflocal, flagdict, myrows, apply, writeflags, display=''):
810 ''' Setup the parameters of each agent and call the agentflagger tool
812 aflocal --> local instance of the agentflagger tool (assumes the MS is open)
813 flagdict --> a dictionary of the flag commands to parse to tool
814 myrows --> selected rows to apply/unapply flags
815 apply --> a boolean to control whether to apply or unapply the flags
816 writeflags --> used by mode=rflag only
817 display --> used by mode='rflag' only
819 The original flagdict dictionary is not modified '''
822 if flagdict.__len__() <= 0:
823 casalog.post('There are no flag cmds in list', 'SEVERE')
824 return False
826 # Do not modify original dictionary
827 myflagcmd = copy.deepcopy(flagdict)
828 commands = []
830 # Setup the agent for each input line and merge timeranges to one command
831 for row in myflagcmd.keys():
832 cmd = OrderedDict()
833 cmd = myflagcmd[row]['command']
835 if 'reason' in cmd:
836 cmd.pop('reason')
838 if (
839 'mode' not in cmd
840 or cmd['mode'] == ''
841 or cmd['mode'] == 'manualflag'
842 ):
843 cmd['mode'] = 'manual'
845 elif cmd['mode'] == 'rflag':
846 cmd['writeflags'] = writeflags
847 cmd['display'] = display
849 # Read ntime
850 readNtime(cmd)
852 # Cast the correct type of some parameters
853 evalParams(cmd)
855 # Add the apply/unapply parameter to dictionary
856 cmd['apply'] = apply
858 # Hold the name of the agent and the cmd row number
859 mode = cmd['mode']
860 agent_name = mode.capitalize()+'_'+str(row)
862 cmd['agentname'] = agent_name
864 # Remove the data selection parameters if there is only one agent for performance reasons.
865 # Explanation: if only one agent exists and the data selection parameters are parsed to it,
866 # it will have to go through the entire MS and check if each data selection given to the agent
867 # matches what the user asked in the selected data.
869 # Only correlation, antenna and timerange will go to the agent
870 # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too
871 if myflagcmd.__len__() == 1:
872 sellist=['scan','field','intent','feed','array','uvrange','observation']
873 for k in sellist:
874 if k in cmd:
875 cmd.pop(k)
877 casalog.post('Parsing parameters of mode %s in row %s'%(mode,row), 'DEBUG')
878 casalog.post('%s'%cmd,'DEBUG')
880 commands.append(cmd)
882 merged = _merge_timerange(commands)
884 if len(myflagcmd) != len(merged):
885 casalog.post('Reduced %d timerange flags into %d compound flags' %
886 (len(myflagcmd), len(merged)))
888 for cmd in merged:
889 # Parse the dictionary of parameters to the tool
890 if (not aflocal.parseagentparameters(cmd)):
891 casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,row), 'WARN')
892 continue
893# return True
894 return myflagcmd
896# TO BE VERIFIED. May not be needed
897def evalParams(params):
898 '''Give correct types to non-string parameters
899 The types are defined in the XML file of the task flagdata
900 Do not repeat any parameter'''
902 # manual parameter
903# if 'autocorr' in params:
904# params['autocorr'] = eval(params['autocorr'].capitalize())
906 # quack parameters
907# if 'quackmode' in params and not params['quackmode'] in ['beg'
908# , 'endb', 'end', 'tail']:
909# raise Exception, \
910# "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \
911# % params['quackmode']
912# if 'quackinterval' in params:
913# params['quackinterval'] = float(params['quackinterval'])
914# if 'quackincrement'in params:
915# if type(params['quackincrement']) == str:
916# params['quackincrement'] = eval(params['quackincrement'].capitalize())
918 # clip parameters
919 if 'clipminmax' in params:
920 val1 = params['clipminmax'][0]
921 val2 = params['clipminmax'][1]
922 params['clipminmax'] = [float(val1), float(val2)]
923# if 'clipoutside' in params:
924# if type(params['clipoutside']) == str:
925# params['clipoutside'] = eval(params['clipoutside'].capitalize())
926# else:
927# params['clipoutside'] = params['clipoutside']
928# if 'channelavg' in params:
929# params['channelavg'] = eval(params['channelavg'].capitalize())
930# if 'clipzeros' in params:
931# params['clipzeros'] = eval(params['clipzeros'].capitalize())
934 # shadow parameter
935# if 'tolerance' in params:
936# params['tolerance'] = float(params['tolerance'])
937#
938# # elevation parameters
939# if 'lowerlimit' in params:
940# params['lowerlimit'] = float(params['lowerlimit'])
941# if 'upperlimit' in params:
942# params['upperlimit'] = float(params['upperlimit'])
944 # extend parameters
945# if 'extendpols' in params:
946# params['extendpols'] = eval(params['extendpols'].capitalize())
947# if 'growtime' in params:
948# params['growtime'] = float(params['growtime'])
949# if 'growfreq' in params:
950# params['growfreq'] = float(params['growfreq'])
951# if 'growaround' in params:
952# params['growaround'] = eval(params['growaround'].capitalize())
953# if 'flagneartime' in params:
954# params['flagneartime'] = eval(params['flagneartime'].capitalize())
955# if 'flagnearfreq' in params:
956# params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize())
958 # tfcrop parameters
959# if 'combinescans' in params:
960# params['combinescans'] = eval(params['combinescans'].capitalize())
961# if 'timecutoff' in params:
962# params['timecutoff'] = float(params['timecutoff'])
963# if 'freqcutoff' in params:
964# params['freqcutoff'] = float(params['freqcutoff'])
965# if 'maxnpieces' in params:
966# params['maxnpieces'] = int(params['maxnpieces'])
967# if 'halfwin' in params:
968# params['halfwin'] = int(params['halfwin'])
969# if 'extendflags' in params:
970# params['extendflags'] = eval(params['extendflags'].capitalize())
972 # rflag parameters
973# if 'winsize' in params:
974# params['winsize'] = int(params['winsize']);
976 # This is only necessary when timedev/freqdev are strings
977 if 'timedev' in params:
978 timepar = params['timedev']
979 if isinstance(timepar, list):
980 return
981 try:
982 timepar = eval(timepar)
983 except Exception:
984 timepar = readRFlagThresholdFile(params['timedev'],'timedev');
985 params['timedev'] = timepar
986 if 'freqdev' in params:
987 freqpar = params['freqdev']
988 if isinstance(freqpar, list):
989 return
990 try:
991 freqpar = eval(freqpar)
992 except Exception:
993 freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev');
994 params['freqdev'] = freqpar
995# if 'timedevscale' in params:
996# params['timedevscale'] = float(params['timedevscale']);
997# if 'freqdevscale' in params:
998# params['freqdevscale'] = float(params['freqdevscale']);
999# if 'spectralmin' in params:
1000# params['spectralmin'] = float(params['spectralmin']);
1001# if 'spectralmax' in params:
1002# params['spectralmax'] = float(params['spectralmax']);
1004def writeFlagCommands(msfile, flagdict, applied, add_reason, outfile, append=True):
1005 '''
1006 Writes the flag commands to an ASCII file or to the FLAG_CMD table
1008 msfile --> MS name to save to FLAG_CMD table or empty.
1009 flagdict --> dictionary of commands read from inputfile
1010 applied --> True or False to write to APPLIED column
1011 add_reason --> reason to add to output (replace input reason, if any)
1012 outfile --> if not empty, save to it
1013 append --> if True it will append to outfile, otherwise it removes outfile
1014 before saving to it.
1016 '''
1018 try:
1019 import pylab as pl
1020 except ImportError as exc:
1021 raise ImportError('Failed to load pylab, required in writeFlagCommands: ', exc)
1023 reason2add = False
1024 # Replace blanks from reason and cmdreason with underscores
1025 # and warn the user.
1026 # TODO: I think this is not necessary for new parser!!! Check it
1027 if add_reason != '':
1028 reason2add = True
1029# casalog.post('Replaced blanks with underscores in cmdreason', 'WARN')
1031 # Append to a file
1032 if outfile != '':
1033 if append:
1034# casalog.post('Will append commands to the file '+outfile)
1035 ffout = open(outfile, 'a')
1036 else:
1037# casalog.post('Will save commands to the file '+outfile)
1038 os.system('rm -f '+outfile)
1039 ffout = open(outfile, 'w')
1041 try:
1042 for key in flagdict.keys():
1043 cmdline = ""
1044 cmddict = flagdict[key]['command']
1045 reason = flagdict[key]['reason']
1047 # Add new reason or replace old one with new reason
1048 if reason2add:
1049 reason = add_reason
1051 if reason != '':
1052 cmddict['reason'] = reason
1054 for k,v in cmddict.items():
1055 cmdstr = ""
1056 if isinstance(v, str):
1057 # Add quotes to string values
1058 if v.count("'") > 0:
1059 v = v.strip("'")
1060 if v.count('"') > 0:
1061 v = v.strip("'")
1062 cmdstr = "'"+v+"'"
1063 cmdline = cmdline + k + '=' + str(cmdstr) + ' '
1064 else:
1065 cmdline = cmdline + k + '=' + str(v) + ' '
1067 # Save to output file
1068 ffout.write('%s\n' %cmdline.rstrip())
1069 ffout.close()
1070 except:
1071 raise Exception('Error writing/appending lines to file ' \
1072 + outfile)
1073 ffout.close()
1074 return False
1075 else:
1076 # Append new commands to existing FLAG_CMD table
1077# casalog.post('Saving commands to the FLAG_CMD table')
1078 cmdlist = []
1079 reasonlist = []
1080 for key in flagdict.keys():
1081 cmdline = ""
1082 reason = ""
1083 cmddict = flagdict[key]['command']
1084 # Do not save reason in the COMMAND column
1085 if 'reason' in cmddict:
1086 cmddict.pop('reason')
1087 # Summary cmds should not go to FLAG_CMD
1088 if 'mode' in cmddict and cmddict['mode'] == 'summary':
1089 casalog.post("Commands with mode='summary' are not allowed in the FLAG_CMD table", 'WARN')
1090 continue
1091 # Add to REASON column the user input reason if requested
1092 reason = flagdict[key]['reason']
1093 if reason2add:
1094 reason = add_reason
1096 for k,v in cmddict.items():
1097 cmdstr = ""
1098 if isinstance(v, str):
1099 # Add quotes to string values
1100 if v.count("'") > 0:
1101 v = v.strip("'")
1102 if v.count('"') > 0:
1103 v = v.strip("'")
1104 cmdstr = "'"+v+"'"
1105 cmdline = cmdline + k + '=' + str(cmdstr) + ' '
1107 else:
1108 cmdline = cmdline + k + '=' + str(v) + ' '
1110 cmdline = cmdline.rstrip()
1111 cmdlist.append(cmdline)
1112 reasonlist.append(reason)
1114 # Number of cmds to add to FLAG_CMD
1115 Nadd = cmdlist.__len__()
1117 # Fill in other columns
1118 appliedlist = [applied for i in range(Nadd)]
1119 timelist = [0.0]*Nadd
1120 otherlist = [0]*Nadd
1121 typelist = ['']*Nadd
1123 mstable = msfile + '/FLAG_CMD'
1124 try:
1125 tblocal.open(mstable, nomodify=False)
1126 except:
1127 raise Exception('Error opening FLAG_CMD table '+mstable)
1129 nrows = int(tblocal.nrows())
1130 casalog.post('There are ' + str(nrows)+ ' rows already in FLAG_CMD')
1132 # Add blank rows first
1133 tblocal.addrows(Nadd)
1135 # Fill in the columns
1136 tblocal.putcol('APPLIED', numpy.array(appliedlist), startrow=nrows,nrow=Nadd)
1137 tblocal.putcol('COMMAND', numpy.array(cmdlist), startrow=nrows, nrow=Nadd)
1138 tblocal.putcol('INTERVAL', numpy.array(timelist), startrow=nrows,nrow=Nadd)
1139 tblocal.putcol('LEVEL', numpy.array(otherlist), startrow=nrows,nrow=Nadd)
1140 tblocal.putcol('REASON', numpy.array(reasonlist),startrow=nrows, nrow=Nadd)
1141 tblocal.putcol('SEVERITY', numpy.array(otherlist), startrow=nrows,nrow=Nadd)
1142 tblocal.putcol('TIME', numpy.array(timelist), startrow=nrows,nrow=Nadd)
1143 tblocal.putcol('TYPE', numpy.array(typelist), startrow=nrows,nrow=Nadd)
1145 newrows = int(tblocal.nrows())
1146 newrows -= nrows
1147 if newrows == 0:
1148 casalog.post('Did not save any rows to FLAG_CMD')
1149 else:
1150 casalog.post('Saved ' + str(newrows) + ' rows to FLAG_CMD of %s'%msfile)
1152 tblocal.close()
1154 return True
1156def parseRFlagOutputFromSummary(mode,summary_stats_list, flagcmd):
1157 """
1158 Function to pull out 'rflag' output from the long dictionary, and
1159 (1) write the output files with thresholds. If the filename is specified, use it.
1160 If filename is not specified, make one up.
1161 (2) modify entries in 'cmdline' so that it is ready for savepars.
1162 This is to ensure that 'savepars' saves the contents of the threshold-files
1163 and not just the file-names. It has to save it in the form that flagdata
1164 accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user
1165 need not keep track of threshold text files if they use 'savepars' with action='apply'.
1166 """
1167 # The careful checks are especially for MMS which can produce empty selections/results from
1168 # individual sub-MSs.
1169 if summary_stats_list and type(summary_stats_list) is dict and 'nreport' in summary_stats_list:
1170 nreps = summary_stats_list['nreport']
1171 for rep in range(0,nreps):
1172 repname = 'report'+str(rep)
1173 if summary_stats_list[repname]['type'] == "rflag":
1174 # Pull out the rflag threshold dictionary. This has a 'name' in it.
1175 rflag_thresholds = summary_stats_list[repname]
1176 # Get the rflag id, to later construct a 'name' from to match the above.
1177 rflagid = 0
1178 if mode=='list':
1179 rflagid = int( rflag_thresholds['name'].replace('Rflag_','') )
1180 # Go through the flagcmd list, to find the 'rflags'.....
1181 for key in flagcmd.keys():
1182 # cmdline is a dictionary with flag commands
1183 cmdline = flagcmd[key]['command']
1184 if 'mode' in cmdline and cmdline['mode'] == 'rflag':
1185 # Check for match between input flagcmd and output threshold, via the rflag id
1186 if(key == rflagid):
1187 # If timedev,freqdev are missing from cmdline, add empty ones.
1188 if not 'timedev' in cmdline: # aah. don't confuse it with timedevscale
1189 cmdline['timedev'] = []
1190 if not 'freqdev' in cmdline:
1191 cmdline['freqdev'] = []
1192 # Write RFlag thresholds to these file names.
1193 newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, cmdline['timedev'], cmdline['freqdev'], rflagid)
1194 ## VERIFY for parser
1195 # Modify the flagcmd string, so that savepars sees the contents of the file
1196 if( isinstance(cmdline['timedev'], list)):
1197 cmdline['timedev'] = newtimedev
1198 if( isinstance(cmdline['freqdev'],list)):
1199 cmdline['freqdev'] = newfreqdev
1201 # Remove writeflags from the cmd to prevent it from going into savepars
1202 cmdline.pop('writeflags')
1204def save_rflag_consolidated_files(mode, action, cons_dict, opts_dict, inpfile):
1205 """
1206 Utility for RFlag when running in parallel/MMS. Does parseRFlagOutputFromSummary() on
1207 the consolidated (on the client process) - only when needed for rflag mode (or list
1208 mode when the list of commands contains some RFlag commands).
1210 :param mode: mode parameter as given to the from casatasks.flagdata import flagdata task
1211 :param action: action parameter as given to the flagdata task
1212 :param cons_dict: consolidated dictionary of reports from (RFlag) flagdata commands
1213 :param opts_dict: dictionary with the options needed for parseRFlagOutputFromSummary()
1214 (timedev, freqdev and writeflags, from the task input parameters)
1215 :param inpfile: inpfile parameter as given to the flagdata task
1216 """
1217 if mode in ['rflag', 'list'] and action != 'apply':
1218 import pprint
1219 casalog.post('Saving RFlag return dictionary: {0}'.
1220 format(pprint.pformat(cons_dict)), 'INFO')
1221 # Prepare the list of commands
1222 if mode == 'list':
1223 cmd_list = get_flag_cmd_list(inpfile)
1224 else:
1225 # need only the relevant fields to save the output files in
1226 # parseRFlagOutputFromSummary()
1227 cmd_list = {0: {'command':
1228 dict({'mode': 'rflag'}, **opts_dict)}}
1229 parseRFlagOutputFromSummary(mode, cons_dict, cmd_list)
1231# Not used at the moment!!!!!!!!!!!
1232def readFlagCmdTable(msfile, rows=[], applied=True, reason='any'):
1233 '''Read flag commands from rows of the FLAG_CMD table of msfile
1234 If useapplied=False then include only rows with APPLIED=False
1235 If myreason is anything other than 'any', then select on that'''
1237 #
1238 # Return flagcmd structure:
1239 #
1240 # The flagcmd key is the integer row number from FLAG_CMD
1241 #
1242 # Dictionary structure:
1243 # key : 'id' (string)
1244 # 'mode' (string) flag mode '','clip','shadow','quack'
1245 # 'antenna' (string)
1246 # 'timerange' (string)
1247 # 'reason' (string)
1248 # 'time' (float) in mjd seconds
1249 # 'interval' (float) in mjd seconds
1250 # 'cmd' (string) string (for COMMAND col in FLAG_CMD)
1251 # 'type' (string) 'FLAG' / 'UNFLAG'
1252 # 'applied' (bool) set to True here on read-in
1253 # 'level' (int) set to 0 here on read-in
1254 # 'severity' (int) set to 0 here on read-in
1256 # Open and read columns from FLAG_CMD
1257 mstable = msfile + '/FLAG_CMD'
1259 # Note, tblocal.getcol doesn't allow random row access, read all
1261 try:
1262 tblocal.open(mstable)
1263 f_time = tblocal.getcol('TIME')
1264 f_interval = tblocal.getcol('INTERVAL')
1265 f_type = tblocal.getcol('TYPE')
1266 f_reas = tblocal.getcol('REASON')
1267 f_level = tblocal.getcol('LEVEL')
1268 f_severity = tblocal.getcol('SEVERITY')
1269 f_applied = tblocal.getcol('APPLIED')
1270 f_cmd = tblocal.getcol('COMMAND')
1271 tblocal.close()
1272 except:
1273 casalog.post('Error reading table ' + mstable, 'ERROR')
1274 raise Exception
1276 nrows = f_time.__len__()
1278 if nrows == 0:
1279 casalog.post('FLAG_CMD table in %s is empty, no flags extracted'
1280 % msfile, 'ERROR')
1281 return {}
1283 rowlist = []
1285 # Select by rows
1286 if rows.__len__() > 0:
1287 rowlist = rows
1288 else:
1289 rowlist = range(nrows)
1291 # Select by APPLIED
1292 if not applied:
1293 rowl = []
1294 for i in rowlist:
1295 if not f_applied[i]:
1296 rowl.append(i)
1297 rowlist = rowl
1299 # Select by REASON
1300 myreaslist = []
1301 if type(reason) == str:
1302 if reason != 'any':
1303 myreaslist.append(reason)
1304 elif type(reason) == list:
1305 myreaslist = reason
1306 else:
1307 casalog.post('Cannot read reason; it contains unknown variable types'
1308 , 'ERROR')
1309 return
1311 if myreaslist.__len__() > 0:
1312 rowl = []
1313 for i in rowlist:
1314 if myreaslist.count(f_reas[i]) > 0:
1315 rowl.append(i)
1316 rowlist = rowl
1318 Ncmds = 0
1319 myflagcmd = {}
1321 # Define the parser
1322 myParser = Parser(' ','=')
1323 for i in rowlist:
1324 parsed = {}
1325 flagd = {}
1326 cmd = f_cmd[i]
1327 if cmd == '':
1328 continue
1330 # Parse to a dictionary
1331 parsed = myParser.parse2Dictionary(cmd)
1333 # This is the original row number in FLAG_CMD
1334 flagd['ROW'] = i
1335 flagd['COMMAND'] = parsed
1336 flagd['REASON'] = f_reas[i]
1337 flagd['APPLIED'] = f_applied[i]
1338 flagd['TIME'] = f_time[i]
1339 flagd['INTERVAL'] = f_interval[i]
1340 flagd['TYPE'] = f_type[i]
1341 flagd['LEVEL'] = f_level[i]
1342 flagd['SEVERITY'] = f_severity[i]
1344 myflagcmd[Ncmds] = flagd
1345 Ncmds += 1
1347 casalog.post('Read ' + str(Ncmds)
1348 + ' rows from FLAG_CMD table in ' + msfile)
1350 return myflagcmd
1352def plotFlagCommands(myflags,plotname,t1sdata,t2sdata,):
1353 #
1354 # Function to plot flagging dictionary
1355 # Adapted from J.Marvil
1356 # Updated STM v4.1 2011-11-02 to handle ALMA flags
1357 # Updated STM v4.2 2012-02-16 trim flag times to data times
1358 # Updated STM v4.2 2012-04-10 bug fix in trim flag times to data times
1359 # Updated STM v4.2 2012-04-10 messages to logger
1361 try:
1362 import pylab as pl
1363 except ImportError as exc:
1364 raise ImportError('Failed to load pylab, required in plotFlagCommands: ', exc)
1366 # list of supported colors (in order)
1367 colorlist = [
1368 'red',
1369 'blue',
1370 'green',
1371 'black',
1372 'cyan',
1373 'magenta',
1374 'yellow',
1375 'orange',
1376 ]
1377 ncolors = colorlist.__len__()
1379 # get list of flag keys
1380 keylist = myflags.keys()
1382 # get lists of antennas and reasons
1383 # make plotting dictionary
1384 myants = []
1385 myreas = []
1386 plotflag = {}
1387 ipf = 0
1388 for key in keylist:
1389 antstr = ''
1390 reastr = ''
1391 timstr = ''
1392 if 'antenna' in myflags[key]:
1393 antstr = myflags[key]['antenna']
1394 if 'reason' in myflags[key]:
1395 reastr = myflags[key]['reason']
1396 if 'timerange' in myflags[key]:
1397 timstr = myflags[key]['timerange']
1398 if antstr != '':
1399 # flags that have antenna specified
1400 antlist = antstr.split(',')
1401 nantlist = antlist.__len__()
1402 else:
1403 # Special
1404 antlist = ['All']
1405 nantlist = 1
1406 #
1407 realist = reastr.split(',')
1408 nrealist = realist.__len__()
1409 #
1410 timlist = timstr.split(',')
1411 ntimlist = timlist.__len__()
1412 #
1413 # Break these into nants x ntimes flags
1414 # Trick is assigning multiple reasons
1415 # Normal cases:
1416 # A. One reason, single/multiple antennas x times
1417 # B. Multiple reasons=times, single/multiple antenna(s)
1418 # C. Single time, multiple antennas/reasons
1419 # D. Multiple reasons, no way to correspond with times
1420 #
1421 timmin = 1.0E11
1422 timmax = 0.0
1423 if nrealist == 1:
1424 # simplest case, single reason
1425 reas = realist[0]
1426 if reas == '':
1427 reas = 'Unknown'
1428 if myreas.count(reas) == 0:
1429 myreas.append(reas)
1430 for ia in range(nantlist):
1431 ant = antlist[ia]
1432 if myants.count(ant) == 0:
1433 myants.append(ant)
1434 for it in range(ntimlist):
1435 times = timlist[it]
1436 plotflag[ipf] = {}
1437 plotflag[ipf]['antenna'] = ant
1438 plotflag[ipf]['reason'] = reas
1439 plotflag[ipf]['timerange'] = times
1440 plotflag[ipf]['show'] = True
1441 ipf += 1
1442 elif nrealist == ntimlist:
1443 # corresponding reasons and times
1444 for ia in range(nantlist):
1445 ant = antlist[ia]
1446 if myants.count(ant) == 0:
1447 myants.append(ant)
1448 for it in range(ntimlist):
1449 times = timlist[it]
1450 reas = realist[it]
1451 if reas == '':
1452 reas = 'Unknown'
1453 if myreas.count(reas) == 0:
1454 myreas.append(reas)
1455 plotflag[ipf] = {}
1456 plotflag[ipf]['antenna'] = ant
1457 plotflag[ipf]['reason'] = reas
1458 plotflag[ipf]['timerange'] = times
1459 plotflag[ipf]['show'] = True
1460 ipf += 1
1461 else:
1462 # no correspondence between multiple reasons and ants/times
1463 # assign reason 'Miscellaneous'
1464 reas = 'Miscellaneous'
1465 if myreas.count(reas) == 0:
1466 myreas.append(reas)
1467 for ia in range(nantlist):
1468 ant = antlist[ia]
1469 if myants.count(ant) == 0:
1470 myants.append(ant)
1471 for it in range(ntimlist):
1472 times = timlist[it]
1473 plotflag[ipf] = {}
1474 plotflag[ipf]['antenna'] = ant
1475 plotflag[ipf]['reason'] = reas
1476 plotflag[ipf]['timerange'] = times
1477 plotflag[ipf]['show'] = True
1478 ipf += 1
1480 myants.sort()
1481 nants = myants.__len__()
1482 nreas = myreas.__len__()
1483 casalog.post('Found ' + str(nreas) + ' reasons to plot for '
1484 + str(nants) + ' antennas')
1485 npf = ipf
1486 casalog.post('Found ' + str(npf) + ' total flag ranges to plot')
1488 # sort out times
1489 for ipf in range(npf):
1490 times = plotflag[ipf]['timerange']
1491 if times != '':
1492 if times.count('~') > 0:
1493 t1 = times[:times.find('~')]
1494 t2 = times[times.find('~') + 1:]
1495 else:
1496 t1 = times
1497 t2 = t1
1498 (t1s, t2s) = (qalocal.convert(t1, 's')['value'], qalocal.convert(t2,
1499 's')['value'])
1500 plotflag[ipf]['t1s'] = t1s
1501 plotflag[ipf]['t2s'] = t2s
1502 if t1s < timmin:
1503 timmin = t1s
1504 if t2s > timmax:
1505 timmax = t2s
1506 # min,max times
1507 q1 = qalocal.quantity(timmin, 's')
1508 time1 = qalocal.time(q1, form='ymd', prec=9)[0]
1509 q2 = qalocal.quantity(timmax, 's')
1510 time2 = qalocal.time(q2, form='ymd', prec=9)[0]
1511 casalog.post('Found flag times from ' + time1 + ' to ' + time2)
1513 # sort out blank times
1514 for ipf in range(npf):
1515 times = plotflag[ipf]['timerange']
1516 if times == '':
1517 if t2sdata >= t1sdata > 0:
1518 plotflag[ipf]['t1s'] = t1sdata
1519 plotflag[ipf]['t2s'] = t2sdata
1520 else:
1521 plotflag[ipf]['t1s'] = timmin
1522 plotflag[ipf]['t2s'] = timmax
1524 # if flag times are beyond range of data, trim them
1525 # Added STM 2012-02-16, fixed STM 2012-04-10
1526 ndropped = 0
1527 if t2sdata >= t1sdata > 0 and (timmin < t1sdata or timmax
1528 > t2sdata):
1529 # min,max data times
1530 q1 = qalocal.quantity(t1sdata, 's')
1531 tdata1 = qalocal.time(q1, form='ymd', prec=9)[0]
1532 q2 = qalocal.quantity(t2sdata, 's')
1533 tdata2 = qalocal.time(q2, form='ymd', prec=9)[0]
1534 casalog.post('WARNING: Trimming flag times to data limits '
1535 + tdata1 + ' to ' + tdata2)
1537 for ipf in range(npf):
1538 t1s = plotflag[ipf]['t1s']
1539 t2s = plotflag[ipf]['t2s']
1540 if t1s < t1sdata:
1541 if t2s >= t1sdata:
1542 # truncate to t1sdata
1543 plotflag[ipf]['t1s'] = t1sdata
1544 else:
1545 # entirely outside data range, do not plot
1546 plotflag[ipf]['show'] = False
1547 ndropped += 1
1549 if t2s > t2sdata:
1550 if t1s <= t2sdata:
1551 # truncate to t2sdata
1552 plotflag[ipf]['t2s'] = t2sdata
1553 else:
1554 # entirely outside data range, do not plot
1555 plotflag[ipf]['show'] = False
1556 ndropped += 1
1558 if ndropped > 0:
1559 casalog.post('WARNING: Trimming dropped ' + str(ndropped)
1560 + ' flags entirely')
1562 # make reason dictionary with mapping of colors and offsets (-0.3 to 0.3)
1563 readict = {}
1564 reakeys = []
1565 if nreas > ncolors:
1566 for i in range(nreas):
1567 reas = myreas[i]
1568 readict[reas] = {}
1569 if i < ncolors - 1:
1570 colr = colorlist[i]
1571 readict[reas]['color'] = colr
1572 readict[reas]['index'] = i
1573 offs = 0.3 - float(i) * 0.6 / float(ncolors - 1)
1574 readict[reas]['offset'] = offs
1575 reakeys.append(reas)
1576 else:
1577 colr = colorlist[ncolors - 1]
1578 readict[reas]['color'] = colr
1579 readict[reas]['index'] = ncolors - 1
1580 readict[reas]['offset'] = -0.3
1581 reakeys.append('Other')
1582 readict['Other'] = {}
1583 readict['Other']['color'] = colorlist[ncolors - 1]
1584 readict['Other']['index'] = ncolors - 1
1585 readict['Other']['offset'] = -0.3
1586 else:
1587 for i in range(nreas):
1588 reas = myreas[i]
1589 reakeys.append(reas)
1590 colr = colorlist[i]
1591 offs = 0.3 - float(i) * 0.6 / float(ncolors - 1)
1592 readict[reas] = {}
1593 readict[reas]['color'] = colr
1594 readict[reas]['index'] = i
1595 readict[reas]['offset'] = offs
1596 nlegend = reakeys.__len__()
1597 casalog.post('Will plot ' + str(nlegend) + ' reasons in legend')
1599 antind = 0
1600 if plotname == '':
1601 pl.ion()
1602 else:
1603 pl.ioff()
1605 f1 = pl.figure()
1606 ax1 = f1.add_axes([.15, .1, .75, .85])
1607# ax1.set_ylabel('antenna')
1608# ax1.set_xlabel('time')
1609 # badflags=[]
1610 nplotted = 0
1611 for thisant in myants:
1612 antind += 1
1613 for ipf in range(npf):
1614 if plotflag[ipf]['show'] and plotflag[ipf]['antenna'] \
1615 == thisant:
1616 # plot this flag
1617 thisReason = plotflag[ipf]['reason']
1618 thisColor = readict[thisReason]['color']
1619 thisOffset = readict[thisReason]['offset']
1620 t1s = plotflag[ipf]['t1s']
1621 t2s = plotflag[ipf]['t2s']
1623 ax1.plot([t1s, t2s], [antind + thisOffset, antind
1624 + thisOffset], color=thisColor, lw=2, alpha=.7)
1625 nplotted += 1
1627 casalog.post('Plotted ' + str(nplotted) + ' flags')
1629 myXlim = ax1.get_xlim()
1630 myXrange = myXlim[1] - myXlim[0]
1631 # Pad the time axis?
1632 PadTime = 0.050000000000000003
1633 if PadTime > 0:
1634 xPad = PadTime * myXrange
1635 x0 = myXlim[0] - xPad
1636 x1 = myXlim[1] + xPad
1637 ax1.set_xlim(x0, x1)
1638 myXrange = x1 - x0
1639 else:
1640 # casalog.post(' Rescaled x axis')
1641 x0 = myXlim[0]
1642 x1 = myXlim[1]
1644 legendFontSize = 12
1645 myYupper = nants + nlegend + 1.5
1646 # place legend text
1647 i = nants
1648 x = x0 + 0.050000000000000003 * myXrange
1649 for reas in reakeys:
1650 i += 1
1651 colr = readict[reas]['color']
1652 ax1.text(x, i, reas, color=colr, size=legendFontSize)
1653 ax1.set_ylim([0, myYupper])
1655 ax1.set_yticks(range(1, len(myants) + 1))
1656 ax1.set_yticklabels(myants)
1657 # casalog.post(' Relabled y axis')
1659 nxticks = 3
1660 ax1.set_xticks(pl.linspace(myXlim[0], myXlim[1], nxticks))
1662 mytime = []
1663 myTimestr = []
1664 for itim in range(nxticks):
1665 time = myXlim[0] + (myXlim[1] - myXlim[0]) * float(itim) \
1666 / float(nxticks - 1)
1667 mytime.append(time)
1668 q1 = qalocal.quantity(time, 's')
1669 time1 = qalocal.time(q1, form='ymd', prec=9)[0]
1670 if itim > 0:
1671 time1s = time1[11:]
1672 else:
1673 time1s = time1
1674 myTimestr.append(time1s)
1676 ax1.set_xticklabels(myTimestr)
1677 # casalog.post(myTimestr)
1678 if plotname == '':
1679 pl.draw()
1680 else:
1681 pl.savefig(plotname, dpi=150)
1682 return
1684def evaluateParameters(pardict):
1685 '''Evaluate the correct types in a string with parameters
1687 Keyword arguments:
1688 pardict -- a dictionary with flag commands as returned by parseDictionary()
1690 It evaluates each value to its correct Python type.
1691 '''
1693 cmddict = OrderedDict()
1695 for key,val in pardict.items():
1696 newval = None
1698 if val.startswith('['):
1699 newval = ast.literal_eval(val)
1700 elif val.startswith('{'):
1701 newval = eval(val)
1702 elif val in ['True', 'False']:
1703 if eval(val) in [True, False]:
1704 newval = eval(val)
1705 if newval is None:
1706 try:
1707 # Try int
1708 int(val)
1709 newval = int(val)
1710 except:
1711 # Try float
1712 try:
1713 float(val)
1714 newval = float(val)
1715 except:
1716 # Try string
1717 if val.count("'") > 0:
1718 val = val.strip("'")
1719 elif val.count('"') > 0:
1720 val = val.strip('"')
1722 # CAS-6553 cannot have only one quote. remove it
1723 if val.count("'") == 1:
1724 val = val.replace("'", '')
1725 elif val.count('"') == 1:
1726 val = val.replace('"', '')
1728 newval = str(val).strip()
1730 cmddict[key] = newval
1732 return cmddict
1734def evaluateFlagParameters(pardict, pars):
1735 """Check if a flagdata parameter dictionary is valid
1737 Keyword arguments:
1738 pardict -- a dictionary with flag commands as returned by parseDictionary()
1739 pars -- dictionary of flagdata's parameters (locals() inside flagdata task)
1741 It checks if the parameter exists in the reference tuple.
1742 It checks if the type of the parameter value matches the type in the reference
1743 for that parameter.
1744 It raises an exception if any parameter or type of value do not match.
1746 """
1747 from casatasks import flagdata
1749 # Make a deepcopy of flagdata parameters dictionary for modification
1750 fpars = copy.deepcopy(pars)
1752 # Get the defaults of each parameter
1753 for par in fpars.keys():
1754 fpars[par] = get_task_arg_default(flagdata, par)
1756 # Define the parameters that don't go in an input list in flagdata
1757 removepars = ['vis','inpfile','flagbackup','tbuff','cmdreason','savepars','outfile',
1758 'display','action']
1760 # Remove the parameters we don't need to evaluate
1761 for par in removepars:
1762 if par in fpars:
1763 fpars.pop(par)
1765 # Define the parameters that have variant type in flagdata
1766 dup_pars = ['ntime','observation','addantenna','timedev','timedev','freqdev','freqdev','quackinterval']
1767 dup_types = [0.0,0,{},[],0.0,[],0.0,0]
1769 # Create a tuple from flagdata's parameters
1770 ftup = fpars.items()
1772 # Create a list of the tuples
1773 flist = list(ftup)
1775 # Append the duplicated types
1776 for d in range(len(dup_pars)):
1777 # Create a tuple and append to list
1778 dpar = (dup_pars[d],dup_types[d])
1779 flist.append(dpar)
1782 # Create a tuple of the reference dictionary
1783 reftup = tuple(flist)
1785 # Reference parameters for flagdata task
1786# reftup = (('field','string'),('spw','string'),('antenna','string'),('timerange','string'),('correlation','string'),
1787# ('scan','string'),('intent','string'),('array','string'),('uvrange','string'),
1788# ('observation','string'),
1789# ('observation',0),
1790# ('feed','string'),
1791# ('mode','string'),
1792# ('reason','string'),
1793# # manual
1794# ('autocorr', True),
1795# # clip
1796# ('datacolumn','string'),
1797# ('clipminmax',[]),
1798# ('clipoutside',True),
1799# ('channelavg',False),
1800# ('clipzeros',False),
1801# # elevation
1802# ('lowerlimit',0.0),
1803# ('upperlimit',90.0),
1804# # shadow
1805# ('tolerance',0.0),
1806# ('addantenna','string'),
1807# ('addantenna',{}),
1808# # quack
1809# ('quackinterval',0.0),
1810# ('quackinterval',0),
1811# ('quackmode','string'),
1812# ('quackincrement',False),
1813# # tfcrop
1814# ('ntime','string'),
1815# ('ntime',0.0),
1816# ('combinescans',False),
1817# ('timecutoff',4.0),
1818# ('freqcutoff',3.0),
1819# ('timefit','string'),
1820# ('freqfit','string'),
1821# ('maxnpieces',7),
1822# ('flagdimension','string'),
1823# ('usewindowstats','string'),
1824# ('halfwin',1),
1825# ('extendflags',True),
1826# # rflag
1827# ('winsize',3),
1828# ('timedev','string'),
1829# ('timedev',[]),
1830# ('timedev',0.0),
1831# ('freqdev','string'),
1832# ('freqdev',[]),
1833# ('freqdev',0.0),
1834# ('timedevscale',5.0),
1835# ('freqdevscale',5.0),
1836# ('spectralmax',1000000.0),
1837# ('spectralmin',0.0),
1838# # extend
1839# ('extendpols',True),
1840# ('growtime',50.0),
1841# ('growfreq',50.0),
1842# ('growaround',False),
1843# ('flagneartime',False),
1844# ('flagnearfreq',False),
1845# # summary
1846# ('minrel',0.0),
1847# ('maxrel',1.0),
1848# ('minabs',0),
1849# ('maxabs',1),
1850# ('spwchan',False),
1851# ('spwcorr',False),
1852# ('basecnt',False),
1853# ('name','string'))
1856 reference = defaultdict(list)
1857 for k, v in reftup:
1858 reference[k].append(v)
1860 refkeys = reference.keys()
1862 # Check the user dictionary against the reference
1863 count = 0
1864 for idx in pardict.keys():
1865 mydict = pardict[idx]['command']
1866 count += 1
1867 for key,val in mydict.items():
1868 if key not in refkeys:
1869 raise ValueError('Parameter \'%s\' in row=%s is not a valid flagdata parameter'%(key,idx))
1871 # reference[key] is always a list
1872 refval = reference[key]
1873 vtypes = ""
1874 match = False
1875 for tt in refval:
1876 vtypes = vtypes + str(type(tt)) + ','
1877 if type(val) == type(tt):
1878 # type matches
1879 match = True
1880 if not match:
1881 raise(ValueError,'Parameter %s=%s in row=%s has wrong type.\nSupported types are: %s.'%(key,val,idx,vtypes.rstrip(',')))
1883 casalog.post('Evaluated %s rows of dictionary'%count,'DEBUG1')
1884 return True
1886def evaluateNumpyType(elem):
1887 '''Evaluate if an element is of numpy type.
1888 Cast it to the corresponding Python type
1889 and return the casted value'''
1891 import numpy as np
1893 val = None
1895 if (isinstance(elem, np.int_) or isinstance(elem, np.int8) or
1896 isinstance(elem, np.int16) or isinstance(elem, np.int32) or
1897 isinstance(elem, np.int64)):
1898 val = int(elem)
1900 elif(isinstance(elem,np.float16) or
1901 isinstance(elem,np.float32) or isinstance(elem,np.float64)):
1902 val = float(elem)
1904 elif (hasattr(np, "float")) and isinstance(elem,np.float):
1905 val = float(elem)
1906 # float128 is not available on all platforms
1907 elif (hasattr(np, "float128")) and isinstance(elem,np.float128):
1908 val = float(elem)
1911 else:
1912 # it is none of the above numpy types
1913 val = elem
1915 # return the casted element
1916 return val
1918def parseXML(sdmfile, mytbuff):
1919 '''
1920# readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses
1921# into returned dictionary as flag command strings
1922# sdmfile (string) path to SDM containing Antenna.xml and Flag.xml
1923# mytbuff (float) time interval (start and end) padding (seconds)
1924#
1925# Usage: myflags = readflagxml(sdmfile,tbuff)
1926#
1927# Dictionary structure:
1928# fid : 'id' (string)
1929# 'mode' (string) flag mode ('online')
1930# 'antenna' (string)
1931# 'timerange' (string)
1932# 'reason' (string)
1933# 'time' (float) in mjd seconds
1934# 'interval' (float) in mjd seconds
1935# 'command' (string) string (for COMMAND col in FLAG_CMD)
1936# 'type' (string) 'FLAG' / 'UNFLAG'
1937# 'applied' (bool) set to True here on read-in
1938# 'level' (int) set to 0 here on read-in
1939# 'severity' (int) set to 0 here on read-in
1940#
1941# Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA
1942# Updated STM 2012-02-14 handle spectral window indices, names, IDs
1943# Updated STM 2012-02-21 handle polarization types
1944#
1945# Mode to use for spectral window selection in commands:
1946# spwmode = 0 none (flag all spw)
1947# spwmode = 1 use name
1948# spwmode = -1 use index (counting rows in SpectralWindow.xml)
1949#
1950# Mode to use for polarization selection in commands:
1951# polmode = 0 none (flag all pols/corrs)
1952# polmode = 1 use polarization type
1953#
1954# CURRENT DEFAULT: Use spw names, flag pols
1955 '''
1957 spwmode = 1
1958 polmode = 1
1960#
1961 try:
1962 from xml.dom import minidom
1963 except ImportError as exc:
1964 raise ImportError('Failed to load xml.dom.minidom, required in parseXML: ', exc)
1966 if type(mytbuff) != float:
1967 raise RuntimeError('Found incorrect type for tbuff. type(mytbuff): {}'.
1968 format(type(mytbuff)))
1970 # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends)
1971 flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK)
1972 antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK)
1973 spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK)
1974 if not flagexist:
1975 casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE')
1976 exit(1)
1977 if not antexist:
1978 casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE'
1979 )
1980 exit(1)
1981 if not spwexist:
1982 casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml',
1983 'WARN')
1985 # construct look-up dictionary of name vs. id from Antenna.xml
1986 xmlants = minidom.parse(sdmfile + '/Antenna.xml')
1987 antdict = {}
1988 rowlist = xmlants.getElementsByTagName('row')
1989 for rownode in rowlist:
1990 rowname = rownode.getElementsByTagName('name')
1991 ant = str(rowname[0].childNodes[0].nodeValue).strip()
1992 rowid = rownode.getElementsByTagName('antennaId')
1993 # CAS-4532: remove spaces between content and tags
1994 antid = str(rowid[0].childNodes[0].nodeValue).strip()
1995 antdict[antid] = ant
1996 casalog.post('Found ' + str(rowlist.length)
1997 + ' rows in Antenna.xml')
1999 # construct look-up dictionary of name vs. id from SpectralWindow.xml
2000 if spwexist:
2001 xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml')
2002 spwdict = {}
2003 rowlist = xmlspws.getElementsByTagName('row')
2004 ispw = 0
2005 wvrnominal = False
2006 for rownode in rowlist:
2007 rowid = rownode.getElementsByTagName('spectralWindowId')
2008 # CAS-4532: remove spaces between content and tags
2009 spwid = str(rowid[0].childNodes[0].nodeValue).strip()
2010 spwdict[spwid] = {}
2011 spwdict[spwid]['index'] = ispw
2012 if rownode.getElementsByTagName('name'):
2013 rowname = rownode.getElementsByTagName('name')
2014 spwname = str(rowname[0].childNodes[0].nodeValue).strip()
2015 if spwname == 'WVR#NOMINAL':
2016 wvrnominal = True
2017 spwdict[spwid]['name'] = spwname
2018 else:
2019 spwmode = -1
2021# rowid = rownode.getElementsByTagName('spectralWindowId')
2022# spwid = str(rowid[0].childNodes[0].nodeValue)
2023# spwdict[spwid] = {}
2024# spwdict[spwid]['index'] = ispw
2025 ispw += 1
2026 casalog.post('Found ' + str(rowlist.length)
2027 + ' rows in SpectralWindow.xml')
2029 # report chosen spw and pol modes
2030 if spwmode > 0:
2031 casalog.post('Will construct spw flags using names')
2032 elif spwmode < 0:
2033 casalog.post('Will construct spw flags using table indices')
2034 else:
2035 casalog.post('')
2036 #
2037 if polmode == 0:
2038 casalog.post('Will not set polarization dependent flags (flag all corrs)'
2039 )
2040 else:
2041 casalog.post('Will construct polarization flags using polarizationType'
2042 )
2044 # now read Flag.xml into dictionary row by row
2045 xmlflags = minidom.parse(sdmfile + '/Flag.xml')
2046 flagdict = {}
2047 rowlist = xmlflags.getElementsByTagName('row')
2048 nrows = rowlist.length
2049 newsdm = -1
2050 newspw = -1
2051 newpol = -1
2052 for fid in range(nrows):
2053 rownode = rowlist[fid]
2054 rowfid = rownode.getElementsByTagName('flagId')
2055 fidstr = str(rowfid[0].childNodes[0].nodeValue).strip()
2056 flagdict[fid] = {}
2057 flagdict[fid]['id'] = fidstr
2058 rowid = rownode.getElementsByTagName('antennaId')
2059 antid = rowid[0].childNodes[0].nodeValue
2060 # check if there is a numAntenna specified (new format)
2061 rownant = rownode.getElementsByTagName('numAntenna')
2062 antname = ''
2063 if rownant.__len__() > 0:
2064 xid = antid.split()
2065 nant = int(rownant[0].childNodes[0].nodeValue)
2066 if newsdm < 0:
2067 casalog.post('Found numAntenna=' + str(nant)
2068 + ' must be a new style SDM')
2069 newsdm = 1
2070 # CAS-4698. Flag auto-correlations when there is
2071 # only one antenna
2072 if nant == 1:
2073 aid = xid[2]
2074 ana = antdict[aid]
2075 if antname == '':
2076 antname = ana+'&&*'
2077 else:
2078 antname += ',' + ana
2080 elif nant > 1:
2081 for ia in range(nant):
2082 aid = xid[2 + ia]
2083 ana = antdict[aid]
2084 if antname == '':
2085 antname = ana
2086 else:
2087 antname += ',' + ana
2088 else:
2089 # numAntenna = 0 means flag all antennas
2090 antname = ''
2091 else:
2092 if newsdm < 0:
2093 casalog.post('No numAntenna entry found, must be an old style SDM'
2094 )
2095 newsdm = 0
2096 nant = 1
2097 aid = antid
2098 ana = antdict[aid]
2099 antname = ana
2100 # start and end times in mjd ns
2101 rowstart = rownode.getElementsByTagName('startTime')
2102 start = int(rowstart[0].childNodes[0].nodeValue)
2103 startmjds = float(start) * 1.0E-9 - mytbuff
2104 t = qalocal.quantity(startmjds, 's')
2105 starttime = qalocal.time(t, form='ymd', prec=9)[0]
2106 rowend = rownode.getElementsByTagName('endTime')
2107 end = int(rowend[0].childNodes[0].nodeValue)
2108 endmjds = float(end) * 1.0E-9 + mytbuff
2109 t = qalocal.quantity(endmjds, 's')
2110 endtime = qalocal.time(t, form='ymd', prec=9)[0]
2111 # time and interval for FLAG_CMD use
2112 times = 0.5 * (startmjds + endmjds)
2113 intervs = endmjds - startmjds
2114 flagdict[fid]['time'] = times
2115 flagdict[fid]['interval'] = intervs
2116 # reasons
2117 rowreason = rownode.getElementsByTagName('reason')
2118 reas = str(rowreason[0].childNodes[0].nodeValue)
2119 # Replace any white space with underscores
2120 reason = reas.replace(' ','_')
2121 # NEW SDM ADDITIONS 2011-11-01
2122 rownspw = rownode.getElementsByTagName('numSpectralWindow')
2123 spwstring = ''
2124 if spwmode != 0 and rownspw.__len__() > 0:
2125 nspw = int(rownspw[0].childNodes[0].nodeValue)
2126 # has a new-style spw specification
2127 if newspw < 0:
2128 if not spwexist:
2129 casalog.post('Cannot open ' + sdmfile
2130 + '/SpectralWindow.xml', 'SEVERE')
2131 exit(1)
2132 casalog.post('Found numSpectralWindow=' + str(nspw)
2133 + ' must be a new style SDM')
2134 newspw = 1
2135 if nspw > 0:
2136 rowspwid = rownode.getElementsByTagName('spectralWindowId')
2137 spwids = rowspwid[0].childNodes[0].nodeValue
2138 xspw = spwids.split()
2139 for isp in range(nspw):
2140 spid = str(xspw[2 + isp])
2141 if spwmode > 0:
2142 spstr =spwdict[spid]['name']
2143 # Skip the WVR#Antenna_* if WVR#NOMINAL is present
2144 if spstr.__contains__('WVR#Antenna') and wvrnominal==True:
2145 continue
2146 if spwstring == '':
2147 spwstring = '"'+spstr+'"'
2148 else:
2149 spwstring += ',' + '"'+spstr+'"'
2150 else:
2151 spstr = (spwdict[spid]['index'])
2152 if spwstring == '':
2153 spwstring = spstr
2154 else:
2155 spwstring += ',' + spstr
2156 polstring = ''
2157 rownpol = rownode.getElementsByTagName('numPolarizationType')
2158 if polmode != 0 and rownpol.__len__() > 0:
2159 npol = int(rownpol[0].childNodes[0].nodeValue)
2160 # has a new-style pol specification
2161 if newpol < 0:
2162 casalog.post('Found numPolarizationType=' + str(npol)
2163 + ' must be a new style SDM')
2164 newpol = 1
2165 if npol > 0:
2166 rowpolid = \
2167 rownode.getElementsByTagName('polarizationType')
2168 polids = rowpolid[0].childNodes[0].nodeValue
2169 xpol = polids.split()
2170 for ipol in range(npol):
2171 polid = str(xpol[2 + ipol])
2172 if polstring == '':
2173 polstring = polid
2174 else:
2175 polstring += ',' + polid
2176 #
2177 # Construct antenna name and timerange and reason strings
2178 flagdict[fid]['antenna'] = antname
2179 timestr = starttime + '~' + endtime
2180 flagdict[fid]['timerange'] = timestr
2181 flagdict[fid]['reason'] = reason
2182 # Construct command strings (per input flag)
2183 cmddict = OrderedDict()
2184# cmd = "antenna='" + antname + "' timerange='" + timestr + "'"
2185# cmddict['mode'] = 'manual'
2186 cmddict['antenna'] = antname
2187 cmddict['timerange'] = timestr
2188 if spwstring != '':
2189# cmd += " spw='" + spwstring + "'"
2190 flagdict[fid]['spw'] = spwstring
2191 cmddict['spw'] = spwstring
2192# if polstring != '':
2193# cmd += " poln='" + polstring + "'"
2194# flagdict[fid]['poln'] = polstring
2195 if polstring != '':
2196 # Write the poln translation in correlation
2197 if polstring.count('R')>0:
2198 if polstring.count('L')>0:
2199 corr = 'RR,RL,LR,LL'
2200 else:
2201 corr = 'RR,RL,LR'
2202 elif polstring.count('L')>0:
2203 corr = 'LL,LR,RL'
2204 elif polstring.count('X')>0:
2205 if polstring.count('Y')>0:
2206 corr = 'XX,XY,YX,YY'
2207 else:
2208 corr = 'XX,XY,YX'
2209 elif polstring.count('Y')>0:
2210 corr = 'YY,YX,XY'
2212# cmd += " correlation='" + corr + "'"
2213 cmddict['correlation'] = corr
2214# flagdict[fid]['poln'] = polstring
2215# flagdict[fid]['command'] = cmd
2216 flagdict[fid]['command'] = cmddict
2217 #
2218 flagdict[fid]['type'] = 'FLAG'
2219 flagdict[fid]['applied'] = False
2220 flagdict[fid]['level'] = 0
2221 flagdict[fid]['severity'] = 0
2222 flagdict[fid]['mode'] = 'online'
2224 flags = {}
2225 if rowlist.length > 0:
2226 flags = flagdict
2227 casalog.post('Found ' + str(rowlist.length)
2228 + ' flags in Flag.xml')
2229 else:
2230 casalog.post('No valid flags found in Flag.xml')
2232 # return the dictionary for later use
2233 return flags
2236# END of NEW PARSER FUNCTIONS
2237##############################################################################################
2238##############################################################################################
2241# (CAS-4119): Use absolute paths for input files to ensure that the engines find them
2242def addAbsPath(input_file):
2243 '''Read in the lines from an input file
2244 input_file --> file in disk with a list of strings per line
2246 Re-writes the file changing relative file names to absolute file names
2247 '''
2249 output_file = input_file + ".tmp"
2250 if (type(input_file) == str) & os.path.exists(input_file):
2251 try:
2252 input_file_id = open(input_file, 'r')
2253 except:
2254 raise Exception('Error opening file ' + input_file)
2256 try:
2257 output_file_id = open(output_file, 'w')
2258 except:
2259 output_file_id.close()
2260 raise Exception('Error opening tmp file ' + output_file)
2261 else:
2262 raise Exception('ASCII file not found - please verify the name')
2264 #
2265 # Parse file
2266 try:
2267 for line in input_file_id:
2268 cmd = line.rstrip()
2269 cmd_pars = cmd.split(" ")
2270 for par in cmd_pars:
2271 if ((par.count("inpfile") > 0) or
2272 (par.count("outfile") > 0) or
2273 (par.count("addantenna") > 0) or
2274 (par.count("timedev") > 0) or
2275 (par.count("freqdev") > 0)):
2276 par = par.split("=",1)
2277 file_local = par[1]
2278 if file_local.count("'")>0:
2279 file_local = file_local.split("'")
2280 file_local = file_local[1]
2281 elif file_local.count('"')>0:
2282 file_local = file_local.split('"')
2283 file_local = file_local[1]
2284 else:
2285 continue
2286 file_abs = os.path.abspath(file_local)
2287 cmd = cmd.replace(file_local,file_abs)
2289 output_file_id.write(cmd+"\n")
2291 except:
2292 input_file_id.close()
2293 output_file_id.close()
2294 raise Exception('Error reading lines from file ' \
2295 + input_file)
2297 input_file_id.close()
2298 output_file_id.close()
2299# os.rename(output_file, input_file)
2301 # Return the temporary file
2302 return output_file
2304def makeDict(cmdlist, myreason='any'):
2305 '''Make a dictionary compatible with a FLAG_CMD table structure
2306 and select by reason if any is given
2308 cmdlist --> list of parameters to go into the COMMAND column
2309 myreason --> reason to select from
2311 Returns a dictionary with the the selected rows with the following
2312 structure and default values:
2314 flagd['row'] =
2315 flagd['antenna'] =
2316 flagd['timerange'] =
2317 flagd['applied'] = False
2318 flagd['command'] = ''
2319 flagd['interval'] = 0.0
2320 flagd['level'] = 0
2321 flagd['reason'] = ''
2322 flagd['severity'] = 0
2323 flagd['time'] = 0.0
2324 flagd['type'] = 'FLAG'
2326 '''
2328 if cmdlist.__len__() == 0:
2329 raise Exception('Empty list of commands')
2331 # select by these reasons
2332 myreaslist = []
2333 if type(myreason) == str:
2334 if myreason != 'any':
2335 myreaslist.append(myreason)
2336 elif type(myreason) == list:
2337 myreaslist = myreason
2338 else:
2339 casalog.post('Cannot read reason; it contains unknown variable types',
2340 'ERROR')
2341 return
2343 if debug:
2344 casalog.post("reason in selection: {}".format(myreaslist))
2346 # List of reasons in input file
2347 nrows = cmdlist.__len__()
2349 # If any selection was requested
2350 reaslist = []
2351 for i in range(nrows):
2352 command = cmdlist[i]
2353 if command.__contains__('reason'):
2354 reaslist.append(getReason(command))
2355 else:
2356 # so that cmdlist and reaslist have the same sizes
2357 reaslist.append('NoReasonToMatch')
2359 if debug:
2360 casalog.post("reason in input: {}".format(reaslist))
2363 # Defaults for columns
2364 applied = False
2365 interval = 0.0
2366 level = 0
2367 severity = 0
2368 coltime = 0.0
2369 coltype = 'FLAG'
2370 antenna = ''
2371 timerange = ''
2373 myflagd = {}
2374 ncmds = 0
2375 rowlist = range(nrows)
2377 # If select by reason is requested
2378 if myreaslist.__len__() > 0:
2379 rowl = []
2381 # If selection matches what is in input line
2382 for j in range(reaslist.__len__()):
2383 if myreaslist.count(reaslist[j]) > 0:
2384 rowl.append(j)
2385 rowlist = rowl
2387 try:
2388 for i in rowlist:
2389 flagd = {}
2390 reason = ''
2392 command = cmdlist[i]
2393 if reaslist[i] != 'NoReasonToMatch':
2394 reason = reaslist[i]
2396 # Skip comment lines
2397 if command.startswith('#'):
2398 continue
2399 if command == '':
2400 casalog.post('Ignoring empty command line', 'WARN')
2401 continue
2402 if command.__contains__('summary'):
2403 casalog.post('Mode summary is not allowed in list operation', 'WARN')
2404 continue
2406 # If shadow, remove the addantenna dictionary
2407 if command.__contains__('shadow') and command.__contains__('addantenna'):
2408 i0 = command.rfind('addantenna')
2409 if command[i0+11] == '{':
2410 # It is a dictionary. Remove it from line
2411 i1 = command.rfind('}')
2412 antpar = command[i0+11:i1+1]
2413 temp = command[i0:i1+1]
2414 newcmd = command.replace(temp,'')
2415 antpardict = convertStringToDict(antpar)
2416 flagd['addantenna'] = antpardict
2417 command = newcmd
2419 # CAS-5180: Get antenna and timerange to use when action='plot'
2420 keyvlist = command.split()
2421 if keyvlist.__len__() > 0:
2422 for keyv in keyvlist:
2423 # Take care of '=' signs in values
2424 try:
2425 (xkey, val) = keyv.split('=',1)
2426 except:
2427 casalog.post('Error: not key=value pair for ' + keyv, 'ERROR')
2428 break
2429 xval = val
2430 if xval.count("'") > 0:
2431 xval = xval.strip("'")
2432 if xval.count('"') > 0:
2433 xval = xval.strip('"')
2434 if xkey == 'mode':
2435 flagd['mode'] = xval
2436 elif xkey == 'timerange':
2437 timerange = xval
2438 elif xkey == 'antenna':
2439 antenna = xval
2441 flagd['row'] = str(i)
2442 flagd['applied'] = applied
2443 flagd['reason'] = reason
2444 flagd['timerange'] = timerange
2445 flagd['antenna'] = antenna
2447 # Remove reason from command line
2448 command = command.rstrip()
2449 newline = purgeParameter(command, 'reason')
2451 # Remove any empty parameter (CAS-4015)
2452 command = purgeEmptyPars(newline)
2453 command = command.strip()
2455 flagd['command'] = command
2456 flagd['interval'] = interval
2457 flagd['level'] = level
2458 flagd['severity'] = severity
2459 flagd['time'] = coltime
2460 flagd['type'] = coltype
2461 # Insert into main dictionary
2462 myflagd[ncmds] = flagd
2463 ncmds += 1
2465 except:
2466 raise Exception('Cannot create dictionary')
2468 casalog.post(':makeDict::myflagd=%s'%myflagd,'DEBUG')
2470 return myflagd
2472def readXML(sdmfile, mytbuff):
2473 '''
2474# readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses
2475# into returned dictionary as flag command strings
2476# sdmfile (string) path to SDM containing Antenna.xml and Flag.xml
2477# mytbuff (float) time interval (start and end) padding (seconds)
2478#
2479# Usage: myflags = readflagxml(sdmfile,tbuff)
2480#
2481# Dictionary structure:
2482# fid : 'id' (string)
2483# 'mode' (string) flag mode ('online')
2484# 'antenna' (string)
2485# 'timerange' (string)
2486# 'reason' (string)
2487# 'time' (float) in mjd seconds
2488# 'interval' (float) in mjd seconds
2489# 'command' (string) string (for COMMAND col in FLAG_CMD)
2490# 'type' (string) 'FLAG' / 'UNFLAG'
2491# 'applied' (bool) set to True here on read-in
2492# 'level' (int) set to 0 here on read-in
2493# 'severity' (int) set to 0 here on read-in
2494#
2495# Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA
2496# Updated STM 2012-02-14 handle spectral window indices, names, IDs
2497# Updated STM 2012-02-21 handle polarization types
2498#
2499# Mode to use for spectral window selection in commands:
2500# spwmode = 0 none (flag all spw)
2501# spwmode = 1 use name
2502# spwmode = -1 use index (counting rows in SpectralWindow.xml)
2503#
2504# Mode to use for polarization selection in commands:
2505# polmode = 0 none (flag all pols/corrs)
2506# polmode = 1 use polarization type
2507#
2508# CURRENT DEFAULT: Use spw names, flag pols
2509 '''
2511 spwmode = 1
2512 polmode = 1
2514#
2515 try:
2516 from xml.dom import minidom
2517 except ImportError as exc:
2518 raise ImportError('failed to load xml.dom.minidom, required in readXML: ', exc)
2520 if type(mytbuff) != float:
2521 raise RuntimeError('Found incorrect type for tbuff. type(mytbuff): {}'.
2522 format(type(mytbuff)))
2524 # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends)
2525 flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK)
2526 antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK)
2527 spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK)
2528 if not flagexist:
2529 casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE')
2530 exit(1)
2531 if not antexist:
2532 casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE'
2533 )
2534 exit(1)
2535 if not spwexist:
2536 casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml',
2537 'WARN')
2539 # construct look-up dictionary of name vs. id from Antenna.xml
2540 xmlants = minidom.parse(sdmfile + '/Antenna.xml')
2541 antdict = {}
2542 rowlist = xmlants.getElementsByTagName('row')
2543 for rownode in rowlist:
2544 rowname = rownode.getElementsByTagName('name')
2545 ant = str(rowname[0].childNodes[0].nodeValue)
2546 rowid = rownode.getElementsByTagName('antennaId')
2547 # CAS-4532: remove spaces between content and tags
2548 antid = str(rowid[0].childNodes[0].nodeValue).strip()
2549 antdict[antid] = ant
2550 casalog.post('Found ' + str(rowlist.length)
2551 + ' antennas in Antenna.xml')
2553 # construct look-up dictionary of name vs. id from SpectralWindow.xml
2554 if spwexist:
2555 xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml')
2556 spwdict = {}
2557 rowlist = xmlspws.getElementsByTagName('row')
2558 ispw = 0
2559 for rownode in rowlist:
2560 rowid = rownode.getElementsByTagName('spectralWindowId')
2561 # CAS-4532: remove spaces between content and tags
2562 spwid = str(rowid[0].childNodes[0].nodeValue).strip()
2563 spwdict[spwid] = {}
2564 spwdict[spwid]['index'] = ispw
2565 # SMC: 6/3/2012 ALMA SDM does not have name
2566 if rownode.getElementsByTagName('name'):
2567 rowname = rownode.getElementsByTagName('name')
2568 spw = str(rowname[0].childNodes[0].nodeValue)
2569 spwdict[spwid]['name'] = spw
2570 else:
2571 spwmode = -1
2573# rowid = rownode.getElementsByTagName('spectralWindowId')
2574# spwid = str(rowid[0].childNodes[0].nodeValue)
2575# spwdict[spwid] = {}
2576# spwdict[spwid]['index'] = ispw
2577 ispw += 1
2578 casalog.post('Found ' + str(rowlist.length)
2579 + ' spw in SpectralWindow.xml')
2581 # report chosen spw and pol modes
2582 if spwmode > 0:
2583 casalog.post('Will construct spw flags using names')
2584 elif spwmode < 0:
2585 casalog.post('Will construct spw flags using table indices')
2586 else:
2587 casalog.post('')
2588 #
2589 if polmode == 0:
2590 casalog.post('Will not set polarization dependent flags (flag all corrs)'
2591 )
2592 else:
2593 casalog.post('Will construct polarization flags using polarizationType'
2594 )
2596 # now read Flag.xml into dictionary row by row
2597 xmlflags = minidom.parse(sdmfile + '/Flag.xml')
2598 flagdict = {}
2599 rowlist = xmlflags.getElementsByTagName('row')
2600 nrows = rowlist.length
2601 newsdm = -1
2602 newspw = -1
2603 newpol = -1
2604 for fid in range(nrows):
2605 rownode = rowlist[fid]
2606 rowfid = rownode.getElementsByTagName('flagId')
2607 fidstr = str(rowfid[0].childNodes[0].nodeValue)
2608 flagdict[fid] = {}
2609 flagdict[fid]['id'] = fidstr
2610 rowid = rownode.getElementsByTagName('antennaId')
2611 antid = rowid[0].childNodes[0].nodeValue
2612 # check if there is a numAntenna specified (new format)
2613 rownant = rownode.getElementsByTagName('numAntenna')
2614 antname = ''
2615 if rownant.__len__() > 0:
2616 xid = antid.split()
2617 nant = int(rownant[0].childNodes[0].nodeValue)
2618 if newsdm < 0:
2619 casalog.post('Found numAntenna=' + str(nant)
2620 + ' must be a new style SDM')
2621 newsdm = 1
2622 # CAS-4698. Flag auto-correlations when there is
2623 # only one antenna
2624 if nant == 1:
2625 aid = xid[2]
2626 ana = antdict[aid]
2627 if antname == '':
2628 antname = ana+'&&*'
2629 else:
2630 antname += ',' + ana
2632 elif nant > 1:
2633 for ia in range(nant):
2634 aid = xid[2 + ia]
2635 ana = antdict[aid]
2636 if antname == '':
2637 antname = ana
2638 else:
2639 antname += ',' + ana
2640 else:
2641 # numAntenna = 0 means flag all antennas
2642 antname = ''
2643 else:
2644 if newsdm < 0:
2645 casalog.post('No numAntenna entry found, must be an old style SDM'
2646 )
2647 newsdm = 0
2648 nant = 1
2649 aid = antid
2650 ana = antdict[aid]
2651 antname = ana
2652 # start and end times in mjd ns
2653 rowstart = rownode.getElementsByTagName('startTime')
2654 start = int(rowstart[0].childNodes[0].nodeValue)
2655 startmjds = float(start) * 1.0E-9 - mytbuff
2656 t = qalocal.quantity(startmjds, 's')
2657 starttime = qalocal.time(t, form='ymd', prec=9)[0]
2658 rowend = rownode.getElementsByTagName('endTime')
2659 end = int(rowend[0].childNodes[0].nodeValue)
2660 endmjds = float(end) * 1.0E-9 + mytbuff
2661 t = qalocal.quantity(endmjds, 's')
2662 endtime = qalocal.time(t, form='ymd', prec=9)[0]
2663 # time and interval for FLAG_CMD use
2664 times = 0.5 * (startmjds + endmjds)
2665 intervs = endmjds - startmjds
2666 flagdict[fid]['time'] = times
2667 flagdict[fid]['interval'] = intervs
2668 # reasons
2669 rowreason = rownode.getElementsByTagName('reason')
2670 reas = str(rowreason[0].childNodes[0].nodeValue)
2671 # Replace any white space with underscores
2672 reason = reas.replace(' ','_')
2673 # NEW SDM ADDITIONS 2011-11-01
2674 rownspw = rownode.getElementsByTagName('numSpectralWindow')
2675 spwstring = ''
2676 if spwmode != 0 and rownspw.__len__() > 0:
2677 nspw = int(rownspw[0].childNodes[0].nodeValue)
2678 # has a new-style spw specification
2679 if newspw < 0:
2680 if not spwexist:
2681 casalog.post('Cannot open ' + sdmfile
2682 + '/SpectralWindow.xml', 'SEVERE')
2683 exit(1)
2684 casalog.post('Found SpectralWindow=' + str(nspw)
2685 + ' must be a new style SDM')
2686 newspw = 1
2687 if nspw > 0:
2688 rowspwid = \
2689 rownode.getElementsByTagName('spectralWindowId')
2690 spwids = rowspwid[0].childNodes[0].nodeValue
2691 xspw = spwids.split()
2692 for isp in range(nspw):
2693 spid = str(xspw[2 + isp])
2694 if spwmode > 0:
2695 spstr = spwdict[spid]['name']
2696 else:
2697 spstr = str(spwdict[spid]['index'])
2698 if spwstring == '':
2699 spwstring = spstr
2700 else:
2701 spwstring += ',' + spstr
2702 polstring = ''
2703 rownpol = rownode.getElementsByTagName('numPolarizationType')
2704 if polmode != 0 and rownpol.__len__() > 0:
2705 npol = int(rownpol[0].childNodes[0].nodeValue)
2706 # has a new-style pol specification
2707 if newpol < 0:
2708 casalog.post('Found numPolarizationType=' + str(npol)
2709 + ' must be a new style SDM')
2710 newpol = 1
2711 if npol > 0:
2712 rowpolid = \
2713 rownode.getElementsByTagName('polarizationType')
2714 polids = rowpolid[0].childNodes[0].nodeValue
2715 xpol = polids.split()
2716 for ipol in range(npol):
2717 polid = str(xpol[2 + ipol])
2718 if polstring == '':
2719 polstring = polid
2720 else:
2721 polstring += ',' + polid
2722 #
2723 # Construct antenna name and timerange and reason strings
2724 flagdict[fid]['antenna'] = antname
2725 timestr = starttime + '~' + endtime
2726 flagdict[fid]['timerange'] = timestr
2727 flagdict[fid]['reason'] = reason
2728 # Construct command strings (per input flag)
2729 cmd = "antenna='" + antname + "' timerange='" + timestr + "'"
2730 if spwstring != '':
2731 cmd += " spw='" + spwstring + "'"
2732 flagdict[fid]['spw'] = spwstring
2733# if polstring != '':
2734# cmd += " poln='" + polstring + "'"
2735# flagdict[fid]['poln'] = polstring
2736 if polstring != '':
2737 # Write the poln translation in correlation
2738 if polstring.count('R')>0:
2739 if polstring.count('L')>0:
2740 corr = 'RR,RL,LR,LL'
2741 else:
2742 corr = 'RR,RL,LR'
2743 elif polstring.count('L')>0:
2744 corr = 'LL,LR,RL'
2745 elif polstring.count('X')>0:
2746 if polstring.count('Y')>0:
2747 corr = 'XX,XY,YX,YY'
2748 else:
2749 corr = 'XX,XY,YX'
2750 elif polstring.count('Y')>0:
2751 corr = 'YY,YX,XY'
2753 cmd += " correlation='" + corr + "'"
2754# flagdict[fid]['poln'] = polstring
2755 flagdict[fid]['command'] = cmd
2756 #
2757 flagdict[fid]['type'] = 'FLAG'
2758 flagdict[fid]['applied'] = False
2759 flagdict[fid]['level'] = 0
2760 flagdict[fid]['severity'] = 0
2761 flagdict[fid]['mode'] = 'online'
2763 flags = {}
2764 if rowlist.length > 0:
2765 flags = flagdict
2766 casalog.post('Found ' + str(rowlist.length)
2767 + ' flags in Flag.xml')
2768 else:
2769 casalog.post('No valid flags found in Flag.xml')
2771 # return the dictionary for later use
2772 return flags
2774def getUnion(vis, cmddict):
2775 '''Get a dictionary of a union of all selection parameters from a list of lines:
2776 vis --> MS
2777 cmddict --> dictionary of parameters and values (par=val) such as the one
2778 returned by makeDict()
2779 '''
2781 # Dictionary of parameters to return
2782 dicpars = {
2783 'field':'',
2784 'scan':'',
2785 'antenna':'',
2786 'spw':'',
2787 'timerange':'',
2788 'correlation':'',
2789 'intent':'',
2790 'feed':'',
2791 'array':'',
2792 'uvrange':'',
2793 'observation':''
2794 }
2796 # Strings for each parameter
2797 scans = ''
2798 fields = ''
2799 ants = ''
2800 times = ''
2801 corrs = ''
2802 ints = ''
2803 feeds = ''
2804 arrays = ''
2805 uvs = ''
2806 spws = ''
2807 obs = ''
2809 nrows = cmddict.keys().__len__()
2810# nrows = cmdlist.__len__()
2812# for i in range(nrows):
2813 for k in cmddict.keys():
2814# cmdline = cmdlist[i]
2815 cmdline = cmddict[k]['command']
2817 # Skip if it is a comment line
2818 if cmdline.startswith('#'):
2819 break
2821 # split by white space
2822 keyvlist = cmdline.split()
2823 if keyvlist.__len__() > 0:
2825 # Split by '='
2826 for keyv in keyvlist:
2828 (xkey,xval) = keyv.split('=',1)
2830 # Remove quotes
2831 if type(xval) == str:
2832 if xval.count("'") > 0:
2833 xval = xval.strip("'")
2834 if xval.count('"') > 0:
2835 xval = xval.strip('"')
2837 # Check which parameter
2838 if xkey == "scan":
2839 scans += xval + ','
2841 elif xkey == "field":
2842 fields += xval + ','
2844 elif xkey == "antenna":
2845 ants += xval + ';'
2847 elif xkey == "timerange":
2848 times += xval + ','
2850 elif xkey == "correlation":
2851 corrs += xval + ','
2853 elif xkey == "intent":
2854 ints += xval + ','
2856 elif xkey == "feed":
2857 feeds += xval + ','
2859 elif xkey == "array":
2860 arrays += xval + ','
2862 elif xkey == "uvrange":
2863 uvs += xval + ','
2865 elif xkey == "spw":
2866 spws += xval + ','
2868 elif xkey == "observation":
2869 obs += xval + ','
2872 # Strip out the extra comma at the end
2873 scans = scans.rstrip(',')
2874 fields = fields.rstrip(',')
2875 ants = ants.rstrip(';')
2876 times = times.rstrip(',')
2877 corrs = corrs.rstrip(',')
2878 ints = ints.rstrip(',')
2879 feeds = feeds.rstrip(',')
2880 arrays = arrays.rstrip(',')
2881 uvs = uvs.rstrip(',')
2882 spws = spws.rstrip(',')
2883 obs = obs.rstrip(',')
2885 dicpars['scan'] = scans
2886 dicpars['field'] = fields
2887 # Antennas are handled better within the framework.
2888 dicpars['antenna'] = ''
2889 # Times are handled better within the framework.
2890 dicpars['timerange'] = ''
2891 # Correlations should be handled only by the agents
2892 dicpars['correlation'] = ''
2893 # CAS-4682, do not create union for intents
2894 dicpars['intent'] = ''
2895 dicpars['feed'] = feeds
2896 dicpars['array'] = arrays
2897 dicpars['uvrange'] = uvs
2898 dicpars['spw'] = spws
2899 dicpars['observation'] = obs
2902 # Compress the selection list to reduce MSSelection parsing time.
2903 # 'field','spw','antenna' strings in dicpars will be modified in-place.
2904 compressSelectionList(vis,dicpars);
2906 # Real number of input lines
2907 # Get the number of occurrences of each parameter
2908 npars = getNumPar(cmddict)
2909 nlines = nrows - npars['comment']
2911 # Make the union.
2912 for k,v in npars.items():
2913 if k != 'comment':
2914 if v < nlines:
2915 dicpars[k] = ''
2918 uniondic = dicpars.copy()
2919 # Remove empty parameters from the dictionary
2920 for k,v in dicpars.items():
2921 if v == '':
2922 uniondic.pop(k)
2924 return uniondic
2926def getNumPar(cmddict):
2927 '''Get the number of occurrences of all parameter keys
2928 cmdlist --> list of strings with parameters and values
2929 '''
2931# nrows = cmdlist.__len__()
2933 # Dictionary of number of occurrences to return
2934 npars = {
2935 'field':0,
2936 'scan':0,
2937 'antenna':0,
2938 'spw':0,
2939 'timerange':0,
2940 'correlation':0,
2941 'intent':0,
2942 'feed':0,
2943 'array':0,
2944 'uvrange':0,
2945 'comment':0,
2946 'observation':0
2947 }
2949 ci = 0 # count the number of lines with comments (starting with a #)
2950 si = 0 # count the number of lines with scan
2951 fi = 0 # count the number of lines with field
2952 ai = 0 # count the number of lines with antenna
2953 ti = 0 # count the number of lines with timerange
2954 coi = 0 # count the number of lines with correlation
2955 ii = 0 # count the number of lines with intent
2956 fei = 0 # count the number of lines with feed
2957 ari = 0 # count the number of lines with array
2958 ui = 0 # count the number of lines with uvrange
2959 pi = 0 # count the number of lines with spw
2960 oi = 0 # count the number of lines with observation
2962# for i in range(nrows):
2963 for k in cmddict.keys():
2964# cmdline = cmdlist[i]
2965 cmdline = cmddict[k]['command']
2967 if cmdline.startswith('#'):
2968 ci += 1
2969 npars['comment'] = ci
2970 continue
2972 # split by white space
2973 keyvlist = cmdline.split()
2974 if keyvlist.__len__() > 0:
2976 # Split by '='
2977 for keyv in keyvlist:
2979 # Skip if it is a comment character #
2980 if keyv.count('#') > 0:
2981 break
2983 (xkey,xval) = keyv.split('=',1)
2985 # Remove quotes
2986 if type(xval) == str:
2987 if xval.count("'") > 0:
2988 xval = xval.strip("'")
2989 if xval.count('"') > 0:
2990 xval = xval.strip('"')
2991 # Remove blanks
2992 xval=xval.replace(' ','');
2994 # Check which parameter, if not empty
2995 if xval != "":
2996 if xkey == "scan":
2997 si += 1
2998 npars['scan'] = si
3000 elif xkey == "field":
3001 fi += 1
3002 npars['field'] = fi
3004 elif xkey == "antenna":
3005 ai += 1
3006 npars['antenna'] = ai
3008 elif xkey == "timerange":
3009 ti += 1
3010 npars['timerange'] = ti
3012 elif xkey == "correlation":
3013 coi += 1
3014 npars['correlation'] = coi
3016 elif xkey == "intent":
3017 ii += 1
3018 npars['intent'] = ii
3020 elif xkey == "feed":
3021 fei += 1
3022 npars['feed'] = fei
3024 elif xkey == "array":
3025 ari += 1
3026 npars['array'] = ari
3028 elif xkey == "uvrange":
3029 ui += 1
3030 npars['uvrange'] = ui
3032 elif xkey == "spw":
3033 pi += 1
3034 npars['spw'] = pi
3036 elif xkey == "observation":
3037 oi += 1
3038 npars['observation'] = oi
3041 return npars
3043def compressSelectionList(vis='',dicpars={}):
3044 """
3045 - Find a loose union of data-selection parameters, to reduce the MSSelection parsing load.
3046 - This compressed selection list is only meant to be used with af.selectdata(), because
3047 further selections are handled internally.
3048 - Use MSSelection in its 'onlyparse=True' mode to gather a list of fields, spws, antennas
3049 touched by the selection list. These are the only keys for which MSSelection does not
3050 need to parse the MS, and will cover the expensive cases where complicated antenna
3051 and spw expressions can slow MSSelection down.
3052 """
3053 from numpy import unique;
3055 mslocal.open(vis) #, nomodify=False)
3056 try:
3057 indices = mslocal.msseltoindex(vis=vis,field=dicpars['field'], spw=dicpars['spw'],baseline=dicpars['antenna']);
3058 finally:
3059 mslocal.close()
3061 c_field = str(list(unique(indices['field']))).strip('[]');
3062 c_spw = str(list(unique(indices['spw']))).strip('[]');
3063 c_antenna = str(list( unique( list(indices['antenna1']) + list(indices['antenna2']) ) ) ).strip('[]');
3065 dicpars['field'] = c_field;
3066 dicpars['spw'] = c_spw;
3067 dicpars['antenna'] = c_antenna;
3069 # Programmer note : Other selection parameters that can be compressed accurately
3070 # from MS subtable information alone (no need to parse the main table) are
3071 # 'array', 'observationid', 'state(or intent)'. They are currently un-available
3072 # via mslocal.msseltoindex() and therefore not used here yet.
3074 return;
3076# ONLY for flagcmd task
3077def writeFlagCmd(msfile, myflags, vrows, applied, add_reason, outfile):
3078 '''
3079 Writes the flag commands to FLAG_CMD or to an ASCII file. Used
3080 only by the flagcmd task. This uses the old, simple parser.
3081 For flagdata, use writeFlagCommands().
3083 msfile --> MS
3084 myflags --> dictionary of commands read from inputfile (from readFromTable, etc.)
3085 vrows --> list of valid rows from myflags dictionary to save
3086 applied --> value to update APPLIED column of FLAG_CMD
3087 add_reason --> reason to add to output (replace input reason, if any)
3088 outfile --> if not empty, save to it
3089 Returns the number of commands written to output
3090 '''
3092 nadd = 0
3093 try:
3094 import pylab as pl
3095 except ImportError as exc:
3096 raise ImportError('Failed to load pylab, required in writeFlagCmd: {}'.
3097 format(exc))
3100 # append to a file
3101 if outfile != '':
3102 ffout = open(outfile, 'a')
3104 # Replace blanks from reason and cmdreason with underscores
3105 # and warn the user.
3106 if add_reason != '':
3107 tempreason = add_reason
3108 add_reason = tempreason.replace(' ','_')
3109 casalog.post('Replaced blanks with underscores in cmdreason', 'WARN')
3111 try:
3112 for key in myflags.keys():
3113 # Remove leading and trailing white spaces
3114 cmdline = myflags[key]['command'].strip()
3116 # Add addantenna parameter back
3117 if myflags[key].__contains__('addantenna'):
3118 addantenna = myflags[key]['addantenna']
3119 cmdline = cmdline + ' addantenna=' + str(addantenna) + ' '
3121 reason = myflags[key]['reason']
3122 if add_reason != '':
3123 reason = str(add_reason)
3125 if reason != '':
3126 cmdline = cmdline + ' reason=' + str(reason) + ' '
3128 # For flagcmd saving
3129 newdict = evalString(cmdline)
3130# newdict = evaluateString(cmdline)
3131 cmdline = ''
3132 for k,v in newdict.items():
3133 cmdstr = ""
3134 # Add quotes to non-quoted strings
3135 if isinstance(v, str):
3136 if v.count("'") > 0:
3137 v = v.strip("'")
3138 if v.count('"') > 0:
3139 v = v.strip('"')
3141 # Add quotes to string values
3142 cmdstr = "'"+v+"'"
3143 v = cmdstr
3144 cmdline = cmdline + k + '=' + str(v) + ' '
3146 elif isinstance(v,list):
3147 lstring = ''
3148 for ll in v:
3149 if isinstance(ll,list):
3150 for nl in ll:
3151 lstring = lstring + str(nl)+','
3152 else:
3153 lstring = lstring + str(ll)+','
3154 lstring = lstring.rstrip(',')
3155 cmdline = cmdline + k + '=['+ lstring + '] '
3157 else:
3158 cmdline = cmdline + k + '=' + str(v) + ' '
3160 # Save to output file
3161 ffout.write('%s\n' %cmdline.rstrip())
3164# except:
3165 except Exception as instance:
3166 raise Exception('Error writing lines to file %s. %s' %(outfile,instance))
3167 ffout.close()
3168 return
3170 # Append new commands to existing table
3171 if vrows.__len__() > 0:
3172 # Extract flags from dictionary into list
3173 tim_list = []
3174 intv_list = []
3175 cmd_list = []
3176 reas_list = []
3177 typ_list = []
3178 sev_list = []
3179 lev_list = []
3180 app_list = []
3182 # Only write valid rows that have been applied to MS
3184 for key in vrows:
3185 # do not write line with summary mode
3186 command = myflags[key]['command']
3187 if command.__contains__('summary'):
3188 continue
3190 # Add addantenna back
3191 if myflags[key].__contains__('addantenna'):
3192 addantenna = myflags[key]['addantenna']
3193 command = command + ' addantenna=' + str(addantenna)
3195 # Evaluate the types
3196 newdict = evalString(command)
3197# newdict = evaluateString(command)
3198 cmdline = ''
3199 for k,v in newdict.items():
3200 cmdstr = ""
3201 if isinstance(v, str):
3202 if v.count("'") > 0:
3203 v = v.strip("'")
3204 if v.count('"') > 0:
3205 v = v.strip('"')
3206 # Add quotes to string values
3207 cmdstr = "'"+v+"'"
3208 v = cmdstr
3209 cmdline = cmdline + k + '=' + str(v) + ' '
3210 elif isinstance(v,list):
3211 lstring = ''
3212 for ll in v:
3213 lstring = lstring + str(ll)+','
3214 lstring = lstring.rstrip(',')
3215 cmdline = cmdline + k + '=['+ lstring + '] '
3216 else:
3217 cmdline = cmdline + k + '=' + str(v) + ' '
3219 cmd_list.append(cmdline)
3220 tim_list.append(myflags[key]['time'])
3221 intv_list.append(myflags[key]['interval'])
3222 if add_reason != '':
3223 reas_list.append(add_reason)
3224 else:
3225 reas_list.append(myflags[key]['reason'])
3226 typ_list.append(myflags[key]['type'])
3227 sev_list.append(myflags[key]['severity'])
3228 lev_list.append(myflags[key]['level'])
3229 app_list.append(applied)
3232 # Save to FLAG_CMD table
3233 nadd = cmd_list.__len__()
3235 mstable = msfile + '/FLAG_CMD'
3236 try:
3237 tblocal.open(mstable, nomodify=False)
3238 except:
3239 raise Exception('Error opening FLAG_CMD table ' + mstable)
3240 nrows = int(tblocal.nrows())
3241 casalog.post('There are ' + str(nrows)
3242 + ' rows already in FLAG_CMD', 'DEBUG')
3243 # add blank rows
3244 tblocal.addrows(nadd)
3245 # now fill them in
3246 tblocal.putcol('TIME', numpy.array(tim_list), startrow=nrows, nrow=nadd)
3247 tblocal.putcol('INTERVAL', numpy.array(intv_list), startrow=nrows,
3248 nrow=nadd)
3249 tblocal.putcol('REASON', numpy.array(reas_list), startrow=nrows,
3250 nrow=nadd)
3251 tblocal.putcol('COMMAND', numpy.array(cmd_list), startrow=nrows,
3252 nrow=nadd)
3253 # Other columns
3254 tblocal.putcol('TYPE', numpy.array(typ_list), startrow=nrows, nrow=nadd)
3255 tblocal.putcol('SEVERITY', numpy.array(sev_list), startrow=nrows,
3256 nrow=nadd)
3257 tblocal.putcol('LEVEL', numpy.array(lev_list), startrow=nrows,
3258 nrow=nadd)
3259 tblocal.putcol('APPLIED', numpy.array(app_list), startrow=nrows,
3260 nrow=nadd)
3262 nrows = int(tblocal.nrows())
3263 tblocal.close()
3264 casalog.post('Saved ' + str(nrows) + ' rows to FLAG_CMD')
3266 else:
3267 casalog.post('Saved zero rows to FLAG_CMD; no flags found')
3269 return nrows
3271def getReason(cmdline):
3272 '''Get the reason values from a line with strings
3273 cmdline --> a string with parameters
3274 returns a string with reason values.
3275 '''
3277 reason = ''
3279 # Skip comment lines
3280 if cmdline.startswith('#'):
3281 return reason
3283 # Split by white space
3284 keyvlist = cmdline.split()
3285 if keyvlist.__len__() > 0:
3287 for keyv in keyvlist:
3289 # Split by '='
3290 (xkey,xval) = keyv.split('=',1)
3292 # Remove quotes
3293 if type(xval) == str:
3294 if xval.count("'") > 0:
3295 xval = xval.strip("'")
3296 if xval.count('"') > 0:
3297 xval = xval.strip('"')
3299 # Check if reason is in
3300 if xkey == "reason":
3301 reason = xval
3302 break;
3305 return reason
3307def getLinePars(cmdline, mlist=[]):
3308 '''Get a dictionary of all selection parameters from a line:
3309 cmdline --> a string with parameters
3310 mlist --> a list of mode's parameters to add to the output dictionary
3312 Returns a dictionary.
3313 '''
3315 # Dictionary of parameters to return
3316 dicpars = {}
3318 # Skip comment lines
3319 if cmdline.startswith('#'):
3320 return dicpars
3322 # split by white space
3323 keyvlist = cmdline.split()
3324 if keyvlist.__len__() > 0:
3326 # Split by '='
3327 for keyv in keyvlist:
3329 (xkey,xval) = keyv.split('=',1)
3331 # Remove quotes
3332 if type(xval) == str:
3333 if xval.count("'") > 0:
3334 xval = xval.strip("'")
3335 if xval.count('"') > 0:
3336 xval = xval.strip('"')
3338 # Check which parameter
3339 if xkey == "scan":
3340 dicpars['scan'] = xval
3342 elif xkey == "field":
3343 dicpars['field'] = xval
3345 elif xkey == "antenna":
3346 dicpars['antenna'] = xval
3348 elif xkey == "timerange":
3349 dicpars['timerange'] = xval
3351 elif xkey == "correlation":
3352 dicpars['correlation'] = xval.upper()
3354 elif xkey == "intent":
3355 dicpars['intent'] = xval
3357 elif xkey == "feed":
3358 dicpars['feed'] = xval
3360 elif xkey == "array":
3361 dicpars['array'] = xval
3363 elif xkey == "uvrange":
3364 dicpars['uvrange'] = xval
3366 elif xkey == "spw":
3367 dicpars['spw'] = xval
3369 elif xkey == "observation":
3370 dicpars['observation'] = xval
3372 elif xkey == "mode":
3373 if xval == 'manualflag':
3374 xval = 'manual'
3375 dicpars['mode'] = xval
3377 elif mlist != []:
3378 # Any parameters requested for this mode?
3379 for m in mlist:
3380 if xkey == m:
3381 dicpars[m] = xval
3383 casalog.post(':getLinePars::dicpars=%s'%dicpars, 'DEBUG')
3385 return dicpars
3387def getSelectionPars(cmdline):
3388 '''Get a dictionary of all selection parameters from a line:
3389 cmdline --> a string with parameters
3390 '''
3392 # Dictionary of parameters to return
3393 dicpars = {}
3395 # Skip comment lines
3396 if cmdline.startswith('#'):
3397 return dicpars
3399 # split by white space
3400 keyvlist = cmdline.split()
3401 if keyvlist.__len__() > 0:
3403 # Split by '='
3404 for keyv in keyvlist:
3406 (xkey,xval) = keyv.split('=',1)
3408 # Remove quotes
3409 if type(xval) == str:
3410 if xval.count("'") > 0:
3411 xval = xval.strip("'")
3412 if xval.count('"') > 0:
3413 xval = xval.strip('"')
3415 # Check which parameter
3416 if xkey == "scan":
3417 dicpars['scan'] = xval
3419 elif xkey == "field":
3420 dicpars['field'] = xval
3422 elif xkey == "antenna":
3423 dicpars['antenna'] = xval
3425 elif xkey == "timerange":
3426 dicpars['timerange'] = xval
3428 # Correlation will be handled by the agent
3429 elif xkey == "correlation":
3430 dicpars['correlation'] = ''
3432 elif xkey == "intent":
3433 dicpars['intent'] = xval
3435 elif xkey == "feed":
3436 dicpars['feed'] = xval
3438 elif xkey == "array":
3439 dicpars['array'] = xval
3441 elif xkey == "uvrange":
3442 dicpars['uvrange'] = xval
3444 elif xkey == "spw":
3445 dicpars['spw'] = xval
3447 elif xkey == "observation":
3448 dicpars['observation'] = xval
3451 return dicpars
3453def readNtime(params):
3454 '''Check the value and units of ntime
3455 params --> dictionary of agent's parameters '''
3457 newtime = 0.0
3459 if 'ntime' in params:
3460 ntime = params['ntime']
3462 # Verify the ntime value
3463 if type(ntime) == float or type(ntime) == int:
3464 if ntime <= 0:
3465 raise Exception('Parameter ntime cannot be < = 0')
3466 else:
3467 # units are seconds
3468 newtime = float(ntime)
3470 elif type(ntime) == str:
3471 if ntime == 'scan':
3472 # iteration time step is a scan
3473 newtime = 0.0
3474 else:
3475 # read the units from the string
3476 qtime = qalocal.quantity(ntime)
3478 if qtime['unit'] == 'min':
3479 # convert to seconds
3480 qtime = qalocal.convert(qtime, 's')
3481 elif qtime['unit'] == '':
3482 qtime['unit'] = 's'
3484 # check units
3485 if qtime['unit'] == 's':
3486 newtime = qtime['value']
3487 else:
3488 casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN')
3490 params['ntime'] = float(newtime)
3492def fixType(params):
3493 '''Give correct types to non-string parameters
3494 The types are defined in the XML file of the task flagdata
3495 Do not repeat any parameter'''
3497 # manual parameter
3498 if 'autocorr' in params:
3499 params['autocorr'] = eval(params['autocorr'].capitalize())
3501 # quack parameters
3502 if 'quackmode' in params and not params['quackmode'] in ['beg'
3503 , 'endb', 'end', 'tail']:
3504 raise Exception( \
3505 "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \
3506 % params['quackmode'])
3507 if 'quackinterval' in params:
3508 params['quackinterval'] = float(params['quackinterval'])
3509 if 'quackincrement' in params:
3510 if type(params['quackincrement']) == str:
3511 params['quackincrement'] = eval(params['quackincrement'].capitalize())
3513 # clip parameters
3514 if 'clipminmax' in params:
3515 value01 = params['clipminmax']
3516 # turn string into [min,max] range
3517 value0 = value01.lstrip('[')
3518 value = value0.rstrip(']')
3519 r = value.split(',')
3520 rmin = float(r[0])
3521 rmax = float(r[1])
3522 params['clipminmax'] = [rmin, rmax]
3523 if 'clipoutside' in params:
3524 if type(params['clipoutside']) == str:
3525 params['clipoutside'] = eval(params['clipoutside'].capitalize())
3526 else:
3527 params['clipoutside'] = params['clipoutside']
3528 if 'channelavg' in params:
3529 params['channelavg'] = eval(params['channelavg'].capitalize())
3530 if 'clipzeros' in params:
3531 params['clipzeros'] = eval(params['clipzeros'].capitalize())
3534 # shadow parameter
3535 if 'tolerance' in params:
3536 params['tolerance'] = float(params['tolerance'])
3538 # elevation parameters
3539 if 'lowerlimit' in params:
3540 params['lowerlimit'] = float(params['lowerlimit'])
3541 if 'upperlimit'in params:
3542 params['upperlimit'] = float(params['upperlimit'])
3544 # extend parameters
3545 if 'extendpols' in params:
3546 params['extendpols'] = eval(params['extendpols'].capitalize())
3547 if 'growtime' in params:
3548 params['growtime'] = float(params['growtime'])
3549 if 'growfreq' in params:
3550 params['growfreq'] = float(params['growfreq'])
3551 if 'growaround' in params:
3552 params['growaround'] = eval(params['growaround'].capitalize())
3553 if 'flagneartime' in params:
3554 params['flagneartime'] = eval(params['flagneartime'].capitalize())
3555 if 'flagnearfreq' in params:
3556 params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize())
3558 # tfcrop parameters
3559 if 'combinescans' in params:
3560 params['combinescans'] = eval(params['combinescans'].capitalize())
3561 if 'timecutoff' in params:
3562 params['timecutoff'] = float(params['timecutoff'])
3563 if 'freqcutoff' in params:
3564 params['freqcutoff'] = float(params['freqcutoff'])
3565 if 'maxnpieces' in params:
3566 params['maxnpieces'] = int(params['maxnpieces'])
3567 if 'halfwin' in params:
3568 params['halfwin'] = int(params['halfwin'])
3569 if 'extendflags' in params:
3570 params['extendflags'] = eval(params['extendflags'].capitalize())
3572 # rflag parameters
3573 if 'winsize' in params:
3574 params['winsize'] = int(params['winsize']);
3575 if 'timedev' in params:
3576 timepar = params['timedev']
3577 try:
3578 timepar = eval(timepar)
3579 except Exception:
3580 timepar = readRFlagThresholdFile(params['timedev'],'timedev');
3581 params['timedev'] = timepar
3582 if 'freqdev' in params:
3583 freqpar = params['freqdev']
3584 try:
3585 freqpar = eval(freqpar)
3586 except Exception:
3587 freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev');
3588 params['freqdev'] = freqpar
3589 if 'timedevscale' in params:
3590 params['timedevscale'] = float(params['timedevscale']);
3591 if 'freqdevscale' in params:
3592 params['freqdevscale'] = float(params['freqdevscale']);
3593 if 'spectralmin' in params:
3594 params['spectralmin'] = float(params['spectralmin']);
3595 if 'spectralmax' in params:
3596 params['spectralmax'] = float(params['spectralmax']);
3598def purgeEmptyPars(cmdline):
3599 '''Remove empty parameters from a string:
3600 cmdline --> a string with parameters
3602 returns a string containing only parameters with values
3603 '''
3604 newstr = ''
3606 # split by white space
3607 keyvlist = cmdline.split()
3608 if keyvlist.__len__() > 0:
3610 # Split by '='
3611 for keyv in keyvlist:
3613 (xkey,xval) = keyv.split('=',1)
3615 # Remove quotes
3616 if type(xval) == str:
3617 if xval.count("'") > 0:
3618 xval = xval.strip("'")
3619 if xval.count('"') > 0:
3620 xval = xval.strip('"')
3622 # Write only parameters with values
3623 if xval == '':
3624 continue
3625 else:
3626 newstr = newstr+xkey+'='+xval+' '
3628 else:
3629 casalog.post('String of parameters is empty','WARN')
3631 return newstr
3633def purgeParameter(cmdline, par):
3634 '''Remove parameter from a string:
3635 cmdline --> a string with a parameter to be removed
3636 par --> the parameter to be removed from the string
3638 returns a string containing the remaining parameters
3639 '''
3641 newstr = ''
3643 # split by white space
3644 keyvlist = cmdline.split()
3645 if keyvlist.__len__() > 0:
3647 # Split by '='
3648 for keyv in keyvlist:
3650 (xkey,xval) = keyv.split('=',1)
3652 # Remove quotes
3653 if type(xval) == str:
3654 if xval.count("'") > 0:
3655 xval = xval.strip("'")
3656 if xval.count('"') > 0:
3657 xval = xval.strip('"')
3659 # Write only parameters with values
3660 if xkey == par:
3661 continue
3662 else:
3663 newstr = newstr+xkey+'=' + xval+ ' '
3665 else:
3666 casalog.post('String of parameters is empty','WARN')
3668 return newstr
3670def setupAgent(aflocal, myflagcmd, myrows, apply, writeflags, display=''):
3671 ''' Setup the parameters of each agent and call the agentflagger tool
3673 myflagcmd --> it is a dictionary coming from readFromTable, readFile, etc.
3674 myrows --> selected rows to apply/unapply flags
3675 apply --> it's a boolean to control whether to apply or unapply the flags
3676 writeflags --> used by mode=rflag only
3677 display --> used by mode='rflag only'''
3680 if not myflagcmd.__len__() >0:
3681 casalog.post('There are no flag cmds in list', 'SEVERE')
3682 return
3684 # Parameters for each mode
3685 manualpars = ['autocorr']
3686 unflagpars = []
3687 clippars = ['clipminmax', 'clipoutside','datacolumn', 'channelavg', 'clipzeros']
3688 quackpars = ['quackinterval','quackmode','quackincrement']
3689 shadowpars = ['tolerance', 'addantenna']
3690 elevationpars = ['lowerlimit','upperlimit']
3691 tfcroppars = ['ntime','combinescans','datacolumn','timecutoff','freqcutoff',
3692 'timefit','freqfit','maxnpieces','flagdimension','usewindowstats','halfwin',
3693 'extendflags']
3694 antintpars = ['antint_ref_antenna','minchanfrac','verbose']
3695 extendpars = ['ntime','combinescans','extendpols','growtime','growfreq','growaround',
3696 'flagneartime','flagnearfreq']
3697 rflagpars = ['winsize','timedev','freqdev','timedevscale','freqdevscale','spectralmax',
3698 'spectralmin', 'extendflags']
3701 # dictionary of successful command lines to save to outfile
3702 savelist = {}
3704 # Setup the agent for each input line
3705 for key in myflagcmd.keys():
3706 cmdline = myflagcmd[key]['command']
3707 applied = myflagcmd[key]['applied']
3708 interval = myflagcmd[key]['interval']
3709 level = myflagcmd[key]['level']
3710 reason = myflagcmd[key]['reason']
3711 severity = myflagcmd[key]['severity']
3712 coltime = myflagcmd[key]['time']
3713 coltype = myflagcmd[key]['type']
3714 if debug:
3715 casalog.post('cmdline for key%s'%key)
3716 casalog.post('%s'%cmdline)
3717 casalog.post('applied is %s'%applied)
3719 if cmdline.startswith('#'):
3720 continue
3722 modepars = {}
3723 parslist = {}
3724 mode = ''
3725 valid = True
3726 addantenna = {}
3728 # Get the specific parameters for the mode
3729 if cmdline.__contains__('mode'):
3730 if cmdline.__contains__('manual'):
3731 mode = 'manual'
3732 modepars = getLinePars(cmdline,manualpars)
3733 elif cmdline.__contains__('clip'):
3734 mode = 'clip'
3735 modepars = getLinePars(cmdline,clippars)
3736 elif cmdline.__contains__('quack'):
3737 mode = 'quack'
3738 modepars = getLinePars(cmdline,quackpars)
3739 elif cmdline.__contains__('shadow'):
3740 mode = 'shadow'
3741 modepars = getLinePars(cmdline,shadowpars)
3743 # Get addantenna dictionary
3744 if myflagcmd[key].__contains__('addantenna'):
3745 addantenna = myflagcmd[key]['addantenna']
3746 modepars['addantenna'] = addantenna
3747 else:
3748 # Get antenna filename
3749 if (modepars.__contains__('addantenna')):
3750 ant_par = modepars['addantenna']
3752 # It must be a string
3753 if (type(ant_par) == str and ant_par != ''):
3754 antennafile = modepars['addantenna']
3755 addantenna = readAntennaList(antennafile)
3756 modepars['addantenna'] = addantenna
3758 elif cmdline.__contains__('elevation'):
3759 mode = 'elevation'
3760 modepars = getLinePars(cmdline,elevationpars)
3761 elif cmdline.__contains__('tfcrop'):
3762 mode = 'tfcrop'
3763 modepars = getLinePars(cmdline,tfcroppars)
3764 elif cmdline.__contains__('antint'):
3765 mode = 'antint'
3766 modepars = getLinePars(cmdline,antintpars)
3767 elif cmdline.__contains__('extend') and cmdline.__contains__('extendpols'):
3768 # Necessary to avoid matching the extenflags parameter of rflag/tfcrop
3769 mode = 'extend'
3770 modepars = getLinePars(cmdline,extendpars)
3771 elif cmdline.__contains__('unflag'):
3772 mode = 'unflag'
3773 modepars = getLinePars(cmdline,unflagpars)
3774 elif cmdline.__contains__('rflag'):
3775 mode = 'rflag'
3776 modepars = getLinePars(cmdline,rflagpars)
3778 #####
3779 ### According to 'shadow' file handling, this code should be here...
3780 ### but, it's already done inside fixType.
3781 #####
3782 #if( type(modepars['timedev']) == str and writeflags == True):
3783 # timedev = readRFlagThresholdFile(modepars['timedev'],'timedev')
3784 # modepars['timedev'] = timedev
3785 #if( type(modepars['freqdev']) == str and writeflags == True):
3786 # freqdev = readRFlagThresholdFile(modepars['freqdev'],'freqdev')
3787 # modepars['freqdev'] = freqdev
3789 # Add the writeflags and display parameters
3790 modepars['writeflags'] = writeflags
3791 modepars['display'] = display
3792 else:
3793 # Unknown mode, ignore it
3794 casalog.post('Ignoring unknown mode', 'WARN')
3795# valid = False
3797 else:
3798 # No mode means manual
3799 mode = 'manual'
3800 cmdline = cmdline+' mode=manual'
3801 modepars = getLinePars(cmdline,manualpars)
3804 # Read ntime
3805 readNtime(modepars)
3807 # Cast the correct type to non-string parameters
3808 fixType(modepars)
3810 # Add the apply/unapply parameter to dictionary
3811 modepars['apply'] = apply
3813 # Unapply selected rows only and re-apply the other rows with APPLIED=True
3814# if not apply and myrows.__len__() > 0:
3815# if key in myrows:
3816# modepars['apply'] = False
3817# elif not applied:
3818# casalog.post("Skipping this %s"%modepars,"DEBUG")
3819# continue
3820# elif applied:
3821# modepars['apply'] = True
3822# valid = False
3824 # Keep only cmds that overlap with the unapply cmds
3825 # TODO later
3827 # Hold the name of the agent and the cmd row number
3828 agent_name = mode.capitalize()+'_'+str(key)
3829 modepars['name'] = agent_name
3831 # Remove the data selection parameters if there is only one agent for performance reasons.
3832 # Explanation: if only one agent exists and the data selection parameters are parsed to it,
3833 # it will have to go through the entire MS and check if each data selection given to the agent
3834 # matches what the user asked in the selected data.
3836 # Only correlation, antenna and timerange will go to the agent
3837 # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too
3838 if myflagcmd.__len__() == 1:
3839 sellist=['scan','field','intent','feed','array','uvrange','observation']
3840 for k in sellist:
3841 if k in modepars:
3842 modepars.pop(k)
3844 casalog.post('Parsing parameters of mode %s in row %s'%(mode,key), 'DEBUG')
3845 casalog.post('%s'%modepars, 'DEBUG')
3846 if debug:
3847 casalog.post('Parsing parameters of mode %s in row %s'%(mode,key))
3848 casalog.post(modepars)
3850 # Parse the dictionary of parameters to the tool
3851 if (not aflocal.parseagentparameters(modepars)):
3852 casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,key), 'WARN')
3853 continue
3855 # Save the dictionary of valid agents
3856 if valid:
3857 # add this command line to list to save in outfile
3858 parslist['row'] = key
3859 parslist['command'] = cmdline
3860 parslist['applied'] = applied
3861 parslist['interval'] = interval
3862 parslist['level'] = level
3863 parslist['reason'] = reason
3864 parslist['severity'] = severity
3865 parslist['time'] = coltime
3866 parslist['type'] = coltype
3867 if addantenna != {}:
3868 parslist['addantenna'] = addantenna
3869 savelist[key] = parslist
3871 if debug:
3872 casalog.post('Dictionary of valid commands to save')
3873 casalog.post('%s'%savelist)
3875 return savelist
3877def backupFlags(aflocal=None, msfile='', prename='flagbackup'):
3878 '''Create a backup of the FLAG column
3880 aflocal local version of the agentflagger tool or
3881 msfile name of MS/cal table to backup
3882 prename prefix for name of flag backup file
3884 If msfile is given, aflocal will not be used
3886 Create names like this:
3887 flags.flagcmd_1,
3888 flags.flagdata_1,
3890 Generally <task>_<i>, where i is the smallest
3891 integer giving a name, which does not already exist'''
3893 if msfile != '':
3894 # open msfile and attach it to tool
3895 aflocal = agentflagger()
3896 aflocal.open(msfile)
3898 elif aflocal == None:
3899 casalog.post('Need an MS or a copy of the agentflagger tool to create a backup','WARN')
3900 return
3902 prefix = prename
3903 try:
3904 existing = aflocal.getflagversionlist(printflags=False)
3906 # remove comments from strings
3907 existing = [x[0:x.find(' : ')] for x in existing]
3908 i = 1
3909 while True:
3910 versionname = prefix + '_' + str(i)
3912 if not versionname in existing:
3913 break
3914 else:
3915 i = i + 1
3917 time_string = str(time.strftime('%Y-%m-%d %H:%M:%S'))
3919 casalog.post('Saving current flags to ' + versionname, 'DEBUG')
3921 aflocal.saveflagversion(versionname=versionname,
3922 comment='Flags autosave on ' + time_string, merge='replace')
3923 finally:
3924 if msfile != '':
3925 aflocal.done()
3927 return
3930####
3931#### Set of functions to handle antenna information for shadowing.
3932####
3933#### - extractAntennaInfo : Extract info into a returned dictionary (and text file)
3934#### - antListWrite : Write the dictionary to a text file
3935#### - antListRead : Read the text file and return a dictionary
3936####
3937#### Example :
3938#### alist = extractAntennaInfo(msname='../Data/shadowtest.ms',
3939#### antnamelist=['VLA1','VLA15'], outfile='ttt.txt')
3940####
3941#### alist = antListRead('ttt.txt');
3942####
3943###################################
3944#### Example output text file ( 'ttt.txt' )
3945###################################
3946#
3947#name = VLA1
3948#diameter = 25.0
3949#position = [-1601144.961466915, -5041998.0197185818, 3554864.76811967]
3950#name = VLA15
3951#diameter = 25.0
3952#position = [-1601556.245351332, -5041990.7252590274, 3554684.6464035073]
3953#
3954###################################
3955#### Example output dictionary ( alist )
3956###################################
3957#
3958#{'0': {'diameter ': 25.0,
3959# 'name ': 'VLA1',
3960# 'position ': [-1601144.961466915,
3961# -5041998.0197185818,
3962# 3554864.7681196001]},
3963# '1': {'diameter ': 25.0,
3964# 'name ': 'VLA15',
3965# 'position ': [-1601556.245351332,
3966# -5041990.7252590274,
3967# 3554684.6464035069]}}
3968#
3969##################################
3971def extractAntennaInfo(msname='', antnamelist=[], outfile=''):
3972 """
3973 Function to extract antenna names, positions, and diameters
3974 for a specified subset of the ANTENNA subtable of the specified MS.
3975 - It writes a text file, which can be sent as input for shadow in the task
3976 - It also returns a dictionary, which can be sent as input for shadow in the tool.
3977 This dictionary can also be given as input in the task.
3979 msname : name of MS
3980 antennalist : list of strings (antenna names). Names must match exactly. Case insensitive.
3981 outfile : name of text file. Will be overwritten if exists. If outfile='', no output file is written
3983 Always returns a dictionary containing the same info as in the file
3985 Example :
3986 antinfo = extractAntennaInfo(msname='xxx.ms',antnamelist=['vla1','vla2'],outfile='out.txt');
3987 """
3988 ## Check that the MS exists
3989 if(not os.path.exists(msname)):
3990 casalog.post("Cannot find MS : {}".format(msname))
3991 return False;
3993 ## If outfile exists, delete it
3994 if(os.path.exists(outfile)):
3995 casalog.post("Replacing existing file : {}".format(outfile))
3996 rmcmd = "rm -rf "+outfile;
3997 os.system(rmcmd);
3999 ## Convert input antenna names to upper-case
4000 newants=[];
4001 for ants in antnamelist:
4002 newants.append( ants.upper() );
4003 antnamelist = newants;
4005 ## Read antenna subtable of input MS
4006 tblocal.open(msname+'/ANTENNA');
4007 a_position = (tblocal.getcol('POSITION')).transpose();
4008 a_dish_diameter = tblocal.getcol('DISH_DIAMETER');
4009 a_name = tblocal.getcol('NAME');
4010 tblocal.close();
4012 ## Pick out only selected antennas from this list, and make a dictionary
4013 antlist = {};
4014 counter=0;
4015 for antid in range(0, len(a_name)):
4016 if (a_name[antid]).upper() in antnamelist:
4017 antlist[str(counter)] = { 'name':a_name[antid] , 'position': list(a_position[antid]) , 'diameter': a_dish_diameter[antid] } ;
4018 counter=counter+1;
4020 ## Open a new file and write this info into it, if requested
4021 if(outfile != ''):
4022 casalog.post("Making new file : {}".format(outfile))
4023 writeAntennaList(outfile, antlist);
4024 ## always return the dictionary anyway.
4025 return antlist;
4027##############################################
4028def writeAntennaList(outfile='', antlist={}):
4029 """
4030 Save the antlist dictionary as a text file
4031 """
4032 ofile = open(outfile, 'w');
4033 for apid in sorted(antlist):
4034 apars = antlist[apid];
4035 ofile.write("name=" + str(apars['name']) + '\n');
4036 ofile.write("diameter=" + str(apars['diameter'])+'\n');
4037 ofile.write("position=" + str((apars['position']))+'\n');
4038 ofile.close();
4040##############################################
4041def readAntennaList(infile=''):
4042 """
4043 Read the antlist text file and return a dictionary
4045 A return value of empty {} indicates an error (or, empty file).
4047 The file needs to have 3 entries per antenna, on separate lines.
4048 The diameter and position are in units of meters, with positions in ITRF.
4049 Multiple antennas can be specified by repeating these three lines.
4050 Blank lines are allowed.
4051 Lines can be commented with '#' as the first character.
4053 Example :
4054 name = ea05
4055 diameter = 25.0
4056 position = [-1601144.96146691, -5041998.01971858, 3554864.76811967]
4058 """
4060 if (type(infile) == str) & os.path.exists(infile):
4061 try:
4062 ifile = open(infile,'r');
4063 except:
4064 raise Exception('Error opening file ' + infile)
4066 thelist = ifile.readlines();
4067 ifile.close();
4068 else:
4069 raise Exception( \
4070 'File %s not found - please verify the name'%infile)
4072 cleanlist=[];
4073 for aline in thelist:
4074 if(len(aline)>5 and aline[0] != '#'):
4075 cleanlist.append(aline.rstrip());
4077 #casalog.post('Found ' + str(len(cleanlist)) + ' valid lines out of ' + str(len(thelist));)
4079 if( len(cleanlist) > 0 and len(cleanlist) % 3 != 0 ):
4080 casalog.post("\nThe file needs to have 3 entries per antenna, on separate lines. For example :")
4081 casalog.post("name=ea05")
4082 casalog.post("diameter=25.0")
4083 casalog.post("position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]")
4084 casalog.post("\n")
4085 casalog.post("The diameter and position are in units of meters, with positions in ITRF")
4086 return False;
4088 antlist={};
4089 counter=0;
4090 for aline in range(0,len(cleanlist),3):
4091 antdict = {};
4092 for row in range(0,3):
4093 pars = cleanlist[aline+row].split("=",1);
4094 #casalog.post(aline, row, pars)
4095 if(len(pars) != 2):
4096 casalog.post('Error in parsing : ' + cleanlist[aline+row], 'ERROR')
4097 return {};
4098 else:
4099 if(pars[0].count('name') > 0 ):
4100 antdict[pars[0].rstrip()] = str(pars[1].rsplit()[0]);
4101 if(pars[0].count('diameter') > 0 ):
4102 antdict[pars[0].rstrip()] = float(pars[1]);
4103 if(pars[0].count('position') > 0 ):
4104 plist = pars[1][1:-2].replace('[','').split(',');
4105 if(len(plist) != 3):
4106 casalog.post('Error in parsing : ' + cleanlist[aline+row],
4107 'ERROR')
4108 return {};
4109 else:
4110 qlist=[];
4111 for ind in range(0,3):
4112 qlist.append(float(plist[ind]));
4113 antdict[pars[0].rstrip()] = qlist;
4114 antlist[str(counter)] = antdict;
4115 counter = counter+1;
4117 return antlist;
4119################################################
4120#
4121# Function to pull out RFLAG thresholds from the returned report dictionary.
4122#
4123def writeRFlagThresholdFile(rflag_thresholds={},timedevfile='', freqdevfile='',agent_id=0):
4124 """
4125 Extract the RFLAG output thresholds from the threshold dictionary
4126 Return them as arrays, and optionally, write them into a file.
4127 """
4128 # Decide the output file name.
4129 if( type(timedevfile) == str and timedevfile != '' and timedevfile.count('[')==0 and (not timedevfile.replace('.','').isdigit() ) ):
4130 toutfile = timedevfile
4131 else:
4132 toutfile = 'rflag_output_thresholds_timedev'+str(agent_id)+'.txt'
4134 # Decide the output file name.
4135 if( type(freqdevfile) == str and freqdevfile != '' and freqdevfile.count('[')==0 and (not freqdevfile.replace('.','').isdigit() ) ):
4136 foutfile = freqdevfile
4137 else:
4138 foutfile = 'rflag_output_thresholds_freqdev'+str(agent_id)+'.txt'
4140 # save rflag output in file, and print them everywhere.
4141 casalog.post("Saving RFlag_"+str(agent_id)+" output in : " + toutfile + " and " + foutfile, 'INFO')
4143 ofiletime = open(toutfile, 'w');
4144 ofilefreq = open(foutfile, 'w');
4145 # Construct dictionary from what needs to be stored.
4146 timedict = {'name':rflag_thresholds['name'] , 'timedev': (rflag_thresholds['timedev']).tolist()}
4147 freqdict = {'name':rflag_thresholds['name'] , 'freqdev': (rflag_thresholds['freqdev']).tolist()}
4148 timestr = convertDictToString(timedict)
4149 freqstr = convertDictToString(freqdict)
4150 # Write to file
4151 ofiletime.write(timestr + '\n');
4152 ofilefreq.write(freqstr + '\n');
4153 # Send to logger
4154 casalog.post("RFlag_"+str(agent_id)+" output timedev written to " + toutfile + " : " + timestr, 'INFO');
4155 casalog.post("RFlag_"+str(agent_id)+" output freqdev written to " + foutfile + " : " + freqstr, 'INFO');
4156 # End filed
4157 ofiletime.write('\n');
4158 ofiletime.close();
4159 ofilefreq.write('\n');
4160 ofilefreq.close();
4161 # Returne timedev, freqdev contents. This is for savepars later.
4162 return timedict['timedev'], freqdict['freqdev']
4164##############################################
4165def readRFlagThresholdFile(infile='',inkey=''):
4166 """
4167 Read the input RFlag threshold file, and return dictionaries.
4168 """
4169 if(infile==''):
4170 return [];
4172 if ( not os.path.exists(infile) ):
4173 casalog.post('Cannot find file : ' + infile)
4174 return [];
4176 ifile = open(infile,'r');
4177 thelist = ifile.readlines();
4178 ifile.close();
4180 cleanlist=[];
4181 for aline in thelist:
4182 if(len(aline)>5 and aline[0] != '#'):
4183 cleanlist.append(aline.rstrip().rstrip('\n'))
4185# threshlist={};
4186 threshlist = OrderedDict()
4187 for aline in range(0,len(cleanlist)):
4188 threshlist[str(aline)] = convertStringToDict(cleanlist[aline]);
4189 if inkey in threshlist[str(aline)]:
4190 devthreshold = threshlist[str(aline)][inkey]
4192 # return only the last one. There should be only one anyway.
4193 return devthreshold
4195##############################################
4196## Note - replace all arrays by lists before coming here.
4197def convertDictToString(indict={}):
4198 '''Convert dictionary to string'''
4199 thestr = str(indict);
4200 # Remove newlines and spaces from this string.
4201 thestr = thestr.replace('\n','');
4202 thestr = thestr.replace(' ','');
4203 return thestr;
4205##############################################
4206def convertStringToDict(instr=''):
4207 '''Convert string to dictionary'''
4208 instr = instr.replace('\n','');
4209 try:
4210 thedict = OrderedDict()
4211 thedict = ast.literal_eval(instr)
4212 except Exception as instance:
4213 casalog.post("*** Error converting string %s to dictionary : \'%s\'" % (instr,instance),'ERROR');
4214 return thedict;
4215##############################################
4217def extractRFlagOutputFromSummary(mode,summary_stats_list, flagcmd):
4218 """
4219 Function to pull out 'rflag' output from the long dictionary, and
4220 (1) write the output files with thresholds. If the filename is specified, use it.
4221 If filename is not specified, make one up.
4222 (2) modify entries in 'cmdline' so that it is ready for savepars.
4223 This is to ensure that 'savepars' saves the contents of the threshold-files
4224 and not just the file-names. It has to save it in the form that flagdata
4225 accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user
4226 need not keep track of threshold text files if they use 'savepars' with action='apply'.
4227 """
4228 if type(summary_stats_list) is dict:
4229 nreps = summary_stats_list['nreport']
4230 for rep in range(0,nreps):
4231 repname = 'report'+str(rep)
4232 if summary_stats_list[repname]['type'] == "rflag":
4233 # Pull out the rflag threshold dictionary. This has a 'name' in it.
4234 rflag_thresholds = summary_stats_list[repname]
4235 ##casalog.post(rflag_thresholds)
4236 # Get the rflag id, to later construct a 'name' from to match the above.
4237 rflagid = 0
4238 if mode=='list':
4239 rflagid = int( rflag_thresholds['name'].replace('Rflag_','') )
4240 # Go through the flagcmd list, to find the 'rflags'.....
4241 for key in flagcmd.keys():
4242 cmdline = flagcmd[key]['command'];
4243 if cmdline.__contains__('rflag'):
4244 # Check for match between input flagcmd and output threshold, via the rflag id
4245 if(key==rflagid):
4246 # If timedev,freqdev are missing from cmdline, add empty ones.
4247 if( not cmdline.__contains__('timedev=') ): # aah. don't confuse it with timedevscale
4248 cmdline = cmdline + " timedev=[] ";
4249 if( not cmdline.__contains__('freqdev=') ):
4250 cmdline = cmdline + " freqdev=[] ";
4251 # Pull out timedev, freqdev strings from flagcmd
4252 rflagpars = getLinePars(cmdline , ['timedev','freqdev','writeflags']);
4253 ##casalog.post("cmdline : "+cmdline)
4254 # Write RFlag thresholds to these file names.
4255 newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, rflagpars['timedev'], rflagpars['freqdev'], rflagid)
4256 ## Modify the flagcmd string, so that savepars sees the contents of the file
4257 if( rflagpars['timedev'].__contains__('[') ):
4258 oldstring = 'timedev='+str(rflagpars['timedev'])
4259 newstring = 'timedev='+str(newtimedev).replace(' ','')
4260 ##casalog.post( "time : replacing " + oldstring + newstring)
4261 cmdline = cmdline.replace( oldstring, newstring );
4262 if( rflagpars['freqdev'].__contains__('[') ):
4263 oldstring = 'freqdev='+str(rflagpars['freqdev'])
4264 newstring = 'freqdev='+str(newfreqdev).replace(' ','')
4265 ##casalog.post("freq : replacing " + oldstring + newstring)
4266 cmdline = cmdline.replace( oldstring, newstring );
4267 # Remove writeflags from the cmd to prevent it from going into savepars
4268 oldstring = 'writeflags='+str(rflagpars['writeflags'])
4269 cmdline = cmdline.replace( oldstring, "" );
4271 flagcmd[key]['command'] = cmdline;
4273# Only used in writeFlagCmd
4274def evalString(cmdline):
4275 '''Evaluate the correct types in a string with parameters'''
4277# newcmdline = ""
4278 cmddict = OrderedDict()
4280 try:
4281 pairs = cmdline.split()
4282 except:
4283 raise "Whitespace in parameter values is not yet supported"
4285 for kv in pairs:
4286 newval = None
4287 (key,val) = kv.split('=',1)
4289 # Selection parameters are always string
4290 if key == 'spw' or key == 'scan' or key == 'field' or \
4291 key == 'antenna' or key == 'correlation' or key == 'array' or \
4292 key == 'timerange' or key == 'uvrange' or key == 'feed':
4294 newval = str(val)
4295 cmddict[key] = newval
4296 continue
4298 if val.startswith('['):
4299 newval = ast.literal_eval(val)
4300 elif val.startswith('{'):
4301 newval = eval(val)
4302 elif val == 'True' or val == 'False':
4303 if eval(val) == True or eval(val) == False:
4304 newval = bool(val)
4306 if newval == None:
4307 try:
4308 int(val)
4309 newval = int(val)
4310 except:
4311 # string
4312 newval = str(val)
4313 if newval == None:
4314 try:
4315 float(val)
4316 newval = float(val)
4317 except:
4318 # string
4319 newval = str(val)
4321 cmddict[key] = newval
4323 return cmddict