Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/flaghelper.py: 40%

2349 statements  

« prev     ^ index     » next       coverage.py v7.6.4, created at 2024-11-01 07:19 +0000

1import os 

2import time 

3import ast 

4import copy 

5import numpy 

6import inspect 

7 

8from collections import deque,defaultdict 

9from collections import OrderedDict 

10 

11from casatasks import casalog, flagdata 

12from casatools import table,quanta,ms,agentflagger 

13from .parallel.parallel_task_helper import ParallelTaskHelper 

14 

15###some helper tools 

16tblocal = table() 

17mslocal = ms() 

18qalocal = quanta() 

19 

20''' 

21A set of helper functions for the tasks flagdata and flagcmd. 

22Class Parser: to parse flag commands 

23 

24I/O functions: 

25 get_flag_cmd_list 

26 readFile 

27 readFiles 

28 readAndParse 

29 parseDictionary 

30 parseXML 

31 readAntennaList 

32 writeAntennaList 

33 writeFlagCommands 

34 writeRflagThresholdFile 

35  

36Parameter handling 

37 compressSelectionList 

38 evalParams 

39 selectReason 

40 parseSelectionPars 

41 parseUnion 

42 purgeEmptyPars 

43 purgeParameter 

44 parseAgents 

45  

46Others 

47 backupFlags 

48 convertDictToString 

49 convertStringToDict 

50 extractAntennaInfo 

51 save_rflag_consolidated_files 

52 parseRflagOutputFromSummary 

53  

54''' 

55 

56debug = False 

57 

58def get_task_arg_default( func, arg ): 

59 spec = inspect.getfullargspec(func.__call__) 

60 

61 if arg not in spec.args: 

62 raise Exception("cannot find '%s' among the function arguments" % arg) 

63 return spec.defaults[spec.args.index(arg)-1] 

64 

65class Parser(): 

66 ''' Parser for input files. 

67 primarydivider --> first split the string by this character 

68 secondarydivider --> next split the string by this character 

69  

70 The constructor takes two separators. 

71 It first splits the string by the 'primarydivider'. It 

72 then loops through each entry after the first split and 

73 verifies if the 'secondarydivider' is in the string. If yes, 

74 the string is added to a list. If not, it removes the 

75 whitespace of the initial split, by bringing back two 

76 strings together. This will allow the primarydivider to be 

77 part of the string itself. Last thing, it returns an ordered 

78 dictionary using the imported class OrderedDict. 

79 ''' 

80 def __init__(self,primarydivider,secondarydivider): 

81 self.prime = primarydivider 

82 self.second = secondarydivider 

83 

84 def parse2Dictionary(self,string): 

85 res = self.initialsplit(string) 

86 new = [] 

87 for entry in res: 

88 if self.second not in entry: 

89 new[-1] += ' ' + entry 

90 else: 

91 new.append(entry) 

92# return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) 

93 return OrderedDict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) 

94 

95 def parseNoEval(self,string): 

96 res = self.initialsplit(string) 

97 new = [] 

98 for entry in res: 

99 if self.second not in entry: 

100 new[-1] += ' ' + entry 

101 else: 

102 new.append(entry) 

103# return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) 

104 return OrderedDict((entry[0],entry[1]) for entry in [entry.split(self.second,1) for entry in new]) 

105 

106 def parse2List(self,string): 

107 res = self.initialsplit(string) 

108 new = [] 

109 for entry in res: 

110 if self.second not in entry: 

111 new[-1] += ' ' + entry 

112 else: 

113 new.append(entry) 

114 return new 

115# return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) 

116# return OrderedDict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) 

117 

118 def initialsplit(self,string): 

119 nstring = string.strip() 

120 return nstring.split(self.prime) 

121 

122####################################################### 

123# 

124# Reading functions 

125# 

126####################################################### 

127def isCalTable(msname): 

128 '''Check if a file is a cal table 

129 msname --> filename 

130 Return 1 for cal table, 0 for MS and 2 for MMS''' 

131 

132 try: 

133 tblocal.open(msname) 

134 except: 

135 raise ValueError("Unable to open MS %s" % msname) 

136 

137 tbinfo = tblocal.info() 

138 tblocal.close() 

139 retval = 0 

140 

141 if tbinfo['type'] == 'Calibration': 

142 retval = 1 

143 

144 elif tbinfo['type'] == 'Measurement Set': 

145 # MMS type 

146 if tbinfo['subType'] == 'CONCATENATED' and ParallelTaskHelper.isParallelMS(msname): 

147 retval = 2 

148 else: 

149 # MS type 

150 retval = 0 

151 else: 

152 retval = 0 

153 

154 

155 return retval 

156 

157def addAbsolutePath(input_file): 

158 '''Read in the lines from an input file 

159 input_file --> file in disk with a list of strinZeeuwgs per line 

160  

161 Return a new file saved to disk with relative file names  

162 changed to absolute file names. The new file will have the name 

163 of the input_file + '.tmp' 

164 ''' 

165 

166 # Get a list of the flag commands from the file  

167 cmdlist = readFile(input_file) 

168 

169 # Make a dictionary of the flag commands 

170 flagdict = parseDictionary(cmdlist, 'any', False) 

171 

172 for key in flagdict: 

173 cmddict = flagdict[key]['command'] 

174 if 'addantenna' in cmddict and isinstance(cmddict['addantenna'],str) and \ 

175 cmddict['addantenna'] != '': 

176 cmddict['addantenna'] = os.path.abspath(cmddict['addantenna']) 

177 if 'timedev' in cmddict and isinstance(cmddict['timedev'],str) and \ 

178 cmddict['timedev'] != '': 

179 cmddict['timedev'] = os.path.abspath(cmddict['timedev']) 

180 if 'freqdev' in cmddict and isinstance(cmddict['freqdev'],str) and \ 

181 cmddict['freqdev'] != '': 

182 cmddict['freqdev'] = os.path.abspath(cmddict['freqdev']) 

183 

184 

185 # Convert the dictionary back to a list and save it 

186 output_file = os.path.abspath(input_file + ".tmp") 

187 writeFlagCommands('', flagdict, False, '', output_file, append=False) 

188 

189 # Return the temporary file 

190 return output_file 

191 

192def get_flag_cmd_list(inpfile): 

193 """ 

194 For flagdata list mode, get the list of commands from whatever has been given in the 

195 inpfile input parameter (a file, a list of files, or a Python string with commands). 

196 The output list has one item (command) per line in the input file(s) or string. 

197 

198 :param inpfile: inpfile parameter as passed to task flagdata 

199 

200 :returns: list of commands found in file(s) or string 

201 """ 

202 # inpfile is a file 

203 if isinstance(inpfile, str) and os.path.isfile(inpfile): 

204 flaglist = readFile(inpfile) 

205 nlines = len(flaglist) 

206 casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile)) 

207 

208 # inpfile is a list of files 

209 elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]): 

210 flaglist = readFiles(inpfile) 

211 

212 # Python list of strings 

213 elif isinstance(inpfile, list): 

214 flaglist = inpfile 

215 

216 else: 

217 raise ValueError('Unsupported input list of flag commands or input file does not ' 

218 'exist') 

219 

220 return flaglist 

221 

222def readFile(inputfile): 

223 '''Read in the lines from an input file 

224 inputfile --> file in disk with a list of strings per line 

225  

226 Returns a list. Blank lines are skipped. Boolean values will 

227 be capitalized to avoid errors in the parser later. 

228 ''' 

229 

230 flagfile = inputfile 

231 

232 if (type(flagfile) == str) & os.path.exists(flagfile): 

233 try: 

234 ff = open(flagfile, 'r') 

235 except: 

236 casalog.post('Error opening file ' + flagfile,'ERROR') 

237 raise 

238 else: 

239 casalog.post('ASCII file not found - please verify the name','ERROR') 

240 raise Exception('ASCII file not found') 

241 

242 # Parse file 

243 try: 

244 cmdlist = [] 

245 for line in ff: 

246 cmd = line.rstrip() 

247 if cmd == '': 

248 continue 

249 if cmd.startswith('#'): 

250 continue 

251 uppercmd = cmd.replace('true','True') 

252 cmd = uppercmd.replace('false','False') 

253 

254 cmdlist.append(cmd) 

255 

256 except: 

257 casalog.post('Error reading lines from file '+ff.name, 'SEVERE') 

258 ff.close() 

259 raise 

260 

261 ff.close() 

262 

263 return cmdlist 

264 

265def readFiles(inputfiles): 

266 '''Read in a list of files with flag commands 

267 inputfiles --> list of files in disk 

268 

269 Returns all files concatenated into one single list. Blank lines are skipped.  

270 Boolean values will be capitalized to avoid errors in the parser later. 

271 ''' 

272 

273 if not isinstance(inputfiles, list): 

274 casalog.post('Error opening list of flag commands ' + inputfiles,'ERROR') 

275 raise 

276 

277 cmdlist = [] 

278 

279 # Read files  

280 for flagfile in inputfiles: 

281 cmd = readFile(flagfile) 

282 nlines = len(cmd) 

283 casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile)) 

284 cmdlist = cmdlist + cmd 

285 

286 return cmdlist 

287 

288def readAndParse(inputlist, tbuff=None): 

289 '''Read in a list of flag commands and parse them into a dictionary. 

290 The flag commands can be from a list of files or from a list of flag commands. 

291 If tbuff is given, it will be applied to the timerange parameters. 

292  

293 Note: when tbuff is None, it is best to use the individual functions 

294 readFile(s) and parseDictionary to avoid parsing the file twice! 

295  

296 inputlist --> list of files in disk containing the flag commands or 

297 a list of Python strings with flag commands. 

298 tbuff --> list of time buffers to apply to each timerange. If tbuff 

299 is a list, it should be the same size of the list of inputfiles. 

300 If it is a Double value, it will be applied only to the first 

301 input file.  

302  

303 Returns a list of dictionaries. Blank lines are skipped. Boolean values will 

304 be capitalized to avoid errors in the parser. 

305  

306 Each parameter=value will become a key=value in the dictionary. Example: 

307 inputlist = ["mode='manual' spw='0' autocorr=true", 

308 "mode='shadow'"] 

309  

310 Returns: 

311 [{'mode':'manual','spw':'0','autocorr':True}, 

312 {'mode':'shadow'}] 

313  

314 ''' 

315 if not isinstance(inputlist, list): 

316 casalog.post('Error opening list of flag commands ' + inputlist,'ERROR') 

317 raise 

318 

319 # List of files 

320 if os.path.isfile(inputlist[0]): 

321 isFile = True 

322 

323 # List of strings 

324 else: 

325 isFile = False 

326 

327 if tbuff is None: 

328 doPadding = False 

329 else: 

330 doPadding = True 

331 

332 # Make the list of tbuff a deque  

333# dtbuff = deque() 

334#  

335# if isinstance(tbuff, float): 

336# dtbuff.append(tbuff) 

337# elif isinstance(tbuff, list): 

338# dtbuff = deque(i for i in tbuff) 

339 

340 # List of dictionaries to return  

341 listofdict = [] 

342 

343 # Initialize the parser 

344 myParser = Parser(' ', '=') 

345 

346 # Read files  

347 if isFile: 

348 for flagfile in inputlist: 

349 cmdlist = readFile(flagfile) 

350 nlines = len(cmdlist) 

351 casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile)) 

352 

353 parsedlist = [] 

354 for cmd in cmdlist: 

355 #Get a dictionary without type evaluation 

356 preparsing = myParser.parseNoEval(cmd) 

357 

358 # Evaluate the types  

359 parsed = evaluateParameters(preparsing) 

360 parsedlist.append(parsed) 

361 

362 # Apply time buffer to file 

363 if doPadding: 

364# mytbuff = dtbuff.popleft() 

365 applyTimeBufferList(parsedlist, tbuff) 

366# if dtbuff.__len__() == 0: 

367# doPadding = False 

368 

369 listofdict = listofdict + parsedlist 

370 

371 # It is a list of strings 

372 else: 

373 

374 cmdlist = inputlist 

375 nlines = len(cmdlist) 

376 casalog.post('Read %s command(s) from a Python list of strings'%nlines) 

377 

378 parsedlist = [] 

379 for cmd in cmdlist: 

380 #Get a dictionary without type evaluation 

381 preparsing = myParser.parseNoEval(cmd) 

382 

383 # Evaluate the types  

384 parsed = evaluateParameters(preparsing) 

385 parsedlist.append(parsed) 

386 

387 # Apply time buffer to list 

388 if doPadding: 

389# mytbuff = dtbuff.popleft() 

390 applyTimeBufferList(parsedlist, tbuff) 

391 

392 listofdict = listofdict + parsedlist 

393 

394 return listofdict 

395 

396def applyTimeBufferList(alist, tbuff=None): 

397 ''' Apply in-place a time buffer to ALL timerange parameters of a 

398 list of dictionaries with several flag commands. It will do the following: 

399  

400 alist --> list of dictionaries with flag commands. 

401 Ex: [{'antenna':'DV01', 'timerange':'2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'}, 

402 {'antenna':'DV02', 'timerange':'2013/10/15/10:25:30.110~2013/10/15/10:25:32.230'}, 

403 ...] 

404 tbuff --> float value or list of 2 values of time buffer to apply to all timerange parameters. 

405 When tbuff is a list of 2 values, the first value is applied to the lower time, 

406 the second to the upper time. 

407  

408 * it assumes that timerange has syntax t0~t1 

409 * split timerange in '~' to get t0 and t1 

410 * convert value to time in days using qalocal.totime 

411 * convert days to seconds 

412 * subtract tbuff0 from t0 and add tbuff1 to t1 

413 * convert back to time string with the form 'ymd' using qalocal.time 

414 * write new values back to dictionary 

415  

416 ''' 

417# if not isinstance(tbuff, float): 

418# casalog.post('Time buffer (tbuff) is not of type float', 'WARN') 

419# return 

420 

421 casalog.post('Apply time buffer padding to list of dictionaries', 'DEBUG1') 

422 

423 # When tbuff is float, the range is regular, otherwise it's irregular 

424 if isinstance(tbuff, list) and len(tbuff) == 2: 

425 tbuff0 = tbuff[0] 

426 tbuff1 = tbuff[1] 

427 elif isinstance(tbuff, list) and len(tbuff) == 1: 

428 tbuff0 = tbuff1 = tbuff[0] 

429 elif isinstance(tbuff, float): 

430 tbuff0 = tbuff1 = tbuff 

431 else: 

432 casalog.post('Time buffer (tbuff) is not of type float or list', 'WARN') 

433 return 

434 

435 def get_date_format(timestamp): 

436 '''Guess date-time string format to use in quanta tool time conversion, 

437 depending on whether the date seems to be present (as in 

438 2013/11/15/10:35:05) 

439 ''' 

440 if timestamp.count('/') == 3: 

441 return 'ymd' 

442 else: 

443 return '' 

444 

445 for cmddict in alist: 

446 if 'timerange' in cmddict: 

447 timerange = cmddict['timerange'] 

448 if timerange.find('~') != -1: 

449 t0, t1 = timerange.split('~',1) 

450 date_time_format_t0 = get_date_format(t0) 

451 date_time_format_t1 = get_date_format(t1) 

452 

453 # start time 

454 startTime = qalocal.totime(t0)['value'] 

455 startTimeSec = (startTime * 24 * 3600) - tbuff0 

456 startTimeSec = qalocal.quantity(startTimeSec, 's') 

457 paddedT0 = qalocal.time(startTimeSec, form=date_time_format_t0, prec=9)[0] 

458 # end time 

459 endTime = qalocal.totime(t1)['value'] 

460 endTimeSec = (endTime * 24 * 3600) + tbuff1 

461 endTimeSec = qalocal.quantity(endTimeSec, 's') 

462 paddedT1 = qalocal.time(endTimeSec, form=date_time_format_t1, prec=9)[0] 

463 

464 # update the original dictionary 

465 cmddict['timerange'] = paddedT0+'~'+paddedT1 

466 

467 return 

468 

469def parseDictionary(cmdlist, reason='any', shadow=True): 

470 '''Create a dictionary after parsing a list of flag commands. 

471 If reason is different than 'any', only the selected 

472 rows will be parsed to the final dictionary. 

473  

474 cmdlist --> list of flag commands OR list of dictionaries with flag commands as key:value 

475 reason --> reason or list of reasons to select from 

476 shadow --> True will make a dictionary of the addantenna parameter 

477 and add it to the flag command 

478  

479 Returns a dictionary with the the selected rows. Each parameter 

480 will become a key in the dictionary. 

481 If a mode='shadow' is present in the dictionary and the parameter 

482 addantenna is a filename (string), it will create a dictionary 

483 of the filename and add it to the parsed dictionary. 

484  

485 ''' 

486 if cmdlist.__len__() == 0: 

487 raise Exception('Empty list of commands') 

488 

489 # Gather all requested reasons 

490 myreaslist = [] 

491 if type(reason) == str: 

492 if reason != 'any': 

493 myreaslist.append(reason) 

494 elif type(reason) == list: 

495 myreaslist = reason 

496 else: 

497 casalog.post('Cannot read reason; it contains unknown variable types', 'ERROR') 

498 return 

499 

500 # Separate per ' ', then per '=' 

501 myParser = Parser(' ', '=') 

502 

503 flagdict = {} 

504 row = 0 

505 for cmd in cmdlist: 

506 cmddict = {} 

507 

508 # Simple list of strings ([key1='value1' key2='value2'] 

509 if isinstance(cmd, str): 

510 # Skip comment and empty lines 

511 if cmd.startswith('#'): 

512 continue 

513 if cmd == '': 

514 continue 

515 

516 uppercmd = cmd.replace('true','True') 

517 cmd = uppercmd.replace('false','False') 

518 

519 # Get a dictionary without type evaluation 

520 preparsing = myParser.parseNoEval(cmd) 

521 

522 # Evaluate the types 

523 parsed = evaluateParameters(preparsing) 

524 

525 # List of dictionaries: [{'key1':'value1', 'key2':'value2'}] 

526 elif isinstance(cmd, dict): 

527 parsed = cmd 

528 

529 # Parse the flag commands into a dictionary 

530 mode = '' 

531 antenna = '' 

532 timerange = '' 

533 cmddict['row'] = str(row) 

534 cmddict['id'] = str(row) 

535 cmddict['command'] = parsed 

536 if 'mode' in parsed: 

537 mode = parsed['mode'] 

538 

539 input_reason = '' 

540 if 'reason' in parsed: 

541 input_reason = parsed['reason'] 

542 

543 cmddict['reason'] = input_reason 

544 

545 if 'timerange' in parsed: 

546 timerange = parsed['timerange'] 

547 if 'antenna' in parsed: 

548 antenna = parsed['antenna'] 

549 

550 cmddict['applied'] = False 

551 cmddict['time'] = 0.0 

552 cmddict['interval'] = 0 

553 cmddict['level'] = 0 

554 cmddict['severity'] = 0 

555 cmddict['type'] = '' 

556 cmddict['mode'] = mode 

557 cmddict['timerange'] = timerange 

558 cmddict['antenna'] = antenna 

559 

560 flagdict[row] = cmddict 

561 row += 1 

562 

563 # Select the input cmds based on the requested reasons 

564 selected_dict = {} 

565 cmddict = {} 

566 row = 0 

567 if myreaslist.__len__() > 0: 

568 for key in flagdict: 

569 cmddict = flagdict[key]['command'] 

570 if 'reason' in cmddict: 

571 input_reason = cmddict['reason'] 

572 if selectReason(myreaslist, input_reason): 

573 selected_dict[row] = flagdict[key] 

574 row += 1 

575 

576 if selected_dict.__len__() == 0: 

577 raise Exception('No input lines matching requested reason(s)') 

578 else: 

579 selected_dict = flagdict 

580 

581 # Specific for the shadow mode 

582 if shadow: 

583 cmddict = {} 

584 for key in selected_dict: 

585 cmddict = selected_dict[key]['command'] 

586 if ( 

587 'addantenna' in cmddict 

588 and isinstance(cmddict['addantenna'], str) 

589 and cmddict['addantenna'] != '' 

590 ): 

591 # Create a dictionary and replace the parameter 

592 casalog.post('The addantenna parameter will be parsed as a dictionary', 'DEBUG1') 

593 antdict = readAntennaList(cmddict['addantenna']) 

594 selected_dict[key]['command']['addantenna'] = antdict 

595 

596 return selected_dict 

597 

598def selectReason(reasonlist, pattern): 

599 '''Return True if pattern is in the reasonlist''' 

600 

601 if isinstance(reasonlist, str): 

602 if pattern == reasonlist: 

603 return True 

604 elif isinstance(reasonlist, list): 

605 for r in reasonlist: 

606 if pattern == r: 

607 return True 

608 

609 return False 

610 

611def parseSelectionPars(cmddict): 

612 '''Return only the selection parameters into a dictionary: 

613 cmddict --> one row-dictionary with parameters 

614 The correlation parameter will not be considered. 

615 ''' 

616 

617 # Only these parameters will be included in dictionary 

618 # correlation is handled by the agents 

619 parlist = ['observation','array','feed','scan','field','spw', 

620 'timerange','uvrange','intent','antenna'] 

621 

622 return {par: cmddict[par] for par in parlist if par in cmddict} 

623 

624def parseUnion(vis, flagdict): 

625 '''Get a dictionary of a union of all selection parameters from a dictionary 

626 of flag commands. 

627 vis --> MS 

628 flagdict --> dictionary of parameters and values (par=val) such as the one 

629 returned by parseDictionary() 

630  

631 Returns a new dictionary with the union of the selection parameters. 

632 ''' 

633 # Union dictionary to return 

634 dictpars = {} 

635 

636 # Strings for each parameter 

637 scans = '' 

638 fields = '' 

639 ants = '' 

640 times = '' 

641 corrs = '' 

642 ints = '' 

643 feeds = '' 

644 arrays = '' 

645 uvs = '' 

646 spws = '' 

647 obs = '' 

648 

649 # Counters for each parameter 

650 si = 0 # count the number of lines with scan 

651 fi = 0 # count the number of lines with field 

652 ai = 0 # count the number of lines with antenna 

653 ti = 0 # count the number of lines with timerange 

654 coi = 0 # count the number of lines with correlation 

655 ii = 0 # count the number of lines with intent 

656 fei = 0 # count the number of lines with feed 

657 ari = 0 # count the number of lines with array 

658 ui = 0 # count the number of lines with uvrange 

659 pi = 0 # count the number of lines with spw 

660 oi = 0 # count the number of lines with observation 

661 

662 for k in flagdict.keys(): 

663 # Each key is a dictionary of one flag command 

664 cmddict = flagdict[k]['command'] 

665 

666 for xkey,xval in cmddict.items(): 

667 # Check which parameter 

668 if xkey == "scan": 

669 scans += xval + ',' 

670 si += 1 

671 elif xkey == "field": 

672 fields += xval + ',' 

673 fi += 1 

674 elif xkey == "antenna": 

675 ants += xval + ';' 

676 ai += 1 

677 elif xkey == "timerange": 

678 times += xval + ',' 

679 ti += 1 

680 elif xkey == "correlation": 

681 corrs += xval + ',' 

682 coi += 1 

683 elif xkey == "intent": 

684 ints += xval + ',' 

685 ii += 1 

686 elif xkey == "feed": 

687 feeds += xval + ',' 

688 fei += 1 

689 elif xkey == "array": 

690 arrays += xval + ',' 

691 ari += 1 

692 elif xkey == "uvrange": 

693 uvs += xval + ',' 

694 ui += 1 

695 elif xkey == "spw": 

696 spws += xval + ',' 

697 pi += 1 

698 elif xkey == "observation": 

699 obs += xval + ',' 

700 oi += 1 

701 

702 # Dictionary with number of occurrences of each parameter 

703 npars = { 

704 'field':fi, 

705 'scan':si, 

706 'antenna':ai, 

707 'spw':pi, 

708 'timerange':ti, 

709 'correlation':coi, 

710 'intent':ii, 

711 'feed':fei, 

712 'array':ari, 

713 'uvrange':ui, 

714 'observation':oi 

715 } 

716 

717 # Strip out the extra comma at the end 

718 scans = scans.rstrip(',') 

719 fields = fields.rstrip(',') 

720 ants = ants.rstrip(';') 

721 times = times.rstrip(',') 

722 corrs = corrs.rstrip(',') 

723 ints = ints.rstrip(',') 

724 feeds = feeds.rstrip(',') 

725 arrays = arrays.rstrip(',') 

726 uvs = uvs.rstrip(',') 

727 spws = spws.rstrip(',') 

728 obs = obs.rstrip(',') 

729 

730 dictpars['scan'] = scans 

731 dictpars['field'] = fields 

732 # Antennas are handled better within the framework. 

733 dictpars['antenna'] = '' 

734 # Times are handled better within the framework. 

735 dictpars['timerange'] = '' 

736 # Correlations should be handled only by the agents 

737 dictpars['correlation'] = '' 

738 # CAS-4682, do not create union for intents 

739 dictpars['intent'] = '' 

740 dictpars['feed'] = feeds 

741 dictpars['array'] = arrays 

742 dictpars['uvrange'] = uvs 

743 dictpars['spw'] = spws 

744 dictpars['observation'] = obs 

745 

746 # Compress the selection list to reduce MSSelection parsing time. 

747 # 'field','spw','antenna' strings in dicpars will be modified in-place. 

748 compressSelectionList(vis,dictpars); 

749 

750 # The number of keys in flagdict is the number of flag commands 

751 ncmds = flagdict.__len__() 

752 

753 # Make the union. Only leave non-empty parameters in dictionary 

754 for k,v in npars.items(): 

755 if v < ncmds: 

756 dictpars.pop(k) 

757 

758 return dictpars 

759 

760def _merge_timerange(commands): 

761 ''' merge manual commands that only differ in timerange and agentname 

762 this speeds up manual flagging using large lists 

763 

764 cmd -> list of flagging commands 

765 

766 returns list of commands with unique key 

767 ''' 

768 merged = dict() 

769 lunique = [] 

770 for cmd in commands: 

771 try: 

772 # only merge manual commands with timerange 

773 if (cmd.get('mode') != 'manual') or ('timerange' not in cmd): 

774 raise ValueError 

775 # create sorted list of command keys excluding agentname which 

776 # changes for each manual flag 

777 compound_key = sorted(x for x in cmd.keys() if x not in ('timerange', 'agentname')) 

778 # create compound key of all command keys and their values (e.g. antenna:1) 

779 compound = tuple((x, cmd[x]) for x in compound_key) 

780 

781 # skip invalid timeranges so they don't remove the whole agent group 

782 if '~' in cmd['timerange']: 

783 t0,t1 = cmd['timerange'].split('~', 1) 

784 startTime = qalocal.totime(t0)['value'] 

785 endTime = qalocal.totime(t1)['value'] 

786 if endTime <= startTime: 

787 raise ValueError 

788 

789 # merge timerange duplicate compound keys 

790 try: 

791 merged[compound]['timerange'] += ',' + cmd['timerange'] 

792 except KeyError: 

793 merged[compound] = copy.deepcopy(cmd) 

794 except: 

795 # add merged keys to non-mergeable keys, also on errors like 

796 # non-hashable keys 

797 lunique.extend(merged.values()) 

798 # append non mergeable key 

799 lunique.append(copy.deepcopy(cmd)) 

800 # reset merge to preserve ordering of manual and other flags 

801 # e.g. summary,manual,manual,manual,summary,manual 

802 # to summary,merged-manual,summary,manual 

803 merged = dict() 

804 

805 # add remaining merged keys to non-mergeable keys 

806 lunique.extend(merged.values()) 

807 return lunique 

808 

809def parseAgents(aflocal, flagdict, myrows, apply, writeflags, display=''): 

810 ''' Setup the parameters of each agent and call the agentflagger tool 

811  

812 aflocal --> local instance of the agentflagger tool (assumes the MS is open) 

813 flagdict --> a dictionary of the flag commands to parse to tool 

814 myrows --> selected rows to apply/unapply flags 

815 apply --> a boolean to control whether to apply or unapply the flags 

816 writeflags --> used by mode=rflag only 

817 display --> used by mode='rflag' only 

818  

819 The original flagdict dictionary is not modified ''' 

820 

821 

822 if flagdict.__len__() <= 0: 

823 casalog.post('There are no flag cmds in list', 'SEVERE') 

824 return False 

825 

826 # Do not modify original dictionary 

827 myflagcmd = copy.deepcopy(flagdict) 

828 commands = [] 

829 

830 # Setup the agent for each input line and merge timeranges to one command 

831 for row in myflagcmd.keys(): 

832 cmd = OrderedDict() 

833 cmd = myflagcmd[row]['command'] 

834 

835 if 'reason' in cmd: 

836 cmd.pop('reason') 

837 

838 if ( 

839 'mode' not in cmd 

840 or cmd['mode'] == '' 

841 or cmd['mode'] == 'manualflag' 

842 ): 

843 cmd['mode'] = 'manual' 

844 

845 elif cmd['mode'] == 'rflag': 

846 cmd['writeflags'] = writeflags 

847 cmd['display'] = display 

848 

849 # Read ntime 

850 readNtime(cmd) 

851 

852 # Cast the correct type of some parameters 

853 evalParams(cmd) 

854 

855 # Add the apply/unapply parameter to dictionary 

856 cmd['apply'] = apply 

857 

858 # Hold the name of the agent and the cmd row number 

859 mode = cmd['mode'] 

860 agent_name = mode.capitalize()+'_'+str(row) 

861 

862 cmd['agentname'] = agent_name 

863 

864 # Remove the data selection parameters if there is only one agent for performance reasons. 

865 # Explanation: if only one agent exists and the data selection parameters are parsed to it, 

866 # it will have to go through the entire MS and check if each data selection given to the agent 

867 # matches what the user asked in the selected data. 

868 

869 # Only correlation, antenna and timerange will go to the agent 

870 # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too 

871 if myflagcmd.__len__() == 1: 

872 sellist=['scan','field','intent','feed','array','uvrange','observation'] 

873 for k in sellist: 

874 if k in cmd: 

875 cmd.pop(k) 

876 

877 casalog.post('Parsing parameters of mode %s in row %s'%(mode,row), 'DEBUG') 

878 casalog.post('%s'%cmd,'DEBUG') 

879 

880 commands.append(cmd) 

881 

882 merged = _merge_timerange(commands) 

883 

884 if len(myflagcmd) != len(merged): 

885 casalog.post('Reduced %d timerange flags into %d compound flags' % 

886 (len(myflagcmd), len(merged))) 

887 

888 for cmd in merged: 

889 # Parse the dictionary of parameters to the tool 

890 if (not aflocal.parseagentparameters(cmd)): 

891 casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,row), 'WARN') 

892 continue 

893# return True 

894 return myflagcmd 

895 

896# TO BE VERIFIED. May not be needed 

897def evalParams(params): 

898 '''Give correct types to non-string parameters 

899 The types are defined in the XML file of the task flagdata 

900 Do not repeat any parameter''' 

901 

902 # manual parameter 

903# if 'autocorr' in params: 

904# params['autocorr'] = eval(params['autocorr'].capitalize()) 

905 

906 # quack parameters 

907# if 'quackmode' in params and not params['quackmode'] in ['beg' 

908# , 'endb', 'end', 'tail']: 

909# raise Exception, \ 

910# "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \ 

911# % params['quackmode'] 

912# if 'quackinterval' in params: 

913# params['quackinterval'] = float(params['quackinterval'])  

914# if 'quackincrement'in params: 

915# if type(params['quackincrement']) == str: 

916# params['quackincrement'] = eval(params['quackincrement'].capitalize()) 

917 

918 # clip parameters 

919 if 'clipminmax' in params: 

920 val1 = params['clipminmax'][0] 

921 val2 = params['clipminmax'][1] 

922 params['clipminmax'] = [float(val1), float(val2)] 

923# if 'clipoutside' in params: 

924# if type(params['clipoutside']) == str: 

925# params['clipoutside'] = eval(params['clipoutside'].capitalize()) 

926# else: 

927# params['clipoutside'] = params['clipoutside'] 

928# if 'channelavg' in params: 

929# params['channelavg'] = eval(params['channelavg'].capitalize()) 

930# if 'clipzeros' in params: 

931# params['clipzeros'] = eval(params['clipzeros'].capitalize()) 

932 

933 

934 # shadow parameter 

935# if 'tolerance' in params: 

936# params['tolerance'] = float(params['tolerance']) 

937#  

938# # elevation parameters 

939# if 'lowerlimit' in params: 

940# params['lowerlimit'] = float(params['lowerlimit'])  

941# if 'upperlimit' in params: 

942# params['upperlimit'] = float(params['upperlimit']) 

943 

944 # extend parameters 

945# if 'extendpols' in params: 

946# params['extendpols'] = eval(params['extendpols'].capitalize()) 

947# if 'growtime' in params: 

948# params['growtime'] = float(params['growtime']) 

949# if 'growfreq' in params: 

950# params['growfreq'] = float(params['growfreq']) 

951# if 'growaround' in params: 

952# params['growaround'] = eval(params['growaround'].capitalize()) 

953# if 'flagneartime' in params: 

954# params['flagneartime'] = eval(params['flagneartime'].capitalize()) 

955# if 'flagnearfreq' in params: 

956# params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize()) 

957 

958 # tfcrop parameters 

959# if 'combinescans' in params: 

960# params['combinescans'] = eval(params['combinescans'].capitalize())  

961# if 'timecutoff' in params: 

962# params['timecutoff'] = float(params['timecutoff'])  

963# if 'freqcutoff' in params: 

964# params['freqcutoff'] = float(params['freqcutoff'])  

965# if 'maxnpieces' in params: 

966# params['maxnpieces'] = int(params['maxnpieces'])  

967# if 'halfwin' in params: 

968# params['halfwin'] = int(params['halfwin']) 

969# if 'extendflags' in params: 

970# params['extendflags'] = eval(params['extendflags'].capitalize())  

971 

972 # rflag parameters 

973# if 'winsize' in params: 

974# params['winsize'] = int(params['winsize']); 

975 

976 # This is only necessary when timedev/freqdev are strings 

977 if 'timedev' in params: 

978 timepar = params['timedev'] 

979 if isinstance(timepar, list): 

980 return 

981 try: 

982 timepar = eval(timepar) 

983 except Exception: 

984 timepar = readRFlagThresholdFile(params['timedev'],'timedev'); 

985 params['timedev'] = timepar 

986 if 'freqdev' in params: 

987 freqpar = params['freqdev'] 

988 if isinstance(freqpar, list): 

989 return 

990 try: 

991 freqpar = eval(freqpar) 

992 except Exception: 

993 freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev'); 

994 params['freqdev'] = freqpar 

995# if 'timedevscale' in params: 

996# params['timedevscale'] = float(params['timedevscale']); 

997# if 'freqdevscale' in params: 

998# params['freqdevscale'] = float(params['freqdevscale']); 

999# if 'spectralmin' in params: 

1000# params['spectralmin'] = float(params['spectralmin']); 

1001# if 'spectralmax' in params: 

1002# params['spectralmax'] = float(params['spectralmax']); 

1003 

1004def writeFlagCommands(msfile, flagdict, applied, add_reason, outfile, append=True): 

1005 ''' 

1006 Writes the flag commands to an ASCII file or to the FLAG_CMD table 

1007  

1008 msfile --> MS name to save to FLAG_CMD table or empty. 

1009 flagdict --> dictionary of commands read from inputfile 

1010 applied --> True or False to write to APPLIED column 

1011 add_reason --> reason to add to output (replace input reason, if any) 

1012 outfile --> if not empty, save to it 

1013 append --> if True it will append to outfile, otherwise it removes outfile 

1014 before saving to it. 

1015  

1016 ''' 

1017 

1018 try: 

1019 import pylab as pl 

1020 except ImportError as exc: 

1021 raise ImportError('Failed to load pylab, required in writeFlagCommands: ', exc) 

1022 

1023 reason2add = False 

1024 # Replace blanks from reason and cmdreason with underscores 

1025 # and warn the user. 

1026 # TODO: I think this is not necessary for new parser!!! Check it 

1027 if add_reason != '': 

1028 reason2add = True 

1029# casalog.post('Replaced blanks with underscores in cmdreason', 'WARN') 

1030 

1031 # Append to a file  

1032 if outfile != '': 

1033 if append: 

1034# casalog.post('Will append commands to the file '+outfile)  

1035 ffout = open(outfile, 'a') 

1036 else: 

1037# casalog.post('Will save commands to the file '+outfile)  

1038 os.system('rm -f '+outfile) 

1039 ffout = open(outfile, 'w') 

1040 

1041 try: 

1042 for key in flagdict.keys(): 

1043 cmdline = "" 

1044 cmddict = flagdict[key]['command'] 

1045 reason = flagdict[key]['reason'] 

1046 

1047 # Add new reason or replace old one with new reason 

1048 if reason2add: 

1049 reason = add_reason 

1050 

1051 if reason != '': 

1052 cmddict['reason'] = reason 

1053 

1054 for k,v in cmddict.items(): 

1055 cmdstr = "" 

1056 if isinstance(v, str): 

1057 # Add quotes to string values 

1058 if v.count("'") > 0: 

1059 v = v.strip("'") 

1060 if v.count('"') > 0: 

1061 v = v.strip("'") 

1062 cmdstr = "'"+v+"'" 

1063 cmdline = cmdline + k + '=' + str(cmdstr) + ' ' 

1064 else: 

1065 cmdline = cmdline + k + '=' + str(v) + ' ' 

1066 

1067 # Save to output file 

1068 ffout.write('%s\n' %cmdline.rstrip()) 

1069 ffout.close() 

1070 except: 

1071 raise Exception('Error writing/appending lines to file ' \ 

1072 + outfile) 

1073 ffout.close() 

1074 return False 

1075 else: 

1076 # Append new commands to existing FLAG_CMD table 

1077# casalog.post('Saving commands to the FLAG_CMD table')  

1078 cmdlist = [] 

1079 reasonlist = [] 

1080 for key in flagdict.keys(): 

1081 cmdline = "" 

1082 reason = "" 

1083 cmddict = flagdict[key]['command'] 

1084 # Do not save reason in the COMMAND column 

1085 if 'reason' in cmddict: 

1086 cmddict.pop('reason') 

1087 # Summary cmds should not go to FLAG_CMD 

1088 if 'mode' in cmddict and cmddict['mode'] == 'summary': 

1089 casalog.post("Commands with mode='summary' are not allowed in the FLAG_CMD table", 'WARN') 

1090 continue 

1091 # Add to REASON column the user input reason if requested 

1092 reason = flagdict[key]['reason'] 

1093 if reason2add: 

1094 reason = add_reason 

1095 

1096 for k,v in cmddict.items(): 

1097 cmdstr = "" 

1098 if isinstance(v, str): 

1099 # Add quotes to string values 

1100 if v.count("'") > 0: 

1101 v = v.strip("'") 

1102 if v.count('"') > 0: 

1103 v = v.strip("'") 

1104 cmdstr = "'"+v+"'" 

1105 cmdline = cmdline + k + '=' + str(cmdstr) + ' ' 

1106 

1107 else: 

1108 cmdline = cmdline + k + '=' + str(v) + ' ' 

1109 

1110 cmdline = cmdline.rstrip() 

1111 cmdlist.append(cmdline) 

1112 reasonlist.append(reason) 

1113 

1114 # Number of cmds to add to FLAG_CMD 

1115 Nadd = cmdlist.__len__() 

1116 

1117 # Fill in other columns 

1118 appliedlist = [applied for i in range(Nadd)] 

1119 timelist = [0.0]*Nadd 

1120 otherlist = [0]*Nadd 

1121 typelist = ['']*Nadd 

1122 

1123 mstable = msfile + '/FLAG_CMD' 

1124 try: 

1125 tblocal.open(mstable, nomodify=False) 

1126 except: 

1127 raise Exception('Error opening FLAG_CMD table '+mstable) 

1128 

1129 nrows = int(tblocal.nrows()) 

1130 casalog.post('There are ' + str(nrows)+ ' rows already in FLAG_CMD') 

1131 

1132 # Add blank rows first 

1133 tblocal.addrows(Nadd) 

1134 

1135 # Fill in the columns 

1136 tblocal.putcol('APPLIED', numpy.array(appliedlist), startrow=nrows,nrow=Nadd) 

1137 tblocal.putcol('COMMAND', numpy.array(cmdlist), startrow=nrows, nrow=Nadd) 

1138 tblocal.putcol('INTERVAL', numpy.array(timelist), startrow=nrows,nrow=Nadd) 

1139 tblocal.putcol('LEVEL', numpy.array(otherlist), startrow=nrows,nrow=Nadd) 

1140 tblocal.putcol('REASON', numpy.array(reasonlist),startrow=nrows, nrow=Nadd) 

1141 tblocal.putcol('SEVERITY', numpy.array(otherlist), startrow=nrows,nrow=Nadd) 

1142 tblocal.putcol('TIME', numpy.array(timelist), startrow=nrows,nrow=Nadd) 

1143 tblocal.putcol('TYPE', numpy.array(typelist), startrow=nrows,nrow=Nadd) 

1144 

1145 newrows = int(tblocal.nrows()) 

1146 newrows -= nrows 

1147 if newrows == 0: 

1148 casalog.post('Did not save any rows to FLAG_CMD') 

1149 else: 

1150 casalog.post('Saved ' + str(newrows) + ' rows to FLAG_CMD of %s'%msfile) 

1151 

1152 tblocal.close() 

1153 

1154 return True 

1155 

1156def parseRFlagOutputFromSummary(mode,summary_stats_list, flagcmd): 

1157 """ 

1158 Function to pull out 'rflag' output from the long dictionary, and  

1159 (1) write the output files with thresholds. If the filename is specified, use it. 

1160 If filename is not specified, make one up. 

1161 (2) modify entries in 'cmdline' so that it is ready for savepars.  

1162 This is to ensure that 'savepars' saves the contents of the threshold-files 

1163 and not just the file-names. It has to save it in the form that flagdata  

1164 accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user 

1165 need not keep track of threshold text files if they use 'savepars' with action='apply'. 

1166 """ 

1167 # The careful checks are especially for MMS which can produce empty selections/results from 

1168 # individual sub-MSs. 

1169 if summary_stats_list and type(summary_stats_list) is dict and 'nreport' in summary_stats_list: 

1170 nreps = summary_stats_list['nreport'] 

1171 for rep in range(0,nreps): 

1172 repname = 'report'+str(rep) 

1173 if summary_stats_list[repname]['type'] == "rflag": 

1174 # Pull out the rflag threshold dictionary. This has a 'name' in it. 

1175 rflag_thresholds = summary_stats_list[repname] 

1176 # Get the rflag id, to later construct a 'name' from to match the above. 

1177 rflagid = 0 

1178 if mode=='list': 

1179 rflagid = int( rflag_thresholds['name'].replace('Rflag_','') ) 

1180 # Go through the flagcmd list, to find the 'rflags'..... 

1181 for key in flagcmd.keys(): 

1182 # cmdline is a dictionary with flag commands 

1183 cmdline = flagcmd[key]['command'] 

1184 if 'mode' in cmdline and cmdline['mode'] == 'rflag': 

1185 # Check for match between input flagcmd and output threshold, via the rflag id 

1186 if(key == rflagid): 

1187 # If timedev,freqdev are missing from cmdline, add empty ones. 

1188 if not 'timedev' in cmdline: # aah. don't confuse it with timedevscale 

1189 cmdline['timedev'] = [] 

1190 if not 'freqdev' in cmdline: 

1191 cmdline['freqdev'] = [] 

1192 # Write RFlag thresholds to these file names.  

1193 newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, cmdline['timedev'], cmdline['freqdev'], rflagid) 

1194 ## VERIFY for parser 

1195 # Modify the flagcmd string, so that savepars sees the contents of the file 

1196 if( isinstance(cmdline['timedev'], list)): 

1197 cmdline['timedev'] = newtimedev 

1198 if( isinstance(cmdline['freqdev'],list)): 

1199 cmdline['freqdev'] = newfreqdev 

1200 

1201 # Remove writeflags from the cmd to prevent it from going into savepars 

1202 cmdline.pop('writeflags') 

1203 

1204def save_rflag_consolidated_files(mode, action, cons_dict, opts_dict, inpfile): 

1205 """ 

1206 Utility for RFlag when running in parallel/MMS. Does parseRFlagOutputFromSummary() on 

1207 the consolidated (on the client process) - only when needed for rflag mode (or list 

1208 mode when the list of commands contains some RFlag commands). 

1209 

1210 :param mode: mode parameter as given to the from casatasks.flagdata import flagdata task 

1211 :param action: action parameter as given to the flagdata task 

1212 :param cons_dict: consolidated dictionary of reports from (RFlag) flagdata commands 

1213 :param opts_dict: dictionary with the options needed for parseRFlagOutputFromSummary() 

1214 (timedev, freqdev and writeflags, from the task input parameters) 

1215 :param inpfile: inpfile parameter as given to the flagdata task 

1216 """ 

1217 if mode in ['rflag', 'list'] and action != 'apply': 

1218 import pprint 

1219 casalog.post('Saving RFlag return dictionary: {0}'. 

1220 format(pprint.pformat(cons_dict)), 'INFO') 

1221 # Prepare the list of commands 

1222 if mode == 'list': 

1223 cmd_list = get_flag_cmd_list(inpfile) 

1224 else: 

1225 # need only the relevant fields to save the output files in 

1226 # parseRFlagOutputFromSummary() 

1227 cmd_list = {0: {'command': 

1228 dict({'mode': 'rflag'}, **opts_dict)}} 

1229 parseRFlagOutputFromSummary(mode, cons_dict, cmd_list) 

1230 

1231# Not used at the moment!!!!!!!!!!! 

1232def readFlagCmdTable(msfile, rows=[], applied=True, reason='any'): 

1233 '''Read flag commands from rows of the FLAG_CMD table of msfile 

1234 If useapplied=False then include only rows with APPLIED=False 

1235 If myreason is anything other than 'any', then select on that''' 

1236 

1237 # 

1238 # Return flagcmd structure: 

1239 # 

1240 # The flagcmd key is the integer row number from FLAG_CMD 

1241 # 

1242 # Dictionary structure: 

1243 # key : 'id' (string) 

1244 # 'mode' (string) flag mode '','clip','shadow','quack' 

1245 # 'antenna' (string) 

1246 # 'timerange' (string) 

1247 # 'reason' (string) 

1248 # 'time' (float) in mjd seconds 

1249 # 'interval' (float) in mjd seconds 

1250 # 'cmd' (string) string (for COMMAND col in FLAG_CMD) 

1251 # 'type' (string) 'FLAG' / 'UNFLAG' 

1252 # 'applied' (bool) set to True here on read-in 

1253 # 'level' (int) set to 0 here on read-in 

1254 # 'severity' (int) set to 0 here on read-in 

1255 

1256 # Open and read columns from FLAG_CMD 

1257 mstable = msfile + '/FLAG_CMD' 

1258 

1259 # Note, tblocal.getcol doesn't allow random row access, read all 

1260 

1261 try: 

1262 tblocal.open(mstable) 

1263 f_time = tblocal.getcol('TIME') 

1264 f_interval = tblocal.getcol('INTERVAL') 

1265 f_type = tblocal.getcol('TYPE') 

1266 f_reas = tblocal.getcol('REASON') 

1267 f_level = tblocal.getcol('LEVEL') 

1268 f_severity = tblocal.getcol('SEVERITY') 

1269 f_applied = tblocal.getcol('APPLIED') 

1270 f_cmd = tblocal.getcol('COMMAND') 

1271 tblocal.close() 

1272 except: 

1273 casalog.post('Error reading table ' + mstable, 'ERROR') 

1274 raise Exception 

1275 

1276 nrows = f_time.__len__() 

1277 

1278 if nrows == 0: 

1279 casalog.post('FLAG_CMD table in %s is empty, no flags extracted' 

1280 % msfile, 'ERROR') 

1281 return {} 

1282 

1283 rowlist = [] 

1284 

1285 # Select by rows 

1286 if rows.__len__() > 0: 

1287 rowlist = rows 

1288 else: 

1289 rowlist = range(nrows) 

1290 

1291 # Select by APPLIED 

1292 if not applied: 

1293 rowl = [] 

1294 for i in rowlist: 

1295 if not f_applied[i]: 

1296 rowl.append(i) 

1297 rowlist = rowl 

1298 

1299 # Select by REASON 

1300 myreaslist = [] 

1301 if type(reason) == str: 

1302 if reason != 'any': 

1303 myreaslist.append(reason) 

1304 elif type(reason) == list: 

1305 myreaslist = reason 

1306 else: 

1307 casalog.post('Cannot read reason; it contains unknown variable types' 

1308 , 'ERROR') 

1309 return 

1310 

1311 if myreaslist.__len__() > 0: 

1312 rowl = [] 

1313 for i in rowlist: 

1314 if myreaslist.count(f_reas[i]) > 0: 

1315 rowl.append(i) 

1316 rowlist = rowl 

1317 

1318 Ncmds = 0 

1319 myflagcmd = {} 

1320 

1321 # Define the parser 

1322 myParser = Parser(' ','=') 

1323 for i in rowlist: 

1324 parsed = {} 

1325 flagd = {} 

1326 cmd = f_cmd[i] 

1327 if cmd == '': 

1328 continue 

1329 

1330 # Parse to a dictionary 

1331 parsed = myParser.parse2Dictionary(cmd) 

1332 

1333 # This is the original row number in FLAG_CMD 

1334 flagd['ROW'] = i 

1335 flagd['COMMAND'] = parsed 

1336 flagd['REASON'] = f_reas[i] 

1337 flagd['APPLIED'] = f_applied[i] 

1338 flagd['TIME'] = f_time[i] 

1339 flagd['INTERVAL'] = f_interval[i] 

1340 flagd['TYPE'] = f_type[i] 

1341 flagd['LEVEL'] = f_level[i] 

1342 flagd['SEVERITY'] = f_severity[i] 

1343 

1344 myflagcmd[Ncmds] = flagd 

1345 Ncmds += 1 

1346 

1347 casalog.post('Read ' + str(Ncmds) 

1348 + ' rows from FLAG_CMD table in ' + msfile) 

1349 

1350 return myflagcmd 

1351 

1352def plotFlagCommands(myflags,plotname,t1sdata,t2sdata,): 

1353 # 

1354 # Function to plot flagging dictionary 

1355 # Adapted from J.Marvil 

1356 # Updated STM v4.1 2011-11-02 to handle ALMA flags 

1357 # Updated STM v4.2 2012-02-16 trim flag times to data times 

1358 # Updated STM v4.2 2012-04-10 bug fix in trim flag times to data times 

1359 # Updated STM v4.2 2012-04-10 messages to logger 

1360 

1361 try: 

1362 import pylab as pl 

1363 except ImportError as exc: 

1364 raise ImportError('Failed to load pylab, required in plotFlagCommands: ', exc) 

1365 

1366 # list of supported colors (in order) 

1367 colorlist = [ 

1368 'red', 

1369 'blue', 

1370 'green', 

1371 'black', 

1372 'cyan', 

1373 'magenta', 

1374 'yellow', 

1375 'orange', 

1376 ] 

1377 ncolors = colorlist.__len__() 

1378 

1379 # get list of flag keys 

1380 keylist = myflags.keys() 

1381 

1382 # get lists of antennas and reasons 

1383 # make plotting dictionary 

1384 myants = [] 

1385 myreas = [] 

1386 plotflag = {} 

1387 ipf = 0 

1388 for key in keylist: 

1389 antstr = '' 

1390 reastr = '' 

1391 timstr = '' 

1392 if 'antenna' in myflags[key]: 

1393 antstr = myflags[key]['antenna'] 

1394 if 'reason' in myflags[key]: 

1395 reastr = myflags[key]['reason'] 

1396 if 'timerange' in myflags[key]: 

1397 timstr = myflags[key]['timerange'] 

1398 if antstr != '': 

1399 # flags that have antenna specified 

1400 antlist = antstr.split(',') 

1401 nantlist = antlist.__len__() 

1402 else: 

1403 # Special 

1404 antlist = ['All'] 

1405 nantlist = 1 

1406 # 

1407 realist = reastr.split(',') 

1408 nrealist = realist.__len__() 

1409 # 

1410 timlist = timstr.split(',') 

1411 ntimlist = timlist.__len__() 

1412 # 

1413 # Break these into nants x ntimes flags 

1414 # Trick is assigning multiple reasons 

1415 # Normal cases: 

1416 # A. One reason, single/multiple antennas x times 

1417 # B. Multiple reasons=times, single/multiple antenna(s) 

1418 # C. Single time, multiple antennas/reasons 

1419 # D. Multiple reasons, no way to correspond with times 

1420 # 

1421 timmin = 1.0E11 

1422 timmax = 0.0 

1423 if nrealist == 1: 

1424 # simplest case, single reason 

1425 reas = realist[0] 

1426 if reas == '': 

1427 reas = 'Unknown' 

1428 if myreas.count(reas) == 0: 

1429 myreas.append(reas) 

1430 for ia in range(nantlist): 

1431 ant = antlist[ia] 

1432 if myants.count(ant) == 0: 

1433 myants.append(ant) 

1434 for it in range(ntimlist): 

1435 times = timlist[it] 

1436 plotflag[ipf] = {} 

1437 plotflag[ipf]['antenna'] = ant 

1438 plotflag[ipf]['reason'] = reas 

1439 plotflag[ipf]['timerange'] = times 

1440 plotflag[ipf]['show'] = True 

1441 ipf += 1 

1442 elif nrealist == ntimlist: 

1443 # corresponding reasons and times 

1444 for ia in range(nantlist): 

1445 ant = antlist[ia] 

1446 if myants.count(ant) == 0: 

1447 myants.append(ant) 

1448 for it in range(ntimlist): 

1449 times = timlist[it] 

1450 reas = realist[it] 

1451 if reas == '': 

1452 reas = 'Unknown' 

1453 if myreas.count(reas) == 0: 

1454 myreas.append(reas) 

1455 plotflag[ipf] = {} 

1456 plotflag[ipf]['antenna'] = ant 

1457 plotflag[ipf]['reason'] = reas 

1458 plotflag[ipf]['timerange'] = times 

1459 plotflag[ipf]['show'] = True 

1460 ipf += 1 

1461 else: 

1462 # no correspondence between multiple reasons and ants/times 

1463 # assign reason 'Miscellaneous' 

1464 reas = 'Miscellaneous' 

1465 if myreas.count(reas) == 0: 

1466 myreas.append(reas) 

1467 for ia in range(nantlist): 

1468 ant = antlist[ia] 

1469 if myants.count(ant) == 0: 

1470 myants.append(ant) 

1471 for it in range(ntimlist): 

1472 times = timlist[it] 

1473 plotflag[ipf] = {} 

1474 plotflag[ipf]['antenna'] = ant 

1475 plotflag[ipf]['reason'] = reas 

1476 plotflag[ipf]['timerange'] = times 

1477 plotflag[ipf]['show'] = True 

1478 ipf += 1 

1479 

1480 myants.sort() 

1481 nants = myants.__len__() 

1482 nreas = myreas.__len__() 

1483 casalog.post('Found ' + str(nreas) + ' reasons to plot for ' 

1484 + str(nants) + ' antennas') 

1485 npf = ipf 

1486 casalog.post('Found ' + str(npf) + ' total flag ranges to plot') 

1487 

1488 # sort out times 

1489 for ipf in range(npf): 

1490 times = plotflag[ipf]['timerange'] 

1491 if times != '': 

1492 if times.count('~') > 0: 

1493 t1 = times[:times.find('~')] 

1494 t2 = times[times.find('~') + 1:] 

1495 else: 

1496 t1 = times 

1497 t2 = t1 

1498 (t1s, t2s) = (qalocal.convert(t1, 's')['value'], qalocal.convert(t2, 

1499 's')['value']) 

1500 plotflag[ipf]['t1s'] = t1s 

1501 plotflag[ipf]['t2s'] = t2s 

1502 if t1s < timmin: 

1503 timmin = t1s 

1504 if t2s > timmax: 

1505 timmax = t2s 

1506 # min,max times 

1507 q1 = qalocal.quantity(timmin, 's') 

1508 time1 = qalocal.time(q1, form='ymd', prec=9)[0] 

1509 q2 = qalocal.quantity(timmax, 's') 

1510 time2 = qalocal.time(q2, form='ymd', prec=9)[0] 

1511 casalog.post('Found flag times from ' + time1 + ' to ' + time2) 

1512 

1513 # sort out blank times 

1514 for ipf in range(npf): 

1515 times = plotflag[ipf]['timerange'] 

1516 if times == '': 

1517 if t2sdata >= t1sdata > 0: 

1518 plotflag[ipf]['t1s'] = t1sdata 

1519 plotflag[ipf]['t2s'] = t2sdata 

1520 else: 

1521 plotflag[ipf]['t1s'] = timmin 

1522 plotflag[ipf]['t2s'] = timmax 

1523 

1524 # if flag times are beyond range of data, trim them 

1525 # Added STM 2012-02-16, fixed STM 2012-04-10 

1526 ndropped = 0 

1527 if t2sdata >= t1sdata > 0 and (timmin < t1sdata or timmax 

1528 > t2sdata): 

1529 # min,max data times 

1530 q1 = qalocal.quantity(t1sdata, 's') 

1531 tdata1 = qalocal.time(q1, form='ymd', prec=9)[0] 

1532 q2 = qalocal.quantity(t2sdata, 's') 

1533 tdata2 = qalocal.time(q2, form='ymd', prec=9)[0] 

1534 casalog.post('WARNING: Trimming flag times to data limits ' 

1535 + tdata1 + ' to ' + tdata2) 

1536 

1537 for ipf in range(npf): 

1538 t1s = plotflag[ipf]['t1s'] 

1539 t2s = plotflag[ipf]['t2s'] 

1540 if t1s < t1sdata: 

1541 if t2s >= t1sdata: 

1542 # truncate to t1sdata 

1543 plotflag[ipf]['t1s'] = t1sdata 

1544 else: 

1545 # entirely outside data range, do not plot 

1546 plotflag[ipf]['show'] = False 

1547 ndropped += 1 

1548 

1549 if t2s > t2sdata: 

1550 if t1s <= t2sdata: 

1551 # truncate to t2sdata 

1552 plotflag[ipf]['t2s'] = t2sdata 

1553 else: 

1554 # entirely outside data range, do not plot 

1555 plotflag[ipf]['show'] = False 

1556 ndropped += 1 

1557 

1558 if ndropped > 0: 

1559 casalog.post('WARNING: Trimming dropped ' + str(ndropped) 

1560 + ' flags entirely') 

1561 

1562 # make reason dictionary with mapping of colors and offsets (-0.3 to 0.3) 

1563 readict = {} 

1564 reakeys = [] 

1565 if nreas > ncolors: 

1566 for i in range(nreas): 

1567 reas = myreas[i] 

1568 readict[reas] = {} 

1569 if i < ncolors - 1: 

1570 colr = colorlist[i] 

1571 readict[reas]['color'] = colr 

1572 readict[reas]['index'] = i 

1573 offs = 0.3 - float(i) * 0.6 / float(ncolors - 1) 

1574 readict[reas]['offset'] = offs 

1575 reakeys.append(reas) 

1576 else: 

1577 colr = colorlist[ncolors - 1] 

1578 readict[reas]['color'] = colr 

1579 readict[reas]['index'] = ncolors - 1 

1580 readict[reas]['offset'] = -0.3 

1581 reakeys.append('Other') 

1582 readict['Other'] = {} 

1583 readict['Other']['color'] = colorlist[ncolors - 1] 

1584 readict['Other']['index'] = ncolors - 1 

1585 readict['Other']['offset'] = -0.3 

1586 else: 

1587 for i in range(nreas): 

1588 reas = myreas[i] 

1589 reakeys.append(reas) 

1590 colr = colorlist[i] 

1591 offs = 0.3 - float(i) * 0.6 / float(ncolors - 1) 

1592 readict[reas] = {} 

1593 readict[reas]['color'] = colr 

1594 readict[reas]['index'] = i 

1595 readict[reas]['offset'] = offs 

1596 nlegend = reakeys.__len__() 

1597 casalog.post('Will plot ' + str(nlegend) + ' reasons in legend') 

1598 

1599 antind = 0 

1600 if plotname == '': 

1601 pl.ion() 

1602 else: 

1603 pl.ioff() 

1604 

1605 f1 = pl.figure() 

1606 ax1 = f1.add_axes([.15, .1, .75, .85]) 

1607# ax1.set_ylabel('antenna') 

1608# ax1.set_xlabel('time') 

1609 # badflags=[] 

1610 nplotted = 0 

1611 for thisant in myants: 

1612 antind += 1 

1613 for ipf in range(npf): 

1614 if plotflag[ipf]['show'] and plotflag[ipf]['antenna'] \ 

1615 == thisant: 

1616 # plot this flag 

1617 thisReason = plotflag[ipf]['reason'] 

1618 thisColor = readict[thisReason]['color'] 

1619 thisOffset = readict[thisReason]['offset'] 

1620 t1s = plotflag[ipf]['t1s'] 

1621 t2s = plotflag[ipf]['t2s'] 

1622 

1623 ax1.plot([t1s, t2s], [antind + thisOffset, antind 

1624 + thisOffset], color=thisColor, lw=2, alpha=.7) 

1625 nplotted += 1 

1626 

1627 casalog.post('Plotted ' + str(nplotted) + ' flags') 

1628 

1629 myXlim = ax1.get_xlim() 

1630 myXrange = myXlim[1] - myXlim[0] 

1631 # Pad the time axis? 

1632 PadTime = 0.050000000000000003 

1633 if PadTime > 0: 

1634 xPad = PadTime * myXrange 

1635 x0 = myXlim[0] - xPad 

1636 x1 = myXlim[1] + xPad 

1637 ax1.set_xlim(x0, x1) 

1638 myXrange = x1 - x0 

1639 else: 

1640 # casalog.post(' Rescaled x axis') 

1641 x0 = myXlim[0] 

1642 x1 = myXlim[1] 

1643 

1644 legendFontSize = 12 

1645 myYupper = nants + nlegend + 1.5 

1646 # place legend text 

1647 i = nants 

1648 x = x0 + 0.050000000000000003 * myXrange 

1649 for reas in reakeys: 

1650 i += 1 

1651 colr = readict[reas]['color'] 

1652 ax1.text(x, i, reas, color=colr, size=legendFontSize) 

1653 ax1.set_ylim([0, myYupper]) 

1654 

1655 ax1.set_yticks(range(1, len(myants) + 1)) 

1656 ax1.set_yticklabels(myants) 

1657 # casalog.post(' Relabled y axis') 

1658 

1659 nxticks = 3 

1660 ax1.set_xticks(pl.linspace(myXlim[0], myXlim[1], nxticks)) 

1661 

1662 mytime = [] 

1663 myTimestr = [] 

1664 for itim in range(nxticks): 

1665 time = myXlim[0] + (myXlim[1] - myXlim[0]) * float(itim) \ 

1666 / float(nxticks - 1) 

1667 mytime.append(time) 

1668 q1 = qalocal.quantity(time, 's') 

1669 time1 = qalocal.time(q1, form='ymd', prec=9)[0] 

1670 if itim > 0: 

1671 time1s = time1[11:] 

1672 else: 

1673 time1s = time1 

1674 myTimestr.append(time1s) 

1675 

1676 ax1.set_xticklabels(myTimestr) 

1677 # casalog.post(myTimestr) 

1678 if plotname == '': 

1679 pl.draw() 

1680 else: 

1681 pl.savefig(plotname, dpi=150) 

1682 return 

1683 

1684def evaluateParameters(pardict): 

1685 '''Evaluate the correct types in a string with parameters 

1686  

1687 Keyword arguments: 

1688 pardict -- a dictionary with flag commands as returned by parseDictionary() 

1689  

1690 It evaluates each value to its correct Python type. 

1691 ''' 

1692 

1693 cmddict = OrderedDict() 

1694 

1695 for key,val in pardict.items(): 

1696 newval = None 

1697 

1698 if val.startswith('['): 

1699 newval = ast.literal_eval(val) 

1700 elif val.startswith('{'): 

1701 newval = eval(val) 

1702 elif val in ['True', 'False']: 

1703 if eval(val) in [True, False]: 

1704 newval = eval(val) 

1705 if newval is None: 

1706 try: 

1707 # Try int 

1708 int(val) 

1709 newval = int(val) 

1710 except: 

1711 # Try float 

1712 try: 

1713 float(val) 

1714 newval = float(val) 

1715 except: 

1716 # Try string 

1717 if val.count("'") > 0: 

1718 val = val.strip("'") 

1719 elif val.count('"') > 0: 

1720 val = val.strip('"') 

1721 

1722 # CAS-6553 cannot have only one quote. remove it 

1723 if val.count("'") == 1: 

1724 val = val.replace("'", '') 

1725 elif val.count('"') == 1: 

1726 val = val.replace('"', '') 

1727 

1728 newval = str(val).strip() 

1729 

1730 cmddict[key] = newval 

1731 

1732 return cmddict 

1733 

1734def evaluateFlagParameters(pardict, pars): 

1735 """Check if a flagdata parameter dictionary is valid 

1736 

1737 Keyword arguments: 

1738 pardict -- a dictionary with flag commands as returned by parseDictionary() 

1739 pars -- dictionary of flagdata's parameters (locals() inside flagdata task) 

1740 

1741 It checks if the parameter exists in the reference tuple. 

1742 It checks if the type of the parameter value matches the type in the reference 

1743 for that parameter. 

1744 It raises an exception if any parameter or type of value do not match. 

1745 

1746 """ 

1747 from casatasks import flagdata 

1748 

1749 # Make a deepcopy of flagdata parameters dictionary for modification 

1750 fpars = copy.deepcopy(pars) 

1751 

1752 # Get the defaults of each parameter  

1753 for par in fpars.keys(): 

1754 fpars[par] = get_task_arg_default(flagdata, par) 

1755 

1756 # Define the parameters that don't go in an input list in flagdata 

1757 removepars = ['vis','inpfile','flagbackup','tbuff','cmdreason','savepars','outfile', 

1758 'display','action'] 

1759 

1760 # Remove the parameters we don't need to evaluate 

1761 for par in removepars: 

1762 if par in fpars: 

1763 fpars.pop(par) 

1764 

1765 # Define the parameters that have variant type in flagdata 

1766 dup_pars = ['ntime','observation','addantenna','timedev','timedev','freqdev','freqdev','quackinterval'] 

1767 dup_types = [0.0,0,{},[],0.0,[],0.0,0] 

1768 

1769 # Create a tuple from flagdata's parameters 

1770 ftup = fpars.items() 

1771 

1772 # Create a list of the tuples 

1773 flist = list(ftup) 

1774 

1775 # Append the duplicated types 

1776 for d in range(len(dup_pars)): 

1777 # Create a tuple and append to list 

1778 dpar = (dup_pars[d],dup_types[d]) 

1779 flist.append(dpar) 

1780 

1781 

1782 # Create a tuple of the reference dictionary 

1783 reftup = tuple(flist) 

1784 

1785 # Reference parameters for flagdata task 

1786# reftup = (('field','string'),('spw','string'),('antenna','string'),('timerange','string'),('correlation','string'), 

1787# ('scan','string'),('intent','string'),('array','string'),('uvrange','string'), 

1788# ('observation','string'), 

1789# ('observation',0), 

1790# ('feed','string'), 

1791# ('mode','string'), 

1792# ('reason','string'), 

1793# # manual 

1794# ('autocorr', True), 

1795# # clip 

1796# ('datacolumn','string'), 

1797# ('clipminmax',[]), 

1798# ('clipoutside',True), 

1799# ('channelavg',False), 

1800# ('clipzeros',False), 

1801# # elevation 

1802# ('lowerlimit',0.0), 

1803# ('upperlimit',90.0), 

1804# # shadow 

1805# ('tolerance',0.0), 

1806# ('addantenna','string'), 

1807# ('addantenna',{}), 

1808# # quack 

1809# ('quackinterval',0.0), 

1810# ('quackinterval',0), 

1811# ('quackmode','string'), 

1812# ('quackincrement',False), 

1813# # tfcrop 

1814# ('ntime','string'), 

1815# ('ntime',0.0), 

1816# ('combinescans',False), 

1817# ('timecutoff',4.0), 

1818# ('freqcutoff',3.0), 

1819# ('timefit','string'), 

1820# ('freqfit','string'), 

1821# ('maxnpieces',7), 

1822# ('flagdimension','string'), 

1823# ('usewindowstats','string'), 

1824# ('halfwin',1), 

1825# ('extendflags',True), 

1826# # rflag 

1827# ('winsize',3), 

1828# ('timedev','string'), 

1829# ('timedev',[]), 

1830# ('timedev',0.0), 

1831# ('freqdev','string'), 

1832# ('freqdev',[]), 

1833# ('freqdev',0.0), 

1834# ('timedevscale',5.0), 

1835# ('freqdevscale',5.0), 

1836# ('spectralmax',1000000.0), 

1837# ('spectralmin',0.0), 

1838# # extend 

1839# ('extendpols',True), 

1840# ('growtime',50.0), 

1841# ('growfreq',50.0), 

1842# ('growaround',False), 

1843# ('flagneartime',False), 

1844# ('flagnearfreq',False), 

1845# # summary 

1846# ('minrel',0.0), 

1847# ('maxrel',1.0), 

1848# ('minabs',0), 

1849# ('maxabs',1), 

1850# ('spwchan',False), 

1851# ('spwcorr',False), 

1852# ('basecnt',False), 

1853# ('name','string')) 

1854 

1855 

1856 reference = defaultdict(list) 

1857 for k, v in reftup: 

1858 reference[k].append(v) 

1859 

1860 refkeys = reference.keys() 

1861 

1862 # Check the user dictionary against the reference 

1863 count = 0 

1864 for idx in pardict.keys(): 

1865 mydict = pardict[idx]['command'] 

1866 count += 1 

1867 for key,val in mydict.items(): 

1868 if key not in refkeys: 

1869 raise ValueError('Parameter \'%s\' in row=%s is not a valid flagdata parameter'%(key,idx)) 

1870 

1871 # reference[key] is always a list 

1872 refval = reference[key] 

1873 vtypes = "" 

1874 match = False 

1875 for tt in refval: 

1876 vtypes = vtypes + str(type(tt)) + ',' 

1877 if type(val) == type(tt): 

1878 # type matches 

1879 match = True 

1880 if not match: 

1881 raise(ValueError,'Parameter %s=%s in row=%s has wrong type.\nSupported types are: %s.'%(key,val,idx,vtypes.rstrip(','))) 

1882 

1883 casalog.post('Evaluated %s rows of dictionary'%count,'DEBUG1') 

1884 return True 

1885 

1886def evaluateNumpyType(elem): 

1887 '''Evaluate if an element is of numpy type. 

1888 Cast it to the corresponding Python type 

1889 and return the casted value''' 

1890 

1891 import numpy as np 

1892 

1893 val = None 

1894 

1895 if (isinstance(elem, np.int_) or isinstance(elem, np.int8) or 

1896 isinstance(elem, np.int16) or isinstance(elem, np.int32) or 

1897 isinstance(elem, np.int64)): 

1898 val = int(elem) 

1899 

1900 elif(isinstance(elem,np.float16) or 

1901 isinstance(elem,np.float32) or isinstance(elem,np.float64)): 

1902 val = float(elem) 

1903 

1904 elif (hasattr(np, "float")) and isinstance(elem,np.float): 

1905 val = float(elem) 

1906 # float128 is not available on all platforms 

1907 elif (hasattr(np, "float128")) and isinstance(elem,np.float128): 

1908 val = float(elem) 

1909 

1910 

1911 else: 

1912 # it is none of the above numpy types 

1913 val = elem 

1914 

1915 # return the casted element  

1916 return val 

1917 

1918def parseXML(sdmfile, mytbuff): 

1919 ''' 

1920# readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses 

1921# into returned dictionary as flag command strings 

1922# sdmfile (string) path to SDM containing Antenna.xml and Flag.xml 

1923# mytbuff (float) time interval (start and end) padding (seconds) 

1924# 

1925# Usage: myflags = readflagxml(sdmfile,tbuff) 

1926# 

1927# Dictionary structure: 

1928# fid : 'id' (string) 

1929# 'mode' (string) flag mode ('online') 

1930# 'antenna' (string) 

1931# 'timerange' (string) 

1932# 'reason' (string) 

1933# 'time' (float) in mjd seconds 

1934# 'interval' (float) in mjd seconds 

1935# 'command' (string) string (for COMMAND col in FLAG_CMD) 

1936# 'type' (string) 'FLAG' / 'UNFLAG' 

1937# 'applied' (bool) set to True here on read-in 

1938# 'level' (int) set to 0 here on read-in 

1939# 'severity' (int) set to 0 here on read-in 

1940# 

1941# Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA 

1942# Updated STM 2012-02-14 handle spectral window indices, names, IDs 

1943# Updated STM 2012-02-21 handle polarization types 

1944# 

1945# Mode to use for spectral window selection in commands: 

1946# spwmode = 0 none (flag all spw) 

1947# spwmode = 1 use name 

1948# spwmode = -1 use index (counting rows in SpectralWindow.xml) 

1949# 

1950# Mode to use for polarization selection in commands: 

1951# polmode = 0 none (flag all pols/corrs) 

1952# polmode = 1 use polarization type 

1953# 

1954# CURRENT DEFAULT: Use spw names, flag pols 

1955 ''' 

1956 

1957 spwmode = 1 

1958 polmode = 1 

1959 

1960# 

1961 try: 

1962 from xml.dom import minidom 

1963 except ImportError as exc: 

1964 raise ImportError('Failed to load xml.dom.minidom, required in parseXML: ', exc) 

1965 

1966 if type(mytbuff) != float: 

1967 raise RuntimeError('Found incorrect type for tbuff. type(mytbuff): {}'. 

1968 format(type(mytbuff))) 

1969 

1970 # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends) 

1971 flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK) 

1972 antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK) 

1973 spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK) 

1974 if not flagexist: 

1975 casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE') 

1976 exit(1) 

1977 if not antexist: 

1978 casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE' 

1979 ) 

1980 exit(1) 

1981 if not spwexist: 

1982 casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 

1983 'WARN') 

1984 

1985 # construct look-up dictionary of name vs. id from Antenna.xml 

1986 xmlants = minidom.parse(sdmfile + '/Antenna.xml') 

1987 antdict = {} 

1988 rowlist = xmlants.getElementsByTagName('row') 

1989 for rownode in rowlist: 

1990 rowname = rownode.getElementsByTagName('name') 

1991 ant = str(rowname[0].childNodes[0].nodeValue).strip() 

1992 rowid = rownode.getElementsByTagName('antennaId') 

1993 # CAS-4532: remove spaces between content and tags 

1994 antid = str(rowid[0].childNodes[0].nodeValue).strip() 

1995 antdict[antid] = ant 

1996 casalog.post('Found ' + str(rowlist.length) 

1997 + ' rows in Antenna.xml') 

1998 

1999 # construct look-up dictionary of name vs. id from SpectralWindow.xml 

2000 if spwexist: 

2001 xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml') 

2002 spwdict = {} 

2003 rowlist = xmlspws.getElementsByTagName('row') 

2004 ispw = 0 

2005 wvrnominal = False 

2006 for rownode in rowlist: 

2007 rowid = rownode.getElementsByTagName('spectralWindowId') 

2008 # CAS-4532: remove spaces between content and tags 

2009 spwid = str(rowid[0].childNodes[0].nodeValue).strip() 

2010 spwdict[spwid] = {} 

2011 spwdict[spwid]['index'] = ispw 

2012 if rownode.getElementsByTagName('name'): 

2013 rowname = rownode.getElementsByTagName('name') 

2014 spwname = str(rowname[0].childNodes[0].nodeValue).strip() 

2015 if spwname == 'WVR#NOMINAL': 

2016 wvrnominal = True 

2017 spwdict[spwid]['name'] = spwname 

2018 else: 

2019 spwmode = -1 

2020 

2021# rowid = rownode.getElementsByTagName('spectralWindowId') 

2022# spwid = str(rowid[0].childNodes[0].nodeValue) 

2023# spwdict[spwid] = {} 

2024# spwdict[spwid]['index'] = ispw 

2025 ispw += 1 

2026 casalog.post('Found ' + str(rowlist.length) 

2027 + ' rows in SpectralWindow.xml') 

2028 

2029 # report chosen spw and pol modes 

2030 if spwmode > 0: 

2031 casalog.post('Will construct spw flags using names') 

2032 elif spwmode < 0: 

2033 casalog.post('Will construct spw flags using table indices') 

2034 else: 

2035 casalog.post('') 

2036 # 

2037 if polmode == 0: 

2038 casalog.post('Will not set polarization dependent flags (flag all corrs)' 

2039 ) 

2040 else: 

2041 casalog.post('Will construct polarization flags using polarizationType' 

2042 ) 

2043 

2044 # now read Flag.xml into dictionary row by row 

2045 xmlflags = minidom.parse(sdmfile + '/Flag.xml') 

2046 flagdict = {} 

2047 rowlist = xmlflags.getElementsByTagName('row') 

2048 nrows = rowlist.length 

2049 newsdm = -1 

2050 newspw = -1 

2051 newpol = -1 

2052 for fid in range(nrows): 

2053 rownode = rowlist[fid] 

2054 rowfid = rownode.getElementsByTagName('flagId') 

2055 fidstr = str(rowfid[0].childNodes[0].nodeValue).strip() 

2056 flagdict[fid] = {} 

2057 flagdict[fid]['id'] = fidstr 

2058 rowid = rownode.getElementsByTagName('antennaId') 

2059 antid = rowid[0].childNodes[0].nodeValue 

2060 # check if there is a numAntenna specified (new format) 

2061 rownant = rownode.getElementsByTagName('numAntenna') 

2062 antname = '' 

2063 if rownant.__len__() > 0: 

2064 xid = antid.split() 

2065 nant = int(rownant[0].childNodes[0].nodeValue) 

2066 if newsdm < 0: 

2067 casalog.post('Found numAntenna=' + str(nant) 

2068 + ' must be a new style SDM') 

2069 newsdm = 1 

2070 # CAS-4698. Flag auto-correlations when there is 

2071 # only one antenna 

2072 if nant == 1: 

2073 aid = xid[2] 

2074 ana = antdict[aid] 

2075 if antname == '': 

2076 antname = ana+'&&*' 

2077 else: 

2078 antname += ',' + ana 

2079 

2080 elif nant > 1: 

2081 for ia in range(nant): 

2082 aid = xid[2 + ia] 

2083 ana = antdict[aid] 

2084 if antname == '': 

2085 antname = ana 

2086 else: 

2087 antname += ',' + ana 

2088 else: 

2089 # numAntenna = 0 means flag all antennas 

2090 antname = '' 

2091 else: 

2092 if newsdm < 0: 

2093 casalog.post('No numAntenna entry found, must be an old style SDM' 

2094 ) 

2095 newsdm = 0 

2096 nant = 1 

2097 aid = antid 

2098 ana = antdict[aid] 

2099 antname = ana 

2100 # start and end times in mjd ns 

2101 rowstart = rownode.getElementsByTagName('startTime') 

2102 start = int(rowstart[0].childNodes[0].nodeValue) 

2103 startmjds = float(start) * 1.0E-9 - mytbuff 

2104 t = qalocal.quantity(startmjds, 's') 

2105 starttime = qalocal.time(t, form='ymd', prec=9)[0] 

2106 rowend = rownode.getElementsByTagName('endTime') 

2107 end = int(rowend[0].childNodes[0].nodeValue) 

2108 endmjds = float(end) * 1.0E-9 + mytbuff 

2109 t = qalocal.quantity(endmjds, 's') 

2110 endtime = qalocal.time(t, form='ymd', prec=9)[0] 

2111 # time and interval for FLAG_CMD use 

2112 times = 0.5 * (startmjds + endmjds) 

2113 intervs = endmjds - startmjds 

2114 flagdict[fid]['time'] = times 

2115 flagdict[fid]['interval'] = intervs 

2116 # reasons 

2117 rowreason = rownode.getElementsByTagName('reason') 

2118 reas = str(rowreason[0].childNodes[0].nodeValue) 

2119 # Replace any white space with underscores 

2120 reason = reas.replace(' ','_') 

2121 # NEW SDM ADDITIONS 2011-11-01 

2122 rownspw = rownode.getElementsByTagName('numSpectralWindow') 

2123 spwstring = '' 

2124 if spwmode != 0 and rownspw.__len__() > 0: 

2125 nspw = int(rownspw[0].childNodes[0].nodeValue) 

2126 # has a new-style spw specification 

2127 if newspw < 0: 

2128 if not spwexist: 

2129 casalog.post('Cannot open ' + sdmfile 

2130 + '/SpectralWindow.xml', 'SEVERE') 

2131 exit(1) 

2132 casalog.post('Found numSpectralWindow=' + str(nspw) 

2133 + ' must be a new style SDM') 

2134 newspw = 1 

2135 if nspw > 0: 

2136 rowspwid = rownode.getElementsByTagName('spectralWindowId') 

2137 spwids = rowspwid[0].childNodes[0].nodeValue 

2138 xspw = spwids.split() 

2139 for isp in range(nspw): 

2140 spid = str(xspw[2 + isp]) 

2141 if spwmode > 0: 

2142 spstr =spwdict[spid]['name'] 

2143 # Skip the WVR#Antenna_* if WVR#NOMINAL is present 

2144 if spstr.__contains__('WVR#Antenna') and wvrnominal==True: 

2145 continue 

2146 if spwstring == '': 

2147 spwstring = '"'+spstr+'"' 

2148 else: 

2149 spwstring += ',' + '"'+spstr+'"' 

2150 else: 

2151 spstr = (spwdict[spid]['index']) 

2152 if spwstring == '': 

2153 spwstring = spstr 

2154 else: 

2155 spwstring += ',' + spstr 

2156 polstring = '' 

2157 rownpol = rownode.getElementsByTagName('numPolarizationType') 

2158 if polmode != 0 and rownpol.__len__() > 0: 

2159 npol = int(rownpol[0].childNodes[0].nodeValue) 

2160 # has a new-style pol specification 

2161 if newpol < 0: 

2162 casalog.post('Found numPolarizationType=' + str(npol) 

2163 + ' must be a new style SDM') 

2164 newpol = 1 

2165 if npol > 0: 

2166 rowpolid = \ 

2167 rownode.getElementsByTagName('polarizationType') 

2168 polids = rowpolid[0].childNodes[0].nodeValue 

2169 xpol = polids.split() 

2170 for ipol in range(npol): 

2171 polid = str(xpol[2 + ipol]) 

2172 if polstring == '': 

2173 polstring = polid 

2174 else: 

2175 polstring += ',' + polid 

2176 # 

2177 # Construct antenna name and timerange and reason strings 

2178 flagdict[fid]['antenna'] = antname 

2179 timestr = starttime + '~' + endtime 

2180 flagdict[fid]['timerange'] = timestr 

2181 flagdict[fid]['reason'] = reason 

2182 # Construct command strings (per input flag) 

2183 cmddict = OrderedDict() 

2184# cmd = "antenna='" + antname + "' timerange='" + timestr + "'" 

2185# cmddict['mode'] = 'manual' 

2186 cmddict['antenna'] = antname 

2187 cmddict['timerange'] = timestr 

2188 if spwstring != '': 

2189# cmd += " spw='" + spwstring + "'" 

2190 flagdict[fid]['spw'] = spwstring 

2191 cmddict['spw'] = spwstring 

2192# if polstring != '': 

2193# cmd += " poln='" + polstring + "'" 

2194# flagdict[fid]['poln'] = polstring 

2195 if polstring != '': 

2196 # Write the poln translation in correlation 

2197 if polstring.count('R')>0: 

2198 if polstring.count('L')>0: 

2199 corr = 'RR,RL,LR,LL' 

2200 else: 

2201 corr = 'RR,RL,LR' 

2202 elif polstring.count('L')>0: 

2203 corr = 'LL,LR,RL' 

2204 elif polstring.count('X')>0: 

2205 if polstring.count('Y')>0: 

2206 corr = 'XX,XY,YX,YY' 

2207 else: 

2208 corr = 'XX,XY,YX' 

2209 elif polstring.count('Y')>0: 

2210 corr = 'YY,YX,XY' 

2211 

2212# cmd += " correlation='" + corr + "'" 

2213 cmddict['correlation'] = corr 

2214# flagdict[fid]['poln'] = polstring 

2215# flagdict[fid]['command'] = cmd 

2216 flagdict[fid]['command'] = cmddict 

2217 # 

2218 flagdict[fid]['type'] = 'FLAG' 

2219 flagdict[fid]['applied'] = False 

2220 flagdict[fid]['level'] = 0 

2221 flagdict[fid]['severity'] = 0 

2222 flagdict[fid]['mode'] = 'online' 

2223 

2224 flags = {} 

2225 if rowlist.length > 0: 

2226 flags = flagdict 

2227 casalog.post('Found ' + str(rowlist.length) 

2228 + ' flags in Flag.xml') 

2229 else: 

2230 casalog.post('No valid flags found in Flag.xml') 

2231 

2232 # return the dictionary for later use 

2233 return flags 

2234 

2235 

2236# END of NEW PARSER FUNCTIONS  

2237############################################################################################## 

2238############################################################################################## 

2239 

2240 

2241# (CAS-4119): Use absolute paths for input files to ensure that the engines find them 

2242def addAbsPath(input_file): 

2243 '''Read in the lines from an input file 

2244 input_file --> file in disk with a list of strings per line 

2245  

2246 Re-writes the file changing relative file names to absolute file names 

2247 ''' 

2248 

2249 output_file = input_file + ".tmp" 

2250 if (type(input_file) == str) & os.path.exists(input_file): 

2251 try: 

2252 input_file_id = open(input_file, 'r') 

2253 except: 

2254 raise Exception('Error opening file ' + input_file) 

2255 

2256 try: 

2257 output_file_id = open(output_file, 'w') 

2258 except: 

2259 output_file_id.close() 

2260 raise Exception('Error opening tmp file ' + output_file) 

2261 else: 

2262 raise Exception('ASCII file not found - please verify the name') 

2263 

2264 # 

2265 # Parse file 

2266 try: 

2267 for line in input_file_id: 

2268 cmd = line.rstrip() 

2269 cmd_pars = cmd.split(" ") 

2270 for par in cmd_pars: 

2271 if ((par.count("inpfile") > 0) or 

2272 (par.count("outfile") > 0) or 

2273 (par.count("addantenna") > 0) or 

2274 (par.count("timedev") > 0) or 

2275 (par.count("freqdev") > 0)): 

2276 par = par.split("=",1) 

2277 file_local = par[1] 

2278 if file_local.count("'")>0: 

2279 file_local = file_local.split("'") 

2280 file_local = file_local[1] 

2281 elif file_local.count('"')>0: 

2282 file_local = file_local.split('"') 

2283 file_local = file_local[1] 

2284 else: 

2285 continue 

2286 file_abs = os.path.abspath(file_local) 

2287 cmd = cmd.replace(file_local,file_abs) 

2288 

2289 output_file_id.write(cmd+"\n") 

2290 

2291 except: 

2292 input_file_id.close() 

2293 output_file_id.close() 

2294 raise Exception('Error reading lines from file ' \ 

2295 + input_file) 

2296 

2297 input_file_id.close() 

2298 output_file_id.close() 

2299# os.rename(output_file, input_file) 

2300 

2301 # Return the temporary file 

2302 return output_file 

2303 

2304def makeDict(cmdlist, myreason='any'): 

2305 '''Make a dictionary compatible with a FLAG_CMD table structure 

2306 and select by reason if any is given 

2307  

2308 cmdlist --> list of parameters to go into the COMMAND column 

2309 myreason --> reason to select from 

2310  

2311 Returns a dictionary with the the selected rows with the following  

2312 structure and default values: 

2313  

2314 flagd['row'] = 

2315 flagd['antenna'] = 

2316 flagd['timerange'] = 

2317 flagd['applied'] = False 

2318 flagd['command'] = '' 

2319 flagd['interval'] = 0.0 

2320 flagd['level'] = 0 

2321 flagd['reason'] = '' 

2322 flagd['severity'] = 0 

2323 flagd['time'] = 0.0 

2324 flagd['type'] = 'FLAG' 

2325  

2326 ''' 

2327 

2328 if cmdlist.__len__() == 0: 

2329 raise Exception('Empty list of commands') 

2330 

2331 # select by these reasons 

2332 myreaslist = [] 

2333 if type(myreason) == str: 

2334 if myreason != 'any': 

2335 myreaslist.append(myreason) 

2336 elif type(myreason) == list: 

2337 myreaslist = myreason 

2338 else: 

2339 casalog.post('Cannot read reason; it contains unknown variable types', 

2340 'ERROR') 

2341 return 

2342 

2343 if debug: 

2344 casalog.post("reason in selection: {}".format(myreaslist)) 

2345 

2346 # List of reasons in input file 

2347 nrows = cmdlist.__len__() 

2348 

2349 # If any selection was requested 

2350 reaslist = [] 

2351 for i in range(nrows): 

2352 command = cmdlist[i] 

2353 if command.__contains__('reason'): 

2354 reaslist.append(getReason(command)) 

2355 else: 

2356 # so that cmdlist and reaslist have the same sizes 

2357 reaslist.append('NoReasonToMatch') 

2358 

2359 if debug: 

2360 casalog.post("reason in input: {}".format(reaslist)) 

2361 

2362 

2363 # Defaults for columns 

2364 applied = False 

2365 interval = 0.0 

2366 level = 0 

2367 severity = 0 

2368 coltime = 0.0 

2369 coltype = 'FLAG' 

2370 antenna = '' 

2371 timerange = '' 

2372 

2373 myflagd = {} 

2374 ncmds = 0 

2375 rowlist = range(nrows) 

2376 

2377 # If select by reason is requested 

2378 if myreaslist.__len__() > 0: 

2379 rowl = [] 

2380 

2381 # If selection matches what is in input line 

2382 for j in range(reaslist.__len__()): 

2383 if myreaslist.count(reaslist[j]) > 0: 

2384 rowl.append(j) 

2385 rowlist = rowl 

2386 

2387 try: 

2388 for i in rowlist: 

2389 flagd = {} 

2390 reason = '' 

2391 

2392 command = cmdlist[i] 

2393 if reaslist[i] != 'NoReasonToMatch': 

2394 reason = reaslist[i] 

2395 

2396 # Skip comment lines 

2397 if command.startswith('#'): 

2398 continue 

2399 if command == '': 

2400 casalog.post('Ignoring empty command line', 'WARN') 

2401 continue 

2402 if command.__contains__('summary'): 

2403 casalog.post('Mode summary is not allowed in list operation', 'WARN') 

2404 continue 

2405 

2406 # If shadow, remove the addantenna dictionary 

2407 if command.__contains__('shadow') and command.__contains__('addantenna'): 

2408 i0 = command.rfind('addantenna') 

2409 if command[i0+11] == '{': 

2410 # It is a dictionary. Remove it from line 

2411 i1 = command.rfind('}') 

2412 antpar = command[i0+11:i1+1] 

2413 temp = command[i0:i1+1] 

2414 newcmd = command.replace(temp,'') 

2415 antpardict = convertStringToDict(antpar) 

2416 flagd['addantenna'] = antpardict 

2417 command = newcmd 

2418 

2419 # CAS-5180: Get antenna and timerange to use when action='plot' 

2420 keyvlist = command.split() 

2421 if keyvlist.__len__() > 0: 

2422 for keyv in keyvlist: 

2423 # Take care of '=' signs in values 

2424 try: 

2425 (xkey, val) = keyv.split('=',1) 

2426 except: 

2427 casalog.post('Error: not key=value pair for ' + keyv, 'ERROR') 

2428 break 

2429 xval = val 

2430 if xval.count("'") > 0: 

2431 xval = xval.strip("'") 

2432 if xval.count('"') > 0: 

2433 xval = xval.strip('"') 

2434 if xkey == 'mode': 

2435 flagd['mode'] = xval 

2436 elif xkey == 'timerange': 

2437 timerange = xval 

2438 elif xkey == 'antenna': 

2439 antenna = xval 

2440 

2441 flagd['row'] = str(i) 

2442 flagd['applied'] = applied 

2443 flagd['reason'] = reason 

2444 flagd['timerange'] = timerange 

2445 flagd['antenna'] = antenna 

2446 

2447 # Remove reason from command line  

2448 command = command.rstrip() 

2449 newline = purgeParameter(command, 'reason') 

2450 

2451 # Remove any empty parameter (CAS-4015) 

2452 command = purgeEmptyPars(newline) 

2453 command = command.strip() 

2454 

2455 flagd['command'] = command 

2456 flagd['interval'] = interval 

2457 flagd['level'] = level 

2458 flagd['severity'] = severity 

2459 flagd['time'] = coltime 

2460 flagd['type'] = coltype 

2461 # Insert into main dictionary 

2462 myflagd[ncmds] = flagd 

2463 ncmds += 1 

2464 

2465 except: 

2466 raise Exception('Cannot create dictionary') 

2467 

2468 casalog.post(':makeDict::myflagd=%s'%myflagd,'DEBUG') 

2469 

2470 return myflagd 

2471 

2472def readXML(sdmfile, mytbuff): 

2473 ''' 

2474# readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses 

2475# into returned dictionary as flag command strings 

2476# sdmfile (string) path to SDM containing Antenna.xml and Flag.xml 

2477# mytbuff (float) time interval (start and end) padding (seconds) 

2478# 

2479# Usage: myflags = readflagxml(sdmfile,tbuff) 

2480# 

2481# Dictionary structure: 

2482# fid : 'id' (string) 

2483# 'mode' (string) flag mode ('online') 

2484# 'antenna' (string) 

2485# 'timerange' (string) 

2486# 'reason' (string) 

2487# 'time' (float) in mjd seconds 

2488# 'interval' (float) in mjd seconds 

2489# 'command' (string) string (for COMMAND col in FLAG_CMD) 

2490# 'type' (string) 'FLAG' / 'UNFLAG' 

2491# 'applied' (bool) set to True here on read-in 

2492# 'level' (int) set to 0 here on read-in 

2493# 'severity' (int) set to 0 here on read-in 

2494# 

2495# Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA 

2496# Updated STM 2012-02-14 handle spectral window indices, names, IDs 

2497# Updated STM 2012-02-21 handle polarization types 

2498# 

2499# Mode to use for spectral window selection in commands: 

2500# spwmode = 0 none (flag all spw) 

2501# spwmode = 1 use name 

2502# spwmode = -1 use index (counting rows in SpectralWindow.xml) 

2503# 

2504# Mode to use for polarization selection in commands: 

2505# polmode = 0 none (flag all pols/corrs) 

2506# polmode = 1 use polarization type 

2507# 

2508# CURRENT DEFAULT: Use spw names, flag pols 

2509 ''' 

2510 

2511 spwmode = 1 

2512 polmode = 1 

2513 

2514# 

2515 try: 

2516 from xml.dom import minidom 

2517 except ImportError as exc: 

2518 raise ImportError('failed to load xml.dom.minidom, required in readXML: ', exc) 

2519 

2520 if type(mytbuff) != float: 

2521 raise RuntimeError('Found incorrect type for tbuff. type(mytbuff): {}'. 

2522 format(type(mytbuff))) 

2523 

2524 # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends) 

2525 flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK) 

2526 antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK) 

2527 spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK) 

2528 if not flagexist: 

2529 casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE') 

2530 exit(1) 

2531 if not antexist: 

2532 casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE' 

2533 ) 

2534 exit(1) 

2535 if not spwexist: 

2536 casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 

2537 'WARN') 

2538 

2539 # construct look-up dictionary of name vs. id from Antenna.xml 

2540 xmlants = minidom.parse(sdmfile + '/Antenna.xml') 

2541 antdict = {} 

2542 rowlist = xmlants.getElementsByTagName('row') 

2543 for rownode in rowlist: 

2544 rowname = rownode.getElementsByTagName('name') 

2545 ant = str(rowname[0].childNodes[0].nodeValue) 

2546 rowid = rownode.getElementsByTagName('antennaId') 

2547 # CAS-4532: remove spaces between content and tags 

2548 antid = str(rowid[0].childNodes[0].nodeValue).strip() 

2549 antdict[antid] = ant 

2550 casalog.post('Found ' + str(rowlist.length) 

2551 + ' antennas in Antenna.xml') 

2552 

2553 # construct look-up dictionary of name vs. id from SpectralWindow.xml 

2554 if spwexist: 

2555 xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml') 

2556 spwdict = {} 

2557 rowlist = xmlspws.getElementsByTagName('row') 

2558 ispw = 0 

2559 for rownode in rowlist: 

2560 rowid = rownode.getElementsByTagName('spectralWindowId') 

2561 # CAS-4532: remove spaces between content and tags 

2562 spwid = str(rowid[0].childNodes[0].nodeValue).strip() 

2563 spwdict[spwid] = {} 

2564 spwdict[spwid]['index'] = ispw 

2565 # SMC: 6/3/2012 ALMA SDM does not have name 

2566 if rownode.getElementsByTagName('name'): 

2567 rowname = rownode.getElementsByTagName('name') 

2568 spw = str(rowname[0].childNodes[0].nodeValue) 

2569 spwdict[spwid]['name'] = spw 

2570 else: 

2571 spwmode = -1 

2572 

2573# rowid = rownode.getElementsByTagName('spectralWindowId') 

2574# spwid = str(rowid[0].childNodes[0].nodeValue) 

2575# spwdict[spwid] = {} 

2576# spwdict[spwid]['index'] = ispw 

2577 ispw += 1 

2578 casalog.post('Found ' + str(rowlist.length) 

2579 + ' spw in SpectralWindow.xml') 

2580 

2581 # report chosen spw and pol modes 

2582 if spwmode > 0: 

2583 casalog.post('Will construct spw flags using names') 

2584 elif spwmode < 0: 

2585 casalog.post('Will construct spw flags using table indices') 

2586 else: 

2587 casalog.post('') 

2588 # 

2589 if polmode == 0: 

2590 casalog.post('Will not set polarization dependent flags (flag all corrs)' 

2591 ) 

2592 else: 

2593 casalog.post('Will construct polarization flags using polarizationType' 

2594 ) 

2595 

2596 # now read Flag.xml into dictionary row by row 

2597 xmlflags = minidom.parse(sdmfile + '/Flag.xml') 

2598 flagdict = {} 

2599 rowlist = xmlflags.getElementsByTagName('row') 

2600 nrows = rowlist.length 

2601 newsdm = -1 

2602 newspw = -1 

2603 newpol = -1 

2604 for fid in range(nrows): 

2605 rownode = rowlist[fid] 

2606 rowfid = rownode.getElementsByTagName('flagId') 

2607 fidstr = str(rowfid[0].childNodes[0].nodeValue) 

2608 flagdict[fid] = {} 

2609 flagdict[fid]['id'] = fidstr 

2610 rowid = rownode.getElementsByTagName('antennaId') 

2611 antid = rowid[0].childNodes[0].nodeValue 

2612 # check if there is a numAntenna specified (new format) 

2613 rownant = rownode.getElementsByTagName('numAntenna') 

2614 antname = '' 

2615 if rownant.__len__() > 0: 

2616 xid = antid.split() 

2617 nant = int(rownant[0].childNodes[0].nodeValue) 

2618 if newsdm < 0: 

2619 casalog.post('Found numAntenna=' + str(nant) 

2620 + ' must be a new style SDM') 

2621 newsdm = 1 

2622 # CAS-4698. Flag auto-correlations when there is 

2623 # only one antenna 

2624 if nant == 1: 

2625 aid = xid[2] 

2626 ana = antdict[aid] 

2627 if antname == '': 

2628 antname = ana+'&&*' 

2629 else: 

2630 antname += ',' + ana 

2631 

2632 elif nant > 1: 

2633 for ia in range(nant): 

2634 aid = xid[2 + ia] 

2635 ana = antdict[aid] 

2636 if antname == '': 

2637 antname = ana 

2638 else: 

2639 antname += ',' + ana 

2640 else: 

2641 # numAntenna = 0 means flag all antennas 

2642 antname = '' 

2643 else: 

2644 if newsdm < 0: 

2645 casalog.post('No numAntenna entry found, must be an old style SDM' 

2646 ) 

2647 newsdm = 0 

2648 nant = 1 

2649 aid = antid 

2650 ana = antdict[aid] 

2651 antname = ana 

2652 # start and end times in mjd ns 

2653 rowstart = rownode.getElementsByTagName('startTime') 

2654 start = int(rowstart[0].childNodes[0].nodeValue) 

2655 startmjds = float(start) * 1.0E-9 - mytbuff 

2656 t = qalocal.quantity(startmjds, 's') 

2657 starttime = qalocal.time(t, form='ymd', prec=9)[0] 

2658 rowend = rownode.getElementsByTagName('endTime') 

2659 end = int(rowend[0].childNodes[0].nodeValue) 

2660 endmjds = float(end) * 1.0E-9 + mytbuff 

2661 t = qalocal.quantity(endmjds, 's') 

2662 endtime = qalocal.time(t, form='ymd', prec=9)[0] 

2663 # time and interval for FLAG_CMD use 

2664 times = 0.5 * (startmjds + endmjds) 

2665 intervs = endmjds - startmjds 

2666 flagdict[fid]['time'] = times 

2667 flagdict[fid]['interval'] = intervs 

2668 # reasons 

2669 rowreason = rownode.getElementsByTagName('reason') 

2670 reas = str(rowreason[0].childNodes[0].nodeValue) 

2671 # Replace any white space with underscores 

2672 reason = reas.replace(' ','_') 

2673 # NEW SDM ADDITIONS 2011-11-01 

2674 rownspw = rownode.getElementsByTagName('numSpectralWindow') 

2675 spwstring = '' 

2676 if spwmode != 0 and rownspw.__len__() > 0: 

2677 nspw = int(rownspw[0].childNodes[0].nodeValue) 

2678 # has a new-style spw specification 

2679 if newspw < 0: 

2680 if not spwexist: 

2681 casalog.post('Cannot open ' + sdmfile 

2682 + '/SpectralWindow.xml', 'SEVERE') 

2683 exit(1) 

2684 casalog.post('Found SpectralWindow=' + str(nspw) 

2685 + ' must be a new style SDM') 

2686 newspw = 1 

2687 if nspw > 0: 

2688 rowspwid = \ 

2689 rownode.getElementsByTagName('spectralWindowId') 

2690 spwids = rowspwid[0].childNodes[0].nodeValue 

2691 xspw = spwids.split() 

2692 for isp in range(nspw): 

2693 spid = str(xspw[2 + isp]) 

2694 if spwmode > 0: 

2695 spstr = spwdict[spid]['name'] 

2696 else: 

2697 spstr = str(spwdict[spid]['index']) 

2698 if spwstring == '': 

2699 spwstring = spstr 

2700 else: 

2701 spwstring += ',' + spstr 

2702 polstring = '' 

2703 rownpol = rownode.getElementsByTagName('numPolarizationType') 

2704 if polmode != 0 and rownpol.__len__() > 0: 

2705 npol = int(rownpol[0].childNodes[0].nodeValue) 

2706 # has a new-style pol specification 

2707 if newpol < 0: 

2708 casalog.post('Found numPolarizationType=' + str(npol) 

2709 + ' must be a new style SDM') 

2710 newpol = 1 

2711 if npol > 0: 

2712 rowpolid = \ 

2713 rownode.getElementsByTagName('polarizationType') 

2714 polids = rowpolid[0].childNodes[0].nodeValue 

2715 xpol = polids.split() 

2716 for ipol in range(npol): 

2717 polid = str(xpol[2 + ipol]) 

2718 if polstring == '': 

2719 polstring = polid 

2720 else: 

2721 polstring += ',' + polid 

2722 # 

2723 # Construct antenna name and timerange and reason strings 

2724 flagdict[fid]['antenna'] = antname 

2725 timestr = starttime + '~' + endtime 

2726 flagdict[fid]['timerange'] = timestr 

2727 flagdict[fid]['reason'] = reason 

2728 # Construct command strings (per input flag) 

2729 cmd = "antenna='" + antname + "' timerange='" + timestr + "'" 

2730 if spwstring != '': 

2731 cmd += " spw='" + spwstring + "'" 

2732 flagdict[fid]['spw'] = spwstring 

2733# if polstring != '': 

2734# cmd += " poln='" + polstring + "'" 

2735# flagdict[fid]['poln'] = polstring 

2736 if polstring != '': 

2737 # Write the poln translation in correlation 

2738 if polstring.count('R')>0: 

2739 if polstring.count('L')>0: 

2740 corr = 'RR,RL,LR,LL' 

2741 else: 

2742 corr = 'RR,RL,LR' 

2743 elif polstring.count('L')>0: 

2744 corr = 'LL,LR,RL' 

2745 elif polstring.count('X')>0: 

2746 if polstring.count('Y')>0: 

2747 corr = 'XX,XY,YX,YY' 

2748 else: 

2749 corr = 'XX,XY,YX' 

2750 elif polstring.count('Y')>0: 

2751 corr = 'YY,YX,XY' 

2752 

2753 cmd += " correlation='" + corr + "'" 

2754# flagdict[fid]['poln'] = polstring 

2755 flagdict[fid]['command'] = cmd 

2756 # 

2757 flagdict[fid]['type'] = 'FLAG' 

2758 flagdict[fid]['applied'] = False 

2759 flagdict[fid]['level'] = 0 

2760 flagdict[fid]['severity'] = 0 

2761 flagdict[fid]['mode'] = 'online' 

2762 

2763 flags = {} 

2764 if rowlist.length > 0: 

2765 flags = flagdict 

2766 casalog.post('Found ' + str(rowlist.length) 

2767 + ' flags in Flag.xml') 

2768 else: 

2769 casalog.post('No valid flags found in Flag.xml') 

2770 

2771 # return the dictionary for later use 

2772 return flags 

2773 

2774def getUnion(vis, cmddict): 

2775 '''Get a dictionary of a union of all selection parameters from a list of lines: 

2776 vis --> MS 

2777 cmddict --> dictionary of parameters and values (par=val) such as the one 

2778 returned by makeDict() 

2779 ''' 

2780 

2781 # Dictionary of parameters to return 

2782 dicpars = { 

2783 'field':'', 

2784 'scan':'', 

2785 'antenna':'', 

2786 'spw':'', 

2787 'timerange':'', 

2788 'correlation':'', 

2789 'intent':'', 

2790 'feed':'', 

2791 'array':'', 

2792 'uvrange':'', 

2793 'observation':'' 

2794 } 

2795 

2796 # Strings for each parameter 

2797 scans = '' 

2798 fields = '' 

2799 ants = '' 

2800 times = '' 

2801 corrs = '' 

2802 ints = '' 

2803 feeds = '' 

2804 arrays = '' 

2805 uvs = '' 

2806 spws = '' 

2807 obs = '' 

2808 

2809 nrows = cmddict.keys().__len__() 

2810# nrows = cmdlist.__len__() 

2811 

2812# for i in range(nrows): 

2813 for k in cmddict.keys(): 

2814# cmdline = cmdlist[i] 

2815 cmdline = cmddict[k]['command'] 

2816 

2817 # Skip if it is a comment line 

2818 if cmdline.startswith('#'): 

2819 break 

2820 

2821 # split by white space 

2822 keyvlist = cmdline.split() 

2823 if keyvlist.__len__() > 0: 

2824 

2825 # Split by '=' 

2826 for keyv in keyvlist: 

2827 

2828 (xkey,xval) = keyv.split('=',1) 

2829 

2830 # Remove quotes 

2831 if type(xval) == str: 

2832 if xval.count("'") > 0: 

2833 xval = xval.strip("'") 

2834 if xval.count('"') > 0: 

2835 xval = xval.strip('"') 

2836 

2837 # Check which parameter 

2838 if xkey == "scan": 

2839 scans += xval + ',' 

2840 

2841 elif xkey == "field": 

2842 fields += xval + ',' 

2843 

2844 elif xkey == "antenna": 

2845 ants += xval + ';' 

2846 

2847 elif xkey == "timerange": 

2848 times += xval + ',' 

2849 

2850 elif xkey == "correlation": 

2851 corrs += xval + ',' 

2852 

2853 elif xkey == "intent": 

2854 ints += xval + ',' 

2855 

2856 elif xkey == "feed": 

2857 feeds += xval + ',' 

2858 

2859 elif xkey == "array": 

2860 arrays += xval + ',' 

2861 

2862 elif xkey == "uvrange": 

2863 uvs += xval + ',' 

2864 

2865 elif xkey == "spw": 

2866 spws += xval + ',' 

2867 

2868 elif xkey == "observation": 

2869 obs += xval + ',' 

2870 

2871 

2872 # Strip out the extra comma at the end 

2873 scans = scans.rstrip(',') 

2874 fields = fields.rstrip(',') 

2875 ants = ants.rstrip(';') 

2876 times = times.rstrip(',') 

2877 corrs = corrs.rstrip(',') 

2878 ints = ints.rstrip(',') 

2879 feeds = feeds.rstrip(',') 

2880 arrays = arrays.rstrip(',') 

2881 uvs = uvs.rstrip(',') 

2882 spws = spws.rstrip(',') 

2883 obs = obs.rstrip(',') 

2884 

2885 dicpars['scan'] = scans 

2886 dicpars['field'] = fields 

2887 # Antennas are handled better within the framework. 

2888 dicpars['antenna'] = '' 

2889 # Times are handled better within the framework. 

2890 dicpars['timerange'] = '' 

2891 # Correlations should be handled only by the agents 

2892 dicpars['correlation'] = '' 

2893 # CAS-4682, do not create union for intents 

2894 dicpars['intent'] = '' 

2895 dicpars['feed'] = feeds 

2896 dicpars['array'] = arrays 

2897 dicpars['uvrange'] = uvs 

2898 dicpars['spw'] = spws 

2899 dicpars['observation'] = obs 

2900 

2901 

2902 # Compress the selection list to reduce MSSelection parsing time. 

2903 # 'field','spw','antenna' strings in dicpars will be modified in-place. 

2904 compressSelectionList(vis,dicpars); 

2905 

2906 # Real number of input lines 

2907 # Get the number of occurrences of each parameter 

2908 npars = getNumPar(cmddict) 

2909 nlines = nrows - npars['comment'] 

2910 

2911 # Make the union.  

2912 for k,v in npars.items(): 

2913 if k != 'comment': 

2914 if v < nlines: 

2915 dicpars[k] = '' 

2916 

2917 

2918 uniondic = dicpars.copy() 

2919 # Remove empty parameters from the dictionary 

2920 for k,v in dicpars.items(): 

2921 if v == '': 

2922 uniondic.pop(k) 

2923 

2924 return uniondic 

2925 

2926def getNumPar(cmddict): 

2927 '''Get the number of occurrences of all parameter keys 

2928 cmdlist --> list of strings with parameters and values 

2929 ''' 

2930 

2931# nrows = cmdlist.__len__() 

2932 

2933 # Dictionary of number of occurrences to return 

2934 npars = { 

2935 'field':0, 

2936 'scan':0, 

2937 'antenna':0, 

2938 'spw':0, 

2939 'timerange':0, 

2940 'correlation':0, 

2941 'intent':0, 

2942 'feed':0, 

2943 'array':0, 

2944 'uvrange':0, 

2945 'comment':0, 

2946 'observation':0 

2947 } 

2948 

2949 ci = 0 # count the number of lines with comments (starting with a #) 

2950 si = 0 # count the number of lines with scan 

2951 fi = 0 # count the number of lines with field 

2952 ai = 0 # count the number of lines with antenna 

2953 ti = 0 # count the number of lines with timerange 

2954 coi = 0 # count the number of lines with correlation 

2955 ii = 0 # count the number of lines with intent 

2956 fei = 0 # count the number of lines with feed 

2957 ari = 0 # count the number of lines with array 

2958 ui = 0 # count the number of lines with uvrange 

2959 pi = 0 # count the number of lines with spw 

2960 oi = 0 # count the number of lines with observation 

2961 

2962# for i in range(nrows): 

2963 for k in cmddict.keys(): 

2964# cmdline = cmdlist[i] 

2965 cmdline = cmddict[k]['command'] 

2966 

2967 if cmdline.startswith('#'): 

2968 ci += 1 

2969 npars['comment'] = ci 

2970 continue 

2971 

2972 # split by white space 

2973 keyvlist = cmdline.split() 

2974 if keyvlist.__len__() > 0: 

2975 

2976 # Split by '=' 

2977 for keyv in keyvlist: 

2978 

2979 # Skip if it is a comment character # 

2980 if keyv.count('#') > 0: 

2981 break 

2982 

2983 (xkey,xval) = keyv.split('=',1) 

2984 

2985 # Remove quotes 

2986 if type(xval) == str: 

2987 if xval.count("'") > 0: 

2988 xval = xval.strip("'") 

2989 if xval.count('"') > 0: 

2990 xval = xval.strip('"') 

2991 # Remove blanks 

2992 xval=xval.replace(' ',''); 

2993 

2994 # Check which parameter, if not empty 

2995 if xval != "": 

2996 if xkey == "scan": 

2997 si += 1 

2998 npars['scan'] = si 

2999 

3000 elif xkey == "field": 

3001 fi += 1 

3002 npars['field'] = fi 

3003 

3004 elif xkey == "antenna": 

3005 ai += 1 

3006 npars['antenna'] = ai 

3007 

3008 elif xkey == "timerange": 

3009 ti += 1 

3010 npars['timerange'] = ti 

3011 

3012 elif xkey == "correlation": 

3013 coi += 1 

3014 npars['correlation'] = coi 

3015 

3016 elif xkey == "intent": 

3017 ii += 1 

3018 npars['intent'] = ii 

3019 

3020 elif xkey == "feed": 

3021 fei += 1 

3022 npars['feed'] = fei 

3023 

3024 elif xkey == "array": 

3025 ari += 1 

3026 npars['array'] = ari 

3027 

3028 elif xkey == "uvrange": 

3029 ui += 1 

3030 npars['uvrange'] = ui 

3031 

3032 elif xkey == "spw": 

3033 pi += 1 

3034 npars['spw'] = pi 

3035 

3036 elif xkey == "observation": 

3037 oi += 1 

3038 npars['observation'] = oi 

3039 

3040 

3041 return npars 

3042 

3043def compressSelectionList(vis='',dicpars={}): 

3044 """ 

3045 - Find a loose union of data-selection parameters, to reduce the MSSelection parsing load. 

3046 - This compressed selection list is only meant to be used with af.selectdata(), because 

3047 further selections are handled internally. 

3048 - Use MSSelection in its 'onlyparse=True' mode to gather a list of fields, spws, antennas 

3049 touched by the selection list. These are the only keys for which MSSelection does not  

3050 need to parse the MS, and will cover the expensive cases where complicated antenna 

3051 and spw expressions can slow MSSelection down.  

3052 """ 

3053 from numpy import unique; 

3054 

3055 mslocal.open(vis) #, nomodify=False) 

3056 try: 

3057 indices = mslocal.msseltoindex(vis=vis,field=dicpars['field'], spw=dicpars['spw'],baseline=dicpars['antenna']); 

3058 finally: 

3059 mslocal.close() 

3060 

3061 c_field = str(list(unique(indices['field']))).strip('[]'); 

3062 c_spw = str(list(unique(indices['spw']))).strip('[]'); 

3063 c_antenna = str(list( unique( list(indices['antenna1']) + list(indices['antenna2']) ) ) ).strip('[]'); 

3064 

3065 dicpars['field'] = c_field; 

3066 dicpars['spw'] = c_spw; 

3067 dicpars['antenna'] = c_antenna; 

3068 

3069 # Programmer note : Other selection parameters that can be compressed accurately 

3070 # from MS subtable information alone (no need to parse the main table) are  

3071 # 'array', 'observationid', 'state(or intent)'. They are currently un-available  

3072 # via mslocal.msseltoindex() and therefore not used here yet. 

3073 

3074 return; 

3075 

3076# ONLY for flagcmd task 

3077def writeFlagCmd(msfile, myflags, vrows, applied, add_reason, outfile): 

3078 ''' 

3079 Writes the flag commands to FLAG_CMD or to an ASCII file. Used 

3080 only by the flagcmd task. This uses the old, simple parser. 

3081 For flagdata, use writeFlagCommands(). 

3082  

3083 msfile --> MS 

3084 myflags --> dictionary of commands read from inputfile (from readFromTable, etc.) 

3085 vrows --> list of valid rows from myflags dictionary to save 

3086 applied --> value to update APPLIED column of FLAG_CMD 

3087 add_reason --> reason to add to output (replace input reason, if any) 

3088 outfile --> if not empty, save to it 

3089 Returns the number of commands written to output 

3090 ''' 

3091 

3092 nadd = 0 

3093 try: 

3094 import pylab as pl 

3095 except ImportError as exc: 

3096 raise ImportError('Failed to load pylab, required in writeFlagCmd: {}'. 

3097 format(exc)) 

3098 

3099 

3100 # append to a file  

3101 if outfile != '': 

3102 ffout = open(outfile, 'a') 

3103 

3104 # Replace blanks from reason and cmdreason with underscores 

3105 # and warn the user. 

3106 if add_reason != '': 

3107 tempreason = add_reason 

3108 add_reason = tempreason.replace(' ','_') 

3109 casalog.post('Replaced blanks with underscores in cmdreason', 'WARN') 

3110 

3111 try: 

3112 for key in myflags.keys(): 

3113 # Remove leading and trailing white spaces 

3114 cmdline = myflags[key]['command'].strip() 

3115 

3116 # Add addantenna parameter back 

3117 if myflags[key].__contains__('addantenna'): 

3118 addantenna = myflags[key]['addantenna'] 

3119 cmdline = cmdline + ' addantenna=' + str(addantenna) + ' ' 

3120 

3121 reason = myflags[key]['reason'] 

3122 if add_reason != '': 

3123 reason = str(add_reason) 

3124 

3125 if reason != '': 

3126 cmdline = cmdline + ' reason=' + str(reason) + ' ' 

3127 

3128 # For flagcmd saving 

3129 newdict = evalString(cmdline) 

3130# newdict = evaluateString(cmdline) 

3131 cmdline = '' 

3132 for k,v in newdict.items(): 

3133 cmdstr = "" 

3134 # Add quotes to non-quoted strings 

3135 if isinstance(v, str): 

3136 if v.count("'") > 0: 

3137 v = v.strip("'") 

3138 if v.count('"') > 0: 

3139 v = v.strip('"') 

3140 

3141 # Add quotes to string values 

3142 cmdstr = "'"+v+"'" 

3143 v = cmdstr 

3144 cmdline = cmdline + k + '=' + str(v) + ' ' 

3145 

3146 elif isinstance(v,list): 

3147 lstring = '' 

3148 for ll in v: 

3149 if isinstance(ll,list): 

3150 for nl in ll: 

3151 lstring = lstring + str(nl)+',' 

3152 else: 

3153 lstring = lstring + str(ll)+',' 

3154 lstring = lstring.rstrip(',') 

3155 cmdline = cmdline + k + '=['+ lstring + '] ' 

3156 

3157 else: 

3158 cmdline = cmdline + k + '=' + str(v) + ' ' 

3159 

3160 # Save to output file 

3161 ffout.write('%s\n' %cmdline.rstrip()) 

3162 

3163 

3164# except: 

3165 except Exception as instance: 

3166 raise Exception('Error writing lines to file %s. %s' %(outfile,instance)) 

3167 ffout.close() 

3168 return 

3169 

3170 # Append new commands to existing table 

3171 if vrows.__len__() > 0: 

3172 # Extract flags from dictionary into list 

3173 tim_list = [] 

3174 intv_list = [] 

3175 cmd_list = [] 

3176 reas_list = [] 

3177 typ_list = [] 

3178 sev_list = [] 

3179 lev_list = [] 

3180 app_list = [] 

3181 

3182 # Only write valid rows that have been applied to MS 

3183 

3184 for key in vrows: 

3185 # do not write line with summary mode 

3186 command = myflags[key]['command'] 

3187 if command.__contains__('summary'): 

3188 continue 

3189 

3190 # Add addantenna back 

3191 if myflags[key].__contains__('addantenna'): 

3192 addantenna = myflags[key]['addantenna'] 

3193 command = command + ' addantenna=' + str(addantenna) 

3194 

3195 # Evaluate the types 

3196 newdict = evalString(command) 

3197# newdict = evaluateString(command) 

3198 cmdline = '' 

3199 for k,v in newdict.items(): 

3200 cmdstr = "" 

3201 if isinstance(v, str): 

3202 if v.count("'") > 0: 

3203 v = v.strip("'") 

3204 if v.count('"') > 0: 

3205 v = v.strip('"') 

3206 # Add quotes to string values 

3207 cmdstr = "'"+v+"'" 

3208 v = cmdstr 

3209 cmdline = cmdline + k + '=' + str(v) + ' ' 

3210 elif isinstance(v,list): 

3211 lstring = '' 

3212 for ll in v: 

3213 lstring = lstring + str(ll)+',' 

3214 lstring = lstring.rstrip(',') 

3215 cmdline = cmdline + k + '=['+ lstring + '] ' 

3216 else: 

3217 cmdline = cmdline + k + '=' + str(v) + ' ' 

3218 

3219 cmd_list.append(cmdline) 

3220 tim_list.append(myflags[key]['time']) 

3221 intv_list.append(myflags[key]['interval']) 

3222 if add_reason != '': 

3223 reas_list.append(add_reason) 

3224 else: 

3225 reas_list.append(myflags[key]['reason']) 

3226 typ_list.append(myflags[key]['type']) 

3227 sev_list.append(myflags[key]['severity']) 

3228 lev_list.append(myflags[key]['level']) 

3229 app_list.append(applied) 

3230 

3231 

3232 # Save to FLAG_CMD table 

3233 nadd = cmd_list.__len__() 

3234 

3235 mstable = msfile + '/FLAG_CMD' 

3236 try: 

3237 tblocal.open(mstable, nomodify=False) 

3238 except: 

3239 raise Exception('Error opening FLAG_CMD table ' + mstable) 

3240 nrows = int(tblocal.nrows()) 

3241 casalog.post('There are ' + str(nrows) 

3242 + ' rows already in FLAG_CMD', 'DEBUG') 

3243 # add blank rows 

3244 tblocal.addrows(nadd) 

3245 # now fill them in 

3246 tblocal.putcol('TIME', numpy.array(tim_list), startrow=nrows, nrow=nadd) 

3247 tblocal.putcol('INTERVAL', numpy.array(intv_list), startrow=nrows, 

3248 nrow=nadd) 

3249 tblocal.putcol('REASON', numpy.array(reas_list), startrow=nrows, 

3250 nrow=nadd) 

3251 tblocal.putcol('COMMAND', numpy.array(cmd_list), startrow=nrows, 

3252 nrow=nadd) 

3253 # Other columns 

3254 tblocal.putcol('TYPE', numpy.array(typ_list), startrow=nrows, nrow=nadd) 

3255 tblocal.putcol('SEVERITY', numpy.array(sev_list), startrow=nrows, 

3256 nrow=nadd) 

3257 tblocal.putcol('LEVEL', numpy.array(lev_list), startrow=nrows, 

3258 nrow=nadd) 

3259 tblocal.putcol('APPLIED', numpy.array(app_list), startrow=nrows, 

3260 nrow=nadd) 

3261 

3262 nrows = int(tblocal.nrows()) 

3263 tblocal.close() 

3264 casalog.post('Saved ' + str(nrows) + ' rows to FLAG_CMD') 

3265 

3266 else: 

3267 casalog.post('Saved zero rows to FLAG_CMD; no flags found') 

3268 

3269 return nrows 

3270 

3271def getReason(cmdline): 

3272 '''Get the reason values from a line with strings 

3273 cmdline --> a string with parameters 

3274 returns a string with reason values. 

3275 ''' 

3276 

3277 reason = '' 

3278 

3279 # Skip comment lines 

3280 if cmdline.startswith('#'): 

3281 return reason 

3282 

3283 # Split by white space 

3284 keyvlist = cmdline.split() 

3285 if keyvlist.__len__() > 0: 

3286 

3287 for keyv in keyvlist: 

3288 

3289 # Split by '=' 

3290 (xkey,xval) = keyv.split('=',1) 

3291 

3292 # Remove quotes 

3293 if type(xval) == str: 

3294 if xval.count("'") > 0: 

3295 xval = xval.strip("'") 

3296 if xval.count('"') > 0: 

3297 xval = xval.strip('"') 

3298 

3299 # Check if reason is in 

3300 if xkey == "reason": 

3301 reason = xval 

3302 break; 

3303 

3304 

3305 return reason 

3306 

3307def getLinePars(cmdline, mlist=[]): 

3308 '''Get a dictionary of all selection parameters from a line: 

3309 cmdline --> a string with parameters 

3310 mlist --> a list of mode's parameters to add to the output dictionary 

3311  

3312 Returns a dictionary. 

3313 ''' 

3314 

3315 # Dictionary of parameters to return 

3316 dicpars = {} 

3317 

3318 # Skip comment lines 

3319 if cmdline.startswith('#'): 

3320 return dicpars 

3321 

3322 # split by white space 

3323 keyvlist = cmdline.split() 

3324 if keyvlist.__len__() > 0: 

3325 

3326 # Split by '=' 

3327 for keyv in keyvlist: 

3328 

3329 (xkey,xval) = keyv.split('=',1) 

3330 

3331 # Remove quotes 

3332 if type(xval) == str: 

3333 if xval.count("'") > 0: 

3334 xval = xval.strip("'") 

3335 if xval.count('"') > 0: 

3336 xval = xval.strip('"') 

3337 

3338 # Check which parameter 

3339 if xkey == "scan": 

3340 dicpars['scan'] = xval 

3341 

3342 elif xkey == "field": 

3343 dicpars['field'] = xval 

3344 

3345 elif xkey == "antenna": 

3346 dicpars['antenna'] = xval 

3347 

3348 elif xkey == "timerange": 

3349 dicpars['timerange'] = xval 

3350 

3351 elif xkey == "correlation": 

3352 dicpars['correlation'] = xval.upper() 

3353 

3354 elif xkey == "intent": 

3355 dicpars['intent'] = xval 

3356 

3357 elif xkey == "feed": 

3358 dicpars['feed'] = xval 

3359 

3360 elif xkey == "array": 

3361 dicpars['array'] = xval 

3362 

3363 elif xkey == "uvrange": 

3364 dicpars['uvrange'] = xval 

3365 

3366 elif xkey == "spw": 

3367 dicpars['spw'] = xval 

3368 

3369 elif xkey == "observation": 

3370 dicpars['observation'] = xval 

3371 

3372 elif xkey == "mode": 

3373 if xval == 'manualflag': 

3374 xval = 'manual' 

3375 dicpars['mode'] = xval 

3376 

3377 elif mlist != []: 

3378 # Any parameters requested for this mode? 

3379 for m in mlist: 

3380 if xkey == m: 

3381 dicpars[m] = xval 

3382 

3383 casalog.post(':getLinePars::dicpars=%s'%dicpars, 'DEBUG') 

3384 

3385 return dicpars 

3386 

3387def getSelectionPars(cmdline): 

3388 '''Get a dictionary of all selection parameters from a line: 

3389 cmdline --> a string with parameters 

3390 ''' 

3391 

3392 # Dictionary of parameters to return 

3393 dicpars = {} 

3394 

3395 # Skip comment lines 

3396 if cmdline.startswith('#'): 

3397 return dicpars 

3398 

3399 # split by white space 

3400 keyvlist = cmdline.split() 

3401 if keyvlist.__len__() > 0: 

3402 

3403 # Split by '=' 

3404 for keyv in keyvlist: 

3405 

3406 (xkey,xval) = keyv.split('=',1) 

3407 

3408 # Remove quotes 

3409 if type(xval) == str: 

3410 if xval.count("'") > 0: 

3411 xval = xval.strip("'") 

3412 if xval.count('"') > 0: 

3413 xval = xval.strip('"') 

3414 

3415 # Check which parameter 

3416 if xkey == "scan": 

3417 dicpars['scan'] = xval 

3418 

3419 elif xkey == "field": 

3420 dicpars['field'] = xval 

3421 

3422 elif xkey == "antenna": 

3423 dicpars['antenna'] = xval 

3424 

3425 elif xkey == "timerange": 

3426 dicpars['timerange'] = xval 

3427 

3428 # Correlation will be handled by the agent 

3429 elif xkey == "correlation": 

3430 dicpars['correlation'] = '' 

3431 

3432 elif xkey == "intent": 

3433 dicpars['intent'] = xval 

3434 

3435 elif xkey == "feed": 

3436 dicpars['feed'] = xval 

3437 

3438 elif xkey == "array": 

3439 dicpars['array'] = xval 

3440 

3441 elif xkey == "uvrange": 

3442 dicpars['uvrange'] = xval 

3443 

3444 elif xkey == "spw": 

3445 dicpars['spw'] = xval 

3446 

3447 elif xkey == "observation": 

3448 dicpars['observation'] = xval 

3449 

3450 

3451 return dicpars 

3452 

3453def readNtime(params): 

3454 '''Check the value and units of ntime 

3455 params --> dictionary of agent's parameters ''' 

3456 

3457 newtime = 0.0 

3458 

3459 if 'ntime' in params: 

3460 ntime = params['ntime'] 

3461 

3462 # Verify the ntime value 

3463 if type(ntime) == float or type(ntime) == int: 

3464 if ntime <= 0: 

3465 raise Exception('Parameter ntime cannot be < = 0') 

3466 else: 

3467 # units are seconds 

3468 newtime = float(ntime) 

3469 

3470 elif type(ntime) == str: 

3471 if ntime == 'scan': 

3472 # iteration time step is a scan 

3473 newtime = 0.0 

3474 else: 

3475 # read the units from the string 

3476 qtime = qalocal.quantity(ntime) 

3477 

3478 if qtime['unit'] == 'min': 

3479 # convert to seconds 

3480 qtime = qalocal.convert(qtime, 's') 

3481 elif qtime['unit'] == '': 

3482 qtime['unit'] = 's' 

3483 

3484 # check units 

3485 if qtime['unit'] == 's': 

3486 newtime = qtime['value'] 

3487 else: 

3488 casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN') 

3489 

3490 params['ntime'] = float(newtime) 

3491 

3492def fixType(params): 

3493 '''Give correct types to non-string parameters 

3494 The types are defined in the XML file of the task flagdata 

3495 Do not repeat any parameter''' 

3496 

3497 # manual parameter 

3498 if 'autocorr' in params: 

3499 params['autocorr'] = eval(params['autocorr'].capitalize()) 

3500 

3501 # quack parameters 

3502 if 'quackmode' in params and not params['quackmode'] in ['beg' 

3503 , 'endb', 'end', 'tail']: 

3504 raise Exception( \ 

3505 "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \ 

3506 % params['quackmode']) 

3507 if 'quackinterval' in params: 

3508 params['quackinterval'] = float(params['quackinterval']) 

3509 if 'quackincrement' in params: 

3510 if type(params['quackincrement']) == str: 

3511 params['quackincrement'] = eval(params['quackincrement'].capitalize()) 

3512 

3513 # clip parameters 

3514 if 'clipminmax' in params: 

3515 value01 = params['clipminmax'] 

3516 # turn string into [min,max] range 

3517 value0 = value01.lstrip('[') 

3518 value = value0.rstrip(']') 

3519 r = value.split(',') 

3520 rmin = float(r[0]) 

3521 rmax = float(r[1]) 

3522 params['clipminmax'] = [rmin, rmax] 

3523 if 'clipoutside' in params: 

3524 if type(params['clipoutside']) == str: 

3525 params['clipoutside'] = eval(params['clipoutside'].capitalize()) 

3526 else: 

3527 params['clipoutside'] = params['clipoutside'] 

3528 if 'channelavg' in params: 

3529 params['channelavg'] = eval(params['channelavg'].capitalize()) 

3530 if 'clipzeros' in params: 

3531 params['clipzeros'] = eval(params['clipzeros'].capitalize()) 

3532 

3533 

3534 # shadow parameter 

3535 if 'tolerance' in params: 

3536 params['tolerance'] = float(params['tolerance']) 

3537 

3538 # elevation parameters 

3539 if 'lowerlimit' in params: 

3540 params['lowerlimit'] = float(params['lowerlimit']) 

3541 if 'upperlimit'in params: 

3542 params['upperlimit'] = float(params['upperlimit']) 

3543 

3544 # extend parameters 

3545 if 'extendpols' in params: 

3546 params['extendpols'] = eval(params['extendpols'].capitalize()) 

3547 if 'growtime' in params: 

3548 params['growtime'] = float(params['growtime']) 

3549 if 'growfreq' in params: 

3550 params['growfreq'] = float(params['growfreq']) 

3551 if 'growaround' in params: 

3552 params['growaround'] = eval(params['growaround'].capitalize()) 

3553 if 'flagneartime' in params: 

3554 params['flagneartime'] = eval(params['flagneartime'].capitalize()) 

3555 if 'flagnearfreq' in params: 

3556 params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize()) 

3557 

3558 # tfcrop parameters 

3559 if 'combinescans' in params: 

3560 params['combinescans'] = eval(params['combinescans'].capitalize()) 

3561 if 'timecutoff' in params: 

3562 params['timecutoff'] = float(params['timecutoff']) 

3563 if 'freqcutoff' in params: 

3564 params['freqcutoff'] = float(params['freqcutoff']) 

3565 if 'maxnpieces' in params: 

3566 params['maxnpieces'] = int(params['maxnpieces']) 

3567 if 'halfwin' in params: 

3568 params['halfwin'] = int(params['halfwin']) 

3569 if 'extendflags' in params: 

3570 params['extendflags'] = eval(params['extendflags'].capitalize()) 

3571 

3572 # rflag parameters 

3573 if 'winsize' in params: 

3574 params['winsize'] = int(params['winsize']); 

3575 if 'timedev' in params: 

3576 timepar = params['timedev'] 

3577 try: 

3578 timepar = eval(timepar) 

3579 except Exception: 

3580 timepar = readRFlagThresholdFile(params['timedev'],'timedev'); 

3581 params['timedev'] = timepar 

3582 if 'freqdev' in params: 

3583 freqpar = params['freqdev'] 

3584 try: 

3585 freqpar = eval(freqpar) 

3586 except Exception: 

3587 freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev'); 

3588 params['freqdev'] = freqpar 

3589 if 'timedevscale' in params: 

3590 params['timedevscale'] = float(params['timedevscale']); 

3591 if 'freqdevscale' in params: 

3592 params['freqdevscale'] = float(params['freqdevscale']); 

3593 if 'spectralmin' in params: 

3594 params['spectralmin'] = float(params['spectralmin']); 

3595 if 'spectralmax' in params: 

3596 params['spectralmax'] = float(params['spectralmax']); 

3597 

3598def purgeEmptyPars(cmdline): 

3599 '''Remove empty parameters from a string: 

3600 cmdline --> a string with parameters 

3601  

3602 returns a string containing only parameters with values 

3603 ''' 

3604 newstr = '' 

3605 

3606 # split by white space 

3607 keyvlist = cmdline.split() 

3608 if keyvlist.__len__() > 0: 

3609 

3610 # Split by '=' 

3611 for keyv in keyvlist: 

3612 

3613 (xkey,xval) = keyv.split('=',1) 

3614 

3615 # Remove quotes 

3616 if type(xval) == str: 

3617 if xval.count("'") > 0: 

3618 xval = xval.strip("'") 

3619 if xval.count('"') > 0: 

3620 xval = xval.strip('"') 

3621 

3622 # Write only parameters with values 

3623 if xval == '': 

3624 continue 

3625 else: 

3626 newstr = newstr+xkey+'='+xval+' ' 

3627 

3628 else: 

3629 casalog.post('String of parameters is empty','WARN') 

3630 

3631 return newstr 

3632 

3633def purgeParameter(cmdline, par): 

3634 '''Remove parameter from a string: 

3635 cmdline --> a string with a parameter to be removed 

3636 par --> the parameter to be removed from the string 

3637  

3638 returns a string containing the remaining parameters 

3639 ''' 

3640 

3641 newstr = '' 

3642 

3643 # split by white space 

3644 keyvlist = cmdline.split() 

3645 if keyvlist.__len__() > 0: 

3646 

3647 # Split by '=' 

3648 for keyv in keyvlist: 

3649 

3650 (xkey,xval) = keyv.split('=',1) 

3651 

3652 # Remove quotes 

3653 if type(xval) == str: 

3654 if xval.count("'") > 0: 

3655 xval = xval.strip("'") 

3656 if xval.count('"') > 0: 

3657 xval = xval.strip('"') 

3658 

3659 # Write only parameters with values 

3660 if xkey == par: 

3661 continue 

3662 else: 

3663 newstr = newstr+xkey+'=' + xval+ ' ' 

3664 

3665 else: 

3666 casalog.post('String of parameters is empty','WARN') 

3667 

3668 return newstr 

3669 

3670def setupAgent(aflocal, myflagcmd, myrows, apply, writeflags, display=''): 

3671 ''' Setup the parameters of each agent and call the agentflagger tool 

3672  

3673 myflagcmd --> it is a dictionary coming from readFromTable, readFile, etc. 

3674 myrows --> selected rows to apply/unapply flags 

3675 apply --> it's a boolean to control whether to apply or unapply the flags 

3676 writeflags --> used by mode=rflag only 

3677 display --> used by mode='rflag only''' 

3678 

3679 

3680 if not myflagcmd.__len__() >0: 

3681 casalog.post('There are no flag cmds in list', 'SEVERE') 

3682 return 

3683 

3684 # Parameters for each mode 

3685 manualpars = ['autocorr'] 

3686 unflagpars = [] 

3687 clippars = ['clipminmax', 'clipoutside','datacolumn', 'channelavg', 'clipzeros'] 

3688 quackpars = ['quackinterval','quackmode','quackincrement'] 

3689 shadowpars = ['tolerance', 'addantenna'] 

3690 elevationpars = ['lowerlimit','upperlimit'] 

3691 tfcroppars = ['ntime','combinescans','datacolumn','timecutoff','freqcutoff', 

3692 'timefit','freqfit','maxnpieces','flagdimension','usewindowstats','halfwin', 

3693 'extendflags'] 

3694 antintpars = ['antint_ref_antenna','minchanfrac','verbose'] 

3695 extendpars = ['ntime','combinescans','extendpols','growtime','growfreq','growaround', 

3696 'flagneartime','flagnearfreq'] 

3697 rflagpars = ['winsize','timedev','freqdev','timedevscale','freqdevscale','spectralmax', 

3698 'spectralmin', 'extendflags'] 

3699 

3700 

3701 # dictionary of successful command lines to save to outfile 

3702 savelist = {} 

3703 

3704 # Setup the agent for each input line  

3705 for key in myflagcmd.keys(): 

3706 cmdline = myflagcmd[key]['command'] 

3707 applied = myflagcmd[key]['applied'] 

3708 interval = myflagcmd[key]['interval'] 

3709 level = myflagcmd[key]['level'] 

3710 reason = myflagcmd[key]['reason'] 

3711 severity = myflagcmd[key]['severity'] 

3712 coltime = myflagcmd[key]['time'] 

3713 coltype = myflagcmd[key]['type'] 

3714 if debug: 

3715 casalog.post('cmdline for key%s'%key) 

3716 casalog.post('%s'%cmdline) 

3717 casalog.post('applied is %s'%applied) 

3718 

3719 if cmdline.startswith('#'): 

3720 continue 

3721 

3722 modepars = {} 

3723 parslist = {} 

3724 mode = '' 

3725 valid = True 

3726 addantenna = {} 

3727 

3728 # Get the specific parameters for the mode 

3729 if cmdline.__contains__('mode'): 

3730 if cmdline.__contains__('manual'): 

3731 mode = 'manual' 

3732 modepars = getLinePars(cmdline,manualpars) 

3733 elif cmdline.__contains__('clip'): 

3734 mode = 'clip' 

3735 modepars = getLinePars(cmdline,clippars) 

3736 elif cmdline.__contains__('quack'): 

3737 mode = 'quack' 

3738 modepars = getLinePars(cmdline,quackpars) 

3739 elif cmdline.__contains__('shadow'): 

3740 mode = 'shadow' 

3741 modepars = getLinePars(cmdline,shadowpars) 

3742 

3743 # Get addantenna dictionary 

3744 if myflagcmd[key].__contains__('addantenna'): 

3745 addantenna = myflagcmd[key]['addantenna'] 

3746 modepars['addantenna'] = addantenna 

3747 else: 

3748 # Get antenna filename 

3749 if (modepars.__contains__('addantenna')): 

3750 ant_par = modepars['addantenna'] 

3751 

3752 # It must be a string 

3753 if (type(ant_par) == str and ant_par != ''): 

3754 antennafile = modepars['addantenna'] 

3755 addantenna = readAntennaList(antennafile) 

3756 modepars['addantenna'] = addantenna 

3757 

3758 elif cmdline.__contains__('elevation'): 

3759 mode = 'elevation' 

3760 modepars = getLinePars(cmdline,elevationpars) 

3761 elif cmdline.__contains__('tfcrop'): 

3762 mode = 'tfcrop' 

3763 modepars = getLinePars(cmdline,tfcroppars) 

3764 elif cmdline.__contains__('antint'): 

3765 mode = 'antint' 

3766 modepars = getLinePars(cmdline,antintpars) 

3767 elif cmdline.__contains__('extend') and cmdline.__contains__('extendpols'): 

3768 # Necessary to avoid matching the extenflags parameter of rflag/tfcrop 

3769 mode = 'extend' 

3770 modepars = getLinePars(cmdline,extendpars) 

3771 elif cmdline.__contains__('unflag'): 

3772 mode = 'unflag' 

3773 modepars = getLinePars(cmdline,unflagpars) 

3774 elif cmdline.__contains__('rflag'): 

3775 mode = 'rflag' 

3776 modepars = getLinePars(cmdline,rflagpars) 

3777 

3778 #####  

3779 ### According to 'shadow' file handling, this code should be here... 

3780 ### but, it's already done inside fixType.  

3781 ##### 

3782 #if( type(modepars['timedev']) == str and writeflags == True): 

3783 # timedev = readRFlagThresholdFile(modepars['timedev'],'timedev') 

3784 # modepars['timedev'] = timedev 

3785 #if( type(modepars['freqdev']) == str and writeflags == True): 

3786 # freqdev = readRFlagThresholdFile(modepars['freqdev'],'freqdev') 

3787 # modepars['freqdev'] = freqdev 

3788 

3789 # Add the writeflags and display parameters 

3790 modepars['writeflags'] = writeflags 

3791 modepars['display'] = display 

3792 else: 

3793 # Unknown mode, ignore it 

3794 casalog.post('Ignoring unknown mode', 'WARN') 

3795# valid = False 

3796 

3797 else: 

3798 # No mode means manual 

3799 mode = 'manual' 

3800 cmdline = cmdline+' mode=manual' 

3801 modepars = getLinePars(cmdline,manualpars) 

3802 

3803 

3804 # Read ntime 

3805 readNtime(modepars) 

3806 

3807 # Cast the correct type to non-string parameters 

3808 fixType(modepars) 

3809 

3810 # Add the apply/unapply parameter to dictionary  

3811 modepars['apply'] = apply 

3812 

3813 # Unapply selected rows only and re-apply the other rows with APPLIED=True 

3814# if not apply and myrows.__len__() > 0: 

3815# if key in myrows: 

3816# modepars['apply'] = False 

3817# elif not applied: 

3818# casalog.post("Skipping this %s"%modepars,"DEBUG") 

3819# continue 

3820# elif applied: 

3821# modepars['apply'] = True 

3822# valid = False 

3823 

3824 # Keep only cmds that overlap with the unapply cmds 

3825 # TODO later 

3826 

3827 # Hold the name of the agent and the cmd row number 

3828 agent_name = mode.capitalize()+'_'+str(key) 

3829 modepars['name'] = agent_name 

3830 

3831 # Remove the data selection parameters if there is only one agent for performance reasons. 

3832 # Explanation: if only one agent exists and the data selection parameters are parsed to it,  

3833 # it will have to go through the entire MS and check if each data selection given to the agent 

3834 # matches what the user asked in the selected data. 

3835 

3836 # Only correlation, antenna and timerange will go to the agent 

3837 # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too 

3838 if myflagcmd.__len__() == 1: 

3839 sellist=['scan','field','intent','feed','array','uvrange','observation'] 

3840 for k in sellist: 

3841 if k in modepars: 

3842 modepars.pop(k) 

3843 

3844 casalog.post('Parsing parameters of mode %s in row %s'%(mode,key), 'DEBUG') 

3845 casalog.post('%s'%modepars, 'DEBUG') 

3846 if debug: 

3847 casalog.post('Parsing parameters of mode %s in row %s'%(mode,key)) 

3848 casalog.post(modepars) 

3849 

3850 # Parse the dictionary of parameters to the tool 

3851 if (not aflocal.parseagentparameters(modepars)): 

3852 casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,key), 'WARN') 

3853 continue 

3854 

3855 # Save the dictionary of valid agents 

3856 if valid: 

3857 # add this command line to list to save in outfile 

3858 parslist['row'] = key 

3859 parslist['command'] = cmdline 

3860 parslist['applied'] = applied 

3861 parslist['interval'] = interval 

3862 parslist['level'] = level 

3863 parslist['reason'] = reason 

3864 parslist['severity'] = severity 

3865 parslist['time'] = coltime 

3866 parslist['type'] = coltype 

3867 if addantenna != {}: 

3868 parslist['addantenna'] = addantenna 

3869 savelist[key] = parslist 

3870 

3871 if debug: 

3872 casalog.post('Dictionary of valid commands to save') 

3873 casalog.post('%s'%savelist) 

3874 

3875 return savelist 

3876 

3877def backupFlags(aflocal=None, msfile='', prename='flagbackup'): 

3878 '''Create a backup of the FLAG column 

3879  

3880 aflocal local version of the agentflagger tool or 

3881 msfile name of MS/cal table to backup 

3882 prename prefix for name of flag backup file 

3883  

3884 If msfile is given, aflocal will not be used 

3885  

3886 Create names like this: 

3887 flags.flagcmd_1, 

3888 flags.flagdata_1, 

3889  

3890 Generally <task>_<i>, where i is the smallest 

3891 integer giving a name, which does not already exist''' 

3892 

3893 if msfile != '': 

3894 # open msfile and attach it to tool 

3895 aflocal = agentflagger() 

3896 aflocal.open(msfile) 

3897 

3898 elif aflocal == None: 

3899 casalog.post('Need an MS or a copy of the agentflagger tool to create a backup','WARN') 

3900 return 

3901 

3902 prefix = prename 

3903 try: 

3904 existing = aflocal.getflagversionlist(printflags=False) 

3905 

3906 # remove comments from strings 

3907 existing = [x[0:x.find(' : ')] for x in existing] 

3908 i = 1 

3909 while True: 

3910 versionname = prefix + '_' + str(i) 

3911 

3912 if not versionname in existing: 

3913 break 

3914 else: 

3915 i = i + 1 

3916 

3917 time_string = str(time.strftime('%Y-%m-%d %H:%M:%S')) 

3918 

3919 casalog.post('Saving current flags to ' + versionname, 'DEBUG') 

3920 

3921 aflocal.saveflagversion(versionname=versionname, 

3922 comment='Flags autosave on ' + time_string, merge='replace') 

3923 finally: 

3924 if msfile != '': 

3925 aflocal.done() 

3926 

3927 return 

3928 

3929 

3930#### 

3931#### Set of functions to handle antenna information for shadowing. 

3932#### 

3933#### - extractAntennaInfo : Extract info into a returned dictionary (and text file) 

3934#### - antListWrite : Write the dictionary to a text file 

3935#### - antListRead : Read the text file and return a dictionary 

3936#### 

3937#### Example :  

3938#### alist = extractAntennaInfo(msname='../Data/shadowtest.ms', 

3939#### antnamelist=['VLA1','VLA15'], outfile='ttt.txt') 

3940#### 

3941#### alist = antListRead('ttt.txt'); 

3942#### 

3943################################### 

3944#### Example output text file ( 'ttt.txt' ) 

3945################################### 

3946# 

3947#name = VLA1 

3948#diameter = 25.0 

3949#position = [-1601144.961466915, -5041998.0197185818, 3554864.76811967] 

3950#name = VLA15 

3951#diameter = 25.0 

3952#position = [-1601556.245351332, -5041990.7252590274, 3554684.6464035073] 

3953# 

3954################################### 

3955#### Example output dictionary ( alist ) 

3956################################### 

3957# 

3958#{'0': {'diameter ': 25.0, 

3959# 'name ': 'VLA1', 

3960# 'position ': [-1601144.961466915, 

3961# -5041998.0197185818, 

3962# 3554864.7681196001]}, 

3963# '1': {'diameter ': 25.0, 

3964# 'name ': 'VLA15', 

3965# 'position ': [-1601556.245351332, 

3966# -5041990.7252590274, 

3967# 3554684.6464035069]}} 

3968# 

3969################################## 

3970 

3971def extractAntennaInfo(msname='', antnamelist=[], outfile=''): 

3972 """ 

3973 Function to extract antenna names, positions, and diameters 

3974 for a specified subset of the ANTENNA subtable of the specified MS. 

3975 - It writes a text file, which can be sent as input for shadow in the task 

3976 - It also returns a dictionary, which can be sent as input for shadow in the tool. 

3977 This dictionary can also be given as input in the task. 

3978  

3979 msname : name of MS 

3980 antennalist : list of strings (antenna names). Names must match exactly. Case insensitive. 

3981 outfile : name of text file. Will be overwritten if exists. If outfile='', no output file is written 

3982  

3983 Always returns a dictionary containing the same info as in the file 

3984  

3985 Example :  

3986 antinfo = extractAntennaInfo(msname='xxx.ms',antnamelist=['vla1','vla2'],outfile='out.txt'); 

3987 """ 

3988 ## Check that the MS exists 

3989 if(not os.path.exists(msname)): 

3990 casalog.post("Cannot find MS : {}".format(msname)) 

3991 return False; 

3992 

3993 ## If outfile exists, delete it 

3994 if(os.path.exists(outfile)): 

3995 casalog.post("Replacing existing file : {}".format(outfile)) 

3996 rmcmd = "rm -rf "+outfile; 

3997 os.system(rmcmd); 

3998 

3999 ## Convert input antenna names to upper-case 

4000 newants=[]; 

4001 for ants in antnamelist: 

4002 newants.append( ants.upper() ); 

4003 antnamelist = newants; 

4004 

4005 ## Read antenna subtable of input MS 

4006 tblocal.open(msname+'/ANTENNA'); 

4007 a_position = (tblocal.getcol('POSITION')).transpose(); 

4008 a_dish_diameter = tblocal.getcol('DISH_DIAMETER'); 

4009 a_name = tblocal.getcol('NAME'); 

4010 tblocal.close(); 

4011 

4012 ## Pick out only selected antennas from this list, and make a dictionary 

4013 antlist = {}; 

4014 counter=0; 

4015 for antid in range(0, len(a_name)): 

4016 if (a_name[antid]).upper() in antnamelist: 

4017 antlist[str(counter)] = { 'name':a_name[antid] , 'position': list(a_position[antid]) , 'diameter': a_dish_diameter[antid] } ; 

4018 counter=counter+1; 

4019 

4020 ## Open a new file and write this info into it, if requested 

4021 if(outfile != ''): 

4022 casalog.post("Making new file : {}".format(outfile)) 

4023 writeAntennaList(outfile, antlist); 

4024 ## always return the dictionary anyway. 

4025 return antlist; 

4026 

4027############################################## 

4028def writeAntennaList(outfile='', antlist={}): 

4029 """ 

4030 Save the antlist dictionary as a text file 

4031 """ 

4032 ofile = open(outfile, 'w'); 

4033 for apid in sorted(antlist): 

4034 apars = antlist[apid]; 

4035 ofile.write("name=" + str(apars['name']) + '\n'); 

4036 ofile.write("diameter=" + str(apars['diameter'])+'\n'); 

4037 ofile.write("position=" + str((apars['position']))+'\n'); 

4038 ofile.close(); 

4039 

4040############################################## 

4041def readAntennaList(infile=''): 

4042 """ 

4043 Read the antlist text file and return a dictionary 

4044  

4045 A return value of empty {} indicates an error (or, empty file). 

4046  

4047 The file needs to have 3 entries per antenna, on separate lines. 

4048 The diameter and position are in units of meters, with positions in ITRF. 

4049 Multiple antennas can be specified by repeating these three lines. 

4050 Blank lines are allowed. 

4051 Lines can be commented with '#' as the first character. 

4052  

4053 Example :  

4054 name = ea05 

4055 diameter = 25.0 

4056 position = [-1601144.96146691, -5041998.01971858, 3554864.76811967] 

4057  

4058 """ 

4059 

4060 if (type(infile) == str) & os.path.exists(infile): 

4061 try: 

4062 ifile = open(infile,'r'); 

4063 except: 

4064 raise Exception('Error opening file ' + infile) 

4065 

4066 thelist = ifile.readlines(); 

4067 ifile.close(); 

4068 else: 

4069 raise Exception( \ 

4070 'File %s not found - please verify the name'%infile) 

4071 

4072 cleanlist=[]; 

4073 for aline in thelist: 

4074 if(len(aline)>5 and aline[0] != '#'): 

4075 cleanlist.append(aline.rstrip()); 

4076 

4077 #casalog.post('Found ' + str(len(cleanlist)) + ' valid lines out of ' + str(len(thelist));) 

4078 

4079 if( len(cleanlist) > 0 and len(cleanlist) % 3 != 0 ): 

4080 casalog.post("\nThe file needs to have 3 entries per antenna, on separate lines. For example :") 

4081 casalog.post("name=ea05") 

4082 casalog.post("diameter=25.0") 

4083 casalog.post("position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]") 

4084 casalog.post("\n") 

4085 casalog.post("The diameter and position are in units of meters, with positions in ITRF") 

4086 return False; 

4087 

4088 antlist={}; 

4089 counter=0; 

4090 for aline in range(0,len(cleanlist),3): 

4091 antdict = {}; 

4092 for row in range(0,3): 

4093 pars = cleanlist[aline+row].split("=",1); 

4094 #casalog.post(aline, row, pars) 

4095 if(len(pars) != 2): 

4096 casalog.post('Error in parsing : ' + cleanlist[aline+row], 'ERROR') 

4097 return {}; 

4098 else: 

4099 if(pars[0].count('name') > 0 ): 

4100 antdict[pars[0].rstrip()] = str(pars[1].rsplit()[0]); 

4101 if(pars[0].count('diameter') > 0 ): 

4102 antdict[pars[0].rstrip()] = float(pars[1]); 

4103 if(pars[0].count('position') > 0 ): 

4104 plist = pars[1][1:-2].replace('[','').split(','); 

4105 if(len(plist) != 3): 

4106 casalog.post('Error in parsing : ' + cleanlist[aline+row], 

4107 'ERROR') 

4108 return {}; 

4109 else: 

4110 qlist=[]; 

4111 for ind in range(0,3): 

4112 qlist.append(float(plist[ind])); 

4113 antdict[pars[0].rstrip()] = qlist; 

4114 antlist[str(counter)] = antdict; 

4115 counter = counter+1; 

4116 

4117 return antlist; 

4118 

4119################################################ 

4120# 

4121# Function to pull out RFLAG thresholds from the returned report dictionary. 

4122# 

4123def writeRFlagThresholdFile(rflag_thresholds={},timedevfile='', freqdevfile='',agent_id=0): 

4124 """ 

4125 Extract the RFLAG output thresholds from the threshold dictionary 

4126 Return them as arrays, and optionally, write them into a file. 

4127 """ 

4128 # Decide the output file name. 

4129 if( type(timedevfile) == str and timedevfile != '' and timedevfile.count('[')==0 and (not timedevfile.replace('.','').isdigit() ) ): 

4130 toutfile = timedevfile 

4131 else: 

4132 toutfile = 'rflag_output_thresholds_timedev'+str(agent_id)+'.txt' 

4133 

4134 # Decide the output file name. 

4135 if( type(freqdevfile) == str and freqdevfile != '' and freqdevfile.count('[')==0 and (not freqdevfile.replace('.','').isdigit() ) ): 

4136 foutfile = freqdevfile 

4137 else: 

4138 foutfile = 'rflag_output_thresholds_freqdev'+str(agent_id)+'.txt' 

4139 

4140 # save rflag output in file, and print them everywhere. 

4141 casalog.post("Saving RFlag_"+str(agent_id)+" output in : " + toutfile + " and " + foutfile, 'INFO') 

4142 

4143 ofiletime = open(toutfile, 'w'); 

4144 ofilefreq = open(foutfile, 'w'); 

4145 # Construct dictionary from what needs to be stored. 

4146 timedict = {'name':rflag_thresholds['name'] , 'timedev': (rflag_thresholds['timedev']).tolist()} 

4147 freqdict = {'name':rflag_thresholds['name'] , 'freqdev': (rflag_thresholds['freqdev']).tolist()} 

4148 timestr = convertDictToString(timedict) 

4149 freqstr = convertDictToString(freqdict) 

4150 # Write to file 

4151 ofiletime.write(timestr + '\n'); 

4152 ofilefreq.write(freqstr + '\n'); 

4153 # Send to logger 

4154 casalog.post("RFlag_"+str(agent_id)+" output timedev written to " + toutfile + " : " + timestr, 'INFO'); 

4155 casalog.post("RFlag_"+str(agent_id)+" output freqdev written to " + foutfile + " : " + freqstr, 'INFO'); 

4156 # End filed 

4157 ofiletime.write('\n'); 

4158 ofiletime.close(); 

4159 ofilefreq.write('\n'); 

4160 ofilefreq.close(); 

4161 # Returne timedev, freqdev contents. This is for savepars later. 

4162 return timedict['timedev'], freqdict['freqdev'] 

4163 

4164############################################## 

4165def readRFlagThresholdFile(infile='',inkey=''): 

4166 """ 

4167 Read the input RFlag threshold file, and return dictionaries. 

4168 """ 

4169 if(infile==''): 

4170 return []; 

4171 

4172 if ( not os.path.exists(infile) ): 

4173 casalog.post('Cannot find file : ' + infile) 

4174 return []; 

4175 

4176 ifile = open(infile,'r'); 

4177 thelist = ifile.readlines(); 

4178 ifile.close(); 

4179 

4180 cleanlist=[]; 

4181 for aline in thelist: 

4182 if(len(aline)>5 and aline[0] != '#'): 

4183 cleanlist.append(aline.rstrip().rstrip('\n')) 

4184 

4185# threshlist={}; 

4186 threshlist = OrderedDict() 

4187 for aline in range(0,len(cleanlist)): 

4188 threshlist[str(aline)] = convertStringToDict(cleanlist[aline]); 

4189 if inkey in threshlist[str(aline)]: 

4190 devthreshold = threshlist[str(aline)][inkey] 

4191 

4192 # return only the last one. There should be only one anyway. 

4193 return devthreshold 

4194 

4195############################################## 

4196## Note - replace all arrays by lists before coming here. 

4197def convertDictToString(indict={}): 

4198 '''Convert dictionary to string''' 

4199 thestr = str(indict); 

4200 # Remove newlines and spaces from this string. 

4201 thestr = thestr.replace('\n',''); 

4202 thestr = thestr.replace(' ',''); 

4203 return thestr; 

4204 

4205############################################## 

4206def convertStringToDict(instr=''): 

4207 '''Convert string to dictionary''' 

4208 instr = instr.replace('\n',''); 

4209 try: 

4210 thedict = OrderedDict() 

4211 thedict = ast.literal_eval(instr) 

4212 except Exception as instance: 

4213 casalog.post("*** Error converting string %s to dictionary : \'%s\'" % (instr,instance),'ERROR'); 

4214 return thedict; 

4215############################################## 

4216 

4217def extractRFlagOutputFromSummary(mode,summary_stats_list, flagcmd): 

4218 """ 

4219 Function to pull out 'rflag' output from the long dictionary, and  

4220 (1) write the output files with thresholds. If the filename is specified, use it. 

4221 If filename is not specified, make one up. 

4222 (2) modify entries in 'cmdline' so that it is ready for savepars.  

4223 This is to ensure that 'savepars' saves the contents of the threshold-files 

4224 and not just the file-names. It has to save it in the form that flagdata  

4225 accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user 

4226 need not keep track of threshold text files if they use 'savepars' with action='apply'. 

4227 """ 

4228 if type(summary_stats_list) is dict: 

4229 nreps = summary_stats_list['nreport'] 

4230 for rep in range(0,nreps): 

4231 repname = 'report'+str(rep) 

4232 if summary_stats_list[repname]['type'] == "rflag": 

4233 # Pull out the rflag threshold dictionary. This has a 'name' in it. 

4234 rflag_thresholds = summary_stats_list[repname] 

4235 ##casalog.post(rflag_thresholds) 

4236 # Get the rflag id, to later construct a 'name' from to match the above. 

4237 rflagid = 0 

4238 if mode=='list': 

4239 rflagid = int( rflag_thresholds['name'].replace('Rflag_','') ) 

4240 # Go through the flagcmd list, to find the 'rflags'..... 

4241 for key in flagcmd.keys(): 

4242 cmdline = flagcmd[key]['command']; 

4243 if cmdline.__contains__('rflag'): 

4244 # Check for match between input flagcmd and output threshold, via the rflag id 

4245 if(key==rflagid): 

4246 # If timedev,freqdev are missing from cmdline, add empty ones. 

4247 if( not cmdline.__contains__('timedev=') ): # aah. don't confuse it with timedevscale 

4248 cmdline = cmdline + " timedev=[] "; 

4249 if( not cmdline.__contains__('freqdev=') ): 

4250 cmdline = cmdline + " freqdev=[] "; 

4251 # Pull out timedev, freqdev strings from flagcmd 

4252 rflagpars = getLinePars(cmdline , ['timedev','freqdev','writeflags']); 

4253 ##casalog.post("cmdline : "+cmdline) 

4254 # Write RFlag thresholds to these file names.  

4255 newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, rflagpars['timedev'], rflagpars['freqdev'], rflagid) 

4256 ## Modify the flagcmd string, so that savepars sees the contents of the file 

4257 if( rflagpars['timedev'].__contains__('[') ): 

4258 oldstring = 'timedev='+str(rflagpars['timedev']) 

4259 newstring = 'timedev='+str(newtimedev).replace(' ','') 

4260 ##casalog.post( "time : replacing " + oldstring + newstring) 

4261 cmdline = cmdline.replace( oldstring, newstring ); 

4262 if( rflagpars['freqdev'].__contains__('[') ): 

4263 oldstring = 'freqdev='+str(rflagpars['freqdev']) 

4264 newstring = 'freqdev='+str(newfreqdev).replace(' ','') 

4265 ##casalog.post("freq : replacing " + oldstring + newstring) 

4266 cmdline = cmdline.replace( oldstring, newstring ); 

4267 # Remove writeflags from the cmd to prevent it from going into savepars 

4268 oldstring = 'writeflags='+str(rflagpars['writeflags']) 

4269 cmdline = cmdline.replace( oldstring, "" ); 

4270 

4271 flagcmd[key]['command'] = cmdline; 

4272 

4273# Only used in writeFlagCmd 

4274def evalString(cmdline): 

4275 '''Evaluate the correct types in a string with parameters''' 

4276 

4277# newcmdline = "" 

4278 cmddict = OrderedDict() 

4279 

4280 try: 

4281 pairs = cmdline.split() 

4282 except: 

4283 raise "Whitespace in parameter values is not yet supported" 

4284 

4285 for kv in pairs: 

4286 newval = None 

4287 (key,val) = kv.split('=',1) 

4288 

4289 # Selection parameters are always string 

4290 if key == 'spw' or key == 'scan' or key == 'field' or \ 

4291 key == 'antenna' or key == 'correlation' or key == 'array' or \ 

4292 key == 'timerange' or key == 'uvrange' or key == 'feed': 

4293 

4294 newval = str(val) 

4295 cmddict[key] = newval 

4296 continue 

4297 

4298 if val.startswith('['): 

4299 newval = ast.literal_eval(val) 

4300 elif val.startswith('{'): 

4301 newval = eval(val) 

4302 elif val == 'True' or val == 'False': 

4303 if eval(val) == True or eval(val) == False: 

4304 newval = bool(val) 

4305 

4306 if newval == None: 

4307 try: 

4308 int(val) 

4309 newval = int(val) 

4310 except: 

4311 # string 

4312 newval = str(val) 

4313 if newval == None: 

4314 try: 

4315 float(val) 

4316 newval = float(val) 

4317 except: 

4318 # string 

4319 newval = str(val) 

4320 

4321 cmddict[key] = newval 

4322 

4323 return cmddict 

4324 

4325 

4326 

4327 

4328 

4329