Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_flagdata.py: 74%

437 statements  

« prev     ^ index     » next       coverage.py v7.6.4, created at 2024-11-01 07:19 +0000

1import time 

2import os 

3import sys 

4import copy 

5import pprint 

6import numpy as np 

7 

8from . import flaghelper as fh 

9from .mstools import write_history 

10from .parallel.parallel_task_helper import ParallelTaskHelper 

11from .parallel.parallel_data_helper import ParallelDataHelper 

12# this should be replaced when CASA really moves to Python 2.7 

13from collections import OrderedDict 

14from casatasks import casalog 

15from casatools import ms, agentflagger, quanta, table 

16 

17localqa = quanta() 

18 

19debug = False 

20 

21# Helper class 

22class FlagHelper(ParallelTaskHelper): 

23 def __init__(self, args={}): 

24 self.__args = args 

25 

26 def setupInputFile(self, parname): 

27 '''Create a temporary input file with 

28 absolute pathnames for other input files 

29 such as addantenna, timedev, freqdev. 

30 The temporary input file will have the same 

31 name with an extension .tmp''' 

32 

33 newpar = None 

34 if isinstance(parname, str) and parname != '': 

35 newpar = fh.addAbsolutePath(parname) 

36 elif isinstance(parname, list) and os.path.isfile(parname[0]): 

37 newpar = [] 

38 for i in range(len(parname)): 

39 newpar.append(fh.addAbsolutePath(parname[i])) 

40 

41 return newpar 

42 

43 def setupCluster(self, thistask=''): 

44 '''Get a cluster to execute this task''' 

45 if thistask == '': 

46 thistask = 'flagdata' 

47 

48 ParallelTaskHelper.__init__(self, task_name=thistask, args=self.__args) 

49 

50 def setupRflag(self, devpar): 

51 '''cast rflag's list parameters from numpy types to Python types 

52 devpar --> list of numeric parameters or list of list 

53 such as timedev or freqdev''' 

54 

55 nt = copy.deepcopy(devpar) 

56 for i in range(len(nt)): 

57 if (isinstance(nt[i],list)): 

58 nnt = nt[i] 

59 for j in range(len(nnt)): 

60 elem = fh.evaluateNumpyType(nnt[j]) 

61 # write the casted element back  

62 devpar[i][j] = elem 

63 

64 

65# The flagdata task 

66def flagdata(vis, 

67 mode, 

68 autocorr, # mode manual parameter 

69 inpfile, # mode list parameters 

70 reason, 

71 tbuff, 

72 spw, # data selection parameters 

73 field, 

74 antenna, 

75 uvrange, 

76 timerange, 

77 correlation, 

78 scan, 

79 intent, 

80 array, 

81 observation, 

82 feed, 

83 clipminmax, # mode clip parameters 

84 datacolumn, 

85 clipoutside, 

86 channelavg, 

87 chanbin, 

88 timeavg, 

89 timebin, 

90 clipzeros, 

91 quackinterval, # mode quack parameters 

92 quackmode, 

93 quackincrement, 

94 tolerance, # mode shadow parameter 

95 addantenna, 

96 lowerlimit, # mode elevation parameters 

97 upperlimit, 

98 ntime, # mode tfcrop 

99 combinescans, 

100 timecutoff, 

101 freqcutoff, 

102 timefit, 

103 freqfit, 

104 maxnpieces, 

105 flagdimension, 

106 usewindowstats, 

107 halfwin, 

108 extendflags, 

109 winsize, # rflag parameters 

110 timedev, 

111 freqdev, 

112 timedevscale, 

113 freqdevscale, 

114 spectralmax, 

115 spectralmin, 

116 antint_ref_antenna, # mode antint 

117 minchanfrac, 

118 verbose, 

119 extendpols, # mode extend 

120 growtime, 

121 growfreq, 

122 growaround, 

123 flagneartime, 

124 flagnearfreq, 

125 minrel, # mode summary 

126 maxrel, 

127 minabs, 

128 maxabs, 

129 spwchan, 

130 spwcorr, 

131 basecnt, 

132 fieldcnt, 

133 name, 

134 action, # run or not the tool 

135 display, 

136 flagbackup, 

137 savepars, # save the current parameters to FLAG_CMD or to a file 

138 cmdreason, # reason to save to flag cmd 

139 outfile, # output file to save flag commands 

140 overwrite, # overwrite the outfile file 

141 writeflags # HIDDEN parameter 

142 ): 

143 

144 

145 # 

146 # Task flagdata 

147 # Flags data from an MS or calibration table based on data selection in various ways 

148 

149 casalog.origin('flagdata') 

150 

151 if (action == 'none' or action=='' or action=='calculate'): 

152 flagbackup = False 

153 

154 # SMC: moved the flagbackup to before initializing the cluster. 

155 # Note that with this change, a flag backup will be created even if 

156 # an error happens that prevents the flagger tool from running.  

157 if (mode != 'summary' and flagbackup): 

158 casalog.post('Backup original flags before applying new flags') 

159 fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata') 

160 # Set flagbackup to False because only the controller 

161 # should create a backup 

162 flagbackup = False 

163 

164 # Initialize the helper class 

165 orig_locals = copy.deepcopy(locals()) 

166 FHelper = FlagHelper() 

167 

168 # Check if vis is a MS, MMS or cal table: 

169 # typevis = 1 --> cal table 

170 # typevis = 0 --> MS 

171 # typevis = 2 --> MMS 

172 iscal = False 

173 typevis = fh.isCalTable(vis) 

174 if typevis == 1: 

175 iscal = True 

176 

177 

178 # ***************** Input is MMS -- Parallel Processing *********************** 

179 if typevis == 2 and ParallelDataHelper.isMMSAndNotServer(vis) and\ 

180 action != '' and action != 'none': 

181 

182 # Create a temporary input file with .tmp extension. 

183 # Use this file for all the processing from now on. 

184 if (isinstance(inpfile,str) and inpfile != '') or \ 

185 (isinstance(inpfile, list) and os.path.isfile(inpfile[0])): 

186 inpfile = FHelper.setupInputFile(inpfile) 

187 if inpfile != None: 

188 orig_locals['inpfile'] = inpfile 

189 

190 if outfile != '': 

191 outfile = os.path.abspath(outfile) 

192 orig_locals['outfile'] = outfile 

193 if isinstance(addantenna, str) and addantenna != '': 

194 addantenna = os.path.abspath(addantenna) 

195 orig_locals['addantenna'] = addantenna 

196 if isinstance(timedev, str) and timedev != '': 

197 timedev = os.path.abspath(timedev) 

198 orig_locals['timedev'] = timedev 

199 if isinstance(freqdev, str) and freqdev != '': 

200 freqdev = os.path.abspath(freqdev) 

201 orig_locals['freqdev'] = freqdev 

202 # Conversion np.array->list if needed. np.array can be a problem with MMS, as it 

203 # would require in the servers an import of np.array with the same name used by the 

204 # caller 

205 if isinstance(timedev, np.ndarray): 

206 orig_locals['timedev'] = timedev.tolist() 

207 if isinstance(freqdev, np.ndarray): 

208 orig_locals['freqdev'] = freqdev.tolist() 

209 

210 FHelper.__init__(orig_locals) 

211 

212 # For tests only 

213 # FHelper.bypassParallelProcessing(1) 

214 

215 FHelper.setupCluster('flagdata') 

216 # (CAS-4119): Override summary minabs,maxabs,minrel,maxrel  

217 # so that it is done after consolidating the summaries 

218 

219 # By-pass options to filter summary 

220 filterSummary = False 

221 if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or (minabs != 0) or (maxabs != -1))): 

222 filterSummary = True 

223 

224 myms = ms() 

225 myms.open(vis) 

226 subMS_list = myms.getreferencedtables() 

227 myms.close() 

228 

229 if (minrel != 0.0): 

230 minreal_dict = create_arg_dict(subMS_list,0.0) 

231 FHelper.override_arg('minrel',minreal_dict) 

232 if (maxrel != 1.0): 

233 maxrel_dict = create_arg_dict(subMS_list,1.0) 

234 FHelper.override_arg('maxrel',maxrel_dict) 

235 if (minabs != 0): 

236 minabs_dict = create_arg_dict(subMS_list,0) 

237 FHelper.override_arg('minabs',minabs_dict) 

238 if (maxabs != -1): 

239 maxabs_dict = create_arg_dict(subMS_list,-1) 

240 FHelper.override_arg('maxabs',maxabs_dict) 

241 

242 # By-pass options to filter summary 

243 if savepars: 

244 

245 myms = ms() 

246 myms.open(vis) 

247 subMS_list = myms.getreferencedtables() 

248 myms.close() 

249 

250 savepars_dict = create_arg_dict(subMS_list,False) 

251 FHelper.override_arg('savepars',savepars_dict) 

252 

253 # Execute the parallel engines 

254 retVar = FHelper.go() 

255 

256 # Save overall RFlag values into timedev/freqdev files once the subMS results have 

257 # been consolidated, if needed for RFlag (introduced in CAS-11850) 

258 opts_dict = {'writeflags': writeflags, 'timedev': timedev, 'freqdev': freqdev} 

259 fh.save_rflag_consolidated_files(mode, action, retVar, opts_dict, inpfile) 

260 

261 # In async_mode return the job ids 

262 if ParallelTaskHelper.getAsyncMode(): 

263 return retVar 

264 else: 

265 # Filter summary at MMS level 

266 if (mode == 'summary'): 

267 if filterSummary: 

268 retVar = filter_summary(retVar,minrel,maxrel,minabs,maxabs) 

269 return retVar 

270 # Save parameters at MMS level 

271 elif savepars: 

272 action = 'none' 

273 else: 

274 return retVar 

275 

276 summary_stats={}; 

277 

278 

279 # ***************** Input is a normal MS/cal table **************** 

280 

281 # Create local tools 

282 aflocal = agentflagger() 

283 mslocal = ms() 

284 

285 # Verify the ntime value 

286 newtime = 0.0 

287 if type(ntime) == float or type(ntime) == int: 

288 if ntime <= 0: 

289 raise ValueError('Parameter ntime cannot be < = 0') 

290 else: 

291 # units are seconds 

292 newtime = float(ntime) 

293 

294 elif type(ntime) == str: 

295 if ntime == 'scan': 

296 # iteration time step is a scan 

297 newtime = 0.0 

298 else: 

299 # read the units from the string 

300 qtime = localqa.quantity(ntime) 

301 

302 if qtime['unit'] == 'min': 

303 # convert to seconds 

304 qtime = localqa.convert(qtime, 's') 

305 elif qtime['unit'] == '': 

306 qtime['unit'] = 's' 

307 

308 # check units 

309 if qtime['unit'] == 's': 

310 newtime = qtime['value'] 

311 else: 

312 casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN') 

313 

314 casalog.post("New ntime is of type %s and value %s"%(type(newtime),newtime), 'DEBUG') 

315 

316 try: 

317 # Open the MS and attach it to the tool 

318 if ((type(vis) == str) & (os.path.exists(vis))): 

319 aflocal.open(vis, newtime) 

320 else: 

321 raise ValueError('Visibility data set not found - please verify the name') 

322 

323 

324 # Get the parameters for the mode 

325 agent_pars = {} 

326 

327 # By default, write flags to the MS 

328 writeflags = True 

329 

330 # Only the apply action writes to the MS 

331 # action=apply --> write to the MS 

332 # action=calculate --> do not write to the MS 

333 # action='' --> do not run the tool and do not write to the MS 

334 if action != 'apply': 

335 writeflags = False 

336 

337 # Default mode 

338 if mode == '' or mode == 'manualflag': 

339 mode = 'manual' 

340 

341 # Read in the list of commands 

342 # Make a dictionary of the input commands. Select by reason if requested 

343 flagcmd = {} 

344 

345 if mode == 'list': 

346 casalog.post('List mode is active') 

347 doPadding = True 

348 try: 

349 # If tbuff is requested, read and Parse 

350 if tbuff == 0.0 or tbuff == [] or tbuff == None: 

351 doPadding = False 

352 

353 if doPadding: 

354 casalog.post('Will apply time buffer padding') 

355 

356 # inpfile is a file 

357 if isinstance(inpfile, str): 

358 inpfile = [inpfile] 

359 

360 # read in the list and do a simple parsing to apply tbuff 

361 flaglist = fh.readAndParse(inpfile, tbuff) 

362 

363 else: 

364 flaglist = fh.get_flag_cmd_list(inpfile) 

365 

366 

367 # Parse and create a dictionary 

368 flagcmd = fh.parseDictionary(flaglist, reason) 

369 

370 # Validate the dictionary. 

371 # IMPORTANT: if any parameter changes its type, the following 

372 # function needs to be updated. The same if any new parameter is 

373 # added or removed from the task 

374 fh.evaluateFlagParameters(flagcmd,orig_locals) 

375 

376 # List of flag commands in dictionary 

377 vrows = flagcmd.keys() 

378 

379 casalog.post('%s'%flagcmd,'DEBUG1') 

380 

381 

382 except Exception as instance: 

383 casalog.post('%s'%instance,'ERROR') 

384 raise ValueError('Error reading the input list. Make sure the syntax used in ' 

385 'the list follows the rules given in the inline help of the ' 

386 'task.') 

387 

388 casalog.post('Selected ' + str(vrows.__len__()) 

389 + ' commands from combined input list(s) ') 

390 

391 elif mode == 'manual': 

392 agent_pars['autocorr'] = autocorr 

393 casalog.post('Manual mode is active') 

394 

395 

396 elif mode == 'clip': 

397 

398 agent_pars['datacolumn'] = datacolumn.upper() 

399 agent_pars['clipoutside'] = clipoutside 

400 agent_pars['channelavg'] = channelavg 

401 agent_pars['chanbin'] = chanbin 

402 agent_pars['timeavg'] = timeavg 

403 agent_pars['timebin'] = timebin 

404 agent_pars['clipzeros'] = clipzeros 

405 

406 

407 if type(clipminmax) != list: 

408 casalog.post('Error : clipminmax must be a list : [min,max]', 'ERROR') 

409 # If clipminmax = [], do not write it in the dictionary. 

410 # It will be handled by the framework to flag NaNs only 

411 if clipminmax.__len__() == 2: 

412 # Cast to float to avoid the missing decimal point 

413 clipmin = float(clipminmax[0]) 

414 clipmax = float(clipminmax[1]) 

415 clipminmax = [] 

416 clipminmax.append(clipmin) 

417 clipminmax.append(clipmax) 

418 agent_pars['clipminmax'] = clipminmax 

419 

420 casalog.post('Clip mode is active') 

421 

422 elif mode == 'shadow': 

423 

424 agent_pars['tolerance'] = tolerance 

425 

426 if type(addantenna) == str: 

427 if addantenna != '': 

428 # it's a filename, create a dictionary 

429 antdict = fh.readAntennaList(addantenna) 

430 agent_pars['addantenna'] = antdict 

431 

432 elif type(addantenna) == dict: 

433 if addantenna != {}: 

434 agent_pars['addantenna'] = addantenna 

435 

436 casalog.post('Shadow mode is active') 

437 

438 elif mode == 'quack': 

439 agent_pars['quackmode'] = quackmode 

440 agent_pars['quackinterval'] = quackinterval 

441 agent_pars['quackincrement'] = quackincrement 

442 casalog.post('Quack mode is active') 

443 

444 

445 elif mode == 'elevation': 

446 

447 agent_pars['lowerlimit'] = lowerlimit 

448 agent_pars['upperlimit'] = upperlimit 

449 casalog.post('Elevation mode is active') 

450 

451 

452 elif mode == 'tfcrop': 

453 

454 agent_pars['ntime'] = newtime 

455 agent_pars['combinescans'] = combinescans 

456 agent_pars['datacolumn'] = datacolumn.upper() 

457 agent_pars['timecutoff'] = timecutoff 

458 agent_pars['freqcutoff'] = freqcutoff 

459 agent_pars['timefit'] = timefit 

460 agent_pars['freqfit'] = freqfit 

461 agent_pars['maxnpieces'] = maxnpieces 

462 agent_pars['flagdimension'] = flagdimension 

463 agent_pars['usewindowstats'] = usewindowstats 

464 agent_pars['halfwin'] = halfwin 

465 agent_pars['extendflags'] = bool(extendflags) 

466 agent_pars['channelavg'] = channelavg 

467 agent_pars['chanbin'] = chanbin 

468 agent_pars['timeavg'] = timeavg 

469 agent_pars['timebin'] = timebin 

470 casalog.post('Time and Frequency (tfcrop) mode is active') 

471 

472 

473 elif mode == 'rflag': 

474 if newtime != 0.0: 

475 # this means ntime='scan', the default 

476 agent_pars['ntime'] = newtime 

477 

478 agent_pars['combinescans'] = combinescans 

479 agent_pars['datacolumn'] = datacolumn.upper() 

480 agent_pars['winsize'] = winsize 

481 agent_pars['timedevscale'] = timedevscale 

482 agent_pars['freqdevscale'] = freqdevscale 

483 agent_pars['spectralmax'] = spectralmax 

484 agent_pars['spectralmin'] = spectralmin 

485 agent_pars['extendflags'] = bool(extendflags) 

486 

487 # These can be double, doubleArray, or string. 

488 # writeflags=False : calculate and return thresholds. 

489 # writeflags=True : use given thresholds for this run. 

490 if writeflags == True: 

491 if isinstance(timedev, str): 

492 timedev = fh.readRFlagThresholdFile(timedev,'timedev') 

493 if isinstance(freqdev, str): 

494 freqdev = fh.readRFlagThresholdFile(freqdev,'freqdev') 

495 

496 agent_pars['timedev'] = timedev 

497 agent_pars['freqdev'] = freqdev 

498 

499 agent_pars['writeflags'] = writeflags 

500 agent_pars['display'] = display 

501 

502 agent_pars['channelavg'] = channelavg 

503 agent_pars['chanbin'] = chanbin 

504 agent_pars['timeavg'] = timeavg 

505 agent_pars['timebin'] = timebin 

506 

507 casalog.post('Rflag mode is active') 

508 

509 elif mode == 'antint': 

510 agent_pars['antint_ref_antenna'] = antint_ref_antenna 

511 agent_pars['minchanfrac'] = minchanfrac 

512 agent_pars['verbose'] = verbose 

513 

514 elif mode == 'extend': 

515 agent_pars['ntime'] = newtime 

516 agent_pars['combinescans'] = combinescans 

517 agent_pars['extendpols'] = extendpols 

518 agent_pars['growtime'] = growtime 

519 agent_pars['growfreq'] = growfreq 

520 agent_pars['growaround'] = growaround 

521 agent_pars['flagneartime'] = flagneartime 

522 agent_pars['flagnearfreq'] = flagnearfreq 

523 casalog.post('Extend mode is active') 

524 

525 

526 elif mode == 'unflag': 

527 casalog.post('Unflag mode is active') 

528 

529 elif mode == 'summary': 

530 agent_pars['spwchan'] = spwchan 

531 agent_pars['spwcorr'] = spwcorr 

532 agent_pars['basecnt'] = basecnt 

533 agent_pars['fieldcnt'] = fieldcnt 

534 agent_pars['name'] = name 

535 agent_pars['display'] = display 

536 

537 # Disable writeflags and savepars 

538 writeflags = False 

539 savepars = False 

540 casalog.post('Summary mode is active') 

541 

542 

543 # Add the mode to the agent's parameters 

544 agent_pars['mode'] = mode 

545 

546 # Correlation does not go in selectdata, but in the agent's parameters 

547 if correlation != '': 

548 agent_pars['correlation'] = correlation.upper() 

549 

550 # Create a flagcmd dictionary of the interface parameters for saving 

551 if mode != 'list' and mode != 'summary': 

552 # Create a dictionary of the selection parameters 

553 seldic = {} 

554 seldic['field'] = field 

555 seldic['spw'] = spw 

556 seldic['array'] = array 

557 seldic['feed'] = feed 

558 seldic['scan'] = scan 

559 seldic['antenna'] = antenna 

560 seldic['uvrange'] = uvrange 

561 seldic['timerange'] = timerange 

562 seldic['intent'] = intent 

563 seldic['observation'] = str(observation) 

564 

565 # Add the agent's parameters 

566 seldic.update(agent_pars) 

567 

568 tempdict = copy.deepcopy(seldic) 

569 # Remove the empty parameters 

570 for k,v in seldic.items(): 

571 if isinstance(v, str) and v == '': 

572 tempdict.pop(k) 

573 

574 cmddict = {'command':tempdict} 

575 cmddict['reason'] = '' 

576 cmddict['applied'] = False 

577 flagcmd[0] = cmddict 

578 

579 # Number of commands in dictionary 

580 vrows = flagcmd.keys() 

581 casalog.post('There are %s cmds in dictionary of mode %s'%(vrows.__len__(),mode),'DEBUG1') 

582 

583 modified_flagcmd = flagcmd 

584 

585 # Setup global parameters in the agent's dictionary 

586 apply = True 

587 

588 # Hold the name of the agent 

589 agent_name = mode.capitalize() 

590 agent_pars['agentname'] = agent_name 

591 agent_pars['apply'] = apply 

592 

593 ########## Only save the parameters and exit; action = '' 

594 if action == '' or action == 'none': 

595 if savepars == False: 

596 casalog.post('Parameter action=\'\' is only meaningful with savepars=True.', 'WARN') 

597 aflocal.done() 

598 return summary_stats 

599 

600 else: 

601 if iscal and outfile == '': 

602 casalog.post('Saving to FLAG_CMD is not supported for cal tables', 'WARN') 

603 

604 if not overwrite and os.path.exists(outfile): 

605 raise ValueError('You have set overwrite to False. Remove %s before saving the flag commands'%outfile) 

606 

607 else: 

608 fh.writeFlagCommands(vis, flagcmd, writeflags, cmdreason, outfile, False) 

609 

610 aflocal.done() 

611 return summary_stats 

612 

613 

614 ######### From now on it is assumed that action = apply or calculate 

615 

616 # Select the data and parse the agent's parameters 

617 if mode != 'list': 

618 aflocal.selectdata(field=field, spw=spw, array=array, feed=feed, scan=scan, \ 

619 antenna=antenna, uvrange=uvrange, timerange=timerange, \ 

620 intent=intent, observation=str(observation)) 

621 

622 # CAS-3959 Handle channel selection at the FlagAgent level 

623 agent_pars['spw'] = spw 

624 casalog.post('Parsing the parameters for %s mode'%mode, 'DEBUG1') 

625 if (not aflocal.parseagentparameters(agent_pars)): 

626 # casalog.post('Failed to parse parameters for mode %s' %mode, 'ERROR') 

627 raise ValueError('Failed to parse parameters for mode %s' %mode) 

628 

629 casalog.post('%s'%agent_pars, 'DEBUG') 

630 

631 else: 

632 # Select a loose union of the data selection from the list 

633 # The loose union will be calculated for field and spw only; 

634 # antenna, correlation and timerange should be handled by the agent 

635 if vrows.__len__() == 0: 

636 raise ValueError('There are no valid commands in list') 

637 

638 unionpars = {} 

639 

640 if vrows.__len__() == 1: 

641 unionpars = fh.parseSelectionPars(flagcmd[0]['command']) 

642 casalog.post('The selected subset of the MS will be: '); 

643 casalog.post('%s'%unionpars); 

644 

645 aflocal.selectdata(unionpars); 

646 

647 # Parse the parameters for each agent in the list 

648 # Get the returned modified dictionary of flag commands which 

649 # is needed by the rflag agent, when present in a list 

650 modified_flagcmd = fh.parseAgents(aflocal, flagcmd, [], apply, writeflags, display) 

651 

652 # Do display if requested 

653 if display != '': 

654 

655 agent_pars = {} 

656 casalog.post('Parsing the display parameters') 

657 

658 agent_pars['mode'] = 'display' 

659 # need to create different parameters for both, data and report. 

660 if display == 'both': 

661 agent_pars['datadisplay'] = True 

662 agent_pars['reportdisplay'] = True 

663 

664 elif display == 'data': 

665 agent_pars['datadisplay'] = True 

666 

667 elif display == 'report': 

668 agent_pars['reportdisplay'] = True 

669 

670 # CAS-3966 Add datacolumn to display agent parameters 

671 # Check if FLOAT_DATA exist instead 

672 tblocal = table() 

673 tblocal.open(vis) 

674 allcols = tblocal.colnames() 

675 tblocal.close() 

676 float_data = 'FLOAT_DATA' in allcols 

677 if datacolumn.upper() == 'DATA' and float_data == True: 

678 datacolumn = 'FLOAT_DATA' 

679 

680 agent_pars['datacolumn'] = datacolumn.upper() 

681 ret = aflocal.parseagentparameters(agent_pars) 

682 if not ret: 

683 casalog.post('Unable to load the display. Please check the datacolumn','ERROR') 

684 

685 # Disable saving the parameters to avoid inconsistencies 

686 if savepars: 

687 casalog.post('Disabling savepars for the display', 'WARN') 

688 savepars = False 

689 

690 # Initialize the agents 

691 casalog.post('Initializing the agents') 

692 aflocal.init() 

693 

694 # Run the tool 

695 casalog.post('Running the agentflagger tool') 

696 summary_stats_list = aflocal.run(writeflags, True) 

697 

698 # Inform the user when flags are not written to the MS 

699 if not writeflags: 

700 casalog.post("Flags are not written to the MS. (action=\'calculate\')") 

701 

702 

703 # Now, deal with all the modes that return output. 

704 # Rflag : There can be many 'rflags' in list mode. 

705 

706 ## Pull out RFlag outputs. There will be outputs only if writeflags=False 

707 if (mode == 'rflag' or mode== 'list') and (writeflags==False): 

708 casalog.post('Saving RFlag return dictionary: {0}'. 

709 format(pprint.pformat(summary_stats_list)), 'INFO') 

710 fh.parseRFlagOutputFromSummary(mode, summary_stats_list, modified_flagcmd) 

711 

712 

713 # Save the current parameters/list to FLAG_CMD or to output 

714 if savepars: 

715 if not overwrite and os.path.exists(outfile): 

716 raise RuntimeError('You have set overwrite to False. Remove %s before saving the flag commands'%outfile) 

717 

718 # Cal table type 

719 if iscal: 

720 if outfile == '': 

721 casalog.post('Saving to FLAG_CMD is not supported for cal tables', 'WARN') 

722 else: 

723 casalog.post('Saving parameters to '+outfile) 

724 fh.writeFlagCommands(vis, flagcmd, writeflags, cmdreason, outfile, False) 

725 

726 # MS type 

727 else: 

728 if outfile == '': 

729 casalog.post('Saving parameters to FLAG_CMD') 

730 else: 

731 casalog.post('Saving parameters to '+outfile) 

732 

733 fh.writeFlagCommands(vis, flagcmd, writeflags, cmdreason, outfile, False) 

734 

735 finally: 

736 # Destroy the tool 

737 aflocal.done() 

738 

739 retval = True 

740 # Write history to the MS. Only for modes that write to the MS 

741 if not iscal: 

742 if mode != 'summary' and action == 'apply': 

743 try: 

744 param_names = flagdata.__code__.co_varnames[:flagdata.__code__.co_argcount] 

745 vars = locals( ) 

746 param_vals = [vars[p] for p in param_names] 

747 

748 retval &= write_history(mslocal, vis, 'flagdata', param_names, 

749 param_vals, casalog) 

750 

751 except Exception as instance: 

752 casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 

753 'WARN') 

754 

755 # Pull out the 'summary' reports of summary_stats_list. 

756 if mode == 'summary' or mode == 'list': 

757 ordered_summary_list = OrderedDict(summary_stats_list) 

758 nreps = ordered_summary_list['nreport'] 

759 if nreps > 0: 

760 for rep in range(0,nreps): 

761 repname = "report"+str(rep) 

762 if ordered_summary_list[repname]['type'] != "summary": 

763 ordered_summary_list.pop(repname) 

764 else: 

765 # apply requested filtering 

766 summary_stats = ordered_summary_list.pop(repname); 

767 summary_stats = filter_summary(summary_stats,minrel,maxrel,minabs,maxabs) 

768 # add filtered dictionary back 

769 ordered_summary_list[repname] = summary_stats 

770 

771 # Clean up the dictionary before returning it 

772 ordered_summary_list.pop('type') 

773 ordered_summary_list.pop('nreport') 

774 

775 if len(ordered_summary_list) == 1: 

776 repkey = list(ordered_summary_list.keys()) 

777 summary_stats_list = ordered_summary_list.pop(repkey[0]) 

778 else: 

779 # rename the keys of the dictionary according to 

780 # the number of reports left in dictionary 

781 counter = 0 

782 summary_reports = OrderedDict() 

783 for k in ordered_summary_list.keys(): 

784 repname = "report"+str(counter) 

785 summary_reports[repname] = ordered_summary_list[k] 

786 counter += 1 

787 

788 summary_stats_list = dict(summary_reports) 

789 

790 # for when there is no summary mode in a list 

791 else: 

792 summary_stats_list = {} 

793 

794 elif mode == 'rflag' and action == 'calculate': 

795 # keep the dictionary of rflag intact and return it 

796 pass 

797 else: 

798 summary_stats_list = {} 

799 

800 return summary_stats_list 

801 

802# Helper functions 

803def delspace(word, replace): 

804 '''Replace the white space of a string with another character''' 

805 

806 newword = word 

807 if word.count(' ') > 0: 

808 newword = word.replace(' ', replace) 

809 

810 return newword 

811 

812def filter_summary(summary_stats,minrel,maxrel,minabs,maxabs): 

813 

814 if type(summary_stats) is dict: 

815 if 'flagged' in summary_stats and \ 

816 'total' in summary_stats and \ 

817 not 'type' in summary_stats: 

818 if (summary_stats['flagged'] < minabs) or \ 

819 (summary_stats['flagged'] > maxabs and maxabs >= 0) or \ 

820 (summary_stats['flagged'] * 1.0 / summary_stats['total'] < minrel) or \ 

821 (summary_stats['flagged'] * 1.0 / summary_stats['total'] > maxrel): 

822 return None 

823 else: 

824 for x in list(summary_stats.keys()): 

825 res = filter_summary(summary_stats[x],minrel,maxrel,minabs,maxabs) 

826 if res == None: del summary_stats[x] 

827 

828 return summary_stats 

829 

830def create_arg_dict(subMS_list,value): 

831 

832 ret_dict = [] 

833 for subMS in subMS_list: 

834 ret_dict.append(value) 

835 

836 return ret_dict