Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/imagerhelpers/input_parameters.py: 67%

554 statements  

« prev     ^ index     » next       coverage.py v7.6.4, created at 2024-11-01 07:19 +0000

1import os 

2import math 

3import shutil 

4import string 

5import time 

6import re 

7import copy 

8import pprint 

9import functools 

10import inspect 

11from collections import OrderedDict 

12import numpy as np 

13from typing import Tuple 

14import filecmp 

15 

16 

17from casatools import synthesisutils 

18from casatools import table, ms, synthesisutils, quanta 

19from casatools import calibrater 

20from casatasks import casalog 

21from casatasks.private.mslisthelper import check_mslist 

22from casatasks.private.mslisthelper import sort_mslist 

23 

24 

25""" 

26A set of helper functions for the tasks tclean 

27 

28Summary... 

29  

30""" 

31 

32 

33###################################################### 

34###################################################### 

35###################################################### 

36 

37class ImagerParameters(): 

38 def __init__(self, 

39 # Input Data: what gets in 

40 msname='', 

41 

42 # Output Data: what goes out 

43 imagename='', 

44 

45 

46 # The remaining parameters are Control Parameters: 

47 # they control How what gets in goes out 

48 

49 ## Data Selection 

50 field='', 

51 spw='', 

52 timestr='', 

53 uvdist='', 

54 antenna='', 

55 scan='', 

56 obs='', 

57 state='', 

58 datacolumn='corrected', 

59 ## Image Definition 

60 imsize=[1,1], 

61 cell=[10.0,10.0], 

62 phasecenter='', 

63 stokes='I', 

64 projection='SIN', 

65 startmodel='', 

66 ## Spectral Parameters 

67 specmode='mfs', 

68 reffreq='', 

69 nchan=1, 

70 start='', 

71 width='', 

72 outframe='LSRK', 

73 veltype='radio', 

74 restfreq=[''], 

75 sysvel='', 

76 sysvelframe='', 

77 interpolation='nearest', 

78 perchanweightdensity=False, 

79 gridder="standard", 

80 # ftmachine='gridft', 

81 facets=1, 

82 chanchunks=1, 

83 

84 wprojplanes=1, 

85 

86 vptable="", 

87 usepointing=False, 

88 mosweight=False, 

89 aterm=True, 

90 psterm=True, 

91 mterm=True, 

92 wbawp = True, 

93 cfcache = "", 

94 dopbcorr = True, 

95 conjbeams = True, 

96 computepastep =360.0, 

97 rotatepastep =360.0, 

98 pointingoffsetsigdev = [30.0,30.0], 

99 

100 pblimit=0.01, 

101 normtype='flatnoise', 

102 

103 psfcutoff=0.35, 

104 

105 outlierfile='', 

106 restart=True, 

107 

108 weighting='natural', 

109 robust=0.5, 

110 noise='0.0Jy', 

111 npixels=0, 

112 uvtaper=[], 

113 

114 niter=0, 

115 cycleniter=0, 

116 loopgain=0.1, 

117 threshold='0.0Jy', 

118 nsigma=0.0, 

119 cyclefactor=1.0, 

120 minpsffraction=0.1, 

121 maxpsffraction=0.8, 

122 interactive=False, 

123 fullsummary=False, 

124 nmajor=-1, 

125 

126 deconvolver='hogbom', 

127 scales=[], 

128 nterms=1, 

129 scalebias=0.0, 

130 restoringbeam=[], 

131 # mtype='default', 

132 

133 usemask='user', 

134 mask='', 

135 pbmask=0.0, 

136 maskthreshold='', 

137 maskresolution='', 

138 nmask=0, 

139 # autoadjust=False, 

140 

141 sidelobethreshold=5.0, 

142 noisethreshold=3.0, 

143 lownoisethreshold=3.0, 

144 negativethreshold=0.0, 

145 smoothfactor=1.0, 

146 minbeamfrac=0.3, 

147 cutthreshold=0.01, 

148 growiterations=100, 

149 dogrowprune=True, 

150 minpercentchange=0.0, 

151 verbose=False, 

152 fastnoise=True, 

153 fusedthreshold=0.0, 

154 largestscale=-1, 

155 

156 # usescratch=True, 

157 # readonly=True, 

158 savemodel="none", 

159 parallel=False, 

160 

161 workdir='', 

162 

163 ## CFCache params 

164 cflist=[], 

165 

166 ## single-dish imaging params 

167 gridfunction='SF', 

168 convsupport=-1, 

169 truncate="-1", 

170 gwidth="-1", 

171 jwidth="-1", 

172 pointingcolumntouse='direction', 

173 convertfirst='never', 

174 minweight=0.0, 

175 clipminmax=False 

176 ): 

177 self.allparameters = dict(locals()) 

178 del self.allparameters['self'] 

179 

180 self.defaultKey = "0" 

181 # ---- Selection params. For multiple MSs, all are lists. 

182 # For multiple nodes, the selection parameters are modified inside PySynthesisImager 

183 self.allselpars = { 

184 'msname':msname, 'field':field, 'spw':spw, 'scan':scan, 

185 'timestr':timestr, 'uvdist':uvdist, 'antenna':antenna, 'obs':obs,'state':state, 

186 'datacolumn':datacolumn, 

187 'savemodel':savemodel 

188 } 

189 # ---- Imaging/deconvolution parameters 

190 # The outermost dictionary index is image field. 

191 # The '0' or main field's parameters come from the task parameters 

192 # The outlier '1', '2', .... parameters come from the outlier file 

193 self.outlierfile = outlierfile 

194 # Initialize the parameter lists with the 'main' or '0' field's parameters 

195 # ---- Image definition 

196 self.allimpars = { 

197 self.defaultKey: { 

198 # Image 

199 'imagename': imagename, 

200 'nchan': nchan, 

201 'imsize': imsize, 

202 'cell': cell, 

203 'phasecenter': phasecenter, 

204 'stokes': stokes, 

205 # Frequency axis 

206 'specmode': specmode, 

207 'start': start, 

208 'width': width, 

209 'veltype': veltype, 

210 'nterms': nterms, 

211 'restfreq': restfreq, 

212 # Output frame 

213 'outframe': outframe, 

214 'reffreq': reffreq, 

215 'sysvel': sysvel, 

216 'sysvelframe': sysvelframe, 

217 # Projection 

218 'projection': projection, 

219 # Deconvolution 

220 'restart': restart, 

221 'startmodel': startmodel, 

222 'deconvolver': deconvolver 

223 } 

224 } 

225 # ---- Gridding 

226 self.allgridpars = { 

227 self.defaultKey: { 

228 'gridder': gridder, 

229 # aterm group 

230 'aterm': aterm, 

231 'psterm': psterm, 

232 'mterm': mterm, 

233 'wbawp': wbawp, 

234 # cfcache group 

235 'cfcache': cfcache, 

236 'usepointing': usepointing, 

237 'dopbcorr': dopbcorr, 

238 # conjbeams group 

239 'conjbeams': conjbeams, 

240 'computepastep': computepastep, 

241 # 

242 'rotatepastep': rotatepastep, #'mtype':mtype, # 'weightlimit':weightlimit, 

243 'pointingoffsetsigdev': pointingoffsetsigdev, 

244 # facets group 

245 'facets': facets, 

246 'chanchunks': chanchunks, 

247 # interpolation group 

248 'interpolation': interpolation, 

249 'wprojplanes': wprojplanes, 

250 # deconvolver group 

251 'deconvolver': deconvolver, 

252 'vptable': vptable, 

253 'imagename': imagename, 

254 # single-dish specific parameters 

255 # ---- spatial coordinates 

256 'pointingcolumntouse': pointingcolumntouse, 

257 'convertfirst': convertfirst, 

258 # ---- convolution function 

259 'convfunc': gridfunction, 

260 'convsupport': convsupport, 

261 # ---- truncate group 

262 'truncate': truncate, 

263 'gwidth': gwidth, 

264 'jwidth': jwidth, 

265 # ---- minweight group 

266 'minweight': minweight, 

267 'clipminmax': clipminmax 

268 } 

269 } 

270 # ---- Weighting 

271 if True: # Compute rmode and self.weightpars 

272 rmode = 'none' 

273 if (weighting == 'briggsabs'): 

274 rmode = 'abs' 

275 weighting = 'briggs' 

276 elif (weighting == 'briggs'): 

277 rmode = 'norm' 

278 elif (weighting == 'briggsbwtaper'): 

279 rmode = 'bwtaper' 

280 weighting = 'briggs' 

281 

282 self.weightpars = { 

283 'type': weighting, 

284 'rmode': rmode, 

285 'robust': robust, 

286 'noise': noise, 

287 'npixels': npixels, 

288 'uvtaper': uvtaper, 

289 'multifield': mosweight, 

290 'usecubebriggs': perchanweightdensity 

291 } 

292 # ---- Normalizers ( this is where flat noise, flat sky rules will go... ) 

293 self.allnormpars = { 

294 self.defaultKey : { 

295 # pblimit group 

296 'pblimit': pblimit, 

297 'nterms': nterms, 

298 'facets': facets, 

299 # normtype group 

300 'normtype': normtype, 

301 'workdir': workdir, 

302 # deconvolver group 

303 'deconvolver': deconvolver, 

304 'imagename': imagename, 

305 'restoringbeam': restoringbeam, 

306 'psfcutoff': psfcutoff 

307 } 

308 } 

309 # ---- Deconvolution 

310 self.alldecpars = { 

311 self.defaultKey: { 

312 'id':0, 'deconvolver':deconvolver, 'nterms':nterms, 

313 'scales':scales, 'scalebias':scalebias, 'restoringbeam':restoringbeam, 'usemask':usemask, 

314 'mask':mask, 'pbmask':pbmask, 'maskthreshold':maskthreshold, 

315 'maskresolution':maskresolution, 'nmask':nmask, 

316 #'maskresolution':maskresolution, 'nmask':nmask,'autoadjust':autoadjust, 

317 'sidelobethreshold':sidelobethreshold, 'noisethreshold':noisethreshold, 

318 'lownoisethreshold':lownoisethreshold, 'negativethreshold':negativethreshold,'smoothfactor':smoothfactor, 

319 'fusedthreshold':fusedthreshold, 'specmode':specmode,'largestscale':largestscale, 

320 

321 'minbeamfrac':minbeamfrac, 'cutthreshold':cutthreshold, 'growiterations':growiterations, 

322 'dogrowprune':dogrowprune, 'minpercentchange':minpercentchange, 'verbose':verbose, 'fastnoise':fastnoise, 

323 'interactive':interactive, 'startmodel':startmodel, 'nsigma':nsigma, 'imagename':imagename, 'fullsummary':fullsummary 

324 } 

325 } 

326 # ---- Iteration control 

327 self.iterpars = { 

328 'niter':niter, 'cycleniter':cycleniter, 'threshold':threshold, 

329 'loopgain':loopgain, 'interactive':interactive, 

330 'cyclefactor':cyclefactor, 'minpsffraction':minpsffraction, 

331 'maxpsffraction':maxpsffraction, 

332 'savemodel':savemodel,'nsigma':nsigma, 'nmajor':nmajor, 'fullsummary':fullsummary 

333 } 

334 # ---- CFCache params 

335 self.cfcachepars = { 

336 'cflist': cflist 

337 } 

338 # ---- Parameters that may be internally modified for savemodel behavior 

339 self.inpars = { 

340 'savemodel': savemodel, 

341 'interactive': interactive, 

342 'nsigma': nsigma, 

343 'usemask': usemask 

344 } 

345 # ---- List of supported parameters in outlier files. 

346 # All other parameters will default to the global values. 

347 self.outimparlist = [ 

348 'imagename','nchan','imsize','cell','phasecenter','startmodel', 

349 'start','width', 

350 'nterms','reffreq','specmode' 

351 ] 

352 self.outgridparlist = ['gridder','deconvolver','wprojplanes'] 

353 self.outweightparlist = [] 

354 self.outdecparlist = [ 

355 'deconvolver','startmodel','nterms','usemask','mask' 

356 ] 

357 self.outnormparlist = ['deconvolver','weightlimit','nterms'] 

358 

359 ret = self.checkParameters(parallel) 

360 if ret == False: 

361 casalog.post( 

362 'Found errors in input parameters. Please check.', 'WARN' 

363 ) 

364 

365 self.printParameters() 

366 

367 def resetParameters(self): 

368 """reset parameters to the original settting for interactive, nsigma, auto-multithresh when savemodel!='none'""" 

369 if self.inpars["savemodel"] != "none" and ( 

370 self.inpars["interactive"] == True 

371 or self.inpars["usemask"] == "auto-multithresh" 

372 or self.inpars["nsigma"] > 0.0 

373 ): 

374 # in checkAndFixIterationPars(), when saving model is on, the internal params, readonly and usescrath are set to True and False, 

375 # respectively. So this needs to be undone before calling predictModel. 

376 self.iterpars["savemodel"] = self.inpars["savemodel"] 

377 if self.inpars["savemodel"] == "modelcolumn": 

378 for key in self.allselpars: # for all MSes 

379 self.allselpars[key]["readonly"] = False 

380 self.allselpars[key]["usescratch"] = True 

381 

382 elif self.inpars["savemodel"] == "virtual": 

383 for key in self.allselpars: # for all MSes 

384 self.allselpars[key]["readonly"] = False 

385 self.allselpars[key]["usescratch"] = False 

386 

387 def getAllPars(self): 

388 """Return the state of all parameters""" 

389 return self.allparameters 

390 

391 def getSelPars(self): 

392 return self.allselpars 

393 

394 def getImagePars(self): 

395 return self.allimpars 

396 

397 def getGridPars(self): 

398 return self.allgridpars 

399 

400 def getWeightPars(self): 

401 return self.weightpars 

402 

403 def getDecPars(self): 

404 return self.alldecpars 

405 

406 def getIterPars(self): 

407 return self.iterpars 

408 

409 def getNormPars(self): 

410 return self.allnormpars 

411 

412 def getCFCachePars(self): 

413 return self.cfcachepars 

414 

415 def setSelPars(self, selpars): 

416 for key in selpars.keys(): 

417 self.allselpars[key] = selpars[key] 

418 

419 def setImagePars(self, impars): 

420 for key in impars.keys(): 

421 self.allimpars[key] = impars[key] 

422 

423 def setGridPars(self, gridpars): 

424 for key in gridpars.keys(): 

425 self.allgridpars[key] = gridpars[key] 

426 

427 def setWeightPars(self, weightpars): 

428 for key in weightpars.keys(): 

429 self.weightpars[key] = weightpars[key] 

430 

431 def setDecPars(self, decpars): 

432 for key in decpars.keys(): 

433 self.alldecpars[key] = decpars[key] 

434 

435 def setIterPars(self, iterpars): 

436 for key in iterpars.keys(): 

437 self.iterpars[key] = iterpars[key] 

438 

439 def setNormPars(self, normpars): 

440 for key in normpars.keys(): 

441 self.allnormpars[key] = normpars[key] 

442 

443 def checkParameters(self, parallel=False): 

444 # casalog.origin('refimagerhelper.checkParameters') 

445 casalog.post("Verifying Input Parameters") 

446 # Init the error-string 

447 errs = "" 

448 try: 

449 errs += self.checkAndFixSelectionPars() 

450 errs += self.makeImagingParamLists(parallel) 

451 errs += self.checkAndFixIterationPars() 

452 errs += self.checkAndFixNormPars() 

453 

454 for mss in sorted(self.allselpars.keys()): 

455 if self.allimpars["0"]["specmode"] == "cubedata": 

456 self.allselpars[mss]["outframe"] = "Undefined" 

457 if self.allimpars["0"]["specmode"] == "cubesource": 

458 self.allselpars[mss]["outframe"] = "REST" 

459 ### MOVE this segment of code to the constructor so that it's clear which parameters go where ! 

460 ### Copy them from 'impars' to 'normpars' and 'decpars' 

461 self.iterpars["allimages"] = {} 

462 for immod in self.allimpars.keys(): 

463 self.allnormpars[immod]["imagename"] = self.allimpars[immod][ 

464 "imagename" 

465 ] 

466 self.alldecpars[immod]["imagename"] = self.allimpars[immod]["imagename"] 

467 self.allgridpars[immod]["imagename"] = self.allimpars[immod][ 

468 "imagename" 

469 ] 

470 self.iterpars["allimages"][immod] = { 

471 "imagename": self.allimpars[immod]["imagename"], 

472 "multiterm": (self.alldecpars[immod]["deconvolver"] == "mtmfs"), 

473 } 

474 

475 ## Integers need to be NOT numpy versions. 

476 self.fixIntParam(self.allimpars, "imsize") 

477 self.fixIntParam(self.allimpars, "nchan") 

478 self.fixIntParam(self.allimpars, "nterms") 

479 self.fixIntParam(self.allnormpars, "nterms") 

480 self.fixIntParam(self.alldecpars, "nterms") 

481 self.fixIntParam(self.allgridpars, "facets") 

482 self.fixIntParam(self.allgridpars, "chanchunks") 

483 except Exception as exc: 

484 if len(errs) > 0: 

485 # errs string indicates that maybe this exception was our fault, indicate as such and provide the errs string to the user 

486 raise Exception( 

487 "Parameter Errors : \n{}\nThese errors may have caused the '{}'".format( 

488 errs, type(exc) 

489 ) 

490 ) 

491 else: 

492 # something unforseen happened, just re-throw the exception 

493 raise 

494 

495 ## If there are errors, print a message and exit. 

496 if len(errs) > 0: 

497 # casalog.post('Parameter Errors : \n' + errs,'WARN') 

498 raise Exception("Parameter Errors : \n" + errs) 

499 return True 

500 

501 ###### Start : Parameter-checking functions ################## 

502 

503 def checkAndFixSelectionPars(self): 

504 errs = "" 

505 

506 # If it's already a dict with ms0,ms1,etc...leave it be. 

507 ok = True 

508 for kk in self.allselpars.keys(): 

509 if kk.find("ms") != 0: 

510 ok = False 

511 

512 if ok == True: 

513 # casalog.post("Already in correct format") 

514 return errs 

515 

516 #print("allselpars=",self.allselpars) 

517 # msname, field, spw, etc must all be equal-length lists of strings, or all except msname must be of length 1. 

518 if not "msname" in self.allselpars: 

519 errs = errs + "MS name(s) not specified" 

520 else: 

521 if type(self.allselpars['msname']) == list: 

522 #(timesortedvislist, times) = sort_mslist(self.allselpars['msname']) 

523 (timesortedvislist, times, newindex) = self.mslist_timesorting(self.allselpars['msname']) 

524 if timesortedvislist != self.allselpars['msname']: 

525 self.allselpars['msname'] = timesortedvislist 

526 casalog.post("Sorting the vis list by time. The new vis list:"+ str(self.allselpars['msname'])) 

527 for selp in ['spw','field','timestr','uvdist','antenna','scan','obs','state']: 

528 if type(self.allselpars[selp]) == list and len(self.allselpars[selp]) == len(newindex): 

529 self.allselpars[selp] = [self.allselpars[selp][i] for i in newindex] 

530 

531 #msdiff = check_mslist(self.allselpars['msname'], ignore_tables=['SORTED_TABLE', 'ASDM*']) 

532 msdiff = check_mslist(self.allselpars['msname'], ignore_tables=['SORTED_TABLE', 'ASDM*'], testcontent=False) 

533 

534 # Only call this if vis == list and there is mismatch in wtspec columns 

535 # Maybe expanded for other checks later... 

536 if msdiff != {}: 

537 #print("MS diff===",msdiff) 

538 noWtspecmslist=[] 

539 for msfile, diff_info in msdiff.items(): 

540 # check Main  

541 if 'Main' in diff_info: 

542 for diffkey in diff_info['Main']: 

543 if diffkey == "missingcol_a" or diffkey == "missingcol_b": 

544 if ('WEIGHT_SPECTRUM' in diff_info['Main']['missingcol_a'] and 

545 self.allselpars['msname'][0] not in noWtspecmslist): 

546 noWtspecmslist.append(self.allselpars['msname'][0]) 

547 if ('WEIGHT_SPECTRUM' in diff_info['Main']['missingcol_b'] and 

548 msfile not in noWtspecmslist): 

549 noWtspecmslist.append(msfile) 

550 # repalce this by addwtspec(list_of_ms_withoutWtSpec) 

551 #self.checkmsforwtspec` 

552 if noWtspecmslist!=[]: 

553 #print ("OK addwtspec to "+str(noWtspecmslist)) 

554 self.addwtspec(noWtspecmslist) 

555 

556 selkeys = self.allselpars.keys() 

557 

558 # Convert all non-list parameters into lists. 

559 for par in selkeys: 

560 if type(self.allselpars[par]) != list: 

561 self.allselpars[par] = [self.allselpars[par]] 

562 

563 # Check that all are the same length as nvis 

564 # If not, and if they're single, replicate them nvis times 

565 nvis = len(self.allselpars["msname"]) 

566 

567 if nvis==0: 

568 errs = errs + "Input MS list is empty" 

569 return errs 

570 

571 for par in selkeys: 

572 if len(self.allselpars[par]) > 1 and len(self.allselpars[par]) != nvis: 

573 errs = ( 

574 errs 

575 + str(par) 

576 + " must have a single entry, or " 

577 + str(nvis) 

578 + " entries to match vis list \n" 

579 ) 

580 return errs 

581 else: # Replicate them nvis times if needed. 

582 if len(self.allselpars[par]) == 1: 

583 for ms in range(1, nvis): 

584 self.allselpars[par].append(self.allselpars[par][0]) 

585 

586 # Now, all parameters are lists of strings each of length 'nvis'. 

587 # Put them into separate dicts per MS. 

588 selparlist = {} 

589 for ms in range(0, nvis): 

590 selparlist["ms" + str(ms)] = {} 

591 for par in selkeys: 

592 selparlist["ms" + str(ms)][par] = self.allselpars[par][ms] 

593 

594 synu = synthesisutils() 

595 selparlist["ms" + str(ms)] = synu.checkselectionparams( 

596 selparlist["ms" + str(ms)] 

597 ) 

598 synu.done() 

599 

600 # casalog.post(selparlist) 

601 self.allselpars = selparlist 

602 

603 return errs 

604 

605 def makeImagingParamLists(self, parallel): 

606 errs = "" 

607 # casalog.post("specmode=",self.allimpars['0']['specmode'], " parallel=",parallel) 

608 ## Multiple images have been specified. 

609 ## (1) Parse the outlier file and fill a list of imagedefinitions 

610 ## OR (2) Parse lists per input parameter into a list of parameter-sets (imagedefinitions) 

611 ### The code below implements (1) 

612 outlierpars = [] 

613 parseerrors = "" 

614 if len(self.outlierfile) > 0: 

615 outlierpars, parseerrors = self.parseOutlierFile(self.outlierfile) 

616 if parallel: 

617 casalog.post("CALLING checkParallelMFMixModes...") 

618 errs = self.checkParallelMFMixedModes(self.allimpars, outlierpars) 

619 if len(errs): 

620 return errs 

621 

622 if len(parseerrors) > 0: 

623 errs = errs + "Errors in parsing outlier file : " + parseerrors 

624 return errs 

625 

626 # Initialize outlier parameters with defaults 

627 # Update outlier parameters with modifications from outlier files 

628 for immod in range(0, len(outlierpars)): 

629 modelid = str(immod + 1) 

630 self.allimpars[modelid] = copy.deepcopy(self.allimpars["0"]) 

631 self.allimpars[modelid].update(outlierpars[immod]["impars"]) 

632 self.allgridpars[modelid] = copy.deepcopy(self.allgridpars["0"]) 

633 self.allgridpars[modelid].update(outlierpars[immod]["gridpars"]) 

634 self.alldecpars[modelid] = copy.deepcopy(self.alldecpars["0"]) 

635 self.alldecpars[modelid].update(outlierpars[immod]["decpars"]) 

636 self.allnormpars[modelid] = copy.deepcopy(self.allnormpars["0"]) 

637 self.allnormpars[modelid].update(outlierpars[immod]["normpars"]) 

638 self.alldecpars[modelid]["id"] = immod + 1 ## Try to eliminate. 

639 

640 # casalog.post(self.allimpars) 

641 

642 # 

643 # casalog.post("REMOVING CHECKS to check...") 

644 #### This does not handle the conversions of the csys correctly..... 

645 #### 

646 # for immod in self.allimpars.keys() : 

647 # tempcsys = {} 

648 # if 'csys' in self.allimpars[immod]: 

649 # tempcsys = self.allimpars[immod]['csys'] 

650 # 

651 # synu = synthesisutils() 

652 # self.allimpars[immod] = synu.checkimageparams( self.allimpars[immod] ) 

653 # synu.done() 

654 # 

655 # if len(tempcsys.keys())==0: 

656 # self.allimpars[immod]['csys'] = tempcsys 

657 

658 ## Check for name increments, and copy from impars to decpars and normpars. 

659 self.handleImageNames() 

660 

661 return errs 

662 

663 def handleImageNames(self): 

664 

665 for immod in self.allimpars.keys(): 

666 inpname = self.allimpars[immod]["imagename"] 

667 

668 ### If a directory name is embedded in the image name, check that the dir exists. 

669 if inpname.count("/"): 

670 splitname = inpname.split("/") 

671 prefix = splitname[len(splitname) - 1] 

672 dirname = inpname[0 : len(inpname) - len(prefix)] # has '/' at end 

673 if not os.path.exists(dirname): 

674 casalog.post("Making directory : " + dirname, "INFO") 

675 os.mkdir(dirname) 

676 

677 ### Check for name increments 

678 # if self.reusename == False: 

679 

680 if ( 

681 self.allimpars["0"]["restart"] == False 

682 ): # Later, can change this to be field dependent too. 

683 ## Get a list of image names for all fields (to sync name increment ids across fields) 

684 inpnamelist = {} 

685 for immod in self.allimpars.keys(): 

686 inpnamelist[immod] = self.allimpars[immod]["imagename"] 

687 

688 newnamelist = self.incrementImageNameList(inpnamelist) 

689 

690 if len(newnamelist) != len(self.allimpars.keys()): 

691 casalog.post( 

692 "Internal Error : Non matching list lengths in refimagerhelper::handleImageNames. Not updating image names", 

693 "WARN", 

694 ) 

695 else: 

696 for immod in self.allimpars.keys(): 

697 self.allimpars[immod]["imagename"] = newnamelist[immod] 

698 

699 def checkAndFixIterationPars(self): 

700 errs = "" 

701 

702 # Bother checking only if deconvolution iterations are requested 

703 if self.iterpars["niter"] > 0: 

704 # Make sure cycleniter is less than or equal to niter. 

705 if ( 

706 self.iterpars["cycleniter"] <= 0 

707 or self.iterpars["cycleniter"] > self.iterpars["niter"] 

708 ): 

709 if self.iterpars["interactive"] == False: 

710 self.iterpars["cycleniter"] = self.iterpars["niter"] 

711 else: 

712 self.iterpars["cycleniter"] = min(self.iterpars["niter"], 100) 

713 

714 # saving model is done separately outside of iter. control for interactive clean and or automasking cases 

715 

716 if self.iterpars['savemodel']!='none': 

717 if self.iterpars['interactive']==True or self.alldecpars['0']['usemask']=='auto-multithresh' or \ 

718 self.alldecpars['0']['nsigma']>0.0: 

719 self.iterpars['savemodel']='none' 

720 for visid in self.allselpars: 

721 self.allselpars[visid]['readonly']=True 

722 self.allselpars[visid]['usescratch']=False 

723 

724 

725 return errs 

726 

727 def checkAndFixNormPars(self): 

728 errs = "" 

729 

730 # for modelid in self.allnormpars.keys(): 

731 # if len(self.allnormpars[modelid]['workdir'])==0: 

732 # self.allnormpars[modelid]['workdir'] = self.allnormpars['0']['imagename'] + '.workdir' 

733 

734 return errs 

735 

736 ###### End : Parameter-checking functions ################## 

737 

738 ## Parse outlier file and construct a list of imagedefinitions (dictionaries). 

739 def parseOutlierFile(self, outlierfilename=""): 

740 returnlist = [] 

741 errs = "" # must be empty for no error 

742 

743 if len(outlierfilename) > 0 and not os.path.exists(outlierfilename): 

744 errs += "Cannot find outlier file : " + outlierfilename + "\n" 

745 return returnlist, errs 

746 

747 fp = open(outlierfilename, "r") 

748 thelines = fp.readlines() 

749 tempimpar = {} 

750 tempgridpar = {} 

751 tempweightpar = {} 

752 tempdecpar = {} 

753 tempnormpar = {} 

754 for oneline in thelines: 

755 aline = oneline.replace("\n", "") 

756 # aline = oneline.replace(' ','').replace('\n','') 

757 if len(aline) > 0 and aline.find("#") != 0: 

758 parpair = aline.split("=") 

759 parpair[0] = parpair[0].replace(" ", "") 

760 # casalog.post(parpair) 

761 if len(parpair) != 2: 

762 errs += "Error in line containing : " + oneline + "\n" 

763 if parpair[0] == "imagename" and tempimpar != {}: 

764 # returnlist.append({'impars':tempimpar, 'gridpars':tempgridpar, 'weightpars':tempweightpar, 'decpars':tempdecpar} ) 

765 returnlist.append( 

766 { 

767 "impars": tempimpar, 

768 "gridpars": tempgridpar, 

769 "weightpars": tempweightpar, 

770 "decpars": tempdecpar, 

771 "normpars": tempnormpar, 

772 } 

773 ) 

774 tempimpar = {} 

775 tempgridpar = {} 

776 tempweightpar = {} 

777 tempdecpar = {} 

778 tempnormpar = {} 

779 usepar = False 

780 if parpair[0] in self.outimparlist: 

781 tempimpar[parpair[0]] = parpair[1] 

782 usepar = True 

783 if parpair[0] in self.outgridparlist: 

784 tempgridpar[parpair[0]] = parpair[1] 

785 usepar = True 

786 if parpair[0] in self.outweightparlist: 

787 tempweightpar[parpair[0]] = parpair[1] 

788 usepar = True 

789 if parpair[0] in self.outdecparlist: 

790 tempdecpar[parpair[0]] = parpair[1] 

791 usepar = True 

792 if parpair[0] in self.outnormparlist: 

793 tempnormpar[parpair[0]] = parpair[1] 

794 usepar = True 

795 if usepar == False: 

796 casalog.post("Ignoring unknown parameter pair : " + oneline) 

797 

798 if len(errs) == 0: 

799 returnlist.append( 

800 { 

801 "impars": tempimpar, 

802 "gridpars": tempgridpar, 

803 "weightpars": tempweightpar, 

804 "decpars": tempdecpar, 

805 "normpars": tempnormpar, 

806 } 

807 ) 

808 

809 ## Extra parsing for a few parameters. 

810 returnlist = self.evalToTarget(returnlist, "impars", "imsize", "intvec") 

811 returnlist = self.evalToTarget(returnlist, "impars", "nchan", "int") 

812 returnlist = self.evalToTarget(returnlist, "impars", "cell", "strvec") 

813 returnlist = self.evalToTarget(returnlist, "impars", "nterms", "int") 

814 returnlist = self.evalToTarget(returnlist, "decpars", "nterms", "int") 

815 returnlist = self.evalToTarget(returnlist, "normpars", "nterms", "int") 

816 returnlist = self.evalToTarget(returnlist, "gridpars", "wprojplanes", "int") 

817 # returnlist = self.evalToTarget( returnlist, 'impars', 'reffreq', 'strvec' ) 

818 

819 # casalog.post(returnlist) 

820 return returnlist, errs 

821 

822 def evalToTarget(self, globalpars, subparkey, parname, dtype="int"): 

823 try: 

824 for fld in range(0, len(globalpars)): 

825 if parname in globalpars[fld][subparkey]: 

826 if dtype == "int" or dtype == "intvec": 

827 val_e = eval(globalpars[fld][subparkey][parname]) 

828 if dtype == "strvec": 

829 tcell = globalpars[fld][subparkey][parname] 

830 tcell = ( 

831 tcell.replace(" ", "") 

832 .replace("[", "") 

833 .replace("]", "") 

834 .replace("'", "") 

835 ) 

836 tcells = tcell.split(",") 

837 val_e = [] 

838 for cell in tcells: 

839 val_e.append(cell) 

840 

841 globalpars[fld][subparkey][parname] = val_e 

842 except: 

843 casalog.post( 

844 'Cannot evaluate outlier field parameter "' + parname + '"', "ERROR" 

845 ) 

846 

847 return globalpars 

848 

849 def printParameters(self): 

850 casalog.post("SelPars : " + str(self.allselpars), "INFO2") 

851 casalog.post("ImagePars : " + str(self.allimpars), "INFO2") 

852 casalog.post("GridPars : " + str(self.allgridpars), "INFO2") 

853 casalog.post("NormPars : " + str(self.allnormpars), "INFO2") 

854 casalog.post("Weightpars : " + str(self.weightpars), "INFO2") 

855 casalog.post("DecPars : " + str(self.alldecpars), "INFO2") 

856 casalog.post("IterPars : " + str(self.iterpars), "INFO2") 

857 

858 def incrementImageName(self, imagename): 

859 dirname = "." 

860 prefix = imagename 

861 

862 if imagename.count("/"): 

863 splitname = imagename.split("/") 

864 prefix = splitname[len(splitname) - 1] 

865 ### if it has a leading / then absolute path is assumed 

866 dirname = ( 

867 (imagename[0 : len(imagename) - len(prefix)]) 

868 if (imagename[0] == "/") 

869 else ("./" + imagename[0 : len(imagename) - len(prefix)]) 

870 ) # has '/' at end 

871 

872 inamelist = [fn for fn in os.listdir(dirname) if any([fn.startswith(prefix)])] 

873 

874 if len(inamelist) == 0: 

875 newimagename = dirname[2:] + prefix 

876 else: 

877 nlen = len(prefix) 

878 maxid = 1 

879 for iname in inamelist: 

880 startind = iname.find(prefix + "_") 

881 if startind == 0: 

882 idstr = (iname[nlen + 1 : len(iname)]).split(".")[0] 

883 if idstr.isdigit(): 

884 val = eval(idstr) 

885 if val > maxid: 

886 maxid = val 

887 newimagename = dirname[2:] + prefix + "_" + str(maxid + 1) 

888 

889 casalog.post("Using : {}".format(newimagename)) 

890 return newimagename 

891 

892 def incrementImageNameList(self, inpnamelist): 

893 

894 dirnames = {} 

895 prefixes = {} 

896 

897 for immod in inpnamelist.keys(): 

898 imagename = inpnamelist[immod] 

899 dirname = "." 

900 prefix = imagename 

901 

902 if imagename.count("/"): 

903 splitname = imagename.split("/") 

904 prefix = splitname[len(splitname) - 1] 

905 dirname = ( 

906 (imagename[0 : len(imagename) - len(prefix)]) 

907 if (imagename[0] == "/") 

908 else ("./" + imagename[0 : len(imagename) - len(prefix)]) 

909 ) # has '/' at end 

910 

911 dirnames[immod] = dirname 

912 prefixes[immod] = prefix 

913 

914 maxid = 0 

915 for immod in inpnamelist.keys(): 

916 prefix = prefixes[immod] 

917 inamelist = [ 

918 fn for fn in os.listdir(dirnames[immod]) if any([fn.startswith(prefix)]) 

919 ] 

920 nlen = len(prefix) 

921 

922 if len(inamelist) == 0: 

923 locmax = 0 

924 else: 

925 locmax = 1 

926 

927 cleanext = [".image", ".residual", ".model", ".psf", ".sumwt", ".tt0"] 

928 incremented = False 

929 for iname in inamelist: 

930 rootname, ext = os.path.splitext(iname) 

931 if ext in cleanext: 

932 startind = iname.find(prefix + "_") 

933 if startind == 0: 

934 idstr = (iname[nlen + 1 : len(iname)]).split(".")[0] 

935 if idstr.isdigit(): 

936 val = eval(idstr) 

937 incremented = True 

938 if val > locmax: 

939 locmax = val 

940 elif startind == -1: 

941 if ext == ".tt0": 

942 # need one more pass to extract rootname 

943 rootname, ext = os.path.splitext(rootname) 

944 if rootname == prefix: 

945 # the file name with root file name only 

946 incremented = True 

947 

948 if not incremented: 

949 locmax = 0 

950 if locmax > maxid: 

951 maxid = locmax 

952 

953 newimagenamelist = {} 

954 for immod in inpnamelist.keys(): 

955 if maxid == 0: 

956 newimagenamelist[immod] = inpnamelist[immod] 

957 else: 

958 newimagenamelist[immod] = ( 

959 dirnames[immod][2:] + prefixes[immod] + "_" + str(maxid + 1) 

960 ) 

961 

962 # casalog.post('Input : ', inpnamelist) 

963 # casalog.post('Dirs : ', dirnames) 

964 # casalog.post('Pre : ', prefixes) 

965 # casalog.post('Max id : ', maxid) 

966 # casalog.post('Using : ', newimagenamelist) 

967 return newimagenamelist 

968 

969 ## Guard against numpy int32,int64 types which don't convert well across tool boundary. 

970 ## For CAS-8250. Remove when CAS-6682 is done. 

971 def fixIntParam(self, allpars, parname): 

972 for immod in allpars.keys(): 

973 if parname in allpars[immod]: 

974 ims = allpars[immod][parname] 

975 if type(ims) != list: 

976 ims = int(ims) 

977 else: 

978 for el in range(0, len(ims)): 

979 ims[el] = int(ims[el]) 

980 allpars[immod][parname] = ims 

981 

982 # check for non-supported multifield in mixed modes in parallel 

983 # (e.g. combination cube and continuum for main and outlier fields) 

984 def checkParallelMFMixedModes(self, allimpars, outlierpars): 

985 errmsg = "" 

986 casalog.post("outlierpars=={}".format(outlierpars)) 

987 mainspecmode = allimpars["0"]["specmode"] 

988 mainnchan = allimpars["0"]["nchan"] 

989 casalog.post("mainspecmode={} mainnchan={}".format(mainspecmode, mainnchan)) 

990 cubeoutlier = False 

991 contoutlier = False 

992 isnchanmatch = True 

993 for immod in range(0, len(outlierpars)): 

994 if "impars" in outlierpars[immod]: 

995 if "nchan" in outlierpars[immod]["impars"]: 

996 if outlierpars[immod]["impars"]["nchan"] > 1: 

997 cubeoutlier = True 

998 if outlierpars[immod]["impars"]["nchan"] != mainnchan: 

999 isnchanmatch = False 

1000 else: 

1001 contoutlier = True 

1002 else: 

1003 if "specmode" in outlierpars[immod]["impars"]: 

1004 if outlierpars[immod]["impars"]["specmode"] == "mfs": 

1005 contoutlier = True 

1006 if mainspecmode.find("cube") == 0: 

1007 if contoutlier: 

1008 errmsg = "Mixed cube and continuum mode for multifields is currently not supported for parallel mode" 

1009 else: # all cube modes, but need to check if the nchans are the same 

1010 if not isnchanmatch: 

1011 errmsg = "Cubes for multifields with different nchans are currently not supported for parallel mode " 

1012 else: # mfs 

1013 if cubeoutlier: 

1014 errmsg = "Mixed continuum and cube mode for multifields is currently not supported for parallel mode" 

1015 errs = errmsg 

1016 return errs 

1017 

1018 def checkmsforwtspec(self): 

1019 ''' check if WEIGHT_SPECTRUM column exist when  

1020 a list of vis is given. Add the column for an MS 

1021 which does not have one if other MSs have the column. 

1022 This is a workaround for the issue probably in Vi/VB2 

1023 not handling the state change for the optional column 

1024 when dealing with multiples MSs 

1025 ''' 

1026 mycb = calibrater() 

1027 mytb = table() 

1028 haswtspec=False 

1029 mswithnowtspec=[] 

1030 nms = 1 

1031 if type(self.allselpars['msname'])==list: 

1032 nms = len(self.allselpars['msname']) 

1033 

1034 if nms > 1: 

1035 for inms in self.allselpars['msname']: 

1036 mytb.open(inms) 

1037 cols = mytb.colnames() 

1038 mytb.close() 

1039 if 'WEIGHT_SPECTRUM' in cols: 

1040 haswtspec=True 

1041 else: 

1042 mswithnowtspec.append(inms) 

1043 if haswtspec and len(mswithnowtspec) > 0: 

1044 casalog.post("Some of the MSes donot have WEIGHT_SPECTRUM while some other do."+ 

1045 " Automatically adding the column and initialize for those don't to avoid a process failure.","WARN") 

1046 for inms in mswithnowtspec: 

1047 mycb.open(inms, addcorr=False, addmodel=False) 

1048 mycb.initweights(wtmode='weight', dowtsp=True) 

1049 mycb.close() 

1050 # noOp for nms==1  

1051 

1052 def mslist_timesorting(self, mslist): 

1053 '''  

1054 wrapper for mslisthelper.sort_mslist to get a sorting order w.r.t the original 

1055 ''' 

1056 (thenewmslist, times) = sort_mslist(mslist) 

1057 theindex = [] 

1058 for vnew in thenewmslist: 

1059 for vold in mslist: 

1060 if vnew == vold: 

1061 theindex.append(mslist.index(vnew)) 

1062 return (thenewmslist, times, theindex) 

1063 

1064 def addwtspec(self, mslist): 

1065 '''  

1066 Add the column for an MS which does not have one if other MSs have the column. 

1067 This is a workaround for the issue probably in Vi/VB2 

1068 not handling the state change for the optional column 

1069 when dealing with multiples MSs 

1070 ''' 

1071 mycb = calibrater() 

1072 

1073 if len(mslist) > 0: 

1074 casalog.post("Some of the MSes donot have WEIGHT_SPECTRUM while some other do."+ 

1075 " Automatically adding the column and initialize using the existing WEIGHT column for those don't to avoid a process failure.","WARN") 

1076 casalog.post("Adding WEIGHT_SPECTRUM in the following MS(s): "+str(mslist),"WARN") 

1077 for inms in mslist: 

1078 mycb.open(inms, addcorr=False, addmodel=False) 

1079 mycb.initweights(wtmode='weight', dowtsp=True) 

1080 mycb.close() 

1081 mycb.done() 

1082 # noOp for len(mlist) ==0 

1083 

1084 ############################ 

1085 

1086 

1087################################################################################################# 

1088def backupoldfile(thefile=""): 

1089 import copy 

1090 import shutil 

1091 

1092 if thefile == "" or (not os.path.exists(thefile)): 

1093 return 

1094 outpathdir = os.path.realpath(os.path.dirname(thefile)) 

1095 outpathfile = outpathdir + os.path.sep + os.path.basename(thefile) 

1096 k = 0 

1097 backupfile = outpathfile + "." + str(k) 

1098 prevfile = "--------" 

1099 while os.path.exists(backupfile): 

1100 k = k + 1 

1101 prevfile = copy.copy(backupfile) 

1102 if os.path.exists(prevfile) and filecmp.cmp(outpathfile, prevfile): 

1103 ##avoid making multiple copies of the same file 

1104 return 

1105 backupfile = outpathfile + "." + str(k) 

1106 shutil.copy2(outpathfile, backupfile) 

1107 

1108 

1109def saveparams2last(func=None, multibackup=True): 

1110 """This function is a decorator function that allows for 

1111 task.last to be saved even if calling without casashell. Also 

1112 saves unique revisions ...just like the vax/vms style of revision saving 

1113 by default. set multibackup=False to no not have old revisions kept 

1114 """ 

1115 if not func: 

1116 return functools.partial(saveparams2last, multibackup=multibackup) 

1117 

1118 @functools.wraps(func) 

1119 def wrapper_saveparams(*args, **kwargs): 

1120 # multibackup=True 

1121 outfile = func.__name__ + ".last" 

1122 # print('args {} and kwargs {}'.format(args, kwargs)) 

1123 # print('length of args {}, and kwargs {}'.format(len(args), len(kwargs))) 

1124 params = {} 

1125 byIndex = list() 

1126 if len(kwargs) == 0: 

1127 paramsname = list(inspect.signature(func).parameters) 

1128 # params={paramsname[i]: args[i] for i in range(len(args))} 

1129 params = OrderedDict(zip(paramsname, args)) 

1130 byIndex = list(params) 

1131 else: 

1132 params = kwargs 

1133 byIndex = list(params) 

1134 ###for some reason the dictionary is in reverse 

1135 byIndex.reverse() 

1136 # print('@@@@MULTIBACKUP {}, params {}'.format(multibackup, params)) 

1137 if multibackup: 

1138 backupoldfile(outfile) 

1139 with open(outfile, "w") as _f: 

1140 for _i in range(len(byIndex)): 

1141 _f.write("%-20s = %s\n" % (byIndex[_i], repr(params[byIndex[_i]]))) 

1142 _f.write("#" + func.__name__ + "( ") 

1143 for _i in range(len(byIndex)): 

1144 _f.write("%s=%s" % (byIndex[_i], repr(params[byIndex[_i]]))) 

1145 if _i < len(params) - 1: 

1146 _f.write(",") 

1147 _f.write(" )\n") 

1148 ###End of stuff before task is called 

1149 retval = func(*args, **kwargs) 

1150 ###we could do something here post task 

1151 return retval 

1152 

1153 return wrapper_saveparams 

1154 

1155 

1156###################################################### 

1157 

1158 

1159def determineFreqRange( 

1160 vis: str = "", fieldid: int = 0, spw: str = "*" 

1161) -> Tuple[np.double, np.double]: 

1162 _tb = table() 

1163 _ms = ms() 

1164 _su = synthesisutils() 

1165 _qa = quanta() 

1166 minFreq = 1.0e20 

1167 maxFreq = 0.0 

1168 _tb.open(vis) 

1169 fieldids = _tb.getcol("FIELD_ID") 

1170 _tb.done() 

1171 # advisechansel does not work on fieldids not in main 

1172 if fieldid not in fieldids: 

1173 fieldid = fieldids[0] 

1174 frange = _su.advisechansel( 

1175 msname=vis, getfreqrange=True, fieldid=fieldid, spwselection=spw 

1176 ) 

1177 if minFreq > _qa.convert(frange["freqstart"], "Hz")["value"]: 

1178 minFreq = _qa.convert(frange["freqstart"], "Hz")["value"] 

1179 if maxFreq < _qa.convert(frange["freqend"], "Hz")["value"]: 

1180 maxFreq = _qa.convert(frange["freqend"], "Hz")["value"] 

1181 

1182 if minFreq > maxFreq: 

1183 raise Exception(f"Failed to determine frequency range in ms {vis}") 

1184 freqwidth = maxFreq - minFreq 

1185 return (minFreq, freqwidth)