Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/imagerhelpers/input_parameters.py: 12%
554 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-10-31 19:10 +0000
1import os
2import math
3import shutil
4import string
5import time
6import re
7import copy
8import pprint
9import functools
10import inspect
11from collections import OrderedDict
12import numpy as np
13from typing import Tuple
14import filecmp
17from casatools import synthesisutils
18from casatools import table, ms, synthesisutils, quanta
19from casatools import calibrater
20from casatasks import casalog
21from casatasks.private.mslisthelper import check_mslist
22from casatasks.private.mslisthelper import sort_mslist
25"""
26A set of helper functions for the tasks tclean
28Summary...
30"""
33######################################################
34######################################################
35######################################################
37class ImagerParameters():
38 def __init__(self,
39 # Input Data: what gets in
40 msname='',
42 # Output Data: what goes out
43 imagename='',
46 # The remaining parameters are Control Parameters:
47 # they control How what gets in goes out
49 ## Data Selection
50 field='',
51 spw='',
52 timestr='',
53 uvdist='',
54 antenna='',
55 scan='',
56 obs='',
57 state='',
58 datacolumn='corrected',
59 ## Image Definition
60 imsize=[1,1],
61 cell=[10.0,10.0],
62 phasecenter='',
63 stokes='I',
64 projection='SIN',
65 startmodel='',
66 ## Spectral Parameters
67 specmode='mfs',
68 reffreq='',
69 nchan=1,
70 start='',
71 width='',
72 outframe='LSRK',
73 veltype='radio',
74 restfreq=[''],
75 sysvel='',
76 sysvelframe='',
77 interpolation='nearest',
78 perchanweightdensity=False,
79 gridder="standard",
80 # ftmachine='gridft',
81 facets=1,
82 chanchunks=1,
84 wprojplanes=1,
86 vptable="",
87 usepointing=False,
88 mosweight=False,
89 aterm=True,
90 psterm=True,
91 mterm=True,
92 wbawp = True,
93 cfcache = "",
94 dopbcorr = True,
95 conjbeams = True,
96 computepastep =360.0,
97 rotatepastep =360.0,
98 pointingoffsetsigdev = [30.0,30.0],
100 pblimit=0.01,
101 normtype='flatnoise',
103 psfcutoff=0.35,
105 outlierfile='',
106 restart=True,
108 weighting='natural',
109 robust=0.5,
110 noise='0.0Jy',
111 npixels=0,
112 uvtaper=[],
114 niter=0,
115 cycleniter=0,
116 loopgain=0.1,
117 threshold='0.0Jy',
118 nsigma=0.0,
119 cyclefactor=1.0,
120 minpsffraction=0.1,
121 maxpsffraction=0.8,
122 interactive=False,
123 fullsummary=False,
124 nmajor=-1,
126 deconvolver='hogbom',
127 scales=[],
128 nterms=1,
129 scalebias=0.0,
130 restoringbeam=[],
131 # mtype='default',
133 usemask='user',
134 mask='',
135 pbmask=0.0,
136 maskthreshold='',
137 maskresolution='',
138 nmask=0,
139 # autoadjust=False,
141 sidelobethreshold=5.0,
142 noisethreshold=3.0,
143 lownoisethreshold=3.0,
144 negativethreshold=0.0,
145 smoothfactor=1.0,
146 minbeamfrac=0.3,
147 cutthreshold=0.01,
148 growiterations=100,
149 dogrowprune=True,
150 minpercentchange=0.0,
151 verbose=False,
152 fastnoise=True,
153 fusedthreshold=0.0,
154 largestscale=-1,
156 # usescratch=True,
157 # readonly=True,
158 savemodel="none",
159 parallel=False,
161 workdir='',
163 ## CFCache params
164 cflist=[],
166 ## single-dish imaging params
167 gridfunction='SF',
168 convsupport=-1,
169 truncate="-1",
170 gwidth="-1",
171 jwidth="-1",
172 pointingcolumntouse='direction',
173 convertfirst='never',
174 minweight=0.0,
175 clipminmax=False
176 ):
177 self.allparameters = dict(locals())
178 del self.allparameters['self']
180 self.defaultKey = "0"
181 # ---- Selection params. For multiple MSs, all are lists.
182 # For multiple nodes, the selection parameters are modified inside PySynthesisImager
183 self.allselpars = {
184 'msname':msname, 'field':field, 'spw':spw, 'scan':scan,
185 'timestr':timestr, 'uvdist':uvdist, 'antenna':antenna, 'obs':obs,'state':state,
186 'datacolumn':datacolumn,
187 'savemodel':savemodel
188 }
189 # ---- Imaging/deconvolution parameters
190 # The outermost dictionary index is image field.
191 # The '0' or main field's parameters come from the task parameters
192 # The outlier '1', '2', .... parameters come from the outlier file
193 self.outlierfile = outlierfile
194 # Initialize the parameter lists with the 'main' or '0' field's parameters
195 # ---- Image definition
196 self.allimpars = {
197 self.defaultKey: {
198 # Image
199 'imagename': imagename,
200 'nchan': nchan,
201 'imsize': imsize,
202 'cell': cell,
203 'phasecenter': phasecenter,
204 'stokes': stokes,
205 # Frequency axis
206 'specmode': specmode,
207 'start': start,
208 'width': width,
209 'veltype': veltype,
210 'nterms': nterms,
211 'restfreq': restfreq,
212 # Output frame
213 'outframe': outframe,
214 'reffreq': reffreq,
215 'sysvel': sysvel,
216 'sysvelframe': sysvelframe,
217 # Projection
218 'projection': projection,
219 # Deconvolution
220 'restart': restart,
221 'startmodel': startmodel,
222 'deconvolver': deconvolver
223 }
224 }
225 # ---- Gridding
226 self.allgridpars = {
227 self.defaultKey: {
228 'gridder': gridder,
229 # aterm group
230 'aterm': aterm,
231 'psterm': psterm,
232 'mterm': mterm,
233 'wbawp': wbawp,
234 # cfcache group
235 'cfcache': cfcache,
236 'usepointing': usepointing,
237 'dopbcorr': dopbcorr,
238 # conjbeams group
239 'conjbeams': conjbeams,
240 'computepastep': computepastep,
241 #
242 'rotatepastep': rotatepastep, #'mtype':mtype, # 'weightlimit':weightlimit,
243 'pointingoffsetsigdev': pointingoffsetsigdev,
244 # facets group
245 'facets': facets,
246 'chanchunks': chanchunks,
247 # interpolation group
248 'interpolation': interpolation,
249 'wprojplanes': wprojplanes,
250 # deconvolver group
251 'deconvolver': deconvolver,
252 'vptable': vptable,
253 'imagename': imagename,
254 # single-dish specific parameters
255 # ---- spatial coordinates
256 'pointingcolumntouse': pointingcolumntouse,
257 'convertfirst': convertfirst,
258 # ---- convolution function
259 'convfunc': gridfunction,
260 'convsupport': convsupport,
261 # ---- truncate group
262 'truncate': truncate,
263 'gwidth': gwidth,
264 'jwidth': jwidth,
265 # ---- minweight group
266 'minweight': minweight,
267 'clipminmax': clipminmax
268 }
269 }
270 # ---- Weighting
271 if True: # Compute rmode and self.weightpars
272 rmode = 'none'
273 if (weighting == 'briggsabs'):
274 rmode = 'abs'
275 weighting = 'briggs'
276 elif (weighting == 'briggs'):
277 rmode = 'norm'
278 elif (weighting == 'briggsbwtaper'):
279 rmode = 'bwtaper'
280 weighting = 'briggs'
282 self.weightpars = {
283 'type': weighting,
284 'rmode': rmode,
285 'robust': robust,
286 'noise': noise,
287 'npixels': npixels,
288 'uvtaper': uvtaper,
289 'multifield': mosweight,
290 'usecubebriggs': perchanweightdensity
291 }
292 # ---- Normalizers ( this is where flat noise, flat sky rules will go... )
293 self.allnormpars = {
294 self.defaultKey : {
295 # pblimit group
296 'pblimit': pblimit,
297 'nterms': nterms,
298 'facets': facets,
299 # normtype group
300 'normtype': normtype,
301 'workdir': workdir,
302 # deconvolver group
303 'deconvolver': deconvolver,
304 'imagename': imagename,
305 'restoringbeam': restoringbeam,
306 'psfcutoff': psfcutoff
307 }
308 }
309 # ---- Deconvolution
310 self.alldecpars = {
311 self.defaultKey: {
312 'id':0, 'deconvolver':deconvolver, 'nterms':nterms,
313 'scales':scales, 'scalebias':scalebias, 'restoringbeam':restoringbeam, 'usemask':usemask,
314 'mask':mask, 'pbmask':pbmask, 'maskthreshold':maskthreshold,
315 'maskresolution':maskresolution, 'nmask':nmask,
316 #'maskresolution':maskresolution, 'nmask':nmask,'autoadjust':autoadjust,
317 'sidelobethreshold':sidelobethreshold, 'noisethreshold':noisethreshold,
318 'lownoisethreshold':lownoisethreshold, 'negativethreshold':negativethreshold,'smoothfactor':smoothfactor,
319 'fusedthreshold':fusedthreshold, 'specmode':specmode,'largestscale':largestscale,
321 'minbeamfrac':minbeamfrac, 'cutthreshold':cutthreshold, 'growiterations':growiterations,
322 'dogrowprune':dogrowprune, 'minpercentchange':minpercentchange, 'verbose':verbose, 'fastnoise':fastnoise,
323 'interactive':interactive, 'startmodel':startmodel, 'nsigma':nsigma, 'imagename':imagename, 'fullsummary':fullsummary
324 }
325 }
326 # ---- Iteration control
327 self.iterpars = {
328 'niter':niter, 'cycleniter':cycleniter, 'threshold':threshold,
329 'loopgain':loopgain, 'interactive':interactive,
330 'cyclefactor':cyclefactor, 'minpsffraction':minpsffraction,
331 'maxpsffraction':maxpsffraction,
332 'savemodel':savemodel,'nsigma':nsigma, 'nmajor':nmajor, 'fullsummary':fullsummary
333 }
334 # ---- CFCache params
335 self.cfcachepars = {
336 'cflist': cflist
337 }
338 # ---- Parameters that may be internally modified for savemodel behavior
339 self.inpars = {
340 'savemodel': savemodel,
341 'interactive': interactive,
342 'nsigma': nsigma,
343 'usemask': usemask
344 }
345 # ---- List of supported parameters in outlier files.
346 # All other parameters will default to the global values.
347 self.outimparlist = [
348 'imagename','nchan','imsize','cell','phasecenter','startmodel',
349 'start','width',
350 'nterms','reffreq','specmode'
351 ]
352 self.outgridparlist = ['gridder','deconvolver','wprojplanes']
353 self.outweightparlist = []
354 self.outdecparlist = [
355 'deconvolver','startmodel','nterms','usemask','mask'
356 ]
357 self.outnormparlist = ['deconvolver','weightlimit','nterms']
359 ret = self.checkParameters(parallel)
360 if ret == False:
361 casalog.post(
362 'Found errors in input parameters. Please check.', 'WARN'
363 )
365 self.printParameters()
367 def resetParameters(self):
368 """reset parameters to the original settting for interactive, nsigma, auto-multithresh when savemodel!='none'"""
369 if self.inpars["savemodel"] != "none" and (
370 self.inpars["interactive"] == True
371 or self.inpars["usemask"] == "auto-multithresh"
372 or self.inpars["nsigma"] > 0.0
373 ):
374 # in checkAndFixIterationPars(), when saving model is on, the internal params, readonly and usescrath are set to True and False,
375 # respectively. So this needs to be undone before calling predictModel.
376 self.iterpars["savemodel"] = self.inpars["savemodel"]
377 if self.inpars["savemodel"] == "modelcolumn":
378 for key in self.allselpars: # for all MSes
379 self.allselpars[key]["readonly"] = False
380 self.allselpars[key]["usescratch"] = True
382 elif self.inpars["savemodel"] == "virtual":
383 for key in self.allselpars: # for all MSes
384 self.allselpars[key]["readonly"] = False
385 self.allselpars[key]["usescratch"] = False
387 def getAllPars(self):
388 """Return the state of all parameters"""
389 return self.allparameters
391 def getSelPars(self):
392 return self.allselpars
394 def getImagePars(self):
395 return self.allimpars
397 def getGridPars(self):
398 return self.allgridpars
400 def getWeightPars(self):
401 return self.weightpars
403 def getDecPars(self):
404 return self.alldecpars
406 def getIterPars(self):
407 return self.iterpars
409 def getNormPars(self):
410 return self.allnormpars
412 def getCFCachePars(self):
413 return self.cfcachepars
415 def setSelPars(self, selpars):
416 for key in selpars.keys():
417 self.allselpars[key] = selpars[key]
419 def setImagePars(self, impars):
420 for key in impars.keys():
421 self.allimpars[key] = impars[key]
423 def setGridPars(self, gridpars):
424 for key in gridpars.keys():
425 self.allgridpars[key] = gridpars[key]
427 def setWeightPars(self, weightpars):
428 for key in weightpars.keys():
429 self.weightpars[key] = weightpars[key]
431 def setDecPars(self, decpars):
432 for key in decpars.keys():
433 self.alldecpars[key] = decpars[key]
435 def setIterPars(self, iterpars):
436 for key in iterpars.keys():
437 self.iterpars[key] = iterpars[key]
439 def setNormPars(self, normpars):
440 for key in normpars.keys():
441 self.allnormpars[key] = normpars[key]
443 def checkParameters(self, parallel=False):
444 # casalog.origin('refimagerhelper.checkParameters')
445 casalog.post("Verifying Input Parameters")
446 # Init the error-string
447 errs = ""
448 try:
449 errs += self.checkAndFixSelectionPars()
450 errs += self.makeImagingParamLists(parallel)
451 errs += self.checkAndFixIterationPars()
452 errs += self.checkAndFixNormPars()
454 for mss in sorted(self.allselpars.keys()):
455 if self.allimpars["0"]["specmode"] == "cubedata":
456 self.allselpars[mss]["outframe"] = "Undefined"
457 if self.allimpars["0"]["specmode"] == "cubesource":
458 self.allselpars[mss]["outframe"] = "REST"
459 ### MOVE this segment of code to the constructor so that it's clear which parameters go where !
460 ### Copy them from 'impars' to 'normpars' and 'decpars'
461 self.iterpars["allimages"] = {}
462 for immod in self.allimpars.keys():
463 self.allnormpars[immod]["imagename"] = self.allimpars[immod][
464 "imagename"
465 ]
466 self.alldecpars[immod]["imagename"] = self.allimpars[immod]["imagename"]
467 self.allgridpars[immod]["imagename"] = self.allimpars[immod][
468 "imagename"
469 ]
470 self.iterpars["allimages"][immod] = {
471 "imagename": self.allimpars[immod]["imagename"],
472 "multiterm": (self.alldecpars[immod]["deconvolver"] == "mtmfs"),
473 }
475 ## Integers need to be NOT numpy versions.
476 self.fixIntParam(self.allimpars, "imsize")
477 self.fixIntParam(self.allimpars, "nchan")
478 self.fixIntParam(self.allimpars, "nterms")
479 self.fixIntParam(self.allnormpars, "nterms")
480 self.fixIntParam(self.alldecpars, "nterms")
481 self.fixIntParam(self.allgridpars, "facets")
482 self.fixIntParam(self.allgridpars, "chanchunks")
483 except Exception as exc:
484 if len(errs) > 0:
485 # errs string indicates that maybe this exception was our fault, indicate as such and provide the errs string to the user
486 raise Exception(
487 "Parameter Errors : \n{}\nThese errors may have caused the '{}'".format(
488 errs, type(exc)
489 )
490 )
491 else:
492 # something unforseen happened, just re-throw the exception
493 raise
495 ## If there are errors, print a message and exit.
496 if len(errs) > 0:
497 # casalog.post('Parameter Errors : \n' + errs,'WARN')
498 raise Exception("Parameter Errors : \n" + errs)
499 return True
501 ###### Start : Parameter-checking functions ##################
503 def checkAndFixSelectionPars(self):
504 errs = ""
506 # If it's already a dict with ms0,ms1,etc...leave it be.
507 ok = True
508 for kk in self.allselpars.keys():
509 if kk.find("ms") != 0:
510 ok = False
512 if ok == True:
513 # casalog.post("Already in correct format")
514 return errs
516 #print("allselpars=",self.allselpars)
517 # msname, field, spw, etc must all be equal-length lists of strings, or all except msname must be of length 1.
518 if not "msname" in self.allselpars:
519 errs = errs + "MS name(s) not specified"
520 else:
521 if type(self.allselpars['msname']) == list:
522 #(timesortedvislist, times) = sort_mslist(self.allselpars['msname'])
523 (timesortedvislist, times, newindex) = self.mslist_timesorting(self.allselpars['msname'])
524 if timesortedvislist != self.allselpars['msname']:
525 self.allselpars['msname'] = timesortedvislist
526 casalog.post("Sorting the vis list by time. The new vis list:"+ str(self.allselpars['msname']))
527 for selp in ['spw','field','timestr','uvdist','antenna','scan','obs','state']:
528 if type(self.allselpars[selp]) == list and len(self.allselpars[selp]) == len(newindex):
529 self.allselpars[selp] = [self.allselpars[selp][i] for i in newindex]
531 #msdiff = check_mslist(self.allselpars['msname'], ignore_tables=['SORTED_TABLE', 'ASDM*'])
532 msdiff = check_mslist(self.allselpars['msname'], ignore_tables=['SORTED_TABLE', 'ASDM*'], testcontent=False)
534 # Only call this if vis == list and there is mismatch in wtspec columns
535 # Maybe expanded for other checks later...
536 if msdiff != {}:
537 #print("MS diff===",msdiff)
538 noWtspecmslist=[]
539 for msfile, diff_info in msdiff.items():
540 # check Main
541 if 'Main' in diff_info:
542 for diffkey in diff_info['Main']:
543 if diffkey == "missingcol_a" or diffkey == "missingcol_b":
544 if ('WEIGHT_SPECTRUM' in diff_info['Main']['missingcol_a'] and
545 self.allselpars['msname'][0] not in noWtspecmslist):
546 noWtspecmslist.append(self.allselpars['msname'][0])
547 if ('WEIGHT_SPECTRUM' in diff_info['Main']['missingcol_b'] and
548 msfile not in noWtspecmslist):
549 noWtspecmslist.append(msfile)
550 # repalce this by addwtspec(list_of_ms_withoutWtSpec)
551 #self.checkmsforwtspec`
552 if noWtspecmslist!=[]:
553 #print ("OK addwtspec to "+str(noWtspecmslist))
554 self.addwtspec(noWtspecmslist)
556 selkeys = self.allselpars.keys()
558 # Convert all non-list parameters into lists.
559 for par in selkeys:
560 if type(self.allselpars[par]) != list:
561 self.allselpars[par] = [self.allselpars[par]]
563 # Check that all are the same length as nvis
564 # If not, and if they're single, replicate them nvis times
565 nvis = len(self.allselpars["msname"])
567 if nvis==0:
568 errs = errs + "Input MS list is empty"
569 return errs
571 for par in selkeys:
572 if len(self.allselpars[par]) > 1 and len(self.allselpars[par]) != nvis:
573 errs = (
574 errs
575 + str(par)
576 + " must have a single entry, or "
577 + str(nvis)
578 + " entries to match vis list \n"
579 )
580 return errs
581 else: # Replicate them nvis times if needed.
582 if len(self.allselpars[par]) == 1:
583 for ms in range(1, nvis):
584 self.allselpars[par].append(self.allselpars[par][0])
586 # Now, all parameters are lists of strings each of length 'nvis'.
587 # Put them into separate dicts per MS.
588 selparlist = {}
589 for ms in range(0, nvis):
590 selparlist["ms" + str(ms)] = {}
591 for par in selkeys:
592 selparlist["ms" + str(ms)][par] = self.allselpars[par][ms]
594 synu = synthesisutils()
595 selparlist["ms" + str(ms)] = synu.checkselectionparams(
596 selparlist["ms" + str(ms)]
597 )
598 synu.done()
600 # casalog.post(selparlist)
601 self.allselpars = selparlist
603 return errs
605 def makeImagingParamLists(self, parallel):
606 errs = ""
607 # casalog.post("specmode=",self.allimpars['0']['specmode'], " parallel=",parallel)
608 ## Multiple images have been specified.
609 ## (1) Parse the outlier file and fill a list of imagedefinitions
610 ## OR (2) Parse lists per input parameter into a list of parameter-sets (imagedefinitions)
611 ### The code below implements (1)
612 outlierpars = []
613 parseerrors = ""
614 if len(self.outlierfile) > 0:
615 outlierpars, parseerrors = self.parseOutlierFile(self.outlierfile)
616 if parallel:
617 casalog.post("CALLING checkParallelMFMixModes...")
618 errs = self.checkParallelMFMixedModes(self.allimpars, outlierpars)
619 if len(errs):
620 return errs
622 if len(parseerrors) > 0:
623 errs = errs + "Errors in parsing outlier file : " + parseerrors
624 return errs
626 # Initialize outlier parameters with defaults
627 # Update outlier parameters with modifications from outlier files
628 for immod in range(0, len(outlierpars)):
629 modelid = str(immod + 1)
630 self.allimpars[modelid] = copy.deepcopy(self.allimpars["0"])
631 self.allimpars[modelid].update(outlierpars[immod]["impars"])
632 self.allgridpars[modelid] = copy.deepcopy(self.allgridpars["0"])
633 self.allgridpars[modelid].update(outlierpars[immod]["gridpars"])
634 self.alldecpars[modelid] = copy.deepcopy(self.alldecpars["0"])
635 self.alldecpars[modelid].update(outlierpars[immod]["decpars"])
636 self.allnormpars[modelid] = copy.deepcopy(self.allnormpars["0"])
637 self.allnormpars[modelid].update(outlierpars[immod]["normpars"])
638 self.alldecpars[modelid]["id"] = immod + 1 ## Try to eliminate.
640 # casalog.post(self.allimpars)
642 #
643 # casalog.post("REMOVING CHECKS to check...")
644 #### This does not handle the conversions of the csys correctly.....
645 ####
646 # for immod in self.allimpars.keys() :
647 # tempcsys = {}
648 # if 'csys' in self.allimpars[immod]:
649 # tempcsys = self.allimpars[immod]['csys']
650 #
651 # synu = synthesisutils()
652 # self.allimpars[immod] = synu.checkimageparams( self.allimpars[immod] )
653 # synu.done()
654 #
655 # if len(tempcsys.keys())==0:
656 # self.allimpars[immod]['csys'] = tempcsys
658 ## Check for name increments, and copy from impars to decpars and normpars.
659 self.handleImageNames()
661 return errs
663 def handleImageNames(self):
665 for immod in self.allimpars.keys():
666 inpname = self.allimpars[immod]["imagename"]
668 ### If a directory name is embedded in the image name, check that the dir exists.
669 if inpname.count("/"):
670 splitname = inpname.split("/")
671 prefix = splitname[len(splitname) - 1]
672 dirname = inpname[0 : len(inpname) - len(prefix)] # has '/' at end
673 if not os.path.exists(dirname):
674 casalog.post("Making directory : " + dirname, "INFO")
675 os.mkdir(dirname)
677 ### Check for name increments
678 # if self.reusename == False:
680 if (
681 self.allimpars["0"]["restart"] == False
682 ): # Later, can change this to be field dependent too.
683 ## Get a list of image names for all fields (to sync name increment ids across fields)
684 inpnamelist = {}
685 for immod in self.allimpars.keys():
686 inpnamelist[immod] = self.allimpars[immod]["imagename"]
688 newnamelist = self.incrementImageNameList(inpnamelist)
690 if len(newnamelist) != len(self.allimpars.keys()):
691 casalog.post(
692 "Internal Error : Non matching list lengths in refimagerhelper::handleImageNames. Not updating image names",
693 "WARN",
694 )
695 else:
696 for immod in self.allimpars.keys():
697 self.allimpars[immod]["imagename"] = newnamelist[immod]
699 def checkAndFixIterationPars(self):
700 errs = ""
702 # Bother checking only if deconvolution iterations are requested
703 if self.iterpars["niter"] > 0:
704 # Make sure cycleniter is less than or equal to niter.
705 if (
706 self.iterpars["cycleniter"] <= 0
707 or self.iterpars["cycleniter"] > self.iterpars["niter"]
708 ):
709 if self.iterpars["interactive"] == False:
710 self.iterpars["cycleniter"] = self.iterpars["niter"]
711 else:
712 self.iterpars["cycleniter"] = min(self.iterpars["niter"], 100)
714 # saving model is done separately outside of iter. control for interactive clean and or automasking cases
716 if self.iterpars['savemodel']!='none':
717 if self.iterpars['interactive']==True or self.alldecpars['0']['usemask']=='auto-multithresh' or \
718 self.alldecpars['0']['nsigma']>0.0:
719 self.iterpars['savemodel']='none'
720 for visid in self.allselpars:
721 self.allselpars[visid]['readonly']=True
722 self.allselpars[visid]['usescratch']=False
725 return errs
727 def checkAndFixNormPars(self):
728 errs = ""
730 # for modelid in self.allnormpars.keys():
731 # if len(self.allnormpars[modelid]['workdir'])==0:
732 # self.allnormpars[modelid]['workdir'] = self.allnormpars['0']['imagename'] + '.workdir'
734 return errs
736 ###### End : Parameter-checking functions ##################
738 ## Parse outlier file and construct a list of imagedefinitions (dictionaries).
739 def parseOutlierFile(self, outlierfilename=""):
740 returnlist = []
741 errs = "" # must be empty for no error
743 if len(outlierfilename) > 0 and not os.path.exists(outlierfilename):
744 errs += "Cannot find outlier file : " + outlierfilename + "\n"
745 return returnlist, errs
747 fp = open(outlierfilename, "r")
748 thelines = fp.readlines()
749 tempimpar = {}
750 tempgridpar = {}
751 tempweightpar = {}
752 tempdecpar = {}
753 tempnormpar = {}
754 for oneline in thelines:
755 aline = oneline.replace("\n", "")
756 # aline = oneline.replace(' ','').replace('\n','')
757 if len(aline) > 0 and aline.find("#") != 0:
758 parpair = aline.split("=")
759 parpair[0] = parpair[0].replace(" ", "")
760 # casalog.post(parpair)
761 if len(parpair) != 2:
762 errs += "Error in line containing : " + oneline + "\n"
763 if parpair[0] == "imagename" and tempimpar != {}:
764 # returnlist.append({'impars':tempimpar, 'gridpars':tempgridpar, 'weightpars':tempweightpar, 'decpars':tempdecpar} )
765 returnlist.append(
766 {
767 "impars": tempimpar,
768 "gridpars": tempgridpar,
769 "weightpars": tempweightpar,
770 "decpars": tempdecpar,
771 "normpars": tempnormpar,
772 }
773 )
774 tempimpar = {}
775 tempgridpar = {}
776 tempweightpar = {}
777 tempdecpar = {}
778 tempnormpar = {}
779 usepar = False
780 if parpair[0] in self.outimparlist:
781 tempimpar[parpair[0]] = parpair[1]
782 usepar = True
783 if parpair[0] in self.outgridparlist:
784 tempgridpar[parpair[0]] = parpair[1]
785 usepar = True
786 if parpair[0] in self.outweightparlist:
787 tempweightpar[parpair[0]] = parpair[1]
788 usepar = True
789 if parpair[0] in self.outdecparlist:
790 tempdecpar[parpair[0]] = parpair[1]
791 usepar = True
792 if parpair[0] in self.outnormparlist:
793 tempnormpar[parpair[0]] = parpair[1]
794 usepar = True
795 if usepar == False:
796 casalog.post("Ignoring unknown parameter pair : " + oneline)
798 if len(errs) == 0:
799 returnlist.append(
800 {
801 "impars": tempimpar,
802 "gridpars": tempgridpar,
803 "weightpars": tempweightpar,
804 "decpars": tempdecpar,
805 "normpars": tempnormpar,
806 }
807 )
809 ## Extra parsing for a few parameters.
810 returnlist = self.evalToTarget(returnlist, "impars", "imsize", "intvec")
811 returnlist = self.evalToTarget(returnlist, "impars", "nchan", "int")
812 returnlist = self.evalToTarget(returnlist, "impars", "cell", "strvec")
813 returnlist = self.evalToTarget(returnlist, "impars", "nterms", "int")
814 returnlist = self.evalToTarget(returnlist, "decpars", "nterms", "int")
815 returnlist = self.evalToTarget(returnlist, "normpars", "nterms", "int")
816 returnlist = self.evalToTarget(returnlist, "gridpars", "wprojplanes", "int")
817 # returnlist = self.evalToTarget( returnlist, 'impars', 'reffreq', 'strvec' )
819 # casalog.post(returnlist)
820 return returnlist, errs
822 def evalToTarget(self, globalpars, subparkey, parname, dtype="int"):
823 try:
824 for fld in range(0, len(globalpars)):
825 if parname in globalpars[fld][subparkey]:
826 if dtype == "int" or dtype == "intvec":
827 val_e = eval(globalpars[fld][subparkey][parname])
828 if dtype == "strvec":
829 tcell = globalpars[fld][subparkey][parname]
830 tcell = (
831 tcell.replace(" ", "")
832 .replace("[", "")
833 .replace("]", "")
834 .replace("'", "")
835 )
836 tcells = tcell.split(",")
837 val_e = []
838 for cell in tcells:
839 val_e.append(cell)
841 globalpars[fld][subparkey][parname] = val_e
842 except:
843 casalog.post(
844 'Cannot evaluate outlier field parameter "' + parname + '"', "ERROR"
845 )
847 return globalpars
849 def printParameters(self):
850 casalog.post("SelPars : " + str(self.allselpars), "INFO2")
851 casalog.post("ImagePars : " + str(self.allimpars), "INFO2")
852 casalog.post("GridPars : " + str(self.allgridpars), "INFO2")
853 casalog.post("NormPars : " + str(self.allnormpars), "INFO2")
854 casalog.post("Weightpars : " + str(self.weightpars), "INFO2")
855 casalog.post("DecPars : " + str(self.alldecpars), "INFO2")
856 casalog.post("IterPars : " + str(self.iterpars), "INFO2")
858 def incrementImageName(self, imagename):
859 dirname = "."
860 prefix = imagename
862 if imagename.count("/"):
863 splitname = imagename.split("/")
864 prefix = splitname[len(splitname) - 1]
865 ### if it has a leading / then absolute path is assumed
866 dirname = (
867 (imagename[0 : len(imagename) - len(prefix)])
868 if (imagename[0] == "/")
869 else ("./" + imagename[0 : len(imagename) - len(prefix)])
870 ) # has '/' at end
872 inamelist = [fn for fn in os.listdir(dirname) if any([fn.startswith(prefix)])]
874 if len(inamelist) == 0:
875 newimagename = dirname[2:] + prefix
876 else:
877 nlen = len(prefix)
878 maxid = 1
879 for iname in inamelist:
880 startind = iname.find(prefix + "_")
881 if startind == 0:
882 idstr = (iname[nlen + 1 : len(iname)]).split(".")[0]
883 if idstr.isdigit():
884 val = eval(idstr)
885 if val > maxid:
886 maxid = val
887 newimagename = dirname[2:] + prefix + "_" + str(maxid + 1)
889 casalog.post("Using : {}".format(newimagename))
890 return newimagename
892 def incrementImageNameList(self, inpnamelist):
894 dirnames = {}
895 prefixes = {}
897 for immod in inpnamelist.keys():
898 imagename = inpnamelist[immod]
899 dirname = "."
900 prefix = imagename
902 if imagename.count("/"):
903 splitname = imagename.split("/")
904 prefix = splitname[len(splitname) - 1]
905 dirname = (
906 (imagename[0 : len(imagename) - len(prefix)])
907 if (imagename[0] == "/")
908 else ("./" + imagename[0 : len(imagename) - len(prefix)])
909 ) # has '/' at end
911 dirnames[immod] = dirname
912 prefixes[immod] = prefix
914 maxid = 0
915 for immod in inpnamelist.keys():
916 prefix = prefixes[immod]
917 inamelist = [
918 fn for fn in os.listdir(dirnames[immod]) if any([fn.startswith(prefix)])
919 ]
920 nlen = len(prefix)
922 if len(inamelist) == 0:
923 locmax = 0
924 else:
925 locmax = 1
927 cleanext = [".image", ".residual", ".model", ".psf", ".sumwt", ".tt0"]
928 incremented = False
929 for iname in inamelist:
930 rootname, ext = os.path.splitext(iname)
931 if ext in cleanext:
932 startind = iname.find(prefix + "_")
933 if startind == 0:
934 idstr = (iname[nlen + 1 : len(iname)]).split(".")[0]
935 if idstr.isdigit():
936 val = eval(idstr)
937 incremented = True
938 if val > locmax:
939 locmax = val
940 elif startind == -1:
941 if ext == ".tt0":
942 # need one more pass to extract rootname
943 rootname, ext = os.path.splitext(rootname)
944 if rootname == prefix:
945 # the file name with root file name only
946 incremented = True
948 if not incremented:
949 locmax = 0
950 if locmax > maxid:
951 maxid = locmax
953 newimagenamelist = {}
954 for immod in inpnamelist.keys():
955 if maxid == 0:
956 newimagenamelist[immod] = inpnamelist[immod]
957 else:
958 newimagenamelist[immod] = (
959 dirnames[immod][2:] + prefixes[immod] + "_" + str(maxid + 1)
960 )
962 # casalog.post('Input : ', inpnamelist)
963 # casalog.post('Dirs : ', dirnames)
964 # casalog.post('Pre : ', prefixes)
965 # casalog.post('Max id : ', maxid)
966 # casalog.post('Using : ', newimagenamelist)
967 return newimagenamelist
969 ## Guard against numpy int32,int64 types which don't convert well across tool boundary.
970 ## For CAS-8250. Remove when CAS-6682 is done.
971 def fixIntParam(self, allpars, parname):
972 for immod in allpars.keys():
973 if parname in allpars[immod]:
974 ims = allpars[immod][parname]
975 if type(ims) != list:
976 ims = int(ims)
977 else:
978 for el in range(0, len(ims)):
979 ims[el] = int(ims[el])
980 allpars[immod][parname] = ims
982 # check for non-supported multifield in mixed modes in parallel
983 # (e.g. combination cube and continuum for main and outlier fields)
984 def checkParallelMFMixedModes(self, allimpars, outlierpars):
985 errmsg = ""
986 casalog.post("outlierpars=={}".format(outlierpars))
987 mainspecmode = allimpars["0"]["specmode"]
988 mainnchan = allimpars["0"]["nchan"]
989 casalog.post("mainspecmode={} mainnchan={}".format(mainspecmode, mainnchan))
990 cubeoutlier = False
991 contoutlier = False
992 isnchanmatch = True
993 for immod in range(0, len(outlierpars)):
994 if "impars" in outlierpars[immod]:
995 if "nchan" in outlierpars[immod]["impars"]:
996 if outlierpars[immod]["impars"]["nchan"] > 1:
997 cubeoutlier = True
998 if outlierpars[immod]["impars"]["nchan"] != mainnchan:
999 isnchanmatch = False
1000 else:
1001 contoutlier = True
1002 else:
1003 if "specmode" in outlierpars[immod]["impars"]:
1004 if outlierpars[immod]["impars"]["specmode"] == "mfs":
1005 contoutlier = True
1006 if mainspecmode.find("cube") == 0:
1007 if contoutlier:
1008 errmsg = "Mixed cube and continuum mode for multifields is currently not supported for parallel mode"
1009 else: # all cube modes, but need to check if the nchans are the same
1010 if not isnchanmatch:
1011 errmsg = "Cubes for multifields with different nchans are currently not supported for parallel mode "
1012 else: # mfs
1013 if cubeoutlier:
1014 errmsg = "Mixed continuum and cube mode for multifields is currently not supported for parallel mode"
1015 errs = errmsg
1016 return errs
1018 def checkmsforwtspec(self):
1019 ''' check if WEIGHT_SPECTRUM column exist when
1020 a list of vis is given. Add the column for an MS
1021 which does not have one if other MSs have the column.
1022 This is a workaround for the issue probably in Vi/VB2
1023 not handling the state change for the optional column
1024 when dealing with multiples MSs
1025 '''
1026 mycb = calibrater()
1027 mytb = table()
1028 haswtspec=False
1029 mswithnowtspec=[]
1030 nms = 1
1031 if type(self.allselpars['msname'])==list:
1032 nms = len(self.allselpars['msname'])
1034 if nms > 1:
1035 for inms in self.allselpars['msname']:
1036 mytb.open(inms)
1037 cols = mytb.colnames()
1038 mytb.close()
1039 if 'WEIGHT_SPECTRUM' in cols:
1040 haswtspec=True
1041 else:
1042 mswithnowtspec.append(inms)
1043 if haswtspec and len(mswithnowtspec) > 0:
1044 casalog.post("Some of the MSes donot have WEIGHT_SPECTRUM while some other do."+
1045 " Automatically adding the column and initialize for those don't to avoid a process failure.","WARN")
1046 for inms in mswithnowtspec:
1047 mycb.open(inms, addcorr=False, addmodel=False)
1048 mycb.initweights(wtmode='weight', dowtsp=True)
1049 mycb.close()
1050 # noOp for nms==1
1052 def mslist_timesorting(self, mslist):
1053 '''
1054 wrapper for mslisthelper.sort_mslist to get a sorting order w.r.t the original
1055 '''
1056 (thenewmslist, times) = sort_mslist(mslist)
1057 theindex = []
1058 for vnew in thenewmslist:
1059 for vold in mslist:
1060 if vnew == vold:
1061 theindex.append(mslist.index(vnew))
1062 return (thenewmslist, times, theindex)
1064 def addwtspec(self, mslist):
1065 '''
1066 Add the column for an MS which does not have one if other MSs have the column.
1067 This is a workaround for the issue probably in Vi/VB2
1068 not handling the state change for the optional column
1069 when dealing with multiples MSs
1070 '''
1071 mycb = calibrater()
1073 if len(mslist) > 0:
1074 casalog.post("Some of the MSes donot have WEIGHT_SPECTRUM while some other do."+
1075 " Automatically adding the column and initialize using the existing WEIGHT column for those don't to avoid a process failure.","WARN")
1076 casalog.post("Adding WEIGHT_SPECTRUM in the following MS(s): "+str(mslist),"WARN")
1077 for inms in mslist:
1078 mycb.open(inms, addcorr=False, addmodel=False)
1079 mycb.initweights(wtmode='weight', dowtsp=True)
1080 mycb.close()
1081 mycb.done()
1082 # noOp for len(mlist) ==0
1084 ############################
1087#################################################################################################
1088def backupoldfile(thefile=""):
1089 import copy
1090 import shutil
1092 if thefile == "" or (not os.path.exists(thefile)):
1093 return
1094 outpathdir = os.path.realpath(os.path.dirname(thefile))
1095 outpathfile = outpathdir + os.path.sep + os.path.basename(thefile)
1096 k = 0
1097 backupfile = outpathfile + "." + str(k)
1098 prevfile = "--------"
1099 while os.path.exists(backupfile):
1100 k = k + 1
1101 prevfile = copy.copy(backupfile)
1102 if os.path.exists(prevfile) and filecmp.cmp(outpathfile, prevfile):
1103 ##avoid making multiple copies of the same file
1104 return
1105 backupfile = outpathfile + "." + str(k)
1106 shutil.copy2(outpathfile, backupfile)
1109def saveparams2last(func=None, multibackup=True):
1110 """This function is a decorator function that allows for
1111 task.last to be saved even if calling without casashell. Also
1112 saves unique revisions ...just like the vax/vms style of revision saving
1113 by default. set multibackup=False to no not have old revisions kept
1114 """
1115 if not func:
1116 return functools.partial(saveparams2last, multibackup=multibackup)
1118 @functools.wraps(func)
1119 def wrapper_saveparams(*args, **kwargs):
1120 # multibackup=True
1121 outfile = func.__name__ + ".last"
1122 # print('args {} and kwargs {}'.format(args, kwargs))
1123 # print('length of args {}, and kwargs {}'.format(len(args), len(kwargs)))
1124 params = {}
1125 byIndex = list()
1126 if len(kwargs) == 0:
1127 paramsname = list(inspect.signature(func).parameters)
1128 # params={paramsname[i]: args[i] for i in range(len(args))}
1129 params = OrderedDict(zip(paramsname, args))
1130 byIndex = list(params)
1131 else:
1132 params = kwargs
1133 byIndex = list(params)
1134 ###for some reason the dictionary is in reverse
1135 byIndex.reverse()
1136 # print('@@@@MULTIBACKUP {}, params {}'.format(multibackup, params))
1137 if multibackup:
1138 backupoldfile(outfile)
1139 with open(outfile, "w") as _f:
1140 for _i in range(len(byIndex)):
1141 _f.write("%-20s = %s\n" % (byIndex[_i], repr(params[byIndex[_i]])))
1142 _f.write("#" + func.__name__ + "( ")
1143 for _i in range(len(byIndex)):
1144 _f.write("%s=%s" % (byIndex[_i], repr(params[byIndex[_i]])))
1145 if _i < len(params) - 1:
1146 _f.write(",")
1147 _f.write(" )\n")
1148 ###End of stuff before task is called
1149 retval = func(*args, **kwargs)
1150 ###we could do something here post task
1151 return retval
1153 return wrapper_saveparams
1156######################################################
1159def determineFreqRange(
1160 vis: str = "", fieldid: int = 0, spw: str = "*"
1161) -> Tuple[np.double, np.double]:
1162 _tb = table()
1163 _ms = ms()
1164 _su = synthesisutils()
1165 _qa = quanta()
1166 minFreq = 1.0e20
1167 maxFreq = 0.0
1168 _tb.open(vis)
1169 fieldids = _tb.getcol("FIELD_ID")
1170 _tb.done()
1171 # advisechansel does not work on fieldids not in main
1172 if fieldid not in fieldids:
1173 fieldid = fieldids[0]
1174 frange = _su.advisechansel(
1175 msname=vis, getfreqrange=True, fieldid=fieldid, spwselection=spw
1176 )
1177 if minFreq > _qa.convert(frange["freqstart"], "Hz")["value"]:
1178 minFreq = _qa.convert(frange["freqstart"], "Hz")["value"]
1179 if maxFreq < _qa.convert(frange["freqend"], "Hz")["value"]:
1180 maxFreq = _qa.convert(frange["freqend"], "Hz")["value"]
1182 if minFreq > maxFreq:
1183 raise Exception(f"Failed to determine frequency range in ms {vis}")
1184 freqwidth = maxFreq - minFreq
1185 return (minFreq, freqwidth)