Coverage for /home/casatest/venv/lib/python3.12/site-packages/casatasks/private/imagerhelpers/input_parameters.py: 33%
554 statements
« prev ^ index » next coverage.py v7.10.4, created at 2025-08-21 07:43 +0000
1import os
2import math
3import shutil
4import string
5import time
6import re
7import copy
8import pprint
9import functools
10import inspect
11from collections import OrderedDict
12import numpy as np
13from typing import Tuple
14import filecmp
17from casatools import synthesisutils
18from casatools import table, ms, synthesisutils, quanta
19from casatools import calibrater
20from casatasks import casalog
21from casatasks.private.mslisthelper import check_mslist
22from casatasks.private.mslisthelper import sort_mslist
25"""
26A set of helper functions for the tasks tclean
28Summary...
30"""
33######################################################
34######################################################
35######################################################
38class ImagerParameters:
39 def __init__(
40 self,
41 # Input Data: what gets in
42 msname="",
43 # Output Data: what goes out
44 imagename="",
45 # The remaining parameters are Control Parameters:
46 # they control How what gets in goes out
47 # Data Selection
48 field="",
49 spw="",
50 timestr="",
51 uvdist="",
52 antenna="",
53 scan="",
54 obs="",
55 state="",
56 datacolumn="corrected",
57 # Image Definition
58 imsize=[1, 1],
59 cell=[10.0, 10.0],
60 phasecenter="",
61 stokes="I",
62 projection="SIN",
63 startmodel="",
64 # Spectral Parameters
65 specmode="mfs",
66 reffreq="",
67 nchan=1,
68 start="",
69 width="",
70 outframe="LSRK",
71 veltype="radio",
72 restfreq=[""],
73 sysvel="",
74 sysvelframe="",
75 interpolation="nearest",
76 perchanweightdensity=False,
77 gridder="standard",
78 # ftmachine='gridft',
79 facets=1,
80 chanchunks=1,
81 wprojplanes=1,
82 vptable="",
83 usepointing=False,
84 mosweight=False,
85 aterm=True,
86 psterm=True,
87 mterm=True,
88 wbawp=True,
89 cfcache="",
90 dopbcorr=True,
91 conjbeams=True,
92 computepastep=360.0,
93 rotatepastep=360.0,
94 pointingoffsetsigdev=[30.0, 30.0],
95 # Normalizer group
96 pblimit=0.01,
97 normtype="flatnoise",
98 psfcutoff=0.35,
99 makesingledishnormalizer=False,
100 outlierfile="",
101 restart=True,
102 weighting="natural",
103 robust=0.5,
104 noise="0.0Jy",
105 npixels=0,
106 uvtaper=[],
107 niter=0,
108 cycleniter=0,
109 loopgain=0.1,
110 threshold="0.0Jy",
111 nsigma=0.0,
112 cyclefactor=1.0,
113 minpsffraction=0.1,
114 maxpsffraction=0.8,
115 interactive=False,
116 fullsummary=False,
117 nmajor=-1,
118 deconvolver="hogbom",
119 scales=[],
120 nterms=1,
121 scalebias=0.0,
122 restoringbeam=[],
123 # mtype='default',
124 usemask="user",
125 mask="",
126 pbmask=0.0,
127 maskthreshold="",
128 maskresolution="",
129 nmask=0,
130 # autoadjust=False,
131 sidelobethreshold=5.0,
132 noisethreshold=3.0,
133 lownoisethreshold=3.0,
134 negativethreshold=0.0,
135 smoothfactor=1.0,
136 minbeamfrac=0.3,
137 cutthreshold=0.01,
138 growiterations=100,
139 dogrowprune=True,
140 minpercentchange=0.0,
141 verbose=False,
142 fastnoise=True,
143 fusedthreshold=0.0,
144 largestscale=-1,
145 # usescratch=True,
146 # readonly=True,
147 calcres=True,
148 calcpsf=True,
149 savemodel="none",
150 parallel=False,
151 workdir="",
152 # CFCache params
153 cflist=[],
154 # single-dish imaging params
155 gridfunction="SF",
156 convsupport=-1,
157 truncate="-1",
158 gwidth="-1",
159 jwidth="-1",
160 pointingcolumntouse="direction",
161 convertfirst="never",
162 minweight=0.0,
163 clipminmax=False,
164 ):
166 self.allparameters = dict(locals())
167 del self.allparameters["self"]
169 self.defaultKey = "0"
170 # ---- Selection params. For multiple MSs, all are lists.
171 # For multiple nodes, the selection parameters are modified inside
172 # PySynthesisImager
173 self.allselpars = {
174 "msname": msname,
175 "field": field,
176 "spw": spw,
177 "scan": scan,
178 "timestr": timestr,
179 "uvdist": uvdist,
180 "antenna": antenna,
181 "obs": obs,
182 "state": state,
183 "datacolumn": datacolumn,
184 "savemodel": savemodel,
185 }
186 # ---- Imaging/deconvolution parameters
187 # The outermost dictionary index is image field.
188 # The '0' or main field's parameters come from the task parameters
189 # The outlier '1', '2', .... parameters come from the outlier file
190 self.outlierfile = outlierfile
191 # Initialize the parameter lists with the 'main' or '0' field's
192 # parameters
193 # ---- Image definition
194 self.allimpars = {
195 self.defaultKey: {
196 # Image
197 "imagename": imagename,
198 "nchan": nchan,
199 "imsize": imsize,
200 "cell": cell,
201 "phasecenter": phasecenter,
202 "stokes": stokes,
203 # Frequency axis
204 "specmode": specmode,
205 "start": start,
206 "width": width,
207 "veltype": veltype,
208 "nterms": nterms,
209 "restfreq": restfreq,
210 # Output frame
211 "outframe": outframe,
212 "reffreq": reffreq,
213 "sysvel": sysvel,
214 "sysvelframe": sysvelframe,
215 # Projection
216 "projection": projection,
217 # Deconvolution
218 "restart": restart,
219 "startmodel": startmodel,
220 "deconvolver": deconvolver,
221 }
222 }
223 # ---- Gridding
224 self.allgridpars = {
225 self.defaultKey: {
226 "gridder": gridder,
227 # aterm group
228 "aterm": aterm,
229 "psterm": psterm,
230 "mterm": mterm,
231 "wbawp": wbawp,
232 # cfcache group
233 "cfcache": cfcache,
234 "usepointing": usepointing,
235 "dopbcorr": dopbcorr,
236 # conjbeams group
237 "conjbeams": conjbeams,
238 "computepastep": computepastep,
239 #
240 "rotatepastep": rotatepastep, #'mtype':mtype, # 'weightlimit':weightlimit,
241 "pointingoffsetsigdev": pointingoffsetsigdev,
242 # facets group
243 "facets": facets,
244 "chanchunks": chanchunks,
245 # interpolation group
246 "interpolation": interpolation,
247 "wprojplanes": wprojplanes,
248 # deconvolver group
249 "deconvolver": deconvolver,
250 "vptable": vptable,
251 "imagename": imagename,
252 # single-dish specific parameters
253 # ---- spatial coordinates
254 "pointingcolumntouse": pointingcolumntouse,
255 "convertfirst": convertfirst,
256 # ---- convolution function
257 "convfunc": gridfunction,
258 "convsupport": convsupport,
259 # ---- truncate group
260 "truncate": truncate,
261 "gwidth": gwidth,
262 "jwidth": jwidth,
263 # ---- minweight group
264 "minweight": minweight,
265 "clipminmax": clipminmax,
266 }
267 }
268 # ---- Weighting
269 if True: # Compute rmode and self.weightpars
270 rmode = "none"
271 if weighting == "briggsabs":
272 rmode = "abs"
273 weighting = "briggs"
274 elif weighting == "briggs":
275 rmode = "norm"
276 elif weighting == "briggsbwtaper":
277 rmode = "bwtaper"
278 weighting = "briggs"
280 self.weightpars = {
281 "type": weighting,
282 "rmode": rmode,
283 "robust": robust,
284 "noise": noise,
285 "npixels": npixels,
286 "uvtaper": uvtaper,
287 "multifield": mosweight,
288 "usecubebriggs": perchanweightdensity,
289 }
290 # ---- Normalizers ( this is where flat noise, flat sky rules will go... )
291 self.allnormpars = {
292 self.defaultKey: {
293 # pblimit group
294 "pblimit": pblimit,
295 "nterms": nterms,
296 "facets": facets,
297 # normtype group
298 "normtype": normtype,
299 "workdir": workdir,
300 # deconvolver group
301 "deconvolver": deconvolver,
302 "imagename": imagename,
303 "restoringbeam": restoringbeam,
304 "psfcutoff": psfcutoff,
305 "makesingledishnormalizer": makesingledishnormalizer,
306 "calcres": calcres,
307 "calcpsf": calcpsf,
308 }
309 }
310 # ---- Deconvolution
311 self.alldecpars = {
312 self.defaultKey: {
313 "id": 0,
314 "deconvolver": deconvolver,
315 "nterms": nterms,
316 "scales": scales,
317 "scalebias": scalebias,
318 "restoringbeam": restoringbeam,
319 "usemask": usemask,
320 "mask": mask,
321 "pbmask": pbmask,
322 "maskthreshold": maskthreshold,
323 "maskresolution": maskresolution,
324 "nmask": nmask,
325 #'maskresolution':maskresolution, 'nmask':nmask,'autoadjust':autoadjust,
326 "sidelobethreshold": sidelobethreshold,
327 "noisethreshold": noisethreshold,
328 "lownoisethreshold": lownoisethreshold,
329 "negativethreshold": negativethreshold,
330 "smoothfactor": smoothfactor,
331 "fusedthreshold": fusedthreshold,
332 "specmode": specmode,
333 "largestscale": largestscale,
334 "minbeamfrac": minbeamfrac,
335 "cutthreshold": cutthreshold,
336 "growiterations": growiterations,
337 "dogrowprune": dogrowprune,
338 "minpercentchange": minpercentchange,
339 "verbose": verbose,
340 "fastnoise": fastnoise,
341 "interactive": interactive,
342 "startmodel": startmodel,
343 "nsigma": nsigma,
344 "imagename": imagename,
345 "fullsummary": fullsummary,
346 }
347 }
348 # ---- Iteration control
349 self.iterpars = {
350 "niter": niter,
351 "cycleniter": cycleniter,
352 "threshold": threshold,
353 "loopgain": loopgain,
354 "interactive": interactive,
355 "cyclefactor": cyclefactor,
356 "minpsffraction": minpsffraction,
357 "maxpsffraction": maxpsffraction,
358 "savemodel": savemodel,
359 "nsigma": nsigma,
360 "nmajor": nmajor,
361 "fullsummary": fullsummary,
362 }
363 # ---- CFCache params
364 self.cfcachepars = {"cflist": cflist}
365 # ---- Parameters that may be internally modified for savemodel behavior
366 self.inpars = {
367 "savemodel": savemodel,
368 "interactive": interactive,
369 "nsigma": nsigma,
370 "usemask": usemask,
371 }
372 # ---- List of supported parameters in outlier files.
373 # All other parameters will default to the global values.
374 self.outimparlist = [
375 "imagename",
376 "nchan",
377 "imsize",
378 "cell",
379 "phasecenter",
380 "startmodel",
381 "start",
382 "width",
383 "nterms",
384 "reffreq",
385 "specmode",
386 ]
387 self.outgridparlist = ["gridder", "deconvolver", "wprojplanes"]
388 self.outweightparlist = []
389 self.outdecparlist = [
390 "deconvolver",
391 "startmodel",
392 "nterms",
393 "usemask",
394 "mask",
395 ]
396 self.outnormparlist = ["deconvolver", "weightlimit", "nterms"]
398 ret = self.checkParameters(parallel)
399 if ret == False:
400 casalog.post(
401 "Found errors in input parameters. Please check.", "WARN"
402 )
404 self.printParameters()
406 def resetParameters(self):
407 """reset parameters to the original settting for interactive, nsigma, auto-multithresh when savemodel!='none'"""
408 if self.inpars["savemodel"] != "none" and (
409 self.inpars["interactive"] == True
410 or self.inpars["usemask"] == "auto-multithresh"
411 or self.inpars["nsigma"] > 0.0
412 ):
413 # in checkAndFixIterationPars(), when saving model is on, the internal params, readonly and usescrath are set to True and False,
414 # respectively. So this needs to be undone before calling predictModel.
415 self.iterpars["savemodel"] = self.inpars["savemodel"]
416 if self.inpars["savemodel"] == "modelcolumn":
417 for key in self.allselpars: # for all MSes
418 self.allselpars[key]["readonly"] = False
419 self.allselpars[key]["usescratch"] = True
421 elif self.inpars["savemodel"] == "virtual":
422 for key in self.allselpars: # for all MSes
423 self.allselpars[key]["readonly"] = False
424 self.allselpars[key]["usescratch"] = False
426 def getAllPars(self):
427 """Return the state of all parameters"""
428 return self.allparameters
430 def getSelPars(self):
431 return self.allselpars
433 def getImagePars(self):
434 return self.allimpars
436 def getGridPars(self):
437 return self.allgridpars
439 def getWeightPars(self):
440 return self.weightpars
442 def getDecPars(self):
443 return self.alldecpars
445 def getIterPars(self):
446 return self.iterpars
448 def getNormPars(self):
449 return self.allnormpars
451 def getCFCachePars(self):
452 return self.cfcachepars
454 def setSelPars(self, selpars):
455 for key in selpars.keys():
456 self.allselpars[key] = selpars[key]
458 def setImagePars(self, impars):
459 for key in impars.keys():
460 self.allimpars[key] = impars[key]
462 def setGridPars(self, gridpars):
463 for key in gridpars.keys():
464 self.allgridpars[key] = gridpars[key]
466 def setWeightPars(self, weightpars):
467 for key in weightpars.keys():
468 self.weightpars[key] = weightpars[key]
470 def setDecPars(self, decpars):
471 for key in decpars.keys():
472 self.alldecpars[key] = decpars[key]
474 def setIterPars(self, iterpars):
475 for key in iterpars.keys():
476 self.iterpars[key] = iterpars[key]
478 def setNormPars(self, normpars):
479 for key in normpars.keys():
480 self.allnormpars[key] = normpars[key]
482 def checkParameters(self, parallel=False):
483 # casalog.origin('refimagerhelper.checkParameters')
484 casalog.post("Verifying Input Parameters")
485 # Init the error-string
486 errs = ""
487 try:
488 errs += self.checkAndFixSelectionPars()
489 errs += self.makeImagingParamLists(parallel)
490 errs += self.checkAndFixIterationPars()
491 errs += self.checkAndFixNormPars()
493 for mss in sorted(self.allselpars.keys()):
494 if self.allimpars["0"]["specmode"] == "cubedata":
495 self.allselpars[mss]["outframe"] = "Undefined"
496 if self.allimpars["0"]["specmode"] == "cubesource":
497 self.allselpars[mss]["outframe"] = "REST"
498 ### MOVE this segment of code to the constructor so that it's clear which parameters go where !
499 ### Copy them from 'impars' to 'normpars' and 'decpars'
500 self.iterpars["allimages"] = {}
501 for immod in self.allimpars.keys():
502 self.allnormpars[immod]["imagename"] = self.allimpars[immod][
503 "imagename"
504 ]
505 self.alldecpars[immod]["imagename"] = self.allimpars[immod][
506 "imagename"
507 ]
508 self.allgridpars[immod]["imagename"] = self.allimpars[immod][
509 "imagename"
510 ]
511 self.iterpars["allimages"][immod] = {
512 "imagename": self.allimpars[immod]["imagename"],
513 "multiterm": (
514 self.alldecpars[immod]["deconvolver"] == "mtmfs"
515 ),
516 }
518 ## Integers need to be NOT numpy versions.
519 self.fixIntParam(self.allimpars, "imsize")
520 self.fixIntParam(self.allimpars, "nchan")
521 self.fixIntParam(self.allimpars, "nterms")
522 self.fixIntParam(self.allnormpars, "nterms")
523 self.fixIntParam(self.alldecpars, "nterms")
524 self.fixIntParam(self.allgridpars, "facets")
525 self.fixIntParam(self.allgridpars, "chanchunks")
526 except Exception as exc:
527 if len(errs) > 0:
528 # errs string indicates that maybe this exception was our fault, indicate as such and provide the errs string to the user
529 raise Exception(
530 "Parameter Errors : \n{}\nThese errors may have caused the '{}'".format(
531 errs, type(exc)
532 )
533 )
534 else:
535 # something unforseen happened, just re-throw the exception
536 raise
538 ## If there are errors, print a message and exit.
539 if len(errs) > 0:
540 # casalog.post('Parameter Errors : \n' + errs,'WARN')
541 raise Exception("Parameter Errors : \n" + errs)
542 return True
544 ###### Start : Parameter-checking functions ##################
546 def checkAndFixSelectionPars(self):
547 errs = ""
549 # If it's already a dict with ms0,ms1,etc...leave it be.
550 ok = True
551 for kk in self.allselpars.keys():
552 if kk.find("ms") != 0:
553 ok = False
555 if ok == True:
556 # casalog.post("Already in correct format")
557 return errs
559 # print("allselpars=",self.allselpars)
560 # msname, field, spw, etc must all be equal-length lists of strings, or all except msname must be of length 1.
561 if not "msname" in self.allselpars:
562 errs = errs + "MS name(s) not specified"
563 else:
564 if type(self.allselpars["msname"]) == list:
565 # (timesortedvislist, times) = sort_mslist(self.allselpars['msname'])
566 (timesortedvislist, times, newindex) = self.mslist_timesorting(
567 self.allselpars["msname"]
568 )
569 if timesortedvislist != self.allselpars["msname"]:
570 self.allselpars["msname"] = timesortedvislist
571 casalog.post(
572 "Sorting the vis list by time. The new vis list:"
573 + str(self.allselpars["msname"])
574 )
575 for selp in [
576 "spw",
577 "field",
578 "timestr",
579 "uvdist",
580 "antenna",
581 "scan",
582 "obs",
583 "state",
584 ]:
585 if type(self.allselpars[selp]) == list and len(
586 self.allselpars[selp]
587 ) == len(newindex):
588 self.allselpars[selp] = [
589 self.allselpars[selp][i] for i in newindex
590 ]
592 # msdiff = check_mslist(self.allselpars['msname'], ignore_tables=['SORTED_TABLE', 'ASDM*'])
593 msdiff = check_mslist(
594 self.allselpars["msname"],
595 ignore_tables=["SORTED_TABLE", "ASDM*"],
596 testcontent=False,
597 )
599 # Only call this if vis == list and there is mismatch in wtspec columns
600 # Maybe expanded for other checks later...
601 if msdiff != {}:
602 # print("MS diff===",msdiff)
603 noWtspecmslist = []
604 for msfile, diff_info in msdiff.items():
605 # check Main
606 if "Main" in diff_info:
607 for diffkey in diff_info["Main"]:
608 if (
609 diffkey == "missingcol_a"
610 or diffkey == "missingcol_b"
611 ):
612 if (
613 "WEIGHT_SPECTRUM"
614 in diff_info["Main"]["missingcol_a"]
615 and self.allselpars["msname"][0]
616 not in noWtspecmslist
617 ):
618 noWtspecmslist.append(
619 self.allselpars["msname"][0]
620 )
621 if (
622 "WEIGHT_SPECTRUM"
623 in diff_info["Main"]["missingcol_b"]
624 and msfile not in noWtspecmslist
625 ):
626 noWtspecmslist.append(msfile)
627 # repalce this by addwtspec(list_of_ms_withoutWtSpec)
628 # self.checkmsforwtspec`
629 if noWtspecmslist != []:
630 # print ("OK addwtspec to "+str(noWtspecmslist))
631 self.addwtspec(noWtspecmslist)
633 selkeys = self.allselpars.keys()
635 # Convert all non-list parameters into lists.
636 for par in selkeys:
637 if type(self.allselpars[par]) != list:
638 self.allselpars[par] = [self.allselpars[par]]
640 # Check that all are the same length as nvis
641 # If not, and if they're single, replicate them nvis times
642 nvis = len(self.allselpars["msname"])
644 if nvis == 0:
645 errs = errs + "Input MS list is empty"
646 return errs
648 for par in selkeys:
649 if (
650 len(self.allselpars[par]) > 1
651 and len(self.allselpars[par]) != nvis
652 ):
653 errs = (
654 errs
655 + str(par)
656 + " must have a single entry, or "
657 + str(nvis)
658 + " entries to match vis list \n"
659 )
660 return errs
661 else: # Replicate them nvis times if needed.
662 if len(self.allselpars[par]) == 1:
663 for ms in range(1, nvis):
664 self.allselpars[par].append(
665 self.allselpars[par][0]
666 )
668 # Now, all parameters are lists of strings each of length 'nvis'.
669 # Put them into separate dicts per MS.
670 selparlist = {}
671 for ms in range(0, nvis):
672 selparlist["ms" + str(ms)] = {}
673 for par in selkeys:
674 selparlist["ms" + str(ms)][par] = self.allselpars[par][ms]
676 synu = synthesisutils()
677 selparlist["ms" + str(ms)] = synu.checkselectionparams(
678 selparlist["ms" + str(ms)]
679 )
680 synu.done()
682 # casalog.post(selparlist)
683 self.allselpars = selparlist
685 return errs
687 def makeImagingParamLists(self, parallel):
688 errs = ""
689 # casalog.post("specmode=",self.allimpars['0']['specmode'], " parallel=",parallel)
690 ## Multiple images have been specified.
691 ## (1) Parse the outlier file and fill a list of imagedefinitions
692 ## OR (2) Parse lists per input parameter into a list of parameter-sets (imagedefinitions)
693 ### The code below implements (1)
694 outlierpars = []
695 parseerrors = ""
696 if len(self.outlierfile) > 0:
697 outlierpars, parseerrors = self.parseOutlierFile(self.outlierfile)
698 if parallel:
699 casalog.post("CALLING checkParallelMFMixModes...")
700 errs = self.checkParallelMFMixedModes(
701 self.allimpars, outlierpars
702 )
703 if len(errs):
704 return errs
706 if len(parseerrors) > 0:
707 errs = errs + "Errors in parsing outlier file : " + parseerrors
708 return errs
710 # Initialize outlier parameters with defaults
711 # Update outlier parameters with modifications from outlier files
712 for immod in range(0, len(outlierpars)):
713 modelid = str(immod + 1)
714 self.allimpars[modelid] = copy.deepcopy(self.allimpars["0"])
715 self.allimpars[modelid].update(outlierpars[immod]["impars"])
716 self.allgridpars[modelid] = copy.deepcopy(self.allgridpars["0"])
717 self.allgridpars[modelid].update(outlierpars[immod]["gridpars"])
718 self.alldecpars[modelid] = copy.deepcopy(self.alldecpars["0"])
719 self.alldecpars[modelid].update(outlierpars[immod]["decpars"])
720 self.allnormpars[modelid] = copy.deepcopy(self.allnormpars["0"])
721 self.allnormpars[modelid].update(outlierpars[immod]["normpars"])
722 self.alldecpars[modelid]["id"] = immod + 1 ## Try to eliminate.
724 # casalog.post(self.allimpars)
726 #
727 # casalog.post("REMOVING CHECKS to check...")
728 #### This does not handle the conversions of the csys correctly.....
729 ####
730 # for immod in self.allimpars.keys() :
731 # tempcsys = {}
732 # if 'csys' in self.allimpars[immod]:
733 # tempcsys = self.allimpars[immod]['csys']
734 #
735 # synu = synthesisutils()
736 # self.allimpars[immod] = synu.checkimageparams( self.allimpars[immod] )
737 # synu.done()
738 #
739 # if len(tempcsys.keys())==0:
740 # self.allimpars[immod]['csys'] = tempcsys
742 ## Check for name increments, and copy from impars to decpars and normpars.
743 self.handleImageNames()
745 return errs
747 def handleImageNames(self):
749 for immod in self.allimpars.keys():
750 inpname = self.allimpars[immod]["imagename"]
752 ### If a directory name is embedded in the image name, check that the dir exists.
753 if inpname.count("/"):
754 splitname = inpname.split("/")
755 prefix = splitname[len(splitname) - 1]
756 dirname = inpname[
757 0 : len(inpname) - len(prefix)
758 ] # has '/' at end
759 if not os.path.exists(dirname):
760 casalog.post("Making directory : " + dirname, "INFO")
761 os.mkdir(dirname)
763 ### Check for name increments
764 # if self.reusename == False:
766 if (
767 self.allimpars["0"]["restart"] == False
768 ): # Later, can change this to be field dependent too.
769 ## Get a list of image names for all fields (to sync name increment ids across fields)
770 inpnamelist = {}
771 for immod in self.allimpars.keys():
772 inpnamelist[immod] = self.allimpars[immod]["imagename"]
774 newnamelist = self.incrementImageNameList(inpnamelist)
776 if len(newnamelist) != len(self.allimpars.keys()):
777 casalog.post(
778 "Internal Error : Non matching list lengths in refimagerhelper::handleImageNames. Not updating image names",
779 "WARN",
780 )
781 else:
782 for immod in self.allimpars.keys():
783 self.allimpars[immod]["imagename"] = newnamelist[immod]
785 def checkAndFixIterationPars(self):
786 errs = ""
788 # Bother checking only if deconvolution iterations are requested
789 if self.iterpars["niter"] > 0:
790 # Make sure cycleniter is less than or equal to niter.
791 if (
792 self.iterpars["cycleniter"] <= 0
793 or self.iterpars["cycleniter"] > self.iterpars["niter"]
794 ):
795 if self.iterpars["interactive"] == False:
796 self.iterpars["cycleniter"] = self.iterpars["niter"]
797 else:
798 self.iterpars["cycleniter"] = min(
799 self.iterpars["niter"], 100
800 )
802 # saving model is done separately outside of iter. control for interactive clean and or automasking cases
804 if self.iterpars["savemodel"] != "none":
805 if (
806 self.iterpars["interactive"] == True
807 or self.alldecpars["0"]["usemask"] == "auto-multithresh"
808 or self.alldecpars["0"]["nsigma"] > 0.0
809 ):
810 self.iterpars["savemodel"] = "none"
811 for visid in self.allselpars:
812 self.allselpars[visid]["readonly"] = True
813 self.allselpars[visid]["usescratch"] = False
815 return errs
817 def checkAndFixNormPars(self):
818 errs = ""
820 # for modelid in self.allnormpars.keys():
821 # if len(self.allnormpars[modelid]['workdir'])==0:
822 # self.allnormpars[modelid]['workdir'] = self.allnormpars['0']['imagename'] + '.workdir'
824 return errs
826 ###### End : Parameter-checking functions ##################
828 ## Parse outlier file and construct a list of imagedefinitions (dictionaries).
829 def parseOutlierFile(self, outlierfilename=""):
830 returnlist = []
831 errs = "" # must be empty for no error
833 if len(outlierfilename) > 0 and not os.path.exists(outlierfilename):
834 errs += "Cannot find outlier file : " + outlierfilename + "\n"
835 return returnlist, errs
837 fp = open(outlierfilename, "r")
838 thelines = fp.readlines()
839 tempimpar = {}
840 tempgridpar = {}
841 tempweightpar = {}
842 tempdecpar = {}
843 tempnormpar = {}
844 for oneline in thelines:
845 aline = oneline.replace("\n", "")
846 # aline = oneline.replace(' ','').replace('\n','')
847 if len(aline) > 0 and aline.find("#") != 0:
848 parpair = aline.split("=")
849 parpair[0] = parpair[0].replace(" ", "")
850 # casalog.post(parpair)
851 if len(parpair) != 2:
852 errs += "Error in line containing : " + oneline + "\n"
853 if parpair[0] == "imagename" and tempimpar != {}:
854 # returnlist.append({'impars':tempimpar, 'gridpars':tempgridpar, 'weightpars':tempweightpar, 'decpars':tempdecpar} )
855 returnlist.append(
856 {
857 "impars": tempimpar,
858 "gridpars": tempgridpar,
859 "weightpars": tempweightpar,
860 "decpars": tempdecpar,
861 "normpars": tempnormpar,
862 }
863 )
864 tempimpar = {}
865 tempgridpar = {}
866 tempweightpar = {}
867 tempdecpar = {}
868 tempnormpar = {}
869 usepar = False
870 if parpair[0] in self.outimparlist:
871 tempimpar[parpair[0]] = parpair[1]
872 usepar = True
873 if parpair[0] in self.outgridparlist:
874 tempgridpar[parpair[0]] = parpair[1]
875 usepar = True
876 if parpair[0] in self.outweightparlist:
877 tempweightpar[parpair[0]] = parpair[1]
878 usepar = True
879 if parpair[0] in self.outdecparlist:
880 tempdecpar[parpair[0]] = parpair[1]
881 usepar = True
882 if parpair[0] in self.outnormparlist:
883 tempnormpar[parpair[0]] = parpair[1]
884 usepar = True
885 if usepar == False:
886 casalog.post(
887 "Ignoring unknown parameter pair : " + oneline
888 )
890 if len(errs) == 0:
891 returnlist.append(
892 {
893 "impars": tempimpar,
894 "gridpars": tempgridpar,
895 "weightpars": tempweightpar,
896 "decpars": tempdecpar,
897 "normpars": tempnormpar,
898 }
899 )
901 ## Extra parsing for a few parameters.
902 returnlist = self.evalToTarget(
903 returnlist, "impars", "imsize", "intvec"
904 )
905 returnlist = self.evalToTarget(returnlist, "impars", "nchan", "int")
906 returnlist = self.evalToTarget(returnlist, "impars", "cell", "strvec")
907 returnlist = self.evalToTarget(returnlist, "impars", "nterms", "int")
908 returnlist = self.evalToTarget(returnlist, "decpars", "nterms", "int")
909 returnlist = self.evalToTarget(returnlist, "normpars", "nterms", "int")
910 returnlist = self.evalToTarget(
911 returnlist, "gridpars", "wprojplanes", "int"
912 )
913 # returnlist = self.evalToTarget( returnlist, 'impars', 'reffreq', 'strvec' )
915 # casalog.post(returnlist)
916 return returnlist, errs
918 def evalToTarget(self, globalpars, subparkey, parname, dtype="int"):
919 try:
920 for fld in range(0, len(globalpars)):
921 if parname in globalpars[fld][subparkey]:
922 if dtype == "int" or dtype == "intvec":
923 val_e = eval(globalpars[fld][subparkey][parname])
924 if dtype == "strvec":
925 tcell = globalpars[fld][subparkey][parname]
926 tcell = (
927 tcell.replace(" ", "")
928 .replace("[", "")
929 .replace("]", "")
930 .replace("'", "")
931 )
932 tcells = tcell.split(",")
933 val_e = []
934 for cell in tcells:
935 val_e.append(cell)
937 globalpars[fld][subparkey][parname] = val_e
938 except:
939 casalog.post(
940 'Cannot evaluate outlier field parameter "' + parname + '"',
941 "ERROR",
942 )
944 return globalpars
946 def printParameters(self):
947 casalog.post("SelPars : " + str(self.allselpars), "INFO2")
948 casalog.post("ImagePars : " + str(self.allimpars), "INFO2")
949 casalog.post("GridPars : " + str(self.allgridpars), "INFO2")
950 casalog.post("NormPars : " + str(self.allnormpars), "INFO2")
951 casalog.post("Weightpars : " + str(self.weightpars), "INFO2")
952 casalog.post("DecPars : " + str(self.alldecpars), "INFO2")
953 casalog.post("IterPars : " + str(self.iterpars), "INFO2")
955 def incrementImageName(self, imagename):
956 dirname = "."
957 prefix = imagename
959 if imagename.count("/"):
960 splitname = imagename.split("/")
961 prefix = splitname[len(splitname) - 1]
962 ### if it has a leading / then absolute path is assumed
963 dirname = (
964 (imagename[0 : len(imagename) - len(prefix)])
965 if (imagename[0] == "/")
966 else ("./" + imagename[0 : len(imagename) - len(prefix)])
967 ) # has '/' at end
969 inamelist = [
970 fn for fn in os.listdir(dirname) if any([fn.startswith(prefix)])
971 ]
973 if len(inamelist) == 0:
974 newimagename = dirname[2:] + prefix
975 else:
976 nlen = len(prefix)
977 maxid = 1
978 for iname in inamelist:
979 startind = iname.find(prefix + "_")
980 if startind == 0:
981 idstr = (iname[nlen + 1 : len(iname)]).split(".")[0]
982 if idstr.isdigit():
983 val = eval(idstr)
984 if val > maxid:
985 maxid = val
986 newimagename = dirname[2:] + prefix + "_" + str(maxid + 1)
988 casalog.post("Using : {}".format(newimagename))
989 return newimagename
991 def incrementImageNameList(self, inpnamelist):
993 dirnames = {}
994 prefixes = {}
996 for immod in inpnamelist.keys():
997 imagename = inpnamelist[immod]
998 dirname = "."
999 prefix = imagename
1001 if imagename.count("/"):
1002 splitname = imagename.split("/")
1003 prefix = splitname[len(splitname) - 1]
1004 dirname = (
1005 (imagename[0 : len(imagename) - len(prefix)])
1006 if (imagename[0] == "/")
1007 else ("./" + imagename[0 : len(imagename) - len(prefix)])
1008 ) # has '/' at end
1010 dirnames[immod] = dirname
1011 prefixes[immod] = prefix
1013 maxid = 0
1014 for immod in inpnamelist.keys():
1015 prefix = prefixes[immod]
1016 inamelist = [
1017 fn
1018 for fn in os.listdir(dirnames[immod])
1019 if any([fn.startswith(prefix)])
1020 ]
1021 nlen = len(prefix)
1023 if len(inamelist) == 0:
1024 locmax = 0
1025 else:
1026 locmax = 1
1028 cleanext = [
1029 ".image",
1030 ".residual",
1031 ".model",
1032 ".psf",
1033 ".sumwt",
1034 ".tt0",
1035 ]
1036 incremented = False
1037 for iname in inamelist:
1038 rootname, ext = os.path.splitext(iname)
1039 if ext in cleanext:
1040 startind = iname.find(prefix + "_")
1041 if startind == 0:
1042 idstr = (iname[nlen + 1 : len(iname)]).split(".")[0]
1043 if idstr.isdigit():
1044 val = eval(idstr)
1045 incremented = True
1046 if val > locmax:
1047 locmax = val
1048 elif startind == -1:
1049 if ext == ".tt0":
1050 # need one more pass to extract rootname
1051 rootname, ext = os.path.splitext(rootname)
1052 if rootname == prefix:
1053 # the file name with root file name only
1054 incremented = True
1056 if not incremented:
1057 locmax = 0
1058 if locmax > maxid:
1059 maxid = locmax
1061 newimagenamelist = {}
1062 for immod in inpnamelist.keys():
1063 if maxid == 0:
1064 newimagenamelist[immod] = inpnamelist[immod]
1065 else:
1066 newimagenamelist[immod] = (
1067 dirnames[immod][2:]
1068 + prefixes[immod]
1069 + "_"
1070 + str(maxid + 1)
1071 )
1073 # casalog.post('Input : ', inpnamelist)
1074 # casalog.post('Dirs : ', dirnames)
1075 # casalog.post('Pre : ', prefixes)
1076 # casalog.post('Max id : ', maxid)
1077 # casalog.post('Using : ', newimagenamelist)
1078 return newimagenamelist
1080 ## Guard against numpy int32,int64 types which don't convert well across tool boundary.
1081 ## For CAS-8250. Remove when CAS-6682 is done.
1082 def fixIntParam(self, allpars, parname):
1083 for immod in allpars.keys():
1084 if parname in allpars[immod]:
1085 ims = allpars[immod][parname]
1086 if type(ims) != list:
1087 ims = int(ims)
1088 else:
1089 for el in range(0, len(ims)):
1090 ims[el] = int(ims[el])
1091 allpars[immod][parname] = ims
1093 # check for non-supported multifield in mixed modes in parallel
1094 # (e.g. combination cube and continuum for main and outlier fields)
1095 def checkParallelMFMixedModes(self, allimpars, outlierpars):
1096 errmsg = ""
1097 casalog.post("outlierpars=={}".format(outlierpars))
1098 mainspecmode = allimpars["0"]["specmode"]
1099 mainnchan = allimpars["0"]["nchan"]
1100 casalog.post(
1101 "mainspecmode={} mainnchan={}".format(mainspecmode, mainnchan)
1102 )
1103 cubeoutlier = False
1104 contoutlier = False
1105 isnchanmatch = True
1106 for immod in range(0, len(outlierpars)):
1107 if "impars" in outlierpars[immod]:
1108 if "nchan" in outlierpars[immod]["impars"]:
1109 if outlierpars[immod]["impars"]["nchan"] > 1:
1110 cubeoutlier = True
1111 if outlierpars[immod]["impars"]["nchan"] != mainnchan:
1112 isnchanmatch = False
1113 else:
1114 contoutlier = True
1115 else:
1116 if "specmode" in outlierpars[immod]["impars"]:
1117 if outlierpars[immod]["impars"]["specmode"] == "mfs":
1118 contoutlier = True
1119 if mainspecmode.find("cube") == 0:
1120 if contoutlier:
1121 errmsg = "Mixed cube and continuum mode for multifields is currently not supported for parallel mode"
1122 else: # all cube modes, but need to check if the nchans are the same
1123 if not isnchanmatch:
1124 errmsg = "Cubes for multifields with different nchans are currently not supported for parallel mode "
1125 else: # mfs
1126 if cubeoutlier:
1127 errmsg = "Mixed continuum and cube mode for multifields is currently not supported for parallel mode"
1128 errs = errmsg
1129 return errs
1131 def checkmsforwtspec(self):
1132 """check if WEIGHT_SPECTRUM column exist when
1133 a list of vis is given. Add the column for an MS
1134 which does not have one if other MSs have the column.
1135 This is a workaround for the issue probably in Vi/VB2
1136 not handling the state change for the optional column
1137 when dealing with multiples MSs
1138 """
1139 mycb = calibrater()
1140 mytb = table()
1141 haswtspec = False
1142 mswithnowtspec = []
1143 nms = 1
1144 if type(self.allselpars["msname"]) == list:
1145 nms = len(self.allselpars["msname"])
1147 if nms > 1:
1148 for inms in self.allselpars["msname"]:
1149 mytb.open(inms)
1150 cols = mytb.colnames()
1151 mytb.close()
1152 if "WEIGHT_SPECTRUM" in cols:
1153 haswtspec = True
1154 else:
1155 mswithnowtspec.append(inms)
1156 if haswtspec and len(mswithnowtspec) > 0:
1157 casalog.post(
1158 "Some of the MSes donot have WEIGHT_SPECTRUM while some other do."
1159 + " Automatically adding the column and initialize for those don't to avoid a process failure.",
1160 "WARN",
1161 )
1162 for inms in mswithnowtspec:
1163 mycb.open(inms, addcorr=False, addmodel=False)
1164 mycb.initweights(wtmode="weight", dowtsp=True)
1165 mycb.close()
1166 # noOp for nms==1
1168 def mslist_timesorting(self, mslist):
1169 """
1170 wrapper for mslisthelper.sort_mslist to get a sorting order w.r.t the original
1171 """
1172 (thenewmslist, times) = sort_mslist(mslist)
1173 theindex = []
1174 for vnew in thenewmslist:
1175 for vold in mslist:
1176 if vnew == vold:
1177 theindex.append(mslist.index(vnew))
1178 return (thenewmslist, times, theindex)
1180 def addwtspec(self, mslist):
1181 """
1182 Add the column for an MS which does not have one if other MSs have the column.
1183 This is a workaround for the issue probably in Vi/VB2
1184 not handling the state change for the optional column
1185 when dealing with multiples MSs
1186 """
1187 mycb = calibrater()
1189 if len(mslist) > 0:
1190 casalog.post(
1191 "Some of the MSes donot have WEIGHT_SPECTRUM while some other do."
1192 + " Automatically adding the column and initialize using the existing WEIGHT column for those don't to avoid a process failure.",
1193 "WARN",
1194 )
1195 casalog.post(
1196 "Adding WEIGHT_SPECTRUM in the following MS(s): "
1197 + str(mslist),
1198 "WARN",
1199 )
1200 for inms in mslist:
1201 mycb.open(inms, addcorr=False, addmodel=False)
1202 mycb.initweights(wtmode="weight", dowtsp=True)
1203 mycb.close()
1204 mycb.done()
1205 # noOp for len(mlist) ==0
1207 ############################
1210#################################################################################################
1211def backupoldfile(thefile=""):
1212 import copy
1213 import shutil
1215 if thefile == "" or (not os.path.exists(thefile)):
1216 return
1217 outpathdir = os.path.realpath(os.path.dirname(thefile))
1218 outpathfile = outpathdir + os.path.sep + os.path.basename(thefile)
1219 k = 0
1220 backupfile = outpathfile + "." + str(k)
1221 prevfile = "--------"
1222 while os.path.exists(backupfile):
1223 k = k + 1
1224 prevfile = copy.copy(backupfile)
1225 if os.path.exists(prevfile) and filecmp.cmp(outpathfile, prevfile):
1226 ##avoid making multiple copies of the same file
1227 return
1228 backupfile = outpathfile + "." + str(k)
1229 shutil.copy2(outpathfile, backupfile)
1232def saveparams2last(func=None, multibackup=True):
1233 """This function is a decorator function that allows for
1234 task.last to be saved even if calling without casashell. Also
1235 saves unique revisions ...just like the vax/vms style of revision saving
1236 by default. set multibackup=False to no not have old revisions kept
1237 """
1238 if not func:
1239 return functools.partial(saveparams2last, multibackup=multibackup)
1241 @functools.wraps(func)
1242 def wrapper_saveparams(*args, **kwargs):
1243 # multibackup=True
1244 outfile = func.__name__ + ".last"
1245 # print('args {} and kwargs {}'.format(args, kwargs))
1246 # print('length of args {}, and kwargs {}'.format(len(args), len(kwargs)))
1247 params = {}
1248 byIndex = list()
1249 if len(kwargs) == 0:
1250 paramsname = list(inspect.signature(func).parameters)
1251 # params={paramsname[i]: args[i] for i in range(len(args))}
1252 params = OrderedDict(zip(paramsname, args))
1253 byIndex = list(params)
1254 else:
1255 params = kwargs
1256 byIndex = list(params)
1257 ###for some reason the dictionary is in reverse
1258 byIndex.reverse()
1259 # print('@@@@MULTIBACKUP {}, params {}'.format(multibackup, params))
1260 if multibackup:
1261 backupoldfile(outfile)
1262 with open(outfile, "w") as _f:
1263 for _i in range(len(byIndex)):
1264 _f.write(
1265 "%-20s = %s\n" % (byIndex[_i], repr(params[byIndex[_i]]))
1266 )
1267 _f.write("#" + func.__name__ + "( ")
1268 for _i in range(len(byIndex)):
1269 _f.write("%s=%s" % (byIndex[_i], repr(params[byIndex[_i]])))
1270 if _i < len(params) - 1:
1271 _f.write(",")
1272 _f.write(" )\n")
1273 ###End of stuff before task is called
1274 retval = func(*args, **kwargs)
1275 ###we could do something here post task
1276 return retval
1278 return wrapper_saveparams
1281######################################################
1284def determineFreqRange(
1285 vis: str = "", fieldid: int = 0, spw: str = "*"
1286) -> Tuple[np.double, np.double]:
1287 _tb = table()
1288 _ms = ms()
1289 _su = synthesisutils()
1290 _qa = quanta()
1291 minFreq = 1.0e20
1292 maxFreq = 0.0
1293 _tb.open(vis)
1294 fieldids = _tb.getcol("FIELD_ID")
1295 _tb.done()
1296 # advisechansel does not work on fieldids not in main
1297 if fieldid not in fieldids:
1298 fieldid = fieldids[0]
1299 frange = _su.advisechansel(
1300 msname=vis, getfreqrange=True, fieldid=fieldid, spwselection=spw
1301 )
1302 if minFreq > _qa.convert(frange["freqstart"], "Hz")["value"]:
1303 minFreq = _qa.convert(frange["freqstart"], "Hz")["value"]
1304 if maxFreq < _qa.convert(frange["freqend"], "Hz")["value"]:
1305 maxFreq = _qa.convert(frange["freqend"], "Hz")["value"]
1307 if minFreq > maxFreq:
1308 raise Exception(f"Failed to determine frequency range in ms {vis}")
1309 freqwidth = maxFreq - minFreq
1310 return (minFreq, freqwidth)