Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_mstransform.py: 39%
219 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-10-31 17:39 +0000
2import os, re
3import shutil
4import string
5import copy
6import math
7import time
9from casatools import table, quanta, ms, mstransformer
10from casatasks import casalog
11from .parallel.parallel_data_helper import ParallelDataHelper
12from . import flaghelper as fh
13from .update_spw import update_spwchan
14from .mstools import write_history
15from .callibrary import callibrary
17def mstransform(
18 vis,
19 outputvis, # output
20 createmms, # MMS --> partition
21 separationaxis,
22 numsubms,
23 tileshape, # tiling
24 field,
25 spw,
26 scan,
27 antenna,
28 correlation,
29 timerange,
30 intent,
31 array,
32 uvrange,
33 observation,
34 feed,
35 datacolumn,
36 realmodelcol,
37 keepflags,
38 usewtspectrum,
39 combinespws, # spw combination --> cvel
40 chanaverage, # channel averaging --> split
41 chanbin,
42 hanning, # Hanning --> cvel
43 regridms, # regridding to new frame --> cvel
44 mode,
45 nchan,
46 start,
47 width,
48 nspw, # spw separation
49 interpolation,
50 phasecenter,
51 restfreq,
52 outframe,
53 veltype,
54 preaverage,
55 timeaverage, # time averaging --> split
56 timebin,
57 timespan,
58 maxuvwdistance,
59 docallib,
60 callib,
61 douvcontsub,
62 fitspw,
63 fitorder,
64 want_cont,
65 denoising_lib,
66 nthreads,
67 niter,
68 disableparallel, # HIDDEN parameter to create MMS in sequential
69 ddistart, # HIDDEN internal parameter for the sub-table re-indexing
70 taql, # HIDDEN internal parameter
71 monolithic_processing, # HIDDEN parameter for the MMS monolithic processing
72 reindex # HIDDEN parameter for use in the pipeline context only
73 ):
75 ''' This task can replace split, cvel, partition and hanningsmooth '''
77 casalog.origin('mstransform')
79 ''' HEURISTICS:
80 input MS --> output MS
81 input MMS --> output MMS
83 user can set createmms=True to create the following:
84 input MS --> output MMS
86 '''
87 # Initialize the helper class
88 pdh = ParallelDataHelper('mstransform', locals())
90 # When dealing with MMS, process in parallel or sequential
91 # disableparallel is a hidden parameter. Only for debugging purposes!
92 if disableparallel:
93 pdh.bypassParallelProcessing(1)
94 else:
95 pdh.bypassParallelProcessing(0)
97 # Validate input and output parameters
98 pdh.setupIO()
100 # Process the input Multi-MS
101 if ParallelDataHelper.isMMSAndNotServer(vis) == True and monolithic_processing == False:
102 '''
103 retval{'status': True, 'axis':''} --> can run in parallel
104 retval{'status': False, 'axis':'value'} --> treat MMS as monolithic MS, set new axis for output MMS
105 retval{'status': False, 'axis':''} --> treat MMS as monolithic MS, create an output MS
106 '''
108 retval = pdh.validateInputParams()
109 # Cannot create an output MMS.
110 if retval['status'] == False and retval['axis'] == '':
111 casalog.post('Cannot process MMS with the requested transformations','WARN')
112 casalog.post('Use task listpartition to see the contents of the MMS')
113 casalog.post('Will create an output MS','WARN')
114 createmms = False
116 # MMS is processed as monolithic MS.
117 elif retval['status'] == False and retval['axis'] != '':
118 createmms = True
119 pdh.override__args('createmms', True)
120 pdh.override__args('monolithic_processing', True)
121 separationaxis = retval['axis']
122 pdh.override__args('separationaxis', retval['axis'])
123 casalog.post("Will process the input MMS as a monolithic MS",'WARN')
124 casalog.post("Will create an output MMS with separation axis \'%s\'"%retval['axis'],'WARN')
126 # MMS is processed in parallel
127 else:
128 createmms = False
129 try:
130 pdh.override__args('createmms', False)
131 pdh.setupCluster('mstransform')
132 pdh.go()
133 except Exception as instance:
134 casalog.post('%s'%instance,'ERROR')
135 return False
137 return True
139 # Create an output Multi-MS
140 if createmms == True:
142 # Check the heuristics of separationaxis and the requested transformations
143 pval = pdh.validateOutputParams()
144 if pval == 0:
145 raise Exception('Cannot create MMS using separationaxis=%s with some of the requested transformations.' % separationaxis)
147 pdh.setupCluster('mstransform')
148 pdh.go()
149 monolithic_processing = False
150 return
153 # Create a local copy of the MSTransform tool
154 mtlocal = mstransformer()
155 mslocal = ms()
156 qalocal = quanta()
158 try:
160 # Gather all the parameters in a dictionary.
161 config = {}
163 if keepflags:
164 taqlstr = ''
165 else:
166 taqlstr = "NOT (FLAG_ROW OR ALL(FLAG))"
168 # MMS taql selection
169 if taql != '' and taql != None:
170 if not keepflags:
171 taqlstr = taqlstr + " AND "+taql
172 else:
173 taqlstr = taql
175 config = pdh.setupParameters(inputms=vis, outputms=outputvis, field=field,
176 spw=spw, array=array, scan=scan, antenna=antenna, correlation=correlation,
177 uvrange=uvrange,timerange=timerange, intent=intent, observation=str(observation),
178 feed=feed, taql=taqlstr)
180 # ddistart will be used in the tool when re-indexing the spw table
181 config['ddistart'] = ddistart
183 # re-index parameter is used by the pipeline to not re-index any sub-table and the associated IDs
184 config['reindex'] = reindex
186 config['datacolumn'] = datacolumn
187 dc = datacolumn.upper()
188 # Make real a virtual MODEL column in the output MS
189 if "MODEL" in dc or dc == 'ALL':
190 config['realmodelcol'] = realmodelcol
192 config['usewtspectrum'] = usewtspectrum
194 # Add the tile shape parameter
195 if tileshape.__len__() == 1:
196 # The only allowed values are 0 or 1
197 if tileshape[0] != 0 and tileshape[0] != 1:
198 raise ValueError('When tileshape has one element, it should be either 0 or 1.')
200 elif tileshape.__len__() != 3:
201 # The 3 elements are: correlations, channels, rows
202 raise ValueError('Parameter tileshape must have 1 or 3 elements.')
204 config['tileshape'] = tileshape
206 if combinespws:
207 casalog.post('Combine spws %s into new output spw'%spw)
208 config['combinespws'] = True
210 # Only parse chanaverage if chanbin is valid
211 if chanaverage and isinstance(chanbin, int) and chanbin <= 1:
212 raise ValueError('Parameter chanbin must be > 1 to do channel averaging')
214 # Validate the case of int or list chanbin
215 if chanaverage and pdh.validateChanBin():
216 casalog.post('Parse channel averaging parameters')
217 config['chanaverage'] = True
219 # convert numpy types, until CAS-6493 is not fixed
220 chanbin = fh.evaluateNumpyType(chanbin)
221 config['chanbin'] = chanbin
223 if hanning:
224 casalog.post('Apply Hanning smoothing')
225 config['hanning'] = True
227 if regridms:
228 casalog.post('Parse regridding parameters')
229 config['regridms'] = True
230 # Reset the defaults depending on the mode
231 # Only add non-empty string parameters to config dictionary
232 start, width = pdh.defaultRegridParams()
233 config['mode'] = mode
234 config['nchan'] = nchan
235 if start != '':
236 config['start'] = start
237 if width != '':
238 config['width'] = width
239 if nspw > 1:
240 casalog.post('Separate MS into %s spws'%nspw)
241 config['nspw'] = nspw
242 config['interpolation'] = interpolation
243 if restfreq != '':
244 config['restfreq'] = restfreq
245 if outframe != '':
246 config['outframe'] = outframe
247 if phasecenter != '':
248 config['phasecenter'] = phasecenter
249 config['veltype'] = veltype
250 config['preaverage'] = preaverage
252 # Only parse timeaverage parameters when timebin > 0s
253 if timeaverage:
254 tb = qalocal.convert(qalocal.quantity(timebin), 's')['value']
255 if not tb > 0:
256 raise ValueError("Parameter timebin must be > '0s' to do time averaging")
258 if timeaverage:
259 casalog.post('Parse time averaging parameters')
260 config['timeaverage'] = True
261 config['timebin'] = timebin
262 config['timespan'] = timespan
263 config['maxuvwdistance'] = maxuvwdistance
265 if docallib:
266 casalog.post('Parse docallib parameters')
267 mycallib = callibrary()
268 mycallib.read(callib)
269 config['calibration'] = True
270 config['callib'] = mycallib.cld
272 if douvcontsub:
273 casalog.post('Parse uvcontsub parameters')
274 casalog.post("The parameter douvcontsub of mstransform is deprecated and will "
275 "be removed in an upcoming release. The task uvcontsub can be used "
276 "as an alternative.", "WARN")
277 config['uvcontsub'] = True
278 uvcontsub_config = {}
279 uvcontsub_config['fitspw'] = fitspw
280 uvcontsub_config['fitorder'] = fitorder
281 uvcontsub_config['want_cont'] = want_cont
282 uvcontsub_config['denoising_lib'] = denoising_lib
283 uvcontsub_config['nthreads'] = nthreads
284 uvcontsub_config['niter'] = niter
285 config['uvcontsublib'] = dict(uvcontsub_config)
287 # Configure the tool and all the parameters
289 casalog.post('%s'%config, 'DEBUG')
290 mtlocal.config(config)
292 # Open the MS, select the data and configure the output
293 mtlocal.open()
295 # Run the tool
296 casalog.post('Apply the transformations')
297 mtlocal.run()
299 finally:
300 mtlocal.done()
303 # Update the FLAG_CMD sub-table to reflect any spw/channels selection
304 # If the spw selection is by name or FLAG_CMD contains spw with names, skip the updating
305 if ((spw != '') and (spw != '*')) or chanaverage == True:
306 isopen = False
308 try:
309 mytb = table()
310 mytb.open(outputvis + '/FLAG_CMD', nomodify=False)
311 isopen = True
312 nflgcmds = mytb.nrows()
314 if nflgcmds > 0:
315 updateFlagCmd = False
317 # If spw selection is by name in FLAG_CMD, do not update, CAS-7751
318 mycmd = mytb.getcell('COMMAND', 0)
319 cmdlist = mycmd.split()
320 for cmd in cmdlist:
321 # Match only spw indices, not names
322 if cmd.__contains__('spw'):
323 cmd = cmd.strip("spw=")
324 spwstr = re.search('^[^a-zA-Z]+$', cmd)
325 if spwstr != None and spwstr.string.__len__() > 0:
326 updateFlagCmd = True
327 break
330 if updateFlagCmd:
331 mademod = False
332 cmds = mytb.getcol('COMMAND')
333 widths = {}
334 if hasattr(chanbin, 'has_key'):
335 widths = chanbin
336 else:
337 if hasattr(chanbin, '__iter__') and len(chanbin) > 1:
338 for i in range(len(chanbin)):
339 widths[i] = chanbin[i]
340 elif chanbin != 1:
341 numspw = len(mslocal.msseltoindex(vis=vis,
342 spw='*')['spw'])
343 if hasattr(chanbin, '__iter__'):
344 w = chanbin[0]
345 else:
346 w = chanbin
347 for i in range(numspw):
348 widths[i] = w
349 for rownum in range(nflgcmds):
350 # Matches a bare number or a string quoted any way.
351 spwmatch = re.search(r'spw\s*=\s*(\S+)', cmds[rownum])
352 if spwmatch:
353 sch1 = spwmatch.groups()[0]
354 sch1 = re.sub(r"[\'\"]", '', sch1) # Dequote
355 # Provide a default in case the split selection excludes
356 # cmds[rownum]. update_spwchan() will throw an exception
357 # in that case.
358 cmd = ''
359 try:
360 sch2 = update_spwchan(vis, spw, sch1, truncate=True,
361 widths=widths)
362 if sch2:
363 repl = ''
364 if sch2 != '*':
365 repl = "spw='" + sch2 + "'"
366 cmd = cmds[rownum].replace(spwmatch.group(), repl)
367 #except: # cmd[rownum] no longer applies.
368 except Exception as e:
369 casalog.post("Error %s updating row %d of FLAG_CMD" % (e,rownum),'WARN')
370 casalog.post('sch1 = ' + sch1, 'DEBUG1')
371 casalog.post('cmd = ' + cmd, 'DEBUG1')
372 if cmd != cmds[rownum]:
373 mademod = True
374 cmds[rownum] = cmd
375 if mademod:
376 casalog.post('Updating FLAG_CMD', 'INFO')
377 mytb.putcol('COMMAND', cmds)
379 else:
380 casalog.post('FLAG_CMD table contains spw selection by name. Will not update it!','DEBUG')
383 finally:
384 if isopen:
385 mytb.close()
386 mslocal = None
387 mytb = None
390 # Write history to output MS, not the input ms.
391 try:
392 param_names = mstransform.__code__.co_varnames[:mstransform.__code__.co_argcount]
393 vars = locals( )
394 param_vals = [vars[p] for p in param_names]
396 write_history(mslocal, outputvis, 'mstransform', param_names, param_vals, casalog)
397 except Exception as instance:
398 casalog.post("*** Error \'%s\' updating HISTORY" % (instance),'WARN')
400 mslocal = None