Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_virtualconcat.py: 6%
259 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-10-31 19:10 +0000
1import os
2import shutil
3import stat
4import time
6from . import partitionhelper as ph
7from .parallel.parallel_task_helper import ParallelTaskHelper
8from .mslisthelper import check_mslist, sort_mslist
9from .mstools import write_history
11from casatools import calibrater, quanta
12from casatools import ms as mstool
13from casatools import table as tbtool
14from casatasks import casalog
16_cb = calibrater()
17_qa = quanta()
20def virtualconcat(vislist,concatvis,freqtol,dirtol,respectname,
21 visweightscale,keepcopy,copypointing):
22 """
23 Concatenate visibility data sets creating a Multi-MS.
25 Combine the input datasets into a Multi-MS.
26 NOTE: The input datasets are moved into the Multi-MS and may be modified
27 to account for subtable reference changes.
28 If none of the input MSs have any scratch columns, none are created.
29 Otherwise scratch columns are created and initialized in those MSs
30 which don't have a complete set.
33 Keyword arguments:
34 vis -- Name of input visibility files (MSs, only use real paths, no symlinks)
35 default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms']
36 concatvis -- Name of the output visibility file (MMS)
37 default: none; example: concatvis='src2.ms'
38 freqtol -- Frequency shift tolerance for considering data as the same spwid
39 default: '' means always combine
40 example: freqtol='10MHz' will not combine spwid unless they are
41 within 10 MHz
42 dirtol -- Direction shift tolerance for considering data as the same field
43 default: ;; means always combine
44 example: dirtol='1.arcsec' will not combine data for a field unless
45 their phase center is less than 1 arcsec.
46 respectname -- If true, fields with a different name are not merged even if their
47 direction agrees (within dirtol)
48 default: True
49 visweightscale -- list of the weight scales to be applied to the individual MSs
50 default: [] (don't modify weights, equivalent to setting scale to 1 for each MS)
51 keepcopy -- If true, a copy of the input MSs is kept in their original place.
52 default: false
53 copypointing -- If true, the POINTING table information will be present in the output.
54 If false, the result is an empty POINTING table.
55 default: True
57 """
59 ###
60 #Python script
62 tempdir = ''
63 originalvis = vislist
64 try:
65 casalog.origin('virtualconcat')
66 t = tbtool()
67 m = mstool()
69 #break the reference between vis and vislist as we modify vis
70 if(type(vislist)==str):
71 vis=[vislist]
72 else:
73 vis=list(vislist)
74 #dto. for concavis
75 theconcatvis = concatvis
77 #sanitize the members of vis
78 for i in range(len(vis)):
79 if type(vis[i]) == str:
80 vis[i] = vis[i].rstrip('/')
81 if os.path.islink(vis[i]):
82 raise ValueError('Parameter vis must only contain real paths, not symbolic links.\n'
83 +vis[i]+' is a link.')
84 else:
85 raise ValueError('Parameter vis must only contain strings.')
88 doweightscale = False
89 if(len(visweightscale)>0):
90 if (len(visweightscale) != len(vis)):
91 raise ValueError('parameter visweightscale must have same number of elements as parameter vis')
92 for factor in visweightscale:
93 if factor<0.:
94 raise ValueError('parameter visweightscale must only contain positive numbers')
95 elif factor!=1.:
96 doweightscale=True
98 if((type(concatvis)!=str) or (len(concatvis.split()) < 1)):
99 raise ValueError('Parameter concatvis is invalid.')
101 if(vis.count(concatvis) > 0):
102 raise ValueError('Parameter concatvis must not be equal to one of the members of parameter vis.')
104 if(os.path.exists(concatvis)):
105 raise ValueError('The output MMS must not yet exist.')
107 # test the consistency of the setup of the different MSs
108 casalog.post('Checking MS setup consistency ...', 'INFO')
109 try:
110 mydiff = check_mslist(vislist, ignore_tables=['SORTED_TABLE', 'ASDM*'], testcontent=False)
111 except Exception as instance:
112 raise RuntimeError("*** Error \'%s\' while checking MS setup consistency" % (instance))
114 if mydiff != {}:
115 casalog.post('The setup of the input MSs is not fully consistent. The concatenation may fail', 'WARN')
116 casalog.post('and/or the affected columns may contain partially only default data.', 'WARN')
117 casalog.post(str(mydiff), 'WARN')
119 # process the input MSs in chronological order
120 casalog.post('Checking order of MS list ...', 'INFO')
121 try:
122 sortedvis, sortedtimes, sortedvisweightscale = sort_mslist(vis, visweightscale)
123 except Exception as instance:
124 raise RuntimeError("*** Error \'%s\' while sorting MSs chronologially." % (instance))
127 if((type(concatvis)!=str) or (len(concatvis.split()) < 1)):
128 raise ValueError('parameter concatvis is invalid')
130 existingconcatvis = False
131 if(vis.count(concatvis) > 0):
132 existingconcatvis = True
133 cvisindex = sortedvis.index(concatvis)
134 if not concatvis == sortedvis[0]:
135 raise RuntimeError('If concatvis is set to the name of an existing MS in vis, it must be the chronologically first.'+\
136 '\n I.e. in this case you should set concatvis to '+sortedvis[0])
137 sortedvis.pop(cvisindex)
138 if doweightscale:
139 vwscale = sortedvisweightscale[cvisindex]
140 sortedvisweightscale.pop(cvisindex)
141 sortedvisweightscale = [vwscale] + sortedvisweightscale # move the corresponding weight to the front
143 if not vis == sortedvis:
144 casalog.post('The list of input MSs is not in chronological order and needed to be sorted.' , 'INFO')
145 casalog.post('The chronological order in which the concatenation will take place is:' , 'INFO')
146 if existingconcatvis:
147 casalog.post(' MJD '+str(_qa.splitdate(_qa.quantity(sortedtimes[0],'s'))['mjd'])+': '+concatvis, 'INFO')
148 for name in sortedvis:
149 casalog.post(' MJD '+str(_qa.splitdate(_qa.quantity(sortedtimes[sortedvis.index(name)],'s'))['mjd'])+': '+name, 'INFO')
150 if doweightscale:
151 casalog.post('In this new order, the weights are:'+str(sortedvisweightscale) , 'INFO')
154 # replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists)
155 vis = sortedvis
156 visweightscale = sortedvisweightscale
158 # if there are MMSs among the input, make their constituents the new input
159 mmslist = []
160 ismaster = []
161 for elvis in vis:
162 ismaster.append(True) # may be revised later
163 if(ParallelTaskHelper.isParallelMS(elvis)):
164 mmslist.append(elvis)
165 if len(mmslist)>0:
166 casalog.post('*** The following input measurement sets are multi-MSs', 'INFO')
167 for mname in mmslist:
168 casalog.post('*** '+mname, 'INFO')
169 oldvis = vis
170 oldvisweightscale = visweightscale
171 vis = []
172 visweightscale = []
173 ismaster = [] # reset ismaster
174 i = 0
175 for elvis in oldvis:
176 if elvis in mmslist: # append the subMSs individually
177 m.open(elvis)
178 mses = m.getreferencedtables()
179 m.close()
180 mses.sort()
181 mastername = os.path.basename(os.path.dirname(os.path.realpath(elvis+'/ANTENNA')))
182 for mname in mses:
183 # casalog.post('subms: ', mname)
184 vis.append(mname)
185 if doweightscale:
186 visweightscale.append(oldvisweightscale[i])
187 if os.path.basename(mname) == mastername:
188 ismaster.append(True)
189 else:
190 ismaster.append(False)
191 else:
192 vis.append(elvis)
193 if doweightscale:
194 visweightscale.append(oldvisweightscale[i])
195 ismaster.append(True)
196 i += 1
199 if keepcopy:
200 casalog.post('*** keepcopy==True: creating copy of input MSs to keep ...' , 'INFO')
201 tempdir = 'concat_tmp_'+str(time.time())
202 os.mkdir(tempdir)
203 for elvis in originalvis:
204 shutil.move(elvis,tempdir) # keep timestamps and permissions
205 shutil.copytree(tempdir+'/'+elvis, elvis, True) # symlinks=True
207 casalog.post('Concatenating ...' , 'INFO')
209 if not copypointing: # delete the rows of all pointing tables
210 casalog.post('*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO')
211 tmptabname = 'TMPPOINTING'+str(time.time())
212 tmptabname2 = 'TMPPOINTING2'+str(time.time())
213 shutil.rmtree(tmptabname, ignore_errors=True)
214 shutil.rmtree(tmptabname2, ignore_errors=True)
215 shutil.move(vis[0]+'/POINTING', tmptabname)
216 t.open(tmptabname)
217 if(t.nrows()>0):
218 ttab = t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True)
219 ttab.close()
220 t.close()
221 shutil.rmtree(tmptabname, ignore_errors=True)
222 else: # the POINTING table is already empty
223 t.close()
224 casalog.post('*** Input POINTING table was already empty.', 'INFO')
225 shutil.move(tmptabname, tmptabname2)
227 for i in range(len(vis)): # replace the POINTING tables by the empty one
228 os.system('rm -rf '+vis[i]+'/POINTING')
229 shutil.copytree(tmptabname2, vis[i]+'/POINTING')
230 shutil.rmtree(tmptabname2, ignore_errors=True)
232 if(len(vis) >0): # (note: in case len is 1, we only copy, essentially)
233 theconcatvis = vis[0]
234 if(len(vis)==1):
235 shutil.copytree(vis[0], concatvis, True)
236 vis.remove(vis[0])
238 # Determine if scratch columns should be considered at all
239 # by checking if any of the MSs has them.
241 considerscrcols = False
242 considercorr = False
243 considermodel = False
244 needscrcols = []
245 needmodel = []
246 needcorr = []
247 if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))):
248 # check if all scratch columns are present
249 t.open(theconcatvis)
250 if (t.colnames().count('MODEL_DATA')==1):
251 considermodel = True
252 if(t.colnames().count('CORRECTED_DATA')==1):
253 considercorr = True
255 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0
256 or t.colnames().count('MODEL_DATA')==0)
257 needmodel.append(t.colnames().count('MODEL_DATA')==0)
258 needcorr.append(t.colnames().count('CORRECTED_DATA')==0)
259 t.close()
261 else:
262 raise ValueError('Visibility data set '+theconcatvis+' not found - please verify the name')
264 for elvis in vis : ###Oh no Elvis does not exist Mr Bill
265 if(not os.path.exists(elvis)):
266 raise ValueError('Visibility data set '+elvis+' not found - please verify the name')
268 # check if all scratch columns are present
269 t.open(elvis)
270 if (t.colnames().count('MODEL_DATA')==1):
271 considermodel = True
272 if(t.colnames().count('CORRECTED_DATA')==1):
273 considercorr = True
275 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0
276 or t.colnames().count('MODEL_DATA')==0)
277 needmodel.append(t.colnames().count('MODEL_DATA')==0)
278 needcorr.append(t.colnames().count('CORRECTED_DATA')==0)
279 t.close()
281 considerscrcols = (considercorr or considermodel) # there are scratch columns
284 # start actual work, file existence has already been checked
286 if(considerscrcols and needscrcols[0]):
287 # create scratch cols
288 casalog.post('creating scratch columns in '+theconcatvis , 'INFO')
289 _cb.open(theconcatvis,
290 addcorr=(considercorr and needcorr[0]),
291 addmodel=(considermodel and needmodel[0])) # calibrator-open creates scratch columns
292 _cb.close()
294 # scale the weights of the first MS in the chain
295 if doweightscale:
296 wscale = visweightscale[0]
297 if(wscale==1.):
298 casalog.post('Will leave the weights for this MS unchanged.', 'INFO')
299 else:
300 casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO')
301 t.open(theconcatvis, nomodify=False)
302 for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']:
303 if (colname in t.colnames()) and (t.iscelldefined(colname,0)):
304 for j in range(0,t.nrows()):
305 a = t.getcell(colname, j)
306 a *= wscale
307 t.putcell(colname, j, a)
308 t.close()
310 m.open(theconcatvis,nomodify=False)
311 mmsmembers = [theconcatvis]
313 auxfile = 'concat_aux_'+str(time.time())
315 i = 0
316 for elvis in vis :
317 i += 1
319 mmsmembers.append(elvis)
320 casalog.post('adding '+elvis+' to multi-MS '+concatvis, 'INFO')
322 wscale = 1.
323 if doweightscale:
324 wscale = visweightscale[i]
325 if(wscale==1.):
326 casalog.post('Will leave the weights for this MS unchanged.', 'INFO')
327 else:
328 casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO')
330 if(considerscrcols and needscrcols[i]):
331 # create scratch cols
332 casalog.post('creating scratch columns for '+elvis, 'INFO')
333 _cb.open(elvis,
334 addcorr=(considercorr and needcorr[i]),
335 addmodel=(considermodel and needmodel[i])) # calibrator-open creates scratch columns
336 _cb.close()
338 m.virtconcatenate(msfile=elvis,
339 auxfilename=auxfile,
340 freqtol=freqtol,dirtol=dirtol,respectname=respectname,
341 weightscale=wscale)
342 #end for
343 os.remove(auxfile)
344 m.close()
346 # Write history to MS
347 try:
348 param_names = virtualconcat.__code__.co_varnames[:virtualconcat.__code__.co_argcount]
349 vars = locals( )
350 param_vals = [vars[p] for p in param_names]
351 write_history(mstool(), theconcatvis, 'virtualconcat', param_names,
352 param_vals, casalog)
353 except Exception as instance:
354 casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
355 'WARN')
358 # concatenate the POINTING tables
359 masterptable = mmsmembers[0]+'/POINTING'
360 ptablemembers = []
361 if os.path.exists(masterptable) and copypointing:
362 casalog.post('Concatenating the POINTING tables ...', 'INFO')
363 for i in range(len(mmsmembers)):
364 ptable = mmsmembers[i]+'/POINTING'
365 if ismaster[i] and os.path.exists(ptable):
366 casalog.post(' '+ptable, 'INFO')
367 shutil.move(ptable, ptable+str(i))
368 ptablemembers.append(ptable+str(i))
369 #end for
370 t.createmultitable(masterptable, ptablemembers, 'SUBTBS')
371 # endif
373 # Get all available subtables
374 thesubtables = ph.getSubtables(mmsmembers[0])
376 # Remove the SOURCE and HISTORY tables, which will be the only copied.
377 # All other sub-tables will be linked to first subms
378 thesubtables.remove('SOURCE')
379 thesubtables.remove('HISTORY')
380 subtabs_to_omit = thesubtables
382 ph.makeMMS(concatvis, mmsmembers,
383 True, # copy subtables from first to all other members
384 subtabs_to_omit) # excluding tables which will be linked
386 # remove the remaining "hulls" of the emptied input MMSs (if there are any)
387 for elvis in mmslist:
388 shutil.rmtree(elvis)
390 if keepcopy:
391 for elvis in originalvis:
392 shutil.move(tempdir+'/'+elvis, elvis)
393 os.rmdir(tempdir)
395 finally:
396 if keepcopy and tempdir!='':
397 casalog.post("Restoring original MSs ...")
398 for elvis in originalvis:
399 if os.path.exists(tempdir+'/'+elvis):
400 shutil.rmtree(elvis)
401 shutil.move(tempdir+'/'+elvis, elvis)
402 try:
403 os.rmdir(tempdir)
404 except OSError:
405 pass