Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_virtualconcat.py: 75%

259 statements  

« prev     ^ index     » next       coverage.py v7.6.4, created at 2024-11-01 07:19 +0000

1import os 

2import shutil 

3import stat 

4import time 

5 

6from . import partitionhelper as ph 

7from .parallel.parallel_task_helper import ParallelTaskHelper 

8from .mslisthelper import check_mslist, sort_mslist 

9from .mstools import write_history 

10 

11from casatools import calibrater, quanta 

12from casatools import ms as mstool 

13from casatools import table as tbtool 

14from casatasks import casalog 

15 

16_cb = calibrater() 

17_qa = quanta() 

18 

19 

20def virtualconcat(vislist,concatvis,freqtol,dirtol,respectname, 

21 visweightscale,keepcopy,copypointing): 

22 """ 

23 Concatenate visibility data sets creating a Multi-MS. 

24  

25 Combine the input datasets into a Multi-MS. 

26 NOTE: The input datasets are moved into the Multi-MS and may be modified 

27 to account for subtable reference changes. 

28 If none of the input MSs have any scratch columns, none are created. 

29 Otherwise scratch columns are created and initialized in those MSs 

30 which don't have a complete set. 

31 

32 

33 Keyword arguments: 

34 vis -- Name of input visibility files (MSs, only use real paths, no symlinks) 

35 default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms'] 

36 concatvis -- Name of the output visibility file (MMS) 

37 default: none; example: concatvis='src2.ms' 

38 freqtol -- Frequency shift tolerance for considering data as the same spwid 

39 default: '' means always combine 

40 example: freqtol='10MHz' will not combine spwid unless they are 

41 within 10 MHz 

42 dirtol -- Direction shift tolerance for considering data as the same field 

43 default: ;; means always combine 

44 example: dirtol='1.arcsec' will not combine data for a field unless 

45 their phase center is less than 1 arcsec. 

46 respectname -- If true, fields with a different name are not merged even if their  

47 direction agrees (within dirtol) 

48 default: True 

49 visweightscale -- list of the weight scales to be applied to the individual MSs 

50 default: [] (don't modify weights, equivalent to setting scale to 1 for each MS) 

51 keepcopy -- If true, a copy of the input MSs is kept in their original place. 

52 default: false 

53 copypointing -- If true, the POINTING table information will be present in the output. 

54 If false, the result is an empty POINTING table. 

55 default: True 

56 

57 """ 

58 

59 ### 

60 #Python script 

61 

62 tempdir = '' 

63 originalvis = vislist 

64 try: 

65 casalog.origin('virtualconcat') 

66 t = tbtool() 

67 m = mstool() 

68 

69 #break the reference between vis and vislist as we modify vis 

70 if(type(vislist)==str): 

71 vis=[vislist] 

72 else: 

73 vis=list(vislist) 

74 #dto. for concavis 

75 theconcatvis = concatvis 

76 

77 #sanitize the members of vis 

78 for i in range(len(vis)): 

79 if type(vis[i]) == str: 

80 vis[i] = vis[i].rstrip('/') 

81 if os.path.islink(vis[i]): 

82 raise ValueError('Parameter vis must only contain real paths, not symbolic links.\n' 

83 +vis[i]+' is a link.') 

84 else: 

85 raise ValueError('Parameter vis must only contain strings.') 

86 

87 

88 doweightscale = False 

89 if(len(visweightscale)>0): 

90 if (len(visweightscale) != len(vis)): 

91 raise ValueError('parameter visweightscale must have same number of elements as parameter vis') 

92 for factor in visweightscale: 

93 if factor<0.: 

94 raise ValueError('parameter visweightscale must only contain positive numbers') 

95 elif factor!=1.: 

96 doweightscale=True 

97 

98 if((type(concatvis)!=str) or (len(concatvis.split()) < 1)): 

99 raise ValueError('Parameter concatvis is invalid.') 

100 

101 if(vis.count(concatvis) > 0): 

102 raise ValueError('Parameter concatvis must not be equal to one of the members of parameter vis.') 

103 

104 if(os.path.exists(concatvis)): 

105 raise ValueError('The output MMS must not yet exist.') 

106 

107 # test the consistency of the setup of the different MSs 

108 casalog.post('Checking MS setup consistency ...', 'INFO') 

109 try: 

110 mydiff = check_mslist(vislist, ignore_tables=['SORTED_TABLE', 'ASDM*'], testcontent=False) 

111 except Exception as instance: 

112 raise RuntimeError("*** Error \'%s\' while checking MS setup consistency" % (instance)) 

113 

114 if mydiff != {}: 

115 casalog.post('The setup of the input MSs is not fully consistent. The concatenation may fail', 'WARN') 

116 casalog.post('and/or the affected columns may contain partially only default data.', 'WARN') 

117 casalog.post(str(mydiff), 'WARN') 

118 

119 # process the input MSs in chronological order 

120 casalog.post('Checking order of MS list ...', 'INFO') 

121 try: 

122 sortedvis, sortedtimes, sortedvisweightscale = sort_mslist(vis, visweightscale) 

123 except Exception as instance: 

124 raise RuntimeError("*** Error \'%s\' while sorting MSs chronologially." % (instance)) 

125 

126 

127 if((type(concatvis)!=str) or (len(concatvis.split()) < 1)): 

128 raise ValueError('parameter concatvis is invalid') 

129 

130 existingconcatvis = False 

131 if(vis.count(concatvis) > 0): 

132 existingconcatvis = True 

133 cvisindex = sortedvis.index(concatvis) 

134 if not concatvis == sortedvis[0]: 

135 raise RuntimeError('If concatvis is set to the name of an existing MS in vis, it must be the chronologically first.'+\ 

136 '\n I.e. in this case you should set concatvis to '+sortedvis[0]) 

137 sortedvis.pop(cvisindex) 

138 if doweightscale: 

139 vwscale = sortedvisweightscale[cvisindex] 

140 sortedvisweightscale.pop(cvisindex) 

141 sortedvisweightscale = [vwscale] + sortedvisweightscale # move the corresponding weight to the front 

142 

143 if not vis == sortedvis: 

144 casalog.post('The list of input MSs is not in chronological order and needed to be sorted.' , 'INFO') 

145 casalog.post('The chronological order in which the concatenation will take place is:' , 'INFO') 

146 if existingconcatvis: 

147 casalog.post(' MJD '+str(_qa.splitdate(_qa.quantity(sortedtimes[0],'s'))['mjd'])+': '+concatvis, 'INFO') 

148 for name in sortedvis: 

149 casalog.post(' MJD '+str(_qa.splitdate(_qa.quantity(sortedtimes[sortedvis.index(name)],'s'))['mjd'])+': '+name, 'INFO') 

150 if doweightscale: 

151 casalog.post('In this new order, the weights are:'+str(sortedvisweightscale) , 'INFO') 

152 

153 

154 # replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists) 

155 vis = sortedvis 

156 visweightscale = sortedvisweightscale 

157 

158 # if there are MMSs among the input, make their constituents the new input 

159 mmslist = [] 

160 ismaster = [] 

161 for elvis in vis: 

162 ismaster.append(True) # may be revised later 

163 if(ParallelTaskHelper.isParallelMS(elvis)): 

164 mmslist.append(elvis) 

165 if len(mmslist)>0: 

166 casalog.post('*** The following input measurement sets are multi-MSs', 'INFO') 

167 for mname in mmslist: 

168 casalog.post('*** '+mname, 'INFO') 

169 oldvis = vis 

170 oldvisweightscale = visweightscale 

171 vis = [] 

172 visweightscale = [] 

173 ismaster = [] # reset ismaster 

174 i = 0 

175 for elvis in oldvis: 

176 if elvis in mmslist: # append the subMSs individually 

177 m.open(elvis) 

178 mses = m.getreferencedtables() 

179 m.close() 

180 mses.sort() 

181 mastername = os.path.basename(os.path.dirname(os.path.realpath(elvis+'/ANTENNA'))) 

182 for mname in mses: 

183 # casalog.post('subms: ', mname) 

184 vis.append(mname) 

185 if doweightscale: 

186 visweightscale.append(oldvisweightscale[i]) 

187 if os.path.basename(mname) == mastername: 

188 ismaster.append(True) 

189 else: 

190 ismaster.append(False) 

191 else: 

192 vis.append(elvis) 

193 if doweightscale: 

194 visweightscale.append(oldvisweightscale[i]) 

195 ismaster.append(True) 

196 i += 1 

197 

198 

199 if keepcopy: 

200 casalog.post('*** keepcopy==True: creating copy of input MSs to keep ...' , 'INFO') 

201 tempdir = 'concat_tmp_'+str(time.time()) 

202 os.mkdir(tempdir) 

203 for elvis in originalvis: 

204 shutil.move(elvis,tempdir) # keep timestamps and permissions 

205 shutil.copytree(tempdir+'/'+elvis, elvis, True) # symlinks=True 

206 

207 casalog.post('Concatenating ...' , 'INFO') 

208 

209 if not copypointing: # delete the rows of all pointing tables 

210 casalog.post('*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO') 

211 tmptabname = 'TMPPOINTING'+str(time.time()) 

212 tmptabname2 = 'TMPPOINTING2'+str(time.time()) 

213 shutil.rmtree(tmptabname, ignore_errors=True) 

214 shutil.rmtree(tmptabname2, ignore_errors=True) 

215 shutil.move(vis[0]+'/POINTING', tmptabname) 

216 t.open(tmptabname) 

217 if(t.nrows()>0): 

218 ttab = t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True) 

219 ttab.close() 

220 t.close() 

221 shutil.rmtree(tmptabname, ignore_errors=True) 

222 else: # the POINTING table is already empty 

223 t.close() 

224 casalog.post('*** Input POINTING table was already empty.', 'INFO') 

225 shutil.move(tmptabname, tmptabname2) 

226 

227 for i in range(len(vis)): # replace the POINTING tables by the empty one 

228 os.system('rm -rf '+vis[i]+'/POINTING') 

229 shutil.copytree(tmptabname2, vis[i]+'/POINTING') 

230 shutil.rmtree(tmptabname2, ignore_errors=True) 

231 

232 if(len(vis) >0): # (note: in case len is 1, we only copy, essentially) 

233 theconcatvis = vis[0] 

234 if(len(vis)==1): 

235 shutil.copytree(vis[0], concatvis, True) 

236 vis.remove(vis[0]) 

237 

238 # Determine if scratch columns should be considered at all 

239 # by checking if any of the MSs has them. 

240 

241 considerscrcols = False 

242 considercorr = False 

243 considermodel = False 

244 needscrcols = [] 

245 needmodel = [] 

246 needcorr = [] 

247 if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))): 

248 # check if all scratch columns are present 

249 t.open(theconcatvis) 

250 if (t.colnames().count('MODEL_DATA')==1): 

251 considermodel = True 

252 if(t.colnames().count('CORRECTED_DATA')==1): 

253 considercorr = True 

254 

255 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 

256 or t.colnames().count('MODEL_DATA')==0) 

257 needmodel.append(t.colnames().count('MODEL_DATA')==0) 

258 needcorr.append(t.colnames().count('CORRECTED_DATA')==0) 

259 t.close() 

260 

261 else: 

262 raise ValueError('Visibility data set '+theconcatvis+' not found - please verify the name') 

263 

264 for elvis in vis : ###Oh no Elvis does not exist Mr Bill 

265 if(not os.path.exists(elvis)): 

266 raise ValueError('Visibility data set '+elvis+' not found - please verify the name') 

267 

268 # check if all scratch columns are present 

269 t.open(elvis) 

270 if (t.colnames().count('MODEL_DATA')==1): 

271 considermodel = True 

272 if(t.colnames().count('CORRECTED_DATA')==1): 

273 considercorr = True 

274 

275 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 

276 or t.colnames().count('MODEL_DATA')==0) 

277 needmodel.append(t.colnames().count('MODEL_DATA')==0) 

278 needcorr.append(t.colnames().count('CORRECTED_DATA')==0) 

279 t.close() 

280 

281 considerscrcols = (considercorr or considermodel) # there are scratch columns 

282 

283 

284 # start actual work, file existence has already been checked 

285 

286 if(considerscrcols and needscrcols[0]): 

287 # create scratch cols  

288 casalog.post('creating scratch columns in '+theconcatvis , 'INFO') 

289 _cb.open(theconcatvis, 

290 addcorr=(considercorr and needcorr[0]), 

291 addmodel=(considermodel and needmodel[0])) # calibrator-open creates scratch columns 

292 _cb.close() 

293 

294 # scale the weights of the first MS in the chain 

295 if doweightscale: 

296 wscale = visweightscale[0] 

297 if(wscale==1.): 

298 casalog.post('Will leave the weights for this MS unchanged.', 'INFO') 

299 else: 

300 casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO') 

301 t.open(theconcatvis, nomodify=False) 

302 for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']: 

303 if (colname in t.colnames()) and (t.iscelldefined(colname,0)): 

304 for j in range(0,t.nrows()): 

305 a = t.getcell(colname, j) 

306 a *= wscale 

307 t.putcell(colname, j, a) 

308 t.close() 

309 

310 m.open(theconcatvis,nomodify=False) 

311 mmsmembers = [theconcatvis] 

312 

313 auxfile = 'concat_aux_'+str(time.time()) 

314 

315 i = 0 

316 for elvis in vis : 

317 i += 1 

318 

319 mmsmembers.append(elvis) 

320 casalog.post('adding '+elvis+' to multi-MS '+concatvis, 'INFO') 

321 

322 wscale = 1. 

323 if doweightscale: 

324 wscale = visweightscale[i] 

325 if(wscale==1.): 

326 casalog.post('Will leave the weights for this MS unchanged.', 'INFO') 

327 else: 

328 casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO') 

329 

330 if(considerscrcols and needscrcols[i]): 

331 # create scratch cols  

332 casalog.post('creating scratch columns for '+elvis, 'INFO') 

333 _cb.open(elvis, 

334 addcorr=(considercorr and needcorr[i]), 

335 addmodel=(considermodel and needmodel[i])) # calibrator-open creates scratch columns 

336 _cb.close() 

337 

338 m.virtconcatenate(msfile=elvis, 

339 auxfilename=auxfile, 

340 freqtol=freqtol,dirtol=dirtol,respectname=respectname, 

341 weightscale=wscale) 

342 #end for 

343 os.remove(auxfile) 

344 m.close() 

345 

346 # Write history to MS 

347 try: 

348 param_names = virtualconcat.__code__.co_varnames[:virtualconcat.__code__.co_argcount] 

349 vars = locals( ) 

350 param_vals = [vars[p] for p in param_names] 

351 write_history(mstool(), theconcatvis, 'virtualconcat', param_names, 

352 param_vals, casalog) 

353 except Exception as instance: 

354 casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 

355 'WARN') 

356 

357 

358 # concatenate the POINTING tables 

359 masterptable = mmsmembers[0]+'/POINTING' 

360 ptablemembers = [] 

361 if os.path.exists(masterptable) and copypointing: 

362 casalog.post('Concatenating the POINTING tables ...', 'INFO') 

363 for i in range(len(mmsmembers)): 

364 ptable = mmsmembers[i]+'/POINTING' 

365 if ismaster[i] and os.path.exists(ptable): 

366 casalog.post(' '+ptable, 'INFO') 

367 shutil.move(ptable, ptable+str(i)) 

368 ptablemembers.append(ptable+str(i)) 

369 #end for 

370 t.createmultitable(masterptable, ptablemembers, 'SUBTBS') 

371 # endif 

372 

373 # Get all available subtables 

374 thesubtables = ph.getSubtables(mmsmembers[0]) 

375 

376 # Remove the SOURCE and HISTORY tables, which will be the only copied. 

377 # All other sub-tables will be linked to first subms 

378 thesubtables.remove('SOURCE') 

379 thesubtables.remove('HISTORY') 

380 subtabs_to_omit = thesubtables 

381 

382 ph.makeMMS(concatvis, mmsmembers, 

383 True, # copy subtables from first to all other members  

384 subtabs_to_omit) # excluding tables which will be linked 

385 

386 # remove the remaining "hulls" of the emptied input MMSs (if there are any) 

387 for elvis in mmslist: 

388 shutil.rmtree(elvis) 

389 

390 if keepcopy: 

391 for elvis in originalvis: 

392 shutil.move(tempdir+'/'+elvis, elvis) 

393 os.rmdir(tempdir) 

394 

395 finally: 

396 if keepcopy and tempdir!='': 

397 casalog.post("Restoring original MSs ...") 

398 for elvis in originalvis: 

399 if os.path.exists(tempdir+'/'+elvis): 

400 shutil.rmtree(elvis) 

401 shutil.move(tempdir+'/'+elvis, elvis) 

402 try: 

403 os.rmdir(tempdir) 

404 except OSError: 

405 pass