Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_mstransform.py: 72%

219 statements  

« prev     ^ index     » next       coverage.py v7.6.4, created at 2024-11-01 07:19 +0000

1 

2import os, re 

3import shutil 

4import string 

5import copy 

6import math 

7import time 

8 

9from casatools import table, quanta, ms, mstransformer 

10from casatasks import casalog 

11from .parallel.parallel_data_helper import ParallelDataHelper 

12from . import flaghelper as fh 

13from .update_spw import update_spwchan 

14from .mstools import write_history 

15from .callibrary import callibrary 

16 

17def mstransform( 

18 vis, 

19 outputvis, # output 

20 createmms, # MMS --> partition 

21 separationaxis, 

22 numsubms, 

23 tileshape, # tiling 

24 field, 

25 spw, 

26 scan, 

27 antenna, 

28 correlation, 

29 timerange, 

30 intent, 

31 array, 

32 uvrange, 

33 observation, 

34 feed, 

35 datacolumn, 

36 realmodelcol, 

37 keepflags, 

38 usewtspectrum, 

39 combinespws, # spw combination --> cvel 

40 chanaverage, # channel averaging --> split 

41 chanbin, 

42 hanning, # Hanning --> cvel 

43 regridms, # regridding to new frame --> cvel 

44 mode, 

45 nchan, 

46 start, 

47 width, 

48 nspw, # spw separation 

49 interpolation, 

50 phasecenter, 

51 restfreq, 

52 outframe, 

53 veltype, 

54 preaverage, 

55 timeaverage, # time averaging --> split 

56 timebin, 

57 timespan, 

58 maxuvwdistance, 

59 docallib, 

60 callib, 

61 douvcontsub, 

62 fitspw, 

63 fitorder, 

64 want_cont, 

65 denoising_lib, 

66 nthreads, 

67 niter, 

68 disableparallel, # HIDDEN parameter to create MMS in sequential 

69 ddistart, # HIDDEN internal parameter for the sub-table re-indexing 

70 taql, # HIDDEN internal parameter 

71 monolithic_processing, # HIDDEN parameter for the MMS monolithic processing 

72 reindex # HIDDEN parameter for use in the pipeline context only 

73 ): 

74 

75 ''' This task can replace split, cvel, partition and hanningsmooth ''' 

76 

77 casalog.origin('mstransform') 

78 

79 ''' HEURISTICS: 

80 input MS --> output MS 

81 input MMS --> output MMS 

82  

83 user can set createmms=True to create the following: 

84 input MS --> output MMS 

85  

86 ''' 

87 # Initialize the helper class 

88 pdh = ParallelDataHelper('mstransform', locals()) 

89 

90 # When dealing with MMS, process in parallel or sequential 

91 # disableparallel is a hidden parameter. Only for debugging purposes! 

92 if disableparallel: 

93 pdh.bypassParallelProcessing(1) 

94 else: 

95 pdh.bypassParallelProcessing(0) 

96 

97 # Validate input and output parameters 

98 pdh.setupIO() 

99 

100 # Process the input Multi-MS 

101 if ParallelDataHelper.isMMSAndNotServer(vis) == True and monolithic_processing == False: 

102 ''' 

103 retval{'status': True, 'axis':''} --> can run in parallel  

104 retval{'status': False, 'axis':'value'} --> treat MMS as monolithic MS, set new axis for output MMS 

105 retval{'status': False, 'axis':''} --> treat MMS as monolithic MS, create an output MS 

106 ''' 

107 

108 retval = pdh.validateInputParams() 

109 # Cannot create an output MMS. 

110 if retval['status'] == False and retval['axis'] == '': 

111 casalog.post('Cannot process MMS with the requested transformations','WARN') 

112 casalog.post('Use task listpartition to see the contents of the MMS') 

113 casalog.post('Will create an output MS','WARN') 

114 createmms = False 

115 

116 # MMS is processed as monolithic MS.  

117 elif retval['status'] == False and retval['axis'] != '': 

118 createmms = True 

119 pdh.override__args('createmms', True) 

120 pdh.override__args('monolithic_processing', True) 

121 separationaxis = retval['axis'] 

122 pdh.override__args('separationaxis', retval['axis']) 

123 casalog.post("Will process the input MMS as a monolithic MS",'WARN') 

124 casalog.post("Will create an output MMS with separation axis \'%s\'"%retval['axis'],'WARN') 

125 

126 # MMS is processed in parallel 

127 else: 

128 createmms = False 

129 try: 

130 pdh.override__args('createmms', False) 

131 pdh.setupCluster('mstransform') 

132 pdh.go() 

133 except Exception as instance: 

134 casalog.post('%s'%instance,'ERROR') 

135 return False 

136 

137 return True 

138 

139 # Create an output Multi-MS 

140 if createmms == True: 

141 

142 # Check the heuristics of separationaxis and the requested transformations 

143 pval = pdh.validateOutputParams() 

144 if pval == 0: 

145 raise Exception('Cannot create MMS using separationaxis=%s with some of the requested transformations.' % separationaxis) 

146 

147 pdh.setupCluster('mstransform') 

148 pdh.go() 

149 monolithic_processing = False 

150 return 

151 

152 

153 # Create a local copy of the MSTransform tool 

154 mtlocal = mstransformer() 

155 mslocal = ms() 

156 qalocal = quanta() 

157 

158 try: 

159 

160 # Gather all the parameters in a dictionary. 

161 config = {} 

162 

163 if keepflags: 

164 taqlstr = '' 

165 else: 

166 taqlstr = "NOT (FLAG_ROW OR ALL(FLAG))" 

167 

168 # MMS taql selection 

169 if taql != '' and taql != None: 

170 if not keepflags: 

171 taqlstr = taqlstr + " AND "+taql 

172 else: 

173 taqlstr = taql 

174 

175 config = pdh.setupParameters(inputms=vis, outputms=outputvis, field=field, 

176 spw=spw, array=array, scan=scan, antenna=antenna, correlation=correlation, 

177 uvrange=uvrange,timerange=timerange, intent=intent, observation=str(observation), 

178 feed=feed, taql=taqlstr) 

179 

180 # ddistart will be used in the tool when re-indexing the spw table 

181 config['ddistart'] = ddistart 

182 

183 # re-index parameter is used by the pipeline to not re-index any sub-table and the associated IDs 

184 config['reindex'] = reindex 

185 

186 config['datacolumn'] = datacolumn 

187 dc = datacolumn.upper() 

188 # Make real a virtual MODEL column in the output MS 

189 if "MODEL" in dc or dc == 'ALL': 

190 config['realmodelcol'] = realmodelcol 

191 

192 config['usewtspectrum'] = usewtspectrum 

193 

194 # Add the tile shape parameter 

195 if tileshape.__len__() == 1: 

196 # The only allowed values are 0 or 1 

197 if tileshape[0] != 0 and tileshape[0] != 1: 

198 raise ValueError('When tileshape has one element, it should be either 0 or 1.') 

199 

200 elif tileshape.__len__() != 3: 

201 # The 3 elements are: correlations, channels, rows 

202 raise ValueError('Parameter tileshape must have 1 or 3 elements.') 

203 

204 config['tileshape'] = tileshape 

205 

206 if combinespws: 

207 casalog.post('Combine spws %s into new output spw'%spw) 

208 config['combinespws'] = True 

209 

210 # Only parse chanaverage if chanbin is valid 

211 if chanaverage and isinstance(chanbin, int) and chanbin <= 1: 

212 raise ValueError('Parameter chanbin must be > 1 to do channel averaging') 

213 

214 # Validate the case of int or list chanbin 

215 if chanaverage and pdh.validateChanBin(): 

216 casalog.post('Parse channel averaging parameters') 

217 config['chanaverage'] = True 

218 

219 # convert numpy types, until CAS-6493 is not fixed 

220 chanbin = fh.evaluateNumpyType(chanbin) 

221 config['chanbin'] = chanbin 

222 

223 if hanning: 

224 casalog.post('Apply Hanning smoothing') 

225 config['hanning'] = True 

226 

227 if regridms: 

228 casalog.post('Parse regridding parameters') 

229 config['regridms'] = True 

230 # Reset the defaults depending on the mode 

231 # Only add non-empty string parameters to config dictionary 

232 start, width = pdh.defaultRegridParams() 

233 config['mode'] = mode 

234 config['nchan'] = nchan 

235 if start != '': 

236 config['start'] = start 

237 if width != '': 

238 config['width'] = width 

239 if nspw > 1: 

240 casalog.post('Separate MS into %s spws'%nspw) 

241 config['nspw'] = nspw 

242 config['interpolation'] = interpolation 

243 if restfreq != '': 

244 config['restfreq'] = restfreq 

245 if outframe != '': 

246 config['outframe'] = outframe 

247 if phasecenter != '': 

248 config['phasecenter'] = phasecenter 

249 config['veltype'] = veltype 

250 config['preaverage'] = preaverage 

251 

252 # Only parse timeaverage parameters when timebin > 0s 

253 if timeaverage: 

254 tb = qalocal.convert(qalocal.quantity(timebin), 's')['value'] 

255 if not tb > 0: 

256 raise ValueError("Parameter timebin must be > '0s' to do time averaging") 

257 

258 if timeaverage: 

259 casalog.post('Parse time averaging parameters') 

260 config['timeaverage'] = True 

261 config['timebin'] = timebin 

262 config['timespan'] = timespan 

263 config['maxuvwdistance'] = maxuvwdistance 

264 

265 if docallib: 

266 casalog.post('Parse docallib parameters') 

267 mycallib = callibrary() 

268 mycallib.read(callib) 

269 config['calibration'] = True 

270 config['callib'] = mycallib.cld 

271 

272 if douvcontsub: 

273 casalog.post('Parse uvcontsub parameters') 

274 casalog.post("The parameter douvcontsub of mstransform is deprecated and will " 

275 "be removed in an upcoming release. The task uvcontsub can be used " 

276 "as an alternative.", "WARN") 

277 config['uvcontsub'] = True 

278 uvcontsub_config = {} 

279 uvcontsub_config['fitspw'] = fitspw 

280 uvcontsub_config['fitorder'] = fitorder 

281 uvcontsub_config['want_cont'] = want_cont 

282 uvcontsub_config['denoising_lib'] = denoising_lib 

283 uvcontsub_config['nthreads'] = nthreads 

284 uvcontsub_config['niter'] = niter 

285 config['uvcontsublib'] = dict(uvcontsub_config) 

286 

287 # Configure the tool and all the parameters 

288 

289 casalog.post('%s'%config, 'DEBUG') 

290 mtlocal.config(config) 

291 

292 # Open the MS, select the data and configure the output 

293 mtlocal.open() 

294 

295 # Run the tool 

296 casalog.post('Apply the transformations') 

297 mtlocal.run() 

298 

299 finally: 

300 mtlocal.done() 

301 

302 

303 # Update the FLAG_CMD sub-table to reflect any spw/channels selection 

304 # If the spw selection is by name or FLAG_CMD contains spw with names, skip the updating 

305 if ((spw != '') and (spw != '*')) or chanaverage == True: 

306 isopen = False 

307 

308 try: 

309 mytb = table() 

310 mytb.open(outputvis + '/FLAG_CMD', nomodify=False) 

311 isopen = True 

312 nflgcmds = mytb.nrows() 

313 

314 if nflgcmds > 0: 

315 updateFlagCmd = False 

316 

317 # If spw selection is by name in FLAG_CMD, do not update, CAS-7751 

318 mycmd = mytb.getcell('COMMAND', 0) 

319 cmdlist = mycmd.split() 

320 for cmd in cmdlist: 

321 # Match only spw indices, not names 

322 if cmd.__contains__('spw'): 

323 cmd = cmd.strip("spw=") 

324 spwstr = re.search('^[^a-zA-Z]+$', cmd) 

325 if spwstr != None and spwstr.string.__len__() > 0: 

326 updateFlagCmd = True 

327 break 

328 

329 

330 if updateFlagCmd: 

331 mademod = False 

332 cmds = mytb.getcol('COMMAND') 

333 widths = {} 

334 if hasattr(chanbin, 'has_key'): 

335 widths = chanbin 

336 else: 

337 if hasattr(chanbin, '__iter__') and len(chanbin) > 1: 

338 for i in range(len(chanbin)): 

339 widths[i] = chanbin[i] 

340 elif chanbin != 1: 

341 numspw = len(mslocal.msseltoindex(vis=vis, 

342 spw='*')['spw']) 

343 if hasattr(chanbin, '__iter__'): 

344 w = chanbin[0] 

345 else: 

346 w = chanbin 

347 for i in range(numspw): 

348 widths[i] = w 

349 for rownum in range(nflgcmds): 

350 # Matches a bare number or a string quoted any way. 

351 spwmatch = re.search(r'spw\s*=\s*(\S+)', cmds[rownum]) 

352 if spwmatch: 

353 sch1 = spwmatch.groups()[0] 

354 sch1 = re.sub(r"[\'\"]", '', sch1) # Dequote 

355 # Provide a default in case the split selection excludes 

356 # cmds[rownum]. update_spwchan() will throw an exception 

357 # in that case. 

358 cmd = '' 

359 try: 

360 sch2 = update_spwchan(vis, spw, sch1, truncate=True, 

361 widths=widths) 

362 if sch2: 

363 repl = '' 

364 if sch2 != '*': 

365 repl = "spw='" + sch2 + "'" 

366 cmd = cmds[rownum].replace(spwmatch.group(), repl) 

367 #except: # cmd[rownum] no longer applies. 

368 except Exception as e: 

369 casalog.post("Error %s updating row %d of FLAG_CMD" % (e,rownum),'WARN') 

370 casalog.post('sch1 = ' + sch1, 'DEBUG1') 

371 casalog.post('cmd = ' + cmd, 'DEBUG1') 

372 if cmd != cmds[rownum]: 

373 mademod = True 

374 cmds[rownum] = cmd 

375 if mademod: 

376 casalog.post('Updating FLAG_CMD', 'INFO') 

377 mytb.putcol('COMMAND', cmds) 

378 

379 else: 

380 casalog.post('FLAG_CMD table contains spw selection by name. Will not update it!','DEBUG') 

381 

382 

383 finally: 

384 if isopen: 

385 mytb.close() 

386 mslocal = None 

387 mytb = None 

388 

389 

390 # Write history to output MS, not the input ms. 

391 try: 

392 param_names = mstransform.__code__.co_varnames[:mstransform.__code__.co_argcount] 

393 vars = locals( ) 

394 param_vals = [vars[p] for p in param_names] 

395 

396 write_history(mslocal, outputvis, 'mstransform', param_names, param_vals, casalog) 

397 except Exception as instance: 

398 casalog.post("*** Error \'%s\' updating HISTORY" % (instance),'WARN') 

399 

400 mslocal = None 

401 

402 

403