Coverage for /wheeldirectory/casa-6.7.0-12-py3.10.el8/lib/py/lib/python3.10/site-packages/casatasks/private/task_importasdm.py: 86%
226 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-11-01 07:19 +0000
1import os
2import shutil
4from casatasks import casalog
5from casatools import table, agentflagger, ms, imager, measures, msmetadata, sdm
6from .mstools import write_history
7from . import flaghelper as fh
8from . import convertephem as ce
9from .parallel.parallel_data_helper import ParallelDataHelper
11_tb = table()
13def importasdm(
14 asdm=None,
15 vis=None,
16 createmms=None,
17 separationaxis=None,
18 numsubms=None,
19 corr_mode=None,
20 srt=None,
21 time_sampling=None,
22 ocorr_mode=None,
23 compression=None,
24 lazy=None,
25 asis=None,
26 wvr_corrected_data=None,
27 scans=None,
28 ignore_time=None,
29 process_syspower=None,
30 process_caldevice=None,
31 process_pointing=None,
32 process_flags=None,
33 tbuff=None,
34 applyflags=None,
35 savecmds=None,
36 outfile=None,
37 flagbackup=None,
38 verbose=None,
39 overwrite=None,
40 bdfflags=None,
41 with_pointing_correction=None,
42 convert_ephem2geo=None,
43 polyephem_tabtimestep=None
44 ):
45 """Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
46 The conversion of the ALMA SDM archive format into a measurement set. This version
47 is under development and is geared to handling many spectral windows of different
48 shapes.
50 Keyword arguments:
51 asdm -- Name of input ASDM file (directory)
52 default: none; example: asdm='ExecBlock3'
54 vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
55 default: none
57 createmms -- Create a Multi-MS
58 default: False
60 corr_mode -- correlation mode to be considered on input. Could
61 be one or more of the following, ao, co, ac, or all
62 default: all
64 srt -- spectral resolution type. Could be one or more of
65 the following, fr, ca, bw, or all
66 default: all
68 time_sampling -- specifies the time sampling, INTEGRATION and/or
69 SUBINTEGRATION. could be one or more of the following
70 i, si, or all.
71 default: all
73 ocorr_mode -- output data for correlation mode AUTO_ONLY
74 (ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
75 default: ca
77 compression -- produces comrpressed columns in the resulting measurement set.
78 default: False
80 lazy -- Make the MS DATA column read the ASDM Binary data directly
81 (faster import, smaller MS)
82 default: False
84 asis -- creates verbatim copies of the ASDM tables in
85 the output measurement set. The value given to
86 this option must be a list of table names separated
87 by space characters; the wildcard character '*' is
88 allowed in table names.
90 wvr_corrected_data -- specifies which values are considered in the
91 ASDM binary data to fill the DATA column in
92 the MAIN table of the MS. Expected values for
93 this option are 'no' for the uncorrected data
94 (this is the default), 'yes' for the corrected
95 data and 'both' for corrected and uncorrected
96 data. In the latter case, two measurement sets
97 are created, one containing the uncorrected
98 data and the other one, whose name is suffixed
99 by '-wvr-corrected', containing the corrected
100 data.
102 scans -- processes only the scans specified in the option's value. This value is a semicolon
103 separated list of scan specifications. A scan specification consists in an exec bock index
104 followed by the character ':' followed by a comma separated list of scan indexes or scan
105 index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are
106 1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2"
107 are valid values for the option. "3:" alone will be interpreted as 'all the scans of the
108 exec block#3'. An scan index or a scan index range not preceded by an exec block index will
109 be interpreted as 'all the scans with such indexes in all the exec blocks'. By default
110 all the scans are considered.
112 ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
113 and Weather are processed independently of the time range of the selected exec block / scan.
115 process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
116 default: True
118 process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
119 default: True
121 process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
122 If the parameter is set to False the resulting MS will have an empty POINTING table.
123 default: True
125 process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
126 default: True
128 >>> process_flags expandable parameter
129 tbuff -- Time padding buffer (in seconds).
130 default: 0.0
132 applyflags -- Apply the online flags to the MS.
133 default: False
135 savecmds -- Save the online flags to an ASCII file.
136 default: False
138 outfile -- Filename to save the online flags.
139 default: ''
141 flagbackup -- Backup the FLAG column in the .flagversions.
142 default: True
144 verbose -- produce log output as asdm2MS is being run.
146 overwrite -- Over write an existing MS.
148 bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
149 default: false
151 with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
152 to the value to be written in MS::Pointing::direction
153 default: false
155 convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
157 polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
158 Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
159 default: 0.
161 """
163 # Python script
165 casalog.origin('importasdm')
167 # fill in empty vis parameter
168 if vis is None or type(vis) is str and len(vis) == 0:
169 vis = asdm+".ms"
171 # make local sdm tool - CASA6 only, CASA5 uses asdm2MS executable
172 sdmlocal = sdm(asdm)
174 # make agentflagger tool local
175 aflocal = agentflagger()
177 # make table tool local
178 tblocal = table()
180 viso = ''
181 visoc = '' # for the wvr corrected version, if needed
182 if len(vis) > 0:
183 viso = vis
184 tmps = vis.rstrip('.ms')
185 if tmps == vis:
186 visoc = vis + '-wvr-corrected'
187 else:
188 visoc = tmps + '-wvr-corrected.ms'
189 else:
190 viso = asdm.rstrip("/") + '.ms'
191 visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
192 vis = asdm.rstrip("/")
194 # Compression
195 if compression:
196 # viso = viso + '.compressed'
197 viso = viso.rstrip('.ms') + '.compressed.ms'
198 visoc = visoc.rstrip('.ms') + '.compressed.ms'
200 vistoproc = [] # the output MSs to post-process
201 if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
202 vistoproc.append(viso)
203 if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'):
204 vistoproc.append(visoc)
206 for ff in vistoproc:
207 if not overwrite and os.path.exists(ff):
208 raise Exception('You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff)
210 # If viso+".flagversions" then process differently depending on the value of overwrite..
211 #
212 if flagbackup:
213 for myviso in vistoproc:
214 dotFlagversion = myviso + '.flagversions'
215 if os.path.exists(dotFlagversion):
216 if overwrite:
217 casalog.post("Found '" + dotFlagversion
218 + "' . It'll be deleted before running the filler."
219 )
220 os.system('rm -rf %s' % dotFlagversion)
221 else:
222 casalog.post("Found '%s' but can't overwrite it." % dotFlagversion)
223 raise Exception("Found '%s' but can't overwrite it." % dotFlagversion)
225 # Make outfile always a list
226 if isinstance(outfile, str):
227 if outfile == '':
228 outfile = []
229 else:
230 noutfile = [outfile]
231 outfile = noutfile
233 if savecmds:
234 if len(outfile) == 0:
235 # Create default names for the online flags
236 for myviso in vistoproc:
237 outfile.append(myviso.replace('.ms','_cmd.txt'))
238 elif len(outfile) != len(vistoproc):
239 casalog.post('List of outfile names does not match list of MSs','WARN')
240 casalog.post('Will save online flags to temporary filenames', 'WARN')
241 outfile = []
242 for myviso in vistoproc:
243 online_file = myviso.replace('.ms','_TEMP_cmd.txt')
244 outfile.append(online_file)
246 if not overwrite:
247 for of in outfile:
248 if os.path.exists(of):
249 raise RuntimeError("Cannot overwrite online flags file '%s'; overwrite is set to False."% of)
252 if (polyephem_tabtimestep!=None) and (type(polyephem_tabtimestep)==int or type(polyephem_tabtimestep)==float):
253 if polyephem_tabtimestep>0:
254 casalog.post('Will tabulate all attached polynomial ephemerides with a time step of '
255 +str(polyephem_tabtimestep)+' days.')
256 if polyephem_tabtimestep>1.:
257 casalog.post('A tabulation timestep of <= 1 days is recommended.', 'WARN')
258 # one more addition to the asdm2MS execution string
260 try:
261 exitcode = sdmlocal.toms( vis, createmms, separationaxis, numsubms, corr_mode, srt, time_sampling,
262 ocorr_mode, compression, lazy, asis, wvr_corrected_data, scans,
263 ignore_time, process_syspower, process_caldevice, process_pointing,
264 process_flags, tbuff, applyflags, savecmds, outfile, flagbackup,
265 verbose, overwrite, bdfflags,
266 with_pointing_correction, convert_ephem2geo,
267 polyephem_tabtimestep )
269 if exitcode != True:
270 casalog.post("initial creation of the measurement set failed", 'SEVERE')
271 raise Exception('ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.')
272 except Exception as instance:
273 # I think an exception is more likely than a non-zero error code and will often involve a partial fill into the MS
274 casalog.post("initial creation of the measurementset failed", 'SEVERE')
275 casalog.post("Note: if an MS exists then additional steps usually done after that initial (partial) fill were not completed", 'SEVERE')
276 casalog.post("The MS should only be used for looking into why the fill failed, it should not be used for any additional processing", 'SEVERE')
277 raise Exception('ASDM conversion error. Check the logs for additional details.') from instance
280 #
281 # Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
282 # sets if it appears the filler did not find any corrected data.
283 #
284 if not os.path.exists(visoc):
285 vistoproc = [myviso for myviso in vistoproc if myviso != visoc]
287 if convert_ephem2geo:
288 for myviso in vistoproc:
289 ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO
291 for myviso in vistoproc:
292 theephemfields = ce.findattachedephemfields(myviso,field='*')
293 if len(theephemfields)>0:
294 # until asdm2MS or sdm.toms does this internally: recalc the UVW coordinates for ephem fields
295 imt = imager()
296 imt.open(myviso, usescratch=False)
297 imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
298 imt.close()
300 if len(theephemfields)>0:
301 # also set the direction column in the SOURCE table
302 tblocal.open(myviso+'/FIELD', nomodify=False)
303 sourceids = tblocal.getcol('SOURCE_ID')
304 ftimes = tblocal.getcol('TIME')
305 ftimekw = tblocal.getcolkeywords('TIME')
306 tmpa = tblocal.getcol('PHASE_DIR')
307 origphasedir = tmpa
309 affectedsids = []
310 thesamplefields = []
311 for fld in theephemfields: # determine all source ids used by the ephem fields
312 if not (sourceids[fld] in affectedsids): # this source id wasn't handled yet
313 affectedsids.append(sourceids[fld])
314 thesamplefields.append(fld)
315 # need to temporarily change the offset (not all mosaics have an element at (0,0))
316 tmpa[0][0][fld]=0.
317 tmpa[1][0][fld]=0.
318 #endif
319 #endfor
320 tblocal.putcol('PHASE_DIR', tmpa)
321 tblocal.close()
323 tblocal.open(myviso+'/SOURCE')
324 sourceposref = tblocal.getcolkeywords('DIRECTION')['MEASINFO']['Ref']
325 tblocal.close()
327 directions = []
328 melocal = measures()
329 msmdlocal = msmetadata()
330 msmdlocal.open(myviso)
332 for fld in thesamplefields:
333 thedirmeas = msmdlocal.phasecenter(fld)
334 if thedirmeas['refer']!=sourceposref:
335 casalog.post('Ephemeris is in '+thedirmeas['refer']+' instead of '+sourceposref
336 +' frame.\nEntry in SOURCE table will be converted to '+sourceposref, 'WARN')
337 melocal.doframe(thedirmeas)
338 thedirmeas = melocal.measure(thedirmeas, sourceposref)
340 directions.append([thedirmeas['m0']['value'], thedirmeas['m1']['value']])
341 thetime = melocal.epoch(v0=str(ftimes[fld])+'s', rf=ftimekw['MEASINFO']['Ref'])
342 casalog.post("Will set SOURCE direction for SOURCE_ID "+str(sourceids[fld])
343 +" to ephemeris phase center for time "+str(thetime['m0']['value'])+" "+thetime['m0']['unit']+" "+thetime['refer'])
344 #endfor
345 msmdlocal.close()
347 # restore original PHASE_DIR
348 tblocal.open(myviso+'/FIELD', nomodify=False)
349 tblocal.putcol('PHASE_DIR', origphasedir)
350 tblocal.close()
352 # write source directions
353 tblocal.open(myviso+'/SOURCE', nomodify=False)
354 ssourceids = tblocal.getcol('SOURCE_ID')
355 sdirs = tblocal.getcol('DIRECTION')
356 for row in range(0,len(ssourceids)):
357 for i in range(0,len(affectedsids)):
358 if ssourceids[row]==affectedsids[i]:
359 sdirs[0][row] = directions[i][0]
360 sdirs[1][row] = directions[i][1]
361 break
362 #endfor
363 #endfor
364 tblocal.putcol('DIRECTION', sdirs) # write back corrected directions
365 tblocal.close()
367 #end if
368 #end for
370 ##############################################################################################3
371 # CAS-7369 - Create an output Multi-MS (MMS)
372 if createmms:
373 fpars = { }
374 # Get the default parameters of partition
375 import inspect
376 from casatasks import partition as pt
377 for k,v in inspect.signature(pt).parameters.items( ):
378 fpars[k] = v.default
380 # Call the cluster for each MS
381 for myviso in vistoproc:
382 casalog.origin('importasdm')
384 # Move original MS to tempdir
385 tempname = myviso+'.temp.ms'
386 outputmms = myviso
387 shutil.move(myviso, tempname)
389 # Get the proper column
390 datacolumn = 'DATA'
391 dcols = ['DATA', 'FLOAT_DATA']
392 _tb.open(tempname)
393 for dc in dcols:
394 if dc in _tb.colnames( ):
395 datacolumn = dc
396 break
397 _tb.close( )
399 fpars['datacolumn'] = datacolumn
401 casalog.post('Will create a Multi-MS for: '+myviso)
403 fpars['vis'] = tempname
404 fpars['flagbackup'] = False
405 fpars['outputvis'] = outputmms
406 fpars['separationaxis'] = separationaxis
407 fpars['numsubms'] = numsubms
409 # Run partition only at the MPIServers
410 pdh = ParallelDataHelper('partition', fpars)
412 # Get a cluster
413 pdh.setupCluster(thistask='partition')
414 try:
415 pdh.go()
417 # Remove original MS
418 shutil.rmtree(tempname)
420 except Exception as instance:
421 # Restore MS in case of error in MMS creation
422 shutil.move(tempname, myviso)
423 raise
425 casalog.origin('importasdm')
427 # Create a .flagversions for the MS or MMS
428 if flagbackup:
429 for myviso in vistoproc:
430 if os.path.exists(myviso):
431 aflocal.open(myviso)
432 aflocal.saveflagversion('Original',
433 comment='Original flags at import into CASA',
434 merge='save')
435 aflocal.done()
437 # Importasdm Flag Parsing
438 if os.access(asdm + '/Flag.xml', os.F_OK):
439 # Find Flag.xml
440 casalog.post('Found Flag.xml in SDM')
442 # Find Antenna.xml
443 if os.access(asdm + '/Antenna.xml', os.F_OK):
444 casalog.post('Found Antenna.xml in SDM')
446 else:
447 raise Exception('Failed to find Antenna.xml in SDM')
449 # Find SpectralWindow.xml
450 if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
451 casalog.post('Found SpectralWindow.xml in SDM')
453 else:
454 raise Exception('Failed to find SpectralWindow.xml in SDM')
456 #
457 # Parse Flag.xml into flag dictionary
458 #
459 if process_flags:
460 flagcmds = fh.parseXML(asdm, float(tbuff))
461 onlinekeys = flagcmds.keys()
462 nflags = onlinekeys.__len__()
464 # Apply flags to the MS
465 if nflags > 0:
466 idx = 0
467 for myviso in vistoproc:
468 if applyflags:
469 # Open the MS and attach it to the tool
470 aflocal.open(myviso)
471 # Select the data
472 aflocal.selectdata()
473 # Setup the agent's parameters
474 fh.parseAgents(aflocal, flagcmds, [], True, True, '')
475 # Initialize the agents
476 aflocal.init()
477 # Run the tool
478 aflocal.run(True, True)
479 casalog.post('Applied %s flag commands to %s'%(nflags,myviso))
480 # Destroy the tool and de-attach the MS
481 aflocal.done()
482 # Save to FLAG_CMD table. APPLIED is set to True.
483 fh.writeFlagCommands(myviso, flagcmds, True, '', '', True)
484 else:
485 casalog.post('Will not apply flags to %s (apply_flags=False), use flagcmd to apply'%myviso)
487 # Write to FLAG_CMD, APPLIED is set to False
488 fh.writeFlagCommands(myviso, flagcmds, False, '', '', True)
490 # Save the flag cmds to an ASCII file
491 if savecmds:
492 # Save to standard filename
493 fh.writeFlagCommands(myviso, flagcmds, False, '', outfile[idx], False)
494 casalog.post('Saved %s flag commands to %s'%(nflags,outfile[idx]))
495 idx += 1
497 else:
498 casalog.post('There are no flag commands to process')
500 else:
501 casalog.post('There is no Flag.xml in ASDM', 'WARN')
504 # CAS-7369. HISTORY should be written after createmms is tested
505 #
506 # Populate the HISTORY table of the MS with information about the context in which it's been created
507 #
508 try:
509 mslocal = ms()
510 param_names = importasdm.__code__.co_varnames[:importasdm.__code__.co_argcount]
511 vars = locals( )
512 param_vals = [vars[p] for p in param_names]
514 for myviso in vistoproc:
515 write_history(mslocal, myviso, 'importasdm', param_names, param_vals, casalog)
517 except Exception as instance:
518 casalog.post("*** Error \'%s\' updating HISTORY" % (instance),'WARN')
519 finally:
520 if mslocal:
521 mslocal = None