#########################################################################
# test_task_importasdm.py
# Copyright (C) 2018
# Associated Universities, Inc. Washington DC, USA.
#
# This script is free software; you can redistribute it and/or modify it
# under the terms of the GNU Library General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
# License for more details.
#
# Features tested:
#    1) Is the import performed without raising exceptions
#    2) Do all expected tables exist
#    3) Can the MS be opened
#    4) Do the tables contain expected values
#    5) Is exportasdm performed without raising exceptions
#    6) Is the created ASDM well-formed (XML) and complete
#    7) Can the resulting ASDM be reimported without raising exceptions
#    8) Does it have the same number of integrations as the original
#    9) Can a mixed mode ASDM be imported in lazy mode
#   10) Does the lazy mode when used with auto-only and the FLOAT_DATA
#       column produce an equivalent MS as the non-lazy mode does with the
#       same data selection
#
# Based on the requirements listed in casadocs found here:
# https://casadocs.readthedocs.io/en/stable/api/tt/casatasks.data.importasdm.html
#
##########################################################################
import os
import sys
import shutil
import numpy
import unittest

from casatools import ctsys, ms, table, measures, calibrater, agentflagger
from casatasks import importasdm, flagdata, flagcmd, exportasdm, casalog
from casatasks.private import flaghelper as fh
from casatasks.private import convertephem as ce
from casatestutils import testhelper as th

tblocal = table( )
mslocal = ms( )

myname = 'test_importasdm'

rootpath = ctsys.resolve('unittest/importasdm/')
# default ASDM dataset name
myasdm_dataset_name = 'uid___X5f_X18951_X1'
myms_dataset_name = 'M51.ms'

# name of the resulting MS
msname = myasdm_dataset_name+'.ms'

# name of the exported ASDM
asdmname = myms_dataset_name+'.asdm'

# name of the reimported MS
reimp_msname = 'reimported-'+myms_dataset_name

def checktable(themsname, thename, theexpectation):
    global myname
    tblocal.open(themsname+"/"+thename)
    if thename == "":
        thename = "MAIN"
    for mycell in theexpectation:
        print("%s: comparing %s" % (myname, mycell))
        value = tblocal.getcell(mycell[0], mycell[1])
        in_agreement = False

        # bool and str cases
        if type(value)is bool or type(value) is str:
            # must be equal
            in_agreement = (value == mycell[2])
        else:
            # see if value is array
            try:
                isarray = value.__len__
            except:
                # it's not an array
                # zero tolerance?
                if mycell[3] == 0:
                    in_agreement = (value == mycell[2])
                else:
                    in_agreement = ( abs(value - mycell[2]) < mycell[3]) 
            else:
                # it's an array
                # zero tolerance?
                if mycell[3] == 0:
                    in_agreement =  (value == mycell[2]).all() 
                else:
                    in_agreement = (abs(value - mycell[2]) < mycell[3]).all() 
        if not in_agreement:
            print("%s:  Error in MS subtable %s:" % (myname, thename))
            print("     column %s row %s contains %s" % (mycell[0], mycell[1], value))
            print("     expected value is %s" % mycell[2])
            tblocal.close()
            return False
    tblocal.close()
    print("%s: table %s as expected." % (myname, thename))
    return True

def countlines(txtfile):
    count = 0
    with open(txtfile,'r') as f:
        for count,l in enumerate(f,1):
            pass
    return count
        
#########################

def verify_asdm(asdmname, withPointing):
    print("Verifying asdm %s" % asdmname)
    if(not os.path.exists(asdmname)):
        print("asdm %s doesn't exist." % asdmname)
        raise Exception
    # test for the existence of all obligatory tables
    allTables = [ "Antenna.xml",
                  "ASDM.xml",
                 # "CalData.xml",
                 # "CalDelay.xml",
                 # "CalReduction.xml",
                  "ConfigDescription.xml",
                  "CorrelatorMode.xml",
                  "DataDescription.xml",
                  "ExecBlock.xml",
                  "Feed.xml",
                  "Field.xml",
                 #"FocusModel.xml",
                 #"Focus.xml",
                  "Main.xml",
                  "PointingModel.xml",
                  "Polarization.xml",
                  "Processor.xml",
                  "Receiver.xml",
                  "SBSummary.xml",
                  "Scan.xml",
                  "Source.xml",
                  "SpectralWindow.xml",
                  "State.xml",
                  "Station.xml",
                  "Subscan.xml",
                  "SwitchCycle.xml"
                  ]
    isOK = True
    for fileName in allTables:
        filePath = asdmname+'/'+fileName
        if(not os.path.exists(filePath)):
            print("ASDM table file %s doesn't exist." % filePath)
            isOK = False
        else:
            # test if well formed
            rval = os.system('xmllint --noout '+filePath)
            if(rval !=0):
                print("Table %s is not a well formed XML document." % filePath)
                isOK = False

    print("Note: xml validation not possible since ASDM DTDs (schemas) not yet online.")
        
    if(not os.path.exists(asdmname+"/ASDMBinary")):
        print("ASDM binary directory %s/ASDMBinary doesn't exist." % asdmname)
        isOK = False

    if(withPointing and not os.path.exists(asdmname+"/Pointing.bin")):
        print("ASDM binary file %s/Pointing.bin doesn't exist." % asdmname)
        isOK = False

    if (not isOK):
        raise Exception

# Setup for different data importing
class test_base(unittest.TestCase):
    def setUp_m51(self):
        res = None
        if(os.path.exists(myasdm_dataset_name)):
            shutil.rmtree(myasdm_dataset_name)

        datapath=os.path.join(rootpath,myasdm_dataset_name)
        shutil.copytree(datapath, myasdm_dataset_name)
        datapath=os.path.join(rootpath,myms_dataset_name)
        shutil.copytree(datapath, myms_dataset_name)

    def setUp_xosro(self):
        self.asdm = 'X_osro_013.55979.93803716435'  #EVLA SDM
        datapath=os.path.join(rootpath,self.asdm)
        if(not os.path.lexists(self.asdm)):
            os.system('ln -s '+datapath+' '+self.asdm)

    def setUp_polyuranus(self):
        self.asdm = 'polyuranus'  # EVLA SDM with ephemeris
        datapath = os.path.join(rootpath, self.asdm)
        if (not os.path.lexists(self.asdm)):
            os.system('ln -s '+datapath+' '+self.asdm)

    def setUp_autocorr(self):
        self.asdm = 'AutocorrASDM'  # ALMA 
        datapath=os.path.join(rootpath,self.asdm)
        if(not os.path.lexists(self.asdm)):
            os.system('ln -s '+datapath+' '+self.asdm)

    def setUp_acaex(self):
        res = None
        myasdmname = 'uid___A002_X72bc38_X000' # ACA example ASDM with mixed pol/channelisation
        datapath=os.path.join(rootpath, myasdmname)
        os.system('ln -sf '+datapath)

    def setUp_12mex(self):
        res = None
        myasdmname = 'uid___A002_X71e4ae_X317_short' # 12m example ASDM with mixed pol/channelisation
        datapath=os.path.join(rootpath,myasdmname)
        os.system('ln -sf '+datapath)


    def setUp_12mex_bin(self):
        res = None
        myasdmname = 'uid___A002_X71e4ae_X317_short_bin' # 12m example ASDM with mixed pol/channelisation and binary Cal tables
        datapath=os.path.join(rootpath,myasdmname)
        os.system('ln -sf '+datapath)

    def setUp_eph(self):
        res = None
        myasdmname = 'uid___A002_X997a62_X8c-short' # 12m example ASDM with ephemerides
        datapath=os.path.join(rootpath, myasdmname)
        os.system('ln -sf '+datapath)

    def setUp_flags(self):
        res = None
        myasdmname = 'test_uid___A002_X997a62_X8c-short' # Flag.xml is modified
        datapath=os.path.join(rootpath, myasdmname)
        os.system('ln -sf '+datapath)

    def setUp_SD(self):
        res = None
        myasdmname = 'uid___A002_X6218fb_X264' # Single-dish ASDM
        datapath=os.path.join(rootpath, myasdmname)
        os.system('ln -sf '+datapath)

    def setUp_numbin(self):
        res = None
        # need full copies as this test involves renaming some xml files
        datapath = rootpath
        for this_asdm_name in ['alma_numbin_mixed','evla_numbin_2','evla_numbin_4']:
            if (os.path.exists(this_asdm_name)):
                shutil.rmtree(this_asdm_name)
            shutil.copytree(os.path.join(datapath,this_asdm_name), this_asdm_name)

###########################
# beginning of actual test 
class asdm_import1(test_base):
    
    def setUp(self):
        self.setUp_m51()
        
    def tearDown(self):
        os.system('rm -rf '+myasdm_dataset_name)
        shutil.rmtree(myms_dataset_name)
        shutil.rmtree(msname,ignore_errors=True)
        shutil.rmtree(msname+'.flagversions',ignore_errors=True)
        for thisdir in ['reimported-M51.ms','reimported-M51.ms.flagversions','M51.ms.asdm','myinput.ms']:
            shutil.rmtree(thisdir,ignore_errors=True)
                
    def test1(self):
        '''Asdm-import: Test good v1.2 input with filler and inverse filler'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        self.res = importasdm(myasdm_dataset_name)
        self.assertEqual(self.res, None)
        print( "%s: Success! Now checking output ..." % myname)
        mscomponents = set(["table.dat",
#                            "table.f0",
                            "table.f1",
                            "table.f2",
                            "table.f3",
                            "table.f4",
                            "table.f5",
                            "table.f6",
                            "table.f7",
                            "table.f8",
                            "ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(msname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist..." % (myname,msname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error Cannot open MS table %s" % (myname, msname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            # check main table first
            name = ""
            #             col name, row number, expected value, tolerance
            expected = [
                         ['UVW',       42, [ 0., 0., 0. ], 2E-7],
                         ['EXPOSURE',  42, 1.008, 0],
                         ['DATA',      42, [ [10.5526886+0.0j] ], 1E-7]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed'
    
            expected = [
    # old values using TAI     ['UVW',       638, [-65.07623467,   1.05534109, -33.65801386], 1E-8],
                         ['UVW',       638, [-65.14758508, 1.13423277, -33.51712451], 2E-7],
                         ['EXPOSURE',  638, 1.008, 0],
                         ['DATA',      638, [ [0.00362284+0.00340279j] ], 1E-8]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed'
            
            name = "ANTENNA"
            expected = [ ['OFFSET',       1, [ 0.,  0.,  0.], 0],
                         ['POSITION',     1, [2202242.5520, -5445215.1570, -2485305.0920], 0.0001],
                         ['DISH_DIAMETER',1, 12.0, 0]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table ANTENNA failed'
            
            name = "POINTING"
            expected = [ ['DIRECTION',       10, [[ 1.94681283],[ 1.19702955]], 1E-8],
                         ['INTERVAL',        10, 0.048, 0],
                         ['TARGET',          10, [[ 1.94681283], [ 1.19702955]], 1E-8],
                         ['TIME',            10, 4758823736.016000, 1E-6],
                         ['TIME_ORIGIN',     10, 0., 0],
                         ['POINTING_OFFSET', 10, [[ 0.],[ 0.]], 0],
                         ['ENCODER',         10, [ 1.94851533,  1.19867576], 1E-8 ]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table POINTING failed'

            name = "PROCESSOR"
            expected = [ ['FLAG_ROW',            0, False, 0],
                         ['MODE_ID',             0, 0, 0],
                         ['TYPE',                0, 'CORRELATOR', 0],
                         ['TYPE_ID',             0, -1, 0],
                         ['SUB_TYPE',            0, 'ALMA_BASELINE', 0]
                       ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success'] = False
                retValue['error_msgs'] = retValue['error_msgs']+'Check of table PROCESSOR failed'
                
        self.assertTrue(retValue['success'],retValue['error_msgs'])
                
        myvis = myms_dataset_name
        os.system('rm -rf exportasdm-output.asdm myinput.ms')
        os.system('cp -R ' + myvis + ' myinput.ms')
        try:
            print("\n>>>> Test of exportasdm: input MS is %s" % myvis)
            print("(a simulated input MS with pointing table)")
            try:
                exportasdm(
                    vis = 'myinput.ms',
                    asdm = 'exportasdm-output.asdm',
                    archiveid="S002",
                    apcorrected=False
                )
            except Exception as exc:
                self.fail('Unexpected exception: {}'.format(exc))

            os.system('rm -rf '+asdmname+'; mv exportasdm-output.asdm '+asdmname)
            verify_asdm(asdmname, True)
        except:
            print('%s: *** Unexpected error exporting MS to ASDM, regression failed ***' % myname)
            raise
            
        try:
            print("Reimporting the created ASDM ....")
            importasdm(asdm=asdmname, vis=reimp_msname, wvr_corrected_data='no')
            print("Testing existence of reimported MS ....")
            if(not os.path.exists(reimp_msname)):
                print("MS %s doesn't exist." % reimp_msname)
                raise Exception
            print("Testing equivalence of the original and the reimported MS.")
            tblocal.open(myms_dataset_name)
            nrowsorig = tblocal.nrows()
            print("Original MS contains %s integrations." % nrowsorig)
            tblocal.close()
            tblocal.open(reimp_msname)
            nrowsreimp = tblocal.nrows()
            tblocal.close()
            print("Reimported MS contains %s integrations." % nrowsreimp)
            if(not nrowsreimp==nrowsorig):
                print("Numbers of integrations disagree.")
                raise Exception
        except:
            print('%s: *** Unexpected error reimporting the exported ASDM, regression failed ***' % myname)
            raise

class asdm_import2(test_base):
    
    def setUp(self):
        self.setUp_m51()
        
    def tearDown(self):
        shutil.rmtree(myasdm_dataset_name)
        shutil.rmtree(myms_dataset_name)
        shutil.rmtree(msname,ignore_errors=True)
        shutil.rmtree(msname+'.flagversions',ignore_errors=True)
        for thisdir in ['reimported-M51.ms','reimported-M51.ms.flagversions','M51.ms.asdm','myinput.ms']:
            shutil.rmtree(thisdir,ignore_errors=True)
                
    def test_import2(self):
        '''Asdm-import: Test good v1.2 input with filler and inverse filler '''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        self.res = importasdm(myasdm_dataset_name)
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["table.dat",
#                            "table.f0",
                            "table.f1",
                            "table.f2",
                            "table.f3",
                            "table.f4",
                            "table.f5",
                            "table.f6",
                            "table.f7",
                            "table.f8",
                            "ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(msname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,msname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error Cannot open MS table %s" % (myname,msname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            # check main table first
            name = ""
            #             col name, row number, expected value, tolerance
            expected = [
                         ['UVW',       42, [ 0., 0., 0. ], 2E-7],
                         ['EXPOSURE',  42, 1.008, 0],
                         ['DATA',      42, [ [10.5526886+0.0j] ], 1E-7]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed'
    
            expected = [
    # old values using TAI     ['UVW',       638, [-65.07623467,   1.05534109, -33.65801386], 1E-8],
                         ['UVW',       638, [-65.14758508, 1.13423277, -33.51712451], 2E-7],
                         ['EXPOSURE',  638, 1.008, 0],
                         ['DATA',      638, [ [0.00362284+0.00340279j] ], 1E-8]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed'
            
            name = "ANTENNA"
            expected = [ ['OFFSET',       1, [ 0.,  0.,  0.], 0],
                         ['POSITION',     1, [2202242.5520, -5445215.1570, -2485305.0920], 0.0001],
                         ['DISH_DIAMETER',1, 12.0, 0]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table ANTENNA failed'
            
            name = "POINTING"
            expected = [ ['DIRECTION',       10, [[ 1.94681283],[ 1.19702955]], 1E-8],
                         ['INTERVAL',        10, 0.048, 0],
                         ['TARGET',          10, [[ 1.94681283], [ 1.19702955]], 1E-8],
                         ['TIME',            10, 4758823736.016000, 1E-6],
                         ['TIME_ORIGIN',     10, 0., 0],
                         ['POINTING_OFFSET', 10, [[ 0.],[ 0.]], 0],
                         ['ENCODER',         10, [ 1.94851533,  1.19867576], 1E-8 ]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table POINTING failed'
                
            name = "PROCESSOR"
            expected = [ ['FLAG_ROW',            0, False, 0],
                         ['MODE_ID',             0, 0, 0],
                         ['TYPE',                0, 'CORRELATOR', 0],
                         ['TYPE_ID',             0, -1, 0],
                         ['SUB_TYPE',            0, 'ALMA_BASELINE', 0]
                       ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success'] = False
                retValue['error_msgs'] = retValue['error_msgs']+'Check of table PROCESSOR failed'
                
        self.assertTrue(retValue['success'],retValue['error_msgs'])
                
        myvis = myms_dataset_name
        os.system('rm -rf exportasdm-output.asdm myinput.ms')
        os.system('cp -R ' + myvis + ' myinput.ms')
        try:
            print("\n>>>> Test of exportasdm: input MS  is %s" % myvis)
            print("(a simulated input MS with pointing table)")
            try:
                exportasdm(
                    vis = 'myinput.ms',
                    asdm = 'exportasdm-output.asdm',
                    archiveid="S002",
                    apcorrected=False
                )
            except Exception as exc:
                self.fail('Unexpected exception: {}'.format(exc))

            os.system('rm -rf '+asdmname+'; mv exportasdm-output.asdm '+asdmname)
            verify_asdm(asdmname, True)
        except:
            print('%s: *** Unexpected error exporting MS to ASDM, regression failed ***' % myname)
            raise
            
        try:
            print("Reimporting the created ASDM ....")
            importasdm(asdm=asdmname, vis=reimp_msname, wvr_corrected_data='no')
            print("Testing existence of reimported MS ....")
            if(not os.path.exists(reimp_msname)):
                print("MS %s doesn't exist." % reimp_msname)
                raise Exception
            print("Testing equivalence of the original and the reimported MS.")
            tblocal.open(myms_dataset_name)
            nrowsorig = tblocal.nrows()
            print("Original MS contains %s integrations." % nrowsorig)
            tblocal.close()
            tblocal.open(reimp_msname)
            nrowsreimp = tblocal.nrows()
            tblocal.close()
            print("Reimported MS contains %s integrations." % nrowsreimp)
            if(not nrowsreimp==nrowsorig):
                print("Numbers of integrations disagree.")
                raise Exception
        except:
            print('%s: *** Unexpected error reimporting the exported ASDM, regression failed ***' % myname)
            raise
        
class asdm_import3(test_base):
    # most of these (except test_CAS4532) were formerly found in test_importevla
    
    def setUp(self):
        # multiple SDMs used here, do not trust self.asdm to be correct, this just make a link to the data
        self.setUp_xosro()
        self.setUp_polyuranus()
        
    def tearDown(self):
        for myasdmname in ['X_osro_013.55979.93803716435', 'polyuranus']:
            os.system('rm -rf '+myasdmname)  # a link
            shutil.rmtree(myasdmname+".ms",ignore_errors=True)
            shutil.rmtree(myasdmname+".ms.flagversions",ignore_errors=True)
        os.system('rm -rf X_osro_013.55979.93803716435_cmd.txt')

    # functions to duplicate what importevla did after filling the MS - shadow and zero-level flagging
    def getmsmjds(self,msname):
        # Get start and end times from MS, return in mjds
        # this might take too long for large MS
        # NOTE: could also use values from OBSERVATION table col TIME_RANGE
        mslocal2 = ms()
        success = True
        ms_startmjds = None
        ms_endmjds = None
        try:
            mslocal2.open(msname)
            timd = mslocal2.range(['time'])
            mslocal2.close()
        except:
            success = False
            print('Error opening MS ' + vis)
        if success: 
            ms_startmjds = timd['time'][0]
            ms_endmjds = timd['time'][1]
        else:
            print('WARNING: Could not open vis as MS to find times')
        return (ms_startmjds, ms_endmjds)

    def flagzero(self,startMJDsec, endMJDsec, flagpol):
        # returns a dictionary of flag commands related to zero data values
        # over the time interval given in the arguments
        # startMJDsec : start of interval, MJD, seconds
        # endMJDsec : end of interval, MJD, seconds
        # if flagpol = True, then flag all polarizations, else just RR and LL
        # there will be either a single command (flagpol=True) or
        # two commands (flagpol=False)

        result = {}

        flagz = {}
        flagz['time'] = 0.5*(startMJDsec+endMJDsec)
        flagz['interval'] = endMJDsec - startMJDsec 
        flagz['level'] = 0
        flagz['severity'] = 0
        flagz['type'] = 'FLAG'
        flagz['applied'] = False
        flagz['antenna'] = ''
        flagz['mode'] = 'clip'
        
        if flagpol:
            flagz['reason'] = 'CLIP_ZERO_ALL'
            command = {}
            command['mode'] = 'clip'
            command['clipzeros'] = True
            command['correlation'] = 'ABS_ALL'
            flagz['command'] = command
            flagz['id'] = 'ZERO_ALL'
            result[len(result)] = flagz.copy()
        else:
            flagz['reason'] = 'CLIP_ZERO_RR'
            command = {}
            command['mode'] = 'clip'
            command['clipzeros'] = True
            command['correlation'] = 'ABS_RR'
            flagz['command'] = command
            flagz['id'] = 'ZERO_RR'
            result[len(result)] = flagz.copy()
            
            flagz['reason'] = 'CLIP_ZERO_LL'
            command = {}
            command['mode'] = 'clip'
            command['clipzeros'] = True
            command['correlation'] = 'ABS_LL'
            flagz['command'] = command
            flagz['id'] = 'ZERO_LL'
            result[len(result)] = flagz.copy()

        return result

    def flagshadow(self,startMJDsec, endMJDsec, tolerance, addantenna):
        # create a set of flag commands related to shadowed data
        # the tolerance is given by tolerance
        # addantenna : may be a filename or a string
    
        result = {}
        command = {}
        # flag shadowed data
        result['time'] = 0.5 * (startMJDsec + endMJDsec)
        result['interval'] = endMJDsec - startMJDsec
        result['level'] = 0
        result['severity'] = 0
        result['type'] = 'FLAG'
        result['applied'] = False
        result['antenna'] = ''
        result['mode'] = 'shadow'
        result['reason'] = 'SHADOW'
    
        command['mode'] = 'shadow'
        command['tolerance'] = tolerance

        if type(addantenna) == str:
            if addantenna != '':
                # it's a filename, create a dictionary
                antdict = fh.readAntennaList(addantenna)
                command['addantenna'] = antdict
                    
        elif type(addantenna) == dict:
            if addantenna != {}:
                command['addantenna'] = addantenna
            
        result['command'] = command
        result['id'] = 'SHADOW'

        return result

    def applyflags(self,allflags, mspath):
        aflocal = agentflagger()
        if (len(allflags)) > 0:
            aflocal.open(mspath)
            aflocal.selectdata()
            fh.parseAgents(aflocal,allflags,[],True,True,'')
            aflocal.init()
            stats = aflocal.run(True,True)
            aflocal.done()
            # writes these to the FLAG_CMD table, append is True
            fh.writeFlagCommands(mspath,allflags,True,'','',True)
       
    def test_CAS4532(self):
        '''importasdm CAS-4532: white spaces on Antenna.xml'''
        # The X_osro_scan1/Antenna.xml and SpectralWindow.xml 
        # contain white spaces between some of the contents and 
        # the tags. This should not cause any error in the XML 
        # parser from fh.readXML
        self.asdm = 'X_osro_013.55979.93803716435'
        
        flagcmddict = fh.readXML(self.asdm, 0.0)
        self.assertTrue(flagcmddict, 'Some XML file may contain white spaces not handled by readXML')
        
        self.assertEqual(flagcmddict.keys().__len__(),214)

    def test_evlatest1(self):
        '''test of importing evla data, test1: Good input asdm'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }  

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm+".ms"

        # note that ocorr_mode defaulted to "co" for importevla and with_pointing_correction defaulted to True, so both must be set here
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='2',ocorr_mode='co',with_pointing_correction=True)
        print("%s: Success! Now checking output ..." % myname)
        # this is probably not the best way to test sucess, too dependent on table system details, which might change without invalidating the result
        # but this is how it was written. These are the components it currently produces.
        # not a complete list of all of the table.* files in each directory
        mscomponents = set(["table.dat",
                            "table.f1",
                            "table.f2",
                            "table.f3",
                            "table.f4",
                            "table.f5",
                            "table.f6",
                            "table.f7",
                            "table.f8",
                            # importevla copied these over, importasdm does not
                            #"Antenna.xml",
                            #"Flag.xml",
                            #"SpectralWindow.xml",
                            "ANTENNA/table.dat",
                            "ANTENNA/table.f0",
                            "CALDEVICE/table.dat",
                            "CALDEVICE/table.f0",
                            "DATA_DESCRIPTION/table.dat",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.dat",
                            "FEED/table.f0",
                            "FIELD/table.dat",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.dat",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.dat",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.dat",
                            "OBSERVATION/table.f0",
                            "POINTING/table.dat",
                            "POINTING/table.f0",
                            "POLARIZATION/table.dat",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.dat",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.dat",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.dat",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.dat",
                            "STATE/table.f0",
                            "SYSCAL/table.f0",
                            "SYSCAL/table.dat",
                            "SYSPOWER/table.dat",
                            "SYSPOWER/table.f0",
                            "WEATHER/table.dat",
                            "WEATHER/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(msname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,msname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,msname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            # check main table first
            name = ""
            #             col name, row number, expected value, tolerance
            expected = [
                         ['UVW',       42, [  1607.46416559, -1241.44101203 , 584.54265781], 2E-7],
                         ['EXPOSURE',  42, 1.0, 0]
                       ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed'
    
            expected = [
                         ['UVW',       638, [14.21738626, 722.5956813 , 57.58092887], 2E-7],
                         ['EXPOSURE',  638, 1.0, 0]
                       ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed'
            
            name = "ANTENNA"
            expected = [ ['OFFSET',       1, [ -4.80000000e-12,  0.,  0.], 0],
                         ['POSITION',     1, [-1599644.8611, -5042953.6623, 3554197.0332], 0.0001],
                         ['DISH_DIAMETER',1, 25.0, 0]
                         ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Check of table ANTENNA failed'
            
            name = "PROCESSOR"
            expected = [ ['FLAG_ROW',            0, False, 0],
                         ['MODE_ID',             0, 0, 0],
                         ['TYPE',                0, 'CORRELATOR', 0],
                         ['TYPE_ID',             0, -1, 0],
                         ['SUB_TYPE',            0, 'NRAO_WIDAR', 0]
                       ]
            results = checktable(msname, name, expected)
            if not results:
                retValue['success'] = False
                retValue['error_msgs'] = retValue['error_msgs']+'Check of table PROCESSOR failed'
                
        self.assertTrue(retValue['success'])

    def test_evlatest2(self):
        '''test of importing evla data, test2: Good input asdm with polynomial ephemeris'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        self.asdm = 'polyuranus'
        msname = self.asdm+".ms"

        # note that ocorr_mode defaulted to "co" for importevla and with_pointing_correction defaulted to True, so both must be set here
        # polyephem_tabtimestep defaults to 0.001 when not set, check that this interval is what is produced
        # importevla did not have the convert_ephem2geo step so it must be disabled here to duplicate that behavior
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='0:5',ocorr_mode='co',with_pointing_correction=True,convert_ephem2geo=False)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["table.dat",
                            "table.f1",
                            "table.f2",
                            "table.f3",
                            "table.f4",
                            "table.f5",
                            "table.f6",
                            "table.f7",
                            "table.f8",
                            # importevla copied these over, importasdm does not
                            #"Antenna.xml",
                            #"Flag.xml",
                            #"SpectralWindow.xml",
                            "ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "WEATHER/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FIELD/EPHEM0_URANUS_57545.8.tab/table.f0",  # the ephemeris table!
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0",
                            "WEATHER/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(msname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,msname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            mslocal.open(msname)
            mssum = mslocal.summary()
            mslocal.close()

            if(mssum['scan_5']['0']['FieldName']=='URANUS' and mssum['field_2']['direction']['m0']['value']==0.37832756704958537):
                print("%s: MS summary as expected." % myname)
                retValue['success']=True
            else:
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Unexepcted mssummary.'

            # the ephemeris table
            
            tblocal.open(msname+"/FIELD/EPHEM0_URANUS_57545.8.tab")
            # check that the MJD interval is the expected (default) value of 0.001d
            mjd = tblocal.getcol('MJD')
            tblocal.close()
            mjdInterval = (mjd[1:] - mjd[0:-1]).mean()
            self.assertAlmostEqual(mjdInterval, 0.001, places=5)
                
        self.assertTrue(retValue['success'])

    def test_evla_ephemeris(self):
        '''test of importing evla data with polynomial ephemeris and different polyephem_tabtimestep values'''

        self.asdm = 'polyuranus'
        msname = self.asdm+".ms"

        # the default polyephem_tabtimestep is checked in test_evlatest2

        # a negative value should be the same as the default
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='0:5',ocorr_mode='co',with_pointing_correction=True,convert_ephem2geo=False,polyephem_tabtimestep=-1)
        print("%s: Success! Now checking ephemeris table ..." % myname)
        tblocal.open(msname+"/FIELD/EPHEM0_URANUS_57545.8.tab")
        # check that the MJD interval is the expected (default) value of 0.001d
        mjd = tblocal.getcol('MJD')
        tblocal.close()
        mjdInterval = (mjd[1:] - mjd[0:-1]).mean()
        self.assertAlmostEqual(mjdInterval, 0.001, places=7)
        print("%s: Success! Now checking a different polyephem_tabtimestep value")

        # msname must first be removed before it can be filled again here
        shutil.rmtree(msname)
        # and flagversions tree
        shutil.rmtree(msname+".flagversions")

        # an arbitrary non-default value shouild set the interval
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='0:5',ocorr_mode='co',with_pointing_correction=True,convert_ephem2geo=False,polyephem_tabtimestep=0.00725)
        print("%s: Success! Now checking ephemeris table ..." % myname)
        tblocal.open(msname+"/FIELD/EPHEM0_URANUS_57545.8.tab")
        # check that the MJD interval is the value that was used
        mjd = tblocal.getcol('MJD')
        tblocal.close()
        mjdInterval = (mjd[1:] - mjd[0:-1]).mean()
        self.assertAlmostEqual(mjdInterval, 0.00725, places=7)
        print("%s: Success!")


    def test_evla_apply1(self):
        '''test of importing evla data, apply all flags, and save to file'''
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)
            
        # this applies and saves the online commands, which is all importasdm can do
        # note that there is no ephemeris data here so those arguments are not used in this call
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='2', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=True, applyflags=True, savecmds=True, flagbackup=False)

        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK. doing flagzero and shadow flagging" % myname)

                (startMJDs,endMJDs) = self.getmsmjds(msname)
                flagz = self.flagzero(startMJDs,endMJDs,True)
                flagh = self.flagshadow(startMJDs,endMJDs,0.0,'')
                allflags = flagz.copy()
                allflags[len(allflags)] = flagh
                self.applyflags(allflags,msname)
                fh.writeFlagCommands(msname,allflags,False,'',cmdfile,True)
                print("%s : Checking flags" % myname)
            
                # Check flags
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],2446080)
        
                # Check output file existence
                self.assertTrue(os.path.exists(cmdfile))
        
                # Check file content
                linecount = countlines(cmdfile)
                self.assertEqual(linecount,216,'unexpected number of flag commands in saved ascii file : %s' % linecount)
            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except Exception as instance:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'
        self.assertTrue(retValue['success'])
        
    def test_evla_apply2(self):
        '''test of importing evla data and applying the online flags'''
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
            
        # importasdm can everything importevla did in this case
        self.res  = importasdm(asdm=self.asdm, vis=msname, scans='2',ocorr_mode='co', with_pointing_correction=True,
                               process_flags=True, applyflags=True, savecmds=False, flagbackup=False)
        # Check flags
        res = flagdata(vis=msname, mode='summary')
        self.assertEqual(res['flagged'],2446080)
        self.assertEqual(res['scan']['2']['flagged'],2446080)
        
    def test_evla_apply3(self):
        '''test of importing evla data, do not apply online flags; post fill - apply clip zeros on RR and LL and save to file'''
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)
            
        # do NOT use the online flags here
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='2,13', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=False,flagbackup=False)

        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK, doing flagzero with flagpol=False" % myname)
                (startMJDs,endMJDs) = self.getmsmjds(msname)
                flagz = self.flagzero(startMJDs,endMJDs,False)
                self.applyflags(flagz,msname)
                fh.writeFlagCommands(msname,flagz,False,'',cmdfile,True)

                print("%s : Checking flags" % myname)

                # Check flags - not the most useful test case
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],0,'There are no zeros in this data set')
                self.assertEqual(res['scan']['2']['flagged'],0,'No flags should have been applied')
                self.assertEqual(res['scan']['13']['flagged'],0,'No flags should have been applied')
        
                # Check output file existence
                self.assertTrue(os.path.exists(cmdfile))
        
                # Check file content
                linecount = countlines(cmdfile)
                self.assertEqual(linecount,2,'Only clip zeros should be saved to file (2) : %s' % linecount)

            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'
        self.assertTrue(retValue['success'])

    def test_evla_apply4(self):
        '''test of importing evla data: Save online flags to FLAG_CMD and file; do not apply'''

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)
            
        # importasdm can do all of this
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='2', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=True, applyflags=False,savecmds=True, flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        ok = True
        try:
            mslocal.open(msname)
            mslocal.close()
        except:
            print( "%s: Error  Cannot open MS table %s" % (myname, msname))
            ok = False

        self.assertTrue(ok,'Error opening MS')

        # No flags were applied
        res = flagdata(vis=msname, mode='summary')
        self.assertEqual(res['flagged'],0)
        
        # Apply only row 213 using flagcmd
        # The command in row 213 is the following:
        # antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199' 
        # spw='EVLA_X#A0C0#0' correlation='LL,LR,RL
        flagcmd(vis=msname, action='apply', tablerows=213)
        
        # Check flags. RR should no be flagged
        res = flagdata(vis=msname, mode='summary')
        self.assertEqual(res['correlation']['RR']['flagged'],0,'RR should not be flagged')
        self.assertEqual(res['correlation']['LL']['flagged'],29440)
        self.assertEqual(res['correlation']['LR']['flagged'],29440)
        self.assertEqual(res['correlation']['RL']['flagged'],29440)
        self.assertEqual(res['antenna']['ea06']['flagged'],88320)
        self.assertEqual(res['antenna']['ea07']['flagged'],3840,'Only a few baselines should be flagged')
        self.assertEqual(res['antenna']['ea08']['flagged'],3840,'Only a few baselines should be flagged')
        
        # Check output file existence       
        self.assertTrue(os.path.exists(cmdfile))
        
        # Check file content
        linecount = countlines(cmdfile)
        self.assertEqual(linecount, 214, 'Only Online cmds should have been saved to file')

    def test_evla_apply5(self):
        '''test of importing evla data: Apply only shadow flags'''

        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        if os.path.exists(msname):
            os.system('rm -rf '+msname)
 
        self.res = importasdm(asdm=self.asdm, vis=msname, ocorr_mode='co', with_pointing_correction=True,
                              process_flags=False, flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK. shadow flagging" % myname)

                (startMJDs,endMJDs) = self.getmsmjds(msname)
                flagh = self.flagshadow(startMJDs,endMJDs,0.0,'')
                allflags = {}
                allflags[0] = flagh
                self.applyflags(allflags,msname)

                # This data set doesn't have shadow. Not a very useful sdm.
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],0,'There are shadowed antenna in this data set')

            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'

        self.assertTrue(retValue['success'])

    def test_evla_savepars(self):
        '''test importing evla data: save the flag commands and do not apply'''
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)

        self.res = importasdm(asdm=self.asdm, vis=msname,scans='11~13', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=True, applyflags=False, savecmds=True, flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK. doing flagzero and shadow flagging" % myname)

                (startMJDs,endMJDs) = self.getmsmjds(msname)
                flagz = self.flagzero(startMJDs,endMJDs,True)
                flagh = self.flagshadow(startMJDs,endMJDs,0.0,'')
                allflags = flagz.copy()
                allflags[len(allflags)] = flagh

                # do NOT apply, but only append them to the command file
                fh.writeFlagCommands(msname,allflags,False,'',cmdfile,True)

                # Check flags - non should be applied
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],0,'No flags should have been applied')

                # Check output file existence
                self.assertTrue(os.path.exists(cmdfile))
        
                # Check file content
                linecount = countlines(cmdfile)
                self.assertEqual(linecount, 216, 'Online, shadow and clip zeros should be saved to file')

                # NOW apply the flags
                # Apply flags using flagdata
                flagdata(vis=msname, mode='list', inpfile=cmdfile)
        
                # and check that they've been applied as expected
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],6090624)

            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'
        self.assertTrue(retValue['success'])
                
    def test_evla_apply1_flagdata(self):
        '''test of importing evla data, apply onlineflags, use flagdata to do shadow and zero level clipping'''
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)
            
        # this applies and saves the online commands, which is all importasdm can do
        # note that there is no ephemeris data here so those arguments are not used in this call
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='2', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=True, applyflags=True, savecmds=True, outfile=cmdfile, flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK. doing flagzero and shadow flagging" % myname)

                # workaround here. flagdata cannot append to outfile, just overwrite it
                # create 2 independent cmdfiles and then append them to the one produced by importasdm
                clipCmdfile = msname.replace('.ms','_clip_cmd.txt')
                shadowCmdfile = msname.replace('.ms','_shadow_cmd.txt')

                flagdata(vis=msname, mode='clip',clipzeros=True,savepars=True,outfile=clipCmdfile,cmdreason='CLIP_ZERO_ALL')
                flagdata(vis=msname, mode='shadow',savepars=True,outfile=shadowCmdfile,cmdreason='SHADOW')

                # concatenate them
                os.system('cat %s >> %s' % (clipCmdfile,cmdfile))
                os.system('cat %s >> %s' % (shadowCmdfile,cmdfile))
                # remove the single line files
                os.system('rm %s' % clipCmdfile)
                os.system('rm %s' % shadowCmdfile)
                
                print("%s : Checking flags" % myname)
            
                # Check flags
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],2446080)
        
                # Check output file existence
                self.assertTrue(os.path.exists(cmdfile))
                
                # Check file content
                linecount = countlines(cmdfile)
                self.assertEqual(linecount,216,'unexpected number of flag commands in saved ascii file : %s' % linecount)
            except AssertionError as error: 
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'
        self.assertTrue(retValue['success'])
        
    def test_evla_apply3_flagdata(self):
        '''test of importing evla data, do not apply online flags; post fill - save to file, use flagdata'''
        # flagdata can not just clip the auto-correlation polationarizations. Can not duplicate intent of test_evla_apply3
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)
            
        # do NOT use the online flags here
        self.res = importasdm(asdm=self.asdm, vis=msname, scans='2,13', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=False,flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK, doing flagzero" % myname)  # equivalent to flagpol=False
                flagdata(vis=msname, mode='clip', clipzeros=True, correlation="ABS_RR,ABS_LL", savepars=True, cmdreason='CLIP_ZERO_AUTO_ONLY', outfile=cmdfile)
                
                print("%s : Checking flags" % myname)

                # Check flags - not the most useful test case
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],0,'There are no zeros in this data set')
                self.assertEqual(res['scan']['2']['flagged'],0,'No flags should have been applied')
                self.assertEqual(res['scan']['13']['flagged'],0,'No flags should have been applied')
        
                # Check output file existence
                self.assertTrue(os.path.exists(cmdfile))
        
                # Check file content
                linecount = countlines(cmdfile)
                self.assertEqual(linecount,1,'Only clip zeros should be saved to file (1) : %s' % linecount)

            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'
        self.assertTrue(retValue['success'])

    def test_evla_apply5_flagdata(self):
        '''test of importing evla data: Apply only shadow flags; using flagdata'''

        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        if os.path.exists(msname):
            os.system('rm -rf '+msname)
 
        self.res = importasdm(asdm=self.asdm, vis=msname, ocorr_mode='co', with_pointing_correction=True,
                              process_flags=False, flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK. shadow flagging" % myname)
                flagdata(vis=msname, mode='shadow', savepars=True)

                # This data set doesn't have shadow. Not a very useful sdm.
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],0,'There are shadowed antenna in this data set')

            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                print("%s : post fill flagging and checking failed." % myname)
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'

        self.assertTrue(retValue['success'])

    def test_evla_savepars_flagdata(self):
        '''test importing evla data: save the flag commands and do not apply; using flagdata'''
        retValue = {'success':True, 'msg':"", 'error_msgs':''}

        self.asdm = 'X_osro_013.55979.93803716435'
        msname = self.asdm + ".ms"
        cmdfile = msname.replace('.ms','_cmd.txt')

        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)

        self.res = importasdm(asdm=self.asdm, vis=msname,scans='11~13', ocorr_mode='co', with_pointing_correction=True,
                              process_flags=True, applyflags=False, savecmds=True, outfile=cmdfile, flagbackup=False)
        print("%s: importasdm success. Checking that filled MS can be opened as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname, msname))
            retValue['success'] = False
            retValue['error_msgs'] = retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()

        if (retValue['success']):
            try:
                print("%s : OK. doing flagzero and shadow flagging" % myname)
                # workaround here. flagdata cannot append to outfile, just overwrite it
                # create 2 independent cmdfiles and then append them to the one produced by importasdm
                clipCmdfile = msname.replace('.ms','_clip_cmd.txt')
                shadowCmdfile = msname.replace('.ms','_shadow_cmd.txt')
                
                # no NOT apply
                flagdata(vis=msname, mode='clip',clipzeros=True,action='calculate',savepars=True,outfile=clipCmdfile,cmdreason='CLIP_ZERO_ALL')
                flagdata(vis=msname, mode='shadow',action='calculate',savepars=True,outfile=shadowCmdfile,cmdreason='SHADOW')

                # concatenate them
                os.system('cat %s >> %s' % (clipCmdfile,cmdfile))
                os.system('cat %s >> %s' % (shadowCmdfile,cmdfile))
                # remove the single line files
                os.system('rm %s' % clipCmdfile)
                os.system('rm %s' % shadowCmdfile)

                # Check flags - non should be applied
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],0,'No flags should have been applied')

                # Check output file existence
                self.assertTrue(os.path.exists(cmdfile))
        
                # Check file content
                linecount = countlines(cmdfile)
                self.assertEqual(linecount, 216, 'Online, shadow and clip zeros should be saved to file')

                # NOW apply the flags
                # Apply flags using flagdata
                flagdata(vis=msname, mode='list', inpfile=cmdfile)
        
                # and check that they've been applied as expected
                res = flagdata(vis=msname, mode='summary')
                self.assertEqual(res['flagged'],6090624)

            except AssertionError as error:
                print('%s : assertion error while testing flags after filling: %s' % (myname, str(error)))
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+str(error)
            except:
                retValue['success'] = False
                retValue['error_msgs']=retValue['error_msgs']+'Unexpected post-fill flagging result'
        self.assertTrue(retValue['success'])
                
class asdm_import4(test_base):
    
    def setUp(self):
        self.setUp_autocorr()
        
    def tearDown(self):
        os.system('rm -rf '+self.asdm)
        os.system('rm -rf x54.ms*')
        os.system('rm -rf scan3.ms* autocorr.ms*')
        os.system('rm -rf fbackup1.ms*')
        os.system('rm -rf scan3flags.txt scan3flags1.txt')
        
    def test_autocorr(self):
        '''importasdm: auto-correlations should be written to online flags'''
        outfile='scan3flags.txt'
        importasdm(asdm=self.asdm, vis='x54.ms', scans='3', savecmds=True, outfile=outfile)
        self.assertTrue(os.path.exists(outfile))
        with open(outfile,'r') as ff:
            cmds = ff.readlines()
            # auto-correlation should have been written to online flags               
            self.assertTrue(cmds[0].__contains__('&&*'))

    def test_autocorr_no_append(self):
        '''importasdm: online flags should not be appended to existing file'''
        outfile='scan3flags1.txt'
        importasdm(asdm=self.asdm, vis='x54.ms', scans='3', savecmds=True, outfile=outfile)
        self.assertTrue(os.path.exists(outfile))
        with open(outfile,'r') as ff:
            cmds = ff.readlines()
            self.assertEqual(cmds.__len__(), 2832)
        
        # Run again and verify that the online flags are not overwritten
        os.system('rm -rf x54.ms*')
        with self.assertRaises(RuntimeError):
            importasdm(asdm=self.asdm, vis='x54.ms', scans='3', savecmds=True,
                       outfile=outfile, overwrite=False)
        print('Expected Error!')
        with open(outfile,'r') as ff:
            cmds = ff.readlines()
            self.assertEqual(cmds.__len__(), 2832)
       
    def test_flagautocorr1(self):
        '''importasdm: test that auto-correlations from online flags are correctly flagged'''        
        importasdm(asdm=self.asdm, vis='scan3.ms', scans='3', applyflags=True)
        res = flagdata('scan3.ms',mode='summary', basecnt=True)
        self.assertEqual(res['flagged'], 298)
        self.assertEqual(res['baseline']['DA44&&DA44']['flagged'], 76)
        self.assertEqual(res['baseline']['PM03&&PM03']['flagged'], 16)

    # CAS-4698, CSV-2576
    def test_flagautocorr2(self):
        '''importasdm: apply auto-correlations from online flags in flagcmd''' 
        outputms = 'autocorr.ms'       
        importasdm(asdm=self.asdm, vis=outputms, scans='3', applyflags=False)
        
        # Flag with flagcmd
        flagcmd(vis=outputms, action='apply', flagbackup=False)
        res = flagdata(vis=outputms, mode='summary', basecnt=True)
        self.assertEqual(res['flagged'], 298)
        self.assertEqual(res['baseline']['DA44&&DA44']['flagged'], 76)
        self.assertEqual(res['baseline']['PM03&&PM03']['flagged'], 16)

    # CAS-5286
    def test_flagautocorr3(self):
        '''importasdm: apply auto-correlations using autocorr=true in flagdata''' 
        outputms = 'autocorr.ms'       
        importasdm(asdm=self.asdm, vis=outputms, scans='3', process_flags=False,
                   flagbackup=False)
         # Applys with flagdata. Because all the data has PROCESSOR TYPE RADIOMETER
        # none of the auto-correlations should be flagged
        flagdata(vis=outputms, mode='manual', autocorr=True, flagbackup=False)
        res = flagdata(vis=outputms, mode='summary', basecnt=True)
        self.assertEqual(res['flagged'], 0)
        self.assertEqual(res['baseline']['DA44&&DA44']['flagged'], 0)
        self.assertEqual(res['baseline']['PM03&&PM03']['flagged'], 0)

    def test_flagbackup(self):
        '''importasdm: Create a flagbackup by default''' 
        outputms = 'fbackup1.ms'
        fbackup = outputms+'.flagversions' 
        
        # Do not create a flagbackup
        importasdm(asdm=self.asdm, vis=outputms, scans='3', flagbackup=False)
        self.assertFalse(os.path.exists(fbackup))
        # Create a backup by default
        importasdm(asdm=self.asdm, vis=outputms, scans='3', overwrite=True)
        self.assertTrue(os.path.exists(fbackup))
        
        # Delete it and create again using the parameter
        os.system('rm -rf '+outputms+'*')
        importasdm(asdm=self.asdm, vis=outputms, scans='3', flagbackup=True)
        self.assertTrue(os.path.exists(fbackup))

class asdm_import5(test_base):
    
    def setUp(self):
        self.setUp_m51()
        
    def tearDown(self):
        shutil.rmtree(myasdm_dataset_name)
        shutil.rmtree(myms_dataset_name)
        shutil.rmtree(msname,ignore_errors=True)
        shutil.rmtree(msname+'.flagversions',ignore_errors=True)
        os.system('rm -rf reference.ms* reimported-M51.ms*')
        
                
    def test1_lazy1(self):
        '''Asdm-import: Test good v1.2 input with default filler in lazy mode, with_pointing_correction=True'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        self.res = importasdm(myasdm_dataset_name, lazy=True, with_pointing_correction=True)
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["table.f17asdmindex",
                            "ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(msname+"/"+name, os.F_OK):
                print('%s: Error  %s/%s doesn\'t exist ...' % (myname,msname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(msname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,msname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+msname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            importasdm(asdm=myasdm_dataset_name, vis='reference.ms', lazy=False, overwrite=True)

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.compTables('reference.ms', msname, ['FLAG', 'FLAG_CATEGORY', 'DATA','WEIGHT_SPECTRUM'], 
                                                    0.001) 
                retValue['success'] = th.compVarColTables('reference.ms', msname, 'DATA',1E-5) 
                retValue['success'] = th.compVarColTables('reference.ms', msname, 'FLAG') 

                for subtname in ["ANTENNA",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 #"POINTING", # expect difference since with_pointing_correction=True
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL"]:
                    
                    print("\n*** Subtable %s" % subtname)
                    excllist = []
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                      msname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                        
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            msname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print("ERROR for table %s" % subtname)

                print("\n*** Subtable POINTING")
                try:
                    retValue['success'] = retValue['success'] and not (th.compTables('reference.ms/POINTING', # expect difference since with_pointing_correction=True
                                                                                     msname+'/POINTING', [], 0.0))
                except:
                    retValue['success'] = False
                    print("ERROR: POINTING tables should differ in this test.")
            


                
        self.assertTrue(retValue['success'],retValue['error_msgs'])

class asdm_import6(test_base):

    def setUp(self):
        self.setUp_acaex()
        
    def tearDown(self):
        myasdmname = 'uid___A002_X72bc38_X000'
        os.system('rm '+myasdmname) # a link
        shutil.rmtree(msname,ignore_errors=True)
        shutil.rmtree(msname+'.flagversions',ignore_errors=True)
        shutil.rmtree(myasdmname+'.ms',ignore_errors=True)
        shutil.rmtree(myasdmname+'.ms.flagversions',ignore_errors=True)
        os.system('rm -rf reference.ms*')
               
    def test6_lazy1(self):
        '''Asdm-import: Test good ACA ASDM with mixed pol/channelisation input with default filler in lazy mode'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X72bc38_X000'
        themsname = myasdmname + ".ms"

        self.res = importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3') # only the first 3 scans to save time
        self.assertEqual(self.res, None)
        #test that scratch columns can be created from lazy import
        cblocal = calibrater()
        cblocal.open(themsname)
        cblocal.close()
        tblocal = table()
        tblocal.open(themsname)
        self.assertIn('CORRECTED_DATA', tblocal.getdesc())
        self.assertIn('MODEL_DATA', tblocal.getdesc())
        tblocal.close()

        print("%s: Success! Now checking output ..." % myname)
        # crude test of success ...
        # the following tables should exist as directies containing table.dat and table.f0
        expectedTabs = ["ANTENNA","CALDEVICE","DATA_DESCRIPTION","FEED","FIELD",
                        "FLAG_CMD","HISTORY","OBSERVATION","POINTING","POLARIZATION",
                        "PROCESSOR","SOURCE","SPECTRAL_WINDOW","STATE","SYSCAL",
                        "SYSPOWER","WEATHER"]
        for name in expectedTabs:
            for fname in ["table.dat","table.f0"]:
                tabname = name+"/"+fname
                thisName = themsname+"/"+tabname
                if not os.access(thisName, os.F_OK):
                    print("%s: Error  %s/%s doesn't exist ..." % (myname,thisName,name))
                    retValue['success']=False
                    retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+tabname+' does not exist'
                else:
                    print("%s: %s/%s present." % (myname,tabname,fname))
        if (retValue['success']):
            print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
            try:
                mslocal.open(themsname)
            except:
                print("%s: Error  Cannot open MS table %s" % (myname,themsname))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
            else:
                mslocal.close()
                print("%s: OK. Checking tables in detail ..." % myname)
    
                importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3')

                if(os.path.exists('reference.ms')):
                    retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                           +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
                    if not retValue['success']:
                        print("ERROR: DATA does not agree with reference.")
                    else:
                        print("DATA columns agree.")
                    retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                   +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
                    if not retValueTmp:
                        print("ERROR: FLAG does not agree with reference.")
                    else:
                        print("FLAG columns agree.")

                    retValue['success'] = retValue['success'] and retValueTmp

                    for subtname in expectedTabs:
                        print("\n*** Subtable %s" % subtname)
                        excllist = []
                        if subtname=='SOURCE':
                            excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                        if subtname=='SYSCAL':
                            excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                        if subtname=='CALDEVICE':
                            excllist=["NOISE_CAL","CAL_EFF"]
                        if subtname=='HISTORY':
                            excllist=['MESSAGE']
                        if subtname=='SPECTRAL_WINDOW':
                            excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
                            for colname in excllist: 
                                retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                          themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                        try:    
                            retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                                themsname+'/'+subtname, excllist, 
                                                                0.01) and retValue['success']
                        except:
                            retValue['success'] = False
                            print("ERROR for table %s" % subtname)

                    if retValue['success']:
                        # For this MS, the WEATHER/DEW_POINT_FLAG is all True and
                        # the other *FLAG columns are all false, just check PRESSURE_FLAG
                        tblocal.open(themsname+'/WEATHER')
                        dewptFlag = tblocal.getcol('DEW_POINT_FLAG')
                        pressFlag = tblocal.getcol('PRESSURE_FLAG')
                        tblocal.close()
                        retValue['success'] = (dewptFlag.sum()==len(dewptFlag)) and (pressFlag.sum()==0)
                        if not (retValue['success']):
                            print("ERROR for WEATHER table. Expected values of DEW_POINT_FLAG and PRESSURE_FLAG are not seen.")
                
        self.assertTrue(retValue['success'],retValue['error_msgs'])


class asdm_import7(test_base):

    def setUp(self):
        self.setUp_12mex()
        self.setUp_eph()
        self.setUp_SD()
        
    def tearDown(self):
        for myasdmname in ['uid___A002_X71e4ae_X317_short', 'uid___A002_X997a62_X8c-short', 'uid___A002_X6218fb_X264']:
            os.system('rm -f '+myasdmname) # a link
            shutil.rmtree(myasdmname+".ms",ignore_errors=True)
            shutil.rmtree(myasdmname+'.ms.flagversions',ignore_errors=True)
        shutil.rmtree("reference.ms",ignore_errors=True)
        shutil.rmtree("reference.ms.flagversions",ignore_errors=True)
        shutil.rmtree("uid___A002_X997a62_X8c-short.interp.ms",ignore_errors=True)
        shutil.rmtree("uid___A002_X71e4ae_X317_short.split.ms",ignore_errors=True)
               
    def test7_lazy1(self):
        '''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname+".ms"

        self.res = importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4') # only the first 4 scans to save time
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["ANTENNA/table.dat",
                            "CALDEVICE/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "SYSPOWER/table.dat",
                            "WEATHER/table.dat",
                            "ANTENNA/table.f0",
                            "CALDEVICE/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0",
                            "SYSPOWER/table.f0",
                            "WEATHER/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(themsname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(themsname)
            mslocal.close()
            print("%s: MS can be opened. Now testing the changing of the asdmref ..." % myname)
            mslocal.open(themsname)
            mslocal.asdmref("./moved_"+myasdmname)
            mslocal.close()
            os.system("mv "+myasdmname+" moved_"+myasdmname)
            
            mslocal.open(themsname)
            
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,themsname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            importasdm(asdm="moved_"+myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~4')

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                       +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
                if not retValue['success']:
                    print("ERROR: DATA does not agree with reference.")
                else:
                    print("DATA columns agree.")

                retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                               +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0
                if not retValueTmp:
                    print("ERROR: WEIGHT does not agree with reference.")
                else:
                    print("WEIGHT columns agree.")
                    
                retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
                if not retValueTmp2:
                    print("ERROR: FLAG does not agree with reference.")
                else:
                    print("FLAG columns agree.")

                retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2

                for subtname in ["ANTENNA",
                                 "CALDEVICE",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL",
                                 "SYSPOWER",
                                 "WEATHER"]:
                    
                    print("\n*** Subtable %s" % subtname)
                    excllist = []
                    if subtname=='CALDEVICE':
                        excllist=['NOISE_CAL','CAL_EFF']
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SYSCAL':
                        excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
                        for colname in excllist:
                            if colname!='ASSOC_NATURE':
                                retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                          themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                    if subtname=='POLARIZATION':
                        excllist=['CORR_TYPE', 'CORR_PRODUCT']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
                                                                      themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            themsname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print("ERROR for table %s" % subtname)

                try:
                    # test that the PRESSURE column has the expected units
                    wxcalOK = tblocal.open(themsname+'/WEATHER')
                    if wxcalOK:
                        wxcalOK = tblocal.getcolkeyword("PRESSURE","QuantumUnits")[0] == 'hPa'
                        tblocal.close()
                    retValue['success'] = wxcalOK and retValue['success']
                    if not wxcalOK:
                        print("PRESSURE column in WEATHER table is missing or has incorrect units")
                except:
                    retValue['success'] = False
                    print("ERROR getting units of PRESSURE column in WEATHER table.")

                try:
                    # test that the SDM_WINDOW_FUNCTION column exists and has the exepcted values
                    winFuncOK = tblocal.open(themsname+'/SPECTRAL_WINDOW')
                    if winFuncOK:
                        winFuncCol = tblocal.getcol('SDM_WINDOW_FUNCTION')
                        tblocal.close()
                        # test values here
                        # expect 55 rows, rows 1:24 are HANNING, the rest are UNIFORM
                        indx = numpy.arange(len(winFuncCol))
                        winFuncOK = winFuncOK and (numpy.array_equal(indx[winFuncCol=="HANNING"],(numpy.arange(24)+1)))
                        winFuncOK = winFuncOK and (len(indx[winFuncCol=="UNIFORM"])==31)

                    retValue['success'] = winFuncOK and retValue['success']
                    if not winFuncOK:
                        print("SDM_WINDOW_FUNCTION column in the SPECTRAL_WINDOW table is missing or has incorrect values")
                except:
                    retValue['success'] = False
                    print("ERROR checking the value of the SDM_WINDOW_FUNCTION column in the SPECTRAL_WINDOW table.")

                try:
                    # test that the SDM_NUM_BIN column exists and has the exepcted values
                    numBinOK = tblocal.open(themsname+'/SPECTRAL_WINDOW')
                    if numBinOK:
                        numBinCol = tblocal.getcol('SDM_NUM_BIN')
                        tblocal.close()
                        # all values are 1
                        numBinOK = numpy.all(numBinCol==1)

                    retValue['success'] = numBinOK and retValue['success']
                    if not numBinOK:
                        print("SDM_NUM_BIN column in the SPECTRAL_WINDOW table is missing or has incorrect values")
                except:
                    retValue['success'] = False
                    print("ERROR checking the value of the SDM_NUM_BIN column in the SPECTRAL_WINDOW table.")

                try:
                    # test that the SDM_CORR_BIT column exists and has the exepcted values
                    corrBitOK = tblocal.open(themsname+'/SPECTRAL_WINDOW')
                    if corrBitOK:
                        corrBitCol = tblocal.getcol('SDM_CORR_BIT')
                        tblocal.close()
                        # test values here - all are expected to UNKNOWN (correlationBit is not present in this sdm)
                        corrBitOK = numpy.all(corrBitCol == "UNKNOWN")

                    retValue['success'] = corrBitOK and retValue['success']
                    if not corrBitOK:
                        print("SDM_CORR_BIT column in the SPECTRAL_WINDOW table is missing or has incorrect values")
                except:
                    retValue['success'] = False
                    print("ERROR checking the value of the SDM_CORR_BIT column in the SPECTRAL_WINDOW table.")

                try:
                    # test that the PROCESSOR table SUB_TYPE has the expected values
                    subTypeOK = tblocal.open(themsname+'/PROCESSOR')
                    if subTypeOK:
                        subTypeCol = tblocal.getcol('SUB_TYPE')
                        tblocal.close()
                        subTypeOK = (len(subTypeCol)==3) and (subTypeCol[0]=='ALMA_BASELINE') and (subTypeCol[1]=='ALMA_RADIOMETER') and (subTypeCol[2]==subTypeCol[0])

                    retValue['success'] = subTypeOK and retValue['success']
                    if not subTypeOK:
                        print("SUB_TYPE column in the PROCESSOR table is missing or has incorrect values")
                except:
                    retValue['success'] = False
                    print("ERROR checking the value of the SUB_TYPE column in the PROCESSOR table.")

                try:
                    # split this ms to check that SUB_TYPE is carried over correctly into the new ms
                    mslocal.open(themsname)
                    splitname = myasdmname + ".split.ms"
                    mslocal.split(splitname,spw='1,3,5,7')
                    mslocal.close()
                    splitOK = tblocal.open(splitname+'/PROCESSOR')
                    if  splitOK:
                        subTypeCol = tblocal.getcol('SUB_TYPE')
                        tblocal.close()
                        splitOK = (len(subTypeCol)==3) and (subTypeCol[0]=='ALMA_BASELINE') and (subTypeCol[1]=='ALMA_RADIOMETER') and (subTypeCol[2]==subTypeCol[0])
                    retValue['success'] = splitOK and retValue['success']
                    if not splitOK:
                        print("SUB_TYPE column in the PROCESSOR table after the split has incorrect values")
                except:
                    retValue['success'] = False
                    print("ERROR checking the value of the SUB_TYPE column in the PROCESSOR table after a split.")
                        
        os.system("mv moved_"+myasdmname+" "+myasdmname)
                
        self.assertTrue(retValue['success'],retValue['error_msgs'])

    def test7_lazy2(self):
        '''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode with reading the BDF flags'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname+".ms"

        self.res = importasdm(myasdmname, vis=themsname, lazy=True, bdfflags=True) 
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(themsname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(themsname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,themsname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            importasdm(asdm=myasdmname, vis='reference.ms', lazy=True, overwrite=True, bdfflags=False)

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
                if not retValue['success']:
                    print("ERROR: DATA does not agree with reference.")
                else:
                    print("DATA columns agree.")
                retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                            +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") != 0
                if not retValueTmp:
                    print("ERROR: FLAG columns do agree with reference but they shouldn't.")
                else:
                    print("FLAG columns do not agree as expected.")

                retValue['success'] = retValue['success'] and retValueTmp

                for subtname in ["ANTENNA",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL"]:
                    
                    print("\n*** Subtable %s" % subtname)
                    excllist = []
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SYSCAL':
                        excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
                        for colname in excllist:
                            if colname!='ASSOC_NATURE':
                                retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                          themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                    if subtname=='POLARIZATION':
                        excllist=['CORR_TYPE', 'CORR_PRODUCT']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
                                                                      themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            themsname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print("ERROR for table %s" % subtname)
            
        self.assertTrue(retValue['success'],retValue['error_msgs'])

    def test7_lazy3(self):
        '''Asdm-import: Test good 12 m ASDM with Ephemeris table in lazy mode'''

        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X997a62_X8c-short'
        themsname = myasdmname+".ms"

        # do this fill 2 times, first with the default polyephem_tabtimestep and
        # second with a set value. Both should pass with the same test values
        # since that parameter doesn't matter

        for tabtimestep in [None, 0.0075]:
            # make sure there's nothing there from a previous fill
            shutil.rmtree(themsname,ignore_errors=True)
            shutil.rmtree(themsname+'.flagversions',ignore_errors=True)

            if tabtimestep is None:
                # default value
                print("fill to %s with default polyephem_tabtimestep" % themsname)
                self.res = importasdm(myasdmname, vis=themsname, lazy=True, convert_ephem2geo=True, 
                                      process_pointing=False, flagbackup=False)
            else:
                print("fill to %s with polyephem_tabtimestep = %s, value should not be used here" % (themsname, tabtimestep))
                # a specific value, this should produce a warning message - not checked here
                self.res = importasdm(myasdmname, vis=themsname, lazy=True, convert_ephem2geo=True,
                                      process_pointing=False, flagbackup=False, polyephem_tabtimestep=tabtimestep)
                
            self.assertEqual(self.res, None)
            print("%s: Success! Now checking output ..." % myname)
            mscomponents = set(["FIELD/table.dat",
                                "FIELD/EPHEM0_Mars_57034.9.tab",
                                "FIELD/EPHEM1_Titania_57034.9.tab"
                              ])
            for name in mscomponents:
                if not os.access(themsname+"/"+name, os.F_OK):
                    print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                    retValue['success']=False
                    retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
                else:
                    print("%s: %s present." % (myname,name))
            print("%s: MS exists. All relevant tables present. Try opening as MS ..." % myname)
            
            try:
                mslocal.open(themsname)
            except:
                print("%s: Error  Cannot open MS table %" % (myname,themsname))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname


            print("%s:  testing FIELD values in ms.summary()" % myname)
            try:
                mssum = mslocal.summary()
                # only Mars appears here because this short SDM only contains a single scan and that uses Mars
                self.assertEqual(mssum['scan_1']['0']['FieldName'],'Mars')
                self.assertAlmostEqual(mssum['field_0']['direction']['m0']['value'],-0.4770797859505159,15)
                self.assertAlmostEqual(mssum['field_0']['direction']['m1']['value'],-0.2154815444753364,15)
            except:
                print("%s: Error ms summary has an unexpect source or direction value" % myname)
                retValue['success']=False
                retValue['error_msgs']=retValue['err_msg']+'Unexpected source or direction value in ms summary '+thismsname + '\n'

            
            mslocal.close()

            ephems = []
            # values from indivual rows were picked for no particular reason except verify they've not changed
            ephems.append({'name':"FIELD/EPHEM0_Mars_57034.9.tab",
                           'nrows':27,
                           'rows':[{'row':10,
                                    'values':{'MJD':57035.0416666667,
                                              'RA': 332.7140437500,
                                              'DEC':-12.3273469444,
                                              'Rho':  2.0246094801,
                                              'RadVel':723729.775}},
                                   {'row':22,
                                    'values':{'MJD':57035.2083333333,
                                              'RA': 332.8387870833,
                                              'DEC':-12.2793975000,
                                              'Rho':2.0254053469,
                                              'RadVel':705588.203}}
                                  ]
                          }
                         )

            ephems.append({'name':"FIELD/EPHEM1_Titania_57034.9.tab",
                           'nrows':45,
                           'rows':[{'row':17,
                                    'values':{'MJD':57035.0555555556,
                                              'RA': 11.8131666667,
                                              'DEC': 4.3657500000,
                                              'Rho':20.1508836737,
                                              'RadVel':2730048.084}},
                                   {'row':40,
                                    'values':{'MJD':57035.2152777778,
                                              'RA':11.8160416667,
                                              'DEC':4.3661111111,
                                              'Rho':20.1537364617,
                                              'RadVel':2711142.170}}
                                  ]
                          }
                        )

            # tolerances by column name (digits after the decimal)
            tolerances = {'MJD':10, 'RA':10, 'DEC':10, 'Rho':10, 'RadVel':3}
            
            for ephem in ephems:
                print("%s: Testing various things in ephemeris %s ..." % (myname,ephem['name']))

                tblocal.open(themsname+"/"+ephem['name'])
                kw = tblocal.getkeywords()
                nrows = tblocal.nrows()
                if not nrows==ephem['nrows']:
                    print("%s: Error. unexpected number of rows in ephemeris : %s" % (myname,ephem['name']))
                    retValue['success']=False
                    retValue['error_msgs']=retValue['error_msgs']+' Unexpected number of rows in ephemeris table :'+ ephem['name'] + '\n'

                for row in ephem['rows']:
                    thisRow = row['row']
                    for colname in row['values']:
                        thisVal = tblocal.getcell(colname,thisRow)
                        self.assertAlmostEqual(thisVal,row['values'][colname],tolerances[colname])

                # unfilled columns
                self.assertEqual((tblocal.getcol('diskLong') != 0.0).sum(),0)
                self.assertEqual((tblocal.getcol('diskLat') != 0.0).sum(),0)

                tblocal.close()
                geodist = kw['GeoDist'] # (km above reference ellipsoid)
                geolat = kw['GeoLat'] # (deg)
                geolong = kw['GeoLong'] # (deg)
                if not (geodist==geolat==geolong==0.):
                    print("%s: ERROR." % myname)
                    retValue['success']=False
                    retValue['error_msgs']=retValue['error_msgs']+' Ephemeris was not converted to GEO for '+themsname+'\n'
                prsys = kw['posrefsys']
                if not (prsys=="ICRF/ICRS"):
                    print("%s: ERROR." % myname)
                    retValue['success']=False
                    retValue['error_msgs']=retValue['error_msgs']+' posrefsys keyword is not ICRF/ICRS '+themsname+'\n'

        self.assertTrue(retValue['success'],retValue['error_msgs'])
        print("%s: OK." % myname)


    def test7_lazy4(self):
        '''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode selecting only AUTO data, writing to FLOAT_DATA'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname+".ms"

        self.res = importasdm(myasdmname, vis=themsname, ocorr_mode="ao", lazy=True, scans='0:1~4') # only the first 4 scans to save time
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["ANTENNA/table.dat",
                            "CALDEVICE/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "SYSPOWER/table.dat",
                            "ANTENNA/table.f0",
                            "CALDEVICE/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0",
                            "SYSPOWER/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(themsname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(themsname)
            mslocal.close()
            print("%s: MS can be opened. Now testing the changing of the asdmref ..." % myname)
            mslocal.open(themsname)
            mslocal.asdmref("./moved_"+myasdmname)
            mslocal.close()
            os.system("mv "+myasdmname+" moved_"+myasdmname)
            
            mslocal.open(themsname)
            
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,themsname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            importasdm(asdm="moved_"+myasdmname, vis='reference.ms', ocorr_mode="ao", lazy=False, overwrite=True, scans='0:1~3')

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.FLOAT_DATA,t2.FLOAT_DATA, 1.e-06)))") == 0
                if not retValue['success']:
                    print("ERROR: DATA does not agree with reference.")
                else:
                    print("DATA columns agree.")

                retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0
                if not retValueTmp:
                    print("ERROR: WEIGHT does not agree with reference.")
                else:
                    print("WEIGHT columns agree.")
                    
                retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                            +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
                if not retValueTmp2:
                    print("ERROR: FLAG does not agree with reference.")
                else:
                    print("FLAG columns agree.")

                retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2

                for subtname in ["ANTENNA",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL"]:
                    
                    print("\n*** Subtable %s" % subtname)
                    excllist = []
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SYSCAL':
                        excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
                        for colname in excllist:
                            if colname!='ASSOC_NATURE':
                                retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                          themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                    if subtname=='POLARIZATION':
                        excllist=['CORR_TYPE', 'CORR_PRODUCT']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
                                                                      themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            themsname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print("ERROR for table %s" % subtname)

        os.system("mv moved_"+myasdmname+" "+myasdmname)
                
        self.assertTrue(retValue['success'],retValue['error_msgs'])
        
    def test7_lazy5(self):
        '''Asdm-import: Test TP asdm with default filler in lazy mode selecting only AUTO data, writing to FLOAT_DATA'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname+".ms"

        self.res = importasdm(myasdmname, vis=themsname, ocorr_mode="ao", bdfflags=True, applyflags=True, lazy=True)
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["ANTENNA/table.dat",
                            "CALDEVICE/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "SYSPOWER/table.dat",
                            "WEATHER/table.dat",
                            "ANTENNA/table.f0",
                            "CALDEVICE/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0",
                            "SYSPOWER/table.f0",
                            "WEATHER/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(themsname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(themsname)
            mslocal.close()
            print("%s: MS can be opened. Now testing the changing of the asdmref ..." %  myname)
            mslocal.open(themsname)
            mslocal.asdmref("./moved_"+myasdmname)
            mslocal.close()
            os.system("mv "+myasdmname+" moved_"+myasdmname)
            
            mslocal.open(themsname)
            
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,themsname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            importasdm(asdm="moved_"+myasdmname, vis='reference.ms', ocorr_mode="ao", lazy=False, bdfflags=True, applyflags=True, overwrite=True)

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.FLOAT_DATA,t2.FLOAT_DATA, 1.e-06)))") == 0
                if not retValue['success']:
                    print("ERROR: DATA does not agree with reference.")
                else:
                    print("DATA columns agree.")

                retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0
                if not retValueTmp:
                    print("ERROR: WEIGHT does not agree with reference.")
                else:
                    print("WEIGHT columns agree.")
                    
                retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                            +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
                if not retValueTmp2:
                    print("ERROR: FLAG does not agree with reference.")
                else:
                    print("FLAG columns agree.")

                retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2

                for subtname in ["ANTENNA",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL",
                                 "WEATHER"]:
                    
                    print("\n*** Subtable %s" % subtname)
                    excllist = []
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SYSCAL':
                        excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
                        for colname in excllist:
                            if colname!='ASSOC_NATURE':
                                retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                          themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                    if subtname=='POLARIZATION':
                        excllist=['CORR_TYPE', 'CORR_PRODUCT']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
                                                                      themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            themsname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print("ERROR for table %s" % subtname)

        os.system("mv moved_"+myasdmname+" "+myasdmname)
                
        self.assertTrue(retValue['success'],retValue['error_msgs'])
        
    def test7_skiprows1(self):
        '''Asdm-import: Test TP asdm, comparing output when duplicate DATA rows are skipped versus not-skipped, lazy and regular, with bdflagging on'''

        # this test relies on the stand-alone executable for features not available in the tool
        print("Skipping asdm_import7.test7_skiprows - uses stand-alone executable")
        return

        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname+".ms"

        # the same tests are done for lazy being False and True
        for lazy in [False, True]:
            print("%s Test loop with lazy = %s" % (myname,lazy))
            # always start with a clean slate
            shutil.rmtree(themsname,True)
            shutil.rmtree('referemce.ms',True)
            # use importasdm, which always looks for and skips duplicate DATA rows.
            self.res = importasdm(myasdmname, vis=themsname, ocorr_mode="ao", bdfflags=True, lazy=lazy, overwrite=True)
            self.assertEqual(self.res, None)
            print("%s: Success! Now checking output ..." % myname)
            mscomponents = set(["ANTENNA/table.dat",
                                "CALDEVICE/table.dat",
                                "DATA_DESCRIPTION/table.dat",
                                "FEED/table.dat",
                                "FIELD/table.dat",
                                "FLAG_CMD/table.dat",
                                "HISTORY/table.dat",
                                "OBSERVATION/table.dat",
                                "POINTING/table.dat",
                                "POLARIZATION/table.dat",
                                "PROCESSOR/table.dat",
                                "SOURCE/table.dat",
                                "SPECTRAL_WINDOW/table.dat",
                                "STATE/table.dat",
                                "SYSCAL/table.dat",
                                "SYSPOWER/table.dat",
                                "WEATHER/table.dat",
                                "ANTENNA/table.f0",
                                "CALDEVICE/table.f0",
                                "DATA_DESCRIPTION/table.f0",
                                "FEED/table.f0",
                                "FIELD/table.f0",
                                "FLAG_CMD/table.f0",
                                "HISTORY/table.f0",
                                "OBSERVATION/table.f0",
                                "POINTING/table.f0",
                                "POLARIZATION/table.f0",
                                "PROCESSOR/table.f0",
                                "SOURCE/table.f0",
                                "SPECTRAL_WINDOW/table.f0",
                                "STATE/table.f0",
                                "SYSCAL/table.f0",
                                "SYSPOWER/table.f0",
                                "WEATHER/table.f0"
                                ])
            for name in mscomponents:
                if not os.access(themsname+"/"+name, os.F_OK):
                    print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                    retValue['success']=False
                    retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
                else:
                    print("%s: %spresent." % (myname,name))
            print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
            try:
                mslocal.open(themsname)
                print("%s: MS can be opened" % myname)
                mslocal.close()
            
            except:
                print("%s: Error  Cannot open MS table %s" % (myname,themsname))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
            else:
                print("%s: OK. Generating a reference MS with first integration checking turned off" % myname)

                # this must be done using asdm2MS and bdflags2MS directly
                asdm2MScmd = 'asdm2MS --ocm "ao" --checkdupints false'
                if lazy:
                    asdm2MScmd = asdm2MScmd + " --lazy"
                asdm2MScmd = asdm2MScmd + " " + myasdmname + " reference.ms"
                print('%s Running asdm2MS standalone invoked as:' % myname)
                print(asdm2MScmd)
                exitcode = os.system(asdm2MScmd)
                if exitcode != 0:
                    print("%s asdm2MS terminated with exit code %s" % (myname,exitcode))
                    retValue['success'] = False
                    retValue['error_msgs']=retValue['error_msgs']+' standalone execution of asdm2MS failed'
                    # this should break out of the main loop over lazy values
                    break
                
                bdflags2MScmd = 'bdflags2MS -f ALL --ocm "ao" --checkdupints false'
                if lazy:
                    bdflags2MScmd = bdflags2MScmd + " --lazy=true"
                bdflags2MScmd = bdflags2MScmd + " " + myasdmname + " reference.ms"
                print('%s Running bdflags2MS standalone invoked as:' % myname)
                print(bdflags2MScmd)
                exitcode = os.system(bdflags2MScmd)
                if exitcode != 0:
                    print("%s bdflags2MS terminated with exit code %s" % (myname,exitcode))
                    retValue['success'] = False
                    retValue['error_msgs']=retValue['error_msgs']+' standalone execution of bdflags2MS failed'
                    # this should break out of the main loop over lazy values
                    break                    

                # at this point, reference.ms should exist, with all of the auto rows (ao), including duplicates, with BDF flags applied
                if(os.path.exists('reference.ms')):
                    # and the test here is to make sure that the two MSs are identical in the Main table. The same values in 
                    # the same order except for gaps where rows from the larger table are skipped. 

                    # expected size (rows)
                    msSize = 31972
                    refSize = 32040
                    
                    # expected gaps start at these rows and are always 4 rows long, row numbers in reference.ms
                    gaps = [2280,3176,5328,7120,9276,10172,11432,12328,14480,15376,18752,19648,21800,26636,27896,28792,30944]

                    mstb = table()
                    mstb.open(themsname)
                    if mstb.nrows() != msSize:
                        print('%s MS size is not of the expected number of rows : %s != %s' % (myname,mstb.nrows(),msSize))
                        retValue['success'] = False
                        retValue['error_msgs'] = retValue['error_msgs'] + 'bad size for MS'
                    
                    reftb = table()
                    reftb.open('reference.ms')
                    if reftb.nrows() != refSize:
                        print('%s Reference MS size is not of the expected number of rows : %s != %s' % (myname,reftb.nrows(),refSize))
                        retValue['success'] = False
                        retValue['error_msgs'] = retValue['error_msgs'] + 'bad size for reference MS'

                    if retValue['success']:
                        refrow = 0
                        msrow = 0
                        gapStart = -1
                        gapIndex = -1
                        # assumes set of column names is the same
                        cols = mstb.colnames()

                        while((msrow < mstb.nrows()) and (refrow < reftb.nrows())):
                            matchFound = True
                            for col in cols:
                                if mstb.iscelldefined(col,msrow) != reftb.iscelldefined(col,refrow):
                                    # defined in one cell, not defined in the other
                                    matchFound = False
                                else:
                                    if (mstb.iscelldefined(col,msrow)):
                                        msVal = mstb.getcell(col,msrow)
                                        refVal = reftb.getcell(col,refrow)
                                        # assumes the type is the same for the same column in both tables
                                        if isinstance(msVal,numpy.ndarray):
                                            matchFound = numpy.array_equal(msVal,refVal)
                                        else:
                                            matchFound = msVal == refVal
                                if not matchFound:
                                    # no point in checking the other columns
                                    break
                            if matchFound:
                                if gapStart >= 0:
                                    # a gap has ended, verify that it was exactly 4 rows long
                                    if (refrow-gapStart) != 4:
                                        print('%s Unexpected gap length not equal to 4 rows. Gap length = %s starting at row %s' % (myname,refrow-gapStart,refRow))
                                        retValue['success'] = False
                                        retValue['error_msgs'] = 'Unexpected gap length not equal to 4 rows'
                                    gapStart = -1
                                msrow = msrow + 1
                            else:
                                # do not increment msrow in this case, keep looking for a match
                                if gapStart < 0:
                                    # new gap, increment index and verify it's at the expected place
                                    gapIndex = gapIndex+1
                                    gapStart = refrow
                                    if gapIndex > len(gaps):
                                        print('%s Unexpected gap seen past end of known gaps. Starting at row %s' % (myname,refrow))
                                        retValue['success'] = False
                                        retValue['error_msgs'] = 'Unexpected gap after end of known gaps'
                                    else:
                                        if gapStart != gaps[gapIndex]:
                                            print('%s Unexpected gap start at row %s expected at row %s' % (myname,gapStart,gaps[gapIndex]
))
                                            retValue['success'] = False
                                            retValue['error_msgs'] = 'Unexpected gap start row'
                            # refrow is always incremented
                            refrow = refrow + 1

                            # bail out on failure
                            if not retValue['success']:
                                break
                    
                    reftb.close()
                    mstb.close()
            # break out of the lazy loop on failure
            if not retValue['success']:
                break
                    
        self.assertTrue(retValue['success'],retValue['error_msgs'])
        
    def test7_bdflags1(self):
        '''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler selecting "co" on output and using the BDF flags'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname+".ms"

        self.res = importasdm(myasdmname, vis=themsname, ocorr_mode="co", bdfflags=True) 
        self.assertEqual(self.res, None)
        print("%s: Success! Now checking output ..." % myname)
        mscomponents = set(["ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(themsname+"/"+name, os.F_OK):
                print("%s: Error  %s/%s doesn't exist ..." % (myname,themsname,name))
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
            else:
                print("%s: %s present." % (myname,name))
        print("%s: MS exists. All tables present. Try opening as MS ..." % myname)
        try:
            mslocal.open(themsname)
        except:
            print("%s: Error  Cannot open MS table %s" % (myname,themsname))
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
        else:
            mslocal.close()
            print("%s: OK. Checking tables in detail ..." % myname)
    
            importasdm(asdm=myasdmname, vis='reference.ms', overwrite=True, ocorr_mode="co", bdfflags=False)

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
                if not retValue['success']:
                    print("ERROR: DATA does not agree with reference.")
                else:
                    print("DATA columns agree.")
                retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                            +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") != 0
                if not retValueTmp:
                    print("ERROR: FLAG columns do agree with reference but they shouldn't.")
                else:
                    print("FLAG columns do not agree as expected.")

                retValue['success'] = retValue['success'] and retValueTmp

                for subtname in ["ANTENNA",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL"]:
                    
                    print("\n*** Subtable %s" % subtname)
                    excllist = []
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SYSCAL':
                        excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
                        for colname in excllist:
                            if colname!='ASSOC_NATURE':
                                retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                          themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                    if subtname=='POLARIZATION':
                        excllist=['CORR_TYPE', 'CORR_PRODUCT']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
                                                                      themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            themsname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print("ERROR for table %s" % subtname)
        self.assertTrue(retValue['success'],retValue['error_msgs'])


class asdm_import8(test_base):
    # these are more like unit tests, difficult to test without invoking all of importasdm
    # currently included tests on SDM_NUM_BIN, test on verbose argument
    
    def setUp(self):
        self.setUp_numbin()
        self.setUp_SD()

    def tearDown(self):
        # pass
        for this_asdm_name in ['alma_numbin_mixed','evla_numbin_2','evla_numbin_4','uid___A002_X6218fb_X264']:
            os.system('rm -rf '+this_asdm_name+"*")

    def doNumTest(self, testName, asdm_name, ms_name, spWin_name, execBlock_name, expWinFunCol, expNumBinCol, expResCol, expCorrBitVal):
        retValue = {'success': True, 'error_msgs': '' } 
        print("%s: testing SDM columns in %s writing to %s" % (testName, asdm_name, ms_name))

        originalSpWin = None
        originalExecBlock = None

        if spWin_name is not None:
            print("%s: using %s for SpectralWindow.xml" % (testName, spWin_name))
            spWin_path = os.path.join(asdm_name,spWin_name)
            originalSpWin = os.path.join(asdm_name,"SpectralWindow.xml.original")
            if not os.path.exists(spWin_path):
                msg = spWin_path+" does not exist"
                retValue['success'] = False
                retValue['error_msgs'] = msg
                return retValue
            if os.path.exists(originalSpWin):
                msg = originalSpWin+" already exists, will not overwrite"
                retValue['success'] = False
                retValue['error_msgs'] = msg
                return retValue
            shutil.move(os.path.join(asdm_name,'SpectralWindow.xml'),originalSpWin)
            shutil.copyfile(spWin_path,os.path.join(asdm_name,'SpectralWindow.xml'))

        if execBlock_name is not None:
            print("%s: using %s for ExecBlock.xml" % (testName, execBlock_name))
            execBlock_path = os.path.join(asdm_name,execBlock_name)
            originalExecBlock = os.path.join(asdm_name,"ExecBlock.xml.original")
            if not os.path.exists(execBlock_path):
                msg = execBlock_path+" does not exist"
                retValue['success'] = False
                retValue['error_msgs'] = msg
                return retValue
            if os.path.exists(originalExecBlock):
                msg = originalExecBlock+" already exists, will not overwrite"
                retValue['success'] = False
                retValue['error_msgs'] = msg
                return retValue
            shutil.move(os.path.join(asdm_name,'ExecBlock.xml'),originalExecBlock)
            shutil.copyfile(execBlock_path,os.path.join(asdm_name,'ExecBlock.xml'))
 
        self.res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False)
        # the only table this test cares about is SPECTRAL_WINDOW
        spwName = os.path.join(ms_name,"SPECTRAL_WINDOW")
        if not os.access(spwName,os.F_OK):
            print("%s: Error %s doesn't exist ..." % (testname,spwName))
            retValue['success'] = False
            retValue['error_msgs']=spwName+' does not exist'
        else:
            ok = tblocal.open(spwName)
            if (ok):
                try:
                    winFunCol = tblocal.getcol('SDM_WINDOW_FUNCTION')
                    if not numpy.all(winFunCol==expWinFunCol):
                        retValue['success'] = False
                        msg = "ERROR Unexpected SDM_WINDOW_FUNCTION values when filling "+asdm_name
                        retValue['error_msgs']=msg
                        print("%s:%s" % (testName,msg))
                except:
                     retValue['success'] = False
                     msg = "ERROR getting/testing SDM_WINDOW_FUNCTION column in "+spwName
                     retValue['error_msgs']=msg
                     print("%s:%s" % (testName,msg))

                try:
                    numBinCol = tblocal.getcol('SDM_NUM_BIN')
                    if not numpy.all(numBinCol==expNumBinCol):
                        retValue['success'] = False
                        msg = "ERROR Unexpected SDM_NUM_BIN values when filling "+asdm_name
                        # there may already be messages in error_msgs
                        if len(retValue['error_msgs']>0):
                            retValue['error_msgs']=retValue['error_msgs']+'\n'
                        retValue['error_msgs']=retValue['error_msgs']+msg
                        print("%s:%s" % (testName,msg))
                except:
                    retValue['success'] = False
                    msg = "ERROR getting/testing SDM_NUM_BIN column in "+spwName
                    # there may already be messages in error_msgs
                    if len(retValue['error_msgs'])>0:
                        retValue['error_msgs']=retValue['error_msgs']+'\n'
                    retValue['error_msgs']=retValue['error_msgs']+msg
                    print("%s:%s" % (testName,msg))

                # only test RESOLUTION values if expResCol is not None
                if expResCol is not None:
                    try:
                        resCol = tblocal.getcol('RESOLUTION')
                        # only test first value in each row, assumes all values in a row are equal
                        resCol = resCol[0,:]
                        if not numpy.all(resCol==expResCol):
                            retValue['success'] = False
                            msg = "ERROR Unexpected RESOLUTION values when filling "+asdm_name
                            # there may already be messages in error_msgs
                            if len(retValue['error_msgs']>0):
                                retValue['error_msgs']=retValue['error_msgs']+'\n'
                            retValue['error_msgs']=retValue['error_msgs']+msg
                            print("%s:%s" % (testName,msg))
                    except:
                        retValue['success'] = False
                        msg = "ERROR getting/testing RESOLUTION column in "+spwName
                        # there may already be messages in error_msgs
                        if len(retValue['error_msgs'])>0:
                            retValue['error_msgs']=retValue['error_msgs']+'\n'
                        retValue['error_msgs']=retValue['error_msgs']+msg
                        print("%s:%s" % (testName,msg))

                # finally, check that SDM_CORR_BIT exists and is filled.
                # the expected value should be a string and all values being tested should
                # equal that string.
                try:
                    corrBitCol = tblocal.getcol('SDM_CORR_BIT')
                    if not numpy.all(corrBitCol==expCorrBitVal):
                        retValue['success'] = False
                        msg = "ERROR Unexpected SDM_CORR_BIT values when filling "+asdm_name
                        # there may already be messages in error_msgs
                        if len(retValue['error_msgs'])>0:
                            retValue['error_msgs']=retValue['error_msgs']+'\n'
                        retValue['error_msgs']=retValue['error_msgs']+msg
                        print(testName,":",msg)
                except:
                    retValue['success'] = False
                    msg = "ERROR getting/testing SDM_CORR_BIT column in "+spwName
                    # there may already be messages in error_msgs
                    if len(retValue['error_msgs'])>0:
                        retValue['error_msgs']=retValue['error_msgs']+'\n'
                    retValue['error_msgs']=retValue['error_msgs']+msg
                    print(testName,":",msg)

                tblocal.close()
            else:
                msg = "ERROR opening",spwName
                retValue['success'] = False
                retValue['error_msgs'] = msg
                print("%s:%s" % (testName,msg))

        if originalSpWin is not None:
            os.remove(os.path.join(asdm_name,'SpectralWindow.xml'))
            shutil.move(originalSpWin,os.path.join(asdm_name,'SpectralWindow.xml'))
            print("%s: restored original SpectralWindow.xml" % testName)

        if originalExecBlock is not None:
            os.remove(asdm_name+'/ExecBlock.xml')
            shutil.move(originalExecBlock,os.path.join(asdm_name,'ExecBlock.xml'))
            print("%s: restored original ExecBlock.xml" % testName)

        return retValue 

    def test_alma_numbin(self):
        retValue = {'success': True, 'error_msgs': '' } 

        # original SpectralWindow.xml and inferred numBin  values
        asdm_name = 'alma_numbin_mixed'
        ms_name = asdm_name+".ms"
        expWinFunCol = numpy.array(['UNIFORM']*5 + ['HANNING']*8 + ['UNIFORM']*4 + ['HANNING']*18 + ['UNIFORM']*42)
        # expected values, 8 @ 24, 2 @ 27,29,31,33, rest are 1
        expNumBinCol = numpy.ones(77,dtype=numpy.int32)
        expNumBinCol[25] = 8
        for indx in [27,29,31,33]:
            expNumBinCol[indx] = 2
        # SDM_CORR_BIT is all 'UNKNOWN' for the alma ASDMs here
        expCorrBitVal = 'UNKNOWN'
        res = self.doNumTest(myname,asdm_name,ms_name,None,None,expWinFunCol,expNumBinCol,None,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # SpectralWindow.xml with appropriate numBin values, should yield same column values
        ms_name = asdm_name+".numbin.ms"
        res = self.doNumTest(myname,asdm_name,ms_name,'SpectralWindow.xml.numBin',None,expWinFunCol,expNumBinCol,None,expCorrBitVal)
        retValue['success'] = retValue['success'] and res['success']
        retValue['error_msgs'] = retValue['error_msgs'] + res['error_msgs']

        # SpectralWindow.xml with faked resolution and expectedBw values but no numBin, tests other inferred numBin values
        expNumBinCol[5] = 4
        expNumBinCol[7] = 16
        ms_name = asdm_name+".faked.ms"
        res = self.doNumTest(myname,asdm_name,ms_name,'SpectralWindow.xml.faked',None,expWinFunCol,expNumBinCol,None,expCorrBitVal)
        retValue['success'] = retValue['success'] and res['success']
        retValue['error_msgs'] = retValue['error_msgs'] + res['error_msgs']

        # SpectralWindow.xml with faked resolution and expectedBw values and added numBin values, same expected values as previous test
        ms_name = asdm_name+".faked.numBin.ms"
        res = self.doNumTest(myname,asdm_name,ms_name,'SpectralWindow.xml.faked.numBin',None,expWinFunCol,expNumBinCol,None,expCorrBitVal)
        retValue['success'] = retValue['success'] and res['success']
        retValue['error_msgs'] = retValue['error_msgs'] + res['error_msgs']

        self.assertTrue(retValue['success'],retValue['error_msgs'])

    def test_evla_numbin(self):
        retValue = {'success':True, 'error_msgs':''}

        # numbin=2 related tests
        sdm_name = 'evla_numbin_2'

        # original SpectralWindow.xml and inferred numBin values, all equal to 2
        expWinFunCol = numpy.array(['UNIFORM']*16)
        expNumBinCol = numpy.empty(16,dtype=numpy.int32)
        expNumBinCol.fill(2)
        # also should alter resolution to these expected values
        expResCol = numpy.empty(16)
        expResCol.fill(4000000.)
        # SDM_CORR_BIT is all 'BITS_4x4' for the evla ASDMs here
        expCorrBitVal = 'BITS_4x4'
        ms_name = sdm_name+".ms"
        res = self.doNumTest(myname,sdm_name,ms_name,None,None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # SpectralWindow.xml with numBin field and appropriately modified resolution, same expected values
        ms_name = sdm_name+".numBin.ms"
        res = self.doNumTest(myname,sdm_name,ms_name,'SpectralWindow.xml.numBin',None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # SpectralWindow.xml with mostly numBin and alterned resolution, but one row has the original values, same expected values
        ms_name = sdm_name+".mixed.ms"
        res = self.doNumTest(myname,sdm_name,ms_name,'SpectralWindow.xml.mixed',None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # original SpectralWindow.xml but with one non-physical (bad) value of resolution leading to the algorithm giving up and numBin=1 and resolution the original bad value
        ms_name = sdm_name+".bad.ms"
        expNumBinCol[0] = 1
        expResCol[0] = 9000000.
        res = self.doNumTest(myname,sdm_name,ms_name,'SpectralWindow.xml.bad',None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # numbin=4 related tests
        sdm_name = 'evla_numbin_4'
        
        # original SpectralWindow.xml and inferred numBin values, all equal to 4
        expWinFunCol = numpy.array(['UNIFORM']*16)
        expNumBinCol = numpy.empty(16,dtype=numpy.int32)
        expNumBinCol.fill(4)
        # also should alter resolution to these expected values
        expResCol = numpy.empty(16)
        expResCol.fill(8000000.)
        ms_name = sdm_name+".ms"
        res = self.doNumTest(myname,sdm_name,ms_name,None,None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # original SpectralWindow.xml with numBin field and altered resolution, same expected values
        ms_name = sdm_name+".numBin.ms"
        res = self.doNumTest(myname,sdm_name,ms_name,'SpectralWindow.xml.numBin',None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # original SpectralWindow.xml with numBin field and original resolution
        # expected numBin is the same, expected resolution is now the original values
        expResCol /= 4.0
        ms_name = sdm_name+".onlyNumBin.ms"
        res = self.doNumTest(myname,sdm_name,ms_name,'SpectralWindow.xml.onlyNumBin',None,expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        # original SpectralWindow.xml and altered ExecBlock so that the telescope is UNKNOWN
        # numBin is all 1 and expected resolution is the original resolution
        expNumBinCol.fill(1)
        ms_name = sdm_name+".unknownTel.ms"
        res = self.doNumTest(myname,sdm_name,ms_name,None,'ExecBlock.xml.unknownTel',expWinFunCol,expNumBinCol,expResCol,expCorrBitVal)
        retValue['success'] = res['success']
        retValue['error_msgs'] = res['error_msgs']

        self.assertTrue(retValue['success'],retValue['error_msgs'])

    def test_sdm_feed(self):
        '''importasdm: Test filling of SDM columns in FEED table'''
        
        # original Feed.xml
        # alma_numbin_mixed is used because it has 2 frequencyBand values in Receiver.xml : "ALMA_RB_06" and "UNSPECIFIED"
        # and alma_numbin_mixed already includes alternative tables created for testing purposes
        
        asdm_name = 'alma_numbin_mixed'
        ms_name = asdm_name+".ms"

        # first, fill this as is
        res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False, overwrite=True)

        # the only values being tested here are in the FEED table
        feedName = os.path.join(ms_name,"FEED")
        ok = tblocal.open(feedName)
        self.assertTrue(ok, "ERROR Unable to open FEED table after as-is fill in test_sdm_feed")

        # expect 3192 rows
        self.assertTrue(tblocal.nrows() == 3192, "ERROR Unexpected number of rows in FEED table for as-is fill in test_sdm_feed")

        # rcvrGenCol should all be -1
        try:
            rcvrGenCol = tblocal.getcol('SDM_RECEIVER_GENERATION')
            self.assertTrue(numpy.all(rcvrGenCol==-1), "ERROR Unexpected SDM_RECEIVER_GENERATION values in FEED table for as-is fill in test_sdm_feed")
        except:
            self.fail("ERROR unexpected exception getting/testing SDM_RECEIVER_GENERATION column in FEED table for as-is fill in test_sdm_feed")

        # frequencyBand is ALMA_RB_06 for SW <= 34 and UNSPECIFIED for SW > 34
        # shape is (2,3192)
        # (0,:) == (1,:) for all rows
        try:
            spwidCol = tblocal.getcol('SPECTRAL_WINDOW_ID')
            freqBandCol = tblocal.getcol('SDM_FREQUENCY_BAND')
            self.assertTrue(freqBandCol.shape==(2,3192),"ERROR Unexpected shape for SDM_FREQUENCY_BAND column for FEED table for as-is fill in test_sdm_feed")
            self.assertTrue(numpy.all(freqBandCol[0,:]==freqBandCol[1,:]), "ERROR Values for the two receptors in SDM_FREQUENCY_BAND column do not agree with each other for all rows as expected for as-is fill in test_sdm_feed.")
            rows = numpy.arange(3192)
            self.assertTrue(numpy.all(freqBandCol[:,rows[spwidCol<=34]] == "ALMA_RB_06"), "ERROR Expected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID <= 34 for as-is fill in test_sdm_feed")
            self.assertTrue(numpy.all(freqBandCol[:,rows[spwidCol>34]] == "UNSPECIFIED"), "ERROR Expected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID > 34 for as-is fill in test_sdm_feed")
        except:
            self.fail("ERROR unexpected exception getting/testing SDM_FREQUENCY_BAND column from FEED table for as-is fill in test_sdm_feed")

        # test using Receiver.xml.faked.bandname : change the frequencyBand to something else for one spectral window
        # copy in the edited Receiver.xml to add ALMA_RB_07 at SpectralWindow_47 == SPECTRAL_WINDOW_ID of 5
        
        print("Using Receiver.xml.faked,bandname for Receiver.xml");
        fakedRcvr_path = os.path.join(asdm_name, "Receiver.xml.faked.bandname")
        self.assertTrue(os.path.exists(fakedRcvr_path), fakedRcvr_path + " does not exist")
        originalRcvr_path = os.path.join(asdm_name, "Receiver.xml.original")
        self.assertFalse(os.path.exists(originalRcvr_path), originalRcvr_path + " exists, will not overwrite")
        rcvrPath = os.path.join(asdm_name, "Receiver.xml")
        shutil.move(rcvrPath, originalRcvr_path)
        shutil.copy(fakedRcvr_path, rcvrPath)

        # the assertion with error message is deferred to tne end of this block so that at attempt can be made to restore the Receiver.xml file no matter what
        isOK = True
        errMsg = ""
        
        # fill using the faked Receiver table now in place
        res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False, overwrite=True)

        # the only values being tested here are in the FEED table
        feedName = os.path.join(ms_name,"FEED")
        isOK = tblocal.open(feedName)
        if not isOK:
            errMsg = "ERROR Unable to open FEED table after faked Receiver table 1 fill in test_sdm_feed"
        else:
            # expect 3192 rows
            isOK = tblocal.nrows() == 3192
            if not isOK:
                errMsg =  "ERROR Unexpected number of rows in FEED table for faked Receiver table 1 fill in test_sdm_feed"

        if isOK:
            # rcvrGenCol should all be -1
            try:
                rcvrGenCol = tblocal.getcol('SDM_RECEIVER_GENERATION')
                isOK = numpy.all(rcvrGenCol==-1)
                if not isOK:
                    errMsg = "ERROR Unexpected SDM_RECEIVER_GENERATION values in FEED table for faked Receiver table 1 fill in test_sdm_feed"
                    
            except:
                isOK = False
                errMsg = "ERROR Unexpected exception getting/testing SDM_RECEIVER_GENERATION column in FEED table for faked Receiver table 1 fill in test_sdm_feed"

        if isOK:
            # frequencyBand is ALMA_RB_06 for SW <= 34 except 5. ALMA_RB_07 for SW 5 and  UNSPECIFIED for SW > 34
            # shape is (2,3192)
            # (0,:) == (1,:) for all rows
            try:
                spwidCol = tblocal.getcol('SPECTRAL_WINDOW_ID')
                freqBandCol = tblocal.getcol('SDM_FREQUENCY_BAND')
                isOK = freqBandCol.shape==(2,3192)
                if not isOK:
                    errMsg = "ERROR Unexpected shape for SDM_FREQUENCY_BAND column for FEED table for faked Receiver table 1 fill in test_sdm_feed"
                else:
                    isOK = numpy.all(freqBandCol[0,:]==freqBandCol[1,:])
                    if not isOK:
                        errMsg = "ERROR Values for the two receptors in SDM_FREQUENCY_BAND column do not agree with each other for all rows as expected for faked Receiver table 1 fill in test_sdm_feed."
                if isOK:
                    rows = numpy.arange(3192)
                    isOK = numpy.all(freqBandCol[:,rows[(spwidCol<=34) & (spwidCol!=5)]] == "ALMA_RB_06")
                    # isOK = numpy.all(freqBandCol[:,rows[spwidCol<=34]] == "ALMA_RB_06")
                    if not isOK:
                        errMsg = "ERROR Unexpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID <= 34 except SPW 5 for faked Receiver table 1 fill in test_sdm_feed"
                    else:
                        isOK = numpy.all(freqBandCol[:,rows[spwidCol==5]] == "ALMA_RB_07")
                        if not isOK:
                            errMsg = "ERROR Unxpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID == 5"
                        else:
                            isOK = numpy.all(freqBandCol[:,rows[spwidCol>34]] == "UNSPECIFIED")
                            if not isOK:
                                errMsg = "ERROR Unexpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID > 34 for faked Receiver table 1 fill in test_sdm_feed"
            except:
                isOK = False
                errMsg = "ERROR unexpected exception getting/testing SDM_FREQUENCY_BAND column from FEED table for faked Receiver table 1 fill in test_sdm_feed"

        # first, restore the Receiver table to it's original state
        os.remove(rcvrPath)
        shutil.move(originalRcvr_path, rcvrPath)

        self.assertTrue(isOK, errMsg)
        
        # test using Receiver.xml.faked.bandname : change the frequencyBand to something else for one spectral window
        # test using Feed.xml.faked.rcvr_id which changes the rcvr_id to 1 for the spwid 47, which has the original frequencyBand value in the faked receiver file
        # only doing this for antennaId == Antenna_2 which is DA43 which is ANTENNA_ID 2
        # so the two SDM_FREQUENCY_BAND values for that spectral window ID and antenna will be different

        # copy in the edited Receiver.xml to add ALMA_RB_07 at SpectralWindow_47 (SPECTRAL_WINDOW_ID == 5)
        print("Using Receiver.xml.faked,bandname for Receiver.xml");
        fakedRcvr_path = os.path.join(asdm_name, "Receiver.xml.faked.bandname")
        self.assertTrue(os.path.exists(fakedRcvr_path), fakedRcvr_path + " does not exist")
        originalRcvr_path = os.path.join(asdm_name, "Receiver.xml.original")
        self.assertFalse(os.path.exists(originalRcvr_path), originalRcvr_path + " exists, will not overwrite")
        rcvrPath = os.path.join(asdm_name, "Receiver.xml")
        shutil.move(rcvrPath, originalRcvr_path)
        shutil.copy(fakedRcvr_path, rcvrPath)
        
        # copy in the edited Feed.xml to set recevierId to be 1 2 0 1 for one row (SpectralWindow_47 and Antenna_1)
        print("Using Feed.xml.faked,rcvr_id for Feed.xml");
        fakedFeed_path = os.path.join(asdm_name, "Feed.xml.faked.rcvr_id")
        self.assertTrue(os.path.exists(fakedFeed_path), fakedFeed_path + " does not exist")
        originalFeed_path = os.path.join(asdm_name, "Feed.xml.original")
        self.assertFalse(os.path.exists(originalFeed_path), originalFeed_path + " exists, will not overwrite")
        feedPath = os.path.join(asdm_name, "Feed.xml")
        shutil.move(feedPath, originalFeed_path)
        shutil.copy(fakedFeed_path, feedPath)
        
        # the assertion with error message is deferred to tne end of this block so that at attempt can be made to restore the Receiver.xml file no matter what
        isOK = True
        errMsg = ""
        
        # fill using the faked Receiver and Feed tables now in place
        res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False, overwrite=True)

        # the only values being tested here are in the FEED table
        feedName = os.path.join(ms_name,"FEED")
        isOK = tblocal.open(feedName)
        if not isOK:
            errMsg = "ERROR Unable to open FEED table after faked Receiver and Feed used in test_sdm_feed"
        else:
            # expect 3192 rows
            isOK = tblocal.nrows() == 3192
            if not isOK:
                errMsg =  "ERROR Unexpected number of rows in FEED table for faked Receiver and Feed used in test_sdm_feed"

        if isOK:
            # rcvrGenCol should all be -1
            try:
                rcvrGenCol = tblocal.getcol('SDM_RECEIVER_GENERATION')
                isOK = numpy.all(rcvrGenCol==-1)
                if not isOK:
                    errMsg = "ERROR Unexpected SDM_RECEIVER_GENERATION values in FEED table for faked Receiver and Feed used in test_sdm_feed"
                    
            except:
                isOK = False
                errMsg = "ERROR Unexpected exception getting/testing SDM_RECEIVER_GENERATION column in FEED table for faked Receiver and Feed used in test_sdm_feed"

        if isOK:
            # frequencyBand is ALMA_RB_06 for SW <= 34 except 5. ALMA_RB_07 for SW 5 and  UNSPECIFIED for SW > 34
            # shape is (2,3192)
            # (0,:) == (1,:) for all rows except where ANTENNA_ID==2 and SPECTRAL_WINDOW_ID==5 and they should have different values for those two
            try:
                spwidCol = tblocal.getcol('SPECTRAL_WINDOW_ID')
                freqBandCol = tblocal.getcol('SDM_FREQUENCY_BAND')
                antidCol = tblocal.getcol('ANTENNA_ID')
                isOK = freqBandCol.shape==(2,3192)
                if not isOK:
                    errMsg = "ERROR Unexpected shape for SDM_FREQUENCY_BAND column for FEED table for faked Receiver and Feed used in test_sdm_feed"
                else:
                    mask = (antidCol==2) & (spwidCol==5) 
                    isOK = numpy.all(freqBandCol[0,~mask]==freqBandCol[1,~mask])
                    isOK = isOK and numpy.all(freqBandCol[0,mask]!=freqBandCol[1,mask])
                    if not isOK:
                        errMsg = "ERROR Values for the two receptors in SDM_FREQUENCY_BAND column do not have the expected values for faked Receiver and Feed used in test_sdm_feed."
                if isOK:
                    rows = numpy.arange(3192)
                    isOK = numpy.all(freqBandCol[:,rows[(spwidCol<=34) & (spwidCol!=5)]] == "ALMA_RB_06")
                    # isOK = numpy.all(freqBandCol[:,rows[spwidCol<=34]] == "ALMA_RB_06")
                    if not isOK:
                        errMsg = "ERROR Unexpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID <= 34 except SPW 5 for faked Receiver and Feed used in test_sdm_feed"
                    else:
                        # ALMA_RB_07 for spw 5 and antid != 2
                        # and for spw5 and antid 2 it's ALMA_RB_07 for receptor 0 and ALMA_RB_06 for receptor 1
                        spw5mask = spwidCol==5
                        ant2mask = antidCol==2
    
                        isOK = numpy.all(freqBandCol[:,rows[spw5mask & ~ant2mask]] == "ALMA_RB_07")
                        isOK = isOK and numpy.all(freqBandCol[0,rows[spw5mask & ant2mask]] == "ALMA_RB_07")
                        isOK = isOK and numpy.all(freqBandCol[1,rows[spw5mask & ant2mask]] == "ALMA_RB_06")
                        if not isOK:
                            errMsg = "ERROR Unxpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID == 5 for faked Receiver and Feed used in test_sdm_feed"
                        else:
                            isOK = numpy.all(freqBandCol[:,rows[spwidCol>34]] == "UNSPECIFIED")
                            if not isOK:
                                errMsg = "ERROR Unexpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID > 34 for faked Receiver and Feed used in test_sdm_feed"
            except Exception as exc:
                isOK = False
                print(exc)
                errMsg = "ERROR unexpected exception getting/testing SDM_FREQUENCY_BAND column from FEED table for faked Receiver and Feed used in test_sdm_feed"

        # restore the Receiver table to it's original state
        os.remove(rcvrPath)
        shutil.move(originalRcvr_path, rcvrPath)

        # restore the Feed table to it's original state
        os.remove(feedPath)
        shutil.move(originalFeed_path, feedPath)

        self.assertTrue(isOK, errMsg)
        
        # test using Feed.xml.faked.rcvr_gen which adds a receiverGeneration value of 1 to the first row and 2 to the second row, all others are unchanged

        # copy in the edited Feed.xml
        print("Using Feed.xml.faked,rcvr_gen for Feed.xml");
        fakedFeed_path = os.path.join(asdm_name, "Feed.xml.faked.rcvr_gen")
        self.assertTrue(os.path.exists(fakedFeed_path), fakedFeed_path + " does not exist")
        originalFeed_path = os.path.join(asdm_name, "Feed.xml.original")
        self.assertFalse(os.path.exists(originalFeed_path), originalFeed_path + " exists, will not overwrite")
        feedPath = os.path.join(asdm_name, "Feed.xml")
        shutil.move(feedPath, originalFeed_path)
        shutil.copy(fakedFeed_path, feedPath)
        
        # the assertion with error message is deferred to tne end of this block so that at attempt can be made to restore the Receiver.xml file no matter what
        isOK = True
        errMsg = ""
        
        # fill using the faked Receiver and Feed tables now in place
        res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False, overwrite=True)

        # the only values being tested here are in the FEED table
        feedName = os.path.join(ms_name,"FEED")
        isOK = tblocal.open(feedName)
        if not isOK:
            errMsg = "ERROR Unable to open FEED table after faked receiverGeneration used in test_sdm_feed"
        else:
            # expect 3192 rows
            isOK = tblocal.nrows() == 3192
            if not isOK:
                errMsg =  "ERROR Unexpected number of rows in FEED table for faked receiverGeneration used in test_sdm_feed"

        if isOK:
            # rcvrGenCol should all be -1
            try:
                rcvrGenCol = tblocal.getcol('SDM_RECEIVER_GENERATION')
                isOK = rcvrGenCol[0] == 1
                isOK = isOK and rcvrGenCol[1] == 2
                isOK = isOK and numpy.all(rcvrGenCol[2:]==-1)
                if not isOK:
                    errMsg = "ERROR Unexpected SDM_RECEIVER_GENERATION values in FEED table for faked receiverGeneration used in test_sdm_feed"
                    
            except:
                isOK = False
                errMsg = "ERROR Unexpected exception getting/testing SDM_RECEIVER_GENERATION column in FEED table for faked receiverGeneration used in test_sdm_feed"

        if isOK:
            # frequencyBand is ALMA_RB_06 for SW <= 34 and UNSPECIFIED for SW > 34
            # shape is (2,3192)
            # (0,:) == (1,:) for all rows 
            try:
                freqBandCol = tblocal.getcol('SDM_FREQUENCY_BAND')
                isOK = freqBandCol.shape==(2,3192)
                if not isOK:
                    errMsg = "ERROR Unexpected shape for SDM_FREQUENCY_BAND column for FEED table for faked receiverGeneration used in test_sdm_feed"
                else:
                    isOK = numpy.all(freqBandCol[0,:] == freqBandCol[1,:])
                    if not isOK:
                        errMsg = "ERROR Values for the two receptors in SDM_FREQUENCY_BAND column do not have the same values for faked receiverGeneration used in test_sdm_feed."
                if isOK:
                    rows = numpy.arange(3192)
                    isOK = numpy.all(freqBandCol[:,rows[(spwidCol<=34)]] == "ALMA_RB_06")
                    if not isOK:
                        errMsg = "ERROR Unexpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID <= 34 for faked receiverGeneration used in test_sdm_feed"
                    else:
                        isOK = numpy.all(freqBandCol[:,rows[spwidCol>34]] == "UNSPECIFIED")
                        if not isOK:
                            errMsg = "ERROR Unexpected value(s) for SDM_FREQUENCY_BAND for SPECTRAL_WINDOW_ID > 34 for faked receiverGeneration used in test_sdm_feed"
            except:
                isOK = False
                errMsg = "ERROR unexpected exception getting/testing SDM_FREQUENCY_BAND column from FEED table for faked receiverGeneration used in test_sdm_feed"

        # restore the Feed table to it's original state
        os.remove(feedPath)
        shutil.move(originalFeed_path, feedPath)

        self.assertTrue(isOK, errMsg)

    def checkVarColVals(self, colname, valList):
        # using the table currently open in tblocal extract the values from the colname column
        # one cell at a time and check against the values in valList.
        # valList is a list of tuples where the first element of the tuple is a row number and
        # the second is a list of values that should be equal to the values in that row.
        # For all rows not named in valList the cell value for that row should be an empty array.
        # The values in this column can be assumed to be arrays.
        # returns a string that is empty on success and contains an error message on failure.

        usedRows = []
        for rownr, rowval in valList:
            try:
                thisval = tblocal.getcell(colname, rownr)
                if not (numpy.all(thisval == rowval)):
                    return("Unexpected value for %s in row %s" % (colname, rownr))
                usedRows.append(rownr)
            except Exception:
                return("Unexpected exception trying to check expected value from %s in row %s" % (colname, rownr))

        for rownr in range(tblocal.nrows()):
            if rownr not in usedRows:
                try:
                    if (tblocal.getcell(colname,rownr)).size != 0:
                        return("Unexpected non-empty value for %s in row %s" % (colname, rownr))
                except:
                    return("Unexpected exception tring to check expected empty value from %s for row %s" % (colname, rownr))
                
        return("")
    
    def test_sdm_spectral_window(self):
        '''importasdm: Test filling of optional SDM columns in SPECTRAL_WINDOW table'''
        
        # original SpectralWindow.xml
        # and alma_numbin_mixed already includes alternative tables created for testing purposes
        
        asdm_name = 'alma_numbin_mixed'
        ms_name = asdm_name+".ms"

        # first, fill this as is
        res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False, overwrite=True)

        # the only values being tested here are in the SPECTRAL_WINDOW table
        swName = os.path.join(ms_name,"SPECTRAL_WINDOW")
        ok = tblocal.open(swName)
        self.assertTrue(ok, "ERROR Unable to open SPECTRAL_WINDOW table after as-is fill in test_sdm_spectral_window")

        # expect 77 rows
        self.assertTrue(tblocal.nrows() == 77, "ERROR Unexpected number of rows in SPECTRAL_WINDOW table for as-is fill in test_sdm_spectral_window")

        # it should have SDM_QUANTIZATION
        # and none of SDM_ORIGINATING_FREQUENCY_SLICES, SDM_NUM_BIN_FACTORS, SDM_QUANTIZATION_BITS, SDM_REQUANTIZATION, SDM_REQUANTIZATION_BITS
        colNames = tblocal.colnames()
        self.assertTrue("SDM_QUANTIZATION" in colNames, "ERROR SDM_QUANTIZATION not found as expected in SPECTRAL_WINDOW table of as-is fill in test_sdm_spectral_window")
        for thisColName in ["SDM_ORIGINATING_FREQUENCY_SLICES", "SDM_NUM_BIN_FACTORS", "SDM_QUANTIZATION_BITS", "SDM_REQUANTIZATION", "SDM_REQUANTIZATION_BITS"]:
            self.assertTrue(thisColName not in colNames, "ERROR Unexpected presence of %s in SPECTRAL_WINDOW table of as-is fill in test_sdm_spectral_window")
            
        # SDM_QUANTIZATION is true only for the odd elements in [5:12] and [17:34]
        quantCheck = numpy.full(tblocal.nrows(),False)
        quantCheck[5:12:2] = True
        quantCheck[17:34:2] = True
        self.assertTrue(numpy.all(quantCheck == tblocal.getcol("SDM_QUANTIZATION")), "Unexpected values of SDM_QUANTIZATION in SPECTRAL_WINDOW table of as-is fill in test_sdm_spectral_window")

        # the fake SpectralWindow.xml is used, various additional fields in a few rows to test that they end up in the right place
        
        # copy in the edited SpectralWindow.xml
        print("Using SpectralWindow.xml.faked,sdm_cols for SpectralWindow.xml");
        fakedSW_path = os.path.join(asdm_name, "SpectralWindow.xml.faked.sdm_cols")
        self.assertTrue(os.path.exists(fakedSW_path), fakedSW_path + " does not exist")
        originalSW_path = os.path.join(asdm_name, "SpectralWindow.xml.original")
        self.assertFalse(os.path.exists(originalSW_path), originalSW_path + " exists, will not overwrite")
        swPath = os.path.join(asdm_name, "SpectralWindow.xml")
        shutil.move(swPath, originalSW_path)
        shutil.copy(fakedSW_path, swPath)
        
        # the assertion with error message is deferred to tne end of this block so that at attempt can be made to restore the SpectralWindow.xml file no matter what
        isOK = True
        errMsg = ""

        res = importasdm(asdm_name, vis=ms_name, lazy=True, process_syspower=False, process_caldevice=False, process_pointing=False, process_flags=False, overwrite=True)

        # the only values being tested here are in the SPECTRAL_WINDOW table
        swName = os.path.join(ms_name,"SPECTRAL_WINDOW")
        isOK = tblocal.open(swName)
        if not isOK:
            errMsg = "ERROR Unable to open SPECTRAL_WINDOW table after as-is fill in test_sdm_spectral_window"

        # expect 77 rows
        isOK = isOK and (tblocal.nrows() == 77)
        if not isOK:
            errMsg = "ERROR Unexpected number of rows in SPECTRAL_WINDOW table for as-is fill in test_sdm_spectral_window"

        # it should have all of the optional columns
        if isOK:
            colNames = tblocal.colnames()
            for thisColName in ["SDM_ORIGINATING_FREQUENCY_SLICES", "SDM_NUM_BIN_FACTORS", "SDM_QUANTIZATION", "SDM_QUANTIZATION_BITS", "SDM_REQUANTIZATION", "SDM_REQUANTIZATION_BITS"]:
                isOK = (thisColName in colNames)
                if not isOK:
                    errMsg = "ERROR Unexpected presence of %s in SPECTRAL_WINDOW table of as-is fill in test_sdm_spectral_window"
                    break

        # test the individual values where expected
        # SDM_QUANTIZATION and SDM_QUANTIZATION_BITS can be tested on the entire column
        # SDM_QUANTIZATION is True for the odd rows in [5:12] and [17:36] else False
        # SDM_QUANTIZATION_BITS is 3 for the odd rows in [5:12} and [35] and 6 for the odd rows is [17:34] else -1 (where SDM_QUANTIZATION is False)

        if isOK:
            quantCol = tblocal.getcol("SDM_QUANTIZATION")
            quantBitsCol = tblocal.getcol("SDM_QUANTIZATION_BITS")
            quantExp = numpy.full(tblocal.nrows(),False)
            quantExp[5:12:2] = True
            quantExp[17:36:2] = True
            isOK = numpy.all(quantExp == quantCol)
            if not isOK:
                errMsg = "ERROR Unexpected SDM_QUANTIZATION values in SPECTRAL_WINDOW table for faked SpectralWindow.xml in test_sdm_spectral_window"
            else:
                isOK = numpy.all(quantBitsCol[5:12:2]==3)
                isOK = isOK and numpy.all(quantBitsCol[17:34:2]==6)
                isOK = isOK and (quantBitsCol[35]==3)
                if not isOK:
                    errMsg = "ERROR Unpected SDM_QUANTIZATION_BITS values in SPECTRAL_WINDOW table for faked SpectralWindow.xml in test_sdm_spectral_window"

        # test the variable shaped columns
        if isOK:
            expectedOrigFreqSlices = [(2,[100,150]), (3,[110,160,200,210]), (35,[10,20,30])]
            checkRes = self.checkVarColVals("SDM_ORIGINATING_FREQUENCY_SLICES", expectedOrigFreqSlices)
            isOK = len(checkRes) == 0
            if not isOK:
                errMsg = "ERROR %s for faked SpectralWindow.xml in test_sdm_spectral_window" % checkRes

        if isOK:
            expectedNumBinFactors = [(1,[2]), (2,[3,4,5]), (35,[0,1])]
            checkRes = self.checkVarColVals("SDM_NUM_BIN_FACTORS", expectedNumBinFactors)
            isOK = len(checkRes) == 0
            if not isOK:
                errMsg = "ERROR %s for faked SpectralWindow.xml in test_sdm_spectral_window" % checkRes

        if isOK:
            expectedRequantization = [(3,[True,True,False,False]), (4,[False,True,False,True,False]), (35,[True, True, True, False, False, False])]
            checkRes = self.checkVarColVals("SDM_REQUANTIZATION", expectedRequantization)
            isOK + len(checkRes) == 0
            if not isOK:
                errMsg = "ERROR %s for faked SpectralWindow.xml in test_sdm_spectral_window" % checkRes

        if isOK:
            expectedRequantizationBits = [(3,[1,3,5,7]), (4,[2,4,6,8,10]), (35,[3,6,3,1,0,-1])]
            checkRes = self.checkVarColVals("SDM_REQUANTIZATION_BITS", expectedRequantizationBits)
            isOK = len(checkRes) == 0
            if not isOK:
                errMsg = "ERROR %s for faked SpectralWindow.xml in test_sdm_spectral_window" %checkRes
            
        # restore the SpectralWindow table to it's original state
        os.remove(swPath)
        shutil.move(originalSW_path, swPath)
        
        self.assertTrue(isOK, errMsg)

    def test_verbose(self):
        # check that the number of lines added to the log file by importasdm is more with verbose True than False
        # any ASDM is fine

        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname+".ms"

        # count log file lines at start
        logLinesStart = countlines(casalog.logfile())
        importasdm(myasdmname, vis=themsname, ocorr_mode="ao", bdfflags=True, applyflags=True, lazy=True, overwrite=True,verbose=False)
        logLinesMiddle = countlines(casalog.logfile())
        importasdm(myasdmname, vis=themsname, ocorr_mode="ao", bdfflags=True, applyflags=True, lazy=True, overwrite=True,verbose=True)
        logLinesEnd = countlines(casalog.logfile())

        self.assertTrue(((logLinesEnd-logLinesMiddle) > (logLinesMiddle-logLinesStart)), 'verbose test failed, did not produce more lines')

class asdm_import_asis(test_base):

    def setUp(self):
        self.setUp_12mex()
        self.setUp_12mex_bin()
        
    def tearDown(self):
        for myasdmname in ['uid___A002_X71e4ae_X317_short', 'uid___A002_X71e4ae_X317_short_bin']:
            os.unlink(myasdmname)
            shutil.rmtree(myasdmname+".ms",ignore_errors=True)

    def test_asis_caltables(self):
        '''Asdm-import: Test importing good 12 m ASDM with asis parameter for some Cal Tables'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname+".ms"

        asisParam = 'CalData CalDevice CalFlux CalPointing CalSeeing Subscan'
        self.res = importasdm(myasdmname, vis=themsname, asis=asisParam, flagbackup=False) 
        self.assertEqual(self.res, None)
        print("Successful importing! Checking ASIS tables in %s" % themsname)
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALDATA'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALDEVICE'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALFLUX'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALPOINTING'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALSEEING'))

        # importasdm already uses CalDevice, compare with what asis produces for the same table
        tblocal.open(themsname+'/CALDEVICE')
        nrow_caldevice = tblocal.nrows()
        tblocal.close()
        tblocal.open(themsname+'/ASDM_CALDEVICE')
        nrow_asdm_caldevice = tblocal.nrows()
        tblocal.close()
        self.assertEqual(nrow_caldevice,nrow_asdm_caldevice,'CalDevice table imported asis is not equal to CALDEVICE table')

        # check that each asis table has the same number of rows in the output MS as in the XML file
        from xml.dom import minidom
        
        for tabName in asisParam.split():
            
            tabName = tabName.strip()
            xmlTabName = tabName + '.xml'
            msTabName = 'ASDM_'+tabName.upper()

            tblocal.open(themsname+'/'+msTabName)
            nrow_ms_tab = tblocal.nrows()
            tblocal.close()

            xmlTabParsed = minidom.parse(myasdmname+'/'+xmlTabName)
            rowlist = xmlTabParsed.getElementsByTagName('row')

            self.assertEqual(nrow_ms_tab, len(rowlist), '%s table imported asis does not match the nrows in the XML' % tabName)

    def test_asis_bin_caltables(self):
        '''Asdm-import: Test importing good 12 m ASDM with asis parameter for some Cal Tables in binary format'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X71e4ae_X317_short_bin'
        themsname = myasdmname+".ms"

        # some Cal tables that are in binary format
        # this just tests that they fill without throwing an exception
        
        asisParam = 'CalData CalFlux CalPointing CalSeeing'
        self.res = importasdm(myasdmname, vis=themsname, asis=asisParam, flagbackup=False) 
        self.assertEqual(self.res, None)
        print("Successful importing! Checking ASIS tables in %s" % themsname)
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALDATA'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALFLUX'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALPOINTING'))
        self.assertTrue(os.path.exists(themsname+'/ASDM_CALSEEING'))

class asdm_import_use_durations(test_base):

    def setUp(self):
        # nothing to do, uses the data where it exists so nothing to copy over
        pass

    def tearDown(self):
        # the tests all write to either useDir.on.ms or useDir.off.ms
        for this_asdm_name in ['useDirOn.ms','useDir.off.ms']:
            os.system('rm -rf ' + this_asdm_name+"*")

    def checkTables(self, onMSPath, offMSPath, expectDiffs=True):
        # check that the two MS MAIN tables have the expected difference (or are identical if expectDiffs=True
        # returns a non-empty string with the reason for the failure if the tables are not as expected
        # if expectDiffs is True then verify that the WEIGHT
        # in the on version time (INTERVAL/EXPOSURE) is near to value
        # of the WEIGHT in the off version

        # NOTE : EXPOSURE may be zero, when that happens WEIGHT should be one
        # this works ONLY when the WEIGHT column has 2 dimensions, which it does for this test data
        # (not checked here).

        diffCols = []
        if expectDiffs:
            diffCols = ["EXPOSURE","WEIGHT","SIGMA"]

        # skip because they are not filled or are possibly quite large (DATA)
        skipCols = ["FLAG", "FLAG_CATEGORY","WEIGHT_SPECTRUM","DATA"]

        onTbl = table()
        offTbl = table()

        res = onTbl.open(onMSPath)
        if not res:
            return ("unable to open table : " + onMSPath)
        
        res = offTbl.open(offMSPath)
        if not res:
            return ("unable to open table : " + offMSPath)

        result = ""
        for thisCol in onTbl.colnames():
            if (thisCol not in skipCols):
                onCol = onTbl.getcol(thisCol)
                offCol = offTbl.getcol(thisCol)
                if (thisCol in diffCols):
                    # the columns being equal is an error
                    if numpy.array_equal(onCol, offCol):
                        result += " " + thisCol + " is identical "
                    # EXPOSURE and WEIGHT in the onCol should be <= the offCol
                    if thisCol != "SIGMA":
                        if not numpy.all(onCol <= offCol):
                            result += " " + thisCol + " in the on table is not everywhere <= the valuein the off table"
                else:
                    # the columns not being eequal is an error
                    if not numpy.array_equal(onCol, offCol):
                        result += " " + thisCol + " not identical "

        if expectDiffs:
            onWeight = onTbl.getcol("WEIGHT")
            offWeight = offTbl.getcol("WEIGHT")
            onInterval = onTbl.getcol("INTERVAL")
            onExposure = onTbl.getcol("EXPOSURE")
            expOK = numpy.where(onExposure>0.0)
            expZero = numpy.where(onExposure==0.0)
            if not numpy.allclose(onWeight[:,expOK]*(onInterval[expOK]/onExposure[expOK]), offWeight[:,expOK]):
                result += " WEIGHT column in ON is not scaled by EXPOSURE as expected "
            else:
                if not numpy.all(onWeight[:,expZero] == 1.0):
                    result += " WEIGHT column is not 1.0 when EXPOSURE is 0.0 as expected"

        return result

    def test_no_actual_durations(self):
        # this ASDM does not have any actualDurations and so the resulting MS is identical

        sdmPath = rootpath + "uid___X5f_X18951_X1"
        msNameOn = 'useDir.on.ms'
        msNameOff = 'useDir.off.ms'

        self.res = importasdm(asdm=sdmPath, vis="useDir.on.ms", overwrite=True, use_durations=True)
        self.assertEqual(self.res, None)
        
        self.res = importasdm(asdm=sdmPath, vis="useDir.off.ms", overwrite=True)   # default should be use_durations=False
        self.assertEqual(self.res, None)

        checkValue = self.checkTables(msNameOn, msNameOff, expectDiffs = False)
        self.assertTrue((len(checkValue)==0), checkValue)

    def test_actual_durations(self):
        # fill three available SDMs with actualDurations to test use_durations=True
        # note that the last one is the VLA example data that was given when actualDurations
        # were requested to be used by the tool

        sdmList = ["uid___A002_X72bc38_X000","uid___A002_X6218fb_X264","TRFI0001_sb42534863_1_1.59814.8897497338"]

        for sdmItem in sdmList:
            # print(sdmItem)
            sdmPath = rootpath + "/" + sdmItem
            msNameOn = 'useDir.on.ms'
            msNameOff = 'useDir.off.ms'

            self.res = importasdm(asdm=sdmPath, vis="useDir.on.ms", overwrite=True, use_durations=True)
            self.assertEqual(self.res, None)
            
            self.res = importasdm(asdm=sdmPath, vis="useDir.off.ms", overwrite=True)   # default should be use_durations=False
            self.assertEqual(self.res, None)

            checkValue = self.checkTables(msNameOn, msNameOff, expectDiffs = True)
            if len(checkValue)>0:
                checkValue = "failure checking " + sdmItem + ": " + checkValue
                self.fail(checkValue)

if __name__ == '__main__':
    unittest.main()
