@pytest.mark.seven def test_uid___A002_Xc845c0_X7366__cycle5_restore_procedure_hifa_image__regression(): """ Restore from Cycle 5 (with current pipeline) Recipe name: procedure_hifa_image Dataset: uid___A002_Xc845c0_X7366 """ ref_directory = 'pl-regressiontest/uid___A002_Xc845c0_X7366' pt = PipelineTester( visname=['uid___A002_Xc845c0_X7366'], recipe='procedure_hifa_image.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) # copy files for the restore into products folder if not pt.compare_only: input_products = casa_tools.utils.resolve(f'{ref_directory}/products') shutil.copytree(input_products, f'{pt.output_dir}/rawdata') > pt.run() tests/regression/fast/alma_if_fast_test.py:75: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:356: in run self.__do_sanity_checks(context) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7f821b8c14f0> context = <Context(name='pipeline-procedure_hifa_image')> def __do_sanity_checks(self, context: Context): """ Do the following sanity-checks on the pipeline run 1. rawdata, working, products directories are present 2. Non-existence of errorexit-*.txt in working directory 3. *pipeline_manifest.xml is present under the products directory (regression mode only) 4. No weblog rendering failures Args: context: Pipeline context object from the current run. """ # 1. rawdata, working, products directories are present # The rawdata directory is only present for PPR runs missing_directories = regression.missing_directories(context, include_rawdata=self.ppr) if len(missing_directories) > 0: msg = f"The following directories are missing from the pipeline run: {', '.join(missing_directories)}" LOG.warning(msg) pytest.fail(msg) # 2. Non-existence of errorexit-*.txt in working directory if regression.errorexit_present(context): msg = "errorexit-*.txt is present in working directory" LOG.warning(msg) pytest.fail(msg) # 3. *pipeline_manifest.xml is present under the products directory; regression mode only if self.mode == 'regression': if not regression.manifest_present(context): msg = "pipeline_manifest.xml is not present under the products directory" LOG.warning(msg) > pytest.fail(msg) E Failed: pipeline_manifest.xml is not present under the products directory tests/testing_utils.py:498: Failed@pytest.mark.seven def test_uid___A002_Xc46ab2_X15ae__selfcal_restore_procedure_hifa_image__regression(): """Restore selfcal from Cycle 10 (with current pipeline) Recipe name: procedure_hifa_image Dataset: uid___A002_Xc46ab2_X15ae """ ref_directory = 'pl-regressiontest/uid___A002_Xc46ab2_X15ae_selfcal_restore' pt = PipelineTester( visname=['uid___A002_Xc46ab2_X15ae'], recipe='procedure_hifa_image.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) # copy files into products folder for restore if not pt.compare_only: input_products = casa_tools.utils.resolve(f'{ref_directory}/products') shutil.copytree(input_products, f'{pt.output_dir}/rawdata') > pt.run() tests/regression/fast/alma_if_fast_test.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:356: in run self.__do_sanity_checks(context) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7fd9557d9d60> context = <Context(name='pipeline-procedure_hifa_image')> def __do_sanity_checks(self, context: Context): """ Do the following sanity-checks on the pipeline run 1. rawdata, working, products directories are present 2. Non-existence of errorexit-*.txt in working directory 3. *pipeline_manifest.xml is present under the products directory (regression mode only) 4. No weblog rendering failures Args: context: Pipeline context object from the current run. """ # 1. rawdata, working, products directories are present # The rawdata directory is only present for PPR runs missing_directories = regression.missing_directories(context, include_rawdata=self.ppr) if len(missing_directories) > 0: msg = f"The following directories are missing from the pipeline run: {', '.join(missing_directories)}" LOG.warning(msg) pytest.fail(msg) # 2. Non-existence of errorexit-*.txt in working directory if regression.errorexit_present(context): msg = "errorexit-*.txt is present in working directory" LOG.warning(msg) pytest.fail(msg) # 3. *pipeline_manifest.xml is present under the products directory; regression mode only if self.mode == 'regression': if not regression.manifest_present(context): msg = "pipeline_manifest.xml is not present under the products directory" LOG.warning(msg) > pytest.fail(msg) E Failed: pipeline_manifest.xml is not present under the products directory tests/testing_utils.py:498: Failed@pytest.mark.seven @pytest.mark.mpi def test_2022_1_00207_S__uid___A001_X2d20_X373d__PPR__regression(): """Run ALMA polcal+image regression on a multi-EB 7m test dataset with a PPR file. PPR: pl-regressiontest/2022.1.00207.S/PPR.xml Project: 2022.1.00207.S MOUS: uid___A001_X2d20_X373d EBs: uid___A002_X10b6f7c_X41d1 uid___A002_X10b6f7c_X46cc """ ref_directory = 'pl-regressiontest/2022.1.00207.S' pt = PipelineTester( visname=['uid___A002_X10b6f7c_X41d1', 'uid___A002_X10b6f7c_X46cc'], ppr=f'{ref_directory}/PPR.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) > pt.run() tests/regression/fast/alma_if_fast_test.py:122: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:381: in run self.__compare_results(new_file, default_relative_tolerance) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7f11286bd340> new_file = 'uid___A002_X10b6f7c_X41d1.NEW.results.txt' relative_tolerance = 1e-07 def __compare_results(self, new_file: str, relative_tolerance: float) -> None: """ Compare results between new one loaded from file and old one. Args: new_file : file path of new results relative_tolerance : relative tolerance of output value """ with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd: expected_results = expected_fd.readlines() new_results = new_fd.readlines() errors = [] worst_diff = (0, 0) worst_percent_diff = (0, 0) for old, new in zip(expected_results, new_results): try: oldkey, oldval, tol = self.__sanitize_results_string(old) newkey, newval, _ = self.__sanitize_results_string(new) except ValueError as e: errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e)) errors.append(errorstr) continue assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}." tolerance = tol if tol else relative_tolerance if newval is not None: LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance) if oldval != pytest.approx(newval, rel=tolerance): diff = oldval-newval percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100 if abs(diff) > abs(worst_diff[0]): worst_diff = diff, oldkey if abs(percent_diff) > abs(worst_percent_diff[0]): worst_percent_diff = percent_diff, oldkey errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%" errors.append(errorstr) elif oldval is not None: # If only the new value is None, fail errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}" errors.append(errorstr) else: # If old and new values are both None, this is expected, so pass LOG.info('Comparing %s and %s... both values are None.', oldval, newval) [LOG.warning(x) for x in errors] n_errors = len(errors) if n_errors > 0: summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%" errors.append(summary_str) > pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format( n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True) E Failed: Failed to match 6 result values within tolerances : E s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_10.qa.metric.score_gfluxscale_k_spw E values differ by > a relative difference of 1e-07 E expected: 1.0129262614066323 E new: 1.3799197944259296 E diff: -0.36699353301929727 E percent_diff: -36.23102164511561% E s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_10.qa.score.score_gfluxscale_k_spw E values differ by > a relative difference of 1e-07 E expected: 1.0 E new: 0.5 E diff: 0.5 E percent_diff: 50.0% E s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_4.qa.metric.score_gfluxscale_k_spw E values differ by > a relative difference of 1e-07 E expected: 1.0057307923905467 E new: 1.3701173334731587 E diff: -0.3643865410826119 E percent_diff: -36.23102164511563% E s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_4.qa.score.score_gfluxscale_k_spw E values differ by > a relative difference of 1e-07 E expected: 1.0 E new: 0.5 E diff: 0.5 E percent_diff: 50.0% E s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_8.qa.metric.score_gfluxscale_k_spw E values differ by > a relative difference of 1e-07 E expected: 1.0068059315042126 E new: 1.3715820064718118 E diff: -0.36477607496759923 E percent_diff: -36.231021645115625% E s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_8.qa.score.score_gfluxscale_k_spw E values differ by > a relative difference of 1e-07 E expected: 1.0 E new: 0.5 E diff: 0.5 E percent_diff: 50.0% E Worst absolute diff, s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_10.qa.score.score_gfluxscale_k_spw: 0.5 E Worst percentage diff, s21.hifa_gfluxscale.uid___A002_X10b6f7c_X46cc.field_3.spw_10.qa.score.score_gfluxscale_k_spw: 50.0% tests/testing_utils.py:440: Failed@pytest.mark.seven def test_uid___A002_Xef72bb_X9d29__renorm_restore_procedure_hifa_image__regression(): """Restore renorm from Cycle 8 (with current pipeline) Recipe name: procedure_hifa_image Dataset: uid___A002_Xef72bb_X9d29 """ ref_directory = 'pl-regressiontest/uid___A002_Xef72bb_X9d29' pt = PipelineTester( visname=['uid___A002_Xef72bb_X9d29'], recipe='procedure_hifa_image.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) # copy files into products folder for restore if not pt.compare_only: input_products = casa_tools.utils.resolve(f'{ref_directory}/products') shutil.copytree(input_products, f'{pt.output_dir}/rawdata') > pt.run() tests/regression/fast/alma_if_fast_test.py:50: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:356: in run self.__do_sanity_checks(context) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7fca28be24b0> context = <Context(name='pipeline-procedure_hifa_image')> def __do_sanity_checks(self, context: Context): """ Do the following sanity-checks on the pipeline run 1. rawdata, working, products directories are present 2. Non-existence of errorexit-*.txt in working directory 3. *pipeline_manifest.xml is present under the products directory (regression mode only) 4. No weblog rendering failures Args: context: Pipeline context object from the current run. """ # 1. rawdata, working, products directories are present # The rawdata directory is only present for PPR runs missing_directories = regression.missing_directories(context, include_rawdata=self.ppr) if len(missing_directories) > 0: msg = f"The following directories are missing from the pipeline run: {', '.join(missing_directories)}" LOG.warning(msg) pytest.fail(msg) # 2. Non-existence of errorexit-*.txt in working directory if regression.errorexit_present(context): msg = "errorexit-*.txt is present in working directory" LOG.warning(msg) pytest.fail(msg) # 3. *pipeline_manifest.xml is present under the products directory; regression mode only if self.mode == 'regression': if not regression.manifest_present(context): msg = "pipeline_manifest.xml is not present under the products directory" LOG.warning(msg) > pytest.fail(msg) E Failed: pipeline_manifest.xml is not present under the products directory tests/testing_utils.py:498: Faileddef test_TSKY0001__vlass_quicklook_regression(): """Run VLASS quicklook regression Recipe name: procedure_vlassQLIP.xml Dataset: TSKY0001.sb32295801.eb32296475.57549.31722762731_split_withcorrectdata.ms """ ref_directory = 'pl-regressiontest/vlass_quicklook' pt = PipelineTester( visname=['TSKY0001.sb32295801.eb32296475.57549.31722762731_split_withcorrectdata.ms'], recipe='procedure_vlassQLIP.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) ensure_working_dir(pt) if not pt.compare_only: parameter_list_file = casa_tools.utils.resolve( f'{ref_directory}/TSKY0001.sb32295801.eb32296475.57549.31722762731_split_QLIP_parameter.list') shutil.copyfile(parameter_list_file, casa_tools.utils.resolve(f'{pt.output_dir}/working/QLIP_parameter.list')) > pt.run(telescope='vla') tests/regression/fast/vlass_fast_test.py:27: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:356: in run self.__do_sanity_checks(context) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7f2f4078f5f0> context = <Context(name='pipeline-procedure_vlassQLIP')> def __do_sanity_checks(self, context: Context): """ Do the following sanity-checks on the pipeline run 1. rawdata, working, products directories are present 2. Non-existence of errorexit-*.txt in working directory 3. *pipeline_manifest.xml is present under the products directory (regression mode only) 4. No weblog rendering failures Args: context: Pipeline context object from the current run. """ # 1. rawdata, working, products directories are present # The rawdata directory is only present for PPR runs missing_directories = regression.missing_directories(context, include_rawdata=self.ppr) if len(missing_directories) > 0: msg = f"The following directories are missing from the pipeline run: {', '.join(missing_directories)}" LOG.warning(msg) pytest.fail(msg) # 2. Non-existence of errorexit-*.txt in working directory if regression.errorexit_present(context): msg = "errorexit-*.txt is present in working directory" LOG.warning(msg) pytest.fail(msg) # 3. *pipeline_manifest.xml is present under the products directory; regression mode only if self.mode == 'regression': if not regression.manifest_present(context): msg = "pipeline_manifest.xml is not present under the products directory" LOG.warning(msg) > pytest.fail(msg) E Failed: pipeline_manifest.xml is not present under the products directory tests/testing_utils.py:498: Failed