@pytest.mark.twelve def test_E2E6_1_00010_S__uid___A002_Xd0a588_X2239__procedure_hifa_image__regression(): """Run ALMA cal+image regression on a 12m moderate-size test dataset in ASDM. Recipe name: procedure_hifa_calimage Dataset: E2E6.1.00010.S: uid___A002_Xd0a588_X2239 """ ref_directory = 'pl-regressiontest/E2E6.1.00010.S' pt = PipelineTester( visname=['uid___A002_Xd0a588_X2239'], recipe='procedure_hifa_calimage.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) > pt.run() tests/regression/fast/alma_if_fast_test.py:163: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:376: in run self.__compare_results(new_file, default_relative_tolerance) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7fa621f10f80> new_file = 'uid___A002_Xd0a588_X2239.NEW.results.txt' relative_tolerance = 1e-07 def __compare_results(self, new_file: str, relative_tolerance: float) -> None: """ Compare results between new one loaded from file and old one. Args: new_file : file path of new results relative_tolerance : relative tolerance of output value """ with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd: expected_results = expected_fd.readlines() new_results = new_fd.readlines() errors = [] worst_diff = (0, 0) worst_percent_diff = (0, 0) for old, new in zip(expected_results, new_results): try: oldkey, oldval, tol = self.__sanitize_results_string(old) newkey, newval, _ = self.__sanitize_results_string(new) except ValueError as e: errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e)) errors.append(errorstr) continue assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}." tolerance = tol if tol else relative_tolerance if newval is not None: LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance) if oldval != pytest.approx(newval, rel=tolerance): diff = oldval-newval percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100 if abs(diff) > abs(worst_diff[0]): worst_diff = diff, oldkey if abs(percent_diff) > abs(worst_percent_diff[0]): worst_percent_diff = percent_diff, oldkey errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%" errors.append(errorstr) elif oldval is not None: # If only the new value is None, fail errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}" errors.append(errorstr) else: # If old and new values are both None, this is expected, so pass LOG.info('Comparing %s and %s... both values are None.', oldval, newval) [LOG.warning(x) for x in errors] n_errors = len(errors) if n_errors > 0: summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%" errors.append(summary_str) > pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format( n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True) E Failed: Failed to match 1 result value within tolerance : E s21.hif_applycal.uid___A002_Xd0a588_X2239.qa.score.ApplycalFlags E values differ by > a relative difference of 1e-07 E expected: 0.9341105789996004 E new: 1.0 E diff: -0.06588942100039963 E percent_diff: -7.053706753965348% E Worst absolute diff, s21.hif_applycal.uid___A002_Xd0a588_X2239.qa.score.ApplycalFlags: -0.06588942100039963 E Worst percentage diff, s21.hif_applycal.uid___A002_Xd0a588_X2239.qa.score.ApplycalFlags: -7.053706753965348% tests/testing_utils.py:435: Failed@pytest.mark.seven @pytest.mark.mpi def test_2023_1_00228_S__uid___A002_X1199f9e_X7c24__procedure_hifa_calimage_diffgain__regression(): """Run ALMA cal+image regression on a 7m B2B dataset with differential gain calibration. Recipe name: procedure_hifa_calimage_diffgain Dataset: 2023.1.00228.S: uid___A002_X1199f9e_X7c24 """ ref_directory = 'pl-regressiontest/2023.1.00228.S' pt = PipelineTester( visname=['uid___A002_X1199f9e_X7c24'], recipe='procedure_hifa_calimage_diffgain.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) > pt.run() tests/regression/fast/alma_if_fast_test.py:143: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:376: in run self.__compare_results(new_file, default_relative_tolerance) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7effcf2e6d50> new_file = 'uid___A002_X1199f9e_X7c24.NEW.results.txt' relative_tolerance = 1e-07 def __compare_results(self, new_file: str, relative_tolerance: float) -> None: """ Compare results between new one loaded from file and old one. Args: new_file : file path of new results relative_tolerance : relative tolerance of output value """ with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd: expected_results = expected_fd.readlines() new_results = new_fd.readlines() errors = [] worst_diff = (0, 0) worst_percent_diff = (0, 0) for old, new in zip(expected_results, new_results): try: oldkey, oldval, tol = self.__sanitize_results_string(old) newkey, newval, _ = self.__sanitize_results_string(new) except ValueError as e: errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e)) errors.append(errorstr) continue assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}." tolerance = tol if tol else relative_tolerance if newval is not None: LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance) if oldval != pytest.approx(newval, rel=tolerance): diff = oldval-newval percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100 if abs(diff) > abs(worst_diff[0]): worst_diff = diff, oldkey if abs(percent_diff) > abs(worst_percent_diff[0]): worst_percent_diff = percent_diff, oldkey errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%" errors.append(errorstr) elif oldval is not None: # If only the new value is None, fail errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}" errors.append(errorstr) else: # If old and new values are both None, this is expected, so pass LOG.info('Comparing %s and %s... both values are None.', oldval, newval) [LOG.warning(x) for x in errors] n_errors = len(errors) if n_errors > 0: summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%" errors.append(summary_str) > pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format( n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True) E Failed: Failed to match 1 result value within tolerance : E s21.hif_applycal.uid___A002_X1199f9e_X7c24.qa.score.ApplycalFlags E values differ by > a relative difference of 1e-07 E expected: 0.7951601163768378 E new: 1.0 E diff: -0.2048398836231622 E percent_diff: -25.76083475571172% E Worst absolute diff, s21.hif_applycal.uid___A002_X1199f9e_X7c24.qa.score.ApplycalFlags: -0.2048398836231622 E Worst percentage diff, s21.hif_applycal.uid___A002_X1199f9e_X7c24.qa.score.ApplycalFlags: -25.76083475571172% tests/testing_utils.py:435: Faileddef test_uid___A002_X85c183_X36f__procedure_hsd_calimage__regression(): """Run ALMA single-dish cal+image regression on the observation data of M100. Recipe name: procedure_hsd_calimage Dataset: uid___A002_X85c183_X36f """ ref_directory = 'pl-regressiontest/uid___A002_X85c183_X36f' pt = PipelineTester( visname=['uid___A002_X85c183_X36f'], recipe='procedure_hsd_calimage.xml', input_dir=ref_directory, expectedoutput_dir=ref_directory, ) > pt.run() tests/regression/fast/alma_sd_fast_test.py:22: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/testing_utils.py:376: in run self.__compare_results(new_file, default_relative_tolerance) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.testing_utils.PipelineTester object at 0x7ff16c4eba40> new_file = 'uid___A002_X85c183_X36f.NEW.results.txt', relative_tolerance = 1e-07 def __compare_results(self, new_file: str, relative_tolerance: float) -> None: """ Compare results between new one loaded from file and old one. Args: new_file : file path of new results relative_tolerance : relative tolerance of output value """ with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd: expected_results = expected_fd.readlines() new_results = new_fd.readlines() errors = [] worst_diff = (0, 0) worst_percent_diff = (0, 0) for old, new in zip(expected_results, new_results): try: oldkey, oldval, tol = self.__sanitize_results_string(old) newkey, newval, _ = self.__sanitize_results_string(new) except ValueError as e: errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e)) errors.append(errorstr) continue assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}." tolerance = tol if tol else relative_tolerance if newval is not None: LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance) if oldval != pytest.approx(newval, rel=tolerance): diff = oldval-newval percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100 if abs(diff) > abs(worst_diff[0]): worst_diff = diff, oldkey if abs(percent_diff) > abs(worst_percent_diff[0]): worst_percent_diff = percent_diff, oldkey errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%" errors.append(errorstr) elif oldval is not None: # If only the new value is None, fail errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}" errors.append(errorstr) else: # If old and new values are both None, this is expected, so pass LOG.info('Comparing %s and %s... both values are None.', oldval, newval) [LOG.warning(x) for x in errors] n_errors = len(errors) if n_errors > 0: summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%" errors.append(summary_str) > pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format( n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True) E Failed: Failed to match 1 result value within tolerance : E s7.hsd_applycal.uid___A002_X85c183_X36f.qa.score.ApplycalFlags E values differ by > a relative difference of 1e-07 E expected: 0.9579857010836426 E new: 1.0 E diff: -0.04201429891635744 E percent_diff: -4.38569165164284% E Worst absolute diff, s7.hsd_applycal.uid___A002_X85c183_X36f.qa.score.ApplycalFlags: -0.04201429891635744 E Worst percentage diff, s7.hsd_applycal.uid___A002_X85c183_X36f.qa.score.ApplycalFlags: -4.38569165164284% tests/testing_utils.py:435: Failed