Build: #103 failed
Job: Test ManyLinux 2.28 Python 3.12 failed
2023 1 00228 s uid a002 x1199f9e x7c24 procedure hifa calimage diffgain regression: Test case result
The below summarizes the result of the test " 2023 1 00228 s uid a002 x1199f9e x7c24 procedure hifa calimage diffgain regression" in build 103 of PIPESPECS - Test Pipeline main with Casa 6.7.4 - Test ManyLinux 2.28 Python 3.12.
- Description
- 2023 1 00228 s uid a002 x1199f9e x7c24 procedure hifa calimage diffgain regression
- Test class
- tests.regression.fast.alma_if_fast_test
- Method
- test_2023_1_00228_S__uid___A002_X1199f9e_X7c24__procedure_hifa_calimage_diffgain__regression
- Duration
- 810 mins
- Status
- Failed (New Failure)
Error Log
@pytest.mark.seven
@pytest.mark.mpi
def test_2023_1_00228_S__uid___A002_X1199f9e_X7c24__procedure_hifa_calimage_diffgain__regression():
"""Run ALMA cal+image regression on a 7m B2B dataset with differential gain calibration.
Recipe name: procedure_hifa_calimage_diffgain
Dataset: 2023.1.00228.S: uid___A002_X1199f9e_X7c24
"""
ref_directory = 'pl-regressiontest/2023.1.00228.S'
pt = PipelineTester(
visname=['uid___A002_X1199f9e_X7c24'],
recipe='procedure_hifa_calimage_diffgain.xml',
input_dir=ref_directory,
expectedoutput_dir=ref_directory,
)
> pt.run()
tests/regression/fast/alma_if_fast_test.py:142:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/testing_utils.py:381: in run
self.__compare_results(new_file, default_relative_tolerance)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.testing_utils.PipelineTester object at 0x7f98cec24c50>
new_file = 'uid___A002_X1199f9e_X7c24.NEW.results.txt'
relative_tolerance = 1e-07
def __compare_results(self, new_file: str, relative_tolerance: float) -> None:
"""
Compare results between new one loaded from file and old one.
Args:
new_file : file path of new results
relative_tolerance : relative tolerance of output value
"""
with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd:
expected_results = expected_fd.readlines()
new_results = new_fd.readlines()
errors = []
worst_diff = (0, 0)
worst_percent_diff = (0, 0)
for old, new in zip(expected_results, new_results):
try:
oldkey, oldval, tol = self.__sanitize_results_string(old)
newkey, newval, _ = self.__sanitize_results_string(new)
except ValueError as e:
errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e))
errors.append(errorstr)
continue
assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}."
tolerance = tol if tol else relative_tolerance
if newval is not None:
LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance)
if oldval != pytest.approx(newval, rel=tolerance):
diff = oldval-newval
percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100
if abs(diff) > abs(worst_diff[0]):
worst_diff = diff, oldkey
if abs(percent_diff) > abs(worst_percent_diff[0]):
worst_percent_diff = percent_diff, oldkey
errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%"
errors.append(errorstr)
elif oldval is not None:
# If only the new value is None, fail
errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}"
errors.append(errorstr)
else:
# If old and new values are both None, this is expected, so pass
LOG.info('Comparing %s and %s... both values are None.', oldval, newval)
[LOG.warning(x) for x in errors]
n_errors = len(errors)
if n_errors > 0:
summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%"
errors.append(summary_str)
> pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format(
n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True)
E Failed: Failed to match 13 result values within tolerances :
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_29.I
E values differ by > a relative difference of 1e-07
E expected: 0.8378082712078273
E new: 0.8378077335399046
E diff: 5.376679227397929e-07
E percent_diff: 6.417553290141947e-05%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_29.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.8081204912780358
E new: 0.808119972662404
E diff: 5.186156317726898e-07
E percent_diff: 6.417553290258777e-05%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_33.I
E values differ by > a relative difference of 1e-07
E expected: 0.8460116243974625
E new: 0.8460103046383503
E diff: 1.319759112217156e-06
E percent_diff: 0.00015599775158610862%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_33.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.8173528334400738
E new: 0.8173515583880312
E diff: 1.2750520426152434e-06
E percent_diff: 0.0001559977515767341%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_41.I
E values differ by > a relative difference of 1e-07
E expected: 0.7885692224368216
E new: 0.7885703778722577
E diff: -1.1554354361065222e-06
E percent_diff: -0.00014652301956903893%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_41.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.7642712175824183
E new: 0.764272337415684
E diff: -1.119833265672554e-06
E percent_diff: -0.00014652301956560233%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_29.I
E values differ by > a relative difference of 1e-07
E expected: 0.12352226017389038
E new: 0.12352412354044164
E diff: -1.86336655125241e-06
E percent_diff: -0.0015085269235109741%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_29.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.5679953042438604
E new: 0.5680038726059492
E diff: -8.568362088845483e-06
E percent_diff: -0.0015085269235195619%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_33.I
E values differ by > a relative difference of 1e-07
E expected: 0.21527453621962267
E new: 0.2152731154481138
E diff: 1.4207715088587758e-06
E percent_diff: 0.0006599812192415119%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_33.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.9918525112637544
E new: 0.9918459652234575
E diff: 6.5460402969286235e-06
E percent_diff: 0.0006599812192427765%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.I
E values differ by > a relative difference of 1e-07
E expected: 0.20283735907845116
E new: 0.20284323676879906
E diff: -5.877690347905196e-06
E percent_diff: -0.0028977355920079243%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.9381597108479458
E new: 0.9381868962357969
E diff: -2.718538785106972e-05
E percent_diff: -0.0028977355920026125%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.qa.score.score_derived_fluxes_snr
E values differ by > a relative difference of 1e-07
E expected: 0.8782559648560236
E new: 0.8782558045461518
E diff: 1.6030987182169554e-07
E percent_diff: 1.825320615362696e-05%
E Worst absolute diff, s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.qa.metric.score_gfluxscale_k_spw: -2.718538785106972e-05
E Worst percentage diff, s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.I: -0.0028977355920079243%
tests/testing_utils.py:440: Failed
@pytest.mark.seven
@pytest.mark.mpi
def test_2023_1_00228_S__uid___A002_X1199f9e_X7c24__procedure_hifa_calimage_diffgain__regression():
"""Run ALMA cal+image regression on a 7m B2B dataset with differential gain calibration.
Recipe name: procedure_hifa_calimage_diffgain
Dataset: 2023.1.00228.S: uid___A002_X1199f9e_X7c24
"""
ref_directory = 'pl-regressiontest/2023.1.00228.S'
pt = PipelineTester(
visname=['uid___A002_X1199f9e_X7c24'],
recipe='procedure_hifa_calimage_diffgain.xml',
input_dir=ref_directory,
expectedoutput_dir=ref_directory,
)
> pt.run()
tests/regression/fast/alma_if_fast_test.py:142:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/testing_utils.py:381: in run
self.__compare_results(new_file, default_relative_tolerance)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.testing_utils.PipelineTester object at 0x7f98cec24c50>
new_file = 'uid___A002_X1199f9e_X7c24.NEW.results.txt'
relative_tolerance = 1e-07
def __compare_results(self, new_file: str, relative_tolerance: float) -> None:
"""
Compare results between new one loaded from file and old one.
Args:
new_file : file path of new results
relative_tolerance : relative tolerance of output value
"""
with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd:
expected_results = expected_fd.readlines()
new_results = new_fd.readlines()
errors = []
worst_diff = (0, 0)
worst_percent_diff = (0, 0)
for old, new in zip(expected_results, new_results):
try:
oldkey, oldval, tol = self.__sanitize_results_string(old)
newkey, newval, _ = self.__sanitize_results_string(new)
except ValueError as e:
errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e))
errors.append(errorstr)
continue
assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}."
tolerance = tol if tol else relative_tolerance
if newval is not None:
LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance)
if oldval != pytest.approx(newval, rel=tolerance):
diff = oldval-newval
percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100
if abs(diff) > abs(worst_diff[0]):
worst_diff = diff, oldkey
if abs(percent_diff) > abs(worst_percent_diff[0]):
worst_percent_diff = percent_diff, oldkey
errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%"
errors.append(errorstr)
elif oldval is not None:
# If only the new value is None, fail
errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}"
errors.append(errorstr)
else:
# If old and new values are both None, this is expected, so pass
LOG.info('Comparing %s and %s... both values are None.', oldval, newval)
[LOG.warning(x) for x in errors]
n_errors = len(errors)
if n_errors > 0:
summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%"
errors.append(summary_str)
> pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format(
n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True)
E Failed: Failed to match 13 result values within tolerances :
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_29.I
E values differ by > a relative difference of 1e-07
E expected: 0.8378082712078273
E new: 0.8378077335399046
E diff: 5.376679227397929e-07
E percent_diff: 6.417553290141947e-05%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_29.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.8081204912780358
E new: 0.808119972662404
E diff: 5.186156317726898e-07
E percent_diff: 6.417553290258777e-05%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_33.I
E values differ by > a relative difference of 1e-07
E expected: 0.8460116243974625
E new: 0.8460103046383503
E diff: 1.319759112217156e-06
E percent_diff: 0.00015599775158610862%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_33.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.8173528334400738
E new: 0.8173515583880312
E diff: 1.2750520426152434e-06
E percent_diff: 0.0001559977515767341%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_41.I
E values differ by > a relative difference of 1e-07
E expected: 0.7885692224368216
E new: 0.7885703778722577
E diff: -1.1554354361065222e-06
E percent_diff: -0.00014652301956903893%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_1.spw_41.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.7642712175824183
E new: 0.764272337415684
E diff: -1.119833265672554e-06
E percent_diff: -0.00014652301956560233%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_29.I
E values differ by > a relative difference of 1e-07
E expected: 0.12352226017389038
E new: 0.12352412354044164
E diff: -1.86336655125241e-06
E percent_diff: -0.0015085269235109741%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_29.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.5679953042438604
E new: 0.5680038726059492
E diff: -8.568362088845483e-06
E percent_diff: -0.0015085269235195619%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_33.I
E values differ by > a relative difference of 1e-07
E expected: 0.21527453621962267
E new: 0.2152731154481138
E diff: 1.4207715088587758e-06
E percent_diff: 0.0006599812192415119%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_33.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.9918525112637544
E new: 0.9918459652234575
E diff: 6.5460402969286235e-06
E percent_diff: 0.0006599812192427765%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.I
E values differ by > a relative difference of 1e-07
E expected: 0.20283735907845116
E new: 0.20284323676879906
E diff: -5.877690347905196e-06
E percent_diff: -0.0028977355920079243%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.9381597108479458
E new: 0.9381868962357969
E diff: -2.718538785106972e-05
E percent_diff: -0.0028977355920026125%
E s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.qa.score.score_derived_fluxes_snr
E values differ by > a relative difference of 1e-07
E expected: 0.8782559648560236
E new: 0.8782558045461518
E diff: 1.6030987182169554e-07
E percent_diff: 1.825320615362696e-05%
E Worst absolute diff, s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.qa.metric.score_gfluxscale_k_spw: -2.718538785106972e-05
E Worst percentage diff, s16.hifa_gfluxscale.uid___A002_X1199f9e_X7c24.field_4.spw_41.I: -0.0028977355920079243%
tests/testing_utils.py:440: Failed