Build: #95 failed
Job: Test ManyLinux 2.28 Python 3.12 failed
csv 3899 eb2 small procedure hifa calimage regression: Test case result
The below summarizes the result of the test " csv 3899 eb2 small procedure hifa calimage regression" in build 95 of PIPESPECS - Test Pipeline main with Casa 6.7.4 - Test ManyLinux 2.28 Python 3.12.
- Description
- csv 3899 eb2 small procedure hifa calimage regression
- Test class
- tests.regression.fast.alma_if_fast_test
- Method
- test_csv_3899_eb2_small__procedure_hifa_calimage__regression
- Duration
- 281 mins
- Status
- Failed (New Failure)
Error Log
@pytest.mark.twelve
def test_csv_3899_eb2_small__procedure_hifa_calimage__regression():
"""PIPE-2245: Run small ALMA cal+image regression to cover various heuristics
Recipe name: procedure_hifa_calimage
Dataset: CSV-3899-EB2-small
"""
ref_directory = 'pl-regressiontest/CSV-3899-EB2-small'
pt = PipelineTester(
visname=['uid___A002_X1181695_X1c6a4_8ant.ms'],
recipe='procedure_hifa_calimage.xml',
input_dir=ref_directory,
expectedoutput_dir=ref_directory,
)
> pt.run(omp_num_threads=1)
tests/regression/fast/alma_if_fast_test.py:181:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/testing_utils.py:381: in run
self.__compare_results(new_file, default_relative_tolerance)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.testing_utils.PipelineTester object at 0x7faae158d700>
new_file = 'uid___A002_X1181695_X1c6a4_8ant.ms.NEW.results.txt'
relative_tolerance = 1e-07
def __compare_results(self, new_file: str, relative_tolerance: float) -> None:
"""
Compare results between new one loaded from file and old one.
Args:
new_file : file path of new results
relative_tolerance : relative tolerance of output value
"""
with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd:
expected_results = expected_fd.readlines()
new_results = new_fd.readlines()
errors = []
worst_diff = (0, 0)
worst_percent_diff = (0, 0)
for old, new in zip(expected_results, new_results):
try:
oldkey, oldval, tol = self.__sanitize_results_string(old)
newkey, newval, _ = self.__sanitize_results_string(new)
except ValueError as e:
errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e))
errors.append(errorstr)
continue
assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}."
tolerance = tol if tol else relative_tolerance
if newval is not None:
LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance)
if oldval != pytest.approx(newval, rel=tolerance):
diff = oldval-newval
percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100
if abs(diff) > abs(worst_diff[0]):
worst_diff = diff, oldkey
if abs(percent_diff) > abs(worst_percent_diff[0]):
worst_percent_diff = percent_diff, oldkey
errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%"
errors.append(errorstr)
elif oldval is not None:
# If only the new value is None, fail
errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}"
errors.append(errorstr)
else:
# If old and new values are both None, this is expected, so pass
LOG.info('Comparing %s and %s... both values are None.', oldval, newval)
[LOG.warning(x) for x in errors]
n_errors = len(errors)
if n_errors > 0:
summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%"
errors.append(summary_str)
> pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format(
n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True)
E Failed: Failed to match 23 result values within tolerances :
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.after
E values differ by > a relative difference of 1e-07
E expected: 5099936
E new: 5103176
E diff: -3240
E percent_diff: -0.0635302090065444%
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.before
E values differ by > a relative difference of 1e-07
E expected: 5099936
E new: 5101736
E diff: -1800
E percent_diff: -0.035294560559191335%
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.qa.metric.CorrectedampflagQAHandler
E values differ by > a relative difference of 1e-07
E expected: 0.0
E new: 0.0053688072299937365
E diff: -0.0053688072299937365
E percent_diff: 100%
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.qa.score.CorrectedampflagQAHandler
E values differ by > a relative difference of 1e-07
E expected: 1.0
E new: 0.9946311927700062
E diff: 0.005368807229993777
E percent_diff: 0.5368807229993777%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_0.spw_16.qa.metric.score_gfluxscale_amp_time_variation
E values differ by > a relative difference of 1e-07
E expected: 0.15890422094963422
E new: 0.15895858422227874
E diff: -5.436327264451979e-05
E percent_diff: -0.03421134587843994%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_0.spw_16.qa.score.score_gfluxscale_amp_time_variation
E values differ by > a relative difference of 1e-07
E expected: 0.9958440434520864
E new: 0.9958426216434173
E diff: 1.4218086691331067e-06
E percent_diff: 0.00014277423041106084%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_16.I
E values differ by > a relative difference of 1e-07
E expected: 0.3864452726559165
E new: 0.38846230844432605
E diff: -0.00201703578840956
E percent_diff: -0.5219460376749104%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_22.I
E values differ by > a relative difference of 1e-07
E expected: 0.3479005318671154
E new: 0.3559687080382511
E diff: -0.0080681761711357
E percent_diff: -2.3191042933551573%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_22.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.007701514126207
E new: 1.025717471504257
E diff: -0.018015957378050063
E percent_diff: -1.7878267647212942%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_24.I
E values differ by > a relative difference of 1e-07
E expected: 0.32757341992075517
E new: 0.33639694604470255
E diff: -0.008823526123947378
E percent_diff: -2.693602590247377%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_24.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.9744457182701479
E new: 0.9954974538624377
E diff: -0.02105173559228979
E percent_diff: -2.160380531987064%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_16.I
E values differ by > a relative difference of 1e-07
E expected: 0.007882753954217447
E new: 0.007922074070068001
E diff: -3.932011585055378e-05
E percent_diff: -0.4988119136906037%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_22.I
E values differ by > a relative difference of 1e-07
E expected: 0.008878087091949377
E new: 0.009403204584910561
E diff: -0.0005251174929611843
E percent_diff: -5.914759424216049%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_22.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.0515407916400519
E new: 1.1082090210872806
E diff: -0.05666822944722871
E percent_diff: -5.389066206251993%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_22.qa.score.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.0
E new: 0.75
E diff: 0.25
E percent_diff: 25.0%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_24.I
E values differ by > a relative difference of 1e-07
E expected: 0.014043901445720744
E new: 0.014421274744359853
E diff: -0.0003773732986391085
E percent_diff: -2.6870973147856736%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_24.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.6366207338859067
E new: 1.672256908989825
E diff: -0.035636175103918255
E percent_diff: -2.177424150023175%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.qa.metric.score_derived_fluxes_snr
E values differ by > a relative difference of 1e-07
E expected: 2.040571235355487
E new: 2.0439023901874442
E diff: -0.003331154831957228
E percent_diff: -0.16324619176438157%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.qa.score.score_derived_fluxes_snr
E values differ by > a relative difference of 1e-07
E expected: 0.8372305748813088
E new: 0.8372152443898132
E diff: 1.533049149560206e-05
E percent_diff: 0.0018310955136552928%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.after
E values differ by > a relative difference of 1e-07
E expected: 4840736
E new: 4845536
E diff: -4800
E percent_diff: -0.09915847507486464%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.before
E values differ by > a relative difference of 1e-07
E expected: 4840736
E new: 4845536
E diff: -4800
E percent_diff: -0.09915847507486464%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.scan_7.num_rows_flagged.after
E values differ by > a relative difference of 1e-07
E expected: 104640
E new: 109440
E diff: -4800
E percent_diff: -4.587155963302752%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.scan_7.num_rows_flagged.before
E values differ by > a relative difference of 1e-07
E expected: 104640
E new: 109440
E diff: -4800
E percent_diff: -4.587155963302752%
E Worst absolute diff, s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.after: -4800
E Worst percentage diff, s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.qa.metric.CorrectedampflagQAHandler: 100%
tests/testing_utils.py:440: Failed
@pytest.mark.twelve
def test_csv_3899_eb2_small__procedure_hifa_calimage__regression():
"""PIPE-2245: Run small ALMA cal+image regression to cover various heuristics
Recipe name: procedure_hifa_calimage
Dataset: CSV-3899-EB2-small
"""
ref_directory = 'pl-regressiontest/CSV-3899-EB2-small'
pt = PipelineTester(
visname=['uid___A002_X1181695_X1c6a4_8ant.ms'],
recipe='procedure_hifa_calimage.xml',
input_dir=ref_directory,
expectedoutput_dir=ref_directory,
)
> pt.run(omp_num_threads=1)
tests/regression/fast/alma_if_fast_test.py:181:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/testing_utils.py:381: in run
self.__compare_results(new_file, default_relative_tolerance)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.testing_utils.PipelineTester object at 0x7faae158d700>
new_file = 'uid___A002_X1181695_X1c6a4_8ant.ms.NEW.results.txt'
relative_tolerance = 1e-07
def __compare_results(self, new_file: str, relative_tolerance: float) -> None:
"""
Compare results between new one loaded from file and old one.
Args:
new_file : file path of new results
relative_tolerance : relative tolerance of output value
"""
with open(self.expectedoutput_file) as expected_fd, open(new_file) as new_fd:
expected_results = expected_fd.readlines()
new_results = new_fd.readlines()
errors = []
worst_diff = (0, 0)
worst_percent_diff = (0, 0)
for old, new in zip(expected_results, new_results):
try:
oldkey, oldval, tol = self.__sanitize_results_string(old)
newkey, newval, _ = self.__sanitize_results_string(new)
except ValueError as e:
errorstr = "The results: {0} could not be parsed. Error: {1}".format(new, str(e))
errors.append(errorstr)
continue
assert oldkey == newkey, f"Expected key {oldkey} does not match new key {newkey}."
tolerance = tol if tol else relative_tolerance
if newval is not None:
LOG.info('Comparing %s to %s with a rel. tolerance of %s', oldval, newval, tolerance)
if oldval != pytest.approx(newval, rel=tolerance):
diff = oldval-newval
percent_diff = (oldval-newval)/oldval * 100 if oldval != 0 else 100
if abs(diff) > abs(worst_diff[0]):
worst_diff = diff, oldkey
if abs(percent_diff) > abs(worst_percent_diff[0]):
worst_percent_diff = percent_diff, oldkey
errorstr = f"{oldkey}\n\tvalues differ by > a relative difference of {tolerance}\n\texpected: {oldval}\n\tnew: {newval}\n\tdiff: {diff}\n\tpercent_diff: {percent_diff}%"
errors.append(errorstr)
elif oldval is not None:
# If only the new value is None, fail
errorstr = f"{oldkey}\n\tvalue is None\n\texpected: {oldval}\n\tnew: {newval}"
errors.append(errorstr)
else:
# If old and new values are both None, this is expected, so pass
LOG.info('Comparing %s and %s... both values are None.', oldval, newval)
[LOG.warning(x) for x in errors]
n_errors = len(errors)
if n_errors > 0:
summary_str = f"Worst absolute diff, {worst_diff[1]}: {worst_diff[0]}\nWorst percentage diff, {worst_percent_diff[1]}: {worst_percent_diff[0]}%"
errors.append(summary_str)
> pytest.fail("Failed to match {0} result value{1} within tolerance{1} :\n{2}".format(
n_errors, '' if n_errors == 1 else 's', '\n'.join(errors)), pytrace=True)
E Failed: Failed to match 23 result values within tolerances :
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.after
E values differ by > a relative difference of 1e-07
E expected: 5099936
E new: 5103176
E diff: -3240
E percent_diff: -0.0635302090065444%
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.before
E values differ by > a relative difference of 1e-07
E expected: 5099936
E new: 5101736
E diff: -1800
E percent_diff: -0.035294560559191335%
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.qa.metric.CorrectedampflagQAHandler
E values differ by > a relative difference of 1e-07
E expected: 0.0
E new: 0.0053688072299937365
E diff: -0.0053688072299937365
E percent_diff: 100%
E s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.qa.score.CorrectedampflagQAHandler
E values differ by > a relative difference of 1e-07
E expected: 1.0
E new: 0.9946311927700062
E diff: 0.005368807229993777
E percent_diff: 0.5368807229993777%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_0.spw_16.qa.metric.score_gfluxscale_amp_time_variation
E values differ by > a relative difference of 1e-07
E expected: 0.15890422094963422
E new: 0.15895858422227874
E diff: -5.436327264451979e-05
E percent_diff: -0.03421134587843994%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_0.spw_16.qa.score.score_gfluxscale_amp_time_variation
E values differ by > a relative difference of 1e-07
E expected: 0.9958440434520864
E new: 0.9958426216434173
E diff: 1.4218086691331067e-06
E percent_diff: 0.00014277423041106084%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_16.I
E values differ by > a relative difference of 1e-07
E expected: 0.3864452726559165
E new: 0.38846230844432605
E diff: -0.00201703578840956
E percent_diff: -0.5219460376749104%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_22.I
E values differ by > a relative difference of 1e-07
E expected: 0.3479005318671154
E new: 0.3559687080382511
E diff: -0.0080681761711357
E percent_diff: -2.3191042933551573%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_22.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.007701514126207
E new: 1.025717471504257
E diff: -0.018015957378050063
E percent_diff: -1.7878267647212942%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_24.I
E values differ by > a relative difference of 1e-07
E expected: 0.32757341992075517
E new: 0.33639694604470255
E diff: -0.008823526123947378
E percent_diff: -2.693602590247377%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_1.spw_24.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 0.9744457182701479
E new: 0.9954974538624377
E diff: -0.02105173559228979
E percent_diff: -2.160380531987064%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_16.I
E values differ by > a relative difference of 1e-07
E expected: 0.007882753954217447
E new: 0.007922074070068001
E diff: -3.932011585055378e-05
E percent_diff: -0.4988119136906037%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_22.I
E values differ by > a relative difference of 1e-07
E expected: 0.008878087091949377
E new: 0.009403204584910561
E diff: -0.0005251174929611843
E percent_diff: -5.914759424216049%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_22.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.0515407916400519
E new: 1.1082090210872806
E diff: -0.05666822944722871
E percent_diff: -5.389066206251993%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_22.qa.score.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.0
E new: 0.75
E diff: 0.25
E percent_diff: 25.0%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_24.I
E values differ by > a relative difference of 1e-07
E expected: 0.014043901445720744
E new: 0.014421274744359853
E diff: -0.0003773732986391085
E percent_diff: -2.6870973147856736%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.field_3.spw_24.qa.metric.score_gfluxscale_k_spw
E values differ by > a relative difference of 1e-07
E expected: 1.6366207338859067
E new: 1.672256908989825
E diff: -0.035636175103918255
E percent_diff: -2.177424150023175%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.qa.metric.score_derived_fluxes_snr
E values differ by > a relative difference of 1e-07
E expected: 2.040571235355487
E new: 2.0439023901874442
E diff: -0.003331154831957228
E percent_diff: -0.16324619176438157%
E s17.hifa_gfluxscale.uid___A002_X1181695_X1c6a4_8ant.qa.score.score_derived_fluxes_snr
E values differ by > a relative difference of 1e-07
E expected: 0.8372305748813088
E new: 0.8372152443898132
E diff: 1.533049149560206e-05
E percent_diff: 0.0018310955136552928%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.after
E values differ by > a relative difference of 1e-07
E expected: 4840736
E new: 4845536
E diff: -4800
E percent_diff: -0.09915847507486464%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.before
E values differ by > a relative difference of 1e-07
E expected: 4840736
E new: 4845536
E diff: -4800
E percent_diff: -0.09915847507486464%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.scan_7.num_rows_flagged.after
E values differ by > a relative difference of 1e-07
E expected: 104640
E new: 109440
E diff: -4800
E percent_diff: -4.587155963302752%
E s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.scan_7.num_rows_flagged.before
E values differ by > a relative difference of 1e-07
E expected: 104640
E new: 109440
E diff: -4800
E percent_diff: -4.587155963302752%
E Worst absolute diff, s21.hif_applycal.uid___A002_X1181695_X1c6a4_8ant.num_rows_flagged.after: -4800
E Worst percentage diff, s16.hifa_gfluxscaleflag.uid___A002_X1181695_X1c6a4_8ant.qa.metric.CorrectedampflagQAHandler: 100%
tests/testing_utils.py:440: Failed