Line data Source code
1 : //# SingleDishSkyCal.cc: implements SingleDishSkyCal
2 : //# Copyright (C) 2003
3 : //# Associated Universities, Inc. Washington DC, USA.
4 : //#
5 : //# This library is free software; you can redistribute it and/or modify it
6 : //# under the terms of the GNU Library General Public License as published by
7 : //# the Free Software Foundation; either version 2 of the License, or (at your
8 : //# option) any later version.
9 : //#
10 : //# This library is distributed in the hope that it will be useful, but WITHOUT
11 : //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 : //# License for more details.
14 : //#
15 : //# You should have received a copy of the GNU Library General Public License
16 : //# along with this library; if not, write to the Free Software Foundation,
17 : //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 : //#
19 : //# Correspondence concerning AIPS++ should be addressed as follows:
20 : //# Internet email: casa-feedback@nrao.edu.
21 : //# Postal address: AIPS++ Project Office
22 : //# National Radio Astronomy Observatory
23 : //# 520 Edgemont Road
24 : //# Charlottesville, VA 22903-2475 USA
25 : //#
26 : //# $Id$
27 :
28 : //# Includes
29 : #include <iostream>
30 : #include <sstream>
31 : #include <iomanip>
32 : #include <map>
33 : #include <type_traits>
34 :
35 : #include <casacore/casa/OS/File.h>
36 : #include <casacore/casa/Arrays/MaskedArray.h>
37 : #include <casacore/casa/Arrays/MaskArrMath.h>
38 : #include <casacore/tables/TaQL/TableParse.h>
39 : #include <casacore/tables/Tables/Table.h>
40 : #include <casacore/tables/Tables/ScalarColumn.h>
41 : #include <casacore/ms/MeasurementSets/MeasurementSet.h>
42 : #include <casacore/ms/MeasurementSets/MSState.h>
43 : #include <casacore/ms/MeasurementSets/MSSpectralWindow.h>
44 : #include <casacore/ms/MeasurementSets/MSIter.h>
45 : #include <synthesis/MeasurementComponents/MSMetaInfoForCal.h>
46 : #include <synthesis/MeasurementComponents/SingleDishSkyCal.h>
47 : #include <synthesis/CalTables/CTGlobals.h>
48 : #include <synthesis/CalTables/CTMainColumns.h>
49 : #include <synthesis/CalTables/CTInterface.h>
50 : #include <casacore/ms/MSSel/MSSelection.h>
51 : #include <casacore/ms/MSSel/MSSelectionTools.h>
52 : #include <synthesis/Utilities/PointingDirectionCalculator.h>
53 : #include <synthesis/Utilities/PointingDirectionProjector.h>
54 : #include <casacore/coordinates/Coordinates/DirectionCoordinate.h>
55 : #include <casa_sakura/SakuraAlignedArray.h>
56 : #include <libsakura/sakura.h>
57 : #include <cassert>
58 :
59 : // Debug Message Handling
60 : // if SDCALSKY_DEBUG is defined, the macro debuglog and
61 : // debugpost point standard output stream (std::cout and
62 : // std::endl so that debug messages are sent to standard
63 : // output. Otherwise, these macros basically does nothing.
64 : // "Do nothing" behavior is implemented in NullLogger
65 : // and its associating << operator below.
66 : //
67 : // Usage:
68 : // Similar to standard output stream.
69 : //
70 : // debuglog << "Any message" << any_value << debugpost;
71 : //
72 : //#define SDCALSKY_DEBUG
73 :
74 : using namespace casacore;
75 :
76 : namespace {
77 : struct NullLogger {};
78 :
79 : template<class T>
80 626544 : inline NullLogger &operator<<(NullLogger &logger, T /*value*/) {
81 626544 : return logger;
82 : }
83 :
84 : #ifndef SDCALSKY_DEBUG
85 : NullLogger nulllogger;
86 : #endif
87 :
88 : // Helper class to format integers with thousand separators
89 : struct CommaSeparatedThousands : std::numpunct<char> {
90 29 : char_type do_thousands_sep() const override { return ','; }
91 29 : string_type do_grouping() const override { return "\3"; }
92 : };
93 :
94 : }
95 :
96 : #ifdef SDCALSKY_DEBUG
97 : #define debuglog std::cout
98 : #define debugpost std::endl
99 : #else
100 : #define debuglog nulllogger
101 : #define debugpost 0
102 : #endif
103 :
104 : // Local Functions
105 : namespace {
106 : inline Vector<rownr_t> getOffStateIdList(MeasurementSet const &ms) {
107 : String taql("SELECT FLAG_ROW FROM $1 WHERE UPPER(OBS_MODE) ~ m/^OBSERVE_TARGET#OFF_SOURCE/");
108 : Table stateSel = tableCommand(taql, ms.state()).table();
109 : Vector<rownr_t> stateIdList = stateSel.rowNumbers();
110 : debuglog << "stateIdList[" << stateIdList.nelements() << "]=";
111 : for (size_t i = 0; i < stateIdList.nelements(); ++i) {
112 : debuglog << stateIdList[i] << " ";
113 : }
114 : debuglog << debugpost;
115 : return stateIdList;
116 : }
117 :
118 : template<class T>
119 49 : inline std::string toString(casacore::Vector<T> const &v) {
120 49 : std::ostringstream oss;
121 49 : oss << "[";
122 49 : std::string delimiter = "";
123 589 : for (size_t i = 0; i < v.nelements(); ++i) {
124 540 : oss << delimiter << v[i];
125 540 : delimiter = ",";
126 : }
127 49 : oss << "]";
128 98 : return oss.str();
129 49 : }
130 :
131 : // unused
132 : /*
133 : inline casacore::String configureTaqlString(casacore::String const &msName, casacore::Vector<casacore::uInt> stateIdList) {
134 : std::ostringstream oss;
135 : oss << "SELECT FROM " << msName << " WHERE ANTENNA1 == ANTENNA2 && STATE_ID IN "
136 : << toString(stateIdList)
137 : << " ORDER BY FIELD_ID, ANTENNA1, FEED1, DATA_DESC_ID, TIME";
138 : return casacore::String(oss);
139 : }
140 : */
141 :
142 31 : inline void fillNChanParList(casacore::MeasurementSet const &ms, casacore::Vector<casacore::Int> &nChanParList) {
143 31 : casacore::MSSpectralWindow const &msspw = ms.spectralWindow();
144 31 : casacore::ScalarColumn<casacore::Int> nchanCol(msspw, "NUM_CHAN");
145 31 : debuglog << "nchanCol=" << toString(nchanCol.getColumn()) << debugpost;
146 31 : nChanParList = nchanCol.getColumn()(casacore::Slice(0,nChanParList.nelements()));
147 31 : debuglog << "nChanParList=" << nChanParList << debugpost;
148 31 : }
149 :
150 28 : inline void updateWeight(casa::NewCalTable &ct) {
151 28 : casa::CTMainColumns ctmc(ct);
152 28 : casacore::ArrayColumn<casacore::Double> chanWidthCol(ct.spectralWindow(), "CHAN_WIDTH");
153 28 : casacore::Vector<casacore::Int> chanWidth(chanWidthCol.nrow());
154 402 : for (size_t irow = 0; irow < chanWidthCol.nrow(); ++irow) {
155 374 : casacore::Double chanWidthVal = chanWidthCol(irow)(casacore::IPosition(1,0));
156 374 : chanWidth[irow] = abs(chanWidthVal);
157 : }
158 596 : for (size_t irow = 0; irow < ct.nrow(); ++irow) {
159 568 : casacore::Int spwid = ctmc.spwId()(irow);
160 568 : casacore::Double width = chanWidth[spwid];
161 568 : casacore::Double interval = ctmc.interval()(irow);
162 568 : casacore::Float weight = width * interval;
163 568 : casacore::Matrix<casacore::Float> weightMat(ctmc.fparam().shape(irow), weight);
164 568 : ctmc.weight().put(irow, weightMat);
165 568 : }
166 28 : }
167 :
168 : class DataColumnAccessor {
169 : public:
170 112 : DataColumnAccessor(casacore::Table const &ms,
171 : casacore::String const colName="DATA")
172 112 : : dataCol_(ms, colName) {
173 112 : }
174 638 : casacore::Matrix<casacore::Float> operator()(size_t irow) {
175 1276 : return casacore::real(dataCol_(irow));
176 : }
177 : private:
178 : DataColumnAccessor() {}
179 : casacore::ArrayColumn<casacore::Complex> dataCol_;
180 : };
181 :
182 : class FloatDataColumnAccessor {
183 : public:
184 74 : FloatDataColumnAccessor(casacore::Table const &ms)
185 74 : : dataCol_(ms, "FLOAT_DATA") {
186 74 : }
187 2111 : casacore::Matrix<casacore::Float> operator()(size_t irow) {
188 4222 : return dataCol_(irow);
189 : }
190 : private:
191 : FloatDataColumnAccessor() {}
192 : casacore::ArrayColumn<casacore::Float> dataCol_;
193 : };
194 :
195 10 : inline casacore::Int nominalDataDesc(casacore::MeasurementSet const &ms, casacore::Int const ant)
196 : {
197 10 : casacore::Int goodDataDesc = -1;
198 10 : casacore::ScalarColumn<casacore::Int> col(ms.spectralWindow(), "NUM_CHAN");
199 10 : casacore::Vector<casacore::Int> nchanList = col.getColumn();
200 10 : size_t numSpw = col.nrow();
201 10 : casacore::Vector<casacore::Int> spwMap(numSpw);
202 10 : col.attach(ms.dataDescription(), "SPECTRAL_WINDOW_ID");
203 180 : for (size_t i = 0; i < col.nrow(); ++i) {
204 170 : spwMap[col(i)] = i;
205 : }
206 10 : casacore::ScalarColumn<casacore::String> obsModeCol(ms.state(), "OBS_MODE");
207 10 : casacore::Regex regex("^OBSERVE_TARGET#ON_SOURCE.*");
208 100 : for (size_t ispw = 0; ispw < numSpw; ++ispw) {
209 100 : if (nchanList[ispw] == 4) {
210 : // this should be WVR
211 10 : continue;
212 : }
213 : else {
214 90 : std::ostringstream oss;
215 90 : oss << "SELECT FROM $1 WHERE ANTENNA1 == " << ant
216 90 : << " && ANTENNA2 == " << ant << " && DATA_DESC_ID == " << spwMap[ispw]
217 90 : << " ORDER BY STATE_ID";
218 180 : casacore::MeasurementSet msSel(casacore::tableCommand(oss.str(), ms).table());
219 90 : col.attach(msSel, "STATE_ID");
220 90 : casacore::Vector<casacore::Int> stateIdList = col.getColumn();
221 90 : casacore::Int previousStateId = -1;
222 90 : for (size_t i = 0; i < msSel.nrow(); ++i) {
223 10 : casacore::Int stateId = stateIdList[i];
224 10 : if (stateId != previousStateId) {
225 10 : casacore::String obsmode = obsModeCol(stateId);
226 10 : if (regex.match(obsmode.c_str(), obsmode.size()) != casacore::String::npos) {
227 10 : goodDataDesc = spwMap[ispw];
228 10 : break;
229 : }
230 10 : }
231 0 : previousStateId = stateId;
232 : }
233 90 : }
234 :
235 90 : if (goodDataDesc >= 0)
236 10 : break;
237 : }
238 10 : return goodDataDesc;
239 10 : }
240 :
241 10 : inline casacore::Vector<size_t> detectGap(casacore::Vector<casacore::Double> timeList)
242 : {
243 10 : size_t n = timeList.size();
244 20 : casacore::Vector<casacore::Double> timeInterval = timeList(casacore::Slice(1, n-1)) - timeList(casacore::Slice(0, n-1));
245 10 : casacore::Double medianTime = casacore::median(timeInterval);
246 10 : casacore::Double const threshold = medianTime * 5.0;
247 10 : casacore::Vector<size_t> gapIndexList(casacore::IPosition(1, n/2 + 2), new size_t[n/2+2], casacore::TAKE_OVER);
248 10 : gapIndexList[0] = 0;
249 10 : size_t gapIndexCount = 1;
250 252 : for (size_t i = 0; i < timeInterval.size(); ++i) {
251 242 : if (timeInterval[i] > threshold) {
252 19 : gapIndexList[gapIndexCount] = i + 1;
253 19 : gapIndexCount++;
254 : }
255 : }
256 10 : if (gapIndexList[gapIndexCount] != n) {
257 10 : gapIndexList[gapIndexCount] = n;
258 10 : gapIndexCount++;
259 : }
260 10 : debuglog << "Detected " << gapIndexCount << " gaps." << debugpost;
261 10 : casacore::Vector<size_t> ret(casacore::IPosition(1, gapIndexCount), gapIndexList.data(), casacore::COPY);
262 10 : debuglog << "gapList=" << toString(ret) << debugpost;
263 20 : return ret;
264 10 : }
265 :
266 : struct DefaultRasterEdgeDetector
267 : {
268 3 : static size_t N(size_t numData, casacore::Float const /*fraction*/, casacore::Int const /*num*/)
269 : {
270 3 : debuglog << "DefaultRasterEdgeDetector" << debugpost;
271 3 : return max((size_t)1, static_cast<size_t>(sqrt(numData + 1) - 1));
272 : }
273 : };
274 :
275 : struct FixedNumberRasterEdgeDetector
276 : {
277 7 : static size_t N(size_t /*numData*/, casacore::Float const /*fraction*/, casacore::Int const num)
278 : {
279 7 : debuglog << "FixedNumberRasterEdgeDetector" << debugpost;
280 7 : if (num < 0) {
281 0 : throw casacore::AipsError("Negative number of edge points.");
282 : }
283 7 : return (size_t)num;
284 : }
285 : };
286 :
287 : struct FixedFractionRasterEdgeDetector
288 : {
289 17 : static casacore::Int N(size_t numData, casacore::Float const fraction, casacore::Int const /*num*/)
290 : {
291 17 : debuglog << "FixedFractionRasterEdgeDetector" << debugpost;
292 17 : return max((size_t)1, static_cast<size_t>(numData * fraction));
293 : }
294 : };
295 :
296 : template<class Detector>
297 27 : inline casacore::Vector<casacore::Double> detectEdge(casacore::Vector<casacore::Double> timeList, casacore::Double const interval, casacore::Float const fraction, casacore::Int const num)
298 : {
299 : // storage for time range for edges (at head and tail)
300 : // [(start of head edge), (end of head edge),
301 : // (start of tail edge), (end of tail edge)]
302 27 : casacore::Vector<casacore::Double> edgeList(4);
303 27 : size_t numList = timeList.size();
304 27 : size_t numEdge = Detector::N(numList, fraction, num);
305 27 : debuglog << "numEdge = " << numEdge << debugpost;
306 27 : if (numEdge == 0) {
307 0 : throw casacore::AipsError("Zero edge points.");
308 : }
309 27 : else if (timeList.size() > numEdge * 2) {
310 25 : edgeList[0] = timeList[0] - 0.5 * interval;
311 25 : edgeList[1] = timeList[numEdge-1] + 0.5 * interval;
312 25 : edgeList[2] = timeList[numList-numEdge] - 0.5 * interval;
313 25 : edgeList[3] = timeList[numList-1] + 0.5 * interval;
314 : }
315 : else {
316 2 : std::ostringstream oss;
317 2 : oss << "Too many edge points (" << 2.0 * numEdge << " out of "
318 2 : << timeList.size() << " points)";
319 2 : throw casacore::AipsError(oss.str());
320 : // edgeList[0] = timeList[0] - 0.5 * interval;
321 : // edgeList[1] = timeList[numList-1] + 0.5 * interval;
322 : // edgeList[2] = edgeList[0];
323 : // edgeList[3] = edgeList[2];
324 2 : }
325 25 : return edgeList;
326 2 : }
327 :
328 10 : inline casacore::Vector<casacore::String> detectRaster(casacore::MeasurementSet const &ms,
329 : casacore::Int const ant,
330 : casacore::Float const fraction,
331 : casacore::Int const num)
332 : {
333 10 : casacore::Int dataDesc = nominalDataDesc(ms, ant);
334 10 : debuglog << "nominal DATA_DESC_ID=" << dataDesc << debugpost;
335 10 : assert(dataDesc >= 0);
336 10 : if (dataDesc < 0) {
337 0 : return casacore::Vector<casacore::String>();
338 : }
339 :
340 10 : std::ostringstream oss;
341 10 : oss << "SELECT FROM $1 WHERE ANTENNA1 == " << ant
342 10 : << " && ANTENNA2 == " << ant << " && FEED1 == 0 && FEED2 == 0"
343 10 : << " && DATA_DESC_ID == " << dataDesc
344 10 : << " ORDER BY TIME";
345 10 : debuglog << "detectRaster: taql=" << oss.str() << debugpost;
346 20 : casacore::MeasurementSet msSel(casacore::tableCommand(oss.str(), ms).table());
347 10 : casacore::ScalarColumn<casacore::Double> timeCol(msSel, "TIME");
348 10 : casacore::ScalarColumn<casacore::Double> intervalCol(msSel, "INTERVAL");
349 10 : casacore::Vector<casacore::Double> timeList = timeCol.getColumn();
350 10 : casacore::Double interval = casacore::min(intervalCol.getColumn());
351 10 : casacore::Vector<size_t> gapList = detectGap(timeList);
352 10 : casacore::Vector<casacore::String> edgeAsTimeRange(gapList.size() * 2);
353 : typedef casacore::Vector<casacore::Double> (*DetectorFunc)(casacore::Vector<casacore::Double>, casacore::Double const, casacore::Float const, casacore::Int const);
354 10 : DetectorFunc detect = NULL;
355 10 : if (num > 0) {
356 3 : detect = detectEdge<FixedNumberRasterEdgeDetector>;
357 : }
358 7 : else if (fraction > 0.0) {
359 6 : detect = detectEdge<FixedFractionRasterEdgeDetector>;
360 : }
361 : else {
362 1 : detect = detectEdge<DefaultRasterEdgeDetector>;
363 : }
364 35 : for (size_t i = 0; i < gapList.size()-1; ++i) {
365 27 : size_t startRow = gapList[i];
366 27 : size_t endRow = gapList[i+1];
367 27 : size_t len = endRow - startRow;
368 27 : debuglog << "startRow=" << startRow << ", endRow=" << endRow << debugpost;
369 27 : casacore::Vector<casacore::Double> oneRow = timeList(casacore::Slice(startRow, len));
370 29 : casacore::Vector<casacore::Double> edgeList = detect(oneRow, interval, fraction, num);
371 25 : std::ostringstream s;
372 25 : s << std::setprecision(16) << "TIME BETWEEN " << edgeList[0] << " AND " << edgeList[1];
373 25 : edgeAsTimeRange[2*i] = s.str();
374 25 : s.str("");
375 25 : s << std::setprecision(16) << "TIME BETWEEN " << edgeList[2] << " AND " << edgeList[3];
376 25 : edgeAsTimeRange[2*i+1] = s.str();
377 25 : debuglog << "Resulting selection: (" << edgeAsTimeRange[2*i] << ") || ("
378 50 : << edgeAsTimeRange[2*i+1] << ")" << debugpost;
379 27 : }
380 8 : return edgeAsTimeRange;
381 22 : }
382 :
383 : // Formula for weight scaling factor, WF
384 : // 1. only one OFF spectrum is used (i.e. no interpolation)
385 : //
386 : // sigma = sqrt(sigma_ON^2 + sigma_OFF^2)
387 : // = sigma_ON * sqrt(1 + tau_ON / tau_OFF)
388 : //
389 : // weight = 1 / sigma^2
390 : // = 1 / sigma_ON^2 * tau_OFF / (tau_ON + tau_OFF)
391 : // = weight_ON * tau_OFF / (tau_ON + tau_OFF)
392 : //
393 : // WF = tau_OFF / (tau_ON + tau_OFF)
394 : //
395 : struct SimpleWeightScalingScheme
396 : {
397 1351 : static casacore::Float SimpleScale(casacore::Double exposureOn, casacore::Double exposureOff)
398 : {
399 1351 : return exposureOff / (exposureOn + exposureOff);
400 : }
401 : };
402 : // 2. two OFF spectrum is used (linear interpolation)
403 : //
404 : // sigma_OFF = {(t_OFF2 - t_ON)^2 * sigma_OFF1^2
405 : // + (t_ON - t_OFF1)^2 * sigma_OFF2^2}
406 : // / (t_OFF2 - t_OFF1)^2
407 : //
408 : // sigma = sqrt(sigma_ON^2 + sigma_OFF^2)
409 : // = sigma_ON * sqrt(1 + tau_ON / (t_OFF2 - t_OFF1)^2
410 : // * {(t_OFF2 - t_ON)^2 / tau_OFF1
411 : // + (t_ON - t_OFF1)^2 / tau_OFF2})
412 : //
413 : // weight = weight_ON / (1 + tau_ON / (t_OFF2 - t_OFF1)^2
414 : // * {(t_OFF2 - t_ON)^2 / tau_OFF1
415 : // + (t_ON - t_OFF1)^2 / tau_OFF2})
416 : //
417 : // WF = 1.0 / (1 + tau_ON / (t_OFF2 - t_OFF1)^2
418 : // * {(t_OFF2 - t_ON)^2 / tau_OFF1
419 : // + (t_ON - t_OFF1)^2 / tau_OFF2})
420 : //
421 : struct LinearWeightScalingScheme : public SimpleWeightScalingScheme
422 : {
423 3527 : static casacore::Float InterpolateScale(casacore::Double timeOn, casacore::Double exposureOn,
424 : casacore::Double timeOff1, casacore::Double exposureOff1,
425 : casacore::Double timeOff2, casacore::Double exposureOff2)
426 : {
427 3527 : casacore::Double dt = timeOff2 - timeOff1;
428 3527 : casacore::Double dt1 = timeOn - timeOff1;
429 3527 : casacore::Double dt2 = timeOff2 - timeOn;
430 3527 : return 1.0f / (1.0f + exposureOn / (dt * dt)
431 3527 : * (dt2 * dt2 / exposureOff1 + dt1 * dt1 / exposureOff2));
432 : }
433 : };
434 :
435 : // 3. two OFF spectrum is used (nearest interpolation)
436 : //
437 : // formulation is same as case 1.
438 : //
439 : struct NearestWeightScalingScheme : public SimpleWeightScalingScheme
440 : {
441 40 : static casacore::Float InterpolateScale(casacore::Double timeOn, casacore::Double exposureOn,
442 : casacore::Double timeOff1, casacore::Double exposureOff1,
443 : casacore::Double timeOff2, casacore::Double exposureOff2)
444 : {
445 40 : casacore::Double dt1 = abs(timeOn - timeOff1);
446 40 : casacore::Double dt2 = abs(timeOff2 - timeOn);
447 40 : return (dt1 <= dt2) ?
448 24 : SimpleScale(exposureOn, exposureOff1)
449 40 : : SimpleScale(exposureOn, exposureOff2);
450 : }
451 : };
452 :
453 9 : inline bool isEphemeris(casacore::String const &name) {
454 : // Check if given name is included in MDirection types
455 : casacore::Int nall, nextra;
456 : const casacore::uInt *typ;
457 9 : auto *types = casacore::MDirection::allMyTypes(nall, nextra, typ);
458 9 : auto start_extra = nall - nextra;
459 9 : auto capital_name = name;
460 9 : capital_name.upcase();
461 :
462 90 : for (auto i = start_extra; i < nall; ++i) {
463 83 : if (capital_name == types[i]) {
464 2 : return true;
465 : }
466 : }
467 :
468 7 : return false;
469 9 : }
470 :
471 : // Set number of correlations per spw
472 5161 : inline void setNumberOfCorrelationsPerSpw(casacore::MeasurementSet const &ms,
473 : casacore::Vector<casacore::Int> &nCorr) {
474 5161 : casacore::ScalarColumn<casacore::Int> spwIdCol(ms.dataDescription(), "SPECTRAL_WINDOW_ID");
475 5161 : casacore::ScalarColumn<casacore::Int> polIdCol(ms.dataDescription(), "POLARIZATION_ID");
476 5161 : casacore::ScalarColumn<casacore::Int> numCorrCol(ms.polarization(), "NUM_CORR");
477 5161 : auto numSpw = ms.spectralWindow().nrow();
478 5161 : auto const spwIdList = spwIdCol.getColumn();
479 5161 : auto const polIdList = polIdCol.getColumn();
480 5161 : auto const numCorrList = numCorrCol.getColumn();
481 5161 : if (nCorr.size() != numSpw) {
482 0 : nCorr.resize(numSpw);
483 : }
484 : // initialize number of correlations with 0
485 5161 : nCorr = 0;
486 22101 : for (auto i = 0u; i < spwIdList.size(); ++i) {
487 16940 : auto const spwId = spwIdList[i];
488 16940 : auto const polId = polIdList[i];
489 16940 : auto const numCorr = numCorrList[polId];
490 16940 : nCorr[spwId] = numCorr;
491 : }
492 5161 : }
493 : }
494 :
495 : namespace casa { //# NAMESPACE CASA - BEGIN
496 : //
497 : // SingleDishSkyCal
498 : //
499 :
500 : // Constructor
501 51 : SingleDishSkyCal::SingleDishSkyCal(VisSet& vs)
502 : : VisCal(vs),
503 : SolvableVisCal(vs),
504 51 : currAnt_(-1),
505 51 : engineC_(vs.numberSpw(), NULL),
506 51 : engineF_(vs.numberSpw(), NULL),
507 51 : currentSky_(vs.numberSpw(), NULL),
508 51 : currentSkyOK_(vs.numberSpw(), NULL),
509 102 : nCorr_(nSpw())
510 : {
511 51 : debuglog << "SingleDishSkyCal::SingleDishSkyCal(VisSet& vs)" << debugpost;
512 51 : append() = false;
513 :
514 51 : initializeSky();
515 51 : initializeCorr();
516 51 : }
517 :
518 3 : SingleDishSkyCal::SingleDishSkyCal(const MSMetaInfoForCal& msmc)
519 : : VisCal(msmc),
520 : SolvableVisCal(msmc),
521 3 : currAnt_(-1),
522 3 : engineC_(msmc.nSpw(), NULL),
523 3 : engineF_(msmc.nSpw(), NULL),
524 3 : currentSky_(msmc.nSpw(), NULL),
525 3 : currentSkyOK_(msmc.nSpw(), NULL),
526 6 : nCorr_(nSpw())
527 : {
528 3 : debuglog << "SingleDishSkyCal::SingleDishSkyCal(const MSMetaInfoForCal& msmc)" << debugpost;
529 3 : append() = False;
530 :
531 3 : initializeSky();
532 3 : initializeCorr();
533 3 : }
534 :
535 0 : SingleDishSkyCal::SingleDishSkyCal(const Int& nAnt)
536 : : VisCal(nAnt),
537 : SolvableVisCal(nAnt),
538 0 : currAnt_(-1),
539 0 : engineC_(1, NULL),
540 0 : engineF_(1, NULL),
541 0 : currentSky_(1, NULL),
542 0 : currentSkyOK_(1, NULL),
543 0 : nCorr_(nSpw())
544 : {
545 0 : debuglog << "SingleDishSkyCal::SingleDishSkyCal(const Int& nAnt)" << debugpost;
546 0 : append() = false;
547 :
548 0 : initializeSky();
549 0 : initializeCorr();
550 0 : }
551 :
552 : // Destructor
553 54 : SingleDishSkyCal::~SingleDishSkyCal()
554 : {
555 54 : debuglog << "SingleDishSkyCal::~SingleDishSkyCal()" << debugpost;
556 :
557 54 : finalizeSky();
558 54 : }
559 :
560 0 : void SingleDishSkyCal::guessPar(VisBuffer& /*vb*/)
561 : {
562 0 : }
563 :
564 0 : void SingleDishSkyCal::differentiate( CalVisBuffer & /*cvb*/)
565 : {
566 0 : }
567 :
568 0 : void SingleDishSkyCal::differentiate(VisBuffer& /*vb*/, Cube<Complex>& /*V*/,
569 : Array<Complex>& /*dV*/, Matrix<Bool>& /*Vflg*/)
570 : {
571 0 : }
572 :
573 0 : void SingleDishSkyCal::accumulate(SolvableVisCal* /*incr*/,
574 : const Vector<Int>& /*fields*/)
575 : {
576 0 : }
577 :
578 0 : void SingleDishSkyCal::diffSrc(VisBuffer& /*vb*/, Array<Complex>& /*dV*/)
579 : {
580 0 : }
581 :
582 0 : void SingleDishSkyCal::fluxscale(const String& /*outfile*/,
583 : const Vector<Int>& /*refFieldIn*/,
584 : const Vector<Int>& /*tranFieldIn*/,
585 : const Vector<Int>& /*inRefSpwMap*/,
586 : const Vector<String>& /*fldNames*/,
587 : const Float& /*inGainThres*/,
588 : const String& /*antSel*/,
589 : const String& /*timerangeSel*/,
590 : const String& /*scanSel*/,
591 : fluxScaleStruct& /*oFluxScaleStruct*/,
592 : const String& /*oListFile*/,
593 : const Bool& /*incremental*/,
594 : const Int& /*fitorder*/,
595 : const Bool& /*display*/)
596 : {
597 0 : }
598 :
599 0 : void SingleDishSkyCal::listCal(const Vector<Int> /*ufldids*/, const Vector<Int> /*uantids*/,
600 : const Matrix<Int> /*uchanids*/,
601 : //const Int& /*spw*/, const Int& /*chan*/,
602 : const String& /*listfile*/, const Int& /*pagerows*/)
603 : {
604 0 : }
605 :
606 23 : void SingleDishSkyCal::setApply(const Record& apply)
607 : {
608 : // Override interp
609 : // default frequency interpolation option is 'linearflag'
610 23 : Record applyCopy(apply);
611 23 : if (applyCopy.isDefined("interp")) {
612 23 : String interp = applyCopy.asString("interp");
613 23 : if (!interp.contains(',')) {
614 : //fInterpType() = "linearflag";
615 21 : String newInterp = interp + ",linearflag";
616 21 : applyCopy.define("interp", newInterp);
617 21 : }
618 23 : }
619 : else {
620 0 : applyCopy.define("interp", "linear,linearflag");
621 : }
622 :
623 : // call parent method
624 23 : SolvableVisCal::setApply(applyCopy);
625 23 : }
626 :
627 17 : void SingleDishSkyCal::fillCalibrationTable(casacore::MeasurementSet const &reference_data){
628 17 : String dataColName = (reference_data.tableDesc().isColumn("FLOAT_DATA")) ? "FLOAT_DATA" : "DATA";
629 :
630 17 : if ( dataColName == "FLOAT_DATA")
631 7 : fillCalibrationTable<FloatDataColumnAccessor>(reference_data);
632 : else
633 10 : fillCalibrationTable<DataColumnAccessor>(reference_data);
634 17 : }
635 :
636 : template<class DataRealComponentAccessor>
637 17 : void SingleDishSkyCal::fillCalibrationTable(MeasurementSet const &ms) {
638 17 : Int cols[] = {MS::FIELD_ID, MS::ANTENNA1, MS::FEED1,
639 : MS::DATA_DESC_ID};
640 17 : Int *colsp = cols;
641 17 : Block<Int> sortCols(4, colsp, false);
642 17 : MSIter msIter(ms, sortCols, 0.0, false, false);
643 99 : for (msIter.origin(); msIter.more(); msIter++) {
644 41 : Table const current = msIter.table();
645 41 : uInt nrow = current.nrow();
646 41 : debuglog << "nrow = " << nrow << debugpost;
647 41 : if (nrow == 0) {
648 0 : debuglog << "Skip" << debugpost;
649 0 : continue;
650 : }
651 41 : Int ispw = msIter.spectralWindowId();
652 41 : if (nChanParList()[ispw] == 4) {
653 : // Skip WVR
654 : debuglog << "Skip " << ispw
655 0 : << "(nchan " << nChanParList()[ispw] << ")"
656 0 : << debugpost;
657 0 : continue;
658 : }
659 : debuglog << "Process " << ispw
660 41 : << "(nchan " << nChanParList()[ispw] << ")"
661 82 : << debugpost;
662 :
663 41 : Int ifield = msIter.fieldId();
664 41 : ScalarColumn<Int> antennaCol(current, "ANTENNA1");
665 41 : currAnt_ = antennaCol(0);
666 41 : ScalarColumn<Int> feedCol(current, "FEED1");
667 : debuglog << "FIELD_ID " << msIter.fieldId()
668 : << " ANTENNA1 " << currAnt_
669 : << " FEED1 " << feedCol(0)
670 : << " DATA_DESC_ID " << msIter.dataDescriptionId()
671 41 : << debugpost;
672 41 : ScalarColumn<Double> timeCol(current, "TIME");
673 41 : ScalarColumn<Double> exposureCol(current, "EXPOSURE");
674 41 : ScalarColumn<Double> intervalCol(current, "INTERVAL");
675 41 : DataRealComponentAccessor dataCol(current);
676 41 : ArrayColumn<Bool> flagCol(current, "FLAG");
677 41 : ScalarColumn<Bool> flagRowCol(current, "FLAG_ROW");
678 41 : Vector<Double> timeList = timeCol.getColumn();
679 41 : Vector<Double> exposure = exposureCol.getColumn();
680 41 : Vector<Double> interval = intervalCol.getColumn();
681 41 : Vector<Double> timeInterval(timeList.nelements());
682 41 : Slice slice1(0, nrow - 1);
683 41 : Slice slice2(1, nrow - 1);
684 41 : timeInterval(slice1) = timeList(slice2) - timeList(slice1);
685 41 : timeInterval[nrow-1] = DBL_MAX;
686 41 : IPosition cellShape = flagCol.shape(0);
687 41 : size_t nelem = cellShape.product();
688 41 : Matrix<Float> dataSum(cellShape, new Float[nelem], TAKE_OVER);
689 41 : Matrix<Float> weightSum(cellShape, new Float[nelem], TAKE_OVER);
690 41 : dataSum = 0.0f;
691 41 : weightSum = 0.0f;
692 41 : Matrix<Bool> resultMask(cellShape, new Bool[nelem], TAKE_OVER);
693 41 : resultMask = true;
694 41 : Vector<Bool> flagRow = flagRowCol.getColumn();
695 41 : Double threshold = 1.1;
696 41 : Double timeCentroid = 0.0;
697 41 : size_t numSpectra = 0;
698 41 : Double effectiveExposure = 0.0;
699 4865 : for (uInt i = 0; i < nrow; ++i) {
700 2412 : if (flagRow(i)) {
701 0 : continue;
702 : }
703 :
704 2412 : numSpectra++;
705 2412 : timeCentroid += timeList[i];
706 2412 : effectiveExposure += exposure[i];
707 :
708 4824 : Matrix<Bool> mask = !flagCol(i);
709 2412 : MaskedArray<Float> mdata(dataCol(i), mask);
710 2412 : MaskedArray<Float> weight(Matrix<Float>(mdata.shape(), exposure[i]), mask);
711 2412 : dataSum += mdata * weight;
712 2412 : weightSum += weight;
713 :
714 2412 : Double gap = 2.0 * timeInterval[i] /
715 2412 : (interval[i] + interval[(i < nrow-1)?i+1:i]);
716 2412 : if (gap > threshold) {
717 423 : debuglog << "flush accumulated data at row " << i << debugpost;
718 : // Here we can safely use data() since internal storeage
719 : // is contiguous
720 423 : Float *data_ = dataSum.data();
721 423 : const Float *weight_ = weightSum.data();
722 423 : Bool *flag_ = resultMask.data();
723 1567605 : for (size_t j = 0; j < dataSum.nelements(); ++j) {
724 1567182 : if (weight_[j] == 0.0f) {
725 3936 : data_[j] = 0.0;
726 3936 : flag_[j] = false;
727 : }
728 : else {
729 1563246 : data_[j] /= weight_[j];
730 : }
731 : }
732 :
733 423 : currSpw() = ispw;
734 423 : currField() = ifield;
735 423 : timeCentroid /= (Double)numSpectra;
736 423 : refTime() = timeCentroid;
737 423 : interval_ = effectiveExposure;
738 :
739 423 : debuglog << "spw " << ispw << ": solveAllRPar.shape=" << solveAllRPar().shape() << " nPar=" << nPar() << " nChanPar=" << nChanPar() << " nElem=" << nElem() << debugpost;
740 :
741 423 : size_t const nCorr = dataSum.shape()[0];
742 423 : Cube<Float> const rpar = dataSum.addDegenerate(1);
743 423 : Cube<Bool> const parOK = resultMask.addDegenerate(1);
744 1269 : for (size_t iCorr = 0; iCorr < nCorr; ++iCorr) {
745 846 : solveAllRPar().yzPlane(iCorr) = rpar.yzPlane(iCorr);
746 846 : solveAllParOK().yzPlane(iCorr) = parOK.yzPlane(iCorr);
747 846 : solveAllParErr().yzPlane(iCorr) = 0.1; // TODO: this is tentative
748 846 : solveAllParSNR().yzPlane(iCorr) = 1.0; // TODO: this is tentative
749 : }
750 :
751 423 : keepNCT();
752 :
753 423 : dataSum = 0.0f;
754 423 : weightSum = 0.0f;
755 423 : resultMask = true;
756 423 : timeCentroid = 0.0;
757 423 : numSpectra = 0;
758 423 : effectiveExposure = 0.0;
759 423 : }
760 : }
761 : }
762 17 : }
763 :
764 568 : void SingleDishSkyCal::keepNCT() {
765 : // Call parent to do general stuff
766 : // This adds nElem() rows
767 568 : SolvableVisCal::keepNCT();
768 :
769 : // We are adding to the most-recently added rows
770 568 : RefRows rows(ct_->nrow()-nElem(),ct_->nrow()-1,1);
771 568 : Vector<Int> ant(nElem(), currAnt_);
772 :
773 : // update ANTENNA1 and ANTENNA2 with appropriate value
774 568 : CTMainColumns ncmc(*ct_);
775 568 : ncmc.antenna1().putColumnCells(rows,ant);
776 568 : ncmc.antenna2().putColumnCells(rows,ant);
777 :
778 : // update INTERVAL
779 568 : ncmc.interval().putColumnCells(rows,interval_);
780 568 : }
781 :
782 31 : void SingleDishSkyCal::initSolvePar()
783 : {
784 31 : debuglog << "SingleDishSkyCal::initSolvePar()" << debugpost;
785 468 : for (Int ispw=0;ispw<nSpw();++ispw) {
786 :
787 437 : currSpw()=ispw;
788 :
789 437 : switch(parType()) {
790 437 : case VisCalEnum::REAL: {
791 437 : solveAllRPar().resize(nPar(),nChanPar(),nElem());
792 437 : solveAllRPar()=0.0;
793 437 : solveRPar().reference(solveAllRPar());
794 437 : break;
795 : }
796 0 : default: {
797 0 : throw(AipsError("Internal error(Calibrater Module): Unsupported parameter type "
798 0 : "found in SingleDishSkyCal::initSolvePar()"));
799 : break;
800 : }
801 : }//switch
802 :
803 437 : solveAllParOK().resize(solveAllRPar().shape());
804 437 : solveAllParErr().resize(solveAllRPar().shape());
805 437 : solveAllParSNR().resize(solveAllRPar().shape());
806 437 : solveAllParOK()=false;
807 437 : solveAllParErr()=0.0;
808 437 : solveAllParSNR()=0.0;
809 437 : solveParOK().reference(solveAllParOK());
810 437 : solveParErr().reference(solveAllParErr());
811 437 : solveParSNR().reference(solveAllParSNR());
812 : }
813 31 : currSpw()=0;
814 31 : currAnt_ = 0;
815 :
816 31 : interval_.resize(nElem());
817 31 : }
818 :
819 5076 : void SingleDishSkyCal::syncMeta2(const vi::VisBuffer2& vb)
820 : {
821 : // call method in parent class
822 5076 : VisCal::syncMeta2(vb);
823 :
824 : // fill interval array with exposure
825 5076 : interval_.reference(vb.exposure());
826 5076 : debuglog << "SingleDishSkyCal::syncMeta2 interval_= " << interval_ << debugpost;
827 :
828 5076 : setNumberOfCorrelationsPerSpw(vb.ms(), nCorr_);
829 5076 : debuglog << "nCorr_ = " << nCorr_ << debugpost;
830 5076 : debuglog << "currSpw() = " << currSpw() << debugpost;
831 5076 : debuglog << "nPar() = " << nPar() << debugpost;
832 5076 : }
833 :
834 5076 : void SingleDishSkyCal::syncCalMat(const Bool &/*doInv*/)
835 : {
836 5076 : debuglog << "SingleDishSkyCal::syncCalMat" << debugpost;
837 5076 : debuglog << "nAnt()=" << nAnt() << ", nElem()=" << nElem() << ", nBln()=" << nBln() << debugpost;
838 5076 : debuglog << "Spw " << currSpw() << "nchanmat, ncalmat" << nChanMat() << ", " << nCalMat() << debugpost;
839 5076 : debuglog << "nChanPar = " << nChanPar() << debugpost;
840 5076 : currentSky().resize(2, nChanMat(), nCalMat());
841 5076 : currentSky().unique();
842 5076 : currentSkyOK().resize(currentSky().shape());
843 5076 : currentSkyOK().unique();
844 5076 : debuglog << "currentSkyOK.shape()=" << currentSkyOK().shape() << debugpost;
845 :
846 : // sky data from caltable
847 5076 : debuglog << "currRPar().shape()=" << currRPar().shape() << debugpost;
848 5076 : if (currRPar().shape().product() > 0) {
849 5076 : debuglog << "currRPar() = " << currRPar().xzPlane(0) << debugpost;
850 : }
851 :
852 5076 : convertArray(currentSky(), currRPar());
853 5076 : currentSkyOK() = currParOK();
854 5076 : debuglog << "currentTime() = " << setprecision(16) << currTime() << debugpost;
855 5076 : debuglog << "currentSky() = " << currentSky().xzPlane(0) << debugpost;
856 5076 : debuglog << "currentSkyOK() = " << currentSkyOK().xzPlane(0) << debugpost;
857 :
858 : // weight calibration
859 5076 : if (calWt())
860 4464 : syncWtScale();
861 :
862 5076 : debuglog << "SingleDishSkyCal::syncCalMat DONE" << debugpost;
863 5076 : }
864 :
865 0 : void SingleDishSkyCal::syncDiffMat()
866 : {
867 0 : debuglog << "SingleDishSkyCal::syncDiffMat()" << debugpost;
868 0 : }
869 :
870 4464 : void SingleDishSkyCal::syncWtScale()
871 : {
872 4464 : debuglog << "syncWtScale" << debugpost;
873 :
874 : // allocate necessary memory
875 4464 : currWtScale().resize(currentSky().shape());
876 4464 : currWtScale() = 1.0;
877 :
878 : // Calculate the weight scaling
879 4464 : if (tInterpType() == "nearest") {
880 36 : calcWtScale<NearestWeightScalingScheme>();
881 : }
882 : else {
883 4428 : calcWtScale<LinearWeightScalingScheme>();
884 : }
885 :
886 4464 : debuglog << "syncWtScale DONE" << debugpost;
887 4464 : }
888 :
889 : template<class ScalingScheme>
890 4464 : void SingleDishSkyCal::calcWtScale()
891 : {
892 4464 : debuglog << "calcWtScale<ScalingScheme>" << debugpost;
893 4464 : auto const key = std::make_pair(currObs(), currSpw());
894 4464 : debuglog << "for obs " << currObs() << " and spw " << currSpw() << debugpost;
895 4464 : decltype(wtScaleData_)::iterator iter = wtScaleData_.find(key);
896 4464 : map<Int, Matrix<Double> > wtMap;
897 4464 : if (iter == wtScaleData_.end()) {
898 30 : debuglog << "construct weight scaling data for obs " << currObs() << " spw " << currSpw() << debugpost;
899 30 : debuglog << "number of antennas = " << nAnt() << debugpost;
900 371 : for (Int iAnt = 0; iAnt < nAnt(); ++iAnt) {
901 100 : CTInterface cti(*ct_);
902 100 : MSSelection mss;
903 100 : mss.setObservationExpr(String::toString(currObs()));
904 100 : mss.setSpwExpr(String::toString(currSpw()));
905 100 : mss.setAntennaExpr(String::toString(iAnt) + "&&&");
906 100 : TableExprNode ten = mss.toTableExprNode(&cti);
907 100 : NewCalTable temp;
908 : try {
909 147 : getSelectedTable(temp, *ct_, ten, "");
910 94 : } catch (AipsError x) {
911 47 : continue;
912 : }
913 53 : temp = temp.sort("TIME");
914 53 : wtMap.emplace(std::piecewise_construct,
915 : std::forward_as_tuple(iAnt),
916 53 : std::forward_as_tuple(2, temp.nrow()));
917 53 : Matrix<Double> &arr = wtMap.at(iAnt);
918 53 : debuglog << "ant " << iAnt << " arr.shape = " << arr.shape() << debugpost;
919 53 : ScalarColumn<Double> col(temp, "TIME");
920 53 : auto timeCol = arr.row(0);
921 53 : col.getColumn(timeCol, False);
922 53 : col.attach(temp, "INTERVAL");
923 53 : auto intervalCol = arr.row(1);
924 53 : debuglog << "timeCol.size() == " << timeCol.size() << " intervalCol.size() = " << intervalCol.size() << debugpost;
925 53 : col.getColumn(intervalCol, False);
926 : }
927 30 : wtScaleData_.insert(std::make_pair(key, wtMap));
928 : } else {
929 4434 : wtMap = iter->second;
930 : }
931 :
932 : {
933 : // for debugging
934 4464 : debuglog << "wtMap keys: ";
935 9342 : for (decltype(wtMap)::iterator i = wtMap.begin(); i != wtMap.end(); ++i) {
936 4878 : debuglog << i->first << " ";
937 : }
938 4464 : debuglog << debugpost;
939 : }
940 :
941 15088 : for (Int iAnt = 0; iAnt < nAnt(); ++iAnt) {
942 5746 : decltype(wtMap)::iterator i = wtMap.find(iAnt);
943 5746 : if (i == wtMap.end()) {
944 868 : continue;
945 : }
946 4878 : auto const &mat = i->second;
947 4878 : debuglog << "matrix shape " << mat.shape() << debugpost;
948 4878 : Vector<Double> const &timeCol = mat.row(0);
949 4878 : Vector<Double> const &intervalCol = mat.row(1);
950 4878 : size_t nrow = timeCol.size();
951 4878 : debuglog << "timeCol = " << timeCol << debugpost;
952 4878 : debuglog << "intervalCol = " << intervalCol << debugpost;
953 4878 : debuglog << "iAnt " << iAnt << " nrow=" << nrow << debugpost;
954 4878 : if (currTime() < timeCol[0]) {
955 141 : debuglog << "Use nearest OFF weight (0)" << debugpost;
956 141 : currWtScale().xyPlane(iAnt) = ScalingScheme::SimpleScale(interval_[iAnt], intervalCol[0]);
957 : }
958 4737 : else if (currTime() > timeCol[nrow-1]) {
959 976 : debuglog << "Use nearest OFF weight (" << nrow-1 << ")" << debugpost;
960 976 : currWtScale().xyPlane(iAnt) = ScalingScheme::SimpleScale(interval_[iAnt], intervalCol[nrow-1]);
961 : }
962 : else {
963 3761 : debuglog << "Use interpolated OFF weight" << debugpost;
964 85587 : for (size_t irow = 0; irow < nrow ; ++irow) {
965 85587 : if (currTime() == timeCol[irow]) {
966 194 : currWtScale().xyPlane(iAnt) = ScalingScheme::SimpleScale(interval_[iAnt], intervalCol[irow]);
967 194 : break;
968 : }
969 85393 : else if (currTime() < timeCol[irow]) {
970 3567 : currWtScale().xyPlane(iAnt) = ScalingScheme::InterpolateScale(currTime(), interval_[iAnt],
971 : timeCol[irow-1], intervalCol[irow-1],
972 : timeCol[irow], intervalCol[irow]);
973 3567 : break;
974 : }
975 : }
976 : }
977 : }
978 4464 : debuglog << "currWtScale() = " << currWtScale().xzPlane(0) << debugpost;
979 :
980 4464 : debuglog << "calcWtScale<ScalingScheme> DONE" << debugpost;
981 4464 : }
982 :
983 0 : Float SingleDishSkyCal::calcPowerNorm(Array<Float>& /*amp*/, const Array<Bool>& /*ok*/)
984 : {
985 0 : return 0.0f;
986 : }
987 :
988 0 : void SingleDishSkyCal::applyCal(VisBuffer& /*vb*/, Cube<Complex>& /*Vout*/, Bool /*trial*/)
989 : {
990 0 : throw AipsError("Single dish calibration doesn't support applyCal. Please use applyCal2");
991 : }
992 :
993 5076 : void SingleDishSkyCal::applyCal2(vi::VisBuffer2 &vb, Cube<Complex> &Vout, Cube<Float> &Wout,
994 : Bool trial)
995 : {
996 5076 : debuglog << "SingleDishSkyCal::applycal2" << debugpost;
997 5076 : debuglog << "nrow, nchan=" << vb.nRows() << "," << vb.nChannels() << debugpost;
998 5076 : debuglog << "antenna1: " << vb.antenna1() << debugpost;
999 5076 : debuglog << "antenna2: " << vb.antenna2() << debugpost;
1000 5076 : debuglog << "spw: " << vb.spectralWindows() << debugpost;
1001 5076 : debuglog << "field: " << vb.fieldId() << debugpost;
1002 :
1003 : // References to VB2's contents' _data_
1004 5076 : Vector<Bool> flagRv(vb.flagRow());
1005 5076 : Vector<Int> a1v(vb.antenna1());
1006 5076 : Vector<Int> a2v(vb.antenna2());
1007 5076 : Cube<Bool> flagCube(vb.flagCube());
1008 5076 : Cube<Complex> visCube(Vout);
1009 5076 : ArrayIterator<Float> wt(Wout,2);
1010 5076 : Matrix<Float> wtmat;
1011 :
1012 : // Data info/indices
1013 : Int* dataChan;
1014 5076 : Bool* flagR=&flagRv(0);
1015 5076 : Int* a1=&a1v(0);
1016 5076 : Int* a2=&a2v(0);
1017 :
1018 : // iterate rows
1019 5076 : Int nRow=vb.nRows();
1020 5076 : Int nChanDat=vb.nChannels();
1021 5076 : Vector<Int> dataChanv(vb.getChannelNumbers(0)); // All rows have same chans
1022 : // cout << currSpw() << " startChan() = " << startChan() << " nChanMat() = " << nChanMat() << " nChanDat="<<nChanDat <<endl;
1023 :
1024 : // setup calibration engine
1025 5076 : engineC().setNumChannel(nChanDat);
1026 5076 : engineC().setNumPolarization(vb.nCorrelations());
1027 :
1028 5076 : debuglog << "typesize=" << engineC().typesize() << debugpost;
1029 :
1030 : // Matrix slice of visCube
1031 : // TODO: storage must be aligned for future use
1032 5076 : Matrix<Complex> visCubeSlice;
1033 5076 : Matrix<Bool> flagCubeSlice;
1034 :
1035 10566 : for (Int row=0; row<nRow; row++,flagR++,a1++,a2++) {
1036 5490 : debuglog << "spw: " << currSpw() << " antenna: " << *a1 << debugpost;
1037 5490 : assert(*a1 == *a2);
1038 :
1039 : // Solution channel registration
1040 5490 : Int solCh0(0);
1041 5490 : dataChan=&dataChanv(0);
1042 :
1043 : // If cal _parameters_ are not freqDep (e.g., a delay)
1044 : // the startChan() should be the same as the first data channel
1045 5490 : if (freqDepMat() && !freqDepPar())
1046 0 : startChan()=(*dataChan);
1047 :
1048 : // Solution and data array registration
1049 5490 : engineC().sync(currentSky()(0,solCh0,*a1), currentSkyOK()(0,solCh0,*a1));
1050 5490 : visCubeSlice.reference(visCube.xyPlane(row));
1051 5490 : flagCubeSlice.reference(flagCube.xyPlane(row));
1052 :
1053 5490 : if (trial) {
1054 : // only update flag info
1055 0 : engineC().flag(flagCubeSlice);
1056 : }
1057 : else {
1058 : // apply calibration
1059 5490 : engineC().apply(visCubeSlice, flagCubeSlice);
1060 : }
1061 :
1062 : // If requested, update the weights
1063 5490 : if (!trial && calWt()) {
1064 4878 : wtmat.reference(wt.array());
1065 4878 : updateWt2(wtmat,*a1);
1066 : }
1067 :
1068 5490 : if (!trial)
1069 5490 : wt.next();
1070 :
1071 : }
1072 5076 : }
1073 :
1074 31 : void SingleDishSkyCal::selfGatherAndSolve(VisSet& vs, VisEquation& /*ve*/)
1075 : {
1076 31 : debuglog << "SingleDishSkyCal::selfGatherAndSolve()" << debugpost;
1077 :
1078 31 : MeasurementSet const &user_selection = vs.iter().ms();
1079 :
1080 31 : debuglog << "nspw = " << nSpw() << debugpost;
1081 : // Get and store the number of channels per spectral window
1082 31 : fillNChanParList(MeasurementSet(msName()), nChanParList());
1083 :
1084 : // Get and store the number of correlations per spectral window
1085 31 : setNumberOfCorrelationsPerSpw(user_selection, nCorr_);
1086 31 : debuglog << "nCorr_ = " << nCorr_ << debugpost;
1087 :
1088 : // Create a new in-memory calibration table to fill up
1089 31 : createMemCalTable();
1090 :
1091 : // Setup shape of solveAllRPar
1092 31 : nElem() = 1;
1093 31 : initSolvePar();
1094 :
1095 : // Select from user selection reference data associated with science target
1096 31 : MeasurementSet reference_data = selectReferenceData(user_selection);
1097 29 : logSink().origin(LogOrigin("SingleDishSkyCal","selfGatherAndSolve"));
1098 : {
1099 : // Log row numbers in a readable way
1100 29 : std::ostringstream msg;
1101 29 : auto * us_num_fmt = new CommaSeparatedThousands();
1102 29 : std::locale us_like_locale(std::locale::classic(), us_num_fmt);
1103 : // us_like_locale is responsible for deleting us_num_fmt from its destructor
1104 : // Ref: https://en.cppreference.com/w/cpp/locale/locale/locale
1105 : // Constructor (7) + Notes section
1106 : // So we must NOT delete us_num_fmt.
1107 29 : msg.imbue(us_like_locale);
1108 29 : msg << "Selected: " << std::right << std::setw(10) << reference_data.nrow() << " rows of reference data" << '\n'
1109 29 : << "out of : " << std::right << std::setw(10) << user_selection.nrow() << " rows of user-selected data";
1110 29 : logSink() << msg.str() << LogIO::POST;
1111 29 : }
1112 29 : logSink().origin(LogOrigin());
1113 29 : if (reference_data.nrow() == 0)
1114 1 : throw AipsError("No reference integration found in user-selected data. Please double-check your data selection criteria.");
1115 :
1116 : // Fill observing-mode-dependent columns of the calibration table
1117 : // Implementation is observing-mode-specific
1118 28 : fillCalibrationTable(reference_data);
1119 :
1120 : // Fill remaining columns of the calibration table,
1121 : // which are computed the same way for all observing modes
1122 : // ---- 1) FIELD_ID, SCAN_NUMBER, OBSERVATION_ID columns
1123 28 : assignCTScanField(*ct_, msName());
1124 :
1125 : // ---- 2) WEIGHT column
1126 : // update weight without Tsys
1127 : // formula is chanWidth [Hz] * interval [sec]
1128 28 : updateWeight(*ct_);
1129 :
1130 : // Store calibration table on disk
1131 28 : storeNCT();
1132 29 : }
1133 :
1134 54 : void SingleDishSkyCal::initializeSky()
1135 : {
1136 54 : debuglog << "SingleDishSkyCal::initializeSky()" << debugpost;
1137 813 : for (Int ispw=0;ispw<nSpw(); ispw++) {
1138 759 : currentSky_[ispw] = new Cube<Complex>();
1139 759 : currentSkyOK_[ispw] = new Cube<Bool>();
1140 759 : engineC_[ispw] = new SkyCal<Complex, Complex>();
1141 : }
1142 54 : }
1143 :
1144 54 : void SingleDishSkyCal::finalizeSky()
1145 : {
1146 813 : for (Int ispw=0;ispw<nSpw(); ispw++) {
1147 759 : if (currentSky_[ispw]) delete currentSky_[ispw];
1148 759 : if (currentSkyOK_[ispw]) delete currentSkyOK_[ispw];
1149 759 : if (engineC_[ispw]) delete engineC_[ispw];
1150 759 : if (engineF_[ispw]) delete engineF_[ispw];
1151 : }
1152 :
1153 54 : }
1154 :
1155 54 : void SingleDishSkyCal::initializeCorr()
1156 : {
1157 54 : File msPath(msName());
1158 54 : if (msPath.exists()) {
1159 54 : setNumberOfCorrelationsPerSpw(MeasurementSet(msName()), nCorr_);
1160 : } else {
1161 0 : nCorr_ = 1;
1162 : }
1163 54 : }
1164 :
1165 4878 : void SingleDishSkyCal::updateWt2(Matrix<Float> &weight, const Int &antenna1)
1166 : {
1167 : // apply weight scaling factor
1168 4878 : Matrix<Float> const factor = currWtScale().xyPlane(antenna1);
1169 4878 : debuglog << "factor.shape() = " << factor.shape() << debugpost;
1170 4878 : debuglog << "weight.shape() = " << weight.shape() << debugpost;
1171 4878 : debuglog << "weight = " << weight << debugpost;
1172 :
1173 4878 : auto const wtShape = weight.shape();
1174 4878 : size_t const nCorr = wtShape[0];
1175 4878 : size_t const nChan = wtShape[1];
1176 : // for each correlation
1177 14624 : for (size_t iCorr = 0; iCorr < nCorr; ++iCorr) {
1178 9746 : auto wSlice = weight.row(iCorr);
1179 9746 : auto const fSlice = factor.row(iCorr);
1180 9746 : if (fSlice.nelements() == nChan) {
1181 8224 : wSlice *= fSlice;
1182 1522 : } else if (nChan == 1) {
1183 : // take mean of spectral weight factor to apply it to scalar weight
1184 1522 : wSlice *= mean(fSlice);
1185 : } else {
1186 0 : throw AipsError("Shape mismatch between input weight and weight scaling factor");
1187 : }
1188 9746 : }
1189 4878 : }
1190 :
1191 : //
1192 : // SingleDishPositionSwitchCal
1193 : //
1194 :
1195 : // Constructor
1196 27 : SingleDishPositionSwitchCal::SingleDishPositionSwitchCal(VisSet& vs)
1197 : : VisCal(vs),
1198 27 : SingleDishSkyCal(vs)
1199 : {
1200 27 : debuglog << "SingleDishPositionSwitchCal::SingleDishPositionSwitchCal(VisSet& vs)" << debugpost;
1201 27 : }
1202 :
1203 3 : SingleDishPositionSwitchCal::SingleDishPositionSwitchCal(const MSMetaInfoForCal& msmc)
1204 : : VisCal(msmc),
1205 3 : SingleDishSkyCal(msmc)
1206 : {
1207 3 : debuglog << "SingleDishPositionSwitchCal::SingleDishPositionSwitchCal(const MSMetaInfoForCal& msmc)" << debugpost;
1208 3 : }
1209 :
1210 0 : SingleDishPositionSwitchCal::SingleDishPositionSwitchCal(const Int& nAnt)
1211 : : VisCal(nAnt),
1212 0 : SingleDishSkyCal(nAnt)
1213 : {
1214 0 : debuglog << "SingleDishPositionSwitchCal::SingleDishPositionSwitchCal(const Int& nAnt)" << debugpost;
1215 0 : }
1216 :
1217 : // Destructor
1218 60 : SingleDishPositionSwitchCal::~SingleDishPositionSwitchCal()
1219 : {
1220 30 : debuglog << "SingleDishPositionSwitchCal::~SingleDishPositionSwitchCal()" << debugpost;
1221 60 : }
1222 :
1223 12 : MeasurementSet SingleDishPositionSwitchCal::selectReferenceData(MeasurementSet const &user_selection)
1224 : {
1225 12 : std::ostringstream qry;
1226 12 : constexpr auto eol = '\n';
1227 12 : qry << "using style python" << eol;
1228 : qry << "with [" << eol
1229 : << "select" << eol
1230 12 : << " [select TELESCOPE_NAME from ::OBSERVATION][OBSERVATION_ID] as TELESCOPE_NAME," << eol;
1231 : // Purposively not using TAQL's default mscal UDF library alias for derivedmscal
1232 : // to workaround a bug in casacore UDFBase::createUDF
1233 : qry << " derivedmscal.spwcol('NUM_CHAN') as NUM_CHAN" << eol
1234 : << "from" << eol
1235 : << " $1" << eol
1236 : << "] as metadata" << eol
1237 : << "select * from $1 , metadata" << eol
1238 12 : << "where " << eol;
1239 : // Row contains single-dish auto-correlation data,
1240 12 : qry << " ( ANTENNA1 == ANTENNA2 ) and" << eol ;
1241 12 : qry << " ( FEED1 == FEED2 ) and" << eol ;
1242 :
1243 : // has not been marked as invalid,
1244 12 : qry << " ( not(FLAG_ROW) ) and " << eol ;
1245 : // holds at least 1 data marked as valid,
1246 12 : qry << " ( not(all(FLAG)) ) and " << eol;
1247 : // ---- Note: both conditions above are required because FLAG and FLAG_ROW are not synchronized:
1248 : // a valid row (FLAG_ROW=False) may contain only invalid data: all(FLAG)=True
1249 :
1250 : // has observational intent: OBSERVE_TARGET#OFF_SOURCE,
1251 : qry << " ( STATE_ID in [ " << eol
1252 : << " select rowid() " << eol
1253 : << " from ::STATE" << eol
1254 : << " where " << eol
1255 : << " upper(OBS_MODE) ~ m/^OBSERVE_TARGET#OFF_SOURCE/ " << eol
1256 : << " ]" << eol
1257 12 : << " ) and" << eol;
1258 : // excluding - for ALMA - rows from Water Vapor Radiometers spectral windows, which have 4 channels
1259 : qry << " (" << eol
1260 : << " ( metadata.TELESCOPE_NAME != 'ALMA' ) or" << eol
1261 : << " (" << eol
1262 : << " ( metadata.TELESCOPE_NAME == 'ALMA' ) and" << eol
1263 : << " ( metadata.NUM_CHAN != 4 )" << eol
1264 : << " )" << eol
1265 12 : << " )" << eol;
1266 : debuglog << "SingleDishPositionSwitchCal::selectReferenceData(): taql query:" << eol
1267 12 : << qry.str() << debugpost;
1268 36 : return MeasurementSet(tableCommand(qry.str(), user_selection).table());
1269 12 : }
1270 :
1271 11 : void SingleDishPositionSwitchCal::fillCalibrationTable(casacore::MeasurementSet const &reference_data){
1272 11 : String dataColName = (reference_data.tableDesc().isColumn("FLOAT_DATA")) ? "FLOAT_DATA" : "DATA";
1273 :
1274 11 : if ( dataColName == "FLOAT_DATA")
1275 4 : fillCalibrationTable<FloatDataColumnAccessor>(reference_data);
1276 : else
1277 7 : fillCalibrationTable<DataColumnAccessor>(reference_data);
1278 11 : }
1279 :
1280 : template<class DataRealComponentAccessor>
1281 11 : void SingleDishPositionSwitchCal::fillCalibrationTable(casacore::MeasurementSet const &reference_data)
1282 : {
1283 11 : debuglog << "SingleDishPositionSwitchCal::fillCalibrationTable()" << debugpost;
1284 :
1285 : // Sort columns define the granularity at which we average data obtained
1286 : // by observing the reference position associated with the science target
1287 11 : constexpr size_t nSortColumns = 8;
1288 11 : Int columns[nSortColumns] = {
1289 : MS::OBSERVATION_ID,
1290 : MS::PROCESSOR_ID,
1291 : MS::FIELD_ID,
1292 : MS::ANTENNA1,
1293 : MS::FEED1,
1294 : MS::DATA_DESC_ID,
1295 : MS::SCAN_NUMBER,
1296 : MS::STATE_ID
1297 : };
1298 11 : Int *columnsPtr = columns;
1299 11 : Block<Int> sortColumns(nSortColumns, columnsPtr, false);
1300 :
1301 : // Iterator
1302 11 : constexpr Bool doGroupAllTimesTogether = true;
1303 11 : constexpr Double groupAllTimesTogether = doGroupAllTimesTogether ? 0.0 : -1.0;
1304 :
1305 11 : constexpr Bool doAddDefaultSortColumns = false;
1306 11 : constexpr Bool doStoreSortedTableOnDisk = false;
1307 :
1308 11 : MSIter msIter(reference_data, sortColumns,
1309 : groupAllTimesTogether, doAddDefaultSortColumns, doStoreSortedTableOnDisk);
1310 :
1311 : // Main loop: compute values of calibration table's columns
1312 301 : for (msIter.origin(); msIter.more(); msIter++) {
1313 145 : const auto iterTable = msIter.table();
1314 145 : const auto iterRows = iterTable.nrow();
1315 145 : if (iterRows == 0) continue;
1316 :
1317 : // TIME column of calibration table: mean of selected MAIN.TIME
1318 145 : ScalarColumn<Double> timeCol(iterTable, "TIME");
1319 145 : refTime() = mean(timeCol.getColumn());
1320 :
1321 : // FIELD_ID column of calibration table
1322 145 : currSpw() = msIter.spectralWindowId();
1323 :
1324 : // SPECTRAL_WINDOW_ID column of calibration table
1325 145 : currField() = msIter.fieldId();
1326 :
1327 : // ANTENNA1, ANTENNA2 columns of calibration table
1328 145 : ScalarColumn<Int> antenna1Col(iterTable, "ANTENNA1");
1329 145 : currAnt_ = antenna1Col(0);
1330 :
1331 : // INTERVAL column of calibration table: sum of selected MAIN.EXPOSURE
1332 145 : ScalarColumn<Double> exposureCol(iterTable, "EXPOSURE");
1333 145 : const auto exposure = exposureCol.getColumn();
1334 145 : interval_ = sum(exposure);
1335 :
1336 : // SCAN_NUMBER, OBSERVATION_ID columns of calibration table
1337 : // Not computed/updated here
1338 : #ifdef SDCALSKY_DEBUG
1339 : ScalarColumn<Int> scanNumberCol(iterTable, "SCAN_NUMBER");
1340 : const auto scan_number = scanNumberCol(0);
1341 : ScalarColumn<Int> stateIdCol(iterTable, "STATE_ID");
1342 : const auto state_id = stateIdCol(0);
1343 : cout << "field=" << currField()
1344 : << " ant=" << currAnt_
1345 : << " ddid=" << msIter.dataDescriptionId()
1346 : << " spw=" << currSpw()
1347 : << " scan_number=" << scan_number
1348 : << " state_id=" << state_id
1349 : << " nrows=" << iterRows
1350 : << endl;
1351 : #endif
1352 :
1353 : // FPARAM column of calibration table: weighted mean of valid data, weight = exposure
1354 : // + PARAMERR, FLAG, SNR
1355 : // ---- Get data shape from FLAG column
1356 145 : ArrayColumn<Bool> flagCol(iterTable, "FLAG");
1357 145 : const auto dataShape = flagCol.shape(0);
1358 145 : const auto nCorr = dataShape[0];
1359 145 : const auto nChannels = dataShape[1];
1360 : // ---- Initialize accumulators
1361 145 : Matrix<Float> weightedMean(nCorr, nChannels, 0.0f);
1362 145 : Matrix<Float> weightsSums(nCorr, nChannels, 0.0f);
1363 : // ---- Compute weighted mean of valid data
1364 145 : DataRealComponentAccessor dataAccessor(iterTable);
1365 482 : for (std::remove_const<decltype(iterRows)>::type iterRow = 0; iterRow < iterRows ; iterRow++){
1366 674 : Matrix<Bool> dataIsValid = not flagCol(iterRow);
1367 337 : MaskedArray<Float> validData(dataAccessor(iterRow), dataIsValid);
1368 337 : const auto rowExposure = static_cast<Float>(exposure[iterRow]);
1369 337 : weightedMean += validData * rowExposure;
1370 337 : MaskedArray<Float> validWeight(Matrix<Float>(validData.shape(), rowExposure), dataIsValid);
1371 337 : weightsSums += validWeight;
1372 : }
1373 145 : const Matrix<Bool> weightsSumsIsNonZero = ( weightsSums != 0.0f );
1374 145 : weightedMean /= MaskedArray<Float>(weightsSums,weightsSumsIsNonZero);
1375 : // ---- Update solveAll*() members
1376 145 : const Cube<Float> realParam = weightedMean.addDegenerate(1);
1377 145 : const Cube<Bool> realParamIsValid = weightsSumsIsNonZero.addDegenerate(1);
1378 429 : for (std::remove_const<decltype(nCorr)>::type corr = 0; corr < nCorr; corr++) {
1379 284 : solveAllRPar().yzPlane(corr) = realParam.yzPlane(corr); // FPARAM
1380 284 : solveAllParOK().yzPlane(corr) = realParamIsValid.yzPlane(corr); // not FLAG
1381 284 : solveAllParErr().yzPlane(corr) = 0.1; // PARAMERR. TODO: this is tentative
1382 284 : solveAllParSNR().yzPlane(corr) = 1.0; // SNR. TODO: this is tentative
1383 : }
1384 :
1385 : // WEIGHT column of calibration table
1386 : //
1387 :
1388 : // Update in-memory calibration table
1389 145 : keepNCT();
1390 : }
1391 11 : }
1392 :
1393 : //
1394 : // SingleDishRasterCal
1395 : //
1396 :
1397 : // Constructor
1398 10 : SingleDishRasterCal::SingleDishRasterCal(VisSet& vs)
1399 : : VisCal(vs),
1400 : SingleDishSkyCal(vs),
1401 10 : fraction_(0.1),
1402 10 : numEdge_(-1)
1403 : {
1404 10 : debuglog << "SingleDishRasterCal::SingleDishRasterCal(VisSet& vs)" << debugpost;
1405 10 : }
1406 :
1407 0 : SingleDishRasterCal::SingleDishRasterCal(const MSMetaInfoForCal& msmc)
1408 : : VisCal(msmc),
1409 : SingleDishSkyCal(msmc),
1410 0 : fraction_(0.1),
1411 0 : numEdge_(-1)
1412 : {
1413 0 : debuglog << "SingleDishRasterCal::SingleDishRasterCal(const MSMetaInfoForCal& msmc)" << debugpost;
1414 0 : }
1415 :
1416 0 : SingleDishRasterCal::SingleDishRasterCal(const Int& nAnt)
1417 : : VisCal(nAnt),
1418 0 : SingleDishSkyCal(nAnt)
1419 : {
1420 0 : debuglog << "SingleDishRasterCal::SingleDishRasterCal(const Int& nAnt)" << debugpost;
1421 0 : }
1422 :
1423 : // Destructor
1424 20 : SingleDishRasterCal::~SingleDishRasterCal()
1425 : {
1426 10 : debuglog << "SingleDishRasterCal::~SingleDishRasterCal()" << debugpost;
1427 20 : }
1428 :
1429 10 : void SingleDishRasterCal::setSolve(const Record& solve)
1430 : {
1431 : // edge detection parameter for otfraster mode
1432 10 : if (solve.isDefined("fraction")) {
1433 10 : fraction_ = solve.asFloat("fraction");
1434 : }
1435 10 : if (solve.isDefined("numedge")) {
1436 10 : numEdge_ = solve.asInt("numedge");
1437 : }
1438 :
1439 10 : logSink() << "fraction=" << fraction_ << endl
1440 10 : << "numedge=" << numEdge_ << LogIO::POST;
1441 :
1442 : // call parent setSolve
1443 10 : SolvableVisCal::setSolve(solve);
1444 10 : }
1445 :
1446 10 : MeasurementSet SingleDishRasterCal::selectReferenceData(MeasurementSet const &ms)
1447 : {
1448 10 : debuglog << "SingleDishRasterCal::selectReferenceData" << debugpost;
1449 10 : const Record specify;
1450 10 : std::ostringstream oss;
1451 10 : oss << "SELECT FROM $1 WHERE ";
1452 10 : String delimiter = "";
1453 : // for (Int iant = 0; iant < nAnt(); ++iant) {
1454 : // Vector<String> timeRangeList = detectRaster(msName(), iant, fraction_, numEdge_);
1455 : // debuglog << "timeRangeList=" << ::toString(timeRangeList) << debugpost;
1456 : // oss << delimiter;
1457 : // oss << "(ANTENNA1 == " << iant << " && ANTENNA2 == " << iant << " && (";
1458 : // String separator = "";
1459 : // for (size_t i = 0; i < timeRangeList.size(); ++i) {
1460 : // if (timeRangeList[i].size() > 0) {
1461 : // oss << separator << "(" << timeRangeList[i] << ")";
1462 : // separator = " || ";
1463 : // }
1464 : // }
1465 : // oss << "))";
1466 : // debuglog << "oss.str()=" << oss.str() << debugpost;
1467 : // delimiter = " || ";
1468 : // }
1469 : // use ANTENNA 0 for reference antenna
1470 10 : Vector<String> timeRangeList = detectRaster(ms, 0, fraction_, numEdge_);
1471 8 : debuglog << "timeRangeList=" << ::toString(timeRangeList) << debugpost;
1472 8 : oss << delimiter;
1473 8 : oss << "(ANTENNA1 == ANTENNA2 && (";
1474 8 : String separator = "";
1475 72 : for (size_t i = 0; i < timeRangeList.size(); ++i) {
1476 64 : if (timeRangeList[i].size() > 0) {
1477 48 : oss << separator << "(" << timeRangeList[i] << ")";
1478 48 : separator = " || ";
1479 : }
1480 : }
1481 8 : oss << "))";
1482 8 : debuglog << "oss.str()=" << oss.str() << debugpost;
1483 :
1484 : oss //<< ")"
1485 8 : << " ORDER BY FIELD_ID, ANTENNA1, FEED1, DATA_DESC_ID, TIME";
1486 24 : return MeasurementSet(tableCommand(oss.str(), ms).table());
1487 14 : }
1488 :
1489 : //
1490 : // SingleDishOtfCal
1491 : //
1492 :
1493 : // Constructor
1494 14 : SingleDishOtfCal::SingleDishOtfCal(VisSet& vs)
1495 : : VisCal(vs),
1496 : SingleDishSkyCal(vs),
1497 14 : fraction_(0.1),
1498 14 : pixel_scale_(0.5),
1499 14 : msSel_(vs.iter().ms())
1500 : {
1501 14 : debuglog << "SingleDishOtfCal::SingleDishOtfCal(VisSet& vs)" << debugpost;
1502 14 : }
1503 :
1504 : /*
1505 : SingleDishOtfCal::SingleDishOtfCal(const MSMetaInfoForCal& msmc)
1506 : : VisCal(msmc),
1507 : SingleDishSkyCal(msmc),
1508 : fraction_(0.1),
1509 : pixel_scale_(0.5),
1510 : msSel_(vs.iter().ms()) ************need MS!
1511 : {
1512 : debuglog << "SingleDishOtfCal::SingleDishOtfCal(VisSet& vs)" << debugpost;
1513 : }
1514 : */
1515 9 : void SingleDishOtfCal::setSolve(const Record& solve)
1516 : {
1517 : // edge detection parameter for otfraster mode
1518 9 : if (solve.isDefined("fraction")) {
1519 9 : fraction_ = solve.asFloat("fraction");
1520 : }
1521 :
1522 9 : logSink() << "fraction=" << fraction_ << LogIO::POST;
1523 :
1524 : // call parent setSolve
1525 9 : SolvableVisCal::setSolve(solve);
1526 9 : }
1527 :
1528 : /*
1529 : SingleDishOtfCal::SingleDishOtfCal(const Int& nAnt)
1530 : : VisCal(nAnt),
1531 : SingleDishSkyCal(nAnt)
1532 : {
1533 : debuglog << "SingleDishOtfCal::SingleDishOtfCal(const Int& nAnt)" << debugpost;
1534 : }
1535 : */
1536 :
1537 : // Destructor
1538 28 : SingleDishOtfCal::~SingleDishOtfCal()
1539 : {
1540 14 : debuglog << "SingleDishOtfCal::~SingleDishOtfCal()" << debugpost;
1541 28 : }
1542 :
1543 9 : MeasurementSet SingleDishOtfCal::selectReferenceData(MeasurementSet const &ms)
1544 : {
1545 9 : PointingDirectionCalculator calc(ms);
1546 9 : calc.setDirectionListMatrixShape(PointingDirectionCalculator::ROW_MAJOR);
1547 :
1548 : // Check the coordinates system type used to store the pointing measurements
1549 9 : const MSPointing& tbl_pointing = ms.pointing();
1550 9 : MSPointingColumns pointing_cols(tbl_pointing);
1551 9 : const ROArrayMeasColumn< MDirection >& direction_cols = pointing_cols.directionMeasCol();
1552 9 : const MeasRef<MDirection>& direction_ref_frame = direction_cols.getMeasRef();
1553 9 : uInt ref_frame_type = direction_ref_frame.getType();
1554 :
1555 : // If non-celestial coordinates (AZEL*) are used, convert to celestial ones
1556 9 : switch (ref_frame_type) {
1557 0 : case MDirection::AZEL : // Fall through
1558 : case MDirection::AZELSW :
1559 : case MDirection::AZELGEO :
1560 : case MDirection::AZELSWGEO : {
1561 0 : const String& ref_frame_name = MDirection::showType(ref_frame_type);
1562 0 : debuglog << "Reference frame of pointings coordinates is non-celestial: " << ref_frame_name << debugpost;
1563 0 : String j2000(MDirection::showType(MDirection::J2000));
1564 0 : debuglog << "Pointings coordinates will be converted to: " << j2000 << debugpost;
1565 0 : calc.setFrame(j2000);
1566 0 : }
1567 : }
1568 : // Extract edge pointings for each (field_id,antenna,spectral window) triple
1569 : // MeasurementSet 2 specification / FIELD table:
1570 : // . FIELD_ID column is removed
1571 : // . FIELD table is directly indexed using the FIELD_ID value in MAIN
1572 9 : const MSField& tbl_field = ms.field();
1573 9 : const String &field_col_name_str = tbl_field.columnName(MSField::MSFieldEnums::SOURCE_ID);
1574 18 : ScalarColumn<Int> source_id_col(tbl_field, field_col_name_str);
1575 9 : const MSAntenna& tbl_antenna = ms.antenna();
1576 9 : const String &col_name_str = tbl_antenna.columnName(MSAntenna::MSAntennaEnums::NAME);
1577 18 : ScalarColumn<String> antenna_name(tbl_antenna,col_name_str);
1578 9 : const MSSpectralWindow& tbl_spectral_window = ms.spectralWindow();
1579 :
1580 : // make a map between SOURCE_ID and source NAME
1581 9 : const MSSource &tbl_source = ms.source();
1582 18 : ScalarColumn<Int> id_col(tbl_source, tbl_source.columnName(MSSource::MSSourceEnums::SOURCE_ID));
1583 18 : ScalarColumn<String> name_col(tbl_source, tbl_source.columnName(MSSource::MSSourceEnums::NAME));
1584 18 : std::map<Int, String> source_map;
1585 18 : for (uInt irow = 0; irow < tbl_source.nrow(); ++irow) {
1586 9 : auto source_id = id_col(irow);
1587 9 : if (source_map.find(source_id) == source_map.end()) {
1588 9 : source_map[source_id] = name_col(irow);
1589 : }
1590 : }
1591 :
1592 18 : Vector<casacore::rownr_t> rowList;
1593 :
1594 18 : for (uInt field_id=0; field_id < tbl_field.nrow(); ++field_id){
1595 9 : String field_sel(casacore::String::toString<uInt>(field_id));
1596 9 : String const source_name = source_map.at(source_id_col(field_id));
1597 :
1598 : // Set ephemeris flag if source name is the one recognized as a moving source
1599 9 : if (isEphemeris(source_name)) {
1600 2 : calc.setMovingSource(source_name);
1601 : }
1602 : else {
1603 7 : calc.unsetMovingSource();
1604 : }
1605 :
1606 20 : for (uInt ant_id=0; ant_id < tbl_antenna.nrow(); ++ant_id){
1607 11 : String ant_sel(antenna_name(ant_id) + "&&&");
1608 46 : for (uInt spw_id=0; spw_id < tbl_spectral_window.nrow(); ++spw_id){
1609 35 : String spw_sel(casacore::String::toString<uInt>(spw_id));
1610 : // Filter user selection by (field_id,antenna,spectral window) triple
1611 : try {
1612 217 : calc.selectData(ant_sel,spw_sel,field_sel);
1613 : }
1614 26 : catch (AipsError& e) { // Empty selection
1615 : // Note: when the underlying MSSelection is empty
1616 : // MSSelection internally catches an MSSelectionError error
1617 : // but does not re-throw it. It throws instead an AipsError
1618 : // copy-constructed from the MSSelectionError
1619 26 : continue;
1620 26 : }
1621 : debuglog << "field_id: " << field_id
1622 : << " ant_id: " << ant_id
1623 : << " spw: " << spw_id
1624 9 : << " ==> selection rows: " << calc.getNrowForSelectedMS() << debugpost;
1625 : // Get time-interpolated celestial pointing directions for the filtered user selection
1626 9 : Matrix<Double> pointings_dirs = calc.getDirection();
1627 : // Project directions onto image plane
1628 : // pixel_scale_ :
1629 : // . hard-coded to 0.5 in constructor
1630 : // . is applied to the median separation of consecutive pointing directions by the projector
1631 : // . projector pixel size = 0.5*directions_median
1632 9 : debuglog << "pixel_scale:" << pixel_scale_ << debugpost;
1633 9 : OrthographicProjector p(pixel_scale_);
1634 9 : p.setDirection(pointings_dirs);
1635 9 : const Matrix<Double> &pointings_coords = p.project();
1636 : // Extract edges of the observed region for the (field_id,antenna,spectral window) triple
1637 9 : SakuraAlignedArray<Double> pointings_x(pointings_coords.row(0));
1638 9 : SakuraAlignedArray<Double> pointings_y(pointings_coords.row(1));
1639 9 : SakuraAlignedArray<Bool> is_edge_storage(pointings_coords.ncolumn());
1640 9 : Vector<Bool> is_edge = is_edge_storage.casaVector();
1641 9 : is_edge = false;
1642 9 : double pixel_size = 0.0;
1643 : {
1644 : // CAS-9956
1645 : // Mitigation of memory usage due to unexpectedly large number of pixels.
1646 : // Currently CreateMaskNearEdgeDouble requires 2*sizeof(size_t)*num_pixels bytes
1647 : // of memory. If this value exceeds 2GB, the mitigation will be activated.
1648 9 : Double const num_pixels = p.p_size()[0] * p.p_size()[1];
1649 9 : auto const estimated_memory = num_pixels * 2 * sizeof(size_t);
1650 9 : constexpr Double kMaxMemory = 2.0e9;
1651 9 : if (estimated_memory >= kMaxMemory && pixel_scale_ < 1.0) {
1652 0 : LogIO os;
1653 0 : os << LogOrigin("PointingDirectionProjector", "scale_and_center", WHERE);
1654 0 : os << LogIO::WARN << "Estimated Memory: " << estimated_memory << LogIO::POST;
1655 0 : os << LogIO::WARN << "Mitigation of memory usage is activated. pixel scale is set to 1.0" << LogIO::POST;
1656 : // pixel_size can be set to 2.0 since projection grid spacing is estimated from half of median separation
1657 : // between neighboring pixels so that pixel_width will become about 1.0 if pixel_size is 0.
1658 0 : pixel_size = 2.0;
1659 0 : os << LogIO::WARN << "pixel_size is set to " << pixel_size << LogIO::POST;
1660 0 : }
1661 : }
1662 : // libsakura 2.0: setting pixel_size=0.0 means that CreateMaskNearEdgeDouble will
1663 : // . compute the median separation of consecutive pointing coordinates
1664 : // . use an "edge detection pixel size" = 0.5*coordinates_median (pixel scale hard-coded to 0.5)
1665 9 : debuglog << "sakura library function call: parameters info:" << debugpost;
1666 9 : debuglog << "in: fraction: " << fraction_ << debugpost;
1667 9 : debuglog << "in: pixel size: " << pixel_size << debugpost;
1668 9 : debuglog << "in: pixels count: (nx = " << p.p_size()[0] << " , ny = " << p.p_size()[1] << debugpost;
1669 9 : debuglog << "in: pointings_coords.ncolumn(): " << pointings_coords.ncolumn() << debugpost;
1670 18 : LIBSAKURA_SYMBOL(Status) status = LIBSAKURA_SYMBOL(CreateMaskNearEdgeDouble)(
1671 : fraction_, pixel_size,
1672 9 : pointings_coords.ncolumn(), pointings_x.data(), pointings_y.data(),
1673 : nullptr /* blc_x */, nullptr /* blc_y */,
1674 : nullptr /* trc_x */, nullptr /* trc_y */,
1675 : is_edge.data());
1676 9 : bool edges_detection_ok = ( status == LIBSAKURA_SYMBOL(Status_kOK) );
1677 9 : if ( ! edges_detection_ok ) {
1678 0 : debuglog << "sakura error: status=" << status << debugpost;
1679 : }
1680 9 : AlwaysAssert(edges_detection_ok,AipsError);
1681 : // Compute ROW ids of detected edges. ROW "ids" are ROW ids in the MS filtered by user selection.
1682 9 : auto index_2_rowid = calc.getRowIdForOriginalMS();
1683 9 : size_t edges_count = ntrue(is_edge);
1684 9 : size_t rowListIndex = rowList.size();
1685 9 : rowList.resize(rowList.size() + edges_count, True);
1686 7689 : for (size_t i = 0; i < is_edge.size(); ++i){
1687 7680 : if ( is_edge[i] ) {
1688 2028 : rowList[rowListIndex] = index_2_rowid[i]; // i;
1689 2028 : ++rowListIndex;
1690 : }
1691 : }
1692 9 : debuglog << "edges_count=" << edges_count << debugpost;
1693 9 : AlwaysAssert(edges_count > 0, AipsError);
1694 : #ifdef SDCALSKY_DEBUG
1695 : stringstream fname;
1696 : fname << calTableName().c_str() << ".edges."
1697 : << field_id << "_" << ant_id << "_" << spw_id
1698 : << ".csv" ;
1699 : debuglog << "Save pointing directions and coordinates to:" << debugpost;
1700 : debuglog << fname.str() << debugpost;
1701 : ofstream ofs(fname.str());
1702 : AlwaysAssert(ofs.good(), AipsError);
1703 : ofs << "row_id,field_id,ant_id,spw_id,triple_key,dir_0,dir_1,coord_0,coord_1,edge_0,edge_1,is_edge" << endl;
1704 : const auto &d0 = pointings_dirs.row(0);
1705 : const auto &d1 = pointings_dirs.row(1);
1706 : const auto &c0 = pointings_coords.row(0);
1707 : const auto &c1 = pointings_coords.row(1);
1708 : for (uInt j=0; j<d0.size(); j++) {
1709 : ofs << index_2_rowid[j] << ","
1710 : << field_id << "," << ant_id << "," << spw_id << ","
1711 : << field_id << "_" << ant_id << "_" << spw_id << ","
1712 : << d0(j) << "," << d1(j) << ","
1713 : << c0(j) << "," << c1(j) << "," ;
1714 : if ( is_edge[j] ) ofs << c0(j) << "," << c1(j) << "," << 1 << endl;
1715 : else ofs << ",," << 0 << endl;
1716 : }
1717 : #endif
1718 35 : }
1719 11 : }
1720 9 : }
1721 9 : Bool have_off_spectra = (rowList.size() > 0);
1722 9 : AlwaysAssert(have_off_spectra, AipsError);
1723 18 : MeasurementSet msSel = ms(rowList);
1724 9 : debuglog << "rowList = " << rowList << debugpost;
1725 18 : return msSel;
1726 9 : }
1727 :
1728 : } //# NAMESPACE CASA - END
1729 :
|
