Line data Source code
1 : //# MSTransformDataHandler.cc: This file contains the implementation of the MSTransformDataHandler class.
2 : //#
3 : //# CASA - Common Astronomy Software Applications (http://casa.nrao.edu/)
4 : //# Copyright (C) Associated Universities, Inc. Washington DC, USA 2011, All rights reserved.
5 : //# Copyright (C) European Southern Observatory, 2011, All rights reserved.
6 : //#
7 : //# This library is free software; you can redistribute it and/or
8 : //# modify it under the terms of the GNU Lesser General Public
9 : //# License as published by the Free software Foundation; either
10 : //# version 2.1 of the License, or (at your option) any later version.
11 : //#
12 : //# This library is distributed in the hope that it will be useful,
13 : //# but WITHOUT ANY WARRANTY, without even the implied warranty of
14 : //# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : //# Lesser General Public License for more details.
16 : //#
17 : //# You should have received a copy of the GNU Lesser General Public
18 : //# License along with this library; if not, write to the Free Software
19 : //# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20 : //# MA 02111-1307 USA
21 : //# $Id: $
22 :
23 : #include <mstransform/MSTransform/MSTransformDataHandler.h>
24 : #include <casacore/tables/Tables/TableProxy.h>
25 : #include <casacore/tables/TaQL/TableParse.h>
26 : #include <casacore/ms/MSOper/MSMetaData.h>
27 : #include <asdmstman/AsdmStMan.h>
28 :
29 :
30 : using namespace casacore;
31 : namespace casa { //# NAMESPACE CASA - BEGIN
32 :
33 : /////////////////////////////////////////////
34 : /// MSTransformDataHandler implementation ///
35 : /////////////////////////////////////////////
36 :
37 : // -----------------------------------------------------------------------
38 : //
39 : // -----------------------------------------------------------------------
40 1890 : MSTransformDataHandler::MSTransformDataHandler(const String& theMS, Table::TableOption option,
41 : bool virtualModelCol,bool virtualCorrectedCol,
42 1890 : bool reindex) :
43 1890 : ms_p(MeasurementSet(theMS, option)),
44 1890 : mssel_p(ms_p),
45 1890 : msc_p(NULL),
46 1890 : mscIn_p(NULL),
47 1890 : keepShape_p(true),
48 1890 : antennaSel_p(false),
49 1890 : timeBin_p(-1.0),
50 1890 : scanString_p(""),
51 1890 : intentString_p(""),
52 1890 : obsString_p(""),
53 1890 : uvrangeString_p(""),
54 1890 : taqlString_p(""),
55 1890 : timeRange_p(""),
56 1890 : arrayExpr_p(""),
57 1890 : combine_p(""),
58 1890 : fitorder_p(-1),
59 1890 : fitspw_p("*"),
60 1890 : fitoutspw_p("*"),
61 1890 : virtualModelCol_p(virtualModelCol),
62 1890 : virtualCorrectedCol_p(virtualCorrectedCol),
63 5670 : reindex_p(reindex)
64 : {
65 1890 : return;
66 0 : }
67 :
68 : // -----------------------------------------------------------------------
69 : //
70 : // -----------------------------------------------------------------------
71 0 : MSTransformDataHandler::MSTransformDataHandler(const MeasurementSet& ms,
72 : bool virtualModelCol,bool virtualCorrectedCol,
73 0 : bool reindex) :
74 0 : ms_p(ms),
75 0 : mssel_p(ms_p),
76 0 : msc_p(NULL),
77 0 : mscIn_p(NULL),
78 0 : keepShape_p(true),
79 0 : antennaSel_p(false),
80 0 : timeBin_p(-1.0),
81 0 : scanString_p(""),
82 0 : intentString_p(""),
83 0 : obsString_p(""),
84 0 : uvrangeString_p(""),
85 0 : taqlString_p(""),
86 0 : timeRange_p(""),
87 0 : arrayExpr_p(""),
88 0 : combine_p(""),
89 0 : fitorder_p(-1),
90 0 : fitspw_p("*"),
91 0 : fitoutspw_p("*"),
92 0 : virtualModelCol_p(virtualModelCol),
93 0 : virtualCorrectedCol_p(virtualCorrectedCol),
94 0 : reindex_p(reindex)
95 : {
96 0 : return;
97 0 : }
98 :
99 : // -----------------------------------------------------------------------
100 : //
101 : // -----------------------------------------------------------------------
102 1890 : MSTransformDataHandler::~MSTransformDataHandler()
103 : {
104 1890 : if (msc_p) delete msc_p;
105 1890 : msc_p = nullptr;
106 :
107 1890 : if (mscIn_p) delete mscIn_p;
108 1890 : mscIn_p = nullptr;
109 :
110 1890 : msOut_p=MeasurementSet();
111 :
112 : // parseColumnNames unavoidably has a static String and Vector<MS::PredefinedColumns>.
113 : // Collapse them down to free most of that memory.
114 1890 : parseColumnNames("None");
115 :
116 1890 : return;
117 1890 : }
118 :
119 : // -----------------------------------------------------------------------
120 : //
121 : // -----------------------------------------------------------------------
122 1890 : const Vector<MS::PredefinedColumns>& MSTransformDataHandler::parseColumnNames(String col)
123 : {
124 : // Memorize both for efficiency and so that the info
125 : // message at the bottom isn't unnecessarily repeated.
126 1890 : static String my_colNameStr = "";
127 1890 : static Vector<MS::PredefinedColumns> my_colNameVect;
128 :
129 1890 : col.upcase();
130 1890 : if(col == my_colNameStr && col != "")
131 : {
132 0 : return my_colNameVect;
133 : }
134 1890 : else if(col == "NONE")
135 : {
136 1890 : my_colNameStr = "";
137 1890 : my_colNameVect.resize(0);
138 1890 : return my_colNameVect;
139 : }
140 :
141 : uInt nNames;
142 :
143 0 : if(col.contains("ALL"))
144 : {
145 0 : nNames = 3;
146 0 : my_colNameVect.resize(nNames);
147 0 : my_colNameVect[0] = MS::DATA;
148 0 : my_colNameVect[1] = MS::MODEL_DATA;
149 0 : my_colNameVect[2] = MS::CORRECTED_DATA;
150 : }
151 : else
152 : {
153 0 : nNames = dataColStrToEnums(col, my_colNameVect);
154 : }
155 :
156 : // Whether or not the MS has the columns is checked by verifyColumns().
157 : // Unfortunately it cannot be done here because this is a static method.
158 :
159 :
160 : /*
161 : * jagonzal: Redundant logging message (this info is already provided by MSTransformManager)
162 : *
163 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
164 : // "NONE" is used by the destructor
165 : if(col != "NONE")
166 : {
167 : os << LogIO::NORMAL << "Using ";
168 : for(uInt i = 0; i < nNames; ++i)
169 : {
170 : os << MS::columnName(my_colNameVect[i]) << " ";
171 : }
172 :
173 : os << " column" << (my_colNameVect.nelements() > 1 ? "s." : ".") << LogIO::POST;
174 : }
175 : */
176 :
177 0 : my_colNameStr = col;
178 0 : return my_colNameVect;
179 : }
180 :
181 : // -----------------------------------------------------------------------
182 : //
183 : // -----------------------------------------------------------------------
184 1882 : const Vector<MS::PredefinedColumns>& MSTransformDataHandler::parseColumnNames( String col,
185 : bool produceModel,
186 : const MeasurementSet& msref,
187 : bool virtualModelCol,
188 : bool virtualCorrectedCol)
189 : {
190 : // Memorize both for efficiency and so that the info
191 : // message at the bottom isn't unnecessarily repeated.
192 1882 : static String my_colNameStr = "";
193 1882 : static Vector<MS::PredefinedColumns> my_colNameVect;
194 :
195 : // Data columns to pick up if present.
196 1882 : Vector<MS::PredefinedColumns> wanted;
197 :
198 1882 : col.upcase();
199 :
200 : // This version of parseColumnNames does not reuse results of previous calls
201 : // but always checks the given columns because it cannot be certain that msref
202 : // refers to the same MS with every call.
203 :
204 1882 : if (col == "NONE")
205 : {
206 0 : my_colNameStr = "";
207 0 : my_colNameVect.resize(0);
208 0 : return my_colNameVect;
209 : }
210 :
211 3764 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
212 :
213 : // Are we choosy?
214 1882 : const bool doAny = col.contains("ALL") || col.contains("ANY");
215 :
216 : uInt nPoss;
217 1882 : if (doAny)
218 : {
219 366 : nPoss = 5;
220 366 : wanted.resize(nPoss);
221 366 : wanted[0] = MS::DATA;
222 366 : wanted[1] = MS::MODEL_DATA;
223 366 : wanted[2] = MS::CORRECTED_DATA;
224 366 : wanted[3] = MS::FLOAT_DATA;
225 366 : wanted[4] = MS::LAG_DATA;
226 : }
227 : // split name string into individual names
228 : else
229 : {
230 1516 : nPoss = dataColStrToEnums(col, wanted);
231 : }
232 :
233 : // Add MODEL_DATA separately from 'datacolumn' selection
234 1882 : if (produceModel) {
235 3 : const auto nelem = wanted.nelements();
236 3 : wanted.resize(nelem + 1, true);
237 3 : wanted[nelem] = MS::MODEL_DATA;
238 3 : ++nPoss;
239 : // When producing output MS with DATA,MODEL from CORRECTED, setupMS will have
240 : // to add MS::DATA to the output MS, not MS::CORRECTED_DATA.
241 : // An alternative would be to add a special case in setupMS, similar
242 : // to mustWriteOnlyToData, when wanted == CORRECTED_DATA, MODEL_DATA.
243 3 : if (MS::CORRECTED_DATA == wanted[0]) {
244 2 : wanted[0] = MS::DATA;
245 : }
246 : }
247 :
248 1882 : uInt nFound = 0;
249 1882 : my_colNameVect.resize(0);
250 5230 : for (uInt i = 0; i < nPoss; ++i)
251 : {
252 3355 : if (msref.tableDesc().isColumn(MS::columnName(wanted[i])))
253 : {
254 2235 : ++nFound;
255 2235 : my_colNameVect.resize(nFound, true);
256 2235 : my_colNameVect[nFound - 1] = wanted[i];
257 : }
258 : // CAS-5348 (jagonzal): Model parameters check is done at construction time
259 : // (produceModel comes from uvcontsub, doesn't require MODEL in input MS)
260 1120 : else if ((wanted[i] == MS::MODEL_DATA and virtualModelCol) or produceModel)
261 : {
262 3 : ++nFound;
263 3 : my_colNameVect.resize(nFound, true);
264 3 : my_colNameVect[nFound - 1] = wanted[i];
265 : }
266 1117 : else if (wanted[i] == MS::CORRECTED_DATA and virtualCorrectedCol)
267 : {
268 9 : ++nFound;
269 9 : my_colNameVect.resize(nFound, true);
270 9 : my_colNameVect[nFound - 1] = wanted[i];
271 : }
272 1108 : else if (!doAny)
273 : {
274 7 : ostringstream ostr;
275 7 : ostr << "Desired column (" << MS::columnName(wanted[i])
276 7 : << ") not found in the input MS (" << msref.tableName()
277 7 : << ").";
278 7 : throw(AipsError(ostr.str()));
279 7 : }
280 : }
281 1875 : if (nFound == 0) throw(AipsError("Did not find and select any data columns."));
282 :
283 1875 : my_colNameStr = col;
284 1875 : return my_colNameVect;
285 1889 : }
286 :
287 : // -----------------------------------------------------------------------
288 : //
289 : // -----------------------------------------------------------------------
290 1516 : uInt MSTransformDataHandler::dataColStrToEnums(const String& col, Vector<MS::PredefinedColumns>& colvec)
291 : {
292 3032 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
293 1516 : String tmpNames(col);
294 1516 : Vector<String> tokens;
295 1516 : tmpNames.upcase();
296 :
297 : // split name string into individual names
298 : char * pch;
299 1516 : Int it = 0;
300 1516 : pch = strtok((char*) tmpNames.c_str(), " ,");
301 3038 : while (pch != NULL)
302 : {
303 1522 : tokens.resize(it + 1, true);
304 1522 : tokens[it] = String(pch);
305 1522 : ++it;
306 1522 : pch = strtok(NULL, " ,");
307 : }
308 :
309 1516 : uInt nNames = tokens.nelements();
310 :
311 1516 : uInt nFound = 0;
312 3038 : for (uInt i = 0; i < nNames; ++i)
313 : {
314 1522 : colvec.resize(nFound + 1, true);
315 1522 : colvec[nFound] = MS::UNDEFINED_COLUMN;
316 :
317 1522 : if ( tokens[i] == "OBSERVED"
318 1522 : || tokens[i] == "DATA"
319 3044 : || tokens[i] == MS::columnName(MS::DATA))
320 : {
321 822 : colvec[nFound++] = MS::DATA;
322 : }
323 700 : else if ( tokens[i] == "FLOAT"
324 700 : || tokens[i] == "FLOAT_DATA"
325 1400 : || tokens[i] == MS::columnName(MS::FLOAT_DATA))
326 : {
327 460 : colvec[nFound++] = MS::FLOAT_DATA;
328 : }
329 240 : else if ( tokens[i] == "LAG"
330 240 : || tokens[i] == "LAG_DATA"
331 480 : || tokens[i] == MS::columnName(MS::LAG_DATA))
332 : {
333 0 : colvec[nFound++] = MS::LAG_DATA;
334 : }
335 240 : else if ( tokens[i] == "MODEL"
336 202 : || tokens[i] == "MODEL_DATA"
337 442 : || tokens[i] == MS::columnName(MS::MODEL_DATA))
338 : {
339 38 : colvec[nFound++] = MS::MODEL_DATA;
340 : }
341 202 : else if ( tokens[i] == "CORRECTED"
342 0 : || tokens[i] == "CORRECTED_DATA"
343 202 : || tokens[i] == MS::columnName(MS::CORRECTED_DATA))
344 : {
345 202 : colvec[nFound++] = MS::CORRECTED_DATA;
346 : }
347 : // "NONE" is used by the destructor
348 0 : else if (tmpNames != "NONE")
349 : {
350 0 : os << LogIO::SEVERE;
351 :
352 0 : if (nFound == 0)
353 : {
354 0 : colvec[0] = MS::DATA;
355 0 : os << "Unrecognized data column " << tokens[i] << "...trying DATA.";
356 : }
357 : else
358 : {
359 0 : os << "Skipping unrecognized data column " << tokens[i];
360 : }
361 :
362 0 : os << LogIO::POST;
363 : }
364 : }
365 1516 : return nFound;
366 1516 : }
367 :
368 : // -----------------------------------------------------------------------
369 : //
370 : // -----------------------------------------------------------------------
371 1882 : bool MSTransformDataHandler::setmsselect( const String& spw, const String& field,
372 : const String& baseline, const String& scan,
373 : const String& uvrange, const String& taql,
374 : const Vector<Int>& step, const String& subarray,
375 : const String& correlation, const String& intent,
376 : const String& obs, const String& feed)
377 : {
378 3764 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
379 : bool ok;
380 :
381 1882 : String myspwstr(spw == "" ? "*" : spw);
382 3764 : Record selrec = ms_p.msseltoindex(myspwstr, field);
383 :
384 1882 : ok = selectSource(selrec.asArrayInt("field"));
385 :
386 : // All of the requested selection functions will be tried, even if an
387 : // earlier one has indicated its failure. This allows all of the selection
388 : // strings to be tested, yielding more complete feedback for the user
389 : // (fewer retries). This is a matter of taste, though. If the selections
390 : // turn out to be slow, this function should return on the first false.
391 :
392 1882 : if (!selectSpw(myspwstr, step))
393 : {
394 0 : os << LogIO::SEVERE << "No channels selected." << LogIO::POST;
395 0 : ok = false;
396 : }
397 :
398 1882 : if (baseline != "")
399 : {
400 786 : Vector<Int> antid(0);
401 786 : Vector<String> antstr(1, baseline);
402 786 : selectAntenna(antid, antstr);
403 786 : }
404 :
405 1882 : scanString_p = scan;
406 1882 : intentString_p = intent;
407 1882 : obsString_p = obs;
408 1882 : uvrangeString_p = uvrange;
409 1882 : taqlString_p = taql;
410 1882 : feedString_p = feed;
411 :
412 1882 : if (subarray != "") selectArray(subarray);
413 :
414 1882 : if (!selectCorrelations(correlation))
415 : {
416 0 : os << LogIO::SEVERE << "No correlations selected." << LogIO::POST;
417 0 : ok = false;
418 : }
419 :
420 1882 : return ok;
421 1882 : }
422 :
423 : // -----------------------------------------------------------------------
424 : //
425 : // -----------------------------------------------------------------------
426 1882 : bool MSTransformDataHandler::selectSource(const Vector<Int>& fieldid)
427 : {
428 3764 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
429 :
430 1882 : bool cando = true;
431 :
432 1882 : if (fieldid.nelements() < 1)
433 : {
434 1806 : fieldid_p = Vector<Int> (1, -1);
435 : }
436 76 : else if (fieldid.nelements() > ms_p.field().nrow())
437 : {
438 0 : os << LogIO::SEVERE << "More fields were requested than are in the input MS." << LogIO::POST;
439 0 : cando = false;
440 : }
441 76 : else if (max(fieldid) >= static_cast<Int> (ms_p.field().nrow()))
442 : {
443 : // Arriving here is very unlikely since if fieldid came from MSSelection
444 : // bad fields were presumably already quietly dropped.
445 0 : os << LogIO::SEVERE << "At least 1 field was requested that is not in the input MS." << LogIO::POST;
446 0 : cando = false;
447 : }
448 : else
449 : {
450 76 : fieldid_p = fieldid;
451 : }
452 :
453 1882 : if (fieldid_p.nelements() == 1 && fieldid_p[0] < 0)
454 : {
455 1806 : fieldid_p.resize(ms_p.field().nrow());
456 1806 : indgen(fieldid_p);
457 : }
458 :
459 1882 : return cando;
460 1882 : }
461 :
462 : // -----------------------------------------------------------------------
463 : //
464 : // -----------------------------------------------------------------------
465 1882 : bool MSTransformDataHandler::selectSpw(const String& spwstr,const Vector<Int>& steps)
466 : {
467 3764 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
468 :
469 1882 : MSSelection mssel;
470 1882 : String myspwstr(spwstr == "" ? "*" : spwstr);
471 1882 : spwString_p = myspwstr;
472 :
473 1882 : mssel.setSpwExpr(myspwstr);
474 :
475 1882 : widths_p = steps.copy();
476 1882 : if (widths_p.nelements() < 1)
477 : {
478 0 : widths_p.resize(1);
479 0 : widths_p[0] = 1;
480 : }
481 : else
482 : {
483 5010 : for (uInt k = 0; k < widths_p.nelements(); ++k)
484 : {
485 3128 : if (widths_p[k] == 0)
486 : {
487 0 : os << LogIO::WARN << "0 cannot be used for channel width...using 1 instead." << LogIO::POST;
488 0 : widths_p[k] = 1;
489 : }
490 : }
491 : }
492 :
493 : // Each row should have spw, start, stop, step
494 : // A single width is a default, but multiple widths should be used literally.
495 1882 : Matrix<Int> chansel = mssel.getChanList(&ms_p, 1);
496 :
497 1882 : if (chansel.nrow() > 0)
498 : {
499 : // Use myspwstr if it selected anything...
500 1882 : spw_p = chansel.column(0);
501 1882 : chanStart_p = chansel.column(1);
502 1882 : chanEnd_p = chansel.column(2);
503 1882 : chanStep_p = chansel.column(3);
504 :
505 1882 : uInt nspw = chanEnd_p.nelements();
506 1882 : nchan_p.resize(nspw);
507 :
508 : // A single width is a default, but multiple widths should be used literally.
509 1882 : if (widths_p.nelements() > 1 && widths_p.nelements() != spw_p.nelements())
510 : {
511 : os << LogIO::SEVERE
512 : << "Mismatch between the # of widths specified by width and the # of spws."
513 0 : << LogIO::POST;
514 0 : return false;
515 : }
516 :
517 : // Copy the default width to all spws.
518 1882 : if (widths_p.nelements() < nspw)
519 : {
520 1047 : widths_p.resize(nspw, true);
521 10776 : for (uInt k = 1; k < nspw; ++k)
522 : {
523 9729 : widths_p[k] = widths_p[0];
524 : }
525 : }
526 :
527 14739 : for (uInt k = 0; k < nspw; ++k)
528 : {
529 : // CAS-2224, triggered by spw='0:2' (as opposed to '0:2~2').
530 12857 : if (chanStep_p[k] == 0) chanStep_p[k] = 1;
531 :
532 12857 : nchan_p[k] = 1 + (chanEnd_p[k] - chanStart_p[k]) / (chanStep_p[k]* widths_p[k]);
533 12857 : if (nchan_p[k] < 1) nchan_p[k] = 1;
534 : }
535 : }
536 : else
537 : {
538 : // Select everything and rely on widths.
539 0 : MSSpWindowColumns mySpwTab(ms_p.spectralWindow());
540 0 : uInt nspw = mySpwTab.nrow();
541 :
542 0 : nchan_p = mySpwTab.numChan().getColumn();
543 :
544 0 : spw_p.resize(nspw);
545 0 : indgen(spw_p);
546 :
547 0 : chanStart_p.resize(nspw);
548 0 : chanStep_p.resize(nspw);
549 0 : for (uInt k = 0; k < nspw; ++k)
550 : {
551 0 : chanStart_p[k] = 0;
552 0 : chanEnd_p[k] = nchan_p[k] - 1;
553 0 : chanStep_p[k] = 1;
554 : }
555 :
556 0 : if (widths_p.nelements() != spw_p.nelements())
557 : {
558 0 : if (widths_p.nelements() == 1)
559 : {
560 0 : widths_p.resize(spw_p.nelements(), true);
561 0 : for (uInt k = 1; k < spw_p.nelements(); ++k)
562 : {
563 0 : widths_p[k] = widths_p[0];
564 : }
565 :
566 : }
567 : else
568 : {
569 : os << LogIO::SEVERE
570 : << "Mismatch between the # of widths specified by width and the # of spws."
571 0 : << LogIO::POST;
572 0 : return false;
573 : }
574 : }
575 :
576 0 : for (uInt k = 0; k < nspw; ++k)
577 : {
578 0 : nchan_p[k] = 1 + (nchan_p[k] - 1) / widths_p[k];
579 : }
580 0 : }
581 :
582 : // Check for and filter out selected spws that aren't included in DATA_DESCRIPTION.
583 : // (See CAS-1673 for an example.)
584 1882 : std::set<Int> selectedSpwNotInDD(MSTransformDataHandler::findSpwsNotInDD(ms_p, spw_p));
585 1882 : uInt nSelectedSpwNotInDD = selectedSpwNotInDD.size();
586 1882 : if (nSelectedSpwNotInDD > 0)
587 : {
588 122 : os << LogIO::NORMAL << "The following a priori selected input spw(s)\n";
589 122 : for (std::set<Int>::iterator spwit = selectedSpwNotInDD.begin();
590 2332 : spwit != selectedSpwNotInDD.end(); ++spwit)
591 : {
592 2210 : os << spw_p[*spwit] << " ";
593 : }
594 : os << "\nwere not found in DATA_DESCRIPTION (i. e. no rows in the main "
595 : "table reference them) and therefore "
596 122 : "are not included to the output." << LogIO::POST;
597 :
598 122 : uInt nSelSpw = spw_p.nelements();
599 122 : uInt ngoodSelSpwSlots = nSelSpw - nSelectedSpwNotInDD;
600 122 : Vector<Int> spwc(ngoodSelSpwSlots);
601 122 : Vector<Int> chanStartc(ngoodSelSpwSlots);
602 122 : Vector<Int> chanEndc(ngoodSelSpwSlots);
603 122 : Vector<Int> nchanc(ngoodSelSpwSlots);
604 122 : Vector<Int> chanStepc(ngoodSelSpwSlots);
605 122 : std::set<Int>::iterator spwNotDDEnd = selectedSpwNotInDD.end();
606 :
607 122 : uInt j = 0;
608 4854 : for (uInt k = 0; k < nSelSpw; ++k)
609 : {
610 4732 : if (selectedSpwNotInDD.find(k) == spwNotDDEnd)
611 : {
612 2522 : spwc[j] = spw_p[k];
613 2522 : chanStartc[j] = chanStart_p[k];
614 2522 : chanEndc[j] = chanEnd_p[k];
615 2522 : nchanc[j] = nchan_p[k];
616 2522 : chanStepc[j] = chanStep_p[k];
617 2522 : ++j;
618 : }
619 : }
620 122 : spw_p.resize(ngoodSelSpwSlots);
621 122 : spw_p = spwc;
622 122 : chanStart_p.resize(ngoodSelSpwSlots);
623 122 : chanStart_p = chanStartc;
624 122 : chanEnd_p.resize(ngoodSelSpwSlots);
625 122 : chanEnd_p = chanEndc;
626 122 : nchan_p.resize(ngoodSelSpwSlots);
627 122 : nchan_p = nchanc;
628 122 : chanStep_p.resize(ngoodSelSpwSlots);
629 122 : chanStep_p = chanStepc;
630 122 : }
631 :
632 1882 : mssel.getChanSlices(chanSlices_p, &ms_p, 1);
633 1882 : return true;
634 1882 : }
635 :
636 : // -----------------------------------------------------------------------
637 : //
638 : // -----------------------------------------------------------------------
639 1882 : std::set<Int> MSTransformDataHandler::findSpwsNotInDD(MeasurementSet& ms,Vector<Int> spwv)
640 : {
641 1882 : ScalarColumn<Int> spws_in_dd( ms.dataDescription(),
642 1882 : MSDataDescription::columnName(MSDataDescription::SPECTRAL_WINDOW_ID));
643 1882 : std::set<Int> uniqSpwsInDD;
644 1882 : uInt nspwsInDD = spws_in_dd.nrow();
645 :
646 16963 : for (uInt ddrow = 0; ddrow < nspwsInDD; ++ddrow)
647 : {
648 15081 : uniqSpwsInDD.insert(spws_in_dd(ddrow));
649 : }
650 :
651 1882 : std::set<Int> badSelSpwSlots;
652 1882 : std::set<Int>::iterator ddend = uniqSpwsInDD.end();
653 14739 : for (uInt k = 0; k < spwv.nelements(); ++k)
654 : {
655 12857 : if (uniqSpwsInDD.find(spwv[k]) == ddend)
656 : {
657 2210 : badSelSpwSlots.insert(k);
658 : }
659 : }
660 :
661 3764 : return badSelSpwSlots;
662 1882 : }
663 :
664 : // -----------------------------------------------------------------------
665 : //
666 : // -----------------------------------------------------------------------
667 786 : void MSTransformDataHandler::selectAntenna(const Vector<Int>& antennaids,const Vector<String>& antennaSel)
668 : {
669 786 : antennaSel_p = MSTransformDataHandler::pickAntennas(antennaId_p, antennaSelStr_p, antennaids,antennaSel);
670 786 : return;
671 : }
672 :
673 : // -----------------------------------------------------------------------
674 : //
675 : // -----------------------------------------------------------------------
676 786 : bool MSTransformDataHandler::pickAntennas( Vector<Int>& selected_antennaids,
677 : Vector<String>& selected_antenna_strs,
678 : const Vector<Int>& antennaids,
679 : const Vector<String>& antennaSel)
680 : {
681 786 : bool didSelect = true;
682 :
683 786 : if ((antennaids.nelements() == 1) && (antennaids[0] == -1))
684 : {
685 0 : if (antennaSel[0] == "")
686 : {
687 0 : didSelect = false;
688 : }
689 : else
690 : {
691 0 : selected_antennaids.resize();
692 : }
693 : }
694 : else
695 : {
696 786 : selected_antennaids = antennaids;
697 : }
698 :
699 786 : selected_antenna_strs = antennaSel;
700 :
701 786 : return didSelect;
702 : }
703 :
704 : // -----------------------------------------------------------------------
705 : //
706 : // -----------------------------------------------------------------------
707 3 : void MSTransformDataHandler::selectArray(const String& subarray)
708 : {
709 3 : arrayExpr_p = subarray;
710 3 : return;
711 : }
712 :
713 : // -----------------------------------------------------------------------
714 : //
715 : // -----------------------------------------------------------------------
716 1882 : bool MSTransformDataHandler::selectCorrelations(const String& corrstr)
717 : {
718 3764 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
719 :
720 1882 : corrString_p = corrstr;
721 1882 : const bool areSelecting = corrstr != "" && corrstr != "*";
722 :
723 : // Get correlation slices
724 1882 : MSSelection mssel1;
725 1882 : if (areSelecting) mssel1.setPolnExpr(corrstr.c_str());
726 1882 : mssel1.getCorrSlices(corrSlices_p, &ms_p);
727 :
728 : // Get correlation map
729 : // jagonzal (CAS-6951): We have to use another MSSelection because the first one corrupts the correlation
730 : // expression for instance "XX;YY" is turned into "XX" after calling getCorrSlices
731 1882 : MSSelection mssel2;
732 1882 : if (areSelecting) mssel2.setPolnExpr(corrstr.c_str());
733 3764 : return MSTransformDataHandler::getCorrMaps(mssel2, ms_p, inPolOutCorrToInCorrMap_p, areSelecting);
734 1882 : }
735 :
736 : // -----------------------------------------------------------------------
737 : //
738 : // -----------------------------------------------------------------------
739 1882 : bool MSTransformDataHandler::getCorrMaps(MSSelection& mssel,
740 : const MeasurementSet& ms,
741 : Vector<Vector<Int> >& outToIn,
742 : const bool areSelecting)
743 : {
744 :
745 : // ?? This always returns true!!!?!!
746 1882 : bool cando = true;
747 :
748 : // The total number of polids
749 1882 : uInt npol = ms.polarization().nrow();
750 :
751 : // Nominally empty selection for all polids
752 1882 : outToIn.resize(npol);
753 1882 : outToIn.set(Vector<Int> ());
754 1882 : if (areSelecting)
755 : {
756 : // Get the corr indices as an ordered map
757 77 : std::map<Int, Vector<Vector<Int> > > corrmap(mssel.getCorrMap(&ms));
758 :
759 : // Iterate over the ordered map to fill the vector maps
760 176 : for ( auto mi = corrmap.begin( ); mi != corrmap.end( ); ++mi)
761 : {
762 99 : Int pol = mi->first;
763 99 : std::vector<int> correlations_idx = mi->second[0].tovector();
764 99 : std::sort(correlations_idx.begin(), correlations_idx.end());
765 99 : outToIn[pol] = Vector<Int>(correlations_idx);
766 99 : }
767 77 : }
768 : else
769 : { // Make outToIn an identity map.
770 1805 : ScalarColumn<Int> numCorr(ms.polarization(),MSPolarization::columnName(MSPolarization::NUM_CORR));
771 :
772 3967 : for (uInt polid = 0; polid < npol; ++polid)
773 : {
774 2162 : uInt ncorr = numCorr(polid);
775 2162 : outToIn[polid].resize(ncorr);
776 6835 : for (uInt cid = 0; cid < ncorr; ++cid)
777 : {
778 4673 : outToIn[polid][cid] = cid;
779 : }
780 : }
781 1805 : }
782 :
783 1882 : return cando;
784 : }
785 :
786 : // -----------------------------------------------------------------------
787 : //
788 : // -----------------------------------------------------------------------
789 1882 : void MSTransformDataHandler::selectTime(Double timeBin, String timerng)
790 : {
791 1882 : timeBin_p = timeBin;
792 1882 : timeRange_p = timerng;
793 1882 : }
794 :
795 : // -----------------------------------------------------------------------
796 : //
797 : // -----------------------------------------------------------------------
798 1882 : bool MSTransformDataHandler::makeMSBasicStructure(String& msname,
799 : String& colname,
800 : bool produceModel,
801 : bool createWeightSpectrumCols,
802 : const Vector<Int>& tileShape,
803 : const String& combine,
804 : Table::TableOption option)
805 : {
806 3764 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
807 : os << LogIO::DEBUG1 << "Preparing to setup output MS with createWeightSpectrumCols: "
808 1882 : << createWeightSpectrumCols << LogIO::POST;;
809 :
810 1882 : if ((spw_p.nelements() > 0) && (max(spw_p) >= Int(ms_p.spectralWindow().nrow())))
811 : {
812 0 : os << LogIO::SEVERE << "SpectralWindow selection contains elements that do not exist in this MS" << LogIO::POST;
813 0 : ms_p = MeasurementSet();
814 0 : return false;
815 : }
816 :
817 : // Watch out! This throws an AipsError if ms_p doesn't have the requested columns.
818 : const Vector<MS::PredefinedColumns> colNamesTok =
819 1889 : parseColumnNames(colname,produceModel, ms_p,virtualModelCol_p,virtualCorrectedCol_p);
820 :
821 1875 : if (!makeSelection())
822 : {
823 9 : ms_p = MeasurementSet();
824 9 : throw(MSSelectionNullSelection("MSSelectionNullSelection : The selected table has zero rows."));
825 : return false;
826 : }
827 :
828 1864 : mscIn_p = new MSColumns(mssel_p);
829 :
830 : // Note again the parseColumnNames() a few lines back that stops setupMS()
831 : // from being called if the MS doesn't have the requested columns.
832 1864 : MeasurementSet* outpointer = 0;
833 :
834 1864 : if (tileShape.nelements() == 3)
835 : {
836 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0], colNamesTok,
837 : createWeightSpectrumCols, tileShape);
838 : }
839 :
840 : // the following calls MSTileLayout... disabled for now because it
841 : // forces tiles to be the full spw bandwidth in width (gmoellen, 2010/11/07)
842 1864 : else if ((tileShape.nelements() == 1) && (tileShape[0] == 0 || tileShape[0]== 1))
843 : {
844 1864 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0],
845 3728 : mscIn_p->observation().telescopeName()(0),
846 1864 : colNamesTok, createWeightSpectrumCols, tileShape[0]);
847 : }
848 : else {
849 : // Sweep all other cases of bad tileshape to a default one.
850 : // (this probably never happens)
851 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0],
852 0 : mscIn_p->observation().telescopeName()(0),
853 : colNamesTok, createWeightSpectrumCols, 0);
854 : }
855 :
856 1864 : combine_p = combine;
857 :
858 1864 : msOut_p = *outpointer;
859 :
860 : // handle column keywords copy for CORRECTED_DATA -> DATA
861 1864 : if (colNamesTok.nelements() == 1 && colNamesTok[0] == MS::CORRECTED_DATA && mssel_p.isColumn(MS::CORRECTED_DATA)) {
862 194 : TableColumn outCol(msOut_p, "DATA");
863 194 : TableColumn inCol(mssel_p, "CORRECTED_DATA");
864 : // Copy the keywords CORRECTED_DATA -> DATA
865 194 : copyMainTableKeywords(outCol.rwKeywordSet(), inCol.keywordSet());
866 194 : }
867 :
868 1864 : bool ret = true;
869 : try
870 : {
871 1864 : if (option == Table::Scratch)
872 : {
873 : // Set up pointing (has to be done in the copied MS)
874 0 : SetupNewTable pointingSetup(msOut_p.pointingTableName(),MSPointing::requiredTableDesc(), Table::New);
875 0 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::POINTING),Table(pointingSetup));
876 0 : msOut_p.initRefs();
877 :
878 : // Add additional columns to SPECTRAL_WINDOW sub-table
879 0 : addOptionalColumns(mssel_p.spectralWindow(), msOut_p.spectralWindow(), true);
880 :
881 : // Initialize output MS Columns
882 0 : msc_p = new MSColumns(msOut_p);
883 :
884 : // Write transformed SPECTRAL_WINDOW, DATA_DESCRIPTION_ID and POLARIZATION
885 0 : ret &= fillDDTables();
886 0 : }
887 : else
888 : {
889 1864 : ret = fillSubTables(colNamesTok);
890 : }
891 : }
892 0 : catch (AipsError ex)
893 : {
894 0 : ret = false;
895 : os << LogIO::SEVERE
896 : << "Exception filling the sub-tables: " << ex.getMesg() << endl
897 0 : << "Stack Trace: " << ex.getStackTrace()
898 0 : << LogIO::POST;
899 0 : }
900 :
901 1864 : if (!ret)
902 : {
903 0 : delete outpointer;
904 0 : ms_p = MeasurementSet();
905 0 : msOut_p = MeasurementSet();
906 0 : os << LogIO::SEVERE << msname << " left unfinished." << LogIO::POST;
907 0 : return false;
908 : }
909 :
910 : //Detaching the selected part
911 1864 : ms_p = MeasurementSet();
912 :
913 1864 : delete outpointer;
914 1864 : return true;
915 1893 : }
916 :
917 : // -----------------------------------------------------------------------
918 : //
919 : // -----------------------------------------------------------------------
920 0 : bool MSTransformDataHandler::isAllColumns(const Vector<MS::PredefinedColumns>& colNames)
921 : {
922 0 : bool dCol = false, mCol = false, cCol = false;
923 0 : for (uInt i = 0; i < colNames.nelements(); i++)
924 : {
925 0 : if (colNames[i] == MS::DATA) dCol = true;
926 0 : else if (colNames[i] == MS::MODEL_DATA) mCol = true;
927 0 : else if (colNames[i] == MS::CORRECTED_DATA) cCol = true;
928 : // else turn off all?
929 : }
930 :
931 0 : return (dCol && mCol && cCol);
932 : }
933 :
934 : // -----------------------------------------------------------------------
935 : // Modified version of makeSelection that uses the new getter methods
936 : // MSS::getSPWDDIDList() and MSS::getDDIDList()
937 : // -----------------------------------------------------------------------
938 1875 : bool MSTransformDataHandler::makeSelection()
939 : {
940 :
941 3750 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
942 :
943 : // VisSet/MSIter will check if the SORTED exists
944 : // jagonzal (CAS-5327): Commenting this out, since this implies all sorts of memory leaks
945 : // Block<Int> sort;
946 : // ROVisibilityIterator(ms_p, sort);
947 :
948 : const MeasurementSet *elms;
949 1875 : elms = &ms_p;
950 1875 : MeasurementSet sorted;
951 1875 : if (ms_p.keywordSet().isDefined("SORTED_TABLE"))
952 : {
953 590 : sorted = ms_p.keywordSet().asTable("SORTED_TABLE");
954 :
955 : //If ms is not writable and sort is a subselection...use original ms
956 590 : if (ms_p.nrow() == sorted.nrow()) elms = &sorted;
957 : }
958 :
959 1875 : MSSelection thisSelection;
960 1875 : if (fieldid_p.nelements() > 0)
961 : {
962 1875 : thisSelection.setFieldExpr(MSSelection::indexExprStr(fieldid_p));
963 : }
964 :
965 1875 : if (spw_p.nelements() > 0)
966 : {
967 1875 : thisSelection.setSpwExpr(spwString_p);
968 : }
969 :
970 1875 : if (antennaSel_p)
971 : {
972 782 : if (antennaId_p.nelements() > 0)
973 : {
974 0 : thisSelection.setAntennaExpr(MSSelection::indexExprStr(antennaId_p));
975 : }
976 782 : if (antennaSelStr_p[0] != "")
977 : {
978 782 : thisSelection.setAntennaExpr(MSSelection::nameExprStr(antennaSelStr_p));
979 : }
980 :
981 : }
982 :
983 1875 : if (timeRange_p != "")
984 : {
985 39 : thisSelection.setTimeExpr(timeRange_p);
986 : }
987 :
988 :
989 1875 : thisSelection.setUvDistExpr(uvrangeString_p);
990 1875 : thisSelection.setScanExpr(scanString_p);
991 1875 : thisSelection.setStateExpr(intentString_p);
992 1875 : thisSelection.setObservationExpr(obsString_p);
993 :
994 1875 : if (arrayExpr_p != "")
995 : {
996 3 : thisSelection.setArrayExpr(arrayExpr_p);
997 : }
998 :
999 1875 : if (corrString_p != "")
1000 : {
1001 77 : thisSelection.setPolnExpr(corrString_p.c_str());
1002 : }
1003 :
1004 1875 : thisSelection.setTaQLExpr(taqlString_p);
1005 1875 : thisSelection.setFeedExpr(feedString_p);
1006 :
1007 1875 : TableExprNode exprNode = thisSelection.toTableExprNode(elms);
1008 1873 : selTimeRanges_p = thisSelection.getTimeList();
1009 1873 : selObsId_p = thisSelection.getObservationList();
1010 :
1011 : // Get the list of DDI for the selected spws
1012 1873 : spw2ddid_p = thisSelection.getSPWDDIDList(elms);
1013 :
1014 1873 : const MSDataDescription ddtable = elms->dataDescription();
1015 1873 : ScalarColumn<Int> polId(ddtable,MSDataDescription::columnName(MSDataDescription::POLARIZATION_ID));
1016 1873 : const MSPolarization poltable = elms->polarization();
1017 1873 : ArrayColumn<Int> pols(poltable,MSPolarization::columnName(MSPolarization::CORR_TYPE));
1018 :
1019 : // Get the list of DDI for the selected polarizations
1020 1873 : Vector<Int> polDDIList = thisSelection.getDDIDList(elms);
1021 :
1022 : // When polDDIList is empty, do not do an intersection
1023 1873 : bool doIntersection = true;
1024 :
1025 1873 : if (polDDIList.size() == 0){
1026 1796 : doIntersection = false;
1027 : }
1028 :
1029 : // intersection between selected DDI from spw selection and
1030 : // selected DDI from polarization selection
1031 1873 : if (doIntersection) {
1032 77 : Vector<Int> intersectedDDI = set_intersection(spw2ddid_p, polDDIList);
1033 77 : uInt nddids = intersectedDDI.size();
1034 77 : if (nddids > 0){
1035 77 : spw2ddid_p.resize(nddids);
1036 508 : for (uInt ii = 0; ii < nddids; ++ii){
1037 431 : spw2ddid_p[ii] = intersectedDDI[ii];
1038 : }
1039 : }
1040 : else {
1041 : os << LogIO::SEVERE << "None of the selected correlations are in spectral window "
1042 0 : << LogIO::POST;
1043 : }
1044 77 : }
1045 :
1046 :
1047 : // This is actually the number of selected DDI
1048 1873 : uInt nDDIs = spw2ddid_p.size();
1049 :
1050 1873 : inNumCorr_p.resize(nDDIs);
1051 1873 : ncorr_p.resize(nDDIs);
1052 :
1053 : // Map the correlations from input selected DDI to output
1054 12481 : for (uInt k = 0; k < nDDIs; ++k)
1055 : {
1056 10608 : Int ddid = spw2ddid_p[k];
1057 :
1058 : // Number of input correlations for each DDI
1059 : // It reads the nelements of the CORR_TYPE column cell
1060 10608 : inNumCorr_p[k] = pols(polId(ddid)).nelements();
1061 :
1062 : // Corresponding number of output correlations for each DDI
1063 10608 : ncorr_p[k] = inPolOutCorrToInCorrMap_p[polId(ddid)].nelements();
1064 10608 : if (ncorr_p[k] == 0)
1065 : {
1066 : os << LogIO::SEVERE
1067 : << "None of the selected correlations are in spectral window "
1068 0 : << spw_p[k] << LogIO::POST;
1069 0 : return false;
1070 : }
1071 : }
1072 :
1073 :
1074 : // Now remake the selected ms
1075 1873 : if (!(exprNode.isNull()))
1076 : {
1077 1873 : mssel_p = MeasurementSet((*elms)(exprNode));
1078 : }
1079 : else
1080 : {
1081 : // Null take all the ms ...setdata() blank means that
1082 0 : mssel_p = MeasurementSet((*elms));
1083 : }
1084 :
1085 1873 : if (mssel_p.nrow() == 0) return false;
1086 :
1087 : // Setup antNewIndex_p now that mssel_p is ready.
1088 1864 : if (antennaSel_p and reindex_p)
1089 : {
1090 : /*
1091 : // Watch out! getAntenna*List() and getBaselineList() return negative numbers for negated antennas!
1092 : ScalarColumn<Int> ant1c(mssel_p, MS::columnName(MS::ANTENNA1));
1093 : ScalarColumn<Int> ant2c(mssel_p, MS::columnName(MS::ANTENNA2));
1094 : Vector<Int> selAnts(ant1c.getColumn());
1095 : uInt nAnts = selAnts.nelements();
1096 :
1097 : selAnts.resize(2 * nAnts, true);
1098 : selAnts(Slice(nAnts, nAnts)) = ant2c.getColumn();
1099 : nAnts = GenSort<Int>::sort(selAnts, Sort::Ascending,Sort::NoDuplicates);
1100 : selAnts.resize(nAnts, true);
1101 : Int maxAnt = max(selAnts);
1102 : */
1103 :
1104 : // jagonzal: Scanning the main table is extremely inefficient, and depends on TaQL
1105 : // Therefore simply remove the negated antennas from getAntenna1List
1106 744 : vector<Int> antsSel;
1107 :
1108 : // Get antennas selected on position 1
1109 744 : Vector<Int> ant1List = thisSelection.getAntenna1List();
1110 1712 : for (uInt idx=0;idx<ant1List.size();idx++)
1111 : {
1112 968 : if (ant1List(idx) >= 0) antsSel.push_back(ant1List(idx));
1113 : }
1114 :
1115 : // Get antennas selected on position 2
1116 744 : Vector<Int> ant2List = thisSelection.getAntenna2List();
1117 1770 : for (uInt idx=0;idx<ant2List.size();idx++)
1118 : {
1119 1026 : if (ant2List(idx) >= 0) antsSel.push_back(ant2List(idx));
1120 : }
1121 :
1122 : // Sort and remove duplicates
1123 744 : std::sort(antsSel.begin(), antsSel.end());
1124 744 : antsSel.erase(std::unique(antsSel.begin(), antsSel.end()),antsSel.end());
1125 :
1126 744 : Vector<Int> selAnts(antsSel);
1127 744 : uInt nAnts = selAnts.size();
1128 744 : Int maxAnt = max(selAnts);
1129 :
1130 744 : if (maxAnt < 0)
1131 : {
1132 : os << LogIO::SEVERE
1133 : << "The maximum selected antenna number, " << maxAnt
1134 : << ", seems to be < 0."
1135 0 : << LogIO::POST;
1136 0 : return false;
1137 : }
1138 :
1139 744 : antNewIndex_p.resize(maxAnt + 1);
1140 : //So if you see -1 in the main, feed, or pointing tables, fix it
1141 744 : antNewIndex_p.set(-1);
1142 :
1143 1791 : for (uInt k = 0; k < nAnts; ++k)
1144 1047 : antNewIndex_p[selAnts[k]] = k;
1145 :
1146 : //If the total number of output antennas is the same as the input antennas
1147 : //this means that the selection of baselines includes at the end
1148 : //all the input antennas. Therefore setting antenna selection to false.
1149 : //See CAS-11111
1150 744 : if(nAnts == elms->antenna().nrow())
1151 718 : antennaSel_p = false;
1152 1488 : }
1153 : // This still gets tripped up by VLA:OUT.
1154 : else
1155 : {
1156 : // Make a default antNewIndex_p.
1157 1120 : antNewIndex_p.resize(mssel_p.antenna().nrow());
1158 1120 : indgen(antNewIndex_p);
1159 : }
1160 :
1161 1864 : if (mssel_p.nrow() < ms_p.nrow())
1162 : {
1163 : os << LogIO::NORMAL
1164 : << mssel_p.nrow() << " out of " << ms_p.nrow()
1165 : << " rows are going to be considered due to the selection criteria."
1166 766 : << LogIO::POST;
1167 : }
1168 :
1169 1864 : return true;
1170 1879 : }
1171 :
1172 :
1173 : // -----------------------------------------------------------------------
1174 : //
1175 : // -----------------------------------------------------------------------
1176 1864 : MeasurementSet* MSTransformDataHandler::setupMS(const String& MSFileName, const Int nchan,
1177 : const Int nCorr, const String& telescop,
1178 : const Vector<MS::PredefinedColumns>& colNames,
1179 : bool createWeightSpectrumCols,
1180 : const Int obstype, const bool compress,
1181 : const asdmStManUseAlternatives asdmStManUse,
1182 : Table::TableOption option)
1183 : {
1184 : //Choose an appropriate tileshape
1185 1864 : IPosition dataShape(2, nCorr, nchan);
1186 1864 : IPosition tileShape = MSTileLayout::tileShape(dataShape, obstype, telescop);
1187 1864 : return setupMS(MSFileName, nchan, nCorr, colNames, createWeightSpectrumCols,
1188 5592 : tileShape.asVector(),compress, asdmStManUse,option);
1189 1864 : }
1190 :
1191 : // -----------------------------------------------------------------------
1192 : //
1193 : // -----------------------------------------------------------------------
1194 1864 : MeasurementSet* MSTransformDataHandler::setupMS(const String& MSFileName, const Int nchan,
1195 : const Int nCorr,
1196 : const Vector<MS::PredefinedColumns>& colNamesTok,
1197 : bool createWeightSpectrumCols,
1198 : const Vector<Int>& tshape, const bool compress,
1199 : const asdmStManUseAlternatives asdmStManUse,
1200 : Table::TableOption option)
1201 : {
1202 1864 : if (tshape.nelements() != 3) throw(AipsError("TileShape has to have 3 elements "));
1203 :
1204 : // This is more to shush a compiler warning than to warn users.
1205 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1206 1864 : if (tshape[0] != nCorr)
1207 0 : os << LogIO::DEBUG1 << "Warning: using " << tshape[0]
1208 : << " from the tileshape instead of " << nCorr
1209 0 : << " for the number of correlations." << LogIO::POST;
1210 1864 : if (tshape[1] != nchan)
1211 1558 : os << LogIO::DEBUG1 << "Warning: using " << tshape[1]
1212 : << " from the tileshape instead of " << nchan
1213 779 : << " for the number of channels." << LogIO::POST;
1214 :
1215 : // Choose an appropriate tileshape //////////////////
1216 :
1217 1864 : IPosition tileShape(tshape);
1218 :
1219 : // Make the MS table
1220 1864 : TableDesc td = MS::requiredTableDesc();
1221 1864 : Vector<String> tiledDataNames;
1222 :
1223 1864 : if (option == Table::Scratch)
1224 : {
1225 0 : SetupNewTable newtab(MSFileName, td, option);
1226 0 : TableLock lock(TableLock::AutoLocking);
1227 0 : MeasurementSet *ms = new MeasurementSet(newtab, lock);
1228 :
1229 : // Set up default sub-tables for the MS
1230 0 : SetupNewTable dataDescSetup(ms->dataDescriptionTableName(),MSDataDescription::requiredTableDesc(), option);
1231 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::DATA_DESCRIPTION),Table(dataDescSetup));
1232 0 : SetupNewTable polarizationSetup(ms->polarizationTableName(),MSPolarization::requiredTableDesc(), option);
1233 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::POLARIZATION),Table(polarizationSetup));
1234 0 : SetupNewTable spectralWindowSetup(ms->spectralWindowTableName(),MSSpectralWindow::requiredTableDesc(), option);
1235 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::SPECTRAL_WINDOW),Table(spectralWindowSetup));
1236 0 : ms->initRefs();
1237 :
1238 0 : return ms;
1239 0 : }
1240 :
1241 : // Even though we know the data is going to be the same shape throughout I'll
1242 : // still create a column that has a variable shape as this will permit MS's
1243 : // with other shapes to be appended.
1244 1864 : uInt ncols = colNamesTok.nelements();
1245 1864 : const bool mustWriteOnlyToData = mustConvertToData(ncols, colNamesTok);
1246 1864 : if (mustWriteOnlyToData)
1247 : {
1248 1203 : MS::addColumnToDesc(td, MS::DATA, 2);
1249 1203 : if (asdmStManUse == DONT)
1250 : {
1251 1203 : if (compress) MS::addColumnCompression(td, MS::DATA, true);
1252 2406 : String hcolName = String("Tiled") + String("DATA");
1253 1203 : td.defineHypercolumn(hcolName, 3, stringToVector("DATA"));
1254 1203 : tiledDataNames.resize(1);
1255 1203 : tiledDataNames[0] = hcolName;
1256 1203 : }
1257 : }
1258 : else
1259 : {
1260 661 : tiledDataNames.resize(ncols);
1261 1694 : for (uInt i = 0; i < ncols; ++i)
1262 : {
1263 : // Unfortunately MS::PredefinedColumns aren't ordered so that
1264 : // I can just check if colNamesTok[i] is in the "data range".
1265 1033 : if ( colNamesTok[i] == MS::DATA
1266 831 : || colNamesTok[i] == MS::MODEL_DATA
1267 653 : || colNamesTok[i] == MS::CORRECTED_DATA
1268 459 : || colNamesTok[i] == MS::FLOAT_DATA
1269 1864 : || colNamesTok[i] == MS::LAG_DATA)
1270 : {
1271 1033 : if (asdmStManUse == DONT || colNamesTok[i] != MS::DATA)
1272 : {
1273 1033 : MS::addColumnToDesc(td, colNamesTok[i], 2);
1274 1033 : if (compress) MS::addColumnCompression(td, colNamesTok[i], true);
1275 : }
1276 : }
1277 : else
1278 : {
1279 0 : throw(AipsError( MS::columnName(colNamesTok[i]) + " is not a recognized data column "));
1280 : }
1281 1033 : if (asdmStManUse == DONT || colNamesTok[i] != MS::DATA)
1282 : {
1283 1033 : String hcolName = String("Tiled") + MS::columnName(colNamesTok[i]);
1284 1033 : td.defineHypercolumn(hcolName, 3,stringToVector(MS::columnName(colNamesTok[i])));
1285 1033 : tiledDataNames[i] = hcolName;
1286 1033 : }
1287 : }
1288 : }
1289 :
1290 : //other cols for compression
1291 1864 : if (compress && asdmStManUse != USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1292 : {
1293 0 : MS::addColumnCompression(td, MS::WEIGHT, true);
1294 0 : MS::addColumnCompression(td, MS::SIGMA, true);
1295 : }
1296 :
1297 1864 : if (createWeightSpectrumCols) {
1298 324 : MS::addColumnToDesc(td, MS::WEIGHT_SPECTRUM, 2);
1299 324 : MS::addColumnToDesc(td, MS::SIGMA_SPECTRUM, 2);
1300 :
1301 324 : td.defineHypercolumn("TiledWgtSpectrum", 3,stringToVector(MS::columnName(MS::WEIGHT_SPECTRUM)));
1302 324 : td.defineHypercolumn("TiledSigmaSpectrum", 3,stringToVector(MS::columnName(MS::SIGMA_SPECTRUM)));
1303 : }
1304 :
1305 1864 : td.defineHypercolumn("TiledFlagCategory", 4,stringToVector(MS::columnName(MS::FLAG_CATEGORY)));
1306 1864 : td.defineHypercolumn("TiledUVW", 2, stringToVector(MS::columnName(MS::UVW)));
1307 :
1308 1864 : if (asdmStManUse != USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1309 : {
1310 1864 : td.defineHypercolumn("TiledFlag", 3,stringToVector(MS::columnName(MS::FLAG)));
1311 1864 : td.defineHypercolumn("TiledWgt", 2,stringToVector(MS::columnName(MS::WEIGHT)));
1312 1864 : td.defineHypercolumn("TiledSigma", 2,stringToVector(MS::columnName(MS::SIGMA)));
1313 : }
1314 :
1315 1864 : SetupNewTable newtab(MSFileName, td, option);
1316 :
1317 1864 : uInt cache_val = 32768;
1318 :
1319 : // Set the default Storage Manager to be the Incr one
1320 1864 : IncrementalStMan incrStMan("ISMData", cache_val);
1321 1864 : newtab.bindAll(incrStMan, true);
1322 :
1323 : //Override the binding for specific columns
1324 1864 : IncrementalStMan incrStMan0("Array_ID", cache_val);
1325 1864 : newtab.bindColumn(MS::columnName(MS::ARRAY_ID), incrStMan0);
1326 1864 : IncrementalStMan incrStMan1("EXPOSURE", cache_val);
1327 1864 : newtab.bindColumn(MS::columnName(MS::EXPOSURE), incrStMan1);
1328 1864 : IncrementalStMan incrStMan2("FEED1", cache_val);
1329 1864 : newtab.bindColumn(MS::columnName(MS::FEED1), incrStMan2);
1330 1864 : IncrementalStMan incrStMan3("FEED2", cache_val);
1331 1864 : newtab.bindColumn(MS::columnName(MS::FEED2), incrStMan3);
1332 1864 : IncrementalStMan incrStMan4("FIELD_ID", cache_val);
1333 1864 : newtab.bindColumn(MS::columnName(MS::FIELD_ID), incrStMan4);
1334 :
1335 : // jagonzal (CAS-6746): Don't use IncrementalStMan with RW cols ///////////////////////////////////////////////////
1336 : // IncrementalStMan incrStMan5("FLAG_ROW",cache_val/4);
1337 : // newtab.bindColumn(MS::columnName(MS::FLAG_ROW), incrStMan5);
1338 1864 : StandardStMan aipsStManFlagRow("FLAG_ROW", cache_val/4);
1339 1864 : newtab.bindColumn(MS::columnName(MS::FLAG_ROW), aipsStManFlagRow);
1340 : ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
1341 :
1342 1864 : IncrementalStMan incrStMan6("INTERVAL", cache_val);
1343 1864 : newtab.bindColumn(MS::columnName(MS::INTERVAL), incrStMan6);
1344 1864 : IncrementalStMan incrStMan7("OBSERVATION_ID", cache_val);
1345 1864 : newtab.bindColumn(MS::columnName(MS::OBSERVATION_ID), incrStMan7);
1346 1864 : IncrementalStMan incrStMan8("PROCESSOR_ID", cache_val);
1347 1864 : newtab.bindColumn(MS::columnName(MS::PROCESSOR_ID), incrStMan8);
1348 1864 : IncrementalStMan incrStMan9("SCAN_NUMBER", cache_val);
1349 1864 : newtab.bindColumn(MS::columnName(MS::SCAN_NUMBER), incrStMan9);
1350 1864 : IncrementalStMan incrStMan10("STATE_ID", cache_val);
1351 1864 : newtab.bindColumn(MS::columnName(MS::STATE_ID), incrStMan10);
1352 1864 : IncrementalStMan incrStMan11("TIME", cache_val);
1353 1864 : newtab.bindColumn(MS::columnName(MS::TIME), incrStMan11);
1354 1864 : IncrementalStMan incrStMan12("TIME_CENTROID", cache_val);
1355 1864 : newtab.bindColumn(MS::columnName(MS::TIME_CENTROID), incrStMan12);
1356 :
1357 : // Bind ANTENNA1, ANTENNA2 and DATA_DESC_ID to the standardStMan
1358 : // as they may change sufficiently frequently to make the
1359 : // incremental storage manager inefficient for these columns.
1360 1864 : StandardStMan aipsStMan0("ANTENNA1", cache_val);
1361 1864 : newtab.bindColumn(MS::columnName(MS::ANTENNA1), aipsStMan0);
1362 1864 : StandardStMan aipsStMan1("ANTENNA2", cache_val);
1363 1864 : newtab.bindColumn(MS::columnName(MS::ANTENNA2), aipsStMan1);
1364 1864 : StandardStMan aipsStMan2("DATA_DESC_ID", cache_val);
1365 1864 : newtab.bindColumn(MS::columnName(MS::DATA_DESC_ID), aipsStMan2);
1366 :
1367 : // Bind the DATA, FLAG & WEIGHT/SIGMA_SPECTRUM columns to the tiled stman or
1368 : // asdmStMan
1369 1864 : AsdmStMan sm;
1370 :
1371 1864 : if (mustWriteOnlyToData)
1372 : {
1373 1203 : if (asdmStManUse == DONT)
1374 : {
1375 1203 : TiledShapeStMan tiledStMan1Data("TiledDATA", tileShape);
1376 1203 : newtab.bindColumn(MS::columnName(MS::DATA), tiledStMan1Data);
1377 1203 : }
1378 : else
1379 : {
1380 0 : newtab.bindColumn(MS::columnName(MS::DATA), sm);
1381 : }
1382 : }
1383 : else
1384 : {
1385 1694 : for (uInt i = 0; i < ncols; ++i)
1386 : {
1387 1033 : TiledShapeStMan tiledStMan1Data(tiledDataNames[i], tileShape);
1388 1033 : newtab.bindColumn(MS::columnName(colNamesTok[i]), tiledStMan1Data);
1389 1033 : }
1390 661 : if (asdmStManUse != DONT)
1391 : {
1392 0 : newtab.bindColumn(MS::columnName(MS::DATA), sm);
1393 : }
1394 : }
1395 :
1396 3728 : TiledShapeStMan tiledStMan1fc("TiledFlagCategory",IPosition(4, tileShape(0), tileShape(1), 1, tileShape(2)));
1397 1864 : newtab.bindColumn(MS::columnName(MS::FLAG_CATEGORY), tiledStMan1fc);
1398 :
1399 1864 : if (createWeightSpectrumCols) {
1400 324 : TiledShapeStMan tiledStMan2("TiledWgtSpectrum", tileShape);
1401 324 : TiledShapeStMan tiledStMan6("TiledSigmaSpectrum", tileShape);
1402 324 : newtab.bindColumn(MS::columnName(MS::WEIGHT_SPECTRUM), tiledStMan2);
1403 324 : newtab.bindColumn(MS::columnName(MS::SIGMA_SPECTRUM), tiledStMan6);
1404 324 : }
1405 :
1406 3728 : TiledColumnStMan tiledStMan3("TiledUVW",IPosition(2, 3, (tileShape(0) * tileShape(1) * tileShape(2)) / 3));
1407 1864 : newtab.bindColumn(MS::columnName(MS::UVW), tiledStMan3);
1408 1864 : if (asdmStManUse == USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1409 : {
1410 0 : newtab.bindColumn(MS::columnName(MS::FLAG), sm);
1411 0 : newtab.bindColumn(MS::columnName(MS::WEIGHT), sm);
1412 0 : newtab.bindColumn(MS::columnName(MS::SIGMA), sm);
1413 : }
1414 : else
1415 : {
1416 1864 : TiledShapeStMan tiledStMan1f("TiledFlag", tileShape);
1417 1864 : TiledShapeStMan tiledStMan4("TiledWgt", IPosition(2, tileShape(0),
1418 5592 : tileShape(1) * tileShape(2)));
1419 1864 : TiledShapeStMan tiledStMan5("TiledSigma",IPosition(2, tileShape(0),
1420 5592 : tileShape(1) * tileShape(2)));
1421 :
1422 1864 : newtab.bindColumn(MS::columnName(MS::FLAG), tiledStMan1f);
1423 1864 : newtab.bindColumn(MS::columnName(MS::WEIGHT), tiledStMan4);
1424 1864 : newtab.bindColumn(MS::columnName(MS::SIGMA), tiledStMan5);
1425 1864 : }
1426 :
1427 : // Avoid lock overheads by locking the table permanently
1428 1864 : TableLock lock(TableLock::AutoLocking);
1429 1864 : MeasurementSet *ms = new MeasurementSet(newtab, lock);
1430 :
1431 : // Set up the sub-tables for the UVFITS MS (we make new tables with 0 rows)
1432 : // Table::TableOption option = Table::New;
1433 1864 : createSubtables(*ms, option);
1434 :
1435 : // Set the TableInfo
1436 1864 : TableInfo& info(ms->tableInfo());
1437 1864 : info.setType(TableInfo::type(TableInfo::MEASUREMENTSET));
1438 1864 : info.setSubType(String("UVFITS"));
1439 1864 : info.readmeAddLine("This is a measurement set Table holding astronomical observations");
1440 :
1441 1864 : return ms;
1442 1864 : }
1443 :
1444 : // -----------------------------------------------------------------------
1445 : //
1446 : // -----------------------------------------------------------------------
1447 1864 : void MSTransformDataHandler::createSubtables(MeasurementSet& ms, Table::TableOption option)
1448 : {
1449 1864 : SetupNewTable antennaSetup(ms.antennaTableName(),MSAntenna::requiredTableDesc(), option);
1450 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::ANTENNA),Table(antennaSetup));
1451 1864 : SetupNewTable dataDescSetup(ms.dataDescriptionTableName(),MSDataDescription::requiredTableDesc(), option);
1452 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::DATA_DESCRIPTION),Table(dataDescSetup));
1453 1864 : SetupNewTable feedSetup(ms.feedTableName(), MSFeed::requiredTableDesc(),option);
1454 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FEED), Table(feedSetup));
1455 1864 : SetupNewTable flagCmdSetup(ms.flagCmdTableName(),MSFlagCmd::requiredTableDesc(), option);
1456 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FLAG_CMD),Table(flagCmdSetup));
1457 1864 : SetupNewTable fieldSetup(ms.fieldTableName(), MSField::requiredTableDesc(),option);
1458 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FIELD), Table(fieldSetup));
1459 1864 : SetupNewTable historySetup(ms.historyTableName(),MSHistory::requiredTableDesc(), option);
1460 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::HISTORY),Table(historySetup));
1461 1864 : SetupNewTable observationSetup(ms.observationTableName(),MSObservation::requiredTableDesc(), option);
1462 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::OBSERVATION),Table(observationSetup));
1463 1864 : SetupNewTable polarizationSetup(ms.polarizationTableName(),MSPolarization::requiredTableDesc(), option);
1464 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::POLARIZATION),Table(polarizationSetup));
1465 1864 : SetupNewTable processorSetup(ms.processorTableName(),MSProcessor::requiredTableDesc(), option);
1466 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::PROCESSOR),Table(processorSetup));
1467 1864 : SetupNewTable spectralWindowSetup(ms.spectralWindowTableName(),MSSpectralWindow::requiredTableDesc(), option);
1468 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::SPECTRAL_WINDOW),Table(spectralWindowSetup));
1469 1864 : SetupNewTable stateSetup(ms.stateTableName(), MSState::requiredTableDesc(),option);
1470 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::STATE), Table(stateSetup));
1471 :
1472 : // Add the optional Source sub table to allow for specification of the rest frequency
1473 1864 : SetupNewTable sourceSetup(ms.sourceTableName(),MSSource::requiredTableDesc(), option);
1474 1864 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::SOURCE),Table(sourceSetup, 0));
1475 :
1476 : // Update the references to the sub-table keywords
1477 1864 : ms.initRefs();
1478 :
1479 3728 : return;
1480 1864 : }
1481 :
1482 : // -----------------------------------------------------------------------
1483 : //
1484 : // -----------------------------------------------------------------------
1485 1864 : bool MSTransformDataHandler::fillSubTables(const Vector<MS::PredefinedColumns>&)
1486 : {
1487 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1488 1864 : bool success = true;
1489 :
1490 : // Copy the sub-tables before doing anything with the main table.
1491 : // Otherwise MSColumns won't work.
1492 :
1493 : // fill or update
1494 1864 : Timer timer;
1495 :
1496 1864 : timer.mark();
1497 1864 : success &= copyPointing();
1498 1864 : os << LogIO::DEBUG1 << "copyPointing took " << timer.real() << "s." << LogIO::POST;
1499 :
1500 : // Optional columns should be set up before msc_p.
1501 1864 : addOptionalColumns(mssel_p.spectralWindow(), msOut_p.spectralWindow(), true);
1502 :
1503 : // Force the Measures frames for all the time type columns to
1504 : // have the same reference as the TIME column of the main table.
1505 : // Disable the empty table check (with false) because some of the
1506 : // sub-tables (like POINTING) might already have been written.
1507 : // However, empty tables are still empty after setting up the reference codes here.
1508 1864 : msc_p = new MSColumns(msOut_p);
1509 1864 : msc_p->setEpochRef(MEpoch::castType(mscIn_p->timeMeas().getMeasRef().getType()), false);
1510 :
1511 : // UVW is the only other Measures column in the main table.
1512 1864 : msc_p->uvwMeas().setDescRefCode(Muvw::castType(mscIn_p->uvwMeas().getMeasRef().getType()));
1513 :
1514 1864 : if (!mscIn_p->flagCategory().isNull() && mscIn_p->flagCategory().isDefined(0))
1515 : {
1516 417 : msc_p->setFlagCategories(mscIn_p->flagCategories());
1517 : }
1518 :
1519 :
1520 1864 : timer.mark();
1521 1864 : if (!fillDDTables()) return false;
1522 1864 : os << LogIO::DEBUG1 << "fillDDTables took " << timer.real() << "s." << LogIO::POST;
1523 :
1524 : // SourceIDs need to be re-mapped around here
1525 : // (It cannot not be done in selectSource() because mssel_p is not set up yet)
1526 1864 : timer.mark();
1527 1864 : relabelSources();
1528 1864 : os << LogIO::DEBUG1 << "relabelSources took " << timer.real() << "s." << LogIO::POST;
1529 :
1530 1864 : success &= fillFieldTable();
1531 1864 : success &= copySource();
1532 :
1533 1864 : success &= copyAntenna();
1534 : // Feed table writing has to be after antenna
1535 1864 : if (!copyFeed()) return false;
1536 :
1537 1864 : success &= copyFlag_Cmd();
1538 1864 : success &= copyHistory();
1539 1864 : success &= copyObservation();
1540 1864 : success &= copyProcessor();
1541 1864 : success &= copyState();
1542 :
1543 1864 : timer.mark();
1544 1864 : success &= copySyscal();
1545 1864 : os << LogIO::DEBUG1 << "copySyscal took " << timer.real() << "s." << LogIO::POST;
1546 :
1547 1864 : timer.mark();
1548 1864 : success &= copyWeather();
1549 1864 : os << LogIO::DEBUG1 << "copyWeather took " << timer.real() << "s." << LogIO::POST;
1550 :
1551 1864 : timer.mark();
1552 1864 : success &= filterOptSubtable("CALDEVICE");
1553 1864 : os << LogIO::DEBUG1 << "CALDEVICE took " << timer.real() << "s." << LogIO::POST;
1554 :
1555 1864 : timer.mark();
1556 1864 : success &= filterOptSubtable("SYSPOWER");
1557 1864 : os << LogIO::DEBUG1 << "SYSPOWER took " << timer.real() << "s." << LogIO::POST;
1558 :
1559 : // Run this after running the other copy*()s.
1560 : // Maybe there should be an option to *not* run it.
1561 1864 : success &= copyGenericSubtables();
1562 1864 : return success;
1563 1864 : }
1564 :
1565 : // -----------------------------------------------------------------------
1566 : //
1567 : // -----------------------------------------------------------------------
1568 1864 : bool MSTransformDataHandler::fillFieldTable()
1569 : {
1570 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1571 :
1572 1864 : uInt nAddedCols = addOptionalColumns(mssel_p.field(), msOut_p.field(), true);
1573 :
1574 1864 : MSFieldColumns msField(msOut_p.field());
1575 :
1576 1864 : const MSFieldColumns& fieldIn = mscIn_p->field();
1577 1864 : ScalarColumn<String> code(fieldIn.code());
1578 1864 : ArrayColumn<Double> delayDir(fieldIn.delayDir());
1579 1864 : ScalarColumn<bool> flagRow(fieldIn.flagRow());
1580 1864 : ScalarColumn<String> name(fieldIn.name());
1581 1864 : ScalarColumn<Int> numPoly(fieldIn.numPoly());
1582 1864 : ArrayColumn<Double> phaseDir(fieldIn.phaseDir());
1583 1864 : ArrayColumn<Double> refDir(fieldIn.referenceDir());
1584 1864 : ScalarColumn<Int> sourceId(fieldIn.sourceId());
1585 1864 : ScalarColumn<Double> time(fieldIn.time());
1586 :
1587 1864 : String refstr;
1588 1864 : String nameVarRefColDelayDir, nameVarRefColPhaseDir, nameVarRefColRefDir;
1589 :
1590 : // Need to correctly define the direction measures.
1591 :
1592 : // DelayDir
1593 1864 : if (delayDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1594 : {
1595 1480 : delayDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1596 1480 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("Ref",refstr);
1597 : }
1598 :
1599 : // it's a variable ref. column
1600 1864 : if (delayDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1601 : {
1602 384 : delayDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1603 384 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1604 384 : nameVarRefColDelayDir = refstr;
1605 :
1606 384 : Vector<String> refTypeV;
1607 384 : delayDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1608 384 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1609 :
1610 384 : Vector<uInt> refCodeV;
1611 384 : delayDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1612 384 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1613 384 : Int refid = msField.delayDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1614 :
1615 : // Erase the redundant Ref keyword
1616 384 : if (refid >= 0)
1617 : {
1618 384 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1619 : }
1620 384 : }
1621 :
1622 : // PhaseDir
1623 1864 : if (phaseDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1624 : {
1625 1480 : phaseDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1626 1480 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("Ref",refstr);
1627 : }
1628 :
1629 : // It's a variable ref. column
1630 1864 : if (phaseDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1631 : {
1632 384 : phaseDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1633 384 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1634 384 : nameVarRefColPhaseDir = refstr;
1635 :
1636 384 : Vector<String> refTypeV;
1637 384 : phaseDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1638 384 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1639 :
1640 384 : Vector<uInt> refCodeV;
1641 384 : phaseDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1642 384 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1643 :
1644 384 : Int refid = msField.phaseDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1645 384 : if (refid >= 0)
1646 : {
1647 384 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1648 : }
1649 384 : }
1650 :
1651 : // ReferenceDir
1652 1864 : if (refDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1653 : {
1654 1480 : refDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1655 1480 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define(
1656 : "Ref", refstr);
1657 : }
1658 :
1659 : // It's a variable ref. column
1660 1864 : if (refDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1661 : {
1662 384 : refDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1663 384 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1664 384 : nameVarRefColRefDir = refstr;
1665 :
1666 384 : Vector<String> refTypeV;
1667 384 : refDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1668 384 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1669 :
1670 384 : Vector<uInt> refCodeV;
1671 384 : refDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1672 384 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1673 :
1674 384 : Int refid = msField.referenceDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1675 384 : if (refid >= 0)
1676 : {
1677 384 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1678 : }
1679 384 : }
1680 :
1681 : // ...and the time measure...
1682 1864 : time.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1683 1864 : msField.time().rwKeywordSet().asrwRecord("MEASINFO").define("Ref", refstr);
1684 :
1685 1864 : if (!reindex_p)
1686 : {
1687 95 : const MSField &inputField = mssel_p.field();
1688 95 : MSField &outputField = msOut_p.field();
1689 95 : TableCopy::copyRows(outputField, inputField);
1690 95 : copyEphemerisTable(msField);
1691 95 : return true;
1692 : }
1693 :
1694 : // fieldRelabel_p size: nrow of a input MS, -1 for unselected field ids
1695 1769 : fieldRelabel_p.resize(mscIn_p->field().nrow());
1696 1769 : fieldRelabel_p.set(-1);
1697 :
1698 : os << LogIO::DEBUG1 << fieldid_p.nelements()
1699 1769 : << " fields selected out of " << mscIn_p->field().nrow()
1700 1769 : << LogIO::POST;
1701 :
1702 : try {
1703 :
1704 1769 : msOut_p.field().addRow(fieldid_p.nelements());
1705 :
1706 6362 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1707 : {
1708 4593 : fieldRelabel_p[fieldid_p[k]] = k;
1709 :
1710 4593 : msField.code().put(k, code(fieldid_p[k]));
1711 4593 : msField.delayDir().put(k, delayDir(fieldid_p[k]));
1712 4593 : msField.flagRow().put(k, flagRow(fieldid_p[k]));
1713 4593 : msField.name().put(k, name(fieldid_p[k]));
1714 4593 : msField.numPoly().put(k, numPoly(fieldid_p[k]));
1715 4593 : msField.phaseDir().put(k, phaseDir(fieldid_p[k]));
1716 4593 : msField.referenceDir().put(k, refDir(fieldid_p[k]));
1717 4593 : msField.time().put(k, time(fieldid_p[k]));
1718 :
1719 4593 : Int inSrcID = sourceId(fieldid_p[k]);
1720 4593 : if (inSrcID < 0)
1721 : {
1722 519 : msField.sourceId().put(k, -1);
1723 : }
1724 : else
1725 : {
1726 4074 : msField.sourceId().put(k, sourceRelabel_p[inSrcID]);
1727 : }
1728 : }
1729 :
1730 1769 : if (nAddedCols > 0)
1731 : {
1732 339 : copyEphemerisTable(msField);
1733 :
1734 : // need to copy the reference column
1735 339 : if (!nameVarRefColDelayDir.empty())
1736 : {
1737 335 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColDelayDir);
1738 335 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColDelayDir);
1739 1658 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1740 : {
1741 1323 : cdMDirRef.put(k, dM(fieldid_p[k]));
1742 : }
1743 335 : }
1744 :
1745 : // need to copy the reference column
1746 339 : if (!nameVarRefColPhaseDir.empty())
1747 : {
1748 335 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColPhaseDir);
1749 335 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColPhaseDir);
1750 1658 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1751 : {
1752 1323 : cdMDirRef.put(k, dM(fieldid_p[k]));
1753 : }
1754 335 : }
1755 :
1756 : // need to copy the reference column
1757 339 : if (!nameVarRefColRefDir.empty())
1758 : {
1759 335 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColRefDir);
1760 335 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColRefDir);
1761 1658 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1762 : {
1763 1323 : cdMDirRef.put(k, dM(fieldid_p[k]));
1764 : }
1765 335 : }
1766 : }
1767 :
1768 : }
1769 0 : catch (AipsError x)
1770 : {
1771 0 : os << LogIO::EXCEPTION << "Error " << x.getMesg() << " setting up the output FIELD table." << LogIO::POST;
1772 0 : }
1773 0 : catch (...)
1774 : {
1775 0 : throw(AipsError("Unknown exception caught and released in fillFieldTable()"));
1776 0 : }
1777 :
1778 1769 : return true;
1779 1864 : }
1780 :
1781 434 : bool MSTransformDataHandler::copyEphemerisTable(MSFieldColumns & msField)
1782 : {
1783 868 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1784 434 : const MSFieldColumns& fieldIn = mscIn_p->field();
1785 434 : ScalarColumn<Int> eID(fieldIn.ephemerisId());
1786 :
1787 434 : if (eID.hasContent())
1788 : {
1789 : uInt nField;
1790 321 : if (reindex_p)
1791 : {
1792 274 : nField = fieldid_p.nelements();
1793 : }
1794 : else
1795 : {
1796 47 : nField = eID.nrow();
1797 : }
1798 :
1799 321 : String destPathName = Path(msOut_p.field().tableName()).absoluteName();
1800 321 : ScalarColumn<String> name(fieldIn.name());
1801 :
1802 1649 : for (uInt k = 0; k < nField; ++k)
1803 : {
1804 : uInt fieldId;
1805 1328 : if (reindex_p)
1806 : {
1807 1188 : fieldId = fieldid_p[k];
1808 : }
1809 : else
1810 : {
1811 140 : fieldId = k;
1812 : }
1813 :
1814 1328 : Int theEphId = eID(fieldId);
1815 :
1816 : // There is an ephemeris attached to this field
1817 1328 : if (theEphId > -1)
1818 : {
1819 50 : Path ephPath = Path(fieldIn.ephemPath(fieldId));
1820 :
1821 : // Copy the ephemeris table over to the output FIELD table
1822 50 : if (ephPath.length() > 0)
1823 : {
1824 50 : Directory origEphemDir(ephPath);
1825 50 : origEphemDir.copy(destPathName + "/" + ephPath.baseName());
1826 :
1827 : os << LogIO::NORMAL
1828 0 : << "Transferring ephemeris " << ephPath.baseName()
1829 50 : << " for output field " << name(fieldId)
1830 50 : << LogIO::POST;
1831 50 : }
1832 50 : }
1833 :
1834 1328 : if (reindex_p)
1835 : {
1836 1188 : msField.ephemerisId().put(k, theEphId);
1837 : }
1838 : }
1839 321 : }
1840 :
1841 434 : return true;
1842 434 : }
1843 :
1844 : // -----------------------------------------------------------------------
1845 : // Modified version of fillDDTables
1846 : // -----------------------------------------------------------------------
1847 1864 : bool MSTransformDataHandler::fillDDTables()
1848 : {
1849 1864 : fillPolTable();
1850 1864 : fillDDITable();
1851 1864 : fillSPWTable();
1852 :
1853 1864 : return true;
1854 : }
1855 :
1856 : // -----------------------------------------------------------------------
1857 : //
1858 : // -----------------------------------------------------------------------
1859 1864 : bool MSTransformDataHandler::fillPolTable()
1860 : {
1861 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1862 :
1863 : // Input polarization table
1864 1864 : const MSPolarization &poltable = mssel_p.polarization();
1865 1864 : ScalarColumn<Int> numCorr(poltable,MSPolarization::columnName(MSPolarization::NUM_CORR));
1866 1864 : ArrayColumn<Int> corrType(poltable,MSPolarization::columnName(MSPolarization::CORR_TYPE));
1867 1864 : ArrayColumn<Int> corrProd(poltable,MSPolarization::columnName(MSPolarization::CORR_PRODUCT));
1868 1864 : ScalarColumn<bool> flagRow(poltable,MSPolarization::columnName(MSPolarization::FLAG_ROW));
1869 :
1870 : // Output polarization table
1871 1864 : MSPolarizationColumns& msPol(msc_p->polarization());
1872 :
1873 : // Fill output polarization table
1874 1864 : uInt nPol = poltable.nrow(); // nOutputPol = nInputPol (no PolId re-index)
1875 1864 : corrSlice_p.resize(nPol);
1876 4119 : for (uInt polId = 0; polId < nPol; polId++)
1877 : {
1878 2255 : uInt ncorr = inPolOutCorrToInCorrMap_p[polId].nelements();
1879 2255 : const Vector<Int> inCT(corrType(polId));
1880 :
1881 : // Add row
1882 2255 : msOut_p.polarization().addRow();
1883 2255 : msPol.numCorr().put(polId, ncorr);
1884 2255 : msPol.flagRow().put(polId, flagRow(polId));
1885 :
1886 : // Setup correlation slices
1887 2255 : if (ncorr > 0 && ncorr < inCT.nelements())
1888 : {
1889 78 : keepShape_p = false;
1890 :
1891 : // Check whether the requested correlations can be accessed by slicing.
1892 : // That means there must be a constant stride. The most likely (only?)
1893 : // way to violate that is to ask for 3 out of 4 correlations.
1894 78 : if (ncorr > 2)
1895 : {
1896 : os << LogIO::SEVERE
1897 : << "Sorry, the requested correlation selection is not unsupported.\n"
1898 : << "Try selecting fewer or all of the correlations."
1899 0 : << LogIO::POST;
1900 0 : return false;
1901 : }
1902 :
1903 78 : size_t increment = 2;
1904 78 : if (ncorr > 1)
1905 : {
1906 18 : increment = inPolOutCorrToInCorrMap_p[polId][1] - inPolOutCorrToInCorrMap_p[polId][0];
1907 : }
1908 78 : corrSlice_p[polId] = Slice(inPolOutCorrToInCorrMap_p[polId][0],ncorr,increment);
1909 : }
1910 : else
1911 : {
1912 2177 : corrSlice_p[polId] = Slice(0, ncorr);
1913 : }
1914 :
1915 : // Apply slices to correlation type and product
1916 2255 : Vector<Int> outCT;
1917 2255 : const Matrix<Int> inCP(corrProd(polId));
1918 2255 : Matrix<Int> outCP;
1919 2255 : outCT.resize(ncorr);
1920 2255 : outCP.resize(2, ncorr);
1921 7000 : for (uInt k = 0; k < ncorr; ++k)
1922 : {
1923 4745 : Int inCorrInd = inPolOutCorrToInCorrMap_p[polId][k];
1924 :
1925 4745 : outCT[k] = inCT[inCorrInd];
1926 14235 : for (uInt feedind = 0; feedind < 2; ++feedind)
1927 : {
1928 9490 : outCP(feedind, k) = inCP(feedind, inCorrInd);
1929 : }
1930 :
1931 : }
1932 :
1933 : // Fill correlation type and product
1934 2255 : msPol.corrType().put(polId, outCT);
1935 2255 : msPol.corrProduct().put(polId, outCP);
1936 2255 : }
1937 :
1938 1864 : return true;
1939 1864 : }
1940 :
1941 : // -----------------------------------------------------------------------
1942 : //
1943 : // -----------------------------------------------------------------------
1944 1864 : bool MSTransformDataHandler::fillDDITable()
1945 : {
1946 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1947 :
1948 : // Input selected DDIs based on joint SPW-Pol selection
1949 1864 : uInt nddid = spw2ddid_p.size();
1950 :
1951 : // Input ddi table
1952 1864 : const MSDataDescription &inputDDI = mssel_p.dataDescription();
1953 :
1954 1864 : ScalarColumn<Int> polIdCol(inputDDI, MSDataDescription::columnName(MSDataDescription::POLARIZATION_ID));
1955 1864 : ScalarColumn<Int> spwIdCol(inputDDI,MSDataDescription::columnName(MSDataDescription::SPECTRAL_WINDOW_ID));
1956 :
1957 : // Get selected SPWs
1958 1864 : Vector<Int> selectedSpwIds(nddid);
1959 12459 : for (uInt row=0; row<spw2ddid_p.size(); ++row)
1960 : {
1961 10595 : selectedSpwIds[row] = spwIdCol(spw2ddid_p[row]);
1962 : }
1963 :
1964 : // Get list of unique selected SPWs
1965 1864 : bool option(false);
1966 1864 : Sort sortSpws(spw_p.getStorage(option), sizeof(Int));
1967 1864 : sortSpws.sortKey((uInt) 0, TpInt);
1968 1864 : Vector<uInt> spwsortindex, spwuniqinds;
1969 1864 : sortSpws.sort(spwsortindex, spw_p.nelements());
1970 1864 : uInt nuniqSpws = sortSpws.unique(spwuniqinds, spwsortindex);
1971 1864 : spw_uniq_p.resize(nuniqSpws);
1972 :
1973 : // Make map from input to output SPWs
1974 1864 : spwRelabel_p.resize(mscIn_p->spectralWindow().nrow());
1975 1864 : spwRelabel_p.set(-1);
1976 12430 : for (uInt k = 0; k < nuniqSpws; ++k)
1977 : {
1978 10566 : spw_uniq_p[k] = spw_p[spwuniqinds[k]];
1979 10566 : spwRelabel_p[spw_uniq_p[k]] = k;
1980 : }
1981 :
1982 1864 : if (!reindex_p)
1983 : {
1984 95 : MSDataDescription &outputDDI = msOut_p.dataDescription();
1985 95 : TableCopy::copyRows(outputDDI, inputDDI);
1986 95 : return true;
1987 : }
1988 :
1989 : // Output SPECTRAL_WINDOW_ID column
1990 1769 : Vector<Int> newSPWId(nddid);
1991 11171 : for (uInt ddi=0; ddi<nddid; ddi++)
1992 : {
1993 9402 : newSPWId[ddi] = spwRelabel_p[spwIdCol(spw2ddid_p[ddi])];
1994 : }
1995 :
1996 : // Output POLARIZATION_ID column
1997 1769 : Vector<Int> newPolId(nddid);
1998 11171 : for (uInt ddi=0; ddi<nddid; ddi++)
1999 : {
2000 9402 : newPolId[ddi] = polIdCol(spw2ddid_p[ddi]);
2001 : }
2002 :
2003 : // Fill output DDI table
2004 1769 : MSDataDescColumns& outputDDI(msc_p->dataDescription());
2005 11171 : for (uInt ddi=0; ddi<nddid; ddi++)
2006 : {
2007 9402 : msOut_p.dataDescription().addRow();
2008 9402 : outputDDI.flagRow().put(ddi, false);
2009 9402 : outputDDI.polarizationId().put(ddi, newPolId[ddi]);
2010 9402 : outputDDI.spectralWindowId().put(ddi, newSPWId[ddi]);
2011 : }
2012 :
2013 :
2014 1769 : return true;
2015 1864 : }
2016 :
2017 : // -----------------------------------------------------------------------
2018 : //
2019 : // -----------------------------------------------------------------------
2020 1864 : bool MSTransformDataHandler::fillSPWTable()
2021 : {
2022 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2023 :
2024 : // Selected input MS SPW Table Columns (read-only)
2025 1864 : MSSpWindowColumns inSpWCols(mssel_p.spectralWindow());
2026 :
2027 : // SPW Table Columns of output MS (read-write)
2028 1864 : MSSpWindowColumns& msSpW(msc_p->spectralWindow());
2029 :
2030 : // Detect which optional columns of SPECTRAL_WINDOW are present.
2031 : // inSpWCols and msSpW should agree because addOptionalColumns() was done for
2032 : // SPECTRAL_WINDOW in fillAllTables() before making msc_p or calling fillDDTables
2033 1864 : bool haveSpwAN = columnOk(inSpWCols.assocNature());
2034 1864 : bool haveSpwASI = columnOk(inSpWCols.assocSpwId());
2035 1864 : bool haveSpwBN = columnOk(inSpWCols.bbcNo());
2036 1864 : bool haveSpwBS = columnOk(inSpWCols.bbcSideband());
2037 1864 : bool haveSpwDI = columnOk(inSpWCols.dopplerId());
2038 3864 : bool haveSpwSWF = mssel_p.spectralWindow().tableDesc().isColumn("SDM_WINDOW_FUNCTION") &&
2039 2000 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_WINDOW_FUNCTION");
2040 3864 : bool haveSpwSNB = mssel_p.spectralWindow().tableDesc().isColumn("SDM_NUM_BIN") &&
2041 2000 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_NUM_BIN");
2042 3824 : bool haveSpwCorrBit = mssel_p.spectralWindow().tableDesc().isColumn("SDM_CORR_BIT") &&
2043 1960 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_CORR_BIT");
2044 :
2045 1864 : uInt nuniqSpws = spw_uniq_p.size();
2046 :
2047 : // This sets the number of input channels for each spw. But
2048 : // it considers that a SPW ID contains only one set of channels.
2049 : // I hope this is true!!
2050 : // Write to SPECTRAL_WINDOW table
2051 1864 : inNumChan_p.resize(spw_p.nelements());
2052 12437 : for (uInt k = 0; k < spw_p.nelements(); ++k)
2053 : {
2054 10573 : inNumChan_p[k] = inSpWCols.numChan()(spw_p[k]);
2055 : }
2056 :
2057 1864 : if (reindex_p)
2058 : {
2059 1769 : msOut_p.spectralWindow().addRow(nuniqSpws);
2060 : }
2061 : else
2062 : {
2063 95 : const MSSpectralWindow &inputSPW = mssel_p.spectralWindow();
2064 95 : MSSpectralWindow &outputSPW = msOut_p.spectralWindow();
2065 95 : TableCopy::copyRows(outputSPW, inputSPW);
2066 : }
2067 :
2068 :
2069 1864 : Vector<Vector<Int> > spwinds_of_uniq_spws(nuniqSpws);
2070 1864 : totnchan_p.resize(nuniqSpws);
2071 12430 : for (uInt k = 0; k < nuniqSpws; ++k)
2072 : {
2073 10566 : Int maxchan = 0;
2074 10566 : uInt j = 0;
2075 :
2076 10566 : totnchan_p[k] = 0;
2077 10566 : spwinds_of_uniq_spws[k].resize();
2078 184259 : for (uInt spwind = 0; spwind < spw_p.nelements(); ++spwind)
2079 : {
2080 173693 : if (spw_p[spwind] == spw_uniq_p[k])
2081 : {
2082 10573 : Int highchan = nchan_p[spwind] * chanStep_p[spwind] + chanStart_p[spwind];
2083 :
2084 10573 : if (highchan > maxchan) maxchan = highchan;
2085 :
2086 10573 : totnchan_p[k] += nchan_p[spwind];
2087 :
2088 : // The true is necessary to avoid scrambling previously assigned values.
2089 10573 : spwinds_of_uniq_spws[k].resize(j + 1, true);
2090 :
2091 : // Warning! spwinds_of_uniq_spws[k][j] will compile without warning, but dump core at runtime.
2092 10573 : (spwinds_of_uniq_spws[k])[j] = spwind;
2093 :
2094 10573 : ++j;
2095 : }
2096 : }
2097 10566 : if (maxchan > inSpWCols.numChan()(spw_uniq_p[k]))
2098 : {
2099 : os << LogIO::SEVERE
2100 : << " Channel settings wrong; exceeding number of channels in spw "
2101 0 : << spw_uniq_p[k] << LogIO::POST;
2102 0 : return false;
2103 : }
2104 : }
2105 :
2106 : // min_k is an index for getting an spw index via spw_uniq_p[min_k].
2107 : // k is an index for getting an spw index via spw_p[k].
2108 12430 : for (uInt min_k = 0; min_k < nuniqSpws; ++min_k)
2109 : {
2110 10566 : uInt k = spwinds_of_uniq_spws[min_k][0];
2111 10566 : uInt outSPWId = reindex_p? min_k : spw_p[k];
2112 :
2113 10566 : if (spwinds_of_uniq_spws[min_k].nelements() > 1 || nchan_p[k] != inSpWCols.numChan()(spw_p[k]))
2114 : {
2115 1453 : Vector<Double> effBWIn = inSpWCols.effectiveBW()(spw_uniq_p[min_k]);
2116 1453 : Int nOutChan = totnchan_p[min_k];
2117 1453 : Vector<Double> chanFreqOut(nOutChan);
2118 1453 : Vector<Double> chanFreqIn = inSpWCols.chanFreq()(spw_uniq_p[min_k]);
2119 1453 : Vector<Double> chanWidthOut(nOutChan);
2120 1453 : Vector<Double> chanWidthIn = inSpWCols.chanWidth()(spw_uniq_p[min_k]);
2121 1453 : Vector<Double> spwResolOut(nOutChan);
2122 1453 : Vector<Double> spwResolIn = inSpWCols.resolution()(spw_uniq_p[min_k]);
2123 1453 : Vector<Double> effBWOut(nOutChan);
2124 1453 : Int outChan = 0;
2125 1453 : Int outChanNotDropped = 0;
2126 :
2127 1453 : keepShape_p = false;
2128 :
2129 : // The sign of CHAN_WIDTH defaults to +. Its determination assumes that
2130 : // chanFreqIn is monotonic, but not that the sign of the chanWidthIn is correct.
2131 1453 : bool neginc = chanFreqIn[chanFreqIn.nelements() - 1] < chanFreqIn[0];
2132 :
2133 1453 : effBWOut.set(0.0);
2134 1453 : Double totalBW = 0.0;
2135 2913 : for (uInt rangeNum = 0; rangeNum < spwinds_of_uniq_spws[min_k].nelements(); ++rangeNum)
2136 : {
2137 1460 : k = spwinds_of_uniq_spws[min_k][rangeNum];
2138 :
2139 1460 : Int span = chanStep_p[k] * widths_p[k];
2140 :
2141 44468 : for (Int j = 0; j < nchan_p[k]; ++j)
2142 : {
2143 43008 : Int inpChan = chanStart_p[k] + j * span;
2144 :
2145 43008 : if (span >= 1)
2146 : {
2147 43008 : Int lastChan = inpChan + span - 1;
2148 :
2149 43008 : if (lastChan > chanEnd_p[k])
2150 : {
2151 : // The averaging width is not a factor of the number of
2152 : // selected input channels, so the last output bin receives
2153 : // fewer input channels than the other bins.
2154 14 : lastChan = chanEnd_p[k];
2155 :
2156 14 : Int nchan = lastChan - inpChan + 1;
2157 : os << LogIO::NORMAL
2158 : << "The last output channel of spw "
2159 28 : << spw_p[k] << " has only " << nchan
2160 14 : << " input channel";
2161 14 : if (nchan > 1) os << "s.";
2162 14 : os << LogIO::POST;
2163 :
2164 : // jagonzal (CAS-8618): We may have more than one channel dropped per SPW
2165 : // due to multiple channel selections
2166 14 : spwDropChannelMap_p[spw_p[k]].push_back(outChan);
2167 : }
2168 : else
2169 : {
2170 132819 : for (Int inputChan = inpChan;inputChan<=lastChan;inputChan++)
2171 : {
2172 89825 : spwSelectedChannelMap_p[spw_p[k]][outChanNotDropped].push_back(inputChan);
2173 : }
2174 42994 : outChanNotDropped++;
2175 : }
2176 :
2177 43008 : chanFreqOut[outChan] = (chanFreqIn[inpChan]
2178 43008 : + chanFreqIn[lastChan]) / 2;
2179 :
2180 43008 : Double sep = chanFreqIn[lastChan] - chanFreqIn[inpChan];
2181 :
2182 43008 : if (neginc) sep = -sep;
2183 :
2184 : // The internal abs is necessary because the sign of chanWidthIn may be wrong.
2185 43008 : chanWidthOut[outChan] = sep + 0.5 * abs(chanWidthIn[inpChan] + chanWidthIn[lastChan]);
2186 43008 : if (neginc) chanWidthOut[outChan] = -chanWidthOut[outChan];
2187 :
2188 43008 : spwResolOut[outChan] = 0.5 * (spwResolIn[inpChan] + spwResolIn[lastChan]) + sep;
2189 :
2190 132888 : for (Int avgChan = inpChan; avgChan <= lastChan; avgChan += chanStep_p[k])
2191 : {
2192 89880 : effBWOut[outChan] += effBWIn[avgChan];
2193 : }
2194 :
2195 : }
2196 : else
2197 : {
2198 0 : chanFreqOut[outChan] = chanFreqIn[inpChan];
2199 0 : spwResolOut[outChan] = spwResolIn[inpChan];
2200 0 : chanWidthOut[outChan] = chanWidthIn[inpChan];
2201 0 : effBWOut[outChan] = effBWIn[inpChan];
2202 : }
2203 : // Use CHAN_WIDTH instead of EFFECTIVE_BW
2204 43008 : totalBW += abs(chanWidthOut[outChan]);
2205 43008 : ++outChan;
2206 : }
2207 : }
2208 1453 : --outChan;
2209 :
2210 1453 : msSpW.chanFreq().put(outSPWId, chanFreqOut);
2211 1453 : msSpW.refFrequency().put(outSPWId,min(chanFreqOut[0], chanFreqOut[chanFreqOut.size() - 1]));
2212 1453 : msSpW.resolution().put(outSPWId, spwResolOut);
2213 1453 : msSpW.numChan().put(outSPWId, nOutChan);
2214 1453 : msSpW.chanWidth().put(outSPWId, chanWidthOut);
2215 1453 : msSpW.effectiveBW().put(outSPWId, spwResolOut);
2216 1453 : msSpW.totalBandwidth().put(outSPWId, totalBW);
2217 1453 : }
2218 : else
2219 : {
2220 9113 : msSpW.chanFreq().put(outSPWId, inSpWCols.chanFreq()(spw_p[k]));
2221 9113 : msSpW.refFrequency().put(outSPWId, inSpWCols.refFrequency()(spw_p[k]));
2222 9113 : msSpW.resolution().put(outSPWId, inSpWCols.resolution()(spw_p[k]));
2223 9113 : msSpW.numChan().put(outSPWId, inSpWCols.numChan()(spw_p[k]));
2224 9113 : msSpW.chanWidth().put(outSPWId, inSpWCols.chanWidth()(spw_p[k]));
2225 9113 : msSpW.effectiveBW().put(outSPWId, inSpWCols.effectiveBW()(spw_p[k]));
2226 9113 : msSpW.totalBandwidth().put(outSPWId,inSpWCols.totalBandwidth()(spw_p[k]));
2227 : }
2228 :
2229 10566 : msSpW.flagRow().put(outSPWId, inSpWCols.flagRow()(spw_p[k]));
2230 10566 : msSpW.freqGroup().put(outSPWId, inSpWCols.freqGroup()(spw_p[k]));
2231 10566 : msSpW.freqGroupName().put(outSPWId, inSpWCols.freqGroupName()(spw_p[k]));
2232 10566 : msSpW.ifConvChain().put(outSPWId, inSpWCols.ifConvChain()(spw_p[k]));
2233 10566 : msSpW.measFreqRef().put(outSPWId, inSpWCols.measFreqRef()(spw_p[k]));
2234 10566 : msSpW.name().put(outSPWId, inSpWCols.name()(spw_p[k]));
2235 10566 : msSpW.netSideband().put(outSPWId, inSpWCols.netSideband()(spw_p[k]));
2236 :
2237 :
2238 10566 : if (haveSpwBN) msSpW.bbcNo().put(outSPWId, inSpWCols.bbcNo()(spw_p[k]));
2239 10566 : if (haveSpwBS) msSpW.bbcSideband().put(outSPWId, inSpWCols.bbcSideband()(spw_p[k]));
2240 10566 : if (haveSpwDI) msSpW.dopplerId().put(outSPWId, inSpWCols.dopplerId()(spw_p[k]));
2241 10566 : if (haveSpwSWF)
2242 : {
2243 2879 : ScalarColumn<String> inSwfCol(mssel_p.spectralWindow(), "SDM_WINDOW_FUNCTION");
2244 2879 : ScalarColumn<String> outSwfCol(msOut_p.spectralWindow(), "SDM_WINDOW_FUNCTION");
2245 2879 : outSwfCol.put(outSPWId, inSwfCol(spw_p[k]));
2246 2879 : }
2247 10566 : if (haveSpwSNB)
2248 : {
2249 2879 : ScalarColumn<Int> inSnbCol(mssel_p.spectralWindow(), "SDM_NUM_BIN");
2250 2879 : ScalarColumn<Int> outSnbCol(msOut_p.spectralWindow(), "SDM_NUM_BIN");
2251 2879 : outSnbCol.put(outSPWId, inSnbCol(spw_p[k]));
2252 2879 : }
2253 10566 : if (haveSpwCorrBit)
2254 : {
2255 1868 : ScalarColumn<String> inCorrBitCol(mssel_p.spectralWindow(), "SDM_CORR_BIT");
2256 1868 : ScalarColumn<String> outCorrBitCol(msOut_p.spectralWindow(), "SDM_CORR_BIT");
2257 1868 : outCorrBitCol.put(outSPWId, inCorrBitCol(spw_p[k]));
2258 1868 : }
2259 :
2260 10566 : if (haveSpwASI)
2261 : {
2262 4035 : if (reindex_p)
2263 : {
2264 : // Get list of SPWs associated to his one
2265 3301 : std::vector<Int> selectedSPWs = spw_p.tovector();
2266 :
2267 : // Get the list of selected SPWs and association nature
2268 3301 : Array<Int> assocSpwId = inSpWCols.assocSpwId()(spw_p[k]);
2269 3301 : Array<String> assocNature = inSpWCols.assocNature()(spw_p[k]);
2270 :
2271 : // Find which associated SPWs are selected, and store the transformed Id
2272 3301 : std::vector<Int>::iterator findIt;
2273 3301 : std::vector<Int> selectedAssocSpwId;
2274 3301 : std::vector<String> selectedAssocNature;
2275 53953 : for (uInt idx=0;idx<assocSpwId.size();idx++)
2276 : {
2277 50652 : IPosition pos(1,idx);
2278 50652 : Int spw = assocSpwId(pos);
2279 50652 : findIt = find (selectedSPWs.begin(), selectedSPWs.end(), spw);
2280 50652 : if (findIt != selectedSPWs.end())
2281 : {
2282 5672 : selectedAssocSpwId.push_back(spwRelabel_p[spw]);
2283 5672 : if (haveSpwAN) selectedAssocNature.push_back(assocNature(pos));
2284 : }
2285 50652 : }
2286 :
2287 : // Store selected associated SPW Ids
2288 3301 : Vector<Int> selectedAssocSpwIdVector(selectedAssocSpwId);
2289 3301 : msSpW.assocSpwId().put(min_k, selectedAssocSpwIdVector);
2290 :
2291 : // Store selected association nature
2292 3301 : if (haveSpwAN)
2293 : {
2294 3301 : Vector<String> selectedAssocNatureVector(selectedAssocNature);
2295 3301 : msSpW.assocNature().put(min_k, selectedAssocNatureVector);
2296 3301 : }
2297 3301 : }
2298 : else
2299 : {
2300 734 : msSpW.assocNature().put(outSPWId, inSpWCols.assocNature()(outSPWId));
2301 : }
2302 : }
2303 : }
2304 :
2305 :
2306 1864 : return true;
2307 1864 : }
2308 :
2309 : // -----------------------------------------------------------------------
2310 : //
2311 : // -----------------------------------------------------------------------
2312 9659 : uInt MSTransformDataHandler::addOptionalColumns(const Table& inTab, Table& outTab,const bool beLazy)
2313 : {
2314 9659 : uInt nAdded = 0;
2315 9659 : const TableDesc& inTD = inTab.actualTableDesc();
2316 :
2317 : // Only rely on the # of columns if you are sure that inTab and outTab
2318 : // can't have the same # of columns without having _different_ columns,
2319 : // i.e. use beLazy if outTab.actualTableDesc() is in its default state.
2320 9659 : uInt nInCol = inTD.ncolumn();
2321 9659 : if (!beLazy || nInCol > outTab.actualTableDesc().ncolumn())
2322 : {
2323 9116 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2324 :
2325 4558 : Vector<String> oldColNames = inTD.columnNames();
2326 :
2327 67683 : for (uInt k = 0; k < nInCol; ++k)
2328 : {
2329 63125 : if (!outTab.actualTableDesc().isColumn(oldColNames[k]))
2330 : {
2331 : try
2332 : {
2333 28229 : outTab.addColumn(inTD.columnDesc(k), false);
2334 28229 : ++nAdded;
2335 : }
2336 : // NOT AipsError x
2337 0 : catch (...)
2338 : {
2339 : os << LogIO::WARN << "Could not add column "
2340 0 : << oldColNames[k] << " to " << outTab.tableName()
2341 0 : << LogIO::POST;
2342 0 : }
2343 : }
2344 : }
2345 4558 : }
2346 :
2347 9659 : return nAdded;
2348 9659 : }
2349 :
2350 : // -----------------------------------------------------------------------
2351 : //
2352 : // -----------------------------------------------------------------------
2353 1864 : void MSTransformDataHandler::relabelSources()
2354 : {
2355 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2356 :
2357 : //Source is an optional table, so it may not exist
2358 1864 : if (Table::isReadable(mssel_p.sourceTableName()))
2359 : {
2360 : // Note that mscIn_p->field().sourceId() has ALL of the
2361 : // sourceIDs in the input MS, not just the selected ones.
2362 1864 : const Vector<Int>& inSrcIDs = mscIn_p->field().sourceId().getColumn();
2363 :
2364 1864 : Int highestInpSrc = max(inSrcIDs);
2365 :
2366 : // Ensure space for -1.
2367 1864 : if (highestInpSrc < 0) highestInpSrc = 0;
2368 :
2369 1864 : sourceRelabel_p.resize(highestInpSrc + 1);
2370 : // Default to "any".
2371 1864 : sourceRelabel_p.set(-1);
2372 :
2373 : // Enable sourceIDs that are actually referred
2374 : // by selected fields, and remap them using j.
2375 1864 : uInt j = 0;
2376 6692 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
2377 : {
2378 4828 : Int fldInSrcID = inSrcIDs[fieldid_p[k]];
2379 :
2380 4828 : if (fldInSrcID > -1)
2381 : {
2382 : // Multiple fields can use the same
2383 4309 : if (sourceRelabel_p[fldInSrcID] == -1)
2384 : {
2385 : // source in a mosaic.
2386 3827 : sourceRelabel_p[fldInSrcID] = j;
2387 3827 : ++j;
2388 : }
2389 : }
2390 : }
2391 1864 : }
2392 : else
2393 : {
2394 : // Default to "any".
2395 : os << LogIO::NORMAL
2396 : << "The input MS does not have the optional SOURCE table.\n"
2397 0 : << "-1 will be used as a generic source ID." << LogIO::POST;
2398 0 : sourceRelabel_p.resize(1);
2399 0 : sourceRelabel_p.set(-1);
2400 : }
2401 :
2402 3728 : return;
2403 1864 : }
2404 :
2405 : // -----------------------------------------------------------------------
2406 : //
2407 : // -----------------------------------------------------------------------
2408 2216 : void MSTransformDataHandler::copySubtable(const String& tabName, const Table& inTab,const bool doFilter)
2409 : {
2410 2216 : String outName(msOut_p.tableName() + '/' + tabName);
2411 :
2412 2216 : if (PlainTable::tableCache()(outName)) PlainTable::tableCache().remove(outName);
2413 2216 : inTab.deepCopy(outName, Table::New, false, Table::AipsrcEndian, doFilter);
2414 2216 : Table outTab(outName, Table::Update);
2415 2216 : msOut_p.rwKeywordSet().defineTable(tabName, outTab);
2416 2216 : msOut_p.initRefs();
2417 :
2418 4432 : return;
2419 2216 : }
2420 :
2421 : // -----------------------------------------------------------------------
2422 : //
2423 : // -----------------------------------------------------------------------
2424 0 : void MSTransformDataHandler::make_map(std::map<Int, Int>& mapper, const Vector<Int>& inv)
2425 : {
2426 0 : std::set<Int> valSet;
2427 :
2428 : // Strange, but slightly more efficient than going forward.
2429 0 : for (Int i = inv.nelements(); i--;)
2430 : {
2431 0 : valSet.insert(inv[i]);
2432 : }
2433 :
2434 0 : uInt remaval = 0;
2435 0 : for (std::set<Int>::const_iterator vs_iter = valSet.begin(); vs_iter != valSet.end(); ++vs_iter)
2436 : {
2437 0 : mapper[*vs_iter] = remaval;
2438 0 : ++remaval;
2439 : }
2440 :
2441 0 : return;
2442 0 : }
2443 :
2444 : // -----------------------------------------------------------------------
2445 : //
2446 : // -----------------------------------------------------------------------
2447 1864 : bool MSTransformDataHandler::copyPointing()
2448 : {
2449 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2450 :
2451 : //Pointing is allowed to not exist
2452 1864 : if (Table::isReadable(mssel_p.pointingTableName()))
2453 : {
2454 1864 : const MSPointing& oldPoint = mssel_p.pointing();
2455 :
2456 1864 : if ((!antennaSel_p && timeRange_p == "") or !reindex_p)
2457 : {
2458 : // jagonzal: copySubtables works with POINTING because it
2459 : // is not created by default in the method createSubtables
2460 1816 : copySubtable(MS::keywordName(MS::POINTING), oldPoint);
2461 : }
2462 : else
2463 : {
2464 48 : setupNewPointing();
2465 :
2466 48 : if (oldPoint.nrow() > 0)
2467 : {
2468 : // Could be declared as Table&
2469 22 : MSPointing& newPoint = msOut_p.pointing();
2470 :
2471 : // Add optional columns if present in oldPoint.
2472 22 : uInt nAddedCols = addOptionalColumns(oldPoint, newPoint, true);
2473 22 : os << LogIO::DEBUG1 << "POINTING has " << nAddedCols << " optional columns." << LogIO::POST;
2474 :
2475 22 : const MSPointingColumns oldPCs(oldPoint);
2476 22 : MSPointingColumns newPCs(newPoint);
2477 22 : newPCs.setEpochRef(MEpoch::castType(oldPCs.timeMeas().getMeasRef().getType()));
2478 22 : newPCs.setDirectionRef(MDirection::castType(oldPCs.directionMeasCol().getMeasRef().getType()));
2479 22 : newPCs.setEncoderDirectionRef(MDirection::castType(oldPCs.encoderMeas().getMeasRef().getType()));
2480 :
2481 22 : const ScalarColumn<Int>& antIds = oldPCs.antennaId();
2482 22 : const ScalarColumn<Double>& time = oldPCs.time();
2483 22 : ScalarColumn<Int>& outants = newPCs.antennaId();
2484 :
2485 22 : uInt nTRanges = selTimeRanges_p.ncolumn();
2486 :
2487 22 : uInt outRow = 0;
2488 :
2489 : // Int for comparison
2490 22 : Int maxSelAntp1 = antNewIndex_p.nelements();
2491 : // with newAntInd.
2492 122172 : for (uInt inRow = 0; inRow < antIds.nrow(); ++inRow)
2493 : {
2494 122150 : Int newAntInd = antIds(inRow);
2495 122150 : if (antennaSel_p)
2496 : {
2497 0 : newAntInd = newAntInd < maxSelAntp1 ? antNewIndex_p[newAntInd] : -1;
2498 : }
2499 :
2500 122150 : Double t = time(inRow);
2501 :
2502 122150 : if (newAntInd > -1)
2503 : {
2504 122150 : bool matchT = false;
2505 122150 : if (nTRanges == 0)
2506 : {
2507 0 : matchT = true;
2508 : }
2509 : else
2510 : {
2511 242222 : for (uInt tr = 0; tr < nTRanges; ++tr)
2512 : {
2513 122150 : if (t >= selTimeRanges_p(0, tr) && t <= selTimeRanges_p(1, tr))
2514 : {
2515 2078 : matchT = true;
2516 2078 : break;
2517 : }
2518 : }
2519 : }
2520 :
2521 122150 : if (matchT)
2522 : {
2523 2078 : TableCopy::copyRows(newPoint, oldPoint, outRow,inRow, 1, false);
2524 2078 : outants.put(outRow, newAntInd);
2525 2078 : ++outRow;
2526 : }
2527 : }
2528 : }
2529 22 : }
2530 : }
2531 : }
2532 : else
2533 : {
2534 : // Make an empty stub for MSColumns.
2535 0 : setupNewPointing();
2536 : }
2537 :
2538 :
2539 1864 : return true;
2540 1864 : }
2541 :
2542 : // -----------------------------------------------------------------------
2543 : //
2544 : // -----------------------------------------------------------------------
2545 48 : void MSTransformDataHandler::setupNewPointing()
2546 : {
2547 0 : SetupNewTable pointingSetup(msOut_p.pointingTableName(),
2548 48 : MSPointing::requiredTableDesc(), Table::New);
2549 :
2550 : // POINTING can be large, set some sensible defaults for storageMgrs
2551 48 : IncrementalStMan ismPointing("ISMPointing");
2552 48 : StandardStMan ssmPointing("SSMPointing", 32768);
2553 48 : pointingSetup.bindAll(ismPointing, true);
2554 48 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::DIRECTION),ssmPointing);
2555 48 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::TARGET),ssmPointing);
2556 48 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::TIME),ssmPointing);
2557 48 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::POINTING),Table(pointingSetup));
2558 48 : msOut_p.initRefs();
2559 :
2560 96 : return;
2561 48 : }
2562 :
2563 : // -----------------------------------------------------------------------
2564 : //
2565 : // -----------------------------------------------------------------------
2566 1864 : bool MSTransformDataHandler::copySource()
2567 : {
2568 : //Source is an optional table, so it may not exist
2569 1864 : if (Table::isReadable(mssel_p.sourceTableName()))
2570 : {
2571 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2572 :
2573 1864 : const MSSource& oldSource = mssel_p.source();
2574 1864 : MSSource& newSource = msOut_p.source();
2575 :
2576 : // Add optional columns if present in oldSource.
2577 1864 : uInt nAddedCols = addOptionalColumns(oldSource, newSource, true);
2578 1864 : os << LogIO::DEBUG1 << "SOURCE has " << nAddedCols << " optional columns." << LogIO::POST;
2579 :
2580 1864 : const MSSourceColumns incols(oldSource);
2581 1864 : MSSourceColumns outcols(newSource);
2582 :
2583 : // Copy the Measures frame info. This has to be done before filling the rows.
2584 1864 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2585 1864 : outcols.setDirectionRef(MDirection::castType(incols.directionMeas().getMeasRef().getType()));
2586 1864 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2587 1864 : outcols.setFrequencyRef(MFrequency::castType(incols.restFrequencyMeas().getMeasRef().getType()));
2588 1864 : outcols.setRadialVelocityRef(MRadialVelocity::castType(incols.sysvelMeas().getMeasRef().getType()));
2589 :
2590 1864 : if (!reindex_p)
2591 : {
2592 95 : TableCopy::copyRows(newSource, oldSource);
2593 95 : return true;
2594 : }
2595 :
2596 1769 : const ScalarColumn<Int>& inSId = incols.sourceId();
2597 1769 : ScalarColumn<Int>& outSId = outcols.sourceId();
2598 1769 : const ScalarColumn<Int>& inSPW = incols.spectralWindowId();
2599 1769 : ScalarColumn<Int>& outSPW = outcols.spectralWindowId();
2600 :
2601 : // row number in output.
2602 1769 : uInt outrn = 0;
2603 1769 : uInt nInputRows = inSId.nrow();
2604 : // inSidVal is Int.
2605 1769 : Int maxSId = sourceRelabel_p.nelements();
2606 1769 : Int maxSPWId = spwRelabel_p.nelements();
2607 51295 : for (uInt inrn = 0; inrn < nInputRows; ++inrn)
2608 : {
2609 49526 : Int inSidVal = inSId(inrn);
2610 : // -1 means the source is valid for any SPW.
2611 49526 : Int inSPWVal = inSPW(inrn);
2612 49526 : if (inSidVal >= maxSId)
2613 : {
2614 47 : os << LogIO::WARN << "Invalid SOURCE ID in SOURCE table row " << inrn << LogIO::POST;
2615 : }
2616 49526 : if (inSPWVal >= maxSPWId)
2617 : {
2618 3 : os << LogIO::WARN << "Invalid SPW ID in SOURCE table row " << inrn << LogIO::POST;
2619 : }
2620 :
2621 49525 : if ( (inSidVal > -1) &&
2622 49478 : (inSidVal < maxSId) &&
2623 146019 : (sourceRelabel_p[inSidVal] > -1) &&
2624 46968 : ((inSPWVal == -1) || (inSPWVal < maxSPWId && spwRelabel_p[inSPWVal] > -1)))
2625 : {
2626 : // Copy inrn to outrn.
2627 20039 : TableCopy::copyRows(newSource, oldSource, outrn, inrn, 1);
2628 20039 : outSId.put(outrn, sourceRelabel_p[inSidVal]);
2629 20039 : outSPW.put(outrn, inSPWVal > -1 ? spwRelabel_p[inSPWVal] : -1);
2630 20039 : ++outrn;
2631 : }
2632 : }
2633 :
2634 2054 : }
2635 :
2636 1769 : return true;
2637 : }
2638 :
2639 : // -----------------------------------------------------------------------
2640 : //
2641 : // -----------------------------------------------------------------------
2642 1864 : bool MSTransformDataHandler::copyAntenna()
2643 : {
2644 1864 : const MSAntenna& oldAnt = mssel_p.antenna();
2645 1864 : MSAntenna& newAnt = msOut_p.antenna();
2646 1864 : const MSAntennaColumns incols(oldAnt);
2647 1864 : MSAntennaColumns outcols(newAnt);
2648 1864 : bool retval = false;
2649 :
2650 1864 : outcols.setOffsetRef(MPosition::castType(incols.offsetMeas().getMeasRef().getType()));
2651 1864 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2652 :
2653 : //TableCopy::copyRows(newAnt, oldAnt);
2654 : //retval = true;
2655 :
2656 1864 : if (!antennaSel_p or !reindex_p)
2657 : {
2658 1838 : TableCopy::copyRows(newAnt, oldAnt);
2659 1838 : retval = true;
2660 : }
2661 : else
2662 : {
2663 26 : uInt nAnt = antNewIndex_p.nelements();
2664 : // Don't use min() here, it's too overloaded.
2665 26 : if (nAnt > oldAnt.nrow()) nAnt = oldAnt.nrow();
2666 :
2667 99 : for (uInt k = 0; k < nAnt; ++k)
2668 : {
2669 73 : if (antNewIndex_p[k] > -1) TableCopy::copyRows(newAnt, oldAnt, antNewIndex_p[k], k, 1, false);
2670 : }
2671 26 : retval = true;
2672 : }
2673 :
2674 1864 : return retval;
2675 1864 : }
2676 :
2677 : // -----------------------------------------------------------------------
2678 : //
2679 : // -----------------------------------------------------------------------
2680 1864 : bool MSTransformDataHandler::copyFeed()
2681 : {
2682 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2683 :
2684 1864 : const MSFeed& oldFeed = mssel_p.feed();
2685 1864 : MSFeed& newFeed = msOut_p.feed();
2686 1864 : const MSFeedColumns incols(oldFeed);
2687 1864 : MSFeedColumns outcols(newFeed);
2688 :
2689 1864 : outcols.setDirectionRef(MDirection::castType(incols.beamOffsetMeas().getMeasRef().getType()));
2690 1864 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2691 1864 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2692 :
2693 1864 : if ((!antennaSel_p && allEQ(spwRelabel_p, spw_p)) or !reindex_p)
2694 : {
2695 1406 : TableCopy::copyRows(newFeed, oldFeed);
2696 : }
2697 : else
2698 : {
2699 458 : const Vector<Int>& antIds = incols.antennaId().getColumn();
2700 458 : const Vector<Int>& spwIds = incols.spectralWindowId().getColumn();
2701 :
2702 : // Copy selected rows.
2703 458 : uInt totNFeeds = antIds.nelements();
2704 458 : uInt totalSelFeeds = 0;
2705 458 : Int maxSelAntp1 = antNewIndex_p.nelements();
2706 146580 : for (uInt k = 0; k < totNFeeds; ++k)
2707 : {
2708 : // antenna must be selected, and spwId must be -1 (any) or selected.
2709 146122 : if ( antIds[k] < maxSelAntp1 &&
2710 291669 : antNewIndex_p[antIds[k]] > -1 &&
2711 145547 : (spwIds[k] < 0 || spwRelabel_p[spwIds[k]] > -1)
2712 : )
2713 : {
2714 44585 : TableCopy::copyRows(newFeed, oldFeed, totalSelFeeds, k, 1,false);
2715 44585 : ++totalSelFeeds;
2716 : }
2717 : }
2718 :
2719 : // Remap antenna and spw #s.
2720 458 : ScalarColumn<Int>& antCol = outcols.antennaId();
2721 458 : ScalarColumn<Int>& spwCol = outcols.spectralWindowId();
2722 :
2723 45043 : for (uInt k = 0; k < totalSelFeeds; ++k)
2724 : {
2725 :
2726 44585 : antCol.put(k, antNewIndex_p[antCol(k)]);
2727 44585 : if (spwCol(k) > -1) spwCol.put(k, spwRelabel_p[spwCol(k)]);
2728 : }
2729 458 : }
2730 :
2731 : // Check if selected spw is WVR data. WVR data does not have FEED data
2732 : // so it is not a failure if newFeed.nrow == 0
2733 1864 : if (newFeed.nrow() < 1 and spw_p.size() == 1){
2734 : // Using the MSMetaData class
2735 0 : MSMetaData msmeta(&mssel_p, 0);
2736 0 : std::set<uInt> wvrspw = msmeta.getWVRSpw();
2737 0 : for (std::set<uInt>::iterator bbit = wvrspw.begin(); bbit != wvrspw.end(); ++bbit){
2738 0 : if ((uInt)spw_p[0] == *bbit){
2739 0 : os << LogIO::DEBUG1 << "Skipping spw="<<*bbit<<" because it is WVR and has no FEED content" << LogIO::POST;
2740 0 : return true;
2741 : }
2742 : }
2743 0 : }
2744 :
2745 1864 : if (newFeed.nrow() < 1)
2746 : {
2747 : // LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2748 0 : os << LogIO::SEVERE << "No feeds were selected." << LogIO::POST;
2749 0 : return false;
2750 : }
2751 :
2752 1864 : return true;
2753 1864 : }
2754 :
2755 : // -----------------------------------------------------------------------
2756 : // Get the processorId corresponding to a given DDI
2757 : // -----------------------------------------------------------------------
2758 0 : Int MSTransformDataHandler::getProcessorId(Int dataDescriptionId, String msname)
2759 : {
2760 0 : ostringstream taql;
2761 0 : taql << "SELECT PROCESSOR_ID from " << msname;
2762 0 : taql << " WHERE DATA_DESC_ID ==" << dataDescriptionId;
2763 0 : taql << " LIMIT 1";
2764 :
2765 0 : casacore::TableProxy *firstSelectedRow = new TableProxy(tableCommand(taql.str()).table());
2766 0 : Record colWrapper = firstSelectedRow->getVarColumn(String("PROCESSOR_ID"),0,1,1);
2767 0 : casacore::Vector<Int> processorId = colWrapper.asArrayInt("r1");
2768 :
2769 0 : delete firstSelectedRow;
2770 0 : return processorId[0];
2771 0 : }
2772 :
2773 :
2774 : // -----------------------------------------------------------------------
2775 : //
2776 : // -----------------------------------------------------------------------
2777 1864 : bool MSTransformDataHandler::copyFlag_Cmd()
2778 : {
2779 :
2780 : // Like POINTING, FLAG_CMD is supposed to exist but is allowed not to.
2781 1864 : if (Table::isReadable(mssel_p.flagCmdTableName()))
2782 : {
2783 1864 : const MSFlagCmd& oldFlag_Cmd = mssel_p.flagCmd();
2784 :
2785 1864 : if (oldFlag_Cmd.nrow() > 0)
2786 : {
2787 :
2788 : // Could be declared as Table&
2789 297 : MSFlagCmd& newFlag_Cmd = msOut_p.flagCmd();
2790 :
2791 594 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2792 :
2793 : // Add optional columns if present in oldFlag_Cmd.
2794 297 : uInt nAddedCols = addOptionalColumns(oldFlag_Cmd, newFlag_Cmd, true);
2795 297 : os << LogIO::DEBUG1 << "FLAG_CMD has " << nAddedCols << " optional columns." << LogIO::POST;
2796 :
2797 297 : const MSFlagCmdColumns oldFCs(oldFlag_Cmd);
2798 297 : MSFlagCmdColumns newFCs(newFlag_Cmd);
2799 297 : newFCs.setEpochRef(MEpoch::castType(oldFCs.timeMeas().getMeasRef().getType()));
2800 :
2801 297 : TableCopy::copyRows(newFlag_Cmd, oldFlag_Cmd);
2802 :
2803 297 : }
2804 : }
2805 :
2806 1864 : return true;
2807 : }
2808 :
2809 : // -----------------------------------------------------------------------
2810 : //
2811 : // -----------------------------------------------------------------------
2812 1864 : bool MSTransformDataHandler::copyHistory()
2813 : {
2814 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2815 :
2816 1864 : const MSHistory& oldHistory = mssel_p.history();
2817 :
2818 : // Could be declared as Table&
2819 1864 : MSHistory& newHistory = msOut_p.history();
2820 :
2821 : // Add optional columns if present in oldHistory.
2822 1864 : uInt nAddedCols = addOptionalColumns(oldHistory, newHistory, true);
2823 1864 : os << LogIO::DEBUG1 << "HISTORY has " << nAddedCols << " optional columns." << LogIO::POST;
2824 :
2825 1864 : const MSHistoryColumns oldHCs(oldHistory);
2826 1864 : MSHistoryColumns newHCs(newHistory);
2827 1864 : newHCs.setEpochRef(MEpoch::castType(oldHCs.timeMeas().getMeasRef().getType()));
2828 :
2829 1864 : TableCopy::copyRows(newHistory, oldHistory);
2830 :
2831 1864 : return true;
2832 1864 : }
2833 :
2834 : // -----------------------------------------------------------------------
2835 : //
2836 : // -----------------------------------------------------------------------
2837 1864 : bool MSTransformDataHandler::copyObservation()
2838 : {
2839 1864 : const MSObservation& oldObs = mssel_p.observation();
2840 1864 : MSObservation& newObs = msOut_p.observation();
2841 1864 : const MSObservationColumns oldObsCols(oldObs);
2842 1864 : MSObservationColumns newObsCols(newObs);
2843 1864 : newObsCols.setEpochRef(MEpoch::castType(oldObsCols.releaseDateMeas().getMeasRef().getType()));
2844 :
2845 1864 : uInt nObs = selObsId_p.nelements();
2846 1864 : if (nObs > 0 and reindex_p)
2847 : {
2848 21 : for (uInt outrn = 0; outrn < nObs; ++outrn)
2849 : {
2850 12 : TableCopy::copyRows(newObs, oldObs, outrn, selObsId_p[outrn], 1);
2851 : }
2852 :
2853 9 : }
2854 : else
2855 : {
2856 1855 : TableCopy::copyRows(newObs, oldObs);
2857 : }
2858 :
2859 1864 : return true;
2860 1864 : }
2861 :
2862 : // -----------------------------------------------------------------------
2863 : //
2864 : // -----------------------------------------------------------------------
2865 1864 : bool MSTransformDataHandler::copyProcessor()
2866 : {
2867 1864 : const MSProcessor& oldProc = mssel_p.processor();
2868 1864 : MSProcessor& newProc = msOut_p.processor();
2869 1864 : TableCopy::copyRows(newProc, oldProc);
2870 :
2871 1864 : return true;
2872 : }
2873 :
2874 : // -----------------------------------------------------------------------
2875 : //
2876 : // -----------------------------------------------------------------------
2877 1864 : bool MSTransformDataHandler::copyState()
2878 : {
2879 : // STATE is allowed to not exist, even though it is not optional in
2880 : // the MS def'n. For one thing, split dropped it for quite a while.
2881 1864 : if (Table::isReadable(mssel_p.stateTableName()))
2882 : {
2883 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2884 1864 : const MSState& oldState = mssel_p.state();
2885 :
2886 1864 : if (oldState.nrow() > 0)
2887 : {
2888 1654 : if (!intentString_p.empty() and reindex_p)
2889 : {
2890 102 : MSState& newState = msOut_p.state();
2891 102 : const MSStateColumns oldStateCols(oldState);
2892 102 : MSStateColumns newStateCols(newState);
2893 :
2894 : // Initialize stateRemapper_p if necessary.
2895 : // if (stateRemapper_p.size() < 1) make_map(stateRemapper_p, mscIn_p->stateId().getColumn());
2896 :
2897 : // jagonzal (CAS-6351): Do not apply implicit re-indexing //////////////////////////////////////////////////
2898 : //
2899 : // Get list of selected scan intent indexes
2900 102 : MSSelection mssel;
2901 102 : mssel.setStateExpr(intentString_p);
2902 102 : Vector<Int> scanIntentList = mssel.getStateObsModeList(getInputMS());
2903 : //
2904 : // Populate state re-mapper using all selected indexes (not only the implicit ones)
2905 102 : stateRemapper_p.clear();
2906 342 : for (uInt index=0; index < scanIntentList.size(); index++)
2907 : {
2908 240 : stateRemapper_p[scanIntentList(index)] = index;
2909 : }
2910 : ///////////////////////////////////////////////////////////////////////////////////////////////////////////
2911 :
2912 102 : uInt nStates = stateRemapper_p.size();
2913 :
2914 : // stateRemapper_p goes from input to output, as is wanted in most
2915 : // places. Here we need a map going the other way, so make one.
2916 102 : Vector<Int> outStateToInState(nStates);
2917 102 : std::map<Int, Int>::iterator mit;
2918 :
2919 342 : for (mit = stateRemapper_p.begin(); mit != stateRemapper_p.end(); ++mit)
2920 : {
2921 240 : outStateToInState[(*mit).second] = (*mit).first;
2922 : }
2923 :
2924 :
2925 342 : for (uInt outrn = 0; outrn < nStates; ++outrn)
2926 : {
2927 240 : TableCopy::copyRows(newState, oldState, outrn,outStateToInState[outrn], 1);
2928 : }
2929 102 : }
2930 : // jagonzal (CAS-6351): Do not apply implicit re-indexing
2931 : // Therefore just copy the input state sub-table to the output state sub-table
2932 : else
2933 : {
2934 1552 : MSState& newState = msOut_p.state();
2935 1552 : TableCopy::copyRows(newState, oldState);
2936 : }
2937 :
2938 : }
2939 1864 : }
2940 1864 : return true;
2941 : }
2942 :
2943 : // -----------------------------------------------------------------------
2944 : //
2945 : // -----------------------------------------------------------------------
2946 1864 : bool MSTransformDataHandler::copySyscal()
2947 : {
2948 : // SYSCAL is allowed to not exist.
2949 1864 : if (Table::isReadable(mssel_p.sysCalTableName()))
2950 : {
2951 1268 : const MSSysCal& oldSysc = mssel_p.sysCal();
2952 :
2953 1268 : if (oldSysc.nrow() > 0)
2954 : {
2955 : // Add a SYSCAL subtable to msOut_p with 0 rows for now.
2956 625 : Table::TableOption option = Table::New;
2957 625 : TableDesc sysCalTD = MSSysCal::requiredTableDesc();
2958 625 : SetupNewTable sysCalSetup(msOut_p.sysCalTableName(), sysCalTD,option);
2959 625 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::SYSCAL),Table(sysCalSetup, 0));
2960 :
2961 : // update the references to the subtable keywords
2962 625 : msOut_p.initRefs();
2963 :
2964 : // Could be declared as Table&.
2965 625 : MSSysCal& newSysc = msOut_p.sysCal();
2966 :
2967 1250 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2968 :
2969 625 : uInt nAddedCols = addOptionalColumns(oldSysc, newSysc, true);
2970 625 : os << LogIO::DEBUG1 << "SYSCAL has " << nAddedCols << " optional columns." << LogIO::POST;
2971 :
2972 625 : const MSSysCalColumns incols(oldSysc);
2973 625 : MSSysCalColumns outcols(newSysc);
2974 625 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2975 :
2976 625 : if ((!antennaSel_p && allEQ(spwRelabel_p, spw_p)) or !reindex_p)
2977 : {
2978 464 : TableCopy::copyRows(newSysc, oldSysc);
2979 : }
2980 : else
2981 : {
2982 161 : const Vector<Int>& antIds = incols.antennaId().getColumn();
2983 161 : const Vector<Int>& spwIds = incols.spectralWindowId().getColumn();
2984 :
2985 : // Copy selected rows.
2986 161 : uInt totNSyscals = antIds.nelements();
2987 161 : uInt totalSelSyscals = 0;
2988 161 : Int maxSelAntp1 = antNewIndex_p.nelements(); // Int for comparison with antIds.
2989 29846 : for (uInt k = 0; k < totNSyscals; ++k)
2990 : {
2991 : // antenna must be selected, and spwId must be -1 (any) or selected.
2992 29685 : if ( antIds[k] < maxSelAntp1 &&
2993 59370 : antNewIndex_p[antIds[k]] > -1 &&
2994 29685 : (spwIds[k] < 0 || spwRelabel_p[spwIds[k]] > -1)
2995 : )
2996 : {
2997 22738 : TableCopy::copyRows(newSysc, oldSysc, totalSelSyscals,k, 1, false);
2998 22738 : ++totalSelSyscals;
2999 : }
3000 : }
3001 :
3002 : // Remap antenna and spw #s.
3003 161 : ScalarColumn<Int>& antCol = outcols.antennaId();
3004 161 : ScalarColumn<Int>& spwCol = outcols.spectralWindowId();
3005 :
3006 22899 : for (uInt k = 0; k < totalSelSyscals; ++k)
3007 : {
3008 22738 : antCol.put(k, antNewIndex_p[antCol(k)]);
3009 22738 : if (spwCol(k) > -1) spwCol.put(k, spwRelabel_p[spwCol(k)]);
3010 : }
3011 161 : }
3012 625 : }
3013 : }
3014 :
3015 1864 : return true;
3016 : }
3017 :
3018 1864 : bool MSTransformDataHandler::copyWeather()
3019 : {
3020 : // Weather is allowed to not exist.
3021 1864 : if (Table::isReadable(mssel_p.weatherTableName()))
3022 : {
3023 1259 : const MSWeather& oldWeath = mssel_p.weather();
3024 :
3025 1259 : if (oldWeath.nrow() > 0)
3026 : {
3027 : // Add a WEATHER subtable to msOut_p with 0 rows for now.
3028 1259 : Table::TableOption option = Table::New;
3029 1259 : TableDesc weatherTD = MSWeather::requiredTableDesc();
3030 1259 : SetupNewTable weatherSetup(msOut_p.weatherTableName(), weatherTD,option);
3031 1259 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::WEATHER),Table(weatherSetup, 0));
3032 :
3033 : // update the references to the subtable keywords
3034 1259 : msOut_p.initRefs();
3035 :
3036 1259 : MSWeather& newWeath = msOut_p.weather(); // Could be declared as Table&
3037 :
3038 2518 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3039 :
3040 1259 : uInt nAddedCols = addOptionalColumns(oldWeath, newWeath, true);
3041 1259 : os << LogIO::DEBUG1 << "WEATHER has " << nAddedCols << " optional columns." << LogIO::POST;
3042 :
3043 1259 : const MSWeatherColumns oldWCs(oldWeath);
3044 1259 : MSWeatherColumns newWCs(newWeath);
3045 1259 : newWCs.setEpochRef(MEpoch::castType(oldWCs.timeMeas().getMeasRef().getType()));
3046 :
3047 1259 : if (!antennaSel_p or !reindex_p)
3048 : {
3049 1236 : TableCopy::copyRows(newWeath, oldWeath);
3050 : }
3051 : else
3052 : {
3053 23 : const Vector<Int>& antIds(oldWCs.antennaId().getColumn());
3054 23 : ScalarColumn<Int>& outants = newWCs.antennaId();
3055 :
3056 23 : uInt selRow = 0;
3057 23 : Int maxSelAntp1 = antNewIndex_p.nelements();
3058 :
3059 1638 : for (uInt k = 0; k < antIds.nelements(); ++k)
3060 : {
3061 : // Weather station is on antenna?
3062 1615 : if (antIds[k] > -1)
3063 : {
3064 : // remap ant num
3065 0 : if (antIds[k] < maxSelAntp1)
3066 : {
3067 0 : Int newAntInd = antNewIndex_p[antIds[k]];
3068 :
3069 : // Ant selected?
3070 0 : if (newAntInd > -1)
3071 : {
3072 0 : TableCopy::copyRows(newWeath, oldWeath, selRow,k, 1);
3073 0 : outants.put(selRow, newAntInd);
3074 0 : ++selRow;
3075 : }
3076 : }
3077 : }
3078 : else
3079 : {
3080 : // means valid for all antennas.
3081 1615 : TableCopy::copyRows(newWeath, oldWeath, selRow, k, 1);
3082 1615 : outants.put(selRow, -1);
3083 1615 : ++selRow;
3084 : }
3085 : }
3086 23 : }
3087 1259 : }
3088 : }
3089 :
3090 1864 : return true;
3091 : }
3092 :
3093 : // -----------------------------------------------------------------------
3094 : //
3095 : // -----------------------------------------------------------------------
3096 3728 : bool MSTransformDataHandler::filterOptSubtable(const String& subtabname)
3097 : {
3098 7456 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3099 :
3100 : // subtabname is allowed to not exist. Use ms_p instead of mssel_p, because
3101 : // the latter has a random name which does NOT necessarily lead to
3102 : // subtabname. (And if selection took care of subtables, we probably
3103 : // wouldn't have to do it here.)
3104 3728 : if (Table::isReadable(ms_p.tableName() + '/' + subtabname))
3105 : {
3106 1536 : const Table intab(ms_p.tableName() + '/' + subtabname);
3107 :
3108 768 : if (intab.nrow() > 0) {
3109 :
3110 : // Add feed if selecting by it is ever added.
3111 400 : bool doFilter = (antennaSel_p || !allEQ(spwRelabel_p, spw_p)) && reindex_p;
3112 :
3113 400 : copySubtable(subtabname, intab, doFilter);
3114 :
3115 400 : if (doFilter) {
3116 :
3117 : // At this point msOut_p has subtab with 0 rows.
3118 380 : Table outtab(msOut_p.tableName() + '/' + subtabname,Table::Update);
3119 190 : ScalarColumn<Int> inAntIdCol(intab, "ANTENNA_ID");
3120 190 : ScalarColumn<Int> inSpwIdCol(intab, "SPECTRAL_WINDOW_ID");
3121 190 : const Vector<Int>& antIds = inAntIdCol.getColumn();
3122 190 : const Vector<Int>& spwIds = inSpwIdCol.getColumn();
3123 :
3124 : // Copy selected rows.
3125 190 : uInt totNOuttabs = antIds.nelements();
3126 190 : uInt totalSelOuttabs = 0;
3127 :
3128 : // Int for comparison with antIds.
3129 190 : Int maxSelAntp1 = antNewIndex_p.nelements();
3130 :
3131 190 : bool haveRemappingProblem = false;
3132 4423741 : for (uInt inrow = 0; inrow < totNOuttabs; ++inrow)
3133 : {
3134 : // antenna must be selected, and spwId must be -1 (any) or selected.
3135 : // Extra care must be taken because for a while MSes had CALDEVICE
3136 : // and SYSPOWER, but the ANTENNA_ID and SPECTRAL_WINDOW_ID of those
3137 : // sub-tables were not being re-mapped by split.
3138 4423551 : if (antIds[inrow] < maxSelAntp1 && antNewIndex_p[antIds[inrow]] > -1)
3139 : {
3140 :
3141 8807826 : if (spwIds[inrow] < 0 ||
3142 4403913 : (spwIds[inrow] < static_cast<Int> (spwRelabel_p.nelements()) &&
3143 4401458 : spwRelabel_p[spwIds[inrow]] > -1))
3144 : {
3145 831136 : TableCopy::copyRows(outtab, intab, totalSelOuttabs,inrow, 1, false);
3146 831136 : ++totalSelOuttabs;
3147 : }
3148 :
3149 : // Ideally we'd like to catch antenna errors too. They are
3150 : // avoided, but because of the way antNewIndex_p is made,
3151 : // antIds[inrow] >= maxSelAntp1
3152 : // is not necessarily an error. It's not even possible to catch
3153 : // all the spw errors, so we settle for catching the ones we can
3154 : // and reliably avoiding segfaults.
3155 3572777 : else if (spwIds[inrow] >= static_cast<Int> (spwRelabel_p.nelements()))
3156 : {
3157 2455 : haveRemappingProblem = true;
3158 : }
3159 :
3160 : }
3161 : }
3162 :
3163 190 : if (haveRemappingProblem)
3164 : {
3165 : os << LogIO::WARN << "At least one row of "
3166 : << intab.tableName()
3167 : << " has an antenna or spw mismatch.\n"
3168 : << "(Presumably from an older split, sorry.)\n"
3169 : << "If " << subtabname
3170 : << " is important, it should be fixed with tb or browsetable,\n"
3171 : << "or by redoing the split that made "
3172 : << ms_p.tableName() << " (check its history)."
3173 2 : << LogIO::POST;
3174 : }
3175 :
3176 :
3177 : // Remap antenna and spw #s.
3178 190 : ScalarColumn<Int> outAntCol(outtab, "ANTENNA_ID");
3179 190 : ScalarColumn<Int> outSpwCol(outtab, "SPECTRAL_WINDOW_ID");
3180 :
3181 831326 : for (uInt k = 0; k < totalSelOuttabs; ++k)
3182 : {
3183 831136 : outAntCol.put(k, antNewIndex_p[outAntCol(k)]);
3184 831136 : if (outSpwCol(k) > -1) outSpwCol.put(k, spwRelabel_p[outSpwCol(k)]);
3185 : }
3186 190 : }
3187 : }
3188 768 : }
3189 :
3190 3728 : return true;
3191 3728 : }
3192 :
3193 : // -----------------------------------------------------------------------
3194 : //
3195 : // -----------------------------------------------------------------------
3196 1864 : bool MSTransformDataHandler::copyGenericSubtables()
3197 : {
3198 3728 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3199 :
3200 : // Already handled subtables will be removed from this,
3201 : // so a modifiable copy is needed.
3202 1864 : TableRecord inkws(mssel_p.keywordSet());
3203 :
3204 : // Some of the standard subtables need special handling,
3205 : // e.g. DATA_DESCRIPTION, SPECTRAL_WINDOW, and ANTENNA, so they are already
3206 : // defined in msOut_p. Several more (e.g. FLAG_CMD) were also already
3207 : // created by MS::createDefaultSubtables(). Don't try to write over them - a
3208 : // locking error will result.
3209 1864 : const TableRecord& outkws = msOut_p.keywordSet();
3210 30244 : for (uInt i = 0; i < outkws.nfields(); ++i)
3211 : {
3212 28380 : if (outkws.type(i) == TpTable && inkws.isDefined(outkws.name(i)))
3213 : {
3214 26516 : inkws.removeField(outkws.name(i));
3215 : }
3216 : }
3217 :
3218 : // Find ephemerides files
3219 1864 : std::vector<String> ephemerides;
3220 7364 : for (uInt i = 0; i < inkws.nfields(); ++i)
3221 : {
3222 5500 : if (inkws.type(i) == TpTable && inkws.name(i).contains("EPHEM"))
3223 : {
3224 25 : ephemerides.push_back(inkws.name(i));
3225 : }
3226 : }
3227 :
3228 : // Remove ephemerides files
3229 1889 : for (uInt i = 0; i < ephemerides.size(); ++i)
3230 : {
3231 25 : inkws.removeField(ephemerides.at(i));
3232 : }
3233 :
3234 :
3235 : // msOut_p.rwKeywordSet() (pass a reference, not a copy) will put a lock on msOut_p.
3236 1864 : TableCopy::copySubTables(msOut_p.rwKeywordSet(), inkws, msOut_p.tableName(), msOut_p.tableType(), mssel_p);
3237 :
3238 : // TableCopy::copySubTables(Table, Table, bool) includes this other code,
3239 : // which seems to be copying subtables at one level deeper, but not recursively?
3240 1864 : const TableDesc& inDesc = mssel_p.tableDesc();
3241 1864 : const TableDesc& outDesc = msOut_p.tableDesc();
3242 43892 : for (uInt i = 0; i < outDesc.ncolumn(); ++i)
3243 : {
3244 : // Only writable cols can have keywords (and thus subtables) defined.
3245 42028 : if (msOut_p.isColumnWritable(i))
3246 : {
3247 42028 : const String& name = outDesc[i].name();
3248 :
3249 42028 : if (inDesc.isColumn(name))
3250 : {
3251 41695 : TableColumn outCol(msOut_p, name);
3252 41695 : TableColumn inCol(mssel_p, name);
3253 :
3254 41695 : TableCopy::copySubTables(outCol.rwKeywordSet(),
3255 : inCol.keywordSet(),
3256 : msOut_p.tableName(),
3257 : msOut_p.tableType(),
3258 : mssel_p);
3259 : // Copy the keywords if column is [FLOAT_|CORRECTED_]DATA
3260 : // KS NOTE CORRECTED_DATA -> DATA case should be handled separately.
3261 41695 : if (name == "FLOAT_DATA" || name == "DATA" || name == "CORRECTED_DATA")
3262 2019 : copyMainTableKeywords(outCol.rwKeywordSet(), inCol.keywordSet());
3263 :
3264 41695 : }
3265 : }
3266 : }
3267 :
3268 1864 : return true;
3269 1864 : }
3270 :
3271 : // -----------------------------------------------------------------------
3272 : // Method to merge SPW sub-tables from SubMSs to create the MMS level SPW sub-table
3273 : // -----------------------------------------------------------------------
3274 28 : bool MSTransformDataHandler::mergeSpwSubTables(Vector<String> filenames)
3275 : {
3276 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3277 :
3278 28 : String filename_0 = filenames(0);
3279 28 : MeasurementSet ms_0(filename_0,Table::Update);
3280 :
3281 28 : if (Table::isReadable(ms_0.spectralWindowTableName()) and !ms_0.spectralWindow().isNull())
3282 : {
3283 28 : MSSpectralWindow spwTable_0 = ms_0.spectralWindow();
3284 :
3285 28 : if (spwTable_0.nrow() > 0)
3286 : {
3287 56 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3288 56 : << "Merging SPECTRAL_WINDOW sub-tables from all sub-MSs to form MMS-level SPECTRAL_WINDOW sub-table" << LogIO::POST;
3289 :
3290 28 : MSSpWindowColumns spwCols_0(spwTable_0);
3291 :
3292 : // Map subMS with spw_id to merge the FEED table later
3293 28 : Vector<uInt> mapSubmsSpwid;
3294 :
3295 : // subMS_0000 starts with spw 0
3296 28 : uInt spwStart = 0;
3297 28 : mapSubmsSpwid.resize(filenames.size());
3298 28 : mapSubmsSpwid[0] = spwStart;
3299 :
3300 : // for next subMS
3301 28 : uInt rowIndex = spwTable_0.nrow();
3302 28 : spwStart = spwStart + rowIndex;
3303 100 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3304 : {
3305 72 : String filename_i = filenames(subms_index);
3306 72 : MeasurementSet ms_i(filename_i);
3307 72 : MSSpectralWindow spwTable_i = ms_i.spectralWindow();
3308 :
3309 72 : if (spwTable_i.nrow() > 0)
3310 : {
3311 72 : MSSpWindowColumns spwCols_i(spwTable_i);
3312 :
3313 72 : uInt nrow = spwTable_i.nrow();
3314 72 : spwTable_0.addRow(nrow);
3315 :
3316 : // Map of this subMS to spw ID
3317 72 : mapSubmsSpwid[subms_index] = spwStart;
3318 :
3319 : // for next subMS
3320 72 : spwStart = spwStart + nrow;
3321 :
3322 160 : for (uInt subms_row_index=0;subms_row_index<spwTable_i.nrow();subms_row_index++)
3323 : {
3324 88 : spwCols_0.measFreqRef().put(rowIndex,spwCols_i.measFreqRef()(subms_row_index));
3325 88 : spwCols_0.chanFreq().put(rowIndex,spwCols_i.chanFreq()(subms_row_index));
3326 88 : spwCols_0.refFrequency().put(rowIndex,spwCols_i.refFrequency()(subms_row_index));
3327 88 : spwCols_0.chanWidth().put(rowIndex,spwCols_i.chanWidth()(subms_row_index));
3328 88 : spwCols_0.effectiveBW().put(rowIndex,spwCols_i.effectiveBW()(subms_row_index));
3329 88 : spwCols_0.resolution().put(rowIndex,spwCols_i.resolution()(subms_row_index));
3330 88 : spwCols_0.flagRow().put(rowIndex,spwCols_i.flagRow()(subms_row_index));
3331 88 : spwCols_0.freqGroup().put(rowIndex,spwCols_i.freqGroup()(subms_row_index));
3332 88 : spwCols_0.freqGroupName().put(rowIndex,spwCols_i.freqGroupName()(subms_row_index));
3333 88 : spwCols_0.ifConvChain().put(rowIndex,spwCols_i.ifConvChain()(subms_row_index));
3334 88 : spwCols_0.name().put(rowIndex,spwCols_i.name()(subms_row_index));
3335 88 : spwCols_0.netSideband().put(rowIndex,spwCols_i.netSideband()(subms_row_index));
3336 88 : spwCols_0.numChan().put(rowIndex,spwCols_i.numChan()(subms_row_index));
3337 88 : spwCols_0.totalBandwidth().put(rowIndex,spwCols_i.totalBandwidth()(subms_row_index));
3338 :
3339 : // Optional columns
3340 88 : if (columnOk(spwCols_i.bbcNo()))
3341 26 : spwCols_0.bbcNo().put(rowIndex,spwCols_i.bbcNo()(subms_row_index));
3342 :
3343 88 : if (columnOk(spwCols_i.assocSpwId()))
3344 6 : spwCols_0.assocSpwId().put(rowIndex,spwCols_i.assocSpwId()(subms_row_index));
3345 :
3346 88 : if(columnOk(spwCols_i.assocNature()))
3347 6 : spwCols_0.assocNature().put(rowIndex,spwCols_i.assocNature()(subms_row_index));
3348 :
3349 88 : if (columnOk(spwCols_i.bbcSideband()))
3350 0 : spwCols_0.bbcSideband().put(rowIndex,spwCols_i.bbcSideband()(subms_row_index));
3351 :
3352 88 : if (columnOk(spwCols_i.dopplerId()))
3353 4 : spwCols_0.dopplerId().put(rowIndex,spwCols_i.dopplerId()(subms_row_index));
3354 :
3355 88 : if (columnOk(spwCols_i.receiverId()))
3356 0 : spwCols_0.receiverId().put(rowIndex,spwCols_i.receiverId()(subms_row_index));
3357 :
3358 191 : if (spwTable_i.tableDesc().isColumn("SDM_WINDOW_FUNCTION") &&
3359 103 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_WINDOW_FUNCTION"))
3360 : {
3361 15 : ScalarColumn<String> swfCol_i(spwTable_i, "SDM_WINDOW_FUNCTION");
3362 15 : ScalarColumn<String> swfCol_0(spwTable_0, "SDM_WINDOW_FUNCTION");
3363 15 : swfCol_0.put(rowIndex, swfCol_i(subms_row_index));
3364 15 : }
3365 :
3366 191 : if (spwTable_i.tableDesc().isColumn("SDM_NUM_BIN") &&
3367 103 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_NUM_BIN"))
3368 : {
3369 15 : ScalarColumn<Int> snbCol_i(spwTable_i, "SDM_NUM_BIN");
3370 15 : ScalarColumn<Int> snbCol_0(spwTable_0, "SDM_NUM_BIN");
3371 15 : snbCol_0.put(rowIndex, snbCol_i(subms_row_index));
3372 15 : }
3373 191 : if (spwTable_i.tableDesc().isColumn("SDM_CORR_BIT") &&
3374 103 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_CORR_BIT"))
3375 : {
3376 15 : ScalarColumn<String> corrBitCol_i(spwTable_i, "SDM_CORR_BIT");
3377 15 : ScalarColumn<String> corrBitCol_0(spwTable_0, "SDM_CORR_BIT");
3378 15 : corrBitCol_0.put(rowIndex, corrBitCol_i(subms_row_index));
3379 15 : }
3380 :
3381 88 : rowIndex += 1;
3382 : }
3383 72 : }
3384 72 : }
3385 :
3386 : // Merge the other sub-tables using SPW map generated here
3387 28 : mergeDDISubTables(filenames);
3388 28 : mergeFeedSubTables(filenames, mapSubmsSpwid);
3389 28 : mergeSourceSubTables(filenames, mapSubmsSpwid);
3390 28 : mergeFreqOffsetTables(filenames, mapSubmsSpwid);
3391 28 : mergeCalDeviceSubtables(filenames, mapSubmsSpwid);
3392 28 : mergeSysPowerSubtables(filenames, mapSubmsSpwid);
3393 28 : mergeSyscalSubTables(filenames, mapSubmsSpwid);
3394 28 : }
3395 : else
3396 : {
3397 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3398 0 : << "SPECTRAL_WINDOW sub-table found but has no valid content" << LogIO::POST;
3399 0 : return false;
3400 : }
3401 28 : }
3402 : else
3403 : {
3404 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3405 0 : << "SPECTRAL_WINDOW sub-table not found " << LogIO::POST;
3406 0 : return false;
3407 : }
3408 :
3409 28 : return true;
3410 28 : }
3411 :
3412 : // -----------------------------------------------------------------------
3413 : // Method to merge DDI sub-tables from SubMSs to create the MMS-level DDI sub-table
3414 : // -----------------------------------------------------------------------
3415 28 : bool MSTransformDataHandler::mergeDDISubTables(Vector<String> filenames)
3416 : {
3417 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3418 :
3419 28 : String filename_0 = filenames(0);
3420 28 : MeasurementSet ms_0(filename_0,Table::Update);
3421 :
3422 28 : if (Table::isReadable(ms_0.dataDescriptionTableName()) and !ms_0.dataDescription().isNull())
3423 : {
3424 28 : MSDataDescription ddiTable_0 = ms_0.dataDescription();
3425 :
3426 28 : if (ddiTable_0.nrow() > 0)
3427 : {
3428 56 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3429 56 : << "Merging DDI sub-tables from all sub-MSs to form MMS-level DDI sub-table" << LogIO::POST;
3430 :
3431 28 : MSDataDescColumns ddiCols_0(ddiTable_0);
3432 :
3433 28 : uInt rowIndex = ddiTable_0.nrow();
3434 100 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3435 : {
3436 72 : String filename_i = filenames(subms_index);
3437 72 : MeasurementSet ms_i(filename_i);
3438 72 : MSDataDescription dditable_i = ms_i.dataDescription();
3439 :
3440 72 : if (dditable_i.nrow() > 0)
3441 : {
3442 72 : MSDataDescColumns ddicols_i(dditable_i);
3443 :
3444 72 : ddiTable_0.addRow(dditable_i.nrow());
3445 :
3446 162 : for (uInt subms_row_index=0;subms_row_index<dditable_i.nrow();subms_row_index++)
3447 : {
3448 : // get the last spw id entered in the 0th DD table
3449 90 : uInt spwid = ddiCols_0.spectralWindowId().get(rowIndex-1);
3450 :
3451 90 : ddiCols_0.flagRow().put(rowIndex,ddicols_i.flagRow()(subms_row_index));
3452 90 : ddiCols_0.polarizationId().put(rowIndex,ddicols_i.polarizationId()(subms_row_index));
3453 :
3454 : // Take into account that some SPW may be pointed by several DDIs
3455 90 : uInt deltaDDI = 1;
3456 90 : if (subms_row_index>0)
3457 : {
3458 18 : deltaDDI = ddicols_i.spectralWindowId()(subms_row_index) - ddicols_i.spectralWindowId()(subms_row_index-1);
3459 : }
3460 :
3461 90 : ddiCols_0.spectralWindowId().put(rowIndex,spwid+deltaDDI);
3462 90 : rowIndex += 1;
3463 : }
3464 72 : }
3465 72 : }
3466 28 : }
3467 : else
3468 : {
3469 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3470 0 : << "DDI sub-table found but has no valid content" << LogIO::POST;
3471 0 : return false;
3472 : }
3473 28 : }
3474 : else
3475 : {
3476 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3477 0 : << "DDI sub-table not found " << LogIO::POST;
3478 0 : return false;
3479 : }
3480 :
3481 28 : return true;
3482 28 : }
3483 :
3484 :
3485 : // -----------------------------------------------------------------------
3486 : // Method to merge FEED sub-tables from SubMSs to create the MMS-level FEED sub-table
3487 : // -----------------------------------------------------------------------
3488 28 : bool MSTransformDataHandler::mergeFeedSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3489 : {
3490 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3491 :
3492 28 : if (filenames.size() != mapSubmsSpwid.size())
3493 : {
3494 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3495 0 : return false;
3496 : }
3497 :
3498 28 : String filename_0 = filenames(0);
3499 28 : MeasurementSet ms_0(filename_0,Table::Update);
3500 :
3501 28 : if (Table::isReadable(ms_0.feedTableName()) and !ms_0.feed().isNull())
3502 : {
3503 28 : MSFeed feedTable_0 = ms_0.feed();
3504 :
3505 : // CAS-7167. The WVR spw has no FEED content.
3506 : // if (feedTable_0.nrow() >= 0)
3507 : // {
3508 56 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3509 56 : << "Merging FEED sub-tables from all sub-MSs to form MMS-level FEED sub-table" << LogIO::POST;
3510 :
3511 28 : MSFeedColumns feedCols_0(feedTable_0);
3512 :
3513 28 : uInt rowIndex = feedTable_0.nrow();
3514 100 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3515 : {
3516 72 : String filename_i = filenames(subms_index);
3517 72 : MeasurementSet ms_i(filename_i);
3518 72 : MSFeed feedtable_i = ms_i.feed();
3519 :
3520 72 : if (feedtable_i.nrow() > 0)
3521 : {
3522 72 : MSFeedColumns feedcols_i(feedtable_i);
3523 :
3524 72 : feedTable_0.addRow(feedtable_i.nrow());
3525 :
3526 : // Prepare row reference object
3527 72 : RefRows refRow(rowIndex,rowIndex+feedtable_i.nrow()-1);
3528 :
3529 : // Re-index SPW col
3530 72 : Vector<Int> spectralWindowId_output(feedtable_i.nrow(),mapSubmsSpwid[subms_index]);
3531 72 : spectralWindowId_output += feedcols_i.spectralWindowId().getColumn();
3532 72 : feedCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3533 :
3534 : // Columns that can be just copied
3535 72 : feedCols_0.position().putColumnCells(refRow,feedcols_i.position().getColumn());
3536 72 : feedCols_0.beamOffset().putColumnCells(refRow,feedcols_i.beamOffset().getColumn());
3537 72 : feedCols_0.polarizationType().putColumnCells(refRow,feedcols_i.polarizationType().getColumn());
3538 72 : feedCols_0.polResponse().putColumnCells(refRow,feedcols_i.polResponse().getColumn());
3539 72 : feedCols_0.receptorAngle().putColumnCells(refRow,feedcols_i.receptorAngle().getColumn());
3540 72 : feedCols_0.antennaId().putColumnCells(refRow,feedcols_i.antennaId().getColumn());
3541 72 : feedCols_0.beamId().putColumnCells(refRow,feedcols_i.beamId().getColumn());
3542 72 : feedCols_0.feedId().putColumnCells(refRow,feedcols_i.feedId().getColumn());
3543 72 : feedCols_0.interval().putColumnCells(refRow,feedcols_i.interval().getColumn());
3544 72 : feedCols_0.numReceptors().putColumnCells(refRow,feedcols_i.numReceptors().getColumn());
3545 72 : feedCols_0.time().putColumnCells(refRow,feedcols_i.time().getColumn());
3546 :
3547 : // optional columns
3548 72 : if (columnOk(feedcols_i.focusLength()))
3549 : {
3550 0 : feedCols_0.focusLength().putColumnCells(refRow,feedcols_i.focusLength().getColumn());
3551 : }
3552 :
3553 72 : if (columnOk(feedcols_i.phasedFeedId()))
3554 : {
3555 0 : feedCols_0.phasedFeedId().putColumnCells(refRow,feedcols_i.phasedFeedId().getColumn());
3556 : }
3557 :
3558 : // Increment row offset
3559 72 : rowIndex += feedtable_i.nrow();
3560 72 : }
3561 72 : }
3562 :
3563 : // }
3564 : /*
3565 : else
3566 : {
3567 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3568 : << "FEED sub-table found but has no valid content" << LogIO::POST;
3569 : return false;
3570 : }
3571 : */
3572 28 : }
3573 : else
3574 : {
3575 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__) <<
3576 0 : "FEED sub-table not found " << LogIO::POST;
3577 0 : return false;
3578 : }
3579 :
3580 28 : return true;
3581 28 : }
3582 :
3583 : // -----------------------------------------------------------------------
3584 : // Method to merge Source sub-tables from SubMSs to create the MMS-level FEED sub-table
3585 : // -----------------------------------------------------------------------
3586 28 : bool MSTransformDataHandler::mergeSourceSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3587 : {
3588 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3589 :
3590 28 : if (filenames.size() != mapSubmsSpwid.size())
3591 : {
3592 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3593 0 : return false;
3594 : }
3595 :
3596 28 : String filename_0 = filenames(0);
3597 28 : MeasurementSet ms_0(filename_0,Table::Update);
3598 :
3599 28 : if (Table::isReadable(ms_0.sourceTableName()) and !ms_0.source().isNull())
3600 : {
3601 28 : MSSource sourceTable_0 = ms_0.source();
3602 :
3603 28 : if (sourceTable_0.nrow() > 0)
3604 : {
3605 54 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3606 54 : << "Merging SOURCE sub-tables from all sub-MSs to form MMS-level SOURCE sub-table" << LogIO::POST;
3607 :
3608 27 : MSSourceColumns sourceCols_0(sourceTable_0);
3609 :
3610 27 : uInt rowIndex = sourceTable_0.nrow();
3611 98 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3612 : {
3613 71 : String filename_i = filenames(subms_index);
3614 71 : MeasurementSet ms_i(filename_i);
3615 71 : MSSource sourcetable_i = ms_i.source();
3616 :
3617 71 : if (sourcetable_i.nrow() > 0)
3618 : {
3619 71 : MSSourceColumns sourcecols_i(sourcetable_i);
3620 :
3621 71 : sourceTable_0.addRow(sourcetable_i.nrow());
3622 :
3623 : // Prepare row reference object
3624 71 : RefRows refRow(rowIndex,rowIndex+sourcetable_i.nrow()-1);
3625 :
3626 : // Re-index SPW col
3627 71 : Vector<Int> spectralWindowId_output(sourcetable_i.nrow(),mapSubmsSpwid[subms_index]);
3628 71 : spectralWindowId_output += sourcecols_i.spectralWindowId().getColumn();
3629 71 : sourceCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3630 :
3631 : // Columns that can be just copied
3632 71 : sourceCols_0.direction().putColumnCells(refRow,sourcecols_i.direction().getColumn());
3633 71 : sourceCols_0.properMotion().putColumnCells(refRow,sourcecols_i.properMotion().getColumn());
3634 71 : sourceCols_0.calibrationGroup().putColumnCells(refRow,sourcecols_i.calibrationGroup().getColumn());
3635 71 : sourceCols_0.code().putColumnCells(refRow,sourcecols_i.code().getColumn());
3636 71 : sourceCols_0.interval().putColumnCells(refRow,sourcecols_i.interval().getColumn());
3637 71 : sourceCols_0.name().putColumnCells(refRow,sourcecols_i.name().getColumn());
3638 71 : sourceCols_0.numLines().putColumnCells(refRow,sourcecols_i.numLines().getColumn());
3639 71 : sourceCols_0.sourceId().putColumnCells(refRow,sourcecols_i.sourceId().getColumn());
3640 71 : sourceCols_0.time().putColumnCells(refRow,sourcecols_i.time().getColumn());
3641 :
3642 : // Optional columns
3643 71 : if (columnOk(sourcecols_i.position()))
3644 : {
3645 4 : sourceCols_0.position().putColumnCells(refRow,sourcecols_i.position().getColumn());
3646 : }
3647 :
3648 71 : if (columnOk(sourcecols_i.transition()))
3649 : {
3650 6 : sourceCols_0.transition().putColumnCells(refRow,sourcecols_i.transition().getColumn());
3651 : }
3652 :
3653 71 : if (columnOk(sourcecols_i.restFrequency()))
3654 : {
3655 6 : sourceCols_0.restFrequency().putColumnCells(refRow,sourcecols_i.restFrequency().getColumn());
3656 : }
3657 :
3658 71 : if (columnOk(sourcecols_i.sysvel()))
3659 : {
3660 6 : sourceCols_0.sysvel().putColumnCells(refRow,sourcecols_i.sysvel().getColumn());
3661 : }
3662 :
3663 71 : if (columnOk(sourcecols_i.pulsarId()))
3664 : {
3665 0 : sourceCols_0.pulsarId().putColumnCells(refRow,sourcecols_i.pulsarId().getColumn());
3666 : }
3667 :
3668 71 : if (columnOk(sourcecols_i.sourceModel()))
3669 : {
3670 0 : sourceCols_0.sourceModel().putColumnCells(refRow,sourcecols_i.sourceModel().getColumn());
3671 : }
3672 :
3673 : // Increment row offset
3674 71 : rowIndex += sourcetable_i.nrow();
3675 71 : }
3676 71 : }
3677 :
3678 27 : }
3679 : else
3680 : {
3681 2 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3682 2 : << "SOURCE sub-table found but has no valid content" << LogIO::POST;
3683 1 : return false;
3684 : }
3685 28 : }
3686 : else
3687 : {
3688 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3689 0 : << "SOURCE sub-table not found " << LogIO::POST;
3690 0 : return false;
3691 : }
3692 :
3693 27 : return true;
3694 28 : }
3695 :
3696 :
3697 : // -----------------------------------------------------------------------
3698 : // Method to merge Syscal sub-tables from SubMSs to create the MMS-level Syscal sub-table
3699 : // -----------------------------------------------------------------------
3700 28 : bool MSTransformDataHandler::mergeSyscalSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3701 : {
3702 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3703 :
3704 28 : if (filenames.size() != mapSubmsSpwid.size())
3705 : {
3706 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3707 0 : return false;
3708 : }
3709 :
3710 28 : String filename_0 = filenames(0);
3711 28 : MeasurementSet ms_0(filename_0,Table::Update);
3712 :
3713 28 : if(Table::isReadable(ms_0.sysCalTableName()) and !ms_0.sysCal().isNull())
3714 : {
3715 25 : MSSysCal syscalTable_0 = ms_0.sysCal();
3716 :
3717 : // CAS-7167. The WVR spw has no FEED content.
3718 : // if (syscalTable_0.nrow() >= 0)
3719 : // {
3720 50 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3721 50 : << "Merging SYSCAL sub-tables from all sub-MSs to form MMS-level SYSCAL sub-table" << LogIO::POST;
3722 :
3723 25 : MSSysCalColumns syscalCols_0(syscalTable_0);
3724 :
3725 25 : uInt rowIndex = syscalTable_0.nrow();
3726 93 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3727 : {
3728 68 : String filename_i = filenames(subms_index);
3729 68 : MeasurementSet ms_i(filename_i);
3730 68 : MSSysCal syscaltable_i = ms_i.sysCal();
3731 :
3732 68 : if (syscaltable_i.nrow() > 0)
3733 : {
3734 3 : MSSysCalColumns syscalcols_i(syscaltable_i);
3735 :
3736 3 : syscalTable_0.addRow(syscaltable_i.nrow());
3737 :
3738 : // Prepare row reference object
3739 3 : RefRows refRow(rowIndex,rowIndex+syscaltable_i.nrow()-1);
3740 :
3741 : // Re-index SPW col
3742 3 : Vector<Int> spectralWindowId_output(syscaltable_i.nrow(),mapSubmsSpwid[subms_index]);
3743 3 : spectralWindowId_output += syscalcols_i.spectralWindowId().getColumn();
3744 3 : syscalCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3745 :
3746 : // Columns that can be just copied
3747 3 : syscalCols_0.antennaId().putColumnCells(refRow,syscalcols_i.antennaId().getColumn());
3748 3 : syscalCols_0.feedId().putColumnCells(refRow,syscalcols_i.feedId().getColumn());
3749 3 : syscalCols_0.interval().putColumnCells(refRow,syscalcols_i.interval().getColumn());
3750 3 : syscalCols_0.time().putColumnCells(refRow,syscalcols_i.time().getColumn());
3751 :
3752 : // Optional columns
3753 3 : if (columnOk(syscalcols_i.phaseDiff()))
3754 : {
3755 0 : syscalCols_0.phaseDiff().putColumnCells(refRow,syscalcols_i.phaseDiff().getColumn());
3756 : }
3757 :
3758 3 : if (columnOk(syscalcols_i.phaseDiffFlag()))
3759 : {
3760 0 : syscalCols_0.phaseDiffFlag().putColumnCells(refRow,syscalcols_i.phaseDiffFlag().getColumn());
3761 : }
3762 :
3763 3 : if (columnOk(syscalcols_i.tant()))
3764 : {
3765 0 : syscalCols_0.tant().putColumnCells(refRow,syscalcols_i.tant().getColumn());
3766 : }
3767 :
3768 3 : if (columnOk(syscalcols_i.tantFlag()))
3769 : {
3770 3 : syscalCols_0.tantFlag().putColumnCells(refRow,syscalcols_i.tantFlag().getColumn());
3771 : }
3772 :
3773 3 : if (columnOk(syscalcols_i.tantSpectrum()))
3774 : {
3775 0 : syscalCols_0.tantSpectrum().putColumnCells(refRow,syscalcols_i.tantSpectrum().getColumn());
3776 : }
3777 :
3778 3 : if (columnOk(syscalcols_i.tantTsys()))
3779 : {
3780 0 : syscalCols_0.tantTsys().putColumnCells(refRow,syscalcols_i.tantTsys().getColumn());
3781 : }
3782 :
3783 3 : if (columnOk(syscalcols_i.tantTsysFlag()))
3784 : {
3785 3 : syscalCols_0.tantTsysFlag().putColumnCells(refRow,syscalcols_i.tantTsysFlag().getColumn());
3786 : }
3787 :
3788 3 : if (columnOk(syscalcols_i.tantTsysSpectrum()))
3789 : {
3790 0 : syscalCols_0.tantTsysSpectrum().putColumnCells(refRow,syscalcols_i.tantTsysSpectrum().getColumn());
3791 : }
3792 :
3793 3 : if (columnOk(syscalcols_i.tcal()))
3794 : {
3795 0 : syscalCols_0.tcal().putColumnCells(refRow,syscalcols_i.tcal().getColumn());
3796 : }
3797 :
3798 3 : if (columnOk(syscalcols_i.tcalFlag()))
3799 : {
3800 3 : syscalCols_0.tcalFlag().putColumnCells(refRow,syscalcols_i.tcalFlag().getColumn());
3801 : }
3802 :
3803 3 : if (columnOk(syscalcols_i.tcalSpectrum()))
3804 : {
3805 3 : syscalCols_0.tcalSpectrum().putColumnCells(refRow,syscalcols_i.tcalSpectrum().getColumn());
3806 : }
3807 :
3808 3 : if (columnOk(syscalcols_i.trx()))
3809 : {
3810 0 : syscalCols_0.trx().putColumnCells(refRow,syscalcols_i.trx().getColumn());
3811 : }
3812 :
3813 3 : if (columnOk(syscalcols_i.trxFlag()))
3814 : {
3815 3 : syscalCols_0.trxFlag().putColumnCells(refRow,syscalcols_i.trxFlag().getColumn());
3816 : }
3817 :
3818 3 : if (columnOk(syscalcols_i.trxSpectrum()))
3819 : {
3820 3 : syscalCols_0.trxSpectrum().putColumnCells(refRow,syscalcols_i.trxSpectrum().getColumn());
3821 : }
3822 :
3823 3 : if (columnOk(syscalcols_i.tsky()))
3824 : {
3825 0 : syscalCols_0.tsky().putColumnCells(refRow,syscalcols_i.tsky().getColumn());
3826 : }
3827 :
3828 3 : if (columnOk(syscalcols_i.tskyFlag()))
3829 : {
3830 3 : syscalCols_0.tskyFlag().putColumnCells(refRow,syscalcols_i.tskyFlag().getColumn());
3831 : }
3832 :
3833 3 : if (columnOk(syscalcols_i.tskySpectrum()))
3834 : {
3835 3 : syscalCols_0.tskySpectrum().putColumnCells(refRow,syscalcols_i.tskySpectrum().getColumn());
3836 : }
3837 :
3838 3 : if (columnOk(syscalcols_i.tsys()))
3839 : {
3840 0 : syscalCols_0.tsys().putColumnCells(refRow,syscalcols_i.tsys().getColumn());
3841 : }
3842 :
3843 3 : if (columnOk(syscalcols_i.tsysFlag()))
3844 : {
3845 3 : syscalCols_0.tsysFlag().putColumnCells(refRow,syscalcols_i.tsysFlag().getColumn());
3846 : }
3847 :
3848 3 : if (columnOk(syscalcols_i.tsysSpectrum()))
3849 : {
3850 3 : syscalCols_0.tsysSpectrum().putColumnCells(refRow,syscalcols_i.tsysSpectrum().getColumn());
3851 : }
3852 :
3853 : // Increment row offset
3854 3 : rowIndex += syscalcols_i.nrow();
3855 3 : }
3856 68 : }
3857 :
3858 : /*
3859 : }
3860 : else
3861 : {
3862 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3863 : << "SYSCAL sub-table found but has no valid content" << LogIO::POST;
3864 : return false;
3865 : }
3866 : */
3867 25 : }
3868 : else
3869 : {
3870 6 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3871 6 : << "SYSCAL sub-table not found " << LogIO::POST;
3872 3 : return false;
3873 : }
3874 :
3875 25 : return true;
3876 28 : }
3877 :
3878 :
3879 : // -----------------------------------------------------------------------
3880 : // Method to merge FreqOffset sub-tables from SubMSs to create the MMS-level FreqOffset sub-table
3881 : // -----------------------------------------------------------------------
3882 28 : bool MSTransformDataHandler::mergeFreqOffsetTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3883 : {
3884 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3885 :
3886 28 : if (filenames.size() != mapSubmsSpwid.size())
3887 : {
3888 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3889 0 : return false;
3890 : }
3891 :
3892 28 : String filename_0 = filenames(0);
3893 28 : MeasurementSet ms_0(filename_0,Table::Update);
3894 :
3895 28 : if (Table::isReadable(ms_0.freqOffsetTableName()) and !ms_0.freqOffset().isNull())
3896 : {
3897 0 : MSFreqOffset freqoffsetTable_0 = ms_0.freqOffset();
3898 :
3899 0 : if (freqoffsetTable_0.nrow() > 0)
3900 : {
3901 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3902 0 : << "Merging FREQ_OFFSET sub-tables from all sub-MSs to form MMS-level FREQ_OFFSET sub-table" << LogIO::POST;
3903 :
3904 0 : MSFreqOffsetColumns freqoffsetCols_0(freqoffsetTable_0);
3905 :
3906 0 : uInt rowIndex = freqoffsetTable_0.nrow();
3907 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3908 : {
3909 0 : String filename_i = filenames(subms_index);
3910 0 : MeasurementSet ms_i(filename_i);
3911 0 : MSFreqOffset freqoffsettable_i = ms_i.freqOffset();
3912 :
3913 0 : if (freqoffsettable_i.nrow() > 0)
3914 : {
3915 0 : MSFreqOffsetColumns freqoffsetcols_i(freqoffsettable_i);
3916 :
3917 0 : freqoffsetTable_0.addRow(freqoffsettable_i.nrow());
3918 :
3919 : // Prepare row reference object
3920 0 : RefRows refRow(rowIndex,rowIndex+freqoffsettable_i.nrow()-1);
3921 :
3922 : // Re-index SPW col
3923 0 : Vector<Int> spectralWindowId_output(freqoffsettable_i.nrow(),mapSubmsSpwid[subms_index]);
3924 0 : spectralWindowId_output += freqoffsetcols_i.spectralWindowId().getColumn();
3925 0 : freqoffsetCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3926 :
3927 : // Columns that can be just copied
3928 0 : freqoffsetCols_0.antenna1().putColumnCells(refRow,freqoffsetcols_i.antenna1().getColumn());
3929 0 : freqoffsetCols_0.antenna2().putColumnCells(refRow,freqoffsetcols_i.antenna2().getColumn());
3930 0 : freqoffsetCols_0.feedId().putColumnCells(refRow,freqoffsetcols_i.feedId().getColumn());
3931 0 : freqoffsetCols_0.interval().putColumnCells(refRow,freqoffsetcols_i.interval().getColumn());
3932 0 : freqoffsetCols_0.offset().putColumnCells(refRow,freqoffsetcols_i.offset().getColumn());
3933 0 : freqoffsetCols_0.time().putColumnCells(refRow,freqoffsetcols_i.time().getColumn());
3934 :
3935 : // NOTE (jagonzal): FreqOffset does not have optional columns
3936 :
3937 : // Increment row offset
3938 0 : rowIndex += freqoffsettable_i.nrow();
3939 0 : }
3940 0 : }
3941 :
3942 0 : }
3943 : else
3944 : {
3945 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3946 0 : << "FREQ_OFFSET sub-table found but has no valid content" << LogIO::POST;
3947 0 : return false;
3948 : }
3949 :
3950 0 : }
3951 : else
3952 : {
3953 56 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3954 56 : << "FREQ_OFFSET sub-table not found " << LogIO::POST;
3955 28 : return false;
3956 : }
3957 :
3958 0 : return true;
3959 28 : }
3960 :
3961 : // -----------------------------------------------------------------------
3962 : // Method to merge CalDevice sub-tables from SubMSs to create the MMS-level CalDevice sub-table
3963 : // -----------------------------------------------------------------------
3964 28 : bool MSTransformDataHandler::mergeCalDeviceSubtables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3965 : {
3966 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3967 :
3968 28 : if (filenames.size() != mapSubmsSpwid.size())
3969 : {
3970 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3971 0 : return false;
3972 : }
3973 :
3974 28 : String filename_0 = filenames(0);
3975 28 : MeasurementSet ms_0(filename_0,Table::Update);
3976 :
3977 28 : if (Table::isReadable(ms_0.tableName() + "/CALDEVICE"))
3978 : {
3979 8 : Table subtable_0(ms_0.tableName() + "/CALDEVICE", Table::Update);
3980 :
3981 8 : if (subtable_0.nrow() > 0)
3982 : {
3983 16 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3984 16 : << "Merging CALDEVICE sub-tables from all sub-MSs to form MMS-level CALDEVICE sub-table" << LogIO::POST;
3985 :
3986 : // Get RW access to columns
3987 8 : ScalarColumn<Int> antennaIdCol_0(subtable_0, "ANTENNA_ID");
3988 8 : ScalarColumn<Int> feedIdCol_0(subtable_0, "FEED_ID");
3989 8 : ScalarColumn<Int> spectralWindowIdCol_0(subtable_0, "SPECTRAL_WINDOW_ID");
3990 8 : ScalarColumn<Double> timeCol_0(subtable_0, "TIME");
3991 8 : ScalarColumn<Double> intervalCol_0(subtable_0, "INTERVAL");
3992 :
3993 8 : ScalarColumn<Int> numCalLoadCol_0(subtable_0, "NUM_CAL_LOAD");
3994 8 : ArrayColumn<String> calLoadNamesCol_0(subtable_0, "CAL_LOAD_NAMES");
3995 8 : ScalarColumn<Int> numReceptorCol_0(subtable_0, "NUM_RECEPTOR");
3996 8 : ArrayColumn<Float> noiseCalCol_0(subtable_0, "NOISE_CAL");
3997 8 : ArrayColumn<Float> calEffCol_0(subtable_0, "CAL_EFF");
3998 8 : ArrayColumn<Double> temperatureLoadCol_0(subtable_0, "TEMPERATURE_LOAD");
3999 :
4000 : // Get original content of columns
4001 8 : Vector<Int> antennaId_0;
4002 8 : if (columnOk(antennaIdCol_0)) antennaId_0 = antennaIdCol_0.getColumn();
4003 8 : Vector<Int> feedId_0;
4004 8 : if (columnOk(feedIdCol_0)) feedId_0 = feedIdCol_0.getColumn();
4005 8 : Vector<Int> spectralWindowId_0;
4006 8 : if (columnOk(spectralWindowIdCol_0)) spectralWindowId_0 = spectralWindowIdCol_0.getColumn();
4007 8 : Vector<Double> time_0;
4008 8 : if (columnOk(timeCol_0)) time_0 = timeCol_0.getColumn();
4009 8 : Vector<Double> interval_0;
4010 8 : if (columnOk(intervalCol_0)) interval_0 = intervalCol_0.getColumn();
4011 :
4012 8 : Vector<Int> numCalLoad_0;
4013 8 : if (columnOk(numCalLoadCol_0)) numCalLoad_0 = numCalLoadCol_0.getColumn();
4014 8 : Array<String> calLoadNames_0;
4015 8 : if (columnOk(calLoadNamesCol_0)) calLoadNames_0 = calLoadNamesCol_0.getColumn();
4016 8 : Vector<Int> numReceptor_0;
4017 8 : if (columnOk(numReceptorCol_0)) numReceptor_0 = numReceptorCol_0.getColumn();
4018 8 : Array<Float> noiseCal_0;
4019 8 : if (columnOk(noiseCalCol_0)) noiseCal_0 = noiseCalCol_0.getColumn();
4020 8 : Array<Float> calEff_0;
4021 8 : if (columnOk(calEffCol_0)) calEff_0 = calEffCol_0.getColumn();
4022 8 : Array<Double> temperatureLoad_0;
4023 8 : if (columnOk(temperatureLoadCol_0)) temperatureLoad_0 = temperatureLoadCol_0.getColumn();
4024 :
4025 8 : uInt rowIndex = subtable_0.nrow();
4026 26 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
4027 : {
4028 18 : String filename_i = filenames(subms_index);
4029 18 : MeasurementSet ms_i(filename_i);
4030 18 : Table subtable_i(ms_i.tableName() + "/CALDEVICE", Table::Update);
4031 :
4032 18 : if (subtable_i.nrow() > 0)
4033 : {
4034 : // Get RW access to columns
4035 18 : ScalarColumn<Int> antennaIdCol_i(subtable_i, "ANTENNA_ID");
4036 18 : ScalarColumn<Int> feedIdCol_i(subtable_i, "FEED_ID");
4037 18 : ScalarColumn<Int> spectralWindowIdCol_i(subtable_i, "SPECTRAL_WINDOW_ID");
4038 18 : ScalarColumn<Double> timeCol_i(subtable_i, "TIME");
4039 18 : ScalarColumn<Double> intervalCol_i(subtable_i, "INTERVAL");
4040 :
4041 18 : ScalarColumn<Int> numCalLoadCol_i(subtable_i, "NUM_CAL_LOAD");
4042 18 : ArrayColumn<String> calLoadNamesCol_i(subtable_i, "CAL_LOAD_NAMES");
4043 18 : ScalarColumn<Int> numReceptorCol_i(subtable_i, "NUM_RECEPTOR");
4044 18 : ArrayColumn<Float> noiseCalCol_i(subtable_i, "NOISE_CAL");
4045 18 : ArrayColumn<Float> calEffCol_i(subtable_i, "CAL_EFF");
4046 18 : ArrayColumn<Double> temperatureLoadCol_i(subtable_i, "TEMPERATURE_LOAD");
4047 :
4048 : // Get original content of columns
4049 18 : Vector<Int> antennaId_i;
4050 18 : if (columnOk(antennaIdCol_i)) antennaId_i = antennaIdCol_i.getColumn();
4051 18 : Vector<Int> feedId_i;
4052 18 : if (columnOk(feedIdCol_i)) feedId_i = feedIdCol_i.getColumn();
4053 18 : Vector<Int> spectralWindowId_i;
4054 18 : if (columnOk(spectralWindowIdCol_i)) spectralWindowId_i = spectralWindowIdCol_i.getColumn();
4055 18 : Vector<Double> time_i;
4056 18 : if (columnOk(timeCol_i)) time_i = timeCol_i.getColumn();
4057 18 : Vector<Double> interval_i;
4058 18 : if (columnOk(intervalCol_i)) interval_i = intervalCol_i.getColumn();
4059 :
4060 18 : Vector<Int> numCalLoad_i;
4061 18 : if (columnOk(numCalLoadCol_i)) numCalLoad_i = numCalLoadCol_i.getColumn();
4062 18 : Array<String> calLoadNames_i;
4063 18 : if (columnOk(calLoadNamesCol_i)) calLoadNames_i = calLoadNamesCol_i.getColumn();
4064 18 : Vector<Int> numReceptor_i;
4065 18 : if (columnOk(numReceptorCol_i)) numReceptor_i = numReceptorCol_i.getColumn();
4066 18 : Array<Float> noiseCal_i;
4067 18 : if (columnOk(noiseCalCol_i)) noiseCal_i = noiseCalCol_i.getColumn();
4068 18 : Array<Float> calEff_i;
4069 18 : if (columnOk(calEffCol_i)) calEff_i = calEffCol_i.getColumn();
4070 18 : Array<Double> temperatureLoad_i;
4071 18 : if (columnOk(temperatureLoadCol_i)) temperatureLoad_i = temperatureLoadCol_i.getColumn();
4072 :
4073 : // Add n# rows to subtable_i equivalent to n# rows from subtable_0
4074 18 : subtable_0.addRow(subtable_i.nrow());
4075 :
4076 : // Prepare row reference object
4077 18 : RefRows refRow(rowIndex,rowIndex+subtable_i.nrow()-1);
4078 :
4079 : // Re-index SPW col
4080 18 : Vector<Int> spectralWindowId_output(spectralWindowId_i.size(),mapSubmsSpwid[subms_index]);
4081 18 : spectralWindowId_output += spectralWindowId_i;
4082 18 : spectralWindowIdCol_0.putColumnCells(refRow,spectralWindowId_output);
4083 :
4084 : // Columns that can be just copied
4085 18 : if (columnOk(antennaIdCol_i))
4086 : {
4087 18 : antennaIdCol_0.putColumnCells(refRow,antennaId_i);
4088 : }
4089 :
4090 18 : if (columnOk(feedIdCol_i))
4091 : {
4092 18 : feedIdCol_0.putColumnCells(refRow,feedId_i);
4093 : }
4094 :
4095 18 : if (columnOk(timeCol_i))
4096 : {
4097 18 : timeCol_0.putColumnCells(refRow,time_i);
4098 : }
4099 :
4100 18 : if (columnOk(intervalCol_i))
4101 : {
4102 18 : intervalCol_0.putColumnCells(refRow,interval_i);
4103 : }
4104 :
4105 :
4106 18 : if (columnOk(numCalLoadCol_i))
4107 : {
4108 18 : numCalLoadCol_0.putColumnCells(refRow,numCalLoad_i);
4109 : }
4110 :
4111 18 : if (columnOk(calLoadNamesCol_i))
4112 : {
4113 18 : calLoadNamesCol_0.putColumnCells(refRow,calLoadNames_i);
4114 : }
4115 :
4116 18 : if (columnOk(numReceptorCol_i))
4117 : {
4118 18 : numReceptorCol_0.putColumnCells(refRow,numReceptor_i);
4119 : }
4120 :
4121 18 : if (columnOk(noiseCalCol_i))
4122 : {
4123 8 : noiseCalCol_0.putColumnCells(refRow,noiseCal_i);
4124 : }
4125 :
4126 18 : if (columnOk(calEffCol_i))
4127 : {
4128 0 : calEffCol_0.putColumnCells(refRow,calEff_i);
4129 : }
4130 :
4131 18 : if (columnOk(temperatureLoadCol_i))
4132 : {
4133 10 : temperatureLoadCol_0.putColumnCells(refRow,temperatureLoad_i);
4134 : }
4135 :
4136 : // Increment row offset
4137 18 : rowIndex += subtable_i.nrow();
4138 18 : }
4139 18 : }
4140 :
4141 8 : }
4142 : else
4143 : {
4144 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4145 0 : << "CALDEVICE sub-table found but has no valid content" << LogIO::POST;
4146 0 : return false;
4147 : }
4148 8 : }
4149 : else
4150 : {
4151 40 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4152 40 : << "CALDEVICE sub-table not found " << LogIO::POST;
4153 20 : return false;
4154 : }
4155 :
4156 8 : return true;
4157 28 : }
4158 :
4159 :
4160 : // -----------------------------------------------------------------------
4161 : // Method to merge SysPower sub-tables from SubMSs to create the MMS-level SysPower sub-table
4162 : // -----------------------------------------------------------------------
4163 28 : bool MSTransformDataHandler::mergeSysPowerSubtables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
4164 : {
4165 56 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
4166 :
4167 28 : if (filenames.size() != mapSubmsSpwid.size())
4168 : {
4169 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
4170 0 : return false;
4171 : }
4172 :
4173 28 : String filename_0 = filenames(0);
4174 28 : MeasurementSet ms_0(filename_0,Table::Update);
4175 :
4176 28 : if (Table::isReadable(ms_0.tableName() + "/SYSPOWER"))
4177 : {
4178 8 : Table subtable_0(ms_0.tableName() + "/SYSPOWER", Table::Update);
4179 :
4180 8 : if (subtable_0.nrow() > 0)
4181 : {
4182 12 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4183 12 : << "Merging SYSPOWER sub-tables from all sub-MSs to form MMS-level SYSPOWER sub-table" << LogIO::POST;
4184 :
4185 : // Get RW access to columns
4186 6 : ScalarColumn<Int> antennaIdCol_0(subtable_0, "ANTENNA_ID");
4187 6 : ScalarColumn<Int> feedIdCol_0(subtable_0, "FEED_ID");
4188 6 : ScalarColumn<Int> spectralWindowIdCol_0(subtable_0, "SPECTRAL_WINDOW_ID");
4189 6 : ScalarColumn<Double> timeCol_0(subtable_0, "TIME");
4190 6 : ScalarColumn<Double> intervalCol_0(subtable_0, "INTERVAL");
4191 :
4192 6 : ArrayColumn<Float> switchedDiffCol_0(subtable_0, "SWITCHED_DIFF");
4193 6 : ArrayColumn<Float> switchedSumCol_0(subtable_0, "SWITCHED_SUM");
4194 6 : ArrayColumn<Float> requantizerGainCol_0(subtable_0, "REQUANTIZER_GAIN");
4195 :
4196 : // Get original content of columns
4197 6 : Vector<Int> antennaId_0;
4198 6 : if (columnOk(antennaIdCol_0)) antennaId_0 = antennaIdCol_0.getColumn();
4199 6 : Vector<Int> feedId_0;
4200 6 : if (columnOk(feedIdCol_0)) feedId_0 = feedIdCol_0.getColumn();
4201 6 : Vector<Int> spectralWindowId_0;
4202 6 : if (columnOk(spectralWindowIdCol_0)) spectralWindowId_0 = spectralWindowIdCol_0.getColumn();
4203 6 : Vector<Double> time_0;
4204 6 : if (columnOk(timeCol_0)) time_0 = timeCol_0.getColumn();
4205 6 : Vector<Double> interval_0;
4206 6 : if (columnOk(intervalCol_0)) interval_0 = intervalCol_0.getColumn();
4207 :
4208 6 : Array<Float> switchedDiff_0;
4209 6 : if (columnOk(switchedDiffCol_0)) switchedDiff_0 = switchedDiffCol_0.getColumn();
4210 6 : Array<Float> switchedSum_0;
4211 6 : if (columnOk(switchedSumCol_0)) switchedSum_0 = switchedSumCol_0.getColumn();
4212 6 : Array<Float> requantizerGain_0;
4213 6 : if (columnOk(requantizerGainCol_0)) requantizerGain_0 = requantizerGainCol_0.getColumn();
4214 :
4215 6 : uInt rowIndex = subtable_0.nrow();
4216 14 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
4217 : {
4218 8 : String filename_i = filenames(subms_index);
4219 8 : MeasurementSet ms_i(filename_i);
4220 8 : Table subtable_i(ms_i.tableName() + "/SYSPOWER", Table::Update);
4221 :
4222 8 : if (subtable_i.nrow() > 0)
4223 : {
4224 : // Get RW access to columns
4225 8 : ScalarColumn<Int> antennaIdCol_i(subtable_i, "ANTENNA_ID");
4226 8 : ScalarColumn<Int> feedIdCol_i(subtable_i, "FEED_ID");
4227 8 : ScalarColumn<Int> spectralWindowIdCol_i(subtable_i, "SPECTRAL_WINDOW_ID");
4228 8 : ScalarColumn<Double> timeCol_i(subtable_i, "TIME");
4229 8 : ScalarColumn<Double> intervalCol_i(subtable_i, "INTERVAL");
4230 :
4231 8 : ArrayColumn<Float> switchedDiffCol_i(subtable_i, "SWITCHED_DIFF");
4232 8 : ArrayColumn<Float> switchedSumCol_i(subtable_i, "SWITCHED_SUM");
4233 8 : ArrayColumn<Float> requantizerGainCol_i(subtable_i, "REQUANTIZER_GAIN");
4234 :
4235 : // Get original content of columns
4236 8 : Vector<Int> antennaId_i;
4237 8 : if (columnOk(antennaIdCol_i)) antennaId_i = antennaIdCol_i.getColumn();
4238 8 : Vector<Int> feedId_i;
4239 8 : if (columnOk(feedIdCol_i)) feedId_i = feedIdCol_i.getColumn();
4240 8 : Vector<Int> spectralWindowId_i;
4241 8 : if (columnOk(spectralWindowIdCol_i)) spectralWindowId_i = spectralWindowIdCol_i.getColumn();
4242 8 : Vector<Double> time_i;
4243 8 : if (columnOk(timeCol_i)) time_i = timeCol_i.getColumn();
4244 8 : Vector<Double> interval_i;
4245 8 : if (columnOk(intervalCol_i)) interval_i = intervalCol_i.getColumn();
4246 :
4247 8 : Array<Float> switchedDiff_i;
4248 8 : if (columnOk(switchedDiffCol_i)) switchedDiff_i = switchedDiffCol_i.getColumn();
4249 8 : Array<Float> switchedSum_i;
4250 8 : if (columnOk(switchedSumCol_i)) switchedSum_i = switchedSumCol_i.getColumn();
4251 8 : Array<Float> requantizerGain_i;
4252 8 : if (columnOk(requantizerGainCol_i)) requantizerGain_i = requantizerGainCol_i.getColumn();
4253 :
4254 : // Add n# rows to subtable_i equivalent to n# rows from subtable_0
4255 8 : subtable_0.addRow(subtable_i.nrow());
4256 :
4257 : // Prepare row reference object
4258 8 : RefRows refRow(rowIndex,rowIndex+subtable_i.nrow()-1);
4259 :
4260 : // Re-index SPW col
4261 8 : Vector<Int> spectralWindowId_output(spectralWindowId_i.size(),mapSubmsSpwid[subms_index]);
4262 8 : spectralWindowId_output += spectralWindowId_i;
4263 8 : spectralWindowIdCol_0.putColumnCells(refRow,spectralWindowId_output);
4264 :
4265 : // Columns that can be just copied
4266 8 : if (columnOk(antennaIdCol_i))
4267 : {
4268 8 : antennaIdCol_0.putColumnCells(refRow,antennaId_i);
4269 : }
4270 :
4271 8 : if (columnOk(feedIdCol_i))
4272 : {
4273 8 : feedIdCol_0.putColumnCells(refRow,feedId_i);
4274 : }
4275 :
4276 8 : if (columnOk(timeCol_i))
4277 : {
4278 8 : timeCol_0.putColumnCells(refRow,time_i);
4279 : }
4280 :
4281 8 : if (columnOk(intervalCol_i))
4282 : {
4283 8 : intervalCol_0.putColumnCells(refRow,interval_i);
4284 : }
4285 :
4286 :
4287 8 : if (columnOk(switchedDiffCol_i))
4288 : {
4289 8 : switchedDiffCol_0.putColumnCells(refRow,switchedDiff_i);
4290 : }
4291 :
4292 8 : if (columnOk(switchedSumCol_i))
4293 : {
4294 8 : switchedSumCol_0.putColumnCells(refRow,switchedSum_i);
4295 : }
4296 :
4297 8 : if (columnOk(requantizerGainCol_i))
4298 : {
4299 8 : requantizerGainCol_0.putColumnCells(refRow,requantizerGain_i);
4300 : }
4301 :
4302 : // Increment row offset
4303 8 : rowIndex += subtable_i.nrow();
4304 8 : }
4305 8 : }
4306 :
4307 6 : }
4308 : else
4309 : {
4310 4 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4311 4 : << "SYSPOWER sub-table found but has no valid content" << LogIO::POST;
4312 2 : return false;
4313 : }
4314 8 : }
4315 : else
4316 : {
4317 40 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4318 40 : << "SYSPOWER sub-table not found " << LogIO::POST;
4319 20 : return false;
4320 : }
4321 :
4322 6 : return true;
4323 28 : }
4324 :
4325 : // -----------------------------------------------------------------------
4326 : //
4327 : // -----------------------------------------------------------------------
4328 : // template <class T> bool MSTransformDataHandler::columnOk (ArrayColumn<T> column)
4329 : // {
4330 : // bool ret;
4331 : // if (column.isNull()==false and column.hasContent()==true and column.ndimColumn() > 0)
4332 : // {
4333 : // ret = true;
4334 : // }
4335 : // else
4336 : // {
4337 : // ret = false;
4338 : // }
4339 :
4340 : // return ret;
4341 : // }
4342 :
4343 : // // -----------------------------------------------------------------------
4344 : // //
4345 : // // -----------------------------------------------------------------------
4346 : // template <class T> bool MSTransformDataHandler::columnOk (ScalarColumn<T> column)
4347 : // {
4348 : // bool ret;
4349 : // if (column.isNull()==false and column.hasContent()==true)
4350 : // {
4351 : // ret = true;
4352 : // }
4353 : // else
4354 : // {
4355 : // ret = false;
4356 : // }
4357 :
4358 : // return ret;
4359 : // }
4360 :
4361 :
4362 : // -----------------------------------------------------------------------
4363 : // Work-around to copy the keywords of the FLOAT_DATA column to the output MS
4364 : // -----------------------------------------------------------------------
4365 2213 : void MSTransformDataHandler::copyMainTableKeywords (TableRecord& outKeys,
4366 : const TableRecord& inKeys)
4367 : {
4368 2599 : for (uInt i=0; i<inKeys.nfields(); i++) {
4369 386 : if (inKeys.type(i) == TpString) {
4370 : // Add keywords for MAIN table columns such as FLOAT_DATA
4371 386 : String ikey = inKeys.name(i);
4372 386 : if (!outKeys.isDefined (ikey)) {
4373 384 : String keyval;
4374 384 : inKeys.get(ikey, keyval);
4375 384 : outKeys.define(ikey,keyval);
4376 384 : }
4377 :
4378 386 : }
4379 :
4380 : }
4381 2213 : }
4382 :
4383 :
4384 : } //# NAMESPACE CASA - END
|
