Line data Source code
1 : //# FlagAgentBase.h: This file contains the interface definition of the FlagAgentBase class.
2 : //#
3 : //# CASA - Common Astronomy Software Applications (http://casa.nrao.edu/)
4 : //# Copyright (C) Associated Universities, Inc. Washington DC, USA 2011, All rights reserved.
5 : //# Copyright (C) European Southern Observatory, 2011, All rights reserved.
6 : //#
7 : //# This library is free software; you can redistribute it and/or
8 : //# modify it under the terms of the GNU Lesser General Public
9 : //# License as published by the Free software Foundation; either
10 : //# version 2.1 of the License, or (at your option) any later version.
11 : //#
12 : //# This library is distributed in the hope that it will be useful,
13 : //# but WITHOUT ANY WARRANTY, without even the implied warranty of
14 : //# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : //# Lesser General Public License for more details.
16 : //#
17 : //# You should have received a copy of the GNU Lesser General Public
18 : //# License along with this library; if not, write to the Free Software
19 : //# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20 : //# MA 02111-1307 USA
21 : //# $Id: $
22 :
23 : #ifndef FlagAgentBase_H_
24 : #define FlagAgentBase_H_
25 :
26 : #include <map>
27 : // needed for antennaPairMap, polarizationMap, VisMapper, FlagMapper, etc. which should
28 : // probably be split from the rather large FlagDataHandler.h
29 : #include <flagging/Flagging/FlagDataHandler.h>
30 :
31 : #include <stdcasa/thread/AsynchronousTools.h>
32 : #include <flagging/Flagging/FlagReport.h>
33 :
34 : namespace casa { //# NAMESPACE CASA - BEGIN
35 :
36 : class FlagDataHandler;
37 :
38 : // <summary>
39 : // A top level class defining the interface for flagging agents
40 : // </summary>
41 : //
42 : // <use visibility=export>
43 : //
44 : // <prerequisite>
45 : // <li> <linkto class="VisBuffer2:description">FlagDataHandler</linkto>
46 : // <li> <linkto class="VisBuffer2:description">FlagReport</linkto>
47 : // </prerequisite>
48 : //
49 : // <etymology>
50 : // FlagAgentBase stands for a generic class, specific to the flagging operations
51 : // </etymology>
52 : //
53 : // <synopsis>
54 : //
55 : // This is a top-level class defining the interface for flagging agents.
56 : // There are various methods (virtual) that must be re-implemented by the specific derived
57 : // classes, depending on the implemented algorithm. Here we find three categories:
58 : //
59 : // -# Iteration approach methods:
60 : //
61 : // - computeRowFlags(const VisBuffer &visBuffer, FlagMapper &flags, casacore::uInt row)
62 : // - For agents that only depend on meta-data for their flagging operations (for FlagAgentManual,FlagAgentElevation,FlagAgentShadow,FlagAgentQuack)
63 : // - This iteration method can also be used by agents that have to inspect the already existing flags (for FlagAgentSummary, FlagAgentExtension)
64 : //
65 : // - computeInRowFlags(const VisBuffer &visBuffer, VisMapper &visibilities,FlagMapper &flags, casacore::uInt row);
66 : // - For agents that have to look into the visibility points, but regardless of their source baseline, like FlagAgentDisplay
67 : //
68 : // - computeAntennaPairFlags(const VisBuffer &visBuffer,FlagMapper &flags,casacore::Int antenna1,casacore::Int antenna2,vector<casacore::uInt> &rows);
69 : // - For agents that have to look into the visibility points grouped by baseline (FlagAgentTimeFreqCrop,FlagAgentRFlag)
70 : //
71 : // - computeAntennaPairFlags(const VisBuffer &visBuffer, VisMapper &visibilities,FlagMapper &flags,casacore::Int antenna1,casacore::Int antenna2,vector<casacore::uInt> &rows)
72 : // - For agents that have to look into the visibility points grouped by baseline, allowing user-driven navigation (FlagAgentDisplay)
73 : // - NOTE: This method has to be used in combination with iterateAntennaPairsInteractive(antennaPairMap *antennaPairMap_ptr)
74 : //
75 : // -# Configuration methods:
76 : //
77 : // - FlagAgentBase::FlagAgentBase
78 : // - Even though each derived agent has its specific constructor, it is necessary to call the base class constructor to set:
79 : // - The FlagDataHandler implementation pointer
80 : // - The iteration approach: For this the FlagAgentBase class contains an enumeration FlagAgentBase::iteration with the following modes:
81 : // - FlagAgentBase::ROWS: Iterate row by row and flag depending on the corresponding meta-data (for FlagAgentManual,FlagAgentQuack)
82 : // - FlagAgentBase::ROWS_PREPROCESS_BUFFER: Iterate row by row doing a pre-processing common to all the rows of each chunk (for FlagAgentElevation, FlagAgentShadow, FlagAgentSummary)
83 : // - FlagAgentBase::IN_ROWS: Iterate row by row and flag depending on the data column (for FlagAgentClipping)
84 : // - FlagAgentBase::ANTENNA_PAIRS: Iterate per baselines and flag depending on the data column (for FlagAgentTimeFreqCrop, FlagAgentRFlag)
85 : // - FlagAgentBase::ANTENNA_PAIRS_FLAGS: Iterate per baselines accessing the individual flag points (for FlagAgentExtension)
86 : // - FlagAgentBase::ANTENNA_PAIRS_INTERACTIVE: Iterate per baselines interactively accessing the data column (for FlagAgentDisplay)
87 : //
88 : // - setAgentParameters(casacore::Record config)
89 : // - To parse the agent-specific parameters, although there is also an implementation of
90 : // this method in the base class which has to be called to handle the following parameters:
91 : // - datacolumn: To specify the column in which the agent has to operate (see FlagAgentBase::datacolumn enumeration)
92 : // - correlation: To specify the correlation to be inspected for flagging (this also includes visibility expressions)
93 : // - meta-data selection parameters (field, spw, scan, baseline, etc): To feed the agent-level data selection engine (row filtering)
94 : //
95 : // -# Information methods
96 : //
97 : // - FlagReport getReport()
98 : // - To return a specific report to be shown by the display agent, or containing accumulated information (e.g.: summaries, rflag tresholds)
99 : //
100 : // Additionally there are public non-virtual methods to:
101 : //
102 : // - Handle processing in 'background' mode (for parallel flagging)
103 : // - start(): Start service (start thread run method)
104 : // - terminate(): Terminate service (forcing run method to finish)
105 : // - queueProcess(): To signal flagging of the current VisBuffer
106 : // - completeProcess(): Wait until completion of flagging process for the current VisBuffer
107 : //
108 : // - Print out percentage of flags produced by chunk or in total
109 : // - chunkSummary(): Accumulates statistics for each chunk
110 : // - tableSummary(): Accumulates statistics across the entire table selection
111 : //
112 : // </synopsis>
113 : //
114 : // <motivation>
115 : // The motivation for the FlagAgentBase class is having all the iteration and filtering capabilities
116 : // grouped in one single class, with a common interface for all the agents w/o introducing anything
117 : // specific to the implementation of each algorithm, thus improving modularization and maintainability.
118 : // </motivation>
119 : //
120 : //
121 : // <example>
122 : // The top level interface of a flagging agent is quite simple once it is configured, this is due to
123 : // the fact that most of the complexity lies in the FlagDataHandler-FlagAgentBase interaction,
124 : // which is hidden from the application layer (already explained in the FlagDataHandler documentation).
125 : //
126 : // <srcblock>
127 : //
128 : // // Create FlagDataHandler
129 : // FlagDataHandler *dh = new FlagMSHandler(inputFile,iterationMode);
130 : //
131 : // // First of all define a configuration record (e.g.: quack)
132 : // casacore::Record agentConfig;
133 : // agentConfig.define("mode","quack");
134 : // agentConfig.define("quackinterval",(casacore::Double)20);
135 : //
136 : // // Use the factory method to create the agent, and put it into a FlagAgentList
137 : // FlagAgentList agentList;
138 : // FlagAgentBase *agent = FlagAgentBase::create(dh,agentConfig);
139 : // agentList.push_back(agent);
140 : //
141 : // // Iterate over chunks
142 : // while (dh->nextChunk())
143 : // {
144 : // // Iterates over buffers
145 : // while (dh->nextBuffer())
146 : // {
147 : // // Apply agents on current VisBuffer
148 : // agentList.apply();
149 : //
150 : // // Flush flags (only effective if there is a write access to the flag cube)
151 : // dh->flushFlags();
152 : // }
153 : //
154 : // // Print chunk stats from each agent
155 : // agentList.chunkSummary();
156 : // }
157 : //
158 : // // Print total stats from each agent
159 : // agentList.tableSummary();
160 : //
161 : // // Stop flag agent
162 : // agentList.terminate();
163 : //
164 : // </srcblock>
165 : // </example>
166 :
167 :
168 : class FlagAgentBase : public casa::async::Thread {
169 :
170 : public:
171 :
172 : enum datacolumn {
173 :
174 : DATA=0,
175 : CORRECTED,
176 : MODEL,
177 : RESIDUAL,
178 : RESIDUAL_DATA,
179 : FPARAM,
180 : CPARAM,
181 : PARAMERR,
182 : SNR,
183 : WEIGHT_SPECTRUM,
184 : FLOAT_DATA
185 : };
186 :
187 : enum iteration {
188 :
189 : ROWS=0,
190 : ROWS_PREPROCESS_BUFFER,
191 : IN_ROWS,
192 : IN_ROWS_PREPROCESS_BUFFER,
193 : ANTENNA_PAIRS,
194 : ANTENNA_PAIRS_FLAGS,
195 : ANTENNA_PAIRS_INTERACTIVE,
196 : ANTENNA_PAIRS_PREPROCESS_BUFFER
197 : };
198 :
199 : FlagAgentBase(FlagDataHandler *dh, casacore::Record config, casacore::uShort iterationApproach, casacore::Bool writePrivateFlagCube = false, casacore::Bool flag = true);
200 : virtual ~FlagAgentBase ();
201 : static FlagAgentBase *create (FlagDataHandler *dh,casacore::Record config);
202 :
203 : void start();
204 : void terminate ();
205 : void queueProcess();
206 : void chunkSummary();
207 : void tableSummary();
208 : void completeProcess();
209 : void * run ();
210 :
211 : // Set function to activate profiling
212 0 : void setProfiling(bool enable) {profiling_p = enable;}
213 :
214 : // Set function to activate check mode
215 0 : void setCheckMode(bool enable) {checkFlags_p = enable;}
216 :
217 : // Externally visible configuration
218 : casacore::Bool backgroundMode_p;
219 : casacore::LogIO::Command logLevel_p;
220 : casacore::Bool apply_p;
221 : casacore::Bool flag_p;
222 :
223 : // Get a report casacore::Record from the agent, at the end of the run
224 : // The report returned by getReport() can be of multiple types
225 : // -- a single report of type "none" : FlagReport("none",agentName_p)
226 : // -- a single report of type "plot" : FlagReport("plot",agentName_p)
227 : // -- a list of reports :
228 : // FlagReport repList("list");
229 : // repList.addReport( FlagReport("plot",agentName_p) );
230 : // repList.addReport( FlagReport("plot",agentName_p) );
231 : virtual FlagReport getReport();
232 :
233 : protected:
234 :
235 : void initialize();
236 :
237 : // Convenience function to be shared by parallel/non-parallel mode
238 : void runCore();
239 :
240 : void setDataSelection(casacore::Record config);
241 : // TODO: This class must be re-implemented in the derived classes
242 : virtual void setAgentParameters(casacore::Record config);
243 : // Method to sanitize correlation expression and keep going
244 : casacore::String sanitizeCorrExpression(casacore::String corrExpression, std::vector<casacore::String> *corrProducts);
245 :
246 : void generateAllIndex();
247 : void generateRowsIndex(casacore::uInt nRows);
248 : void generateChannelIndex(casacore::uInt nChannels);
249 : void generatePolarizationIndex(casacore::uInt nPolarizations);
250 : std::vector<casacore::uInt> * generateAntennaPairRowsIndex(casacore::Int antenna1, casacore::Int antenna2);
251 :
252 : // Generate index for all rows
253 : void indigen(std::vector<casacore::uInt> &index, casacore::uInt size);
254 :
255 : // For checking ids
256 : bool find(const casacore::Vector<casacore::Int> &validRange, casacore::Int element);
257 :
258 : // For checking ranges
259 : bool find(const casacore::Matrix<casacore::Double> &validRange, casacore::Double element);
260 :
261 : // For checking pairs
262 : bool find(const casacore::Matrix<casacore::Int> &validPairs, casacore::Int element1, casacore::Int element2);
263 :
264 : // For checking columns
265 : bool find(const casacore::Block<int> &columns, int col);
266 :
267 : // Check if a given number is nan (for visibilities,gains and Tsys primarily)
268 : bool isNaN(casacore::Double number);
269 : bool isNaN(casacore::Float number);
270 : bool isZero(casacore::Double number);
271 : bool isZero(casacore::Float number);
272 : bool isNaNOrZero(casacore::Float number);
273 : bool isNaNOrZero(casacore::Double number);
274 :
275 : // Check if buffer has to be processed
276 : bool checkIfProcessBuffer();
277 :
278 : // Common functionality for each visBuffer (don't repeat at the row level)
279 : virtual void preProcessBuffer(const vi::VisBuffer2 &visBuffer);
280 :
281 : // Final consolidation steps after iterating through the buffer rows
282 : virtual void postProcessBuffer();
283 :
284 : // Iterate trough list of rows
285 : void iterateRows();
286 :
287 : // Iterate trough visibilities mapper
288 : void iterateInRows();
289 :
290 : // Iterate trough list of antenna pairs
291 : void iterateAntennaPairs();
292 :
293 : // Iterate trough list of antenna pairs w/o loading visibilities
294 : void iterateAntennaPairsFlags();
295 :
296 : // Methods to interactively iterate trough list of antenna pairs
297 : void processAntennaPair(casacore::Int antenna1,casacore::Int antenna2);
298 : virtual void iterateAntennaPairsInteractive(antennaPairMap *antennaPairMap_ptr);
299 :
300 : // Iter-passes method
301 : virtual void passIntermediate(const vi::VisBuffer2 &visBuffer);
302 : virtual void passFinal(const vi::VisBuffer2 &visBuffer);
303 :
304 : // Mapping functions (abs, real, imag, etc.) as requested by Urvashi
305 : void setVisibilitiesMap(std::vector<casacore::uInt> *rows,VisMapper *visMap);
306 : void setFlagsMap(std::vector<casacore::uInt> *rows, FlagMapper *flagMap);
307 : casacore::Bool checkVisExpression(polarizationMap *polMap);
308 :
309 : // Compute flags for a given visibilities point
310 : virtual bool computeRowFlags(const vi::VisBuffer2 &visBuffer, FlagMapper &flags, casacore::uInt row);
311 :
312 : // Compute flags for a given visibilities point
313 : virtual bool computeInRowFlags(const vi::VisBuffer2 &visBuffer, VisMapper &visibilities,FlagMapper &flags, casacore::uInt row);
314 :
315 : // Compute flags for a given (time,freq) antenna pair map
316 : virtual bool computeAntennaPairFlags(const vi::VisBuffer2 &visBuffer, VisMapper &visibilities,FlagMapper &flags,casacore::Int antenna1,casacore::Int antenna2,std::vector<casacore::uInt> &rows);
317 :
318 : // Compute flags for a given (time,freq) antenna pair map w/o using visibilities
319 : virtual bool computeAntennaPairFlags(const vi::VisBuffer2 &visBuffer,FlagMapper &flags,casacore::Int antenna1,casacore::Int antenna2,std::vector<casacore::uInt> &rows);
320 :
321 : // Common used members that must be accessible to derived classes
322 : FlagDataHandler *flagDataHandler_p;
323 : std::unique_ptr<casacore::LogIO> logger_p;
324 : casacore::String agentName_p;
325 : casacore::String summaryName_p;
326 : casacore::String mode_p;
327 :
328 : // Flag counters
329 : casacore::uInt64 chunkFlags_p;
330 : casacore::uInt64 chunkNaNs_p;
331 : casacore::uInt64 tableFlags_p;
332 : casacore::uInt64 tableNaNs_p;
333 : casacore::uInt64 visBufferFlags_p;
334 : bool flagRow_p;
335 :
336 : // Multithreading configuration and agent id
337 : casacore::Bool multiThreading_p;
338 : casacore::Int nThreads_p;
339 : casacore::Int threadId_p;
340 :
341 : // Running configuration
342 : casacore::Bool prepass_p;
343 :
344 : private:
345 :
346 : vi::VisBuffer2 *visibilityBuffer_p;
347 :
348 : // casacore::MS-related objects
349 : casacore::Cube<casacore::Bool> *commonFlagCube_p;
350 : casacore::Cube<casacore::Bool> *originalFlagCube_p;
351 : casacore::Cube<casacore::Bool> *privateFlagCube_p;
352 :
353 : casacore::Vector<casacore::Bool> *commonFlagRow_p;
354 : casacore::Vector<casacore::Bool> *originalFlagRow_p;
355 : casacore::Vector<casacore::Bool> *privateFlagRow_p;
356 :
357 : // Own data selection ranges
358 : casacore::String arraySelection_p;
359 : casacore::String fieldSelection_p;
360 : casacore::String scanSelection_p;
361 : casacore::String timeSelection_p;
362 : casacore::String spwSelection_p;
363 : casacore::String channelSelection_p;
364 : casacore::String baselineSelection_p;
365 : casacore::String uvwSelection_p;
366 : casacore::String polarizationSelection_p;
367 : casacore::String observationSelection_p;
368 : casacore::String scanIntentSelection_p;
369 : bool filterRows_p;
370 : bool filterPols_p;
371 : bool filterChannels_p;
372 : bool flagAutoCorrelations_p;
373 : casacore::Bool antennaNegation_p;
374 :
375 : // Own data selection indexes
376 : casacore::Vector<casacore::Int> arrayList_p;
377 : casacore::Vector<casacore::Int> fieldList_p;
378 : casacore::Vector<casacore::Int> scanList_p;
379 : casacore::Matrix<casacore::Double> timeList_p;
380 : casacore::Vector<casacore::Int> spwList_p;
381 : casacore::Matrix<casacore::Int> channelList_p;
382 : casacore::Vector<casacore::Int> antenna1List_p;
383 : casacore::Vector<casacore::Int> antenna2List_p;
384 : casacore::Matrix<casacore::Int> baselineList_p;
385 : casacore::Matrix<casacore::Double> uvwList_p;
386 : casacore::Bool uvwUnits_p;
387 : std::map<casacore::Int, casacore::Vector<casacore::Int> > polarizationList_p;
388 : casacore::Vector<casacore::Int> observationList_p;
389 : casacore::Vector<casacore::Int> scanIntentList_p;
390 :
391 : // Thread state parameters
392 : volatile casacore::Bool terminationRequested_p;
393 : volatile casacore::Bool threadTerminated_p;
394 : volatile casacore::Bool processing_p;
395 :
396 : // casacore::Data source configuration
397 : // selection expression to pass to the VisMapper
398 : casacore::String expression_p;
399 : casacore::uShort dataReference_p;
400 :
401 : // Debugging configuration
402 : casacore::Bool profiling_p;
403 : casacore::Bool checkFlags_p;
404 :
405 : // Running mode configuration
406 : casacore::uShort iterationApproach_p;
407 :
408 : // Flagging mode configuration
409 : casacore::Bool writePrivateFlagCube_p;
410 :
411 : protected:
412 : // Lists of elements to process
413 : // jagonzal (CAS-4312): We need channelIndex_p available for the Rflag agent,
414 : // in order to take into account channel selection for the frequency mapping
415 : std::vector<casacore::uInt> rowsIndex_p;
416 : std::vector<casacore::uInt> channelIndex_p;
417 : std::vector<casacore::uInt> polarizationIndex_p;
418 :
419 : // Needed to be protected for timeavg in clip
420 : casacore::String dataColumn_p;
421 :
422 : // Pre-averaging parameters
423 : casacore::Bool timeavg_p;
424 : casacore::Double timebin_p;
425 : casacore::Bool channelavg_p;
426 : casacore::Vector<casacore::Int> chanbin_p;
427 :
428 :
429 : };
430 :
431 : class FlagAgentList
432 : {
433 : public:
434 : FlagAgentList();
435 : ~FlagAgentList();
436 :
437 : // Methods to mimic vector
438 : void push_back(FlagAgentBase *agent_i);
439 : void pop_back();
440 : void clear();
441 : bool empty();
442 : size_t size();
443 :
444 : // Methods to mimic FlagAgentBase
445 : void start();
446 : void terminate ();
447 : void join ();
448 : void apply(bool sequential = false);
449 : void chunkSummary();
450 : void tableSummary();
451 : void setProfiling(bool enable);
452 : void setCheckMode(bool enable);
453 :
454 : // Method to accumulate reports from all agents
455 : FlagReport gatherReports();
456 :
457 : protected:
458 :
459 : private:
460 : std::vector<FlagAgentBase *> container_p;
461 : std::vector<FlagAgentBase *>::iterator iterator_p;
462 : };
463 :
464 : } //# NAMESPACE CASA - END
465 :
466 : #endif /* FlagAgentBase_H_ */
467 :
|
