LCOV - code coverage report
Current view: top level - synthesis/ImagerObjects - CubePartitionMixin.h (source / functions) Hit Total Coverage
Test: casacpp_coverage.info Lines: 0 131 0.0 %
Date: 2024-12-11 20:54:31 Functions: 0 7 0.0 %

          Line data    Source code
       1             : /* -*- mode: c++ -*- */
       2             : //# CubePartitionMixin.h: Parallel cube imaging data partitioning
       3             : //# Copyright (C) 2016
       4             : //# Associated Universities, Inc. Washington DC, USA.
       5             : //#
       6             : //# This library is free software; you can redistribute it and/or modify it
       7             : //# under the terms of the GNU Library General Public License as published by
       8             : //# the Free Software Foundation; either version 2 of the License, or (at your
       9             : //# option) any later version.
      10             : //#
      11             : //# This library is distributed in the hope that it will be useful, but WITHOUT
      12             : //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13             : //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
      14             : //# License for more details.
      15             : //#
      16             : //# You should have received a copy of the GNU Library General Public License
      17             : //# along with this library; if not, write to the Free Software Foundation,
      18             : //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
      19             : //#
      20             : //# Correspondence concerning AIPS++ should be addressed as follows:
      21             : //#        Internet email: casa-feedback@nrao.edu.
      22             : //#        Postal address: AIPS++ Project Office
      23             : //#                        National Radio Astronomy Observatory
      24             : //#                        520 Edgemont Road
      25             : //#                        Charlottesville, VA 22903-2475 USA
      26             : //#
      27             : #ifndef CUBE_PARTITION_MIXIN_H_
      28             : #define CUBE_PARTITION_MIXIN_H_
      29             : 
      30             : #include <synthesis/ImagerObjects/SynthesisUtilMethods.h>
      31             : #include <synthesis/ImagerObjects/ParallelImagerParams.h>
      32             : #include <synthesis/ImagerObjects/ParamFieldIterator.h>
      33             : #include <synthesis/ImagerObjects/MPIGlue.h>
      34             : #include <algorithm>
      35             : #include <vector>
      36             : #include <string>
      37             : #include <unistd.h>
      38             : 
      39             : namespace casa {
      40             : 
      41             : /**
      42             :  * Parameter and input data partitioning for parallel cube imaging (in
      43             :  * ParallelImagerMixin).
      44             :  */
      45             : template <class T>
      46             : class CubePartitionMixin
      47             :         : public T {
      48             : 
      49             : public:
      50           0 :         void concat_images(const string &type) {
      51           0 :                 casacore::LogIO log(casacore::LogOrigin("CubePartitionMixin", "concat_images", WHERE));
      52           0 :                 if (worker_rank >= 0) {
      53           0 :                         const std::string image_types[] =
      54             :                                 {"image", "psf", "model", "residual", "mask", "pb", "weight"};
      55           0 :                         string cwd(getcwd(nullptr, 0));
      56             :                         // wait until all ranks have completed file modifications
      57           0 :                         MPI_Barrier(worker_comm);
      58             :                         // round-robin allocation of image concatenation tasks to worker
      59             :                         // ranks
      60           0 :                         for (casacore::uInt i = (casacore::uInt)worker_rank;
      61           0 :                              i < image_parameters.nfields();
      62           0 :                              i += (casacore::uInt)num_workers) {
      63           0 :                                 std::string imagename =
      64           0 :                                         image_parameters.subRecord(i).asString("imagename");
      65           0 :                                 std::string image_prefix = cwd + "/" + imagename;
      66           0 :                                 std::vector<std::string> images;
      67           0 :                                 for (auto ext : image_types) {
      68           0 :                                         images.clear();
      69           0 :                                         std::string ext_suffix = "." + ext;
      70           0 :                                         std::string concat_imagename =
      71             :                                                 image_prefix + ext_suffix;
      72           0 :                                         for (casacore::uInt j = 0; j < (casacore::uInt)num_workers; ++j) {
      73           0 :                                                 std::string component_imagename =
      74             :                                                         image_prefix + ".n" + std::to_string(j) + ext_suffix;
      75           0 :                                                 if (access(component_imagename.c_str(), F_OK) == 0)
      76           0 :                                                         images.push_back(component_imagename);
      77             :                                         }
      78           0 :                                         if (!images.empty()) {
      79           0 :                                                 std::string cmd = "imageconcat inimages='";
      80           0 :                                                 for (auto im : images) cmd += im + " ";
      81           0 :                                                 cmd += "' outimage='" + concat_imagename + "' type=";
      82           0 :                                                 cmd += type;
      83           0 :                                                 int rc = std::system(cmd.c_str());
      84           0 :                                                 if (rc == -1 || WEXITSTATUS(rc) != 0)
      85             :                                                         log << casacore::LogIO::WARN
      86             :                                                             << ("concatenation of " + concat_imagename
      87             :                                                                 + " failed")
      88           0 :                                                             << casacore::LogIO::POST;
      89           0 :                                         }
      90             :                                 }
      91             :                         }
      92           0 :                 }
      93           0 :         };
      94             : 
      95             : protected:
      96             : 
      97             :         MPI_Comm worker_comm;
      98             : 
      99             :         int num_workers;
     100             : 
     101             :         int worker_rank;
     102             : 
     103             :         casacore::Record image_parameters;
     104             : 
     105             :         ParallelImagerParams
     106           0 :         get_params(MPI_Comm wcomm, ParallelImagerParams &initial) {
     107             : 
     108             :                 // Determine rank among parallel imaging worker processes.
     109           0 :                 worker_comm = wcomm;
     110           0 :                 if (worker_comm != MPI_COMM_NULL) {
     111           0 :                         MPI_Comm_size(worker_comm, &num_workers);
     112           0 :                         MPI_Comm_rank(worker_comm, &worker_rank);
     113             :                 } else {
     114           0 :                         num_workers = 0;
     115           0 :                         worker_rank = -1;
     116             :                 }
     117             : 
     118           0 :                 std::string cwd(getcwd(nullptr, 0));
     119           0 :                 std::vector<std::string> all_worker_suffixes;
     120           0 :                 for (int r = 0; r < num_workers; ++r) {
     121           0 :                         all_worker_suffixes.push_back(".n" + std::to_string(r));
     122             :                 }
     123           0 :                 std::string my_worker_suffix =
     124           0 :                         ((worker_rank >= 0) ? all_worker_suffixes[worker_rank] : "");
     125             : 
     126           0 :                 image_parameters = casacore::Record(initial.image);
     127             : 
     128           0 :                 SynthesisUtilMethods util;
     129           0 :                 ParallelImagerParams result;
     130             : 
     131           0 :                 casacore::Record data_image_partition;
     132           0 :                 if (worker_rank >= 0) {
     133           0 :                         std::string worker_rank_str = std::to_string(worker_rank);
     134             :                         // need a SynthesisImager instance to do cube partitioning
     135           0 :                         std::unique_ptr<SynthesisImager> si(new SynthesisImager());
     136           0 :                         for (casacore::uInt i = 0; i < initial.selection.nfields(); ++i) {
     137           0 :                                 SynthesisParamsSelect selection_pars;
     138           0 :                                 selection_pars.fromRecord(initial.selection.subRecord(i));
     139           0 :                                 si->selectData(selection_pars);
     140             :                         }
     141           0 :                         for (casacore::uInt i = 0; i < initial.image.nfields(); ++i) {
     142           0 :                                 casacore::String i_name = initial.image.name(i);
     143           0 :                                 if (initial.grid.isDefined(i_name)) {
     144           0 :                                         SynthesisParamsImage image_pars;
     145           0 :                                         image_pars.fromRecord(initial.image.subRecord(i_name));
     146           0 :                                         SynthesisParamsGrid grid_pars;
     147           0 :                                         grid_pars.fromRecord(initial.grid.subRecord(i_name));
     148           0 :                                         si->defineImage(image_pars, grid_pars);
     149           0 :                                         casacore::Record csys = si->getcsys();
     150           0 :                                         if (csys.nfields() > 0) {
     151           0 :                                                 int nchan = ((image_pars.nchan == -1)
     152           0 :                                                              ? si->updateNchan()
     153             :                                                              : image_pars.nchan);;
     154           0 :                                                 casacore::Vector<casacore::Int> numchans;
     155           0 :                                                 casacore::Vector<casacore::CoordinateSystem> csystems;
     156             :                                                 // save only that part of the record returned from
     157             :                                                 // util.cubeDataImagePartition that is handled by the
     158             :                                                 // current rank
     159           0 :                                                 data_image_partition.defineRecord(
     160             :                                                         i_name,
     161             :                                                         util.cubeDataImagePartition(
     162           0 :                                                                 initial.selection,
     163           0 :                                                                 *casacore::CoordinateSystem::restore(csys, "coordsys"),
     164             :                                                                 num_workers, nchan, csystems, numchans).
     165           0 :                                                         rwSubRecord(worker_rank_str));
     166           0 :                                         }
     167           0 :                                 }
     168             :                         }
     169           0 :                 }
     170             : 
     171             :                 // selection_params
     172           0 :                 if (worker_rank >= 0) {
     173           0 :                         casacore::Record sel;
     174           0 :                         for (casacore::uInt i = 0; i < data_image_partition.nfields(); ++i) {
     175           0 :                                 casacore::Record &di = data_image_partition.rwSubRecord(i);
     176           0 :                                 for (casacore::uInt f = 0; f < di.nfields(); ++f) {
     177           0 :                                         casacore::String name = di.name(f);
     178           0 :                                         if (name.find("ms") == 0) {
     179           0 :                                                 casacore::Record &ms = di.rwSubRecord(f);
     180           0 :                                                 if (ms.isDefined("spw") && ms.asString("spw") == "-1")
     181           0 :                                                         ms.define("spw", "");
     182           0 :                                                 sel.defineRecord(name, ms);
     183             :                                         }
     184             :                                 }
     185             :                         }
     186           0 :                         result.selection = sel;
     187           0 :                 } else {
     188           0 :                         result.selection = casacore::Record();
     189             :                 }
     190             : 
     191             :                 // image_params
     192           0 :                 if (worker_rank >= 0) {
     193           0 :                         result.image = initial.image;
     194           0 :                         for (casacore::uInt i = 0; i < data_image_partition.nfields(); ++i) {
     195           0 :                                 const casacore::Record &di = data_image_partition.subRecord(i);
     196           0 :                                 casacore::String i_name = data_image_partition.name(i);
     197           0 :                                 casacore::Record &i_image = result.image.rwSubRecord(i_name);
     198           0 :                                 i_image.define("csys", di.asString("coordsys"));
     199           0 :                                 i_image.define("nchan", di.asString("nchan"));
     200           0 :                                 i_image.define(
     201             :                                         "imagename",
     202           0 :                                         i_image.asString("imagename") + casacore::String(my_worker_suffix));
     203             :                         }
     204             :                 } else {
     205           0 :                         result.image = empty_fields(initial.image, "imagename");
     206             :                 }
     207             : 
     208             :                 // FIXME: are grid parameters partitioned by node?
     209             :                 //
     210             :                 // grid params
     211           0 :                 if (worker_rank >= 0) {
     212           0 :                         auto modify_cfcache = [&](const char *field_val) {
     213           0 :                                 return *field_val + my_worker_suffix;
     214             :                         };
     215           0 :                         result.grid =
     216           0 :                                 convert_fields(initial.grid, "cfcache", modify_cfcache);
     217             :                 } else {
     218           0 :                         result.grid = empty_fields(initial.grid, "cfcache");
     219             :                 }
     220             : 
     221             :                 // normalization_params
     222           0 :                 result.normalization =
     223           0 :                         ((worker_rank >= 0) ? initial.normalization : casacore::Record());
     224             : 
     225             :                 // deconvolution params
     226           0 :                 result.deconvolution =
     227           0 :                         ((worker_rank >= 0) ? initial.deconvolution : casacore::Record());
     228             : 
     229             :                 // weight params
     230           0 :                 result.weight =
     231           0 :                         ((worker_rank >= 0) ? initial.weight : casacore::Record());
     232             : 
     233             :                 // iteration params
     234           0 :                 result.iteration = initial.iteration;
     235             : 
     236           0 :                 return result;
     237           0 :         }
     238             : 
     239             : private:
     240             : 
     241             :         // Convenience method to transform certain record fields
     242           0 :         casacore::Record convert_fields(casacore::Record &rec, const char *field,
     243             :                               std::function<std::string(const char *)> fn) {
     244           0 :                 auto modify_field_val = [&](casacore::Record &msRec) {
     245           0 :                         msRec.define(field, fn(msRec.asString(field).c_str()));
     246             :                 };
     247           0 :                 casacore::Record result(rec);
     248           0 :                 for_each(ParamFieldIterator::begin(&result),
     249             :                          ParamFieldIterator::end(&result),
     250             :                          modify_field_val);
     251           0 :                 return result;
     252           0 :         }
     253             : 
     254             :         // Convenience method to clear certain record fields
     255           0 :         casacore::Record empty_fields(casacore::Record &rec, const char *field) {
     256           0 :                 auto modify_field_val = [&](casacore::Record &msRec) {
     257           0 :                         msRec.defineRecord(field, casacore::Record());
     258             :                 };
     259           0 :                 casacore::Record result(rec);
     260           0 :                 for_each(ParamFieldIterator::begin(&result),
     261             :                          ParamFieldIterator::end(&result),
     262             :                          modify_field_val);
     263           0 :                 return result;
     264           0 :         }
     265             : };
     266             : 
     267             : } // namespace casa
     268             : 
     269             : #endif // CUBE_PARTITION_MIXIN_H_

Generated by: LCOV version 1.16