Line data Source code
1 : #include <alma/ASDMBinaries/SDMDataObjectReader.h>
2 :
3 : #include <iostream>
4 : #include <iomanip>
5 : #include <unistd.h>
6 : #include <sys/types.h>
7 : #include <sys/mman.h>
8 : #include <fcntl.h>
9 : #include <sys/stat.h>
10 : #include <errno.h>
11 : #include <exception>
12 : #include <sstream>
13 : #include <cstring>
14 :
15 : #include <alma/ASDMBinaries/CommonDefines.h>
16 :
17 : #if defined( __APPLE__ )
18 : #include <AvailabilityMacros.h>
19 : // 10.5 defines stat64 but Tiger does NOT
20 : #if defined(MAC_OS_X_VERSION_10_4) && ! defined(MAC_OS_X_VERSION_10_5)
21 : #define stat64 stat
22 : #define fstat64 fstat
23 : #endif
24 : #endif
25 :
26 : using namespace CorrelationModeMod;
27 : using namespace CorrelatorTypeMod;
28 : using namespace PrimitiveDataTypeMod;
29 : using namespace ProcessorTypeMod;
30 :
31 : using namespace std;
32 :
33 : namespace asdmbinaries {
34 :
35 : const string SDMDataObjectReader::MIMEBOUNDARY_1 = $MIMEBOUNDARY1;
36 : const string SDMDataObjectReader::MIMEBOUNDARY_2 = $MIMEBOUNDARY2;
37 :
38 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
39 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
40 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
41 : // const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
42 : const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
43 : const std::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
44 : set<string> SDMDataObjectReader::BINATTACHNAMES;
45 : map<string, SDMDataObjectReader::BINATTACHCODES> SDMDataObjectReader::name2code;
46 : const bool SDMDataObjectReader::initClass_ = SDMDataObjectReader::initClass();
47 :
48 : // SDMDataObjectReader:: methods
49 : //
50 86 : SDMDataObjectReader::SDMDataObjectReader() {
51 86 : init();
52 86 : }
53 :
54 86 : SDMDataObjectReader::~SDMDataObjectReader() {
55 86 : done();
56 86 : }
57 :
58 124 : bool SDMDataObjectReader::initClass() {
59 124 : char* binAttachNamesC[] = {(char *) "actualDurations", (char *) "actualTimes", (char *) "autoData", (char *) "flags", (char *) "crossData", (char *) "zeroLags", 0};
60 868 : for (int i = 0; binAttachNamesC[i]; i++) {
61 744 : BINATTACHNAMES.insert(string(binAttachNamesC[i]));
62 : }
63 :
64 124 : name2code["actualDurations"] = ACTUALDURATIONS;
65 124 : name2code["actualTimes"] = ACTUALTIMES;
66 124 : name2code["autoData"] = AUTODATA;
67 124 : name2code["flags"] = FLAGS;
68 124 : name2code["crossData"] = CROSSDATA;
69 124 : name2code["zeroLags"] = ZEROLAGS;
70 :
71 124 : return true;
72 : }
73 :
74 86 : void SDMDataObjectReader::init() {
75 86 : filedes_ = 0;
76 86 : data_ = (char *) 0;
77 86 : position_ = 0;
78 86 : lastPosition_ = 0;
79 86 : endPosition_ = 0;
80 :
81 86 : read_ = UNKNOWN_;
82 :
83 86 : integrationNum_ = 0;
84 86 : }
85 :
86 20564 : string::size_type SDMDataObjectReader::find(const string& s) {
87 :
88 : while (true) {
89 1254303 : while ((position_ < endPosition_) && (data_[position_++] != s.at(0))) {;}
90 :
91 20564 : if (position_ == endPosition_) return string::npos;
92 :
93 20564 : string::size_type lastPosition_ = position_;
94 20564 : unsigned long long int i = 1;
95 :
96 65100 : while ((position_ < endPosition_) && (i < s.length())) {
97 61128 : if (s.at(i) != data_[position_]) break;
98 44536 : i++;
99 44536 : position_++;
100 : }
101 :
102 20564 : if (i == s.length()) return (position_ - s.length());
103 :
104 16592 : if (position_ == endPosition_) return string::npos;
105 :
106 16592 : position_ = lastPosition_; //+1;
107 16592 : }
108 : }
109 :
110 1437 : bool SDMDataObjectReader::compare(const string& s) {
111 1437 : string::size_type index = position_;
112 :
113 1437 : unsigned int i = 0;
114 9489 : while ((index < endPosition_) && (i < s.length()) && (s.at(i++) == data_[index++])) {
115 : ;
116 : }
117 1437 : return (i == s.length());
118 : }
119 :
120 1437 : bool SDMDataObjectReader::EOD() {
121 1437 : return position_ >= endPosition_;
122 : }
123 :
124 :
125 732 : string SDMDataObjectReader::extractXMLHeader(const string& boundary) {
126 : // cout << "Entering extractXMLHeader" << endl;
127 :
128 732 : string boundary_ = "\n--"+boundary;
129 :
130 732 : unsigned long int positionBeg = 0;
131 732 : unsigned long int positionEnd = 0;
132 :
133 : // We assume that we are at the first character of the body.
134 732 : positionBeg = position_;
135 :
136 : // Look for the next boundary supposed to be found right after an the XML header.
137 732 : if ( (positionEnd = find(boundary_)) == string::npos) {
138 0 : ostringstream eos;
139 0 : eos << "A second MIME boundary '" << boundary <<"' could not be found in the MIME message after position " << positionBeg << ".";
140 0 : throw SDMDataObjectReaderException(eos.str());
141 0 : }
142 :
143 732 : string xmlPart = string(data_+positionBeg, positionEnd-positionBeg);
144 :
145 : // cout << "Found XML Header : " << xmlPart << endl;
146 : // cout << "Exiting extractXMLHeader" << endl;
147 :
148 1464 : return xmlPart;
149 732 : }
150 :
151 :
152 3240 : void SDMDataObjectReader::tokenize(const string& str,
153 : vector<string>& tokens,
154 : const string& delimiters) {
155 : // Skip delimiters at beginning.
156 3240 : string::size_type lastPos = str.find_first_not_of(delimiters, 0);
157 : // Find first "non-delimiter".
158 3240 : string::size_type pos = str.find_first_of(delimiters, lastPos);
159 :
160 9720 : while (string::npos != pos || string::npos != lastPos) {
161 : // Found a token, add it to the vector.
162 6480 : tokens.push_back(str.substr(lastPos, pos - lastPos));
163 : // Skip delimiters. Note the "not_of"
164 6480 : lastPos = str.find_first_not_of(delimiters, pos);
165 : // Find next "non-delimiter"
166 6480 : pos = str.find_first_of(delimiters, lastPos);
167 : }
168 3240 : }
169 :
170 :
171 1437 : void SDMDataObjectReader::getFields(const string& header, map<string, string>& fields) {
172 : // We are at the last character of the line preceding the 1st field declaration.
173 1437 : istringstream iss;
174 1437 : iss.str(header);
175 1437 : string line;
176 :
177 1437 : getline(iss, line); // skip an nl
178 :
179 : // Split each line into field-name, field-value
180 4677 : while (getline(iss, line)) {
181 3240 : vector<string> nv;
182 3240 : tokenize(line, nv, ":");
183 3240 : if (nv.size() < 2) {
184 0 : ostringstream oss;
185 0 : oss << "Invalid field in a MIME header '" << header << "', integration #" << integrationNum_;
186 0 : throw SDMDataObjectReaderException (oss.str());
187 0 : }
188 3240 : fields[nv.at(0)] = nv.at(1);
189 3240 : }
190 1437 : }
191 :
192 0 : string SDMDataObjectReader::getContentID() {
193 : // We are at the beginning of an attachment right after its opening boundary.
194 0 : string::size_type lastPosition = position_;
195 :
196 : // Now let's extract the header (which is followed by two successive nl nl)
197 : string::size_type endHeader;
198 0 : if ( (endHeader = find ("\n\n")) == string::npos) {
199 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
200 : }
201 :
202 0 : string header(data_+lastPosition, (endHeader - lastPosition));
203 :
204 0 : map<string, string> fields;
205 0 : getFields(header, fields);
206 :
207 : // Look for the Content-ID field
208 0 : map<string, string>::iterator iter = fields.find("Content-ID");
209 0 : if (iter == fields.end()) {
210 0 : ostringstream oss;
211 0 : oss << "'Content-ID' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
212 0 : throw SDMDataObjectReaderException(oss.str());
213 0 : }
214 0 : return iter->second;
215 0 : }
216 :
217 1437 : string SDMDataObjectReader::getContentLocation() {
218 : // We are at the beginning of an attachment right after its opening boundary.
219 1437 : string::size_type lastPosition = position_;
220 :
221 : // Now let's extract the header (which is followed by two successive nl nl)
222 : string::size_type endHeader;
223 1437 : if ( (endHeader = find ("\n\n")) == string::npos) {
224 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
225 : }
226 :
227 1437 : string header(data_+lastPosition, (endHeader - lastPosition));
228 :
229 1437 : map<string, string> fields;
230 1437 : getFields(header, fields);
231 :
232 : // Look for the Content-Location field
233 1437 : map<string, string>::iterator iter = fields.find("Content-Location");
234 1437 : if (iter == fields.end()) {
235 0 : ostringstream oss;
236 0 : oss << "'Content-Location' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
237 0 : throw SDMDataObjectReaderException(oss.str());
238 0 : }
239 2874 : return iter->second;
240 1437 : }
241 :
242 705 : void SDMDataObjectReader::processBinaryAttachment(const string& boundary, const SDMDataSubset& sdmDataSubset) {
243 :
244 705 : string contentLocation = getContentLocation();
245 :
246 : // Extract the attachment name and the integration number.
247 705 : std::cmatch what;
248 1410 : std::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
249 705 : if (std::regex_match(contentLocation.c_str(), what, r)) {
250 : }
251 : else {
252 0 : ostringstream oss;
253 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of an attachment (approx. char position = " << position_ << ").";
254 0 : throw SDMDataObjectReaderException(oss.str());
255 0 : }
256 :
257 : // We are at the first byte of the body of the binary attachment.
258 705 : string::size_type startAttachPosition = position_; // mcaillat
259 :
260 : // Now look for its closing boundary (or opening boundary of the next one if any).
261 705 : string boundary_ = "\n--"+boundary;
262 : string::size_type endAttachPosition;
263 705 : if ( (endAttachPosition = find(boundary_)) == string::npos) {
264 0 : ostringstream eos;
265 0 : eos << "A MIME boundary '" << boundary << "' terminating a binary attachment starting approx. at character " << startAttachPosition <<" could not be found.";
266 0 : throw SDMDataObjectReaderException(eos.str());
267 0 : }
268 :
269 : // Compute the length of the attachment.
270 705 : string::size_type length = endAttachPosition - startAttachPosition;
271 :
272 :
273 :
274 705 : switch (name2code[what[1]]) {
275 0 : case ACTUALDURATIONS:
276 0 : actualDurations_ = (ACTUALDURATIONSTYPE *) (data_ + startAttachPosition);
277 0 : nActualDurations_ = length / sizeof (ACTUALDURATIONSTYPE);
278 0 : attachmentFlags.set(ACTUALDURATIONS);
279 : //cout << "actualDurations =" << (unsigned long int) actualDurations_ << ", nActualDurations = " << nActualDurations_ << endl;
280 0 : break;
281 0 : case ACTUALTIMES:
282 0 : actualTimes_ = (ACTUALTIMESTYPE *) (data_ + startAttachPosition);
283 0 : nActualTimes_ = length / sizeof( ACTUALTIMESTYPE);
284 0 : attachmentFlags.set(ACTUALTIMES);
285 : //cout << "actualTimes =" << (unsigned long int) actualTimes_ << ", nActualTimes = " << nActualTimes_ << endl;
286 0 : break;
287 366 : case AUTODATA:
288 366 : autoData_ = (AUTODATATYPE *) (data_ + startAttachPosition);
289 366 : nAutoData_ = length / sizeof(AUTODATATYPE);
290 366 : attachmentFlags.set(AUTODATA);
291 : //cout << "autoData =" << (unsigned long int) autoData_ << ", nAutoData = " << nAutoData_ << endl;
292 366 : break;
293 339 : case FLAGS:
294 339 : flags_ = (const FLAGSTYPE *) (data_ + startAttachPosition);
295 339 : nFlags_ = length / sizeof(FLAGSTYPE);
296 339 : attachmentFlags.set(FLAGS);
297 : //cout << "flags =" << (unsigned long int) flags_ << ", nFlags = " << nFlags_ << endl;
298 339 : break;
299 0 : case CROSSDATA:
300 0 : shortCrossData_ = 0;
301 0 : longCrossData_ = 0;
302 0 : floatCrossData_ = 0;
303 0 : switch (sdmDataSubset.crossDataType()) {
304 0 : case INT16_TYPE:
305 0 : shortCrossData_ = (const SHORTCROSSDATATYPE*) (data_ + startAttachPosition);
306 0 : nCrossData_ = length / sizeof (SHORTCROSSDATATYPE);
307 : //cout << "shortCrossData = " << (unsigned long int) shortCrossData_ << ", nShortCrossData = " << nCrossData_ << endl;
308 0 : break;
309 :
310 0 : case INT32_TYPE:
311 0 : longCrossData_ = (const INTCROSSDATATYPE*) (data_ + startAttachPosition);
312 0 : nCrossData_ = length / sizeof (INTCROSSDATATYPE);
313 : //cout << "longCrossData = " << (unsigned long int) longCrossData_ << ", nLongCrossData = " << nCrossData_ << endl;
314 0 : break;
315 :
316 0 : case FLOAT32_TYPE:
317 0 : floatCrossData_ = (const FLOATCROSSDATATYPE*) (data_ + startAttachPosition);
318 0 : nCrossData_ = length / sizeof (FLOATCROSSDATATYPE);
319 0 : break;
320 :
321 0 : default:
322 0 : ostringstream eos;
323 0 : eos << "'" << CPrimitiveDataType::toString(sdmDataSubset.crossDataType())
324 0 : << "' is not a valid primitive data type for CROSSDATA";
325 0 : throw SDMDataObjectReaderException(eos.str());
326 : // cout << "Unrecognized type for cross data" << endl;
327 : }
328 0 : attachmentFlags.set(CROSSDATA);
329 0 : break;
330 0 : case ZEROLAGS:
331 0 : zeroLags_ = (ZEROLAGSTYPE *) (data_ + startAttachPosition);
332 0 : nZeroLags_ = length / sizeof(ZEROLAGSTYPE) ;
333 0 : attachmentFlags.set(ZEROLAGS);
334 : //cout << "zeroLags =" << (unsigned long int) zeroLags_ << ", nZeroLags = " << nZeroLags_ << endl;
335 0 : break;
336 :
337 0 : default:
338 : //cout << "Unrecognized code" << endl;
339 0 : break;
340 : }
341 : // cout << "found a binary attachment" << endl;
342 705 : }
343 :
344 366 : void SDMDataObjectReader::processMIMESDMDataHeader() {
345 : // cout << "Entering processMIMESDMDataHeader" << endl;
346 366 : string sdmDataHeader = extractXMLHeader(MIMEBOUNDARY_1);
347 366 : parser_.parseMemoryHeader(sdmDataHeader, sdmDataObject_);
348 : // cout << "Exiting processMIMESDMDataHeader" << endl;
349 366 : }
350 :
351 366 : void SDMDataObjectReader::processMIMESDMDataSubsetHeader(SDMDataSubset& sdmDataSubset) {
352 : // cout << "Entering processMIMESDMDataSubsetHeader" << endl;
353 366 : string sdmDataSubsetHeader = extractXMLHeader(MIMEBOUNDARY_2);
354 366 : if (sdmDataSubset.owner_->isCorrelation())
355 0 : parser_.parseMemoryCorrSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
356 : else
357 366 : parser_.parseMemoryTPSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
358 : // cout << "Exiting processMIMESDMDataSubsetHeader" << endl;
359 366 : }
360 :
361 0 : void SDMDataObjectReader::processMIMEIntegration() {
362 : // cout << "Entering processMIMEIntegration" << endl;
363 :
364 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
365 0 : string contentLocation = getContentLocation();
366 :
367 0 : std::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
368 0 : ostringstream oss;
369 :
370 : // Extract the Subset name and the integration [, subintegration] number.
371 0 : std::cmatch what;
372 0 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
373 0 : ostringstream oss;
374 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
375 0 : throw SDMDataObjectReaderException(oss.str());
376 0 : }
377 :
378 0 : SDMDataSubset integration(&sdmDataObject_);
379 :
380 : // The SDMDataSubset header.
381 0 : processMIMESDMDataSubsetHeader(integration);
382 :
383 0 : if (integration.aborted_) {
384 : // The [sub]integration has been aborted. Just append its header, without trying to get binary attachment.
385 0 : sdmDataObject_.append(integration);
386 : }
387 : else {
388 : // This is regular [sub]integration, process its binary attachments.
389 0 : attachmentFlags.reset();
390 0 : while (!EOD() && !compare("--")) {
391 0 : processBinaryAttachment(MIMEBOUNDARY_2, integration);
392 : }
393 :
394 0 : if (EOD())
395 0 : throw SDMDataObjectReaderException("Unexpected end of data");
396 :
397 : // Now check if the binary attachments found are compatible with the correlation mode
398 : // and if their sizes are equal to what is announced in the global header.
399 :
400 0 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
401 : // No actualdurations attachment found, ok then set everything to 0.
402 0 : integration.actualDurations_ = 0;
403 0 : integration.nActualDurations_ = 0;
404 : }
405 : else {
406 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
407 0 : ostringstream oss;
408 0 : oss << "Data subset '"<<contentLocation<<"': ";
409 0 : oss << "zize of 'actualDuration' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
410 0 : throw SDMDataObjectReaderException(oss.str());
411 0 : }
412 :
413 0 : integration.actualDurations_ = actualDurations_;
414 0 : integration.nActualDurations_ = nActualDurations_;
415 : }
416 :
417 0 : if (!attachmentFlags.test(ACTUALTIMES)) {
418 : // No actualtimes attachment found, ok then set everything to 0.
419 0 : integration.actualTimes_ = 0;
420 0 : integration.nActualTimes_ = 0;
421 : }
422 : else {
423 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
424 0 : ostringstream oss;
425 0 : oss << "Data subset '"<<contentLocation<<"': ";
426 0 : oss << "size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
427 0 : throw SDMDataObjectReaderException(oss.str());
428 0 : }
429 0 : integration.actualTimes_ = actualTimes_;
430 0 : integration.nActualTimes_ = nActualTimes_;
431 : }
432 :
433 : // The flags are optional. They may be absent.
434 0 : if (!attachmentFlags.test(FLAGS)) {
435 : // No flags binary attachment found, ok then set everything to 0.
436 0 : integration.flags_ = 0;
437 0 : integration.nFlags_ = 0;
438 : }
439 : else {
440 : // flags found
441 : // Check size conformity
442 0 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
443 0 : ostringstream oss;
444 0 : oss << "Data subset '"<<contentLocation<<"': ";
445 0 : oss << "size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
446 0 : throw SDMDataObjectReaderException(oss.str());
447 0 : }
448 :
449 : // Set pointer and size
450 0 : integration.flags_ = flags_;
451 0 : integration.nFlags_ = nFlags_;
452 : }
453 :
454 :
455 : //
456 : // The presence of crossData and autoData depends on the correlation mode.
457 :
458 : // crossDataTypeSize is not currently used
459 : // unsigned int crossDataTypeSize;
460 0 : switch (sdmDataObject_.correlationMode()) {
461 0 : case CROSS_ONLY:
462 0 : if (!attachmentFlags.test(CROSSDATA)) {
463 0 : ostringstream oss;
464 0 : oss << "Data subset '"<<contentLocation<<"': ";
465 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationNum_;
466 0 : throw SDMDataObjectReaderException(oss.str());
467 0 : }
468 :
469 :
470 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
471 0 : ostringstream oss;
472 0 : oss << "Data subset '"<<contentLocation<<"': ";
473 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
474 0 : throw SDMDataObjectReaderException(oss.str());
475 0 : }
476 :
477 :
478 0 : if (attachmentFlags.test(AUTODATA)) {
479 0 : ostringstream oss;
480 0 : oss << "Data subset '"<<contentLocation<<"': ";
481 0 : oss << "found an unexpected attachment 'autoData' in integration #" << integrationNum_ << ".";
482 0 : throw SDMDataObjectReaderException(oss.str());
483 0 : }
484 :
485 0 : integration.shortCrossData_ = 0;
486 0 : integration.longCrossData_ = 0;
487 0 : integration.floatCrossData_ = 0;
488 0 : integration.nCrossData_ = 0;
489 0 : switch (integration.crossDataType()) {
490 0 : case INT32_TYPE:
491 0 : integration.longCrossData_ = longCrossData_;
492 0 : integration.nCrossData_ = nCrossData_;
493 0 : break;
494 :
495 0 : case INT16_TYPE:
496 0 : integration.shortCrossData_ = shortCrossData_;
497 0 : integration.nCrossData_ = nCrossData_;
498 0 : break;
499 :
500 0 : case FLOAT32_TYPE:
501 0 : integration.floatCrossData_ = floatCrossData_;
502 0 : integration.nCrossData_ = nCrossData_;
503 0 : break;
504 :
505 0 : default:
506 0 : throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
507 : }
508 0 : break;
509 :
510 0 : case AUTO_ONLY:
511 0 : if (!attachmentFlags.test(AUTODATA)) {
512 0 : ostringstream oss;
513 0 : oss << "Data subset '"<<contentLocation<<"': ";
514 0 : oss << "a binary attachment 'autoData' was expected.";
515 0 : throw SDMDataObjectReaderException(oss.str());
516 0 : }
517 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
518 0 : ostringstream oss;
519 0 : oss << "Data subset '"<<contentLocation<<"': ";
520 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
521 0 : throw SDMDataObjectReaderException(oss.str());
522 0 : }
523 :
524 :
525 0 : if (attachmentFlags.test(CROSSDATA)) {
526 0 : ostringstream oss;
527 0 : oss << "Data subset '"<<contentLocation<<"': ";
528 0 : oss << "found an unexpected attachment 'crossData'";
529 0 : throw SDMDataObjectReaderException(oss.str());
530 0 : }
531 :
532 0 : integration.autoData_ = autoData_;
533 0 : integration.nAutoData_ = nAutoData_;
534 :
535 0 : break;
536 :
537 0 : case CROSS_AND_AUTO:
538 0 : if (!attachmentFlags.test(AUTODATA)) {
539 0 : ostringstream oss;
540 0 : oss << "Data subset '"<<contentLocation<<"': ";
541 0 : oss << "a binary attachment 'autoData' was expected.";
542 0 : throw SDMDataObjectReaderException(oss.str());
543 0 : }
544 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
545 0 : ostringstream oss;
546 0 : oss << "Data subset '"<<contentLocation<<"': ";
547 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
548 0 : throw SDMDataObjectReaderException(oss.str());
549 0 : }
550 :
551 0 : if (!attachmentFlags.test(CROSSDATA)) {
552 0 : ostringstream oss;
553 0 : oss << "Data subset '"<<contentLocation<<"': ";
554 0 : oss << "a binary attachment 'crossData' was expected.";
555 0 : throw SDMDataObjectReaderException(oss.str());
556 0 : }
557 :
558 : // crossDataTypeSize is not currently used, skip this block
559 : // switch (integration.crossDataType()) {
560 : // case INT32_TYPE:
561 : // crossDataTypeSize = sizeof(INTCROSSDATATYPE);
562 : // break;
563 : // case INT16_TYPE:
564 : // crossDataTypeSize = sizeof(SHORTCROSSDATATYPE);
565 : // break;
566 : // case FLOAT32_TYPE:
567 : // crossDataTypeSize = sizeof(FLOATCROSSDATATYPE);
568 : // break;
569 : // default:
570 : // throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
571 : // }
572 :
573 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
574 0 : ostringstream oss;
575 0 : oss << "Data subset '"<<contentLocation<<"': ";
576 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
577 0 : throw SDMDataObjectReaderException(oss.str());
578 0 : }
579 :
580 :
581 0 : integration.shortCrossData_ = 0;
582 0 : integration.longCrossData_ = 0;
583 0 : integration.floatCrossData_ = 0;
584 0 : integration.nCrossData_ = 0;
585 0 : switch (integration.crossDataType()) {
586 0 : case INT32_TYPE:
587 0 : integration.longCrossData_ = longCrossData_;
588 0 : integration.nCrossData_ = nCrossData_;
589 0 : break;
590 :
591 0 : case INT16_TYPE:
592 0 : integration.shortCrossData_ = shortCrossData_;
593 0 : integration.nCrossData_ = nCrossData_;
594 0 : break;
595 :
596 0 : case FLOAT32_TYPE:
597 0 : integration.floatCrossData_ = floatCrossData_;
598 0 : integration.nCrossData_ = nCrossData_;
599 0 : break;
600 :
601 0 : default:
602 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': '"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
603 : }
604 :
605 0 : integration.autoData_ = autoData_;
606 0 : integration.nAutoData_ = nAutoData_;
607 :
608 :
609 0 : break;
610 0 : default:
611 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': unrecognized correlation mode");
612 : break;
613 : }
614 :
615 :
616 0 : if (attachmentFlags.test(ZEROLAGS)) {
617 : // Refuse the zeroLags attachment if it's not a Correlator or if the correlator is a CORRELATOR_FX (ACA).
618 0 : if ((sdmDataObject_.processorType_ != CORRELATOR) || (sdmDataObject_.correlatorType() == FX))
619 0 : throw SDMDataObjectReaderException("zeroLags are not expected from a correlator CORRELATOR_FX");
620 :
621 0 : if (nZeroLags_ != sdmDataObject_.dataStruct().zeroLags().size()) {
622 0 : ostringstream oss;
623 0 : oss << "Data subset '"<<contentLocation<<"': ";
624 0 : oss << "size of 'zeroLags' attachment (" << nZeroLags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().zeroLags().size() << ")";
625 0 : throw SDMDataObjectReaderException(oss.str());
626 0 : }
627 0 : integration.zeroLags_ = zeroLags_;
628 0 : integration.nZeroLags_ = nZeroLags_;
629 : }
630 :
631 0 : sdmDataObject_.append(integration);
632 : }
633 :
634 0 : if (!compare("--\n--"+MIMEBOUNDARY_1))
635 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
636 :
637 0 : find("--\n--"+MIMEBOUNDARY_1);
638 : // cout << "Exiting processMIMEIntegration" << endl;
639 0 : }
640 :
641 0 : void SDMDataObjectReader::processMIMEIntegrations() {
642 : // cout << "Entering processMIMEIntegrations" << endl;
643 :
644 0 : while (!compare("--")) {
645 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
646 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
647 : // and move to the next --<MIMEBOUNDARY_2>
648 0 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
649 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
650 :
651 : //Process the next integration.
652 0 : integrationNum_++;
653 0 : processMIMEIntegration() ;
654 : }
655 :
656 : // cout << "Exiting processMIMEIntegrations" << endl;
657 0 : }
658 :
659 366 : void SDMDataObjectReader::processMIMESubscan() {
660 : // cout << "Entering processMIMESubscan" << endl;
661 :
662 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
663 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
664 : // and move to the next --<MIMEBOUNDARY_2>
665 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
666 366 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
667 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
668 :
669 :
670 366 : string contentLocation = getContentLocation();
671 : // cout << contentLocation << endl;
672 732 : std::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
673 :
674 : // Extract the Subset name and the suffix number (which must be equal to 1).
675 366 : std::cmatch what;
676 366 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
677 0 : ostringstream oss;
678 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
679 0 : throw SDMDataObjectReaderException(oss.str());
680 0 : }
681 :
682 366 : SDMDataSubset subscan(&sdmDataObject_);
683 :
684 : // The SDMDataSubset header.
685 366 : processMIMESDMDataSubsetHeader(subscan);
686 :
687 :
688 : // Process the binary attachments.
689 366 : attachmentFlags.reset();
690 366 : integrationNum_++;
691 1071 : while(!EOD() && !compare("--")) {
692 705 : processBinaryAttachment(MIMEBOUNDARY_2, subscan);
693 : }
694 :
695 366 : if (EOD()) throw SDMDataObjectReaderException("Unexpected end of data");
696 :
697 : // Now check if the binary attachments found are compatible with the correlation mode.
698 : // and if their sizes are equal to what is announced in the global header.
699 :
700 : // Start with the only mandatory attachment : autoData.
701 366 : if (!attachmentFlags.test(AUTODATA)) {
702 0 : ostringstream oss;
703 0 : oss << "Binary attachment 'autoData' was expected in integration #" << integrationNum_;
704 0 : throw SDMDataObjectReaderException(oss.str());
705 0 : }
706 366 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
707 0 : ostringstream oss;
708 0 : oss << "Size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
709 0 : throw SDMDataObjectReaderException(oss.str());
710 0 : }
711 :
712 366 : subscan.autoData_ = autoData_;
713 366 : subscan.nAutoData_ = nAutoData_;
714 :
715 :
716 : // And now consider the optional attachments.
717 :
718 : // The actualDurations are optional. They may be absent.
719 366 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
720 : // No actualDurations binary attachment found, ok then set everything to 0.
721 366 : subscan.actualDurations_ = 0;
722 366 : subscan.nActualDurations_ = 0;
723 : }
724 : else {
725 : // actualDurations found
726 : // Check size conformity
727 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
728 0 : ostringstream oss;
729 0 : oss << "Size of 'actualDurations' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
730 0 : throw SDMDataObjectReaderException(oss.str());
731 0 : }
732 :
733 : // Set pointer and size
734 0 : subscan.actualDurations_ = actualDurations_;
735 0 : subscan.nActualDurations_ = nActualDurations_;
736 : }
737 :
738 :
739 : // The actualTimes are optional. They may be absent.
740 366 : if (!attachmentFlags.test(ACTUALTIMES)) {
741 : // No actualTimes binary attachment found, ok then set everything to 0.
742 366 : subscan.actualTimes_ = 0;
743 366 : subscan.nActualTimes_ = 0;
744 : }
745 : else {
746 : // actualTimes found
747 : // Check size conformity
748 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
749 0 : ostringstream oss;
750 0 : oss << "Size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
751 0 : throw SDMDataObjectReaderException(oss.str());
752 0 : }
753 :
754 : // Set pointer and size
755 0 : subscan.actualTimes_ = actualTimes_;
756 0 : subscan.nActualTimes_ = nActualTimes_;
757 : }
758 :
759 :
760 : // The flags are optional. They may be absent.
761 366 : if (!attachmentFlags.test(FLAGS)) {
762 : // No flags binary attachment found, ok then set everything to 0.
763 27 : subscan.flags_ = 0;
764 27 : subscan.nFlags_ = 0;
765 : }
766 : else {
767 : // flags found
768 : // Check size conformity
769 339 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
770 0 : ostringstream oss;
771 0 : oss << "Size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
772 0 : throw SDMDataObjectReaderException(oss.str());
773 0 : }
774 :
775 : // Set pointer and size
776 339 : subscan.flags_ = flags_;
777 339 : subscan.nFlags_ = nFlags_;
778 : }
779 :
780 : // And finally let's check that no unexpected binary attachment was found.
781 366 : if (attachmentFlags.test(CROSSDATA)) {
782 0 : ostringstream oss;
783 0 : oss << "Found an unexpected attachment 'crossData' in the binary attachments.";
784 0 : throw SDMDataObjectReaderException(oss.str());
785 0 : }
786 :
787 :
788 366 : if (attachmentFlags.test(ZEROLAGS)) {
789 0 : ostringstream oss;
790 0 : oss << "Found an unexpected attachment 'zeroLags' in the binary attachments.";
791 0 : throw SDMDataObjectReaderException(oss.str());
792 0 : }
793 :
794 366 : sdmDataObject_.tpDataSubset(subscan);
795 :
796 :
797 366 : if (!compare("--\n--"+MIMEBOUNDARY_1))
798 0 : throw SDMDataObjectReaderException("Expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
799 :
800 366 : find("--\n--"+MIMEBOUNDARY_1);
801 366 : }
802 :
803 366 : void SDMDataObjectReader::processMIME() {
804 : // cout << "Entering processMIME" << endl;
805 : // Let's find the opening boundary
806 366 : if (find("--"+MIMEBOUNDARY_1) == string::npos)
807 0 : throw SDMDataObjectReaderException("Expecting a first boundary '--"+MIMEBOUNDARY_1+"'.");
808 :
809 : // Check the Content-Location of the MIME header ...but do nothing special with it...
810 366 : string contentLocation=getContentLocation();
811 :
812 : // Detect SDMDataHeader.
813 366 : processMIMESDMDataHeader();
814 :
815 : // Do we have packed data (i.e. only one SDMDataSubset grouping all the integrations) or one integration per SDMDataSubset (i.e. per timestamp) ?
816 366 : if (sdmDataObject_.hasPackedData()) {
817 366 : processMIMESubscan();
818 : }
819 : else {
820 0 : processMIMEIntegrations();
821 : }
822 :
823 :
824 : // if (sdmDataObject_.isCorrelation()) {
825 : // // Process integrations.
826 : // processMIMEIntegrations();
827 : // }
828 : // else if (sdmDataObject_.isTP()){
829 : // // cout << "TP data" << endl;
830 : // if (sdmDataObject_.dimensionality() == 0) {
831 : // processMIMESubscan();
832 : // }
833 : // else {
834 : // processMIMEIntegrations();
835 : // }
836 : // }
837 : // else {
838 : // // cout << "Unrecognized type of binary data." << endl;
839 : // }
840 : //cout << "Exiting processMIME" << endl;
841 366 : }
842 :
843 366 : const SDMDataObject & SDMDataObjectReader::read(const char * buffer, unsigned long int size, bool fromFile) {
844 366 : if (!fromFile) read_ = MEMORY_;
845 :
846 : // Set up all sensitive pointers and sizes.
847 366 : data_ = (char *)buffer;
848 366 : dataSize_ = size;
849 366 : position_ = 0;
850 366 : endPosition_ = size;
851 :
852 : // And process the MIME message
853 366 : sdmDataObject_.valid_ = true;
854 366 : processMIME();
855 :
856 366 : sdmDataObject_.owns();
857 366 : return sdmDataObject_;
858 : }
859 :
860 0 : const SDMDataObject& SDMDataObjectReader::ref() const {
861 0 : if (read_ == UNKNOWN_)
862 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
863 0 : return sdmDataObject_;
864 : }
865 :
866 0 : const SDMDataObject* SDMDataObjectReader::ptr() const {
867 0 : if (read_ == UNKNOWN_)
868 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
869 0 : return &sdmDataObject_;
870 : }
871 :
872 452 : void SDMDataObjectReader::done() {
873 452 : switch (read_) {
874 86 : case UNKNOWN_:
875 : case MEMORY_:
876 86 : break;
877 366 : case FILE_:
878 366 : munmap((caddr_t) data_, dataSize_);
879 366 : close(filedes_);
880 366 : break;
881 : }
882 452 : sdmDataObject_.done();
883 452 : read_ = UNKNOWN_;
884 452 : }
885 :
886 0 : const SDMDataObject& SDMDataObjectReader::sdmDataObject() {
887 0 : return sdmDataObject_;
888 : }
889 :
890 :
891 366 : const SDMDataObject & SDMDataObjectReader::read(string filename) {
892 :
893 : struct stat fattr; // stat64 fattr;
894 :
895 : unsigned long int filesize;
896 366 : char* data = 0;
897 : // Open the file.
898 366 : errno = 0;
899 : #ifdef __APPLE__
900 : if ( (filedes_ = open(filename.c_str(), O_RDONLY )) == -1) {
901 : #else
902 366 : if ( (filedes_ = open(filename.c_str(), O_RDONLY | O_LARGEFILE)) == -1) {
903 : #endif
904 0 : string message(strerror(errno));
905 0 : throw SDMDataObjectReaderException("Could not open file '" + filename + "'. The message was '" + message + "'");
906 0 : }
907 366 : read_ = FILE_;
908 :
909 : // Get its size.
910 366 : errno = 0;
911 366 : int status = fstat(filedes_,&fattr); // fstat64(filedes_,&fattr);
912 366 : if (status == -1) {
913 0 : string message(strerror(errno));
914 0 : throw SDMDataObjectReaderException("Could not retrieve size of file '" + filename + "'. The message was '" + message + "'");
915 0 : }
916 366 : filesize = fattr.st_size;
917 :
918 : // Map it to virtual memory address space.
919 366 : errno = 0;
920 :
921 366 : data = (char *) mmap((caddr_t)0, filesize, PROT_READ, MAP_SHARED, filedes_, (off_t)0);
922 366 : if ( ((unsigned long) data) == 0xffffffff) {
923 0 : string message(strerror(errno));
924 0 : throw SDMDataObjectReaderException("Could not map file '" + filename + "' to memory. The message was '" + message + "'");
925 0 : }
926 :
927 : // cout << "Successfully mapped file." << endl;
928 : // And delegate to the other read (memory buffer) method.
929 732 : return read(data, filesize, true);
930 : }
931 :
932 : // SDMDataObjectReader::
933 :
934 : } // namespace asdmbinaries
935 :
|
