Line data Source code
1 : #include <alma/ASDMBinaries/SDMDataObjectReader.h>
2 :
3 : #include <iostream>
4 : #include <iomanip>
5 : #include <unistd.h>
6 : #include <sys/types.h>
7 : #include <sys/mman.h>
8 : #include <fcntl.h>
9 : #include <sys/stat.h>
10 : #include <errno.h>
11 : #include <cstring>
12 : #include <exception>
13 : #include <sstream>
14 :
15 : #include <alma/ASDMBinaries/CommonDefines.h>
16 :
17 : #if defined( __APPLE__ )
18 : #include <AvailabilityMacros.h>
19 : // 10.5 defines stat64 but Tiger does NOT
20 : #if defined(MAC_OS_X_VERSION_10_4) && ! defined(MAC_OS_X_VERSION_10_5)
21 : #define stat64 stat
22 : #define fstat64 fstat
23 : #endif
24 : #endif
25 :
26 : using namespace CorrelationModeMod;
27 : using namespace CorrelatorTypeMod;
28 : using namespace PrimitiveDataTypeMod;
29 : using namespace ProcessorTypeMod;
30 :
31 : using namespace std;
32 :
33 : namespace asdmbinaries {
34 :
35 : const string SDMDataObjectReader::MIMEBOUNDARY_1 = $MIMEBOUNDARY1;
36 : const string SDMDataObjectReader::MIMEBOUNDARY_2 = $MIMEBOUNDARY2;
37 :
38 : #ifndef WITHOUT_BOOST
39 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
40 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
41 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
42 : // const boost::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
43 : const boost::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
44 : const boost::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
45 : #else
46 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
47 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
48 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
49 : // const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
50 : const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
51 : const std::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
52 : #endif
53 : set<string> SDMDataObjectReader::BINATTACHNAMES;
54 : map<string, SDMDataObjectReader::BINATTACHCODES> SDMDataObjectReader::name2code;
55 : const bool SDMDataObjectReader::initClass_ = SDMDataObjectReader::initClass();
56 :
57 : // SDMDataObjectReader:: methods
58 : //
59 85 : SDMDataObjectReader::SDMDataObjectReader() {
60 85 : init();
61 85 : }
62 :
63 85 : SDMDataObjectReader::~SDMDataObjectReader() {
64 85 : done();
65 85 : }
66 :
67 3 : bool SDMDataObjectReader::initClass() {
68 3 : char* binAttachNamesC[] = {(char *) "actualDurations", (char *) "actualTimes", (char *) "autoData", (char *) "flags", (char *) "crossData", (char *) "zeroLags", 0};
69 21 : for (int i = 0; binAttachNamesC[i]; i++) {
70 18 : BINATTACHNAMES.insert(string(binAttachNamesC[i]));
71 : }
72 :
73 3 : name2code["actualDurations"] = ACTUALDURATIONS;
74 3 : name2code["actualTimes"] = ACTUALTIMES;
75 3 : name2code["autoData"] = AUTODATA;
76 3 : name2code["flags"] = FLAGS;
77 3 : name2code["crossData"] = CROSSDATA;
78 3 : name2code["zeroLags"] = ZEROLAGS;
79 :
80 3 : return true;
81 : }
82 :
83 85 : void SDMDataObjectReader::init() {
84 85 : filedes_ = 0;
85 85 : data_ = (char *) 0;
86 85 : position_ = 0;
87 85 : lastPosition_ = 0;
88 85 : endPosition_ = 0;
89 :
90 85 : read_ = UNKNOWN_;
91 :
92 85 : integrationNum_ = 0;
93 85 : }
94 :
95 20564 : string::size_type SDMDataObjectReader::find(const string& s) {
96 :
97 : while (true) {
98 1254303 : while ((position_ < endPosition_) && (data_[position_++] != s.at(0))) {;}
99 :
100 20564 : if (position_ == endPosition_) return string::npos;
101 :
102 20564 : string::size_type lastPosition_ = position_;
103 20564 : unsigned long long int i = 1;
104 :
105 65100 : while ((position_ < endPosition_) && (i < s.length())) {
106 61128 : if (s.at(i) != data_[position_]) break;
107 44536 : i++;
108 44536 : position_++;
109 : }
110 :
111 20564 : if (i == s.length()) return (position_ - s.length());
112 :
113 16592 : if (position_ == endPosition_) return string::npos;
114 :
115 16592 : position_ = lastPosition_; //+1;
116 16592 : }
117 : }
118 :
119 1437 : bool SDMDataObjectReader::compare(const string& s) {
120 1437 : string::size_type index = position_;
121 :
122 1437 : unsigned int i = 0;
123 9489 : while ((index < endPosition_) && (i < s.length()) && (s.at(i++) == data_[index++])) {
124 : ;
125 : }
126 1437 : return (i == s.length());
127 : }
128 :
129 1437 : bool SDMDataObjectReader::EOD() {
130 1437 : return position_ >= endPosition_;
131 : }
132 :
133 :
134 732 : string SDMDataObjectReader::extractXMLHeader(const string& boundary) {
135 : // cout << "Entering extractXMLHeader" << endl;
136 :
137 732 : string boundary_ = "\n--"+boundary;
138 :
139 732 : unsigned long int positionBeg = 0;
140 732 : unsigned long int positionEnd = 0;
141 :
142 : // We assume that we are at the first character of the body.
143 732 : positionBeg = position_;
144 :
145 : // Look for the next boundary supposed to be found right after an the XML header.
146 732 : if ( (positionEnd = find(boundary_)) == string::npos) {
147 0 : ostringstream eos;
148 0 : eos << "A second MIME boundary '" << boundary <<"' could not be found in the MIME message after position " << positionBeg << ".";
149 0 : throw SDMDataObjectReaderException(eos.str());
150 0 : }
151 :
152 732 : string xmlPart = string(data_+positionBeg, positionEnd-positionBeg);
153 :
154 : // cout << "Found XML Header : " << xmlPart << endl;
155 : // cout << "Exiting extractXMLHeader" << endl;
156 :
157 1464 : return xmlPart;
158 732 : }
159 :
160 :
161 3240 : void SDMDataObjectReader::tokenize(const string& str,
162 : vector<string>& tokens,
163 : const string& delimiters) {
164 : // Skip delimiters at beginning.
165 3240 : string::size_type lastPos = str.find_first_not_of(delimiters, 0);
166 : // Find first "non-delimiter".
167 3240 : string::size_type pos = str.find_first_of(delimiters, lastPos);
168 :
169 9720 : while (string::npos != pos || string::npos != lastPos) {
170 : // Found a token, add it to the vector.
171 6480 : tokens.push_back(str.substr(lastPos, pos - lastPos));
172 : // Skip delimiters. Note the "not_of"
173 6480 : lastPos = str.find_first_not_of(delimiters, pos);
174 : // Find next "non-delimiter"
175 6480 : pos = str.find_first_of(delimiters, lastPos);
176 : }
177 3240 : }
178 :
179 :
180 1437 : void SDMDataObjectReader::getFields(const string& header, map<string, string>& fields) {
181 : // We are at the last character of the line preceding the 1st field declaration.
182 1437 : istringstream iss;
183 1437 : iss.str(header);
184 1437 : string line;
185 :
186 1437 : getline(iss, line); // skip an nl
187 :
188 : // Split each line into field-name, field-value
189 4677 : while (getline(iss, line)) {
190 3240 : vector<string> nv;
191 3240 : tokenize(line, nv, ":");
192 3240 : if (nv.size() < 2) {
193 0 : ostringstream oss;
194 0 : oss << "Invalid field in a MIME header '" << header << "', integration #" << integrationNum_;
195 0 : throw SDMDataObjectReaderException (oss.str());
196 0 : }
197 3240 : fields[nv.at(0)] = nv.at(1);
198 3240 : }
199 1437 : }
200 :
201 0 : string SDMDataObjectReader::getContentID() {
202 : // We are at the beginning of an attachment right after its opening boundary.
203 0 : string::size_type lastPosition = position_;
204 :
205 : // Now let's extract the header (which is followed by two successive nl nl)
206 : string::size_type endHeader;
207 0 : if ( (endHeader = find ("\n\n")) == string::npos) {
208 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
209 : }
210 :
211 0 : string header(data_+lastPosition, (endHeader - lastPosition));
212 :
213 0 : map<string, string> fields;
214 0 : getFields(header, fields);
215 :
216 : // Look for the Content-ID field
217 0 : map<string, string>::iterator iter = fields.find("Content-ID");
218 0 : if (iter == fields.end()) {
219 0 : ostringstream oss;
220 0 : oss << "'Content-ID' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
221 0 : throw SDMDataObjectReaderException(oss.str());
222 0 : }
223 0 : return iter->second;
224 0 : }
225 :
226 1437 : string SDMDataObjectReader::getContentLocation() {
227 : // We are at the beginning of an attachment right after its opening boundary.
228 1437 : string::size_type lastPosition = position_;
229 :
230 : // Now let's extract the header (which is followed by two successive nl nl)
231 : string::size_type endHeader;
232 1437 : if ( (endHeader = find ("\n\n")) == string::npos) {
233 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
234 : }
235 :
236 1437 : string header(data_+lastPosition, (endHeader - lastPosition));
237 :
238 1437 : map<string, string> fields;
239 1437 : getFields(header, fields);
240 :
241 : // Look for the Content-Location field
242 1437 : map<string, string>::iterator iter = fields.find("Content-Location");
243 1437 : if (iter == fields.end()) {
244 0 : ostringstream oss;
245 0 : oss << "'Content-Location' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
246 0 : throw SDMDataObjectReaderException(oss.str());
247 0 : }
248 2874 : return iter->second;
249 1437 : }
250 :
251 705 : void SDMDataObjectReader::processBinaryAttachment(const string& boundary, const SDMDataSubset& sdmDataSubset) {
252 :
253 705 : string contentLocation = getContentLocation();
254 :
255 : // Extract the attachment name and the integration number.
256 : #ifndef WITHOUT_BOOST
257 : boost::cmatch what;
258 : boost::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
259 : if (boost::regex_match(contentLocation.c_str(), what, r)) {
260 : }
261 : #else
262 705 : std::cmatch what;
263 1410 : std::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
264 705 : if (std::regex_match(contentLocation.c_str(), what, r)) {
265 : }
266 : #endif
267 : else {
268 0 : ostringstream oss;
269 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of an attachment (approx. char position = " << position_ << ").";
270 0 : throw SDMDataObjectReaderException(oss.str());
271 0 : }
272 :
273 : // We are at the first byte of the body of the binary attachment.
274 705 : string::size_type startAttachPosition = position_; // mcaillat
275 :
276 : // Now look for its closing boundary (or opening boundary of the next one if any).
277 705 : string boundary_ = "\n--"+boundary;
278 : string::size_type endAttachPosition;
279 705 : if ( (endAttachPosition = find(boundary_)) == string::npos) {
280 0 : ostringstream eos;
281 0 : eos << "A MIME boundary '" << boundary << "' terminating a binary attachment starting approx. at character " << startAttachPosition <<" could not be found.";
282 0 : throw SDMDataObjectReaderException(eos.str());
283 0 : }
284 :
285 : // Compute the length of the attachment.
286 705 : string::size_type length = endAttachPosition - startAttachPosition;
287 :
288 :
289 :
290 705 : switch (name2code[what[1]]) {
291 0 : case ACTUALDURATIONS:
292 0 : actualDurations_ = (ACTUALDURATIONSTYPE *) (data_ + startAttachPosition);
293 0 : nActualDurations_ = length / sizeof (ACTUALDURATIONSTYPE);
294 0 : attachmentFlags.set(ACTUALDURATIONS);
295 : //cout << "actualDurations =" << (unsigned long int) actualDurations_ << ", nActualDurations = " << nActualDurations_ << endl;
296 0 : break;
297 0 : case ACTUALTIMES:
298 0 : actualTimes_ = (ACTUALTIMESTYPE *) (data_ + startAttachPosition);
299 0 : nActualTimes_ = length / sizeof( ACTUALTIMESTYPE);
300 0 : attachmentFlags.set(ACTUALTIMES);
301 : //cout << "actualTimes =" << (unsigned long int) actualTimes_ << ", nActualTimes = " << nActualTimes_ << endl;
302 0 : break;
303 366 : case AUTODATA:
304 366 : autoData_ = (AUTODATATYPE *) (data_ + startAttachPosition);
305 366 : nAutoData_ = length / sizeof(AUTODATATYPE);
306 366 : attachmentFlags.set(AUTODATA);
307 : //cout << "autoData =" << (unsigned long int) autoData_ << ", nAutoData = " << nAutoData_ << endl;
308 366 : break;
309 339 : case FLAGS:
310 339 : flags_ = (const FLAGSTYPE *) (data_ + startAttachPosition);
311 339 : nFlags_ = length / sizeof(FLAGSTYPE);
312 339 : attachmentFlags.set(FLAGS);
313 : //cout << "flags =" << (unsigned long int) flags_ << ", nFlags = " << nFlags_ << endl;
314 339 : break;
315 0 : case CROSSDATA:
316 0 : shortCrossData_ = 0;
317 0 : longCrossData_ = 0;
318 0 : floatCrossData_ = 0;
319 0 : switch (sdmDataSubset.crossDataType()) {
320 0 : case INT16_TYPE:
321 0 : shortCrossData_ = (const SHORTCROSSDATATYPE*) (data_ + startAttachPosition);
322 0 : nCrossData_ = length / sizeof (SHORTCROSSDATATYPE);
323 : //cout << "shortCrossData = " << (unsigned long int) shortCrossData_ << ", nShortCrossData = " << nCrossData_ << endl;
324 0 : break;
325 :
326 0 : case INT32_TYPE:
327 0 : longCrossData_ = (const INTCROSSDATATYPE*) (data_ + startAttachPosition);
328 0 : nCrossData_ = length / sizeof (INTCROSSDATATYPE);
329 : //cout << "longCrossData = " << (unsigned long int) longCrossData_ << ", nLongCrossData = " << nCrossData_ << endl;
330 0 : break;
331 :
332 0 : case FLOAT32_TYPE:
333 0 : floatCrossData_ = (const FLOATCROSSDATATYPE*) (data_ + startAttachPosition);
334 0 : nCrossData_ = length / sizeof (FLOATCROSSDATATYPE);
335 0 : break;
336 :
337 0 : default:
338 0 : ostringstream eos;
339 0 : eos << "'" << CPrimitiveDataType::toString(sdmDataSubset.crossDataType())
340 0 : << "' is not a valid primitive data type for CROSSDATA";
341 0 : throw SDMDataObjectReaderException(eos.str());
342 : // cout << "Unrecognized type for cross data" << endl;
343 : }
344 0 : attachmentFlags.set(CROSSDATA);
345 0 : break;
346 0 : case ZEROLAGS:
347 0 : zeroLags_ = (ZEROLAGSTYPE *) (data_ + startAttachPosition);
348 0 : nZeroLags_ = length / sizeof(ZEROLAGSTYPE) ;
349 0 : attachmentFlags.set(ZEROLAGS);
350 : //cout << "zeroLags =" << (unsigned long int) zeroLags_ << ", nZeroLags = " << nZeroLags_ << endl;
351 0 : break;
352 :
353 0 : default:
354 : //cout << "Unrecognized code" << endl;
355 0 : break;
356 : }
357 : // cout << "found a binary attachment" << endl;
358 705 : }
359 :
360 366 : void SDMDataObjectReader::processMIMESDMDataHeader() {
361 : // cout << "Entering processMIMESDMDataHeader" << endl;
362 366 : string sdmDataHeader = extractXMLHeader(MIMEBOUNDARY_1);
363 366 : parser_.parseMemoryHeader(sdmDataHeader, sdmDataObject_);
364 : // cout << "Exiting processMIMESDMDataHeader" << endl;
365 366 : }
366 :
367 366 : void SDMDataObjectReader::processMIMESDMDataSubsetHeader(SDMDataSubset& sdmDataSubset) {
368 : // cout << "Entering processMIMESDMDataSubsetHeader" << endl;
369 366 : string sdmDataSubsetHeader = extractXMLHeader(MIMEBOUNDARY_2);
370 366 : if (sdmDataSubset.owner_->isCorrelation())
371 0 : parser_.parseMemoryCorrSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
372 : else
373 366 : parser_.parseMemoryTPSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
374 : // cout << "Exiting processMIMESDMDataSubsetHeader" << endl;
375 366 : }
376 :
377 0 : void SDMDataObjectReader::processMIMEIntegration() {
378 : // cout << "Entering processMIMEIntegration" << endl;
379 :
380 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
381 0 : string contentLocation = getContentLocation();
382 :
383 : #ifndef WITHOUT_BOOST
384 : boost::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
385 : ostringstream oss;
386 :
387 : // Extract the Subset name and the integration [, subintegration] number.
388 : boost::cmatch what;
389 : if (!boost::regex_match(contentLocation.c_str(), what, r)) {
390 : ostringstream oss;
391 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
392 : throw SDMDataObjectReaderException(oss.str());
393 : }
394 : #else
395 0 : std::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
396 0 : ostringstream oss;
397 :
398 : // Extract the Subset name and the integration [, subintegration] number.
399 0 : std::cmatch what;
400 0 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
401 0 : ostringstream oss;
402 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
403 0 : throw SDMDataObjectReaderException(oss.str());
404 0 : }
405 : #endif
406 :
407 0 : SDMDataSubset integration(&sdmDataObject_);
408 :
409 : // The SDMDataSubset header.
410 0 : processMIMESDMDataSubsetHeader(integration);
411 :
412 0 : if (integration.aborted_) {
413 : // The [sub]integration has been aborted. Just append its header, without trying to get binary attachment.
414 0 : sdmDataObject_.append(integration);
415 : }
416 : else {
417 : // This is regular [sub]integration, process its binary attachments.
418 0 : attachmentFlags.reset();
419 0 : while (!EOD() && !compare("--")) {
420 0 : processBinaryAttachment(MIMEBOUNDARY_2, integration);
421 : }
422 :
423 0 : if (EOD())
424 0 : throw SDMDataObjectReaderException("Unexpected end of data");
425 :
426 : // Now check if the binary attachments found are compatible with the correlation mode
427 : // and if their sizes are equal to what is announced in the global header.
428 :
429 0 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
430 : // No actualdurations attachment found, ok then set everything to 0.
431 0 : integration.actualDurations_ = 0;
432 0 : integration.nActualDurations_ = 0;
433 : }
434 : else {
435 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
436 0 : ostringstream oss;
437 0 : oss << "Data subset '"<<contentLocation<<"': ";
438 0 : oss << "zize of 'actualDuration' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
439 0 : throw SDMDataObjectReaderException(oss.str());
440 0 : }
441 :
442 0 : integration.actualDurations_ = actualDurations_;
443 0 : integration.nActualDurations_ = nActualDurations_;
444 : }
445 :
446 0 : if (!attachmentFlags.test(ACTUALTIMES)) {
447 : // No actualtimes attachment found, ok then set everything to 0.
448 0 : integration.actualTimes_ = 0;
449 0 : integration.nActualTimes_ = 0;
450 : }
451 : else {
452 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
453 0 : ostringstream oss;
454 0 : oss << "Data subset '"<<contentLocation<<"': ";
455 0 : oss << "size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
456 0 : throw SDMDataObjectReaderException(oss.str());
457 0 : }
458 0 : integration.actualTimes_ = actualTimes_;
459 0 : integration.nActualTimes_ = nActualTimes_;
460 : }
461 :
462 : // The flags are optional. They may be absent.
463 0 : if (!attachmentFlags.test(FLAGS)) {
464 : // No flags binary attachment found, ok then set everything to 0.
465 0 : integration.flags_ = 0;
466 0 : integration.nFlags_ = 0;
467 : }
468 : else {
469 : // flags found
470 : // Check size conformity
471 0 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
472 0 : ostringstream oss;
473 0 : oss << "Data subset '"<<contentLocation<<"': ";
474 0 : oss << "size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
475 0 : throw SDMDataObjectReaderException(oss.str());
476 0 : }
477 :
478 : // Set pointer and size
479 0 : integration.flags_ = flags_;
480 0 : integration.nFlags_ = nFlags_;
481 : }
482 :
483 :
484 : //
485 : // The presence of crossData and autoData depends on the correlation mode.
486 :
487 : // crossDataTypeSize is not currently used
488 : // unsigned int crossDataTypeSize;
489 0 : switch (sdmDataObject_.correlationMode()) {
490 0 : case CROSS_ONLY:
491 0 : if (!attachmentFlags.test(CROSSDATA)) {
492 0 : ostringstream oss;
493 0 : oss << "Data subset '"<<contentLocation<<"': ";
494 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationNum_;
495 0 : throw SDMDataObjectReaderException(oss.str());
496 0 : }
497 :
498 :
499 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
500 0 : ostringstream oss;
501 0 : oss << "Data subset '"<<contentLocation<<"': ";
502 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
503 0 : throw SDMDataObjectReaderException(oss.str());
504 0 : }
505 :
506 :
507 0 : if (attachmentFlags.test(AUTODATA)) {
508 0 : ostringstream oss;
509 0 : oss << "Data subset '"<<contentLocation<<"': ";
510 0 : oss << "found an unexpected attachment 'autoData' in integration #" << integrationNum_ << ".";
511 0 : throw SDMDataObjectReaderException(oss.str());
512 0 : }
513 :
514 0 : integration.shortCrossData_ = 0;
515 0 : integration.longCrossData_ = 0;
516 0 : integration.floatCrossData_ = 0;
517 0 : integration.nCrossData_ = 0;
518 0 : switch (integration.crossDataType()) {
519 0 : case INT32_TYPE:
520 0 : integration.longCrossData_ = longCrossData_;
521 0 : integration.nCrossData_ = nCrossData_;
522 0 : break;
523 :
524 0 : case INT16_TYPE:
525 0 : integration.shortCrossData_ = shortCrossData_;
526 0 : integration.nCrossData_ = nCrossData_;
527 0 : break;
528 :
529 0 : case FLOAT32_TYPE:
530 0 : integration.floatCrossData_ = floatCrossData_;
531 0 : integration.nCrossData_ = nCrossData_;
532 0 : break;
533 :
534 0 : default:
535 0 : throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
536 : }
537 0 : break;
538 :
539 0 : case AUTO_ONLY:
540 0 : if (!attachmentFlags.test(AUTODATA)) {
541 0 : ostringstream oss;
542 0 : oss << "Data subset '"<<contentLocation<<"': ";
543 0 : oss << "a binary attachment 'autoData' was expected.";
544 0 : throw SDMDataObjectReaderException(oss.str());
545 0 : }
546 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
547 0 : ostringstream oss;
548 0 : oss << "Data subset '"<<contentLocation<<"': ";
549 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
550 0 : throw SDMDataObjectReaderException(oss.str());
551 0 : }
552 :
553 :
554 0 : if (attachmentFlags.test(CROSSDATA)) {
555 0 : ostringstream oss;
556 0 : oss << "Data subset '"<<contentLocation<<"': ";
557 0 : oss << "found an unexpected attachment 'crossData'";
558 0 : throw SDMDataObjectReaderException(oss.str());
559 0 : }
560 :
561 0 : integration.autoData_ = autoData_;
562 0 : integration.nAutoData_ = nAutoData_;
563 :
564 0 : break;
565 :
566 0 : case CROSS_AND_AUTO:
567 0 : if (!attachmentFlags.test(AUTODATA)) {
568 0 : ostringstream oss;
569 0 : oss << "Data subset '"<<contentLocation<<"': ";
570 0 : oss << "a binary attachment 'autoData' was expected.";
571 0 : throw SDMDataObjectReaderException(oss.str());
572 0 : }
573 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
574 0 : ostringstream oss;
575 0 : oss << "Data subset '"<<contentLocation<<"': ";
576 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
577 0 : throw SDMDataObjectReaderException(oss.str());
578 0 : }
579 :
580 0 : if (!attachmentFlags.test(CROSSDATA)) {
581 0 : ostringstream oss;
582 0 : oss << "Data subset '"<<contentLocation<<"': ";
583 0 : oss << "a binary attachment 'crossData' was expected.";
584 0 : throw SDMDataObjectReaderException(oss.str());
585 0 : }
586 :
587 : // crossDataTypeSize is not currently used, skip this block
588 : // switch (integration.crossDataType()) {
589 : // case INT32_TYPE:
590 : // crossDataTypeSize = sizeof(INTCROSSDATATYPE);
591 : // break;
592 : // case INT16_TYPE:
593 : // crossDataTypeSize = sizeof(SHORTCROSSDATATYPE);
594 : // break;
595 : // case FLOAT32_TYPE:
596 : // crossDataTypeSize = sizeof(FLOATCROSSDATATYPE);
597 : // break;
598 : // default:
599 : // throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
600 : // }
601 :
602 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
603 0 : ostringstream oss;
604 0 : oss << "Data subset '"<<contentLocation<<"': ";
605 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
606 0 : throw SDMDataObjectReaderException(oss.str());
607 0 : }
608 :
609 :
610 0 : integration.shortCrossData_ = 0;
611 0 : integration.longCrossData_ = 0;
612 0 : integration.floatCrossData_ = 0;
613 0 : integration.nCrossData_ = 0;
614 0 : switch (integration.crossDataType()) {
615 0 : case INT32_TYPE:
616 0 : integration.longCrossData_ = longCrossData_;
617 0 : integration.nCrossData_ = nCrossData_;
618 0 : break;
619 :
620 0 : case INT16_TYPE:
621 0 : integration.shortCrossData_ = shortCrossData_;
622 0 : integration.nCrossData_ = nCrossData_;
623 0 : break;
624 :
625 0 : case FLOAT32_TYPE:
626 0 : integration.floatCrossData_ = floatCrossData_;
627 0 : integration.nCrossData_ = nCrossData_;
628 0 : break;
629 :
630 0 : default:
631 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': '"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
632 : }
633 :
634 0 : integration.autoData_ = autoData_;
635 0 : integration.nAutoData_ = nAutoData_;
636 :
637 :
638 0 : break;
639 0 : default:
640 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': unrecognized correlation mode");
641 : break;
642 : }
643 :
644 :
645 0 : if (attachmentFlags.test(ZEROLAGS)) {
646 : // Refuse the zeroLags attachment if it's not a Correlator or if the correlator is a CORRELATOR_FX (ACA).
647 0 : if ((sdmDataObject_.processorType_ != CORRELATOR) || (sdmDataObject_.correlatorType() == FX))
648 0 : throw SDMDataObjectReaderException("zeroLags are not expected from a correlator CORRELATOR_FX");
649 :
650 0 : if (nZeroLags_ != sdmDataObject_.dataStruct().zeroLags().size()) {
651 0 : ostringstream oss;
652 0 : oss << "Data subset '"<<contentLocation<<"': ";
653 0 : oss << "size of 'zeroLags' attachment (" << nZeroLags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().zeroLags().size() << ")";
654 0 : throw SDMDataObjectReaderException(oss.str());
655 0 : }
656 0 : integration.zeroLags_ = zeroLags_;
657 0 : integration.nZeroLags_ = nZeroLags_;
658 : }
659 :
660 0 : sdmDataObject_.append(integration);
661 : }
662 :
663 0 : if (!compare("--\n--"+MIMEBOUNDARY_1))
664 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
665 :
666 0 : find("--\n--"+MIMEBOUNDARY_1);
667 : // cout << "Exiting processMIMEIntegration" << endl;
668 0 : }
669 :
670 0 : void SDMDataObjectReader::processMIMEIntegrations() {
671 : // cout << "Entering processMIMEIntegrations" << endl;
672 :
673 0 : while (!compare("--")) {
674 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
675 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
676 : // and move to the next --<MIMEBOUNDARY_2>
677 0 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
678 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
679 :
680 : //Process the next integration.
681 0 : integrationNum_++;
682 0 : processMIMEIntegration() ;
683 : }
684 :
685 : // cout << "Exiting processMIMEIntegrations" << endl;
686 0 : }
687 :
688 366 : void SDMDataObjectReader::processMIMESubscan() {
689 : // cout << "Entering processMIMESubscan" << endl;
690 :
691 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
692 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
693 : // and move to the next --<MIMEBOUNDARY_2>
694 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
695 366 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
696 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
697 :
698 :
699 366 : string contentLocation = getContentLocation();
700 : // cout << contentLocation << endl;
701 : #ifndef WITHOUT_BOOST
702 : boost::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
703 :
704 : // Extract the Subset name and the suffix number (which must be equal to 1).
705 : boost::cmatch what;
706 : if (!boost::regex_match(contentLocation.c_str(), what, r)) {
707 : ostringstream oss;
708 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
709 : throw SDMDataObjectReaderException(oss.str());
710 : }
711 : #else
712 732 : std::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
713 :
714 : // Extract the Subset name and the suffix number (which must be equal to 1).
715 366 : std::cmatch what;
716 366 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
717 0 : ostringstream oss;
718 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
719 0 : throw SDMDataObjectReaderException(oss.str());
720 0 : }
721 : #endif
722 :
723 366 : SDMDataSubset subscan(&sdmDataObject_);
724 :
725 : // The SDMDataSubset header.
726 366 : processMIMESDMDataSubsetHeader(subscan);
727 :
728 :
729 : // Process the binary attachments.
730 366 : attachmentFlags.reset();
731 366 : integrationNum_++;
732 1071 : while(!EOD() && !compare("--")) {
733 705 : processBinaryAttachment(MIMEBOUNDARY_2, subscan);
734 : }
735 :
736 366 : if (EOD()) throw SDMDataObjectReaderException("Unexpected end of data");
737 :
738 : // Now check if the binary attachments found are compatible with the correlation mode.
739 : // and if their sizes are equal to what is announced in the global header.
740 :
741 : // Start with the only mandatory attachment : autoData.
742 366 : if (!attachmentFlags.test(AUTODATA)) {
743 0 : ostringstream oss;
744 0 : oss << "Binary attachment 'autoData' was expected in integration #" << integrationNum_;
745 0 : throw SDMDataObjectReaderException(oss.str());
746 0 : }
747 366 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
748 0 : ostringstream oss;
749 0 : oss << "Size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
750 0 : throw SDMDataObjectReaderException(oss.str());
751 0 : }
752 :
753 366 : subscan.autoData_ = autoData_;
754 366 : subscan.nAutoData_ = nAutoData_;
755 :
756 :
757 : // And now consider the optional attachments.
758 :
759 : // The actualDurations are optional. They may be absent.
760 366 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
761 : // No actualDurations binary attachment found, ok then set everything to 0.
762 366 : subscan.actualDurations_ = 0;
763 366 : subscan.nActualDurations_ = 0;
764 : }
765 : else {
766 : // actualDurations found
767 : // Check size conformity
768 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
769 0 : ostringstream oss;
770 0 : oss << "Size of 'actualDurations' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
771 0 : throw SDMDataObjectReaderException(oss.str());
772 0 : }
773 :
774 : // Set pointer and size
775 0 : subscan.actualDurations_ = actualDurations_;
776 0 : subscan.nActualDurations_ = nActualDurations_;
777 : }
778 :
779 :
780 : // The actualTimes are optional. They may be absent.
781 366 : if (!attachmentFlags.test(ACTUALTIMES)) {
782 : // No actualTimes binary attachment found, ok then set everything to 0.
783 366 : subscan.actualTimes_ = 0;
784 366 : subscan.nActualTimes_ = 0;
785 : }
786 : else {
787 : // actualTimes found
788 : // Check size conformity
789 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
790 0 : ostringstream oss;
791 0 : oss << "Size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
792 0 : throw SDMDataObjectReaderException(oss.str());
793 0 : }
794 :
795 : // Set pointer and size
796 0 : subscan.actualTimes_ = actualTimes_;
797 0 : subscan.nActualTimes_ = nActualTimes_;
798 : }
799 :
800 :
801 : // The flags are optional. They may be absent.
802 366 : if (!attachmentFlags.test(FLAGS)) {
803 : // No flags binary attachment found, ok then set everything to 0.
804 27 : subscan.flags_ = 0;
805 27 : subscan.nFlags_ = 0;
806 : }
807 : else {
808 : // flags found
809 : // Check size conformity
810 339 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
811 0 : ostringstream oss;
812 0 : oss << "Size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
813 0 : throw SDMDataObjectReaderException(oss.str());
814 0 : }
815 :
816 : // Set pointer and size
817 339 : subscan.flags_ = flags_;
818 339 : subscan.nFlags_ = nFlags_;
819 : }
820 :
821 : // And finally let's check that no unexpected binary attachment was found.
822 366 : if (attachmentFlags.test(CROSSDATA)) {
823 0 : ostringstream oss;
824 0 : oss << "Found an unexpected attachment 'crossData' in the binary attachments.";
825 0 : throw SDMDataObjectReaderException(oss.str());
826 0 : }
827 :
828 :
829 366 : if (attachmentFlags.test(ZEROLAGS)) {
830 0 : ostringstream oss;
831 0 : oss << "Found an unexpected attachment 'zeroLags' in the binary attachments.";
832 0 : throw SDMDataObjectReaderException(oss.str());
833 0 : }
834 :
835 366 : sdmDataObject_.tpDataSubset(subscan);
836 :
837 :
838 366 : if (!compare("--\n--"+MIMEBOUNDARY_1))
839 0 : throw SDMDataObjectReaderException("Expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
840 :
841 366 : find("--\n--"+MIMEBOUNDARY_1);
842 366 : }
843 :
844 366 : void SDMDataObjectReader::processMIME() {
845 : // cout << "Entering processMIME" << endl;
846 : // Let's find the opening boundary
847 366 : if (find("--"+MIMEBOUNDARY_1) == string::npos)
848 0 : throw SDMDataObjectReaderException("Expecting a first boundary '--"+MIMEBOUNDARY_1+"'.");
849 :
850 : // Check the Content-Location of the MIME header ...but do nothing special with it...
851 366 : string contentLocation=getContentLocation();
852 :
853 : // Detect SDMDataHeader.
854 366 : processMIMESDMDataHeader();
855 :
856 : // Do we have packed data (i.e. only one SDMDataSubset grouping all the integrations) or one integration per SDMDataSubset (i.e. per timestamp) ?
857 366 : if (sdmDataObject_.hasPackedData()) {
858 366 : processMIMESubscan();
859 : }
860 : else {
861 0 : processMIMEIntegrations();
862 : }
863 :
864 :
865 : // if (sdmDataObject_.isCorrelation()) {
866 : // // Process integrations.
867 : // processMIMEIntegrations();
868 : // }
869 : // else if (sdmDataObject_.isTP()){
870 : // // cout << "TP data" << endl;
871 : // if (sdmDataObject_.dimensionality() == 0) {
872 : // processMIMESubscan();
873 : // }
874 : // else {
875 : // processMIMEIntegrations();
876 : // }
877 : // }
878 : // else {
879 : // // cout << "Unrecognized type of binary data." << endl;
880 : // }
881 : //cout << "Exiting processMIME" << endl;
882 366 : }
883 :
884 366 : const SDMDataObject & SDMDataObjectReader::read(const char * buffer, unsigned long int size, bool fromFile) {
885 366 : if (!fromFile) read_ = MEMORY_;
886 :
887 : // Set up all sensitive pointers and sizes.
888 366 : data_ = (char *)buffer;
889 366 : dataSize_ = size;
890 366 : position_ = 0;
891 366 : endPosition_ = size;
892 :
893 : // And process the MIME message
894 366 : sdmDataObject_.valid_ = true;
895 366 : processMIME();
896 :
897 366 : sdmDataObject_.owns();
898 366 : return sdmDataObject_;
899 : }
900 :
901 0 : const SDMDataObject& SDMDataObjectReader::ref() const {
902 0 : if (read_ == UNKNOWN_)
903 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
904 0 : return sdmDataObject_;
905 : }
906 :
907 0 : const SDMDataObject* SDMDataObjectReader::ptr() const {
908 0 : if (read_ == UNKNOWN_)
909 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
910 0 : return &sdmDataObject_;
911 : }
912 :
913 451 : void SDMDataObjectReader::done() {
914 451 : switch (read_) {
915 85 : case UNKNOWN_:
916 : case MEMORY_:
917 85 : break;
918 366 : case FILE_:
919 366 : munmap((caddr_t) data_, dataSize_);
920 366 : close(filedes_);
921 366 : break;
922 : }
923 451 : sdmDataObject_.done();
924 451 : read_ = UNKNOWN_;
925 451 : }
926 :
927 0 : const SDMDataObject& SDMDataObjectReader::sdmDataObject() {
928 0 : return sdmDataObject_;
929 : }
930 :
931 :
932 366 : const SDMDataObject & SDMDataObjectReader::read(string filename) {
933 :
934 : struct stat fattr; // stat64 fattr;
935 :
936 : unsigned long int filesize;
937 366 : char* data = 0;
938 : // Open the file.
939 366 : errno = 0;
940 : #ifdef __APPLE__
941 : if ( (filedes_ = open(filename.c_str(), O_RDONLY )) == -1) {
942 : #else
943 366 : if ( (filedes_ = open(filename.c_str(), O_RDONLY | O_LARGEFILE)) == -1) {
944 : #endif
945 0 : string message(strerror(errno));
946 0 : throw SDMDataObjectReaderException("Could not open file '" + filename + "'. The message was '" + message + "'");
947 0 : }
948 366 : read_ = FILE_;
949 :
950 : // Get its size.
951 366 : errno = 0;
952 366 : int status = fstat(filedes_,&fattr); // fstat64(filedes_,&fattr);
953 366 : if (status == -1) {
954 0 : string message(strerror(errno));
955 0 : throw SDMDataObjectReaderException("Could not retrieve size of file '" + filename + "'. The message was '" + message + "'");
956 0 : }
957 366 : filesize = fattr.st_size;
958 :
959 : // Map it to virtual memory address space.
960 366 : errno = 0;
961 :
962 366 : data = (char *) mmap((caddr_t)0, filesize, PROT_READ, MAP_SHARED, filedes_, (off_t)0);
963 366 : if ( ((unsigned long) data) == 0xffffffff) {
964 0 : string message(strerror(errno));
965 0 : throw SDMDataObjectReaderException("Could not map file '" + filename + "' to memory. The message was '" + message + "'");
966 0 : }
967 :
968 : // cout << "Successfully mapped file." << endl;
969 : // And delegate to the other read (memory buffer) method.
970 732 : return read(data, filesize, true);
971 : }
972 :
973 : // SDMDataObjectReader::
974 :
975 : } // namespace asdmbinaries
976 :
|
