Line data Source code
1 : #include <alma/ASDMBinaries/SDMDataObjectReader.h>
2 :
3 : #include <iostream>
4 : #include <iomanip>
5 : #include <unistd.h>
6 : #include <sys/types.h>
7 : #include <sys/mman.h>
8 : #include <fcntl.h>
9 : #include <sys/stat.h>
10 : #include <errno.h>
11 : #include <cstring>
12 : #include <exception>
13 : #include <sstream>
14 : #include <cstring> //strerror
15 :
16 : #include <alma/ASDMBinaries/CommonDefines.h>
17 :
18 : #if defined( __APPLE__ )
19 : #include <AvailabilityMacros.h>
20 : // 10.5 defines stat64 but Tiger does NOT
21 : #if defined(MAC_OS_X_VERSION_10_4) && ! defined(MAC_OS_X_VERSION_10_5)
22 : #define stat64 stat
23 : #define fstat64 fstat
24 : #endif
25 : #endif
26 :
27 : using namespace CorrelationModeMod;
28 : using namespace CorrelatorTypeMod;
29 : using namespace PrimitiveDataTypeMod;
30 : using namespace ProcessorTypeMod;
31 :
32 : using namespace std;
33 :
34 : namespace asdmbinaries {
35 :
36 : const string SDMDataObjectReader::MIMEBOUNDARY_1 = $MIMEBOUNDARY1;
37 : const string SDMDataObjectReader::MIMEBOUNDARY_2 = $MIMEBOUNDARY2;
38 :
39 : #ifndef WITHOUT_BOOST
40 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
41 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
42 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
43 : // const boost::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
44 : const boost::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
45 : const boost::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
46 : #else
47 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
48 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
49 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
50 : // const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
51 : const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
52 : const std::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
53 : #endif
54 : set<string> SDMDataObjectReader::BINATTACHNAMES;
55 : map<string, SDMDataObjectReader::BINATTACHCODES> SDMDataObjectReader::name2code;
56 : const bool SDMDataObjectReader::initClass_ = SDMDataObjectReader::initClass();
57 :
58 : // SDMDataObjectReader:: methods
59 : //
60 0 : SDMDataObjectReader::SDMDataObjectReader() {
61 0 : init();
62 0 : }
63 :
64 0 : SDMDataObjectReader::~SDMDataObjectReader() {
65 0 : done();
66 0 : }
67 :
68 11 : bool SDMDataObjectReader::initClass() {
69 11 : char* binAttachNamesC[] = {(char *) "actualDurations", (char *) "actualTimes", (char *) "autoData", (char *) "flags", (char *) "crossData", (char *) "zeroLags", 0};
70 77 : for (int i = 0; binAttachNamesC[i]; i++) {
71 66 : BINATTACHNAMES.insert(string(binAttachNamesC[i]));
72 : }
73 :
74 11 : name2code["actualDurations"] = ACTUALDURATIONS;
75 11 : name2code["actualTimes"] = ACTUALTIMES;
76 11 : name2code["autoData"] = AUTODATA;
77 11 : name2code["flags"] = FLAGS;
78 11 : name2code["crossData"] = CROSSDATA;
79 11 : name2code["zeroLags"] = ZEROLAGS;
80 :
81 11 : return true;
82 : }
83 :
84 0 : void SDMDataObjectReader::init() {
85 0 : filedes_ = 0;
86 0 : data_ = (char *) 0;
87 0 : position_ = 0;
88 0 : lastPosition_ = 0;
89 0 : endPosition_ = 0;
90 :
91 0 : read_ = UNKNOWN_;
92 :
93 0 : integrationNum_ = 0;
94 0 : }
95 :
96 0 : string::size_type SDMDataObjectReader::find(const string& s) {
97 :
98 : while (true) {
99 0 : while ((position_ < endPosition_) && (data_[position_++] != s.at(0))) {;}
100 :
101 0 : if (position_ == endPosition_) return string::npos;
102 :
103 0 : string::size_type lastPosition_ = position_;
104 0 : unsigned long long int i = 1;
105 :
106 0 : while ((position_ < endPosition_) && (i < s.length())) {
107 0 : if (s.at(i) != data_[position_]) break;
108 0 : i++;
109 0 : position_++;
110 : }
111 :
112 0 : if (i == s.length()) return (position_ - s.length());
113 :
114 0 : if (position_ == endPosition_) return string::npos;
115 :
116 0 : position_ = lastPosition_; //+1;
117 0 : }
118 : }
119 :
120 0 : bool SDMDataObjectReader::compare(const string& s) {
121 0 : string::size_type index = position_;
122 :
123 0 : unsigned int i = 0;
124 0 : while ((index < endPosition_) && (i < s.length()) && (s.at(i++) == data_[index++])) {
125 : ;
126 : }
127 0 : return (i == s.length());
128 : }
129 :
130 0 : bool SDMDataObjectReader::EOD() {
131 0 : return position_ >= endPosition_;
132 : }
133 :
134 :
135 0 : string SDMDataObjectReader::extractXMLHeader(const string& boundary) {
136 : // cout << "Entering extractXMLHeader" << endl;
137 :
138 0 : string boundary_ = "\n--"+boundary;
139 :
140 0 : unsigned long int positionBeg = 0;
141 0 : unsigned long int positionEnd = 0;
142 :
143 : // We assume that we are at the first character of the body.
144 0 : positionBeg = position_;
145 :
146 : // Look for the next boundary supposed to be found right after an the XML header.
147 0 : if ( (positionEnd = find(boundary_)) == string::npos) {
148 0 : ostringstream eos;
149 0 : eos << "A second MIME boundary '" << boundary <<"' could not be found in the MIME message after position " << positionBeg << ".";
150 0 : throw SDMDataObjectReaderException(eos.str());
151 0 : }
152 :
153 0 : string xmlPart = string(data_+positionBeg, positionEnd-positionBeg);
154 :
155 : // cout << "Found XML Header : " << xmlPart << endl;
156 : // cout << "Exiting extractXMLHeader" << endl;
157 :
158 0 : return xmlPart;
159 0 : }
160 :
161 :
162 0 : void SDMDataObjectReader::tokenize(const string& str,
163 : vector<string>& tokens,
164 : const string& delimiters) {
165 : // Skip delimiters at beginning.
166 0 : string::size_type lastPos = str.find_first_not_of(delimiters, 0);
167 : // Find first "non-delimiter".
168 0 : string::size_type pos = str.find_first_of(delimiters, lastPos);
169 :
170 0 : while (string::npos != pos || string::npos != lastPos) {
171 : // Found a token, add it to the vector.
172 0 : tokens.push_back(str.substr(lastPos, pos - lastPos));
173 : // Skip delimiters. Note the "not_of"
174 0 : lastPos = str.find_first_not_of(delimiters, pos);
175 : // Find next "non-delimiter"
176 0 : pos = str.find_first_of(delimiters, lastPos);
177 : }
178 0 : }
179 :
180 :
181 0 : void SDMDataObjectReader::getFields(const string& header, map<string, string>& fields) {
182 : // We are at the last character of the line preceding the 1st field declaration.
183 0 : istringstream iss;
184 0 : iss.str(header);
185 0 : string line;
186 :
187 0 : getline(iss, line); // skip an nl
188 :
189 : // Split each line into field-name, field-value
190 0 : while (getline(iss, line)) {
191 0 : vector<string> nv;
192 0 : tokenize(line, nv, ":");
193 0 : if (nv.size() < 2) {
194 0 : ostringstream oss;
195 0 : oss << "Invalid field in a MIME header '" << header << "', integration #" << integrationNum_;
196 0 : throw SDMDataObjectReaderException (oss.str());
197 0 : }
198 0 : fields[nv.at(0)] = nv.at(1);
199 0 : }
200 0 : }
201 :
202 0 : string SDMDataObjectReader::getContentID() {
203 : // We are at the beginning of an attachment right after its opening boundary.
204 0 : string::size_type lastPosition = position_;
205 :
206 : // Now let's extract the header (which is followed by two successive nl nl)
207 : string::size_type endHeader;
208 0 : if ( (endHeader = find ("\n\n")) == string::npos) {
209 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
210 : }
211 :
212 0 : string header(data_+lastPosition, (endHeader - lastPosition));
213 :
214 0 : map<string, string> fields;
215 0 : getFields(header, fields);
216 :
217 : // Look for the Content-ID field
218 0 : map<string, string>::iterator iter = fields.find("Content-ID");
219 0 : if (iter == fields.end()) {
220 0 : ostringstream oss;
221 0 : oss << "'Content-ID' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
222 0 : throw SDMDataObjectReaderException(oss.str());
223 0 : }
224 0 : return iter->second;
225 0 : }
226 :
227 0 : string SDMDataObjectReader::getContentLocation() {
228 : // We are at the beginning of an attachment right after its opening boundary.
229 0 : string::size_type lastPosition = position_;
230 :
231 : // Now let's extract the header (which is followed by two successive nl nl)
232 : string::size_type endHeader;
233 0 : if ( (endHeader = find ("\n\n")) == string::npos) {
234 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
235 : }
236 :
237 0 : string header(data_+lastPosition, (endHeader - lastPosition));
238 :
239 0 : map<string, string> fields;
240 0 : getFields(header, fields);
241 :
242 : // Look for the Content-Location field
243 0 : map<string, string>::iterator iter = fields.find("Content-Location");
244 0 : if (iter == fields.end()) {
245 0 : ostringstream oss;
246 0 : oss << "'Content-Location' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
247 0 : throw SDMDataObjectReaderException(oss.str());
248 0 : }
249 0 : return iter->second;
250 0 : }
251 :
252 0 : void SDMDataObjectReader::processBinaryAttachment(const string& boundary, const SDMDataSubset& sdmDataSubset) {
253 :
254 0 : string contentLocation = getContentLocation();
255 :
256 : // Extract the attachment name and the integration number.
257 : #ifndef WITHOUT_BOOST
258 : boost::cmatch what;
259 : boost::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
260 : if (boost::regex_match(contentLocation.c_str(), what, r)) {
261 : }
262 : #else
263 0 : std::cmatch what;
264 0 : std::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
265 0 : if (std::regex_match(contentLocation.c_str(), what, r)) {
266 : }
267 : #endif
268 : else {
269 0 : ostringstream oss;
270 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of an attachment (approx. char position = " << position_ << ").";
271 0 : throw SDMDataObjectReaderException(oss.str());
272 0 : }
273 :
274 : // We are at the first byte of the body of the binary attachment.
275 0 : string::size_type startAttachPosition = position_; // mcaillat
276 :
277 : // Now look for its closing boundary (or opening boundary of the next one if any).
278 0 : string boundary_ = "\n--"+boundary;
279 : string::size_type endAttachPosition;
280 0 : if ( (endAttachPosition = find(boundary_)) == string::npos) {
281 0 : ostringstream eos;
282 0 : eos << "A MIME boundary '" << boundary << "' terminating a binary attachment starting approx. at character " << startAttachPosition <<" could not be found.";
283 0 : throw SDMDataObjectReaderException(eos.str());
284 0 : }
285 :
286 : // Compute the length of the attachment.
287 0 : string::size_type length = endAttachPosition - startAttachPosition;
288 :
289 :
290 :
291 0 : switch (name2code[what[1]]) {
292 0 : case ACTUALDURATIONS:
293 0 : actualDurations_ = (ACTUALDURATIONSTYPE *) (data_ + startAttachPosition);
294 0 : nActualDurations_ = length / sizeof (ACTUALDURATIONSTYPE);
295 0 : attachmentFlags.set(ACTUALDURATIONS);
296 : //cout << "actualDurations =" << (unsigned long int) actualDurations_ << ", nActualDurations = " << nActualDurations_ << endl;
297 0 : break;
298 0 : case ACTUALTIMES:
299 0 : actualTimes_ = (ACTUALTIMESTYPE *) (data_ + startAttachPosition);
300 0 : nActualTimes_ = length / sizeof( ACTUALTIMESTYPE);
301 0 : attachmentFlags.set(ACTUALTIMES);
302 : //cout << "actualTimes =" << (unsigned long int) actualTimes_ << ", nActualTimes = " << nActualTimes_ << endl;
303 0 : break;
304 0 : case AUTODATA:
305 0 : autoData_ = (AUTODATATYPE *) (data_ + startAttachPosition);
306 0 : nAutoData_ = length / sizeof(AUTODATATYPE);
307 0 : attachmentFlags.set(AUTODATA);
308 : //cout << "autoData =" << (unsigned long int) autoData_ << ", nAutoData = " << nAutoData_ << endl;
309 0 : break;
310 0 : case FLAGS:
311 0 : flags_ = (const FLAGSTYPE *) (data_ + startAttachPosition);
312 0 : nFlags_ = length / sizeof(FLAGSTYPE);
313 0 : attachmentFlags.set(FLAGS);
314 : //cout << "flags =" << (unsigned long int) flags_ << ", nFlags = " << nFlags_ << endl;
315 0 : break;
316 0 : case CROSSDATA:
317 0 : shortCrossData_ = 0;
318 0 : longCrossData_ = 0;
319 0 : floatCrossData_ = 0;
320 0 : switch (sdmDataSubset.crossDataType()) {
321 0 : case INT16_TYPE:
322 0 : shortCrossData_ = (const SHORTCROSSDATATYPE*) (data_ + startAttachPosition);
323 0 : nCrossData_ = length / sizeof (SHORTCROSSDATATYPE);
324 : //cout << "shortCrossData = " << (unsigned long int) shortCrossData_ << ", nShortCrossData = " << nCrossData_ << endl;
325 0 : break;
326 :
327 0 : case INT32_TYPE:
328 0 : longCrossData_ = (const INTCROSSDATATYPE*) (data_ + startAttachPosition);
329 0 : nCrossData_ = length / sizeof (INTCROSSDATATYPE);
330 : //cout << "longCrossData = " << (unsigned long int) longCrossData_ << ", nLongCrossData = " << nCrossData_ << endl;
331 0 : break;
332 :
333 0 : case FLOAT32_TYPE:
334 0 : floatCrossData_ = (const FLOATCROSSDATATYPE*) (data_ + startAttachPosition);
335 0 : nCrossData_ = length / sizeof (FLOATCROSSDATATYPE);
336 0 : break;
337 :
338 0 : default:
339 0 : ostringstream eos;
340 0 : eos << "'" << CPrimitiveDataType::toString(sdmDataSubset.crossDataType())
341 0 : << "' is not a valid primitive data type for CROSSDATA";
342 0 : throw SDMDataObjectReaderException(eos.str());
343 : // cout << "Unrecognized type for cross data" << endl;
344 : }
345 0 : attachmentFlags.set(CROSSDATA);
346 0 : break;
347 0 : case ZEROLAGS:
348 0 : zeroLags_ = (ZEROLAGSTYPE *) (data_ + startAttachPosition);
349 0 : nZeroLags_ = length / sizeof(ZEROLAGSTYPE) ;
350 0 : attachmentFlags.set(ZEROLAGS);
351 : //cout << "zeroLags =" << (unsigned long int) zeroLags_ << ", nZeroLags = " << nZeroLags_ << endl;
352 0 : break;
353 :
354 0 : default:
355 : //cout << "Unrecognized code" << endl;
356 0 : break;
357 : }
358 : // cout << "found a binary attachment" << endl;
359 0 : }
360 :
361 0 : void SDMDataObjectReader::processMIMESDMDataHeader() {
362 : // cout << "Entering processMIMESDMDataHeader" << endl;
363 0 : string sdmDataHeader = extractXMLHeader(MIMEBOUNDARY_1);
364 0 : parser_.parseMemoryHeader(sdmDataHeader, sdmDataObject_);
365 : // cout << "Exiting processMIMESDMDataHeader" << endl;
366 0 : }
367 :
368 0 : void SDMDataObjectReader::processMIMESDMDataSubsetHeader(SDMDataSubset& sdmDataSubset) {
369 : // cout << "Entering processMIMESDMDataSubsetHeader" << endl;
370 0 : string sdmDataSubsetHeader = extractXMLHeader(MIMEBOUNDARY_2);
371 0 : if (sdmDataSubset.owner_->isCorrelation())
372 0 : parser_.parseMemoryCorrSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
373 : else
374 0 : parser_.parseMemoryTPSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
375 : // cout << "Exiting processMIMESDMDataSubsetHeader" << endl;
376 0 : }
377 :
378 0 : void SDMDataObjectReader::processMIMEIntegration() {
379 : // cout << "Entering processMIMEIntegration" << endl;
380 :
381 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
382 0 : string contentLocation = getContentLocation();
383 :
384 : #ifndef WITHOUT_BOOST
385 : boost::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
386 : ostringstream oss;
387 :
388 : // Extract the Subset name and the integration [, subintegration] number.
389 : boost::cmatch what;
390 : if (!boost::regex_match(contentLocation.c_str(), what, r)) {
391 : ostringstream oss;
392 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
393 : throw SDMDataObjectReaderException(oss.str());
394 : }
395 : #else
396 0 : std::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
397 0 : ostringstream oss;
398 :
399 : // Extract the Subset name and the integration [, subintegration] number.
400 0 : std::cmatch what;
401 0 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
402 0 : ostringstream oss;
403 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
404 0 : throw SDMDataObjectReaderException(oss.str());
405 0 : }
406 : #endif
407 :
408 0 : SDMDataSubset integration(&sdmDataObject_);
409 :
410 : // The SDMDataSubset header.
411 0 : processMIMESDMDataSubsetHeader(integration);
412 :
413 0 : if (integration.aborted_) {
414 : // The [sub]integration has been aborted. Just append its header, without trying to get binary attachment.
415 0 : sdmDataObject_.append(integration);
416 : }
417 : else {
418 : // This is regular [sub]integration, process its binary attachments.
419 0 : attachmentFlags.reset();
420 0 : while (!EOD() && !compare("--")) {
421 0 : processBinaryAttachment(MIMEBOUNDARY_2, integration);
422 : }
423 :
424 0 : if (EOD())
425 0 : throw SDMDataObjectReaderException("Unexpected end of data");
426 :
427 : // Now check if the binary attachments found are compatible with the correlation mode
428 : // and if their sizes are equal to what is announced in the global header.
429 :
430 0 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
431 : // No actualdurations attachment found, ok then set everything to 0.
432 0 : integration.actualDurations_ = 0;
433 0 : integration.nActualDurations_ = 0;
434 : }
435 : else {
436 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
437 0 : ostringstream oss;
438 0 : oss << "Data subset '"<<contentLocation<<"': ";
439 0 : oss << "zize of 'actualDuration' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
440 0 : throw SDMDataObjectReaderException(oss.str());
441 0 : }
442 :
443 0 : integration.actualDurations_ = actualDurations_;
444 0 : integration.nActualDurations_ = nActualDurations_;
445 : }
446 :
447 0 : if (!attachmentFlags.test(ACTUALTIMES)) {
448 : // No actualtimes attachment found, ok then set everything to 0.
449 0 : integration.actualTimes_ = 0;
450 0 : integration.nActualTimes_ = 0;
451 : }
452 : else {
453 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
454 0 : ostringstream oss;
455 0 : oss << "Data subset '"<<contentLocation<<"': ";
456 0 : oss << "size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
457 0 : throw SDMDataObjectReaderException(oss.str());
458 0 : }
459 0 : integration.actualTimes_ = actualTimes_;
460 0 : integration.nActualTimes_ = nActualTimes_;
461 : }
462 :
463 : // The flags are optional. They may be absent.
464 0 : if (!attachmentFlags.test(FLAGS)) {
465 : // No flags binary attachment found, ok then set everything to 0.
466 0 : integration.flags_ = 0;
467 0 : integration.nFlags_ = 0;
468 : }
469 : else {
470 : // flags found
471 : // Check size conformity
472 0 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
473 0 : ostringstream oss;
474 0 : oss << "Data subset '"<<contentLocation<<"': ";
475 0 : oss << "size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
476 0 : throw SDMDataObjectReaderException(oss.str());
477 0 : }
478 :
479 : // Set pointer and size
480 0 : integration.flags_ = flags_;
481 0 : integration.nFlags_ = nFlags_;
482 : }
483 :
484 :
485 : //
486 : // The presence of crossData and autoData depends on the correlation mode.
487 :
488 : // crossDataTypeSize is not currently used
489 : // unsigned int crossDataTypeSize;
490 0 : switch (sdmDataObject_.correlationMode()) {
491 0 : case CROSS_ONLY:
492 0 : if (!attachmentFlags.test(CROSSDATA)) {
493 0 : ostringstream oss;
494 0 : oss << "Data subset '"<<contentLocation<<"': ";
495 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationNum_;
496 0 : throw SDMDataObjectReaderException(oss.str());
497 0 : }
498 :
499 :
500 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
501 0 : ostringstream oss;
502 0 : oss << "Data subset '"<<contentLocation<<"': ";
503 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
504 0 : throw SDMDataObjectReaderException(oss.str());
505 0 : }
506 :
507 :
508 0 : if (attachmentFlags.test(AUTODATA)) {
509 0 : ostringstream oss;
510 0 : oss << "Data subset '"<<contentLocation<<"': ";
511 0 : oss << "found an unexpected attachment 'autoData' in integration #" << integrationNum_ << ".";
512 0 : throw SDMDataObjectReaderException(oss.str());
513 0 : }
514 :
515 0 : integration.shortCrossData_ = 0;
516 0 : integration.longCrossData_ = 0;
517 0 : integration.floatCrossData_ = 0;
518 0 : integration.nCrossData_ = 0;
519 0 : switch (integration.crossDataType()) {
520 0 : case INT32_TYPE:
521 0 : integration.longCrossData_ = longCrossData_;
522 0 : integration.nCrossData_ = nCrossData_;
523 0 : break;
524 :
525 0 : case INT16_TYPE:
526 0 : integration.shortCrossData_ = shortCrossData_;
527 0 : integration.nCrossData_ = nCrossData_;
528 0 : break;
529 :
530 0 : case FLOAT32_TYPE:
531 0 : integration.floatCrossData_ = floatCrossData_;
532 0 : integration.nCrossData_ = nCrossData_;
533 0 : break;
534 :
535 0 : default:
536 0 : throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
537 : }
538 0 : break;
539 :
540 0 : case AUTO_ONLY:
541 0 : if (!attachmentFlags.test(AUTODATA)) {
542 0 : ostringstream oss;
543 0 : oss << "Data subset '"<<contentLocation<<"': ";
544 0 : oss << "a binary attachment 'autoData' was expected.";
545 0 : throw SDMDataObjectReaderException(oss.str());
546 0 : }
547 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
548 0 : ostringstream oss;
549 0 : oss << "Data subset '"<<contentLocation<<"': ";
550 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
551 0 : throw SDMDataObjectReaderException(oss.str());
552 0 : }
553 :
554 :
555 0 : if (attachmentFlags.test(CROSSDATA)) {
556 0 : ostringstream oss;
557 0 : oss << "Data subset '"<<contentLocation<<"': ";
558 0 : oss << "found an unexpected attachment 'crossData'";
559 0 : throw SDMDataObjectReaderException(oss.str());
560 0 : }
561 :
562 0 : integration.autoData_ = autoData_;
563 0 : integration.nAutoData_ = nAutoData_;
564 :
565 0 : break;
566 :
567 0 : case CROSS_AND_AUTO:
568 0 : if (!attachmentFlags.test(AUTODATA)) {
569 0 : ostringstream oss;
570 0 : oss << "Data subset '"<<contentLocation<<"': ";
571 0 : oss << "a binary attachment 'autoData' was expected.";
572 0 : throw SDMDataObjectReaderException(oss.str());
573 0 : }
574 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
575 0 : ostringstream oss;
576 0 : oss << "Data subset '"<<contentLocation<<"': ";
577 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
578 0 : throw SDMDataObjectReaderException(oss.str());
579 0 : }
580 :
581 0 : if (!attachmentFlags.test(CROSSDATA)) {
582 0 : ostringstream oss;
583 0 : oss << "Data subset '"<<contentLocation<<"': ";
584 0 : oss << "a binary attachment 'crossData' was expected.";
585 0 : throw SDMDataObjectReaderException(oss.str());
586 0 : }
587 :
588 : // crossDataTypeSize is not currently used, skip this block
589 : // switch (integration.crossDataType()) {
590 : // case INT32_TYPE:
591 : // crossDataTypeSize = sizeof(INTCROSSDATATYPE);
592 : // break;
593 : // case INT16_TYPE:
594 : // crossDataTypeSize = sizeof(SHORTCROSSDATATYPE);
595 : // break;
596 : // case FLOAT32_TYPE:
597 : // crossDataTypeSize = sizeof(FLOATCROSSDATATYPE);
598 : // break;
599 : // default:
600 : // throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
601 : // }
602 :
603 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
604 0 : ostringstream oss;
605 0 : oss << "Data subset '"<<contentLocation<<"': ";
606 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
607 0 : throw SDMDataObjectReaderException(oss.str());
608 0 : }
609 :
610 :
611 0 : integration.shortCrossData_ = 0;
612 0 : integration.longCrossData_ = 0;
613 0 : integration.floatCrossData_ = 0;
614 0 : integration.nCrossData_ = 0;
615 0 : switch (integration.crossDataType()) {
616 0 : case INT32_TYPE:
617 0 : integration.longCrossData_ = longCrossData_;
618 0 : integration.nCrossData_ = nCrossData_;
619 0 : break;
620 :
621 0 : case INT16_TYPE:
622 0 : integration.shortCrossData_ = shortCrossData_;
623 0 : integration.nCrossData_ = nCrossData_;
624 0 : break;
625 :
626 0 : case FLOAT32_TYPE:
627 0 : integration.floatCrossData_ = floatCrossData_;
628 0 : integration.nCrossData_ = nCrossData_;
629 0 : break;
630 :
631 0 : default:
632 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': '"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
633 : }
634 :
635 0 : integration.autoData_ = autoData_;
636 0 : integration.nAutoData_ = nAutoData_;
637 :
638 :
639 0 : break;
640 0 : default:
641 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': unrecognized correlation mode");
642 : break;
643 : }
644 :
645 :
646 0 : if (attachmentFlags.test(ZEROLAGS)) {
647 : // Refuse the zeroLags attachment if it's not a Correlator or if the correlator is a CORRELATOR_FX (ACA).
648 0 : if ((sdmDataObject_.processorType_ != CORRELATOR) || (sdmDataObject_.correlatorType() == FX))
649 0 : throw SDMDataObjectReaderException("zeroLags are not expected from a correlator CORRELATOR_FX");
650 :
651 0 : if (nZeroLags_ != sdmDataObject_.dataStruct().zeroLags().size()) {
652 0 : ostringstream oss;
653 0 : oss << "Data subset '"<<contentLocation<<"': ";
654 0 : oss << "size of 'zeroLags' attachment (" << nZeroLags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().zeroLags().size() << ")";
655 0 : throw SDMDataObjectReaderException(oss.str());
656 0 : }
657 0 : integration.zeroLags_ = zeroLags_;
658 0 : integration.nZeroLags_ = nZeroLags_;
659 : }
660 :
661 0 : sdmDataObject_.append(integration);
662 : }
663 :
664 0 : if (!compare("--\n--"+MIMEBOUNDARY_1))
665 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
666 :
667 0 : find("--\n--"+MIMEBOUNDARY_1);
668 : // cout << "Exiting processMIMEIntegration" << endl;
669 0 : }
670 :
671 0 : void SDMDataObjectReader::processMIMEIntegrations() {
672 : // cout << "Entering processMIMEIntegrations" << endl;
673 :
674 0 : while (!compare("--")) {
675 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
676 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
677 : // and move to the next --<MIMEBOUNDARY_2>
678 0 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
679 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
680 :
681 : //Process the next integration.
682 0 : integrationNum_++;
683 0 : processMIMEIntegration() ;
684 : }
685 :
686 : // cout << "Exiting processMIMEIntegrations" << endl;
687 0 : }
688 :
689 0 : void SDMDataObjectReader::processMIMESubscan() {
690 : // cout << "Entering processMIMESubscan" << endl;
691 :
692 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
693 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
694 : // and move to the next --<MIMEBOUNDARY_2>
695 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
696 0 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
697 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
698 :
699 :
700 0 : string contentLocation = getContentLocation();
701 : // cout << contentLocation << endl;
702 : #ifndef WITHOUT_BOOST
703 : boost::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
704 :
705 : // Extract the Subset name and the suffix number (which must be equal to 1).
706 : boost::cmatch what;
707 : if (!boost::regex_match(contentLocation.c_str(), what, r)) {
708 : ostringstream oss;
709 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
710 : throw SDMDataObjectReaderException(oss.str());
711 : }
712 : #else
713 0 : std::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
714 :
715 : // Extract the Subset name and the suffix number (which must be equal to 1).
716 0 : std::cmatch what;
717 0 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
718 0 : ostringstream oss;
719 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
720 0 : throw SDMDataObjectReaderException(oss.str());
721 0 : }
722 : #endif
723 :
724 0 : SDMDataSubset subscan(&sdmDataObject_);
725 :
726 : // The SDMDataSubset header.
727 0 : processMIMESDMDataSubsetHeader(subscan);
728 :
729 :
730 : // Process the binary attachments.
731 0 : attachmentFlags.reset();
732 0 : integrationNum_++;
733 0 : while(!EOD() && !compare("--")) {
734 0 : processBinaryAttachment(MIMEBOUNDARY_2, subscan);
735 : }
736 :
737 0 : if (EOD()) throw SDMDataObjectReaderException("Unexpected end of data");
738 :
739 : // Now check if the binary attachments found are compatible with the correlation mode.
740 : // and if their sizes are equal to what is announced in the global header.
741 :
742 : // Start with the only mandatory attachment : autoData.
743 0 : if (!attachmentFlags.test(AUTODATA)) {
744 0 : ostringstream oss;
745 0 : oss << "Binary attachment 'autoData' was expected in integration #" << integrationNum_;
746 0 : throw SDMDataObjectReaderException(oss.str());
747 0 : }
748 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
749 0 : ostringstream oss;
750 0 : oss << "Size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
751 0 : throw SDMDataObjectReaderException(oss.str());
752 0 : }
753 :
754 0 : subscan.autoData_ = autoData_;
755 0 : subscan.nAutoData_ = nAutoData_;
756 :
757 :
758 : // And now consider the optional attachments.
759 :
760 : // The actualDurations are optional. They may be absent.
761 0 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
762 : // No actualDurations binary attachment found, ok then set everything to 0.
763 0 : subscan.actualDurations_ = 0;
764 0 : subscan.nActualDurations_ = 0;
765 : }
766 : else {
767 : // actualDurations found
768 : // Check size conformity
769 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
770 0 : ostringstream oss;
771 0 : oss << "Size of 'actualDurations' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
772 0 : throw SDMDataObjectReaderException(oss.str());
773 0 : }
774 :
775 : // Set pointer and size
776 0 : subscan.actualDurations_ = actualDurations_;
777 0 : subscan.nActualDurations_ = nActualDurations_;
778 : }
779 :
780 :
781 : // The actualTimes are optional. They may be absent.
782 0 : if (!attachmentFlags.test(ACTUALTIMES)) {
783 : // No actualTimes binary attachment found, ok then set everything to 0.
784 0 : subscan.actualTimes_ = 0;
785 0 : subscan.nActualTimes_ = 0;
786 : }
787 : else {
788 : // actualTimes found
789 : // Check size conformity
790 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
791 0 : ostringstream oss;
792 0 : oss << "Size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
793 0 : throw SDMDataObjectReaderException(oss.str());
794 0 : }
795 :
796 : // Set pointer and size
797 0 : subscan.actualTimes_ = actualTimes_;
798 0 : subscan.nActualTimes_ = nActualTimes_;
799 : }
800 :
801 :
802 : // The flags are optional. They may be absent.
803 0 : if (!attachmentFlags.test(FLAGS)) {
804 : // No flags binary attachment found, ok then set everything to 0.
805 0 : subscan.flags_ = 0;
806 0 : subscan.nFlags_ = 0;
807 : }
808 : else {
809 : // flags found
810 : // Check size conformity
811 0 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
812 0 : ostringstream oss;
813 0 : oss << "Size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
814 0 : throw SDMDataObjectReaderException(oss.str());
815 0 : }
816 :
817 : // Set pointer and size
818 0 : subscan.flags_ = flags_;
819 0 : subscan.nFlags_ = nFlags_;
820 : }
821 :
822 : // And finally let's check that no unexpected binary attachment was found.
823 0 : if (attachmentFlags.test(CROSSDATA)) {
824 0 : ostringstream oss;
825 0 : oss << "Found an unexpected attachment 'crossData' in the binary attachments.";
826 0 : throw SDMDataObjectReaderException(oss.str());
827 0 : }
828 :
829 :
830 0 : if (attachmentFlags.test(ZEROLAGS)) {
831 0 : ostringstream oss;
832 0 : oss << "Found an unexpected attachment 'zeroLags' in the binary attachments.";
833 0 : throw SDMDataObjectReaderException(oss.str());
834 0 : }
835 :
836 0 : sdmDataObject_.tpDataSubset(subscan);
837 :
838 :
839 0 : if (!compare("--\n--"+MIMEBOUNDARY_1))
840 0 : throw SDMDataObjectReaderException("Expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
841 :
842 0 : find("--\n--"+MIMEBOUNDARY_1);
843 0 : }
844 :
845 0 : void SDMDataObjectReader::processMIME() {
846 : // cout << "Entering processMIME" << endl;
847 : // Let's find the opening boundary
848 0 : if (find("--"+MIMEBOUNDARY_1) == string::npos)
849 0 : throw SDMDataObjectReaderException("Expecting a first boundary '--"+MIMEBOUNDARY_1+"'.");
850 :
851 : // Check the Content-Location of the MIME header ...but do nothing special with it...
852 0 : string contentLocation=getContentLocation();
853 :
854 : // Detect SDMDataHeader.
855 0 : processMIMESDMDataHeader();
856 :
857 : // Do we have packed data (i.e. only one SDMDataSubset grouping all the integrations) or one integration per SDMDataSubset (i.e. per timestamp) ?
858 0 : if (sdmDataObject_.hasPackedData()) {
859 0 : processMIMESubscan();
860 : }
861 : else {
862 0 : processMIMEIntegrations();
863 : }
864 :
865 :
866 : // if (sdmDataObject_.isCorrelation()) {
867 : // // Process integrations.
868 : // processMIMEIntegrations();
869 : // }
870 : // else if (sdmDataObject_.isTP()){
871 : // // cout << "TP data" << endl;
872 : // if (sdmDataObject_.dimensionality() == 0) {
873 : // processMIMESubscan();
874 : // }
875 : // else {
876 : // processMIMEIntegrations();
877 : // }
878 : // }
879 : // else {
880 : // // cout << "Unrecognized type of binary data." << endl;
881 : // }
882 : //cout << "Exiting processMIME" << endl;
883 0 : }
884 :
885 0 : const SDMDataObject & SDMDataObjectReader::read(const char * buffer, unsigned long int size, bool fromFile) {
886 0 : if (!fromFile) read_ = MEMORY_;
887 :
888 : // Set up all sensitive pointers and sizes.
889 0 : data_ = (char *)buffer;
890 0 : dataSize_ = size;
891 0 : position_ = 0;
892 0 : endPosition_ = size;
893 :
894 : // And process the MIME message
895 0 : sdmDataObject_.valid_ = true;
896 0 : processMIME();
897 :
898 0 : sdmDataObject_.owns();
899 0 : return sdmDataObject_;
900 : }
901 :
902 0 : const SDMDataObject& SDMDataObjectReader::ref() const {
903 0 : if (read_ == UNKNOWN_)
904 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
905 0 : return sdmDataObject_;
906 : }
907 :
908 0 : const SDMDataObject* SDMDataObjectReader::ptr() const {
909 0 : if (read_ == UNKNOWN_)
910 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
911 0 : return &sdmDataObject_;
912 : }
913 :
914 0 : void SDMDataObjectReader::done() {
915 0 : switch (read_) {
916 0 : case UNKNOWN_:
917 : case MEMORY_:
918 0 : break;
919 0 : case FILE_:
920 0 : munmap((caddr_t) data_, dataSize_);
921 0 : close(filedes_);
922 0 : break;
923 : }
924 0 : sdmDataObject_.done();
925 0 : read_ = UNKNOWN_;
926 0 : }
927 :
928 0 : const SDMDataObject& SDMDataObjectReader::sdmDataObject() {
929 0 : return sdmDataObject_;
930 : }
931 :
932 :
933 0 : const SDMDataObject & SDMDataObjectReader::read(string filename) {
934 :
935 : struct stat fattr; // stat64 fattr;
936 :
937 : unsigned long int filesize;
938 0 : char* data = 0;
939 : // Open the file.
940 0 : errno = 0;
941 : #ifdef __APPLE__
942 : if ( (filedes_ = open(filename.c_str(), O_RDONLY )) == -1) {
943 : #else
944 0 : if ( (filedes_ = open(filename.c_str(), O_RDONLY | O_LARGEFILE)) == -1) {
945 : #endif
946 0 : string message(strerror(errno));
947 0 : throw SDMDataObjectReaderException("Could not open file '" + filename + "'. The message was '" + message + "'");
948 0 : }
949 0 : read_ = FILE_;
950 :
951 : // Get its size.
952 0 : errno = 0;
953 0 : int status = fstat(filedes_,&fattr); // fstat64(filedes_,&fattr);
954 0 : if (status == -1) {
955 0 : string message(strerror(errno));
956 0 : throw SDMDataObjectReaderException("Could not retrieve size of file '" + filename + "'. The message was '" + message + "'");
957 0 : }
958 0 : filesize = fattr.st_size;
959 :
960 : // Map it to virtual memory address space.
961 0 : errno = 0;
962 :
963 0 : data = (char *) mmap((caddr_t)0, filesize, PROT_READ, MAP_SHARED, filedes_, (off_t)0);
964 0 : if ( ((unsigned long) data) == 0xffffffff) {
965 0 : string message(strerror(errno));
966 0 : throw SDMDataObjectReaderException("Could not map file '" + filename + "' to memory. The message was '" + message + "'");
967 0 : }
968 :
969 : // cout << "Successfully mapped file." << endl;
970 : // And delegate to the other read (memory buffer) method.
971 0 : return read(data, filesize, true);
972 : }
973 :
974 : // SDMDataObjectReader::
975 :
976 : } // namespace asdmbinaries
977 :
|
