Line data Source code
1 : #include <alma/ASDMBinaries/SDMDataObjectStreamReader.h>
2 :
3 : #include <regex>
4 :
5 : #ifdef WITHOUT_BOOST
6 : #include <alma/ASDM/Misc.h>
7 : #endif
8 :
9 : using namespace CorrelationModeMod;
10 : using namespace CorrelatorTypeMod;
11 : using namespace PrimitiveDataTypeMod;
12 : using namespace ProcessorTypeMod;
13 :
14 : using namespace std;
15 :
16 : namespace asdmbinaries {
17 :
18 0 : SDMDataObjectStreamReaderException::SDMDataObjectStreamReaderException():message("SDMDataObjectStreamReaderException:") {;}
19 :
20 0 : SDMDataObjectStreamReaderException::SDMDataObjectStreamReaderException(const string & message): message("SDMDataObjectStreamReaderException:" + message) {;}
21 :
22 0 : const string& SDMDataObjectStreamReaderException::getMessage() { return message; }
23 :
24 850 : SDMDataObjectStreamReader::SDMDataObjectStreamReader() {
25 : // cout << "SDMDataObjectStreamReader::SDMDataObjectStreamReader() : entering" << endl;
26 850 : const char *partNames[] = {"actualDurations", "actualTimes", "autoData", "crossData", "zeroLags", "flags"};
27 850 : set<string> dummy(partNames, partNames+6);
28 850 : s_partNames = dummy;
29 850 : sdmDataSubset = SDMDataSubset(&sdmDataObject);
30 :
31 850 : boundary_1 = "" ;
32 850 : currentState = S_NO_BDF;
33 : // cout << "SDMDataObjectStreamReader::SDMDataObjectStreamReader() : exiting" << endl;
34 850 : }
35 :
36 850 : SDMDataObjectStreamReader::~SDMDataObjectStreamReader() {
37 : //cout << "SDMDataObjectStreamReader::~SDMDataObjectStreamReader : entering" << endl;
38 850 : this->close();
39 : //cout << "SDMDataObjectStreamReader::~SDMDataObjectStreamReader : exiting" << endl;
40 850 : }
41 :
42 2589 : void SDMDataObjectStreamReader::open(const string& path) {
43 2589 : checkState(T_OPEN, "open");
44 2589 : this->path = path;
45 2589 : f.open(path.c_str(), ifstream::in);
46 2589 : if (f.fail())
47 0 : throw SDMDataObjectStreamReaderException("could not open '" + path + "'.");
48 :
49 2589 : boundary_1 = requireMIMEHeader();
50 : // cout << "Boundary = " << boundary_1 << endl;
51 2589 : sdmDataObject.valid_ = true;
52 2589 : requireSDMDataHeaderMIMEPart();
53 2589 : sdmDataObject.owns();
54 2589 : integrationIndex = -1; // We have not yet read any integration.
55 :
56 2589 : currentState = S_AT_BEGINNING;
57 2589 : }
58 :
59 41800 : int64_t SDMDataObjectStreamReader::position() {
60 41800 : return (int64_t) f.tellg();
61 : }
62 :
63 41800 : void SDMDataObjectStreamReader::position(int64_t p) {
64 41800 : f.seekg(p);
65 41800 : }
66 :
67 3835 : void SDMDataObjectStreamReader::close() {
68 : // cout << "SDMDataObjectStreamReader::close -- Entering" << endl;
69 3835 : if (f.is_open()) {
70 2589 : releaseMemory(sdmDataSubset);
71 2589 : for (unsigned int i = 0; i < remainingSubsets.size(); i++)
72 0 : releaseMemory(remainingSubsets[i]);
73 23491 : for (unsigned int i = 0; i < someSubsets.size(); i++)
74 20902 : releaseMemory(someSubsets[i]);
75 2589 : f.close();
76 2589 : currentState = S_NO_BDF;
77 : }
78 : // cout << "SDMDataObjectStreamReader::close -- Exiting" << endl;
79 3835 : }
80 :
81 0 : unsigned int SDMDataObjectStreamReader::currentIntegrationIndex() const { checkState(T_QUERY, "currentIntegrationIndex"); return integrationIndex; }
82 0 : unsigned long long SDMDataObjectStreamReader::currentIntegrationStartsAt() const { checkState(T_QUERY, "currentIntegrationStartAt"); return integrationStartsAt; }
83 0 : string SDMDataObjectStreamReader::title() const { checkState(T_QUERY, "title"); return sdmDataObject.title(); }
84 32 : const ByteOrder* SDMDataObjectStreamReader::byteOrder() const { checkState(T_QUERY, "byteOrder"); return sdmDataObject.byteOrder(); }
85 0 : unsigned long long SDMDataObjectStreamReader::startTime() const { checkState(T_QUERY, "startTime"); return sdmDataObject.startTime(); }
86 4732 : unsigned int SDMDataObjectStreamReader::numTime() const { checkState(T_QUERY, "numTime"); return sdmDataObject.numTime(); }
87 0 : string SDMDataObjectStreamReader::dataOID() const { checkState(T_QUERY, "dataOID"); return sdmDataObject.dataOID(); }
88 0 : string SDMDataObjectStreamReader::execBlockUID() const { checkState(T_QUERY, "execBlockUID"); return sdmDataObject.execBlockUID(); }
89 0 : unsigned int SDMDataObjectStreamReader::execBlockNum() const { checkState(T_QUERY, "execBlockNum"); return sdmDataObject.execBlockNum(); }
90 0 : unsigned int SDMDataObjectStreamReader::scanNum() const { checkState(T_QUERY, "scanNum"); return sdmDataObject.scanNum(); }
91 0 : unsigned int SDMDataObjectStreamReader::subscanNum() const { checkState(T_QUERY, "subscanNum"); return sdmDataObject.subscanNum(); }
92 1843 : unsigned int SDMDataObjectStreamReader::numAntenna() const { checkState(T_QUERY, "numAntenna"); return sdmDataObject.numAntenna(); }
93 2575 : CorrelationMode SDMDataObjectStreamReader::correlationMode() const { checkState(T_QUERY, "correlationMode"); return sdmDataObject.correlationMode(); }
94 0 : OptionalSpectralResolutionType SDMDataObjectStreamReader::spectralResolutionType() const { checkState(T_QUERY, "spectralResolutionType"); return sdmDataObject.spectralResolutionType(); }
95 1843 : ProcessorType SDMDataObjectStreamReader::processorType() const { checkState(T_QUERY, "processorType"); return sdmDataObject.processorType(); }
96 0 : CorrelatorType SDMDataObjectStreamReader::correlatorType() const { checkState(T_QUERY, "correlatorType"); return sdmDataObject.correlatorType(); }
97 1072 : bool SDMDataObjectStreamReader::hasPackedData() const { checkState(T_QUERY, "isTP"); return sdmDataObject.hasPackedData();}
98 0 : bool SDMDataObjectStreamReader::isTP() const { checkState(T_QUERY, "isTP"); return sdmDataObject.isTP();}
99 0 : bool SDMDataObjectStreamReader::isWVR() const {checkState(T_QUERY, "isCorrelation"); return sdmDataObject.isWVR();}
100 0 : bool SDMDataObjectStreamReader::isCorrelation() const {checkState(T_QUERY, "isCorrelation"); return sdmDataObject.isCorrelation();}
101 11608 : const SDMDataObject::DataStruct& SDMDataObjectStreamReader::dataStruct() const { checkState(T_QUERY, "dataStruct"); return sdmDataObject.dataStruct();}
102 0 : bool SDMDataObjectStreamReader::aborted() const {checkState(T_QUERY, "aborted"); return sdmDataObject.aborted();}
103 0 : unsigned long long SDMDataObjectStreamReader::abortTime() const { checkState(T_QUERY, "abortTime"); return sdmDataObject.abortTime(); }
104 0 : string SDMDataObjectStreamReader::abortReason() const { checkState(T_QUERY, "abortReason"); return sdmDataObject.abortReason(); }
105 0 : string SDMDataObjectStreamReader::toString() const { checkState(T_QUERY, "toString"); return sdmDataObject.toString(); }
106 :
107 90891 : void SDMDataObjectStreamReader::checkState(Transitions t, const string& methodName) const {
108 : // cout << "Entering checkState with currentState = " << currentState << " and transition = " << t << " for method " << methodName << endl;
109 90891 : switch (currentState) {
110 2589 : case S_NO_BDF:
111 2589 : if ( t == T_OPEN ) {
112 2589 : return;
113 : }
114 0 : break;
115 :
116 34072 : case S_AT_BEGINNING :
117 34072 : switch (t) {
118 34072 : case T_QUERY:
119 : case T_TEST_END:
120 : case T_READ:
121 : case T_READ_NEXT:
122 : case T_READ_ALL:
123 : case T_CLOSE:
124 34072 : return;
125 0 : default :
126 0 : break;
127 : }
128 0 : break;
129 :
130 54230 : case S_READING:
131 54230 : switch (t) {
132 54230 : case T_TEST_END:
133 : case T_READ:
134 : case T_READ_NEXT:
135 : case T_READ_ALL:
136 : case T_QUERY:
137 : case T_CLOSE:
138 54230 : return;
139 0 : default :
140 0 : break;
141 : }
142 0 : break;
143 :
144 0 : case S_AT_END:
145 0 : switch(t) {
146 0 : case T_TEST_END:
147 : case T_QUERY:
148 : case T_READ_NEXT:
149 : case T_READ_ALL:
150 : case T_CLOSE:
151 0 : return;
152 0 : default:
153 0 : break;
154 : }
155 0 : break;
156 : } // end switch on currentState
157 :
158 : // Any other pair (transition, currentState) will throw an exception.
159 0 : throw SDMDataObjectStreamReaderException("Invalid call of method '" + methodName + "' in the current context.");
160 : }
161 :
162 40982 : bool SDMDataObjectStreamReader::hasSubset() {
163 40982 : checkState(T_TEST_END, "hasSubset");
164 40982 : bool atEnd = currentLine.compare("--"+boundary_1+"--") == 0;
165 40982 : if (atEnd) currentState = S_AT_END;
166 40982 : return !atEnd;
167 : }
168 :
169 589 : const vector<SDMDataSubset>& SDMDataObjectStreamReader::nextSubsets(unsigned int nSubsets) {
170 589 : checkState(T_READ_NEXT, "nextSubsets");
171 :
172 : // Deep empty of the vector nextSubsets
173 : // Firstly free all memory dynamically allocated in every element of the vector.
174 17908 : for (unsigned int i = 0; i < someSubsets.size(); i++)
175 17319 : releaseMemory(someSubsets.at(i));
176 :
177 : // Then clear the vector.
178 589 : someSubsets.clear();
179 :
180 : // Then populate the vector nextSubsets with as many SDMDataSubsets as possible up to a limit
181 : // of nSubsets read from the current position.
182 589 : unsigned int nRead = 0;
183 19363 : while ((nRead < nSubsets) && hasSubset()) {
184 18774 : someSubsets.push_back(SDMDataSubset(&sdmDataObject));
185 18774 : integrationIndex++; nRead++;
186 18774 : requireSDMDataSubsetMIMEPart(someSubsets.back());
187 18774 : string line = nextLine();
188 18774 : }
189 :
190 589 : return someSubsets;
191 : }
192 :
193 0 : const vector<SDMDataSubset>& SDMDataObjectStreamReader::allRemainingSubsets() {
194 : // cout << "SDMDataObjectStreamReader::allRemainingSubsets: entering." << endl;
195 0 : checkState(T_READ_ALL, "allRemainingSubsets");
196 :
197 : // Deep empty of the vector remainingSubsets.
198 : // Firstly free all memory dynamically allocated in every element of the vector.
199 0 : for (unsigned int i = 0; i < remainingSubsets.size(); i++)
200 0 : releaseMemory(remainingSubsets.at(i));
201 :
202 : // Then clear the vector.
203 0 : remainingSubsets.clear();
204 :
205 : // Then populate the vector with a new collection.
206 0 : while (hasSubset()) {
207 0 : remainingSubsets.push_back(SDMDataSubset(&sdmDataObject));
208 0 : integrationIndex++;
209 0 : requireSDMDataSubsetMIMEPart(remainingSubsets.back());
210 0 : string line = nextLine();
211 0 : }
212 :
213 : // cout << "SDMDataObjectStreamReader::allRemainingSubsets: exiting." << endl;
214 0 : return remainingSubsets;
215 : }
216 :
217 23026 : const SDMDataSubset & SDMDataObjectStreamReader::getSubset() {
218 23026 : checkState(T_READ, "getSubset");
219 23026 : integrationIndex++;
220 23026 : requireSDMDataSubsetMIMEPart(sdmDataSubset);
221 23026 : string line = nextLine();
222 :
223 23026 : currentState = S_READING;
224 23026 : return sdmDataSubset;
225 23026 : }
226 :
227 997988 : string SDMDataObjectStreamReader::nextLine() {
228 997988 : unsigned long long whereAmI = f.tellg();
229 997988 : getline(f, currentLine);
230 997988 : if (f.fail()) {
231 0 : ostringstream oss ;
232 0 : oss << "SDMDataObjectStreamReader::nextLine() : I could not read a line in '" << path << "' at position " << whereAmI << ".";
233 0 : throw SDMDataObjectStreamReaderException(oss.str());
234 0 : }
235 : // cout << "nextLine has read '" << currentLine << "'" << endl;
236 997988 : return currentLine;
237 : }
238 :
239 303108 : pair<string, string> SDMDataObjectStreamReader::headerField2Pair(const string& hf){
240 303108 : string name, value;
241 303108 : size_t colonIndex = hf.find(":");
242 303108 : if (colonIndex == string::npos)
243 0 : throw SDMDataObjectStreamReaderException(" could not detect a well formed MIME header field in '"+hf+"'");
244 :
245 303108 : if (colonIndex > 0) {
246 303108 : name = hf.substr(0, colonIndex);
247 : #ifndef WITHOUT_BOOST
248 : boost::algorithm::trim(name);
249 : #else
250 303108 : asdm::trim(name);
251 : #endif
252 : }
253 :
254 303108 : if (colonIndex < hf.size()) {
255 303108 : value = hf.substr(colonIndex+1);
256 : #ifndef WITHOUT_BOOST
257 : boost::algorithm::trim(value);
258 : #else
259 303108 : asdm::trim(value);
260 : #endif
261 : }
262 :
263 606216 : return make_pair(name, value);
264 303108 : }
265 :
266 300519 : pair<string, string> SDMDataObjectStreamReader::requireHeaderField(const string & hf) {
267 300519 : pair<string, string> hf2pair(headerField2Pair(nextLine()));
268 : // cout << hf2pair.first << ", " << hf2pair.second << endl;
269 : #ifndef WITHOUT_BOOST
270 : if (boost::algorithm::to_upper_copy(hf2pair.first) != hf)
271 : throw SDMDataObjectStreamReaderException("read '" + currentLine + "'. Was expecting '" + hf + "'...");
272 : #else
273 300519 : if (asdm::str_toupper(hf2pair.first) != hf)
274 0 : throw SDMDataObjectStreamReaderException("read '" + currentLine + "'. Was expecting '" + hf + "'...");
275 : #endif
276 300519 : return hf2pair;
277 0 : }
278 :
279 44389 : string unquote(const string& s, string& unquoted) {
280 44389 : if (s.size() >= 2)
281 44389 : if (((s.at(0) == '"') && (s.at(s.size()-1) == '"')) || ((s.at(0) == '\'') && (s.at(s.size()-1) == '\''))) {
282 42716 : if (s.size() == 2)
283 0 : unquoted = "";
284 : else
285 42716 : unquoted = s.substr(1, s.size() - 2);
286 : }
287 : else
288 1673 : unquoted = s;
289 : else
290 0 : unquoted = s;
291 44389 : return unquoted;
292 : }
293 :
294 :
295 :
296 44389 : string SDMDataObjectStreamReader::requireBoundaryInCT(const string& ctValue) {
297 44389 : vector<string> cvValueItems;
298 :
299 : #ifndef WITHOUT_BOOST
300 : boost::algorithm::split (cvValueItems, ctValue, boost::algorithm::is_any_of(";"));
301 : vector<string> cvValueItemsNameValue;
302 : for ( vector<string>::const_iterator iter = cvValueItems.begin(); iter != cvValueItems.end() ; iter++ ) {
303 : cvValueItemsNameValue.clear();
304 : boost::algorithm::split(cvValueItemsNameValue, *iter, boost::algorithm::is_any_of("="));
305 : string boundary;
306 : if ((cvValueItemsNameValue.size() > 1) && (boost::algorithm::to_upper_copy(boost::algorithm::trim_copy(cvValueItemsNameValue[0])) == "BOUNDARY") && (unquote(cvValueItemsNameValue[1], boundary).size() > 0))
307 : return boundary;
308 : }
309 : #else
310 44389 : asdm::strsplit (ctValue, ';', cvValueItems);
311 44389 : vector<string> cvValueItemsNameValue;
312 88778 : for ( vector<string>::const_iterator iter = cvValueItems.begin(); iter != cvValueItems.end() ; iter++ ) {
313 88778 : cvValueItemsNameValue.clear();
314 88778 : asdm::strsplit(*iter,'=',cvValueItemsNameValue);
315 88778 : string boundary;
316 88778 : if ((cvValueItemsNameValue.size() > 1) && (asdm::str_toupper(asdm::trim_copy(cvValueItemsNameValue[0])) == "BOUNDARY") && (unquote(cvValueItemsNameValue[1], boundary).size() > 0))
317 88778 : return boundary;
318 88778 : }
319 : #endif
320 0 : throw SDMDataObjectStreamReaderException("could not find a boundary definition in '" + ctValue + "'.");
321 44389 : }
322 :
323 41800 : void SDMDataObjectStreamReader::skipAsLongAsLineStartsWith(const string& start) {
324 41800 : int64_t curpos = 0;
325 : do {
326 41800 : curpos = position();
327 41800 : nextLine();
328 : }
329 41800 : while (currentLine.find(start) == 0);
330 41800 : position(curpos);
331 41800 : return;
332 : }
333 :
334 151554 : void SDMDataObjectStreamReader::skipUntilEmptyLine(int maxSkips) {
335 : // cout << "Entering skipUntilEmptyLine" << endl;
336 151554 : int numSkips = 0;
337 151554 : string line = nextLine();
338 247696 : while ((line.size() != 0) && (numSkips <= maxSkips)) {
339 96142 : line = nextLine();
340 96142 : numSkips += 1;
341 : }
342 :
343 151554 : if (numSkips > maxSkips) {
344 0 : ostringstream oss;
345 0 : oss << "could not find an empty line is less than " << maxSkips + 1 << " lines." << endl;
346 0 : throw SDMDataObjectStreamReaderException(oss.str());
347 0 : }
348 : // cout << "Exiting skipUntilEmptyLine" << endl;
349 151554 : }
350 :
351 :
352 2589 : string SDMDataObjectStreamReader::requireMIMEHeader() {
353 : // MIME-Version
354 2589 : pair<string, string>name_value(headerField2Pair(nextLine()));
355 : // cout << name_value.first << "=" << name_value.second << endl;
356 : // if (currentLine != "MIME-Version: 1.0") // a work around for the case when the very first character is not the expected "M" (happened with some corrupted data).
357 : #ifndef WITHOUT_BOOST
358 : if (! boost::algorithm::iends_with(currentLine, "IME-Version: 1.0"))
359 : #else
360 2589 : std::string versionEnd = "IME-Version: 1.0";
361 2589 : if ((currentLine.size()<=versionEnd.size()) || (currentLine.compare((currentLine.size()-versionEnd.size()),versionEnd.size(),versionEnd)!=0))
362 : #endif
363 0 : throw SDMDataObjectStreamReaderException("'MIME-Version: 1.0' missing at the very beginning of the file '"+path+"'.");
364 :
365 : // Content-Type
366 2589 : boundary_1 = requireBoundaryInCT(requireHeaderField("CONTENT-TYPE").second);
367 :
368 : // cout << "boundary_1 =" << boundary_1 << endl;
369 :
370 : // Content-Description
371 2589 : name_value = requireHeaderField("CONTENT-DESCRIPTION");
372 :
373 : // Content-Location
374 2589 : name_value = requireHeaderField("CONTENT-LOCATION");
375 :
376 : // Look for an empty line in the at most 10 subsequent lines.
377 2589 : skipUntilEmptyLine(10);
378 :
379 5178 : return boundary_1;
380 2589 : }
381 :
382 44389 : string SDMDataObjectStreamReader::accumulateUntilBoundary(const string& boundary, int maxLines) {
383 : // cout << "Entering accumulateUntilBoundary with maxLines = " << maxLines << endl;
384 44389 : int numLines = 0;
385 44389 : string line = nextLine();
386 44389 : string result;
387 108431 : while ((numLines <= maxLines) && (line.find("--"+boundary) == string::npos)) {
388 64042 : result += line;
389 64042 : numLines++;
390 64042 : line = nextLine();
391 : }
392 :
393 44389 : if (numLines > maxLines) {
394 0 : ostringstream oss;
395 0 : oss << "could not find the boundary string '"<< boundary << "' in less than " << maxLines + 1 << " lines." << endl;
396 0 : throw SDMDataObjectStreamReaderException(oss.str());
397 0 : }
398 :
399 88778 : return result;
400 44389 : }
401 :
402 44389 : void SDMDataObjectStreamReader::requireBoundary(const string& boundary, int maxLines) {
403 : // cout << "Entering require boundary with boundary == '" << boundary << "' and maxLines = " << maxLines << endl;
404 44389 : int numLines = 0;
405 44389 : string dashdashBoundary = "--"+boundary;
406 44389 : string line = nextLine();
407 46001 : while ((numLines <= maxLines) && (line.compare(dashdashBoundary) != 0)) {
408 1612 : numLines++;
409 1612 : line = nextLine();
410 : }
411 :
412 44389 : if (numLines > maxLines) {
413 0 : ostringstream oss;
414 0 : oss << "could not find the boundary string '"<< boundary << "' in less than " << maxLines + 1 << " lines." << endl;
415 0 : throw SDMDataObjectStreamReaderException(oss.str());
416 0 : }
417 44389 : }
418 :
419 0 : void SDMDataObjectStreamReader::lookForBinaryPartSize(xmlNode* aNode) {
420 0 : const std::regex UINT("[0-9]+");
421 0 : xmlNode *curNode = NULL;
422 :
423 0 : for (curNode = aNode; curNode ; curNode = curNode->next) {
424 0 : if (curNode->type == XML_ELEMENT_NODE) {
425 0 : if (s_partNames.find(string((char *)curNode->name)) != s_partNames.end()){
426 0 : if (xmlHasProp(curNode, (const xmlChar*) "size")) {
427 0 : xmlChar * value = xmlGetProp(curNode, (const xmlChar *) "size");
428 0 : std::cmatch what;
429 0 : if (std::regex_match((char*) value, what, UINT)) {
430 0 : int64_t result = ::atoi(what[0].first);
431 0 : xmlFree(value);
432 0 : binaryPartSize[string((char *) curNode->name)] = result;
433 : }
434 : else {
435 0 : xmlFree(value);
436 0 : throw SDMDataObjectStreamReaderException("In '" + string((const char*) curNode->name) + "' failed to parse the value of '"+string((const char*) value)+"' as an int.");
437 : }
438 0 : }
439 : else {
440 0 : throw SDMDataObjectStreamReaderException("In '" + string((const char*) curNode->name) + "' could not find the attribute 'size'.");
441 : }
442 : }
443 : }
444 0 : lookForBinaryPartSize(curNode->children);
445 : }
446 0 : }
447 :
448 0 : string SDMDataObjectStreamReader::requireCrossDataType(xmlNode* parent) {
449 0 : string result;
450 :
451 0 : string comparee("crossData");
452 0 : xmlNode * child = parent->children;
453 :
454 0 : while ((child != 0) && (comparee.compare((const char*) child->name) != 0))
455 0 : child = child->next;
456 :
457 0 : if ((child == 0) || (child->type != XML_ELEMENT_NODE)) {
458 0 : ostringstream oss;
459 0 : oss << "could not find the element 'crossData'." << endl;
460 0 : throw SDMDataObjectStreamReaderException(oss.str());
461 0 : }
462 :
463 0 : if (xmlHasProp(child, (const xmlChar*) "type")) {
464 0 : xmlChar * value = xmlGetProp(child, (const xmlChar *) "type");
465 0 : result = string((const char *) value);
466 0 : xmlFree(value);
467 0 : return result;
468 : }
469 : else
470 0 : throw SDMDataObjectStreamReaderException("In '" + string((const char*) child->name) + "' could not find the attribute 'type'.");
471 0 : }
472 :
473 0 : void SDMDataObjectStreamReader::printElementNames(xmlNode * a_node) {
474 0 : xmlNode *cur_node = NULL;
475 :
476 0 : for (cur_node = a_node; cur_node; cur_node = cur_node->next) {
477 0 : if (cur_node->type == XML_ELEMENT_NODE) {
478 0 : cout << "node type: Element, name: " << cur_node->name << endl;
479 : }
480 0 : printElementNames(cur_node->children);
481 : }
482 0 : }
483 :
484 2589 : void SDMDataObjectStreamReader::requireSDMDataHeaderMIMEPart() {
485 : //
486 : // Requires the presense of boundary_1
487 2589 : requireBoundary(boundary_1, 0);
488 :
489 : // Ignore header fields
490 : // requireHeaderField("CONTENT-TYPE")
491 : // requireHeaderField("CONTENT-TRANSFER-ENCODING")
492 : // requireHeaderField("CONTENT-LOCATION")
493 :
494 : //
495 : // Look for an empty line at most distant from 100 lines from here.
496 2589 : skipUntilEmptyLine(10);
497 2589 : string sdmDataHeader = accumulateUntilBoundary(boundary_1, 100);
498 :
499 : /*
500 : xmlDoc * doc = xmlReadMemory(sdmDataHeader.data(), sdmDataHeader.size(), "SDMDataHeader.xml", NULL, XML_PARSE_NOBLANKS);
501 : xmlChar* xmlBuff;
502 : int bufferSize;
503 : xmlDocDumpFormatMemory(doc, &xmlBuff, &bufferSize, 1);
504 : cout << string((const char*) xmlBuff, bufferSize) << endl;
505 : xmlFree(xmlBuff);
506 : xmlNode* root_element = xmlDocGetRootElement(doc);
507 : if ( root_element == NULL || root_element->type != XML_ELEMENT_NODE )
508 : throw SDMDataObjectStreamReaderException("Failed to parse the SDMDataHeader into a DOM structure.");
509 : */
510 :
511 2589 : parser.parseMemoryHeader(sdmDataHeader, sdmDataObject);
512 : unsigned int bps;
513 2589 : if ((bps = sdmDataObject.dataStruct().flags().size()))
514 2447 : binaryPartSize["flags"] = bps;
515 :
516 2589 : if ((bps = sdmDataObject.dataStruct().actualTimes().size()))
517 1997 : binaryPartSize["actualTimes"] = bps;
518 :
519 2589 : if ((bps = sdmDataObject.dataStruct().actualDurations().size()))
520 1997 : binaryPartSize["actualDurations"] = bps;
521 :
522 2589 : if ((bps = sdmDataObject.dataStruct().zeroLags().size()))
523 463 : binaryPartSize["zeroLags"] = bps;
524 :
525 2589 : if ((bps = sdmDataObject.dataStruct().autoData().size()))
526 2515 : binaryPartSize["autoData"] = bps;
527 :
528 2589 : if ((bps = sdmDataObject.dataStruct().crossData().size()))
529 807 : binaryPartSize["crossData"] = bps;
530 2589 : }
531 :
532 41800 : void SDMDataObjectStreamReader::requireSDMDataSubsetMIMEPart(SDMDataSubset & sdmDataSubset) {
533 41800 : integrationStartsAt = f.tellg();
534 :
535 41800 : skipAsLongAsLineStartsWith("--"+boundary_1); // <--- this was added to preserve compatiblity with SDMDataObjectReader, which
536 : // was less strict on the respect of the BDF specifications. (cf CAS-8151)
537 : // Here the problem was caused by two successive occurrences of --MIME_boundary_1
538 : // instead of only one as indicated in the specs at the very beginning of
539 : // an SDMDataSubsetHeader. M.Caillat - 4 decembre 2015
540 83600 : pair<string, string> name_value = requireHeaderField("CONTENT-TYPE");
541 41800 : boundary_2 = requireBoundaryInCT(name_value.second);
542 : // cout << "boundary_2 = " << boundary_2 << endl;
543 41800 : name_value = requireHeaderField("CONTENT-DESCRIPTION");
544 41800 : requireBoundary(boundary_2, 10);
545 41800 : skipUntilEmptyLine(10);
546 :
547 : //
548 : // We assume that the subset header can't be longer than 100 lines.
549 : //
550 41800 : string sdmDataSubsetHeader = accumulateUntilBoundary(boundary_2, 100);
551 :
552 : //
553 : // Empty sdmDataSubset if necessary.
554 : //
555 41800 : releaseMemory(sdmDataSubset);
556 :
557 : // We can start to acquire a new SDMDataSubset.
558 41800 : if (sdmDataObject.isCorrelation())
559 41282 : parser.parseMemoryCorrSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
560 : else
561 518 : parser.parseMemoryTPSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
562 :
563 41800 : attachmentFlags.reset();
564 41800 : std::regex BINARYPARTLOC("([0-9]+/)+(actualDurations|actualTimes|autoData|crossData|zeroLags|flags)\\.bin");
565 41800 : bool done = false;
566 146376 : while (!done) {
567 104576 : name_value = requireHeaderField("CONTENT-TYPE");
568 104576 : name_value = requireHeaderField("CONTENT-LOCATION");
569 :
570 104576 : std::smatch what;
571 : #ifndef WITHOUT_BOOST
572 : const string contentLocation = boost::algorithm::trim_copy(name_value.second);
573 : #else
574 104576 : const string contentLocation = asdm::trim_copy(name_value.second);
575 : #endif
576 104576 : if (!std::regex_search(contentLocation, what, BINARYPARTLOC)) {
577 0 : throw SDMDataObjectStreamReaderException("Invalid field : '" + name_value.first + ":" + name_value.second + "'.");
578 : }
579 : // cout << "Binary part name = " << what[2] << "...";
580 104576 : string binaryPartName = string(what[2]);
581 104576 : if (binaryPartSize.find(binaryPartName) == binaryPartSize.end())
582 0 : throw SDMDataObjectStreamReaderException("The size of '"+binaryPartName+"' was not announced in the data header.!");
583 :
584 104576 : if (binaryPartSize[binaryPartName] == 0)
585 0 : throw SDMDataObjectStreamReaderException("The size of '"+binaryPartName+"' was announced as null. I was not expecting a '"+binaryPartName+"' attachment here.");
586 :
587 104576 : skipUntilEmptyLine(10);
588 104576 : int numberOfCharsPerValue = 0;
589 104576 : char** binaryPartPtrPtr = 0;
590 104576 : if (binaryPartName == "actualDurations") {
591 1468 : attachmentFlags.set(ACTUALDURATIONS);
592 1468 : binaryPartPtrPtr = (char**) &sdmDataSubset.actualDurations_;
593 1468 : sdmDataSubset.nActualDurations_ = binaryPartSize[binaryPartName];
594 1468 : sdmDataSubset.actualDurationsPosition_ = f.tellg();
595 1468 : numberOfCharsPerValue = 8;
596 : }
597 103108 : else if (binaryPartName == "actualTimes") {
598 1468 : attachmentFlags.set(ACTUALTIMES);
599 1468 : binaryPartPtrPtr = (char **) &sdmDataSubset.actualTimes_;
600 1468 : sdmDataSubset.nActualTimes_ = binaryPartSize[binaryPartName];
601 1468 : sdmDataSubset.actualTimesPosition_ = f.tellg();
602 1468 : numberOfCharsPerValue = 8;
603 : }
604 101640 : else if (binaryPartName == "autoData") {
605 41688 : attachmentFlags.set(AUTODATA);
606 41688 : binaryPartPtrPtr = (char **) &sdmDataSubset.autoData_;
607 41688 : sdmDataSubset.nAutoData_ = binaryPartSize[binaryPartName];
608 41688 : sdmDataSubset.autoDataPosition_ = f.tellg();
609 41688 : numberOfCharsPerValue = 4;
610 : }
611 59952 : else if (binaryPartName == "crossData") {
612 27912 : attachmentFlags.set(CROSSDATA);
613 27912 : sdmDataSubset.shortCrossData_ = 0;
614 27912 : sdmDataSubset.longCrossData_ = 0;
615 27912 : sdmDataSubset.floatCrossData_ = 0;
616 27912 : sdmDataSubset.nCrossData_ = binaryPartSize[binaryPartName];
617 27912 : sdmDataSubset.crossDataPosition_ = f.tellg();
618 27912 : PrimitiveDataType pdt = sdmDataSubset.crossDataType();
619 27912 : switch (pdt) {
620 23236 : case INT16_TYPE:
621 23236 : binaryPartPtrPtr = (char **) &sdmDataSubset.shortCrossData_;
622 23236 : numberOfCharsPerValue = 2;
623 23236 : break;
624 2952 : case INT32_TYPE:
625 2952 : binaryPartPtrPtr = (char **) &sdmDataSubset.longCrossData_;
626 2952 : numberOfCharsPerValue = 4;
627 2952 : break;
628 1724 : case FLOAT32_TYPE:
629 1724 : binaryPartPtrPtr = (char **) &sdmDataSubset.floatCrossData_;
630 1724 : numberOfCharsPerValue = 4;
631 1724 : break;
632 0 : default:
633 0 : throw SDMDataObjectStreamReaderException("Invalid data type for cross data '" + CPrimitiveDataType::name(pdt) + "'.");
634 : }
635 : }
636 32040 : else if (binaryPartName == "flags") {
637 9800 : attachmentFlags.set(FLAGS);
638 9800 : binaryPartPtrPtr = (char **) &sdmDataSubset.flags_;
639 9800 : sdmDataSubset.nFlags_ = binaryPartSize[binaryPartName];
640 9800 : sdmDataSubset.flagsPosition_ = f.tellg();
641 9800 : numberOfCharsPerValue = 4;
642 : }
643 22240 : else if (binaryPartName == "zeroLags") {
644 22240 : attachmentFlags.set(ZEROLAGS);
645 22240 : binaryPartPtrPtr = (char **) &sdmDataSubset.zeroLags_;
646 22240 : sdmDataSubset.nZeroLags_ = binaryPartSize[binaryPartName];
647 22240 : sdmDataSubset.zeroLagsPosition_ = f.tellg();
648 22240 : numberOfCharsPerValue = 4;
649 : }
650 :
651 104576 : int64_t numberOfCharsToRead = numberOfCharsPerValue * binaryPartSize[binaryPartName];
652 104576 : *binaryPartPtrPtr = new char[numberOfCharsToRead * sizeof(char)];
653 104576 : if (*binaryPartPtrPtr == 0) {
654 0 : ostringstream oss;
655 0 : oss << "Processing integration # " << integrationIndex << ": I could not get memory to store '" << binaryPartName << "'." << endl;
656 0 : throw SDMDataObjectStreamReaderException(oss.str());
657 0 : }
658 :
659 104576 : f.read(*binaryPartPtrPtr, numberOfCharsToRead * sizeof(char));
660 104576 : if (f.fail()) {
661 0 : ostringstream oss;
662 0 : oss << "Processing integration # " << integrationIndex << ": a problem occurred while reading '" << binaryPartName << "'." << endl;
663 0 : throw SDMDataObjectStreamReaderException(oss.str());
664 0 : }
665 104576 : if (f.eof()) {
666 0 : ostringstream oss;
667 0 : oss << "Processing integration # " << integrationIndex << ": a unexpected end of file occurred while reading '" << binaryPartName << "'." << endl;
668 0 : throw SDMDataObjectStreamReaderException(oss.str());
669 0 : }
670 :
671 104576 : string line = nextLine(); // Absorb the nl right after the last byte of the binary attachment
672 104576 : line = nextLine(); // This should boundary_2
673 :
674 104576 : if (line.find("--" + boundary_2) != 0) {
675 0 : ostringstream oss;
676 0 : oss << "Processing integration # " << integrationIndex << ": unexpected '" << line << "' after the binary part '" << binaryPartName << "'." << endl;
677 0 : throw SDMDataObjectStreamReaderException(oss.str());
678 0 : }
679 :
680 104576 : done = line.compare("--" + boundary_2+"--") == 0;
681 104576 : }
682 :
683 : // Now check if the binary attachments found are compatible with the correlation mode
684 : // and if their sizes are equal to what is announced in the global header.
685 : //
686 : // The presence of crossData and autoData depends on the correlation mode.
687 :
688 41800 : switch (sdmDataObject.correlationMode()) {
689 112 : case CROSS_ONLY:
690 112 : if (!attachmentFlags.test(CROSSDATA)) {
691 0 : ostringstream oss;
692 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
693 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationIndex;
694 0 : throw SDMDataObjectStreamReaderException(oss.str());
695 0 : }
696 :
697 112 : if (attachmentFlags.test(AUTODATA)) {
698 0 : ostringstream oss;
699 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
700 0 : oss << "found an unexpected attachment 'autoData' in integration #" << integrationIndex << ".";
701 0 : throw SDMDataObjectStreamReaderException(oss.str());
702 0 : }
703 112 : break;
704 :
705 13888 : case AUTO_ONLY:
706 13888 : if (!attachmentFlags.test(AUTODATA)) {
707 0 : ostringstream oss;
708 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
709 0 : oss << "a binary attachment 'autoData' was expected.in integration #" << integrationIndex << ".";
710 0 : throw SDMDataObjectStreamReaderException(oss.str());
711 0 : }
712 :
713 13888 : if (attachmentFlags.test(CROSSDATA)) {
714 0 : ostringstream oss;
715 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
716 0 : oss << "found an unexpected attachment 'crossData' in integration #" << integrationIndex << ".";
717 0 : throw SDMDataObjectStreamReaderException(oss.str());
718 0 : }
719 13888 : break;
720 :
721 27800 : case CROSS_AND_AUTO:
722 27800 : if (!attachmentFlags.test(AUTODATA)) {
723 0 : ostringstream oss;
724 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
725 0 : oss << "a binary attachment 'autoData' was expected in integration #" << integrationIndex << ".";
726 0 : throw SDMDataObjectStreamReaderException(oss.str());
727 0 : }
728 :
729 27800 : if (!attachmentFlags.test(CROSSDATA)) {
730 0 : ostringstream oss;
731 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
732 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationIndex << ".";
733 0 : throw SDMDataObjectStreamReaderException(oss.str());
734 0 : }
735 27800 : break;
736 :
737 0 : default:
738 0 : throw SDMDataObjectStreamReaderException("Data subset '"+sdmDataSubset.projectPath()+"': unrecognized correlation mode");
739 : break;
740 : }
741 :
742 :
743 41800 : if (attachmentFlags.test(ZEROLAGS)) {
744 : // Refuse the zeroLags attachment if it's not a Correlator or if the correlator is a CORRELATOR_FX (ACA).
745 22240 : if ((sdmDataObject.processorType_ != CORRELATOR) || (sdmDataObject.correlatorType() == FX))
746 0 : throw SDMDataObjectStreamReaderException("zeroLags are not expected from a correlator CORRELATOR_FX");
747 : }
748 41800 : }
749 :
750 82610 : void SDMDataObjectStreamReader::releaseMemory(SDMDataSubset & sdmDataSubset) {
751 : // cout << "SDMDataObjectStreamReader::releaseMemory : entering." << endl;
752 82610 : if (sdmDataSubset.actualTimes_ != 0) {
753 : // cout << "actualTimes" << endl;
754 1468 : delete[] sdmDataSubset.actualTimes_;
755 1468 : sdmDataSubset.actualTimes_ = 0;
756 : }
757 82610 : sdmDataSubset.nActualTimes_ = 0;
758 :
759 :
760 82610 : if (sdmDataSubset.actualDurations_ != 0) {
761 : // cout << "actualDurations" << endl;
762 1468 : delete[] sdmDataSubset.actualDurations_;
763 1468 : sdmDataSubset.actualDurations_ = 0;
764 : }
765 82610 : sdmDataSubset.nActualDurations_ = 0;
766 :
767 82610 : if (sdmDataSubset.flags_ != 0) {
768 : // cout << "Flags" << endl;
769 9800 : delete[] sdmDataSubset.flags_;
770 9800 : sdmDataSubset.flags_ = 0;
771 : }
772 82610 : sdmDataSubset.nFlags_ = 0;
773 :
774 82610 : if (sdmDataSubset.zeroLags_ != 0) {
775 : // cout << "zeroLags" << endl;
776 22240 : delete[] sdmDataSubset.zeroLags_;
777 22240 : sdmDataSubset.zeroLags_ = 0;
778 : }
779 82610 : sdmDataSubset.nZeroLags_ = 0;
780 :
781 82610 : if (sdmDataSubset.autoData_ != 0) {
782 : //cout << "autoData_ = " << sdmDataSubset.autoData_ << " before deletion" << endl;
783 41688 : delete[] sdmDataSubset.autoData_;
784 41688 : sdmDataSubset.autoData_ = 0;
785 : }
786 82610 : sdmDataSubset.nAutoData_ = 0;
787 :
788 82610 : if (sdmDataSubset.shortCrossData_ != 0) {
789 : //cout << "shortCrossData_ = " << sdmDataSubset.shortCrossData_ << " before deletion" << endl;
790 23236 : delete[] sdmDataSubset.shortCrossData_;
791 23236 : sdmDataSubset.shortCrossData_ = 0;
792 : }
793 :
794 82610 : if (sdmDataSubset.longCrossData_ != 0) {
795 : // cout << "longCrossData" << endl;
796 2952 : delete[] sdmDataSubset.longCrossData_;
797 2952 : sdmDataSubset.longCrossData_ = 0;
798 : }
799 :
800 82610 : if (sdmDataSubset.floatCrossData_ != 0) {
801 : // cout << "floatCrossData" << endl;
802 1724 : delete[] sdmDataSubset.floatCrossData_;
803 1724 : sdmDataSubset.floatCrossData_ = 0;
804 : }
805 82610 : sdmDataSubset.nCrossData_ = 0;
806 :
807 : // cout << "SDMDataObjectStreamReader::releaseMemory : exiting." << endl;
808 82610 : return;
809 : }
810 : } // end namespace asdmbinaries
811 :
812 : using namespace asdmbinaries;
813 :
814 : #ifdef TEST_CLASS
815 : int main (int argC, char* argV[]) {
816 : if (argC < 2) {
817 : cout << "a.out filename" << endl;
818 : exit(1);
819 : }
820 :
821 : SDMDataObjectStreamReader ssr;
822 : try {
823 : ssr.open(argV[1]);
824 : cout << ssr.toString() << endl;
825 : while (ssr.hasSubset()) {
826 : const SDMDataSubset & sdmdss = ssr.getSubset();
827 : cout << "block of data #" << ssr.currentIntegrationIndex() << " starting at byte #" << ssr.currentIntegrationStartsAt() << endl;
828 : cout << sdmdss.toString(128) << endl;
829 : }
830 : }
831 : catch (SDMDataObjectStreamReaderException& e) {
832 : cout << e.getMessage() << endl;
833 : }
834 : catch (SDMDataObjectParserException& e) {
835 : cout << e.getMessage() << endl;
836 : }
837 : catch (SDMDataObjectException& e) {
838 : cout << e.getMessage() << endl;
839 : }
840 : }
841 : #endif
|
