From 51ee6a7dbb3a740ba0cfc693ce5e7a83663f05f4 Mon Sep 17 00:00:00 2001 From: mzernova Date: Tue, 17 Sep 2019 16:05:05 +0300 Subject: [PATCH] 0030964: Data Exchange - use Standard_ReadLineBuffer within OBJ reader Standard_ReadLineBuffer now supports a processing of the special multi-line case with \ at the end of the line. Standard_RedLineBuffer was used to load Stl files --- src/RWObj/RWObj_Reader.cxx | 85 +++++----------------- src/RWStl/RWStl_Reader.cxx | 92 ++++++++++++++---------- src/Standard/Standard_ReadLineBuffer.hxx | 90 ++++++++++++++++++++--- tests/de_mesh/stl_read/D1 | 4 +- 4 files changed, 157 insertions(+), 114 deletions(-) diff --git a/src/RWObj/RWObj_Reader.cxx b/src/RWObj/RWObj_Reader.cxx index 9471491129..157d223a2f 100644 --- a/src/RWObj/RWObj_Reader.cxx +++ b/src/RWObj/RWObj_Reader.cxx @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -44,25 +45,18 @@ IMPLEMENT_STANDARD_RTTIEXT(RWObj_Reader, Standard_Transient) namespace { + // The length of buffer to read (in bytes) + static const size_t THE_BUFFER_SIZE = 4 * 1024; //! Simple wrapper. struct RWObj_ReaderFile { FILE* File; - NCollection_Array1 Line; - Standard_Integer LineBuffLen; - Standard_Integer MaxLineLen; - int64_t Position; int64_t FileLen; //! Constructor opening the file. - RWObj_ReaderFile (const TCollection_AsciiString& theFile, - const Standard_Integer theMaxLineLen = 256) + RWObj_ReaderFile (const TCollection_AsciiString& theFile) : File (OSD_OpenFile (theFile.ToCString(), "rb")), - Line (0, theMaxLineLen - 1), - LineBuffLen (theMaxLineLen), - MaxLineLen (theMaxLineLen), - Position (0), FileLen (0) { if (this->File != NULL) @@ -82,59 +76,6 @@ namespace ::fclose (File); } } - - //! Read line, also considers multi-line syntax (when last line symbol is slash). - bool ReadLine() - { - int64_t aPosPrev = this->Position; - char* aLine = &Line.ChangeFirst(); - for (; ::feof (this->File) == 0 && ::fgets (aLine, MaxLineLen - 1, this->File) != NULL; ) - { - const int64_t aPosNew = ::ftell64 (this->File); - if (aLine[0] == '#') - { - Position = aPosNew; - return true; - } - - const Standard_Integer aNbRead = Standard_Integer(aPosNew - aPosPrev); - bool toReadMore = false; - for (int aTailIter = aNbRead - 1; aTailIter >= 0; --aTailIter) - { - if (aLine[aTailIter] != '\n' - && aLine[aTailIter] != '\r' - && aLine[aTailIter] != '\0') - { - if (aLine[aTailIter] == '\\') - { - // multi-line syntax - aLine[aTailIter] = ' '; - const ptrdiff_t aFullLen = aLine + aTailIter + 1 - &this->Line.First(); - if (LineBuffLen < aFullLen + MaxLineLen) - { - LineBuffLen += MaxLineLen; - this->Line.Resize (0, LineBuffLen - 1, true); - } - aLine = &this->Line.ChangeFirst() + aFullLen; - toReadMore = true; - break; - } - break; - } - } - - if (toReadMore) - { - aPosPrev = aPosNew; - continue; - } - - Position = aPosNew; - return true; - } - return false; - } - }; //! Return TRUE if given polygon has clockwise node order. @@ -215,6 +156,9 @@ Standard_Boolean RWObj_Reader::read (const TCollection_AsciiString& theFile, return Standard_False; } + Standard_ReadLineBuffer aBuffer (THE_BUFFER_SIZE); + aBuffer.SetMultilineMode (true); + const Standard_Integer aNbMiBTotal = Standard_Integer(aFileLen / (1024 * 1024)); Standard_Integer aNbMiBPassed = 0; Message_ProgressSentry aPSentry (theProgress, "Reading text OBJ file", 0, aNbMiBTotal, 1); @@ -222,10 +166,19 @@ Standard_Boolean RWObj_Reader::read (const TCollection_AsciiString& theFile, aTimer.Start(); bool isStart = true; - for (; aFile.ReadLine(); ) + int64_t aPosition = 0; + size_t aLineLen = 0; + int64_t aReadBytes = 0; + const char* aLine = NULL; + for (;;) { + aLine = aBuffer.ReadLine (aFile.File, aLineLen, aReadBytes); + if (aLine == NULL) + { + break; + } ++myNbLines; - const char* aLine = &aFile.Line.First(); + aPosition += aReadBytes; if (aTimer.ElapsedTime() > 1.0) { if (!aPSentry.More()) @@ -233,7 +186,7 @@ Standard_Boolean RWObj_Reader::read (const TCollection_AsciiString& theFile, return false; } - const Standard_Integer aNbMiBRead = Standard_Integer(aFile.Position / (1024 * 1024)); + const Standard_Integer aNbMiBRead = Standard_Integer(aPosition / (1024 * 1024)); for (; aNbMiBPassed < aNbMiBRead; ++aNbMiBPassed) { aPSentry.Next(); } aTimer.Reset(); aTimer.Start(); diff --git a/src/RWStl/RWStl_Reader.cxx b/src/RWStl/RWStl_Reader.cxx index 556f31fd94..b490704706 100644 --- a/src/RWStl/RWStl_Reader.cxx +++ b/src/RWStl/RWStl_Reader.cxx @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,12 @@ namespace static const size_t THE_STL_SIZEOF_FACET = 50; static const size_t THE_STL_MIN_FILE_SIZE = THE_STL_HEADER_SIZE + THE_STL_SIZEOF_FACET; + // The length of buffer to read (in bytes) + static const size_t THE_BUFFER_SIZE = 1024; + + // Buffer to read + Standard_ReadLineBuffer THE_BUFFER (THE_BUFFER_SIZE); + //! Auxiliary tool for merging nodes during STL reading. class MergeNodeTool { @@ -274,12 +281,12 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream, // use method seekpos() to get true 64-bit offset to enable // handling of large files (VS 2010 64-bit) const int64_t aStartPos = GETPOS(theStream.tellg()); - // Note: 1 is added to theUntilPos to be sure to read the last symbol (relevant for files without EOL at the end) - const int64_t aEndPos = (theUntilPos > 0 ? 1 + GETPOS(theUntilPos) : std::numeric_limits::max()); + size_t aLineLen = 0; + const char* aLine; // skip header "solid ..." - theStream.ignore ((std::streamsize)(aEndPos - aStartPos), '\n'); - if (!theStream) + aLine = THE_BUFFER.ReadLine (theStream, aLineLen); + if (aLine == NULL) { Message::DefaultMessenger()->Send ("Error: premature end of file", Message_Fail); return false; @@ -294,11 +301,9 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream, const int aStepB = 1024 * 1024; const Standard_Integer aNbSteps = 1 + Standard_Integer((GETPOS(theUntilPos) - aStartPos) / aStepB); Message_ProgressSentry aPSentry (theProgress, "Reading text STL file", 0, aNbSteps, 1); - int64_t aProgressPos = aStartPos + aStepB; - const int64_t LINELEN = 1024; int aNbLine = 1; - char aLine1[LINELEN], aLine2[LINELEN], aLine3[LINELEN]; + while (aPSentry.More()) { if (GETPOS(theStream.tellg()) > aProgressPos) @@ -307,15 +312,18 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream, aProgressPos += aStepB; } - char facet[LINELEN], outer[LINELEN]; - theStream.getline (facet, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); // "facet normal nx ny nz" - if (str_starts_with (facet, "endsolid", 8)) + aLine = THE_BUFFER.ReadLine (theStream, aLineLen); // "facet normal nx ny nz" + if (aLine == NULL) + { + Message::DefaultMessenger()->Send ("Error: premature end of file", Message_Fail); + return false; + } + if (str_starts_with (aLine, "endsolid", 8)) { // end of STL code break; } - theStream.getline (outer, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); // "outer loop" - if (!str_starts_with (facet, "facet", 5) || !str_starts_with (outer, "outer", 5)) + if (!str_starts_with (aLine, "facet", 5)) { TCollection_AsciiString aStr ("Error: unexpected format of facet at line "); aStr += aNbLine + 1; @@ -323,46 +331,56 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream, return false; } - theStream.getline (aLine1, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); - theStream.getline (aLine2, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); - theStream.getline (aLine3, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); + aLine = THE_BUFFER.ReadLine (theStream, aLineLen); // "outer loop" + if (aLine == NULL || !str_starts_with (aLine, "outer", 5)) + { + TCollection_AsciiString aStr ("Error: unexpected format of facet at line "); + aStr += aNbLine + 1; + Message::DefaultMessenger()->Send (aStr, Message_Fail); + return false; + } + + gp_XYZ aVertex[3]; + Standard_Boolean isEOF = false; + for (Standard_Integer i = 0; i < 3; i++) + { + aLine = THE_BUFFER.ReadLine (theStream, aLineLen); + if (aLine == NULL) + { + isEOF = true; + break; + } + gp_XYZ aReadVertex; + if (!ReadVertex (aLine, aReadVertex.ChangeCoord (1), aReadVertex.ChangeCoord (2), aReadVertex.ChangeCoord (3))) + { + TCollection_AsciiString aStr ("Error: cannot read vertex co-ordinates at line "); + aStr += aNbLine; + Message::DefaultMessenger()->Send (aStr, Message_Fail); + return false; + } + aVertex[i] = aReadVertex; + } // stop reading if end of file is reached; // note that well-formatted file never ends by the vertex line - if (theStream.eof() || GETPOS(theStream.tellg()) >= aEndPos) + if (isEOF) { break; } - if (!theStream) - { - Message::DefaultMessenger()->Send ("Error: premature end of file", Message_Fail); - return false; - } aNbLine += 5; - Standard_Real x1, y1, z1, x2, y2, z2, x3, y3, z3; - if (! ReadVertex (aLine1, x1, y1, z1) || - ! ReadVertex (aLine2, x2, y2, z2) || - ! ReadVertex (aLine3, x3, y3, z3)) - { - TCollection_AsciiString aStr ("Error: cannot read vertex co-ordinates at line "); - aStr += aNbLine; - Message::DefaultMessenger()->Send(aStr, Message_Fail); - return false; - } - // add triangle - int n1 = aMergeTool.AddNode (x1, y1, z1); - int n2 = aMergeTool.AddNode (x2, y2, z2); - int n3 = aMergeTool.AddNode (x3, y3, z3); + int n1 = aMergeTool.AddNode (aVertex[0].X(), aVertex[0].Y(), aVertex[0].Z()); + int n2 = aMergeTool.AddNode (aVertex[1].X(), aVertex[1].Y(), aVertex[1].Z()); + int n3 = aMergeTool.AddNode (aVertex[2].X(), aVertex[2].Y(), aVertex[2].Z()); if (n1 != n2 && n2 != n3 && n3 != n1) { AddTriangle (n1, n2, n3); } - theStream.ignore ((std::streamsize)(aEndPos - GETPOS(theStream.tellg())), '\n'); // skip "endloop" - theStream.ignore ((std::streamsize)(aEndPos - GETPOS(theStream.tellg())), '\n'); // skip "endfacet" + THE_BUFFER.ReadLine (theStream, aLineLen); // skip "endloop" + THE_BUFFER.ReadLine (theStream, aLineLen); // skip "endfacet" aNbLine += 2; } diff --git a/src/Standard/Standard_ReadLineBuffer.hxx b/src/Standard/Standard_ReadLineBuffer.hxx index 94ae850e4c..5e320df008 100644 --- a/src/Standard/Standard_ReadLineBuffer.hxx +++ b/src/Standard/Standard_ReadLineBuffer.hxx @@ -26,6 +26,7 @@ public: //! @param theMaxBufferSizeBytes the length of buffer to read (in bytes) Standard_ReadLineBuffer (size_t theMaxBufferSizeBytes) : myUseReadBufferLastStr(false), + myIsMultilineMode (false), myBufferPos (0), myBytesLastRead (0) { @@ -41,6 +42,7 @@ public: { myReadBufferLastStr.clear(); myUseReadBufferLastStr = false; + myIsMultilineMode = false; myBufferPos = 0; myBytesLastRead = 0; } @@ -71,6 +73,7 @@ public: int64_t& theReadData) { char* aResultLine = NULL; + bool isMultiline = false; theLineLength = 0; theReadData = 0; @@ -97,7 +100,7 @@ public: if (myUseReadBufferLastStr) { theLineLength = myReadBufferLastStr.size(); - aResultLine = myReadBufferLastStr.data(); + aResultLine = &myReadBufferLastStr.front(); myUseReadBufferLastStr = false; } break; @@ -110,9 +113,64 @@ public: // read next line from myReadBuffer while (myBufferPos < myBytesLastRead) { - if (myReadBuffer[myBufferPos] == '\n') + if (myReadBuffer[myBufferPos] == '\\' && myIsMultilineMode) { - isEndLineFound = true; + // multi-line syntax + if (myBufferPos + 1 == myBytesLastRead + ||(myBufferPos + 2 == myBytesLastRead && myReadBuffer[myBufferPos + 1] == '\r')) + { + isMultiline = true; + } + else if (myReadBuffer[myBufferPos + 1] == '\n' + ||(myReadBuffer[myBufferPos + 1] == '\r' && myReadBuffer[myBufferPos + 2] == '\n')) + { + if (myUseReadBufferLastStr) + { + myReadBufferLastStr.insert (myReadBufferLastStr.end(), myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos); + } + else + { + myReadBufferLastStr = std::vector(myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos); + myUseReadBufferLastStr = true; + } + + if (myReadBuffer[myBufferPos + 1] == '\r') + { + myBufferPos += 2; + } + else + { + myBufferPos += 1; + } + + aStartLinePos = myBufferPos + 1; + } + } + else if (myReadBuffer[myBufferPos] == '\n') + { + if (!isMultiline) + { + isEndLineFound = true; + } + else if (myBufferPos == 1 && myReadBuffer[0] == '\r') + { + myReadBufferLastStr.erase (myReadBufferLastStr.end() - 1); + aStartLinePos += 2; + isMultiline = false; + } + else if (myBufferPos == 0) + { + aStartLinePos += 1; + if (myReadBufferLastStr[myReadBufferLastStr.size() - 1] == '\\') + { + myReadBufferLastStr.erase (myReadBufferLastStr.end() - 1); + } + else + { + myReadBufferLastStr.erase (myReadBufferLastStr.end() - 2, myReadBufferLastStr.end()); + } + isMultiline = false; + } } ++myBufferPos; @@ -128,7 +186,7 @@ public: myReadBufferLastStr.insert (myReadBufferLastStr.end(), myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos); myUseReadBufferLastStr = false; theLineLength = myReadBufferLastStr.size(); - aResultLine = myReadBufferLastStr.data(); + aResultLine = &myReadBufferLastStr.front(); } else { @@ -137,7 +195,7 @@ public: myReadBufferLastStr.clear(); } theLineLength = myBufferPos - aStartLinePos; - aResultLine = myReadBuffer.data() + aStartLinePos; + aResultLine = &myReadBuffer.front() + aStartLinePos; } // make string null terminated by replacing '\n' or '\r' (before '\n') symbol to null character. if (theLineLength > 1 && aResultLine[theLineLength - 2] == '\r') @@ -156,14 +214,27 @@ public: // save "unfinished" part of string to additional buffer if (aStartLinePos != myBufferPos) { - myReadBufferLastStr = std::vector(myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos); - myUseReadBufferLastStr = true; + if (myUseReadBufferLastStr) + { + myReadBufferLastStr.insert (myReadBufferLastStr.end(), myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos); + } + else + { + myReadBufferLastStr = std::vector(myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos); + myUseReadBufferLastStr = true; + } } } } return aResultLine; } + //! Returns TRUE when the Multiline Mode is on. + bool IsMultilineMode() const { return myIsMultilineMode; } + + //! Sets or unsets the multi-line mode. + void SetMultilineMode (bool theMultilineMode) { myIsMultilineMode = theMultilineMode; } + protected: //! Read from stl stream. @@ -172,7 +243,7 @@ protected: size_t theLen, size_t& theReadLen) { - theReadLen = (size_t )theStream.read (myReadBuffer.data(), theLen).gcount(); + theReadLen = (size_t )theStream.read (&myReadBuffer.front(), theLen).gcount(); return !theStream.bad(); } @@ -182,7 +253,7 @@ protected: size_t theLen, size_t& theReadLen) { - theReadLen = ::fread (myReadBuffer.data(), 1, theLen, theStream); + theReadLen = ::fread (&myReadBuffer.front(), 1, theLen, theStream); return ::ferror (theStream) == 0; } @@ -191,6 +262,7 @@ protected: std::vector myReadBuffer; //!< Temp read buffer std::vector myReadBufferLastStr; //!< Part of last string of myReadBuffer bool myUseReadBufferLastStr; //!< Flag to use myReadBufferLastStr during next line reading + bool myIsMultilineMode; //!< Flag to process of the special multi-line case at the end of the line size_t myBufferPos; //!< Current position in myReadBuffer size_t myBytesLastRead; //!< The number of characters that were read last time from myReadBuffer. }; diff --git a/tests/de_mesh/stl_read/D1 b/tests/de_mesh/stl_read/D1 index e712b80e01..42d946697d 100644 --- a/tests/de_mesh/stl_read/D1 +++ b/tests/de_mesh/stl_read/D1 @@ -74,7 +74,7 @@ checknbshapes res_one_binary -face 1 puts "\n#======================================================================" puts "# Binary file with no facets -- will be treated as Ascii and generate e r r o r" puts "#======================================================================" -puts "REQUIRED ALL: Error: unexpected format of facet at line 2" +puts "REQUIRED ALL: Error: premature end of file" set fd [open ${imagedir}/${casename}_zero_binary.stl w] fconfigure $fd -translation binary puts -nonewline $fd "stl [string repeat { } 76][binary format i 0]" @@ -84,7 +84,7 @@ readstl res_zero_binary ${imagedir}/${casename}_zero_binary.stl -brep puts "\n#======================================================================" puts "# Empty file" puts "#======================================================================" -puts "REQUIRED ALL: Error: unexpected format of facet at line 2" +puts "REQUIRED ALL: Error: premature end of file" set fd [open ${imagedir}/${casename}_empty.stl w] close $fd readstl res_empty ${imagedir}/${casename}_empty.stl -brep