0030964: Data Exchange - use Standard_ReadLineBuffer within OBJ reader

Standard_ReadLineBuffer now supports a processing of the special multi-line case with \ at the end of the line.

Standard_RedLineBuffer was used to load Stl files

src/RWObj/RWObj_Reader.cxx

@@ -28,6 +28,7 @@
 #include <OSD_Timer.hxx>
 #include <Precision.hxx>
 #include <Standard_CLocaleSentry.hxx>
+#include <Standard_ReadLineBuffer.hxx>
 
 #include <algorithm>
 #include <limits>
@@ -44,25 +45,18 @@ IMPLEMENT_STANDARD_RTTIEXT(RWObj_Reader, Standard_Transient)
 
 namespace
 {
+  // The length of buffer to read (in bytes)
+  static const size_t THE_BUFFER_SIZE = 4 * 1024;
 
   //! Simple wrapper.
   struct RWObj_ReaderFile
   {
     FILE*   File;
-    NCollection_Array1<char> Line;
-    Standard_Integer LineBuffLen;
-    Standard_Integer MaxLineLen;
-    int64_t Position;
     int64_t FileLen;
 
     //! Constructor opening the file.
-    RWObj_ReaderFile (const TCollection_AsciiString& theFile,
+    RWObj_ReaderFile (const TCollection_AsciiString& theFile)
-                      const Standard_Integer theMaxLineLen = 256)
     : File (OSD_OpenFile (theFile.ToCString(), "rb")),
-      Line (0, theMaxLineLen - 1),
-      LineBuffLen (theMaxLineLen),
-      MaxLineLen (theMaxLineLen),
-      Position (0),
       FileLen (0)
     {
       if (this->File != NULL)
@@ -82,59 +76,6 @@ namespace
         ::fclose (File);
       }
     }
-
-    //! Read line, also considers multi-line syntax (when last line symbol is slash).
-    bool ReadLine()
-    {
-      int64_t aPosPrev = this->Position;
-      char* aLine = &Line.ChangeFirst();
-      for (; ::feof (this->File) == 0 && ::fgets (aLine, MaxLineLen - 1, this->File) != NULL; )
-      {
-        const int64_t aPosNew = ::ftell64 (this->File);
-        if (aLine[0] == '#')
-        {
-          Position = aPosNew;
-          return true;
-        }
-
-        const Standard_Integer aNbRead = Standard_Integer(aPosNew - aPosPrev);
-        bool toReadMore = false;
-        for (int aTailIter = aNbRead - 1; aTailIter >= 0; --aTailIter)
-        {
-          if (aLine[aTailIter] != '\n'
-           && aLine[aTailIter] != '\r'
-           && aLine[aTailIter] != '\0')
-          {
-            if (aLine[aTailIter] == '\\')
-            {
-              // multi-line syntax
-              aLine[aTailIter] = ' ';
-              const ptrdiff_t aFullLen = aLine + aTailIter + 1 - &this->Line.First();
-              if (LineBuffLen < aFullLen + MaxLineLen)
-              {
-                LineBuffLen += MaxLineLen;
-                this->Line.Resize (0, LineBuffLen - 1, true);
-              }
-              aLine = &this->Line.ChangeFirst() + aFullLen;
-              toReadMore = true;
-              break;
-            }
-            break;
-          }
-        }
-
-        if (toReadMore)
-        {
-          aPosPrev = aPosNew;
-          continue;
-        }
-
-        Position = aPosNew;
-        return true;
-      }
-      return false;
-    }
-
   };
 
   //! Return TRUE if given polygon has clockwise node order.
@@ -215,6 +156,9 @@ Standard_Boolean RWObj_Reader::read (const TCollection_AsciiString& theFile,
     return Standard_False;
   }
 
+  Standard_ReadLineBuffer aBuffer (THE_BUFFER_SIZE);
+  aBuffer.SetMultilineMode (true);
+
   const Standard_Integer aNbMiBTotal  = Standard_Integer(aFileLen / (1024 * 1024));
   Standard_Integer       aNbMiBPassed = 0;
   Message_ProgressSentry aPSentry (theProgress, "Reading text OBJ file", 0, aNbMiBTotal, 1);
@@ -222,10 +166,19 @@ Standard_Boolean RWObj_Reader::read (const TCollection_AsciiString& theFile,
   aTimer.Start();
 
   bool isStart = true;
-  for (; aFile.ReadLine(); )
+  int64_t aPosition = 0;
+  size_t aLineLen = 0;
+  int64_t aReadBytes = 0;
+  const char* aLine = NULL;
+  for (;;)
   {
+    aLine = aBuffer.ReadLine (aFile.File, aLineLen, aReadBytes);
+    if (aLine == NULL)
+    {
+      break;
+    }
     ++myNbLines;
-    const char* aLine = &aFile.Line.First();
+    aPosition += aReadBytes;
     if (aTimer.ElapsedTime() > 1.0)
     {
       if (!aPSentry.More())
@@ -233,7 +186,7 @@ Standard_Boolean RWObj_Reader::read (const TCollection_AsciiString& theFile,
         return false;
       }
 
-      const Standard_Integer aNbMiBRead = Standard_Integer(aFile.Position / (1024 * 1024));
+      const Standard_Integer aNbMiBRead = Standard_Integer(aPosition / (1024 * 1024));
       for (; aNbMiBPassed < aNbMiBRead; ++aNbMiBPassed) { aPSentry.Next(); }
       aTimer.Reset();
       aTimer.Start();

src/RWStl/RWStl_Reader.cxx

@@ -26,6 +26,7 @@
 #include <OSD_Timer.hxx>
 #include <Precision.hxx>
 #include <Standard_CLocaleSentry.hxx>
+#include <Standard_ReadLineBuffer.hxx>
 
 #include <algorithm>
 #include <limits>
@@ -39,6 +40,12 @@ namespace
   static const size_t THE_STL_SIZEOF_FACET  = 50;
   static const size_t THE_STL_MIN_FILE_SIZE = THE_STL_HEADER_SIZE + THE_STL_SIZEOF_FACET;
 
+  // The length of buffer to read (in bytes)
+  static const size_t THE_BUFFER_SIZE = 1024;
+
+  // Buffer to read
+  Standard_ReadLineBuffer THE_BUFFER (THE_BUFFER_SIZE);
+
   //! Auxiliary tool for merging nodes during STL reading.
   class MergeNodeTool
   {
@@ -274,12 +281,12 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream,
   // use method seekpos() to get true 64-bit offset to enable
   // handling of large files (VS 2010 64-bit)
   const int64_t aStartPos = GETPOS(theStream.tellg());
-  // Note: 1 is added to theUntilPos to be sure to read the last symbol (relevant for files without EOL at the end)
+  size_t aLineLen = 0;
-  const int64_t aEndPos = (theUntilPos > 0 ? 1 + GETPOS(theUntilPos) : std::numeric_limits<int64_t>::max());
+  const char* aLine;
 
   // skip header "solid ..."
-  theStream.ignore ((std::streamsize)(aEndPos - aStartPos), '\n');
+  aLine = THE_BUFFER.ReadLine (theStream, aLineLen);
-  if (!theStream)
+  if (aLine == NULL)
   {
     Message::DefaultMessenger()->Send ("Error: premature end of file", Message_Fail);
     return false;
@@ -294,11 +301,9 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream,
   const int aStepB = 1024 * 1024;
   const Standard_Integer aNbSteps = 1 + Standard_Integer((GETPOS(theUntilPos) - aStartPos) / aStepB);
   Message_ProgressSentry aPSentry (theProgress, "Reading text STL file", 0, aNbSteps, 1);
-
   int64_t aProgressPos = aStartPos + aStepB;
-  const int64_t LINELEN = 1024;
   int aNbLine = 1;
-  char aLine1[LINELEN], aLine2[LINELEN], aLine3[LINELEN];
+
   while (aPSentry.More())
   {
     if (GETPOS(theStream.tellg()) > aProgressPos)
@@ -307,15 +312,18 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream,
       aProgressPos += aStepB;
     }
 
-    char facet[LINELEN], outer[LINELEN];
+    aLine = THE_BUFFER.ReadLine (theStream, aLineLen); // "facet normal nx ny nz"
-    theStream.getline (facet, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); // "facet normal nx ny nz"
+    if (aLine == NULL)
-    if (str_starts_with (facet, "endsolid", 8))
+    {
+      Message::DefaultMessenger()->Send ("Error: premature end of file", Message_Fail);
+      return false;
+    }
+    if (str_starts_with (aLine, "endsolid", 8))
     {
       // end of STL code
       break;
     }
-    theStream.getline (outer, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg()))); // "outer loop"
+    if (!str_starts_with (aLine, "facet", 5))
-    if (!str_starts_with (facet, "facet", 5) || !str_starts_with (outer, "outer", 5))
     {
       TCollection_AsciiString aStr ("Error: unexpected format of facet at line ");
       aStr += aNbLine + 1;
@@ -323,46 +331,56 @@ Standard_Boolean RWStl_Reader::ReadAscii (Standard_IStream& theStream,
       return false;
     }
 
-    theStream.getline (aLine1, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg())));
+    aLine = THE_BUFFER.ReadLine (theStream, aLineLen);  // "outer loop"
-    theStream.getline (aLine2, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg())));
+    if (aLine == NULL || !str_starts_with (aLine, "outer", 5))
-    theStream.getline (aLine3, (std::streamsize)std::min (LINELEN, aEndPos - GETPOS(theStream.tellg())));
+    {
+      TCollection_AsciiString aStr ("Error: unexpected format of facet at line ");
+      aStr += aNbLine + 1;
+      Message::DefaultMessenger()->Send (aStr, Message_Fail);
+      return false;
+    }
+
+    gp_XYZ aVertex[3];
+    Standard_Boolean isEOF = false;
+    for (Standard_Integer i = 0; i < 3; i++)
+    {
+      aLine = THE_BUFFER.ReadLine (theStream, aLineLen);
+      if (aLine == NULL)
+      {
+        isEOF = true;
+        break;
+      }
+      gp_XYZ aReadVertex;
+      if (!ReadVertex (aLine, aReadVertex.ChangeCoord (1), aReadVertex.ChangeCoord (2), aReadVertex.ChangeCoord (3)))
+      {
+        TCollection_AsciiString aStr ("Error: cannot read vertex co-ordinates at line ");
+        aStr += aNbLine;
+        Message::DefaultMessenger()->Send (aStr, Message_Fail);
+        return false;
+      }
+      aVertex[i] = aReadVertex;
+    }
 
     // stop reading if end of file is reached;
     // note that well-formatted file never ends by the vertex line
-    if (theStream.eof() || GETPOS(theStream.tellg()) >= aEndPos)
+    if (isEOF)
     {
       break;
     }
 
-    if (!theStream)
-    {
-      Message::DefaultMessenger()->Send ("Error: premature end of file", Message_Fail);
-      return false;
-    }
     aNbLine += 5;
 
-    Standard_Real x1, y1, z1, x2, y2, z2, x3, y3, z3;
-    if (! ReadVertex (aLine1, x1, y1, z1) ||
-        ! ReadVertex (aLine2, x2, y2, z2) ||
-        ! ReadVertex (aLine3, x3, y3, z3))
-    {
-      TCollection_AsciiString aStr ("Error: cannot read vertex co-ordinates at line ");
-      aStr += aNbLine;
-      Message::DefaultMessenger()->Send(aStr, Message_Fail);
-      return false;
-    }
-
     // add triangle
-    int n1 = aMergeTool.AddNode (x1, y1, z1);
+    int n1 = aMergeTool.AddNode (aVertex[0].X(), aVertex[0].Y(), aVertex[0].Z());
-    int n2 = aMergeTool.AddNode (x2, y2, z2);
+    int n2 = aMergeTool.AddNode (aVertex[1].X(), aVertex[1].Y(), aVertex[1].Z());
-    int n3 = aMergeTool.AddNode (x3, y3, z3);
+    int n3 = aMergeTool.AddNode (aVertex[2].X(), aVertex[2].Y(), aVertex[2].Z());
     if (n1 != n2 && n2 != n3 && n3 != n1)
     {
       AddTriangle (n1, n2, n3);
     }
 
-    theStream.ignore ((std::streamsize)(aEndPos - GETPOS(theStream.tellg())), '\n'); // skip "endloop"
+    THE_BUFFER.ReadLine (theStream, aLineLen); // skip "endloop"
-    theStream.ignore ((std::streamsize)(aEndPos - GETPOS(theStream.tellg())), '\n'); // skip "endfacet"
+    THE_BUFFER.ReadLine (theStream, aLineLen); // skip "endfacet"
 
     aNbLine += 2;
   }

src/Standard/Standard_ReadLineBuffer.hxx

@@ -26,6 +26,7 @@ public:
   //! @param theMaxBufferSizeBytes the length of buffer to read (in bytes)
   Standard_ReadLineBuffer (size_t theMaxBufferSizeBytes)
   : myUseReadBufferLastStr(false),
+    myIsMultilineMode     (false),
     myBufferPos           (0),
     myBytesLastRead       (0)
   {
@@ -41,6 +42,7 @@ public:
   {
     myReadBufferLastStr.clear();
     myUseReadBufferLastStr = false;
+    myIsMultilineMode = false;
     myBufferPos = 0;
     myBytesLastRead = 0;
   }
@@ -71,6 +73,7 @@ public:
                         int64_t& theReadData)
   {
     char* aResultLine = NULL;
+    bool isMultiline = false;
     theLineLength = 0;
     theReadData = 0;
 
@@ -97,7 +100,7 @@ public:
           if (myUseReadBufferLastStr)
           {
             theLineLength = myReadBufferLastStr.size();
-            aResultLine = myReadBufferLastStr.data();
+            aResultLine = &myReadBufferLastStr.front();
             myUseReadBufferLastStr = false;
           }
           break;
@@ -110,9 +113,64 @@ public:
       // read next line from myReadBuffer
       while (myBufferPos < myBytesLastRead)
       {
-        if (myReadBuffer[myBufferPos] == '\n')
+        if (myReadBuffer[myBufferPos] == '\\' && myIsMultilineMode)
         {
-          isEndLineFound = true;
+          // multi-line syntax
+          if (myBufferPos + 1 == myBytesLastRead
+           ||(myBufferPos + 2 == myBytesLastRead && myReadBuffer[myBufferPos + 1] == '\r'))
+          {
+            isMultiline = true;
+          }
+          else if (myReadBuffer[myBufferPos + 1] == '\n'
+                ||(myReadBuffer[myBufferPos + 1] == '\r' && myReadBuffer[myBufferPos + 2] == '\n'))
+          {
+            if (myUseReadBufferLastStr)
+            {
+              myReadBufferLastStr.insert (myReadBufferLastStr.end(), myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos);
+            }
+            else
+            {
+              myReadBufferLastStr = std::vector<char>(myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos);
+              myUseReadBufferLastStr = true;
+            }
+
+            if (myReadBuffer[myBufferPos + 1] == '\r')
+            {
+              myBufferPos += 2;
+            }
+            else
+            {
+              myBufferPos += 1;
+            }
+
+            aStartLinePos = myBufferPos + 1;
+          }
+        }
+        else if (myReadBuffer[myBufferPos] == '\n')
+        {
+          if (!isMultiline)
+          {
+            isEndLineFound = true;
+          }
+          else if (myBufferPos == 1 && myReadBuffer[0] == '\r')
+          {
+            myReadBufferLastStr.erase (myReadBufferLastStr.end() - 1);
+            aStartLinePos += 2;
+            isMultiline = false;
+          }
+          else if (myBufferPos == 0)
+          {
+            aStartLinePos += 1;
+            if (myReadBufferLastStr[myReadBufferLastStr.size() - 1] == '\\')
+            {
+              myReadBufferLastStr.erase (myReadBufferLastStr.end() - 1);
+            }
+            else
+            {
+              myReadBufferLastStr.erase (myReadBufferLastStr.end() - 2, myReadBufferLastStr.end());
+            }
+            isMultiline = false;
+          }
         }
 
         ++myBufferPos;
@@ -128,7 +186,7 @@ public:
           myReadBufferLastStr.insert (myReadBufferLastStr.end(), myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos);
           myUseReadBufferLastStr = false;
           theLineLength = myReadBufferLastStr.size();
-          aResultLine = myReadBufferLastStr.data();
+          aResultLine = &myReadBufferLastStr.front();
         }
         else
         {
@@ -137,7 +195,7 @@ public:
             myReadBufferLastStr.clear();
           }
           theLineLength = myBufferPos - aStartLinePos;
-          aResultLine = myReadBuffer.data() + aStartLinePos;
+          aResultLine = &myReadBuffer.front() + aStartLinePos;
         }
         // make string null terminated by replacing '\n' or '\r' (before '\n') symbol to null character.
         if (theLineLength > 1 && aResultLine[theLineLength - 2] == '\r')
@@ -156,14 +214,27 @@ public:
         // save "unfinished" part of string to additional buffer
         if (aStartLinePos != myBufferPos)
         {
-          myReadBufferLastStr = std::vector<char>(myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos);
+          if (myUseReadBufferLastStr)
-          myUseReadBufferLastStr = true;
+          {
+            myReadBufferLastStr.insert (myReadBufferLastStr.end(), myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos);
+          }
+          else
+          {
+            myReadBufferLastStr = std::vector<char>(myReadBuffer.begin() + aStartLinePos, myReadBuffer.begin() + myBufferPos);
+            myUseReadBufferLastStr = true;
+          }
         }
       }
     }
     return aResultLine;
   }
 
+  //! Returns TRUE when the Multiline Mode is on.
+  bool IsMultilineMode() const { return myIsMultilineMode; }
+
+  //! Sets or unsets the multi-line mode.
+  void SetMultilineMode (bool theMultilineMode) { myIsMultilineMode = theMultilineMode; }
+
 protected:
 
   //! Read from stl stream.
@@ -172,7 +243,7 @@ protected:
                    size_t theLen,
                    size_t& theReadLen)
   {
-    theReadLen = (size_t )theStream.read (myReadBuffer.data(), theLen).gcount();
+    theReadLen = (size_t )theStream.read (&myReadBuffer.front(), theLen).gcount();
     return !theStream.bad();
   }
 
@@ -182,7 +253,7 @@ protected:
                    size_t theLen,
                    size_t& theReadLen)
   {
-    theReadLen = ::fread (myReadBuffer.data(), 1, theLen, theStream);
+    theReadLen = ::fread (&myReadBuffer.front(), 1, theLen, theStream);
     return ::ferror (theStream) == 0;
   }
 
@@ -191,6 +262,7 @@ protected:
   std::vector<char> myReadBuffer;           //!< Temp read buffer
   std::vector<char> myReadBufferLastStr;    //!< Part of last string of myReadBuffer
   bool              myUseReadBufferLastStr; //!< Flag to use myReadBufferLastStr during next line reading
+  bool              myIsMultilineMode;      //!< Flag to process of the special multi-line case at the end of the line
   size_t            myBufferPos;            //!< Current position in myReadBuffer
   size_t            myBytesLastRead;        //!< The number of characters that were read last time from myReadBuffer.
 };

tests/de_mesh/stl_read/D1

@@ -74,7 +74,7 @@ checknbshapes res_one_binary -face 1
 puts "\n#======================================================================"
 puts "# Binary file with no facets -- will be treated as Ascii and generate e r r o r"
 puts "#======================================================================"
-puts "REQUIRED ALL: Error: unexpected format of facet at line 2"
+puts "REQUIRED ALL: Error: premature end of file"
 set fd [open ${imagedir}/${casename}_zero_binary.stl w]
 fconfigure $fd -translation binary
 puts -nonewline $fd "stl [string repeat { } 76][binary format i 0]"
@@ -84,7 +84,7 @@ readstl res_zero_binary ${imagedir}/${casename}_zero_binary.stl -brep
 puts "\n#======================================================================"
 puts "# Empty file"
 puts "#======================================================================"
-puts "REQUIRED ALL: Error: unexpected format of facet at line 2"
+puts "REQUIRED ALL: Error: premature end of file"
 set fd [open ${imagedir}/${casename}_empty.stl w]
 close $fd
 readstl res_empty ${imagedir}/${casename}_empty.stl -brep