commit 7a7fb4aa89ba9f728c55b877cf8d324a77251b73
Author: Tanmai Khanna <khanna.tanmai@gmail.com>
Date:   Wed Aug 19 12:40:49 2020 +0530

    fix break statements

diff --git a/tools/src/hfst-proc/tokenizer.cc b/tools/src/hfst-proc/tokenizer.cc
index f3eaee7f..72d0ccfc 100644
--- a/tools/src/hfst-proc/tokenizer.cc
+++ b/tools/src/hfst-proc/tokenizer.cc
@@ -184,7 +184,6 @@ TokenIOStream::read_escaped()
 std::string
 TokenIOStream::read_delimited(const char delim)
 {
-  wcerr << "\n\n##TEST##\n\n";
   std::string result;
   int c = EOF;
   bool is_wblank = false;
@@ -192,19 +191,19 @@ TokenIOStream::read_delimited(const char delim)
   if(is && c != delim) //Check if wblank is being read
   {
     c = is.get();
-    if(c == EOF || c != '[')
-      break;
-    
-    result += c;
-    if(c == '\\')
-      result += read_escaped();
-    else if(null_flush && c == '\0')
-      do_null_flush();
-    else
+    if(c == '[')
     {
-      int next_char = is.peek();
-      if(next_char == '[')
-        is_wblank = true;
+      result += c;
+      if(c == '\\')
+        result += read_escaped();
+      else if(null_flush && c == '\0')
+        do_null_flush();
+      else
+      {
+        int next_char = is.peek();
+        if(next_char == '[')
+          is_wblank = true;
+      }
     }
   }
   
@@ -224,16 +223,16 @@ TokenIOStream::read_delimited(const char delim)
   if(is_wblank)
   {
     c = is.get();
-    if(c == EOF)
-      break;
-    
-    if(c != delim)
-    {
-      stream_error(std::string("Error in parsing a wordbound blank"));
-    }
-    else
+    if(c != EOF)
     {
-      result += c;
+      if(c != delim)
+      {
+        stream_error(std::string("Error in parsing a wordbound blank"));
+      }
+      else
+      {
+        result += c;
+      }
     }
   }