commit 756aaf2d406d3fb09ae09108d1b4b00cbe408b9f
Author: Tanmai Khanna <khanna.tanmai@gmail.com>
Date:   Wed Aug 19 14:06:37 2020 +0530

    fix read_delimited to work for LUs as well

diff --git a/tools/src/hfst-proc/tokenizer.cc b/tools/src/hfst-proc/tokenizer.cc
index 72d0ccfc..255a2e75 100644
--- a/tools/src/hfst-proc/tokenizer.cc
+++ b/tools/src/hfst-proc/tokenizer.cc
@@ -188,20 +188,20 @@ TokenIOStream::read_delimited(const char delim)
   int c = EOF;
   bool is_wblank = false;
   
-  if(is && c != delim) //Check if wblank is being read
+  if(is && c != delim)
   {
     c = is.get();
-    if(c == '[')
+    if(c != EOF)
     {
       result += c;
       if(c == '\\')
         result += read_escaped();
       else if(null_flush && c == '\0')
         do_null_flush();
-      else
+      else if(c == '[')
       {
         int next_char = is.peek();
-        if(next_char == '[')
+        if(next_char == '[') //Check if wblank is being read
           is_wblank = true;
       }
     }