commit b5d6e0758faac64045086fcf184b8859e5650b56
Author: Daniel Swanson <popcorn.tomato.dude@gmail.com>
Date:   Thu Jun 3 12:14:47 2021 -0500

    utf-32 in monodix and some type cleanup

diff --git a/lttoolbox/compiler.cc b/lttoolbox/compiler.cc
index d4186f2..f4f00fc 100644
--- a/lttoolbox/compiler.cc
+++ b/lttoolbox/compiler.cc
@@ -226,12 +226,12 @@ Compiler::procParDef()
 }
 
 int
-Compiler::matchTransduction(list<int> const &pi,
-                           list<int> const &pd,
+Compiler::matchTransduction(vector<int> const &pi,
+                           vector<int> const &pd,
                            int state, Transducer &t,
                            double const &entry_weight)
 {
-  list<int>::const_iterator left, right, limleft, limright;
+  vector<int>::const_iterator left, right, limleft, limright;
 
   if(direction == COMPILER_RESTRICTION_LR_VAL)
   {
@@ -336,22 +336,18 @@ Compiler::allBlanks()
 
   for(auto c : text)
   {
-    flag = flag && iswspace(c);
+    flag = flag && u_isspace(c);
   }
 
   return flag;
 }
 
 void
-Compiler::readString(list<int> &result, UString const &name)
+Compiler::readString(vector<int> &result, UString const &name)
 {
   if(name == COMPILER_TEXT_NODE)
   {
-    UString value = XMLParseUtil::readValue(reader);
-    for(unsigned int i = 0, limit = value.size(); i < limit; i++)
-    {
-      result.push_back(static_cast<int>(value[i]));
-    }
+    XMLParseUtil::readValueInto32(reader, result);
   }
   else if(name == COMPILER_M_ELEM)
   {
@@ -471,7 +467,7 @@ Compiler::skip(UString &name, UString const &elem, bool open)
 EntryToken
 Compiler::procIdentity(UString const &wsweight, bool ig)
 {
-  list<int> both_sides;
+  vector<int> both_sides;
   double entry_weight = stod(wsweight);
 
   if(!xmlTextReaderIsEmptyElement(reader))
@@ -499,7 +495,7 @@ Compiler::procIdentity(UString const &wsweight, bool ig)
   EntryToken e;
   if(ig)
   {
-    list<int> right;
+    vector<int> right;
     right.push_back(static_cast<int>(L'#'));
     right.insert(right.end(), both_sides.begin(), both_sides.end());
     e.setSingleTransduction(both_sides, right, entry_weight);
@@ -514,7 +510,7 @@ Compiler::procIdentity(UString const &wsweight, bool ig)
 EntryToken
 Compiler::procTransduction(UString const &wsweight)
 {
-  list<int> lhs, rhs;
+  vector<int> lhs, rhs;
   double entry_weight = stod(wsweight);
   UString name;
 
@@ -941,8 +937,7 @@ Compiler::procRegexp()
 {
   EntryToken et;
   xmlTextReaderRead(reader);
-  UString re = XMLParseUtil::readValue(reader);
-  et.setRegexp(re);
+  et.readRegexp(reader);
   xmlTextReaderRead(reader);
   return et;
 }
diff --git a/lttoolbox/compiler.h b/lttoolbox/compiler.h
index b03c14a..5ad073d 100644
--- a/lttoolbox/compiler.h
+++ b/lttoolbox/compiler.h
@@ -216,7 +216,7 @@ private:
    * @param t the transducer
    * @return the last state of the inserted transduction
    */
-  int matchTransduction(list<int> const &lp, list<int> const &rp,
+  int matchTransduction(vector<int> const &lp, vector<int> const &rp,
                         int state, Transducer &t, double const &entry_weight);
   /**
    * Parse the &lt;p&gt; element
@@ -264,7 +264,7 @@ private:
   void skipBlanks(UString &name);
 
 
-  void readString(list<int> &result, UString const &name);
+  void readString(vector<int> &result, UString const &name);
 
   /**
    * Force an element to be empty, and check for it
diff --git a/lttoolbox/entry_token.cc b/lttoolbox/entry_token.cc
index e23db8a..da00e9a 100644
--- a/lttoolbox/entry_token.cc
+++ b/lttoolbox/entry_token.cc
@@ -68,7 +68,7 @@ EntryToken::setParadigm(UString const &np)
 }
 
 void
-EntryToken::setSingleTransduction(list<int> const &pi, list<int> const &pd, double const ew)
+EntryToken::setSingleTransduction(vector<int> const &pi, vector<int> const &pd, double const ew)
 {
   weight = ew;
   leftSide = pi;
@@ -79,7 +79,15 @@ EntryToken::setSingleTransduction(list<int> const &pi, list<int> const &pd, doub
 void
 EntryToken::setRegexp(UString const &r)
 {
-  myregexp = r;
+  //myregexp = r;
+  myregexp = vector<int32_t>(r.begin(), r.end());
+  type = regexp;
+}
+
+void
+EntryToken::readRegexp(xmlTextReaderPtr reader)
+{
+  XMLParseUtil::readValueInto32(reader, myregexp);
   type = regexp;
 }
 
@@ -107,19 +115,19 @@ EntryToken::paradigmName() const
   return parName;
 }
 
-list<int> const &
+vector<int> const &
 EntryToken::left() const
 {
   return leftSide;
 }
 
-list<int> const &
+vector<int> const &
 EntryToken::right() const
 {
   return rightSide;
 }
 
-UString const &
+vector<int32_t> const &
 EntryToken::regExp() const
 {
   return myregexp;
diff --git a/lttoolbox/entry_token.h b/lttoolbox/entry_token.h
index 804162e..0b4b43a 100644
--- a/lttoolbox/entry_token.h
+++ b/lttoolbox/entry_token.h
@@ -18,8 +18,10 @@
 #define _ENTRYTOKEN_
 
 
-#include <list>
+#include <vector>
 #include <lttoolbox/ustring.h>
+#include <lttoolbox/xml_parse_util.h>
+#include <cstdint>
 
 using namespace std;
 
@@ -52,17 +54,17 @@ private:
   /**
    * Left side of transduction (if 'single_transduction')
    */
-  list<int> leftSide;
+  vector<int> leftSide;
 
   /**
    * Right side of transduction (if 'single_transduction')
    */
-  list<int> rightSide;
+  vector<int> rightSide;
 
   /**
    * Regular expression (if 'regexp')
    */
-  UString myregexp;
+  vector<int32_t> myregexp;
 
   /**
    * copy method
@@ -107,7 +109,7 @@ public:
    * @param pd right part
    * @param ew entry weight
    */
-  void setSingleTransduction(list<int> const &pi, list<int> const &pd, double const ew = 0);
+  void setSingleTransduction(vector<int> const &pi, vector<int> const &pd, double const ew = 0);
 
   /**
    * Set regular expression.
@@ -115,6 +117,12 @@ public:
    */
   void setRegexp(UString const &r);
 
+  /**
+   * More efficient version of setRegexp()
+   * @param reader the current xml parser state
+   */
+  void readRegexp(xmlTextReaderPtr reader);
+
   /**
    * eTest EntryToken to detect if is a paradigm.
    * @return true if it is a paradigm.
@@ -143,19 +151,19 @@ public:
    * Retrieve the left part of the paradigm.
    * @return the left part of the paradigm.
    */
-  list<int> const & left() const;
+  vector<int> const & left() const;
 
   /**
    * Retrieve the right part of the paradigm.
    * @return the right part of the paradigm.
    */
-  list<int> const & right() const;
+  vector<int> const & right() const;
 
   /**
    * Retrieve the regular expression specification.
    * @return the regular expression specification.
    */
-  UString const & regExp() const;
+  vector<int32_t> const & regExp() const;
 
   /**
    * Retrieve the weight value of the entry.
diff --git a/lttoolbox/regexp_compiler.cc b/lttoolbox/regexp_compiler.cc
index 340f953..96151d8 100644
--- a/lttoolbox/regexp_compiler.cc
+++ b/lttoolbox/regexp_compiler.cc
@@ -20,6 +20,7 @@
 #include <iostream>
 
 RegexpCompiler::RegexpCompiler() :
+index(0),
 token(0),
 alphabet(0),
 state(0),
@@ -112,14 +113,14 @@ RegexpCompiler::consume(int const t)
 {
   if(token == t)
   {
-    input = input.substr(1);
-    if(input.empty())
+    index++;
+    if(index == input.size())
     {
       token = FIN_FICHERO;
     }
     else
     {
-      token = input[0];
+      token = input[index];
     }
   }
   else
@@ -129,10 +130,11 @@ RegexpCompiler::consume(int const t)
 }
 
 void
-RegexpCompiler::compile(UString const &er)
+RegexpCompiler::compile(vector<int32_t> const &er)
 {
   input = er;
-  token = static_cast<int>(input[0]);
+  token = input[0];
+  index = 0;
   state = transducer.getInitial();
   S();
   transducer.setFinal(state, default_weight);
diff --git a/lttoolbox/regexp_compiler.h b/lttoolbox/regexp_compiler.h
index da428d9..ab7e460 100644
--- a/lttoolbox/regexp_compiler.h
+++ b/lttoolbox/regexp_compiler.h
@@ -22,6 +22,8 @@
 #include <lttoolbox/transducer.h>
 
 #include <set>
+#include <vector>
+#include <cstdint>
 
 using namespace std;
 
@@ -42,7 +44,12 @@ private:
   /**
    * Input string
    */
-  UString input;
+  vector<int32_t> input;
+
+  /**
+   * Location in the input string
+   */
+  size_t index;
 
   /**
    * Alphabet to encode symbols
@@ -201,7 +208,7 @@ public:
    * Function that parses a regular expression and produces a transducer
    * @param er the regular expression
    */
-  void compile(UString const &er);
+  void compile(vector<int32_t> const &er);
 
   /**
    * Set the decoder of symbols
diff --git a/lttoolbox/xml_parse_util.cc b/lttoolbox/xml_parse_util.cc
index d60f62b..eb5da81 100644
--- a/lttoolbox/xml_parse_util.cc
+++ b/lttoolbox/xml_parse_util.cc
@@ -59,3 +59,12 @@ XMLParseUtil::readValue(xmlTextReaderPtr reader)
   const xmlChar* val = xmlTextReaderConstValue(reader);
   return to_ustring(reinterpret_cast<const char*>(val));
 }
+
+void
+XMLParseUtil::readValueInto32(xmlTextReaderPtr reader, vector<int32_t>& vec)
+{
+  const xmlChar* val = xmlTextReaderConstValue(reader);
+  auto sz = xmlStrlen(val);
+  vec.reserve(vec.size() + sz);
+  utf8::utf8to32(val, val+sz, std::back_inserter(vec));
+}
diff --git a/lttoolbox/xml_parse_util.h b/lttoolbox/xml_parse_util.h
index 2dfdd0f..a9b5c3c 100644
--- a/lttoolbox/xml_parse_util.h
+++ b/lttoolbox/xml_parse_util.h
@@ -20,6 +20,8 @@
 #include <libxml/encoding.h>
 #include <libxml/xmlreader.h>
 #include <lttoolbox/ustring.h>
+#include <vector>
+#include <cstdint>
 
 using namespace std;
 
@@ -35,6 +37,7 @@ public:
 
   static UString readName(xmlTextReaderPtr reader);
   static UString readValue(xmlTextReaderPtr reader);
+  static void readValueInto32(xmlTextReaderPtr reader, vector<int32_t>& vec);
 };
 
 #endif