commit 06f58c6f02ee056de5077aa8ff160343b8a91e1e
Author: Daniel Swanson <popcorn.tomato.dude@gmail.com>
Date:   Tue Jun 8 22:00:40 2021 -0500

    add reject-current-rule in t2x and t3x because why not (closes #34)

diff --git a/apertium/interchunk.cc b/apertium/interchunk.cc
index dc44a4f..4fa927c 100644
--- a/apertium/interchunk.cc
+++ b/apertium/interchunk.cc
@@ -460,6 +460,9 @@ Interchunk::interchunk(InputFile& in, UFILE* out)
   }
 
   unsigned int last = input_buffer.getPos();
+  unsigned int prev_last = last;
+  int lastrule_id = -1;
+  set<int> banned_rules;
 
   output = out;
   ms.init(me->getInitial());
@@ -470,8 +473,30 @@ Interchunk::interchunk(InputFile& in, UFILE* out)
     {
       if(lastrule != NULL)
       {
-        applyRule();
-        input_buffer.setPos(last);
+        int words_to_consume = applyRule();
+        if (words_to_consume == -1) {
+          banned_rules.clear();
+          input_buffer.setPos(last);
+        } else if (words_to_consume == 1) {
+          banned_rules.clear();
+          if (prev_last >= input_buffer.getSize()) {
+            input_buffer.setPos(0);
+          } else {
+            input_buffer.setPos(prev_last+1);
+          }
+          while (true) {
+            TransferToken& tt = input_buffer.next();
+            if (tt.getType() == tt_word) {
+              break;
+            }
+          }
+        } else {
+          banned_rules.insert(lastrule_id);
+          input_buffer.setPos(prev_last);
+          input_buffer.next();
+          last = input_buffer.getPos();
+        }
+        lastrule_id = -1;
       }
       else
       {
@@ -480,23 +505,27 @@ Interchunk::interchunk(InputFile& in, UFILE* out)
           tmpword.clear();
           input_buffer.setPos(last);
           input_buffer.next();
+          prev_last = last;
+          banned_rules.clear();
           last = input_buffer.getPos();
           ms.init(me->getInitial());
         }
         else if(tmpblank.size() != 0) {
           write(*tmpblank[0], output);
           tmpblank.clear();
+          prev_last = last;
           last = input_buffer.getPos();
           ms.init(me->getInitial());
         }
       }
     }
-    int val = ms.classifyFinals(me->getFinals());
+    int val = ms.classifyFinals(me->getFinals(), banned_rules);
     if(val != -1)
     {
       size_t lastrule_line = rule_lines[val-1];
       lastrule = rule_map[val-1];
       last = input_buffer.getPos();
+      lastrule_id = val;
 
       last_lword = tmpword.size();
 
@@ -514,38 +543,36 @@ Interchunk::interchunk(InputFile& in, UFILE* out)
 
     switch(current.getType())
     {
-      case tt_word:
-	applyWord(current.getContent());
-        tmpword.push_back(&current.getContent());
-	break;
+    case tt_word:
+      applyWord(current.getContent());
+      tmpword.push_back(&current.getContent());
+      break;
 
-      case tt_blank:
-	ms.step(' ');
-	tmpblank.push_back(&current.getContent());
-	break;
+    case tt_blank:
+      ms.step(' ');
+      tmpblank.push_back(&current.getContent());
+      break;
 
-      case tt_eof:
-	if(tmpword.size() != 0)
-	{
-	  tmpblank.push_back(&current.getContent());
-	  ms.clear();
-	}
-	else
-	{
-      write(current.getContent(), output);
-	  tmpblank.clear();
-	  return;
-	}
-	break;
+    case tt_eof:
+      if(tmpword.size() != 0) {
+        tmpblank.push_back(&current.getContent());
+        ms.clear();
+      }
+      else {
+        write(current.getContent(), output);
+        tmpblank.clear();
+        return;
+      }
+      break;
 
-      default:
-	cerr << "Error: Unknown input token." << endl;
-	return;
+    default:
+      cerr << "Error: Unknown input token." << endl;
+      return;
     }
   }
 }
 
-void
+int
 Interchunk::applyRule()
 {
   unsigned int limit = tmpword.size();
@@ -569,7 +596,7 @@ Interchunk::applyRule()
     word[i] = new InterchunkWord(*tmpword[i]);
   }
 
-  processRule(lastrule);
+  int words_to_consume = processRule(lastrule);
   lastrule = NULL;
 
   if(word)
@@ -585,6 +612,7 @@ Interchunk::applyRule()
   tmpword.clear();
   tmpblank.clear();
   ms.init(me->getInitial());
+  return words_to_consume;
 }
 
 void
diff --git a/apertium/interchunk.dtd b/apertium/interchunk.dtd
index 25e2f66..91af7ec 100644
--- a/apertium/interchunk.dtd
+++ b/apertium/interchunk.dtd
@@ -23,7 +23,7 @@
 
 <!ENTITY % condition "(and|or|not|equal|begins-with|begins-with-list|ends-with|ends-with-list|contains-substring|in)">
 <!ENTITY % container "(var|clip)">
-<!ENTITY % sentence "(let|out|choose|modify-case|call-macro|append)">
+<!ENTITY % sentence "(let|out|choose|modify-case|call-macro|append|reject-current-rule)">
 <!ENTITY % value "(b|clip|lit|lit-tag|var|get-case-from|case-of|concat|chunk)">
 <!ENTITY % stringvalue "(clip|lit|var|get-case-from|case-of)">
 
@@ -425,6 +425,16 @@ get-case-from -->
 <!ELEMENT concat (%value;)+>
 <!-- Concatenates a sequence of values -->
 
+<!ELEMENT reject-current-rule EMPTY>
+<!ATTLIST reject-current-rule shifting (yes|no) #IMPLIED>
+<!--
+      This instruction cancels the execution of the rule being processed.
+      If "shifting" is set to "yes" or is not specified, the matching process
+      consumes exactly one word at the input. If "shifting" is set to "no"
+      then marks the rule to not to be considered in the current matching
+      until the input buffer advances at least one single word
+-->
+
 <!ELEMENT chunk (%value;)+>
 <!--
      Encloses a chunk
diff --git a/apertium/interchunk.h b/apertium/interchunk.h
index bad25f9..7254b9e 100644
--- a/apertium/interchunk.h
+++ b/apertium/interchunk.h
@@ -51,7 +51,7 @@ private:
   UString readBlank(InputFile& in);
   UString readUntil(InputFile& in, int const symbol) const;
   void applyWord(UString const &word_str);
-  void applyRule();
+  int applyRule();
   TransferToken & readToken(InputFile& in);
   bool checkIndex(xmlNode *element, int index, int limit);
   void interchunk_wrapper_null_flush(InputFile& in, UFILE* out);
diff --git a/apertium/mtx_reader.cc b/apertium/mtx_reader.cc
index deb05fb..9281676 100644
--- a/apertium/mtx_reader.cc
+++ b/apertium/mtx_reader.cc
@@ -91,10 +91,7 @@ void MTXReader::emitUInt(int val)
 
 void MTXReader::procCoarseTags()
 {
-  UString tsx_fn_attr = attrib("tag"_u);
-  std::string tsx_fn;
-  utf8::utf16to8(tsx_fn_attr.begin(), tsx_fn_attr.end(), std::back_inserter(tsx_fn));
-  // TODO TODO TODO
+  std::string tsx_fn((const char*) xmlTextReaderGetAttribute(reader, (const xmlChar*) "tag"));
   bool is_abs = ((tsx_fn.size() >= 1 && tsx_fn[0] == '/') ||
                  (tsx_fn.size() >= 2 && tsx_fn[1] == ':'));
   if (!is_abs) {
diff --git a/apertium/postchunk.cc b/apertium/postchunk.cc
index 8084e86..6d2dd9f 100644
--- a/apertium/postchunk.cc
+++ b/apertium/postchunk.cc
@@ -555,6 +555,9 @@ Postchunk::postchunk(InputFile& in, UFILE* out)
   }
 
   unsigned int last = input_buffer.getPos();
+  unsigned int prev_last = last;
+  int lastrule_id = -1;
+  set<int> banned_rules;
 
   output = out;
   ms.init(me->getInitial());
@@ -565,35 +568,59 @@ Postchunk::postchunk(InputFile& in, UFILE* out)
     {
       if(lastrule != NULL)
       {
-	applyRule();
-	input_buffer.setPos(last);
+        int words_to_consume = applyRule();
+        if (words_to_consume == -1) {
+          banned_rules.clear();
+          input_buffer.setPos(last);
+        } else if (words_to_consume == 1) {
+          banned_rules.clear();
+          if (prev_last >= input_buffer.getSize()) {
+            input_buffer.setPos(0);
+          } else {
+            input_buffer.setPos(prev_last+1);
+          }
+          while (true) {
+            TransferToken& tt = input_buffer.next();
+            if (tt.getType() == tt_word) {
+              break;
+            }
+          }
+        } else {
+          banned_rules.insert(lastrule_id);
+          input_buffer.setPos(prev_last);
+          input_buffer.next();
+          last = input_buffer.getPos();
+        }
+        lastrule_id = -1;
       }
       else
       {
-	if(tmpword.size() != 0)
-	{
-	  unchunk(*tmpword[0], output);
-	  tmpword.clear();
-	  input_buffer.setPos(last);
-	  input_buffer.next();
-	  last = input_buffer.getPos();
-	  ms.init(me->getInitial());
-	}
-	else if(tmpblank.size() != 0)
-	{
-      write(*tmpblank[0], output);
-	  tmpblank.clear();
-	  last = input_buffer.getPos();
-	  ms.init(me->getInitial());
-	}
+        if(tmpword.size() != 0) {
+          unchunk(*tmpword[0], output);
+          tmpword.clear();
+          input_buffer.setPos(last);
+          input_buffer.next();
+          prev_last = last;
+          banned_rules.clear();
+          last = input_buffer.getPos();
+          ms.init(me->getInitial());
+        }
+        else if(tmpblank.size() != 0) {
+          write(*tmpblank[0], output);
+          tmpblank.clear();
+          prev_last = last;
+          last = input_buffer.getPos();
+          ms.init(me->getInitial());
+        }
       }
     }
-    int val = ms.classifyFinals(me->getFinals());
+    int val = ms.classifyFinals(me->getFinals(), banned_rules);
     if(val != -1)
     {
       size_t lastrule_line = rule_lines[val-1];
       lastrule = rule_map[val-1];
       last = input_buffer.getPos();
+      lastrule_id = val;
 
       if(trace)
       {
@@ -609,37 +636,35 @@ Postchunk::postchunk(InputFile& in, UFILE* out)
 
     switch(current.getType())
     {
-      case tt_word:
-	applyWord(current.getContent());
-        tmpword.push_back(&current.getContent());
-	break;
+    case tt_word:
+      applyWord(current.getContent());
+      tmpword.push_back(&current.getContent());
+      break;
 
-      case tt_blank:
-	ms.step(' ');
-	tmpblank.push_back(&current.getContent());
-	break;
+    case tt_blank:
+      ms.step(' ');
+      tmpblank.push_back(&current.getContent());
+      break;
 
-      case tt_eof:
-	if(tmpword.size() != 0)
-	{
-	  tmpblank.push_back(&current.getContent());
-	  ms.clear();
-	}
-	else
-	{
-      write(current.getContent(), output);
-	  return;
-	}
-	break;
+    case tt_eof:
+      if(tmpword.size() != 0) {
+        tmpblank.push_back(&current.getContent());
+        ms.clear();
+      }
+      else {
+        write(current.getContent(), output);
+        return;
+      }
+      break;
 
-      default:
-	cerr << "Error: Unknown input token." << endl;
-	return;
+    default:
+      cerr << "Error: Unknown input token." << endl;
+      return;
     }
   }
 }
 
-void
+int
 Postchunk::applyRule()
 {
   UString const chunk = *tmpword[0];
@@ -659,7 +684,7 @@ Postchunk::applyRule()
     word[i] = new InterchunkWord(*tmpword[i-1]);
   }
 
-  processRule(lastrule);
+  int words_to_consume = processRule(lastrule);
   lastrule = NULL;
 
   if(word)
@@ -683,6 +708,7 @@ Postchunk::applyRule()
   tmpword.clear();
   tmpblank.clear();
   ms.init(me->getInitial());
+  return words_to_consume;
 }
 
 void
@@ -852,19 +878,15 @@ Postchunk::unchunk(UString const &chunk, UFILE* output)
         }
         else if(chunk[i] == '<')
         {
-          if(iswdigit(chunk[i+1]))
+          if(u_isdigit(chunk[i+1]))
           {
-            // replace tag
-            // TODO
-            unsigned long value = stoi(chunk.c_str()+i+1) - 1;
-            //unsigned long value = wcstoul(chunk.c_str()+i+1,
-			//		  NULL, 0) - 1;
-            //atoi(chunk.c_str()+i+1)-1;
+            int j = ++i;
+            while (chunk[++i] != '>');
+            unsigned long value = stoi(chunk.substr(j, i-j)) - 1;
             if(vectags.size() > value)
             {
               write(vectags[value], output);
             }
-            while(chunk[++i] != '>');
           }
           else
           {
@@ -882,7 +904,7 @@ Postchunk::unchunk(UString const &chunk, UFILE* output)
           }
           else if(uppercase_first)
           {
-            if(iswalnum(chunk[i])) {
+            if(u_isalnum(chunk[i])) {
               // TODO
               u_fputc(u_toupper(chunk[i]), output);
               uppercase_first = false;
diff --git a/apertium/postchunk.dtd b/apertium/postchunk.dtd
index ea61777..0e4513f 100644
--- a/apertium/postchunk.dtd
+++ b/apertium/postchunk.dtd
@@ -23,7 +23,7 @@
 
 <!ENTITY % condition "(and|or|not|equal|begins-with|begins-with-list|ends-with|ends-with-list|contains-substring|in)">
 <!ENTITY % container "(var|clip)">
-<!ENTITY % sentence "(let|out|choose|modify-case|call-macro|append)">
+<!ENTITY % sentence "(let|out|choose|modify-case|call-macro|append|reject-current-rule)">
 <!ENTITY % value "(b|clip|lit|lit-tag|var|get-case-from|case-of|concat|lu-count|lu|mlu)">
 <!ENTITY % stringvalue "(clip|lit|var|get-case-from|case-of|lu-count)">
 
@@ -414,6 +414,16 @@ get-case-from -->
 <!ELEMENT concat (%value;)+>
 <!-- Concatenates a sequence of values -->
 
+<!ELEMENT reject-current-rule EMPTY>
+<!ATTLIST reject-current-rule shifting (yes|no) #IMPLIED>
+<!--
+      This instruction cancels the execution of the rule being processed.
+      If "shifting" is set to "yes" or is not specified, the matching process
+      consumes exactly one word at the input. If "shifting" is set to "no"
+      then marks the rule to not to be considered in the current matching
+      until the input buffer advances at least one single word
+-->
+
 <!ELEMENT mlu (lu+)>
 <!-- Encloses a multiword -->
 
diff --git a/apertium/postchunk.h b/apertium/postchunk.h
index 43ad7b1..70044e0 100644
--- a/apertium/postchunk.h
+++ b/apertium/postchunk.h
@@ -29,7 +29,7 @@ class Postchunk : public TransferBase
 private:
 
   InterchunkWord **word;
-  
+
   bool in_lu;
   bool in_wblank;
   UString out_wblank;
@@ -44,9 +44,9 @@ private:
   void processCaseOf(xmlNode* element);
   UString processLu(xmlNode* element);
   UString processMlu(xmlNode* element);
-  
+
   UString processChunk(xmlNode* element);
-  
+
   void processLet(xmlNode *localroot);
   void processOut(xmlNode *localroot);
   void processCallMacro(xmlNode *localroot);
@@ -56,7 +56,7 @@ private:
   UString readBlank(InputFile& in);
   UString readUntil(InputFile& in, int const symbol) const;
   void applyWord(UString const &word_str);
-  void applyRule();
+  int applyRule();
   TransferToken & readToken(InputFile& in);
   static void unchunk(UString const &chunk, UFILE *output);
   static vector<UString> getVecTags(UString const &chunk);