commit ffba25dde80ef66c8c3e9ff03e5af0513c9087bf Author: Tanmai Khanna Date: Wed Jul 22 18:09:54 2020 +0530 fix wblank flushing for consecutive postgeneration activation | added lots of tests diff --git a/lttoolbox/fst_processor.cc b/lttoolbox/fst_processor.cc index 6d12abb..1d6ed9c 100644 --- a/lttoolbox/fst_processor.cc +++ b/lttoolbox/fst_processor.cc @@ -2250,6 +2250,11 @@ FSTProcessor::postgeneration(FILE *input, FILE *output) } else { + if(!need_end_wblank) + { + flushWblanks(output); + } + if(isEscaped(val)) { fputwc_unlocked(L'\\', output); @@ -2262,7 +2267,6 @@ FSTProcessor::postgeneration(FILE *input, FILE *output) need_end_wblank = false; } } - flushWblanks(output); } else { diff --git a/tests/data/postgen.dix b/tests/data/postgen.dix new file mode 100644 index 0000000..d76ce94 --- /dev/null +++ b/tests/data/postgen.dix @@ -0,0 +1,91 @@ + + + + + + + + + + + + +

+ el + l +

+
+ +

+ el + l +

+
+
+ + + +

+ la + sela +

+
+ +

+ las + selas +

+
+ +

+ lo + selo +

+
+ +

+ los + selos +

+
+
+ + +
+ +
+ + +

+ de + de +

+ +
+ + +

+ oho + uho +

+ + + +

+ le + +

+ + + + +

+ les + lepetest +

+ + +
+ +
+ diff --git a/tests/lt_proc/__init__.py b/tests/lt_proc/__init__.py index 5455c48..6670fdc 100644 --- a/tests/lt_proc/__init__.py +++ b/tests/lt_proc/__init__.py @@ -139,5 +139,53 @@ class WordboundBlankAnalysisTest(unittest.TestCase, ProcTest): "[[t:b:456123; t:i:90hfbn]]^legge/legge$ [[t:s:xyz789]]^opp/opp$ ^opp/opp$ [[t:b:abc124]]^x/*x$ ^opp/opp$^./.$", ] +class PostgenerationBasicTest(unittest.TestCase, ProcTest): + procdix = "data/postgen.dix" + procflags = ["-p", "-z"] + inputs = [ "xyz ejemplo ~o ho nombre.", + "xyz ~le la pelota.", + "El perro ~de el amigo.", + "abc ~les testword"] + expectedOutputs = [ "xyz ejemplo u ho nombre.", + "xyz se la pelota.", + "El perro del amigo.", + "abc le pe test testword"] + +class PostgenerationWordboundBlankTest(unittest.TestCase, ProcTest): + procdix = "data/postgen.dix" + procflags = ["-p", "-z"] + inputs = [ "xyz ejemplo [[t:i:123456]]~o[[/]] [[t:b:abc123; t:i:123456]]ho[[/]] [[t:b:iopmnb]]nombre[[/]].", + "xyz ejemplo [[t:b:poim230]]~o[[/]] ho [[t:i:mnbj203]]nombre[[/]].", + "xyz ejemplo ~o [[t:b:abc123; t:i:123456]]ho[[/]] [[t:b:iopmnb]]nombre[[/]].", + "xyz ejemplo ~o [[t:b:abc123; t:i:123456]]ho[[/]] ~le la [[t:b:iopmnb]]nombre[[/]].", + "xyz ejemplo [[t:i:1235gb]]~o[[/]] [[t:b:abc123; t:i:123456]]ho[[/]] [[t:b:i4x56fb]]~le[[/]] la nombre.", + "xyz [[t:i:123456]]~le[[/]] [[t:b:123gfv]]la[[/]] pelota.", + "xyz ~le [[t:b:123gfv]]la[[/]] pelota.", + "xyz ejemplo ~o [[t:b:abc123; t:i:123456]]ho[[/]] ~le [[t:b:io1245b]]la[[/]] [[t:b:iopmnb]]nombre[[/]].", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] [[t:i:4_tPUA]]~de[[/]] [[t:b:Z9eiLA]]el[[/]] [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:h5lVhA]]El[[/]] [[t:b:Z9eiLA; t:i:4_tPUA]]perro[[/]] [[t:b:Z9eiLA; t:i:4_tPUA]]~de[[/]] [[t:b:Z9eiLA]]el[[/]] [[t:i:npAFwg]]amigo[[/]][]", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] [[t:i:4_tPUA]]~de[[/]] el [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] ~de [[t:b:Z9eiLA]]el[[/]] [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] ~de el [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:Z9eiLA]]abc[[/]] [[t:i:123456]]~les[[/]] [[t:b:abc123; t:i:123456]]testword[[/]]", + "[[t:b:Z9eiLA]]abc[[/]] ~les [[t:b:abc123; t:i:123456]]testword[[/]]", + "[[t:b:Z9eiLA]]abc[[/]] [[t:i:123456]]~les[[/]] [[t:i:4_tPUA]]~de[[/]] [[t:b:Z9eiLA]]el[[/]] testword[[/]]"] + + expectedOutputs = [ "xyz ejemplo [[t:i:123456; t:b:abc123; t:i:123456]]u ho[[/]] [[t:b:iopmnb]]nombre[[/]].", + "xyz ejemplo [[t:b:poim230]]u ho[[/]] [[t:i:mnbj203]]nombre[[/]].", + "xyz ejemplo [[t:b:abc123; t:i:123456]]u ho[[/]] [[t:b:iopmnb]]nombre[[/]].", + "xyz ejemplo [[t:b:abc123; t:i:123456]]u ho[[/]] se la [[t:b:iopmnb]]nombre[[/]].", + "xyz ejemplo [[t:i:1235gb; t:b:abc123; t:i:123456]]u ho[[/]] [[t:b:i4x56fb]]se la[[/]] nombre.", + "xyz [[t:i:123456; t:b:123gfv]]se la[[/]] pelota.", + "xyz [[t:b:123gfv]]se la[[/]] pelota.", + "xyz ejemplo [[t:b:abc123; t:i:123456]]u ho[[/]] [[t:b:io1245b]]se la[[/]] [[t:b:iopmnb]]nombre[[/]].", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] [[t:i:4_tPUA; t:b:Z9eiLA]]del[[/]] [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:h5lVhA]]El[[/]] [[t:b:Z9eiLA; t:i:4_tPUA]]perro[[/]] [[t:b:Z9eiLA; t:i:4_tPUA; t:b:Z9eiLA]]del[[/]] [[t:i:npAFwg]]amigo[[/]][]", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] [[t:i:4_tPUA]]del[[/]] [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] [[t:b:Z9eiLA]]del[[/]] [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:Z9eiLA]]El[[/]] [[t:s:8AjRFw]]perro[[/]] del [[t:i:wSM6RQ]]amigo[[/]][]", + "[[t:b:Z9eiLA]]abc[[/]] [[t:i:123456]]le pe test[[/]] [[t:b:abc123; t:i:123456]]testword[[/]]", + "[[t:b:Z9eiLA]]abc[[/]] le pe test [[t:b:abc123; t:i:123456]]testword[[/]]", + "[[t:b:Z9eiLA]]abc[[/]] [[t:i:123456]]le pe test[[/]] [[t:i:4_tPUA; t:b:Z9eiLA]]del[[/]] testword[[/]]"] # These fail on some systems: #from null_flush_invalid_stream_format import *