postchunk.cc

Bug Summary

File:	postchunk.cc
Warning:	line 363, column 17 Access to field 'name' results in a dereference of a null pointer (loaded from variable 'leftSide')

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name postchunk.cc -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/tmp/build/apertium/apertium-3.9.12+g928~04ac90c6/apertium -resource-dir /usr/lib/llvm-16/lib/clang/16 -D HAVE_CONFIG_H -I . -I .. -I /usr/include/utf8cpp/ -I /usr/local/include -I /usr/include/libxml2 -I /usr/local/include -D PIC -internal-isystem /usr/lib/llvm-16/bin/../include/c++/v1 -internal-isystem /usr/lib/llvm-16/lib/clang/16/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/14/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -std=c++2b -fdeprecated-macro -fdebug-compilation-dir=/tmp/build/apertium/apertium-3.9.12+g928~04ac90c6/apertium -ferror-limit 19 -fgnuc-version=4.2.1 -fno-implicit-modules -fcxx-exceptions -fexceptions -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/build/apertium/scan-build/2024-09-11-155328-205384-1 -x c++ postchunk.cc

1/*
* Copyright (C) 2005--2015 Universitat d'Alacant / Universidad de Alicante
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <https://www.gnu.org/licenses/>.
*/
17#include <apertium/postchunk.h>

19#include <lttoolbox/xml_walk_util.h>
20#include <lttoolbox/string_utils.h>

22#include <iostream>

24using namespace std;

26Postchunk::Postchunk()
27{}

29bool
30Postchunk::checkIndex(xmlNode *element, int index, int limit)
31{
if(index > limit) // Note: Unlike transfer/interchunk, we allow index==limit!
{
  cerr << "Error in " << (char *) doc->URL << ": line " << element->line << ": index > limit" << endl;
  return false;
}
if(index < 0) {
  cerr << "Error in " << (char *) doc->URL << ": line " << element->line << ": index < 0" << endl;
  return false;
}
if(word[index] == 0)
{
  cerr << "Error in " << (char *) doc->URL << ": line " << element->line << ": Null access at word[index]" << endl;
  return false;
}
return true;
47}

49UString
50Postchunk::evalCachedString(xmlNode* element)
51{
TransferInstr& ti = evalStringCache[element];
switch (ti.getType()) {
case ti_clip_tl:
  if (checkIndex(element, ti.getPos(), lword)) {
    if (gettingLemmaFromWord(ti.getContent()) && lword > 1) {
      if (in_lu) {
        out_wblank = combineWblanks(out_wblank, word[ti.getPos()]->getWblank());
      } else if (in_let_var) {
        var_out_wblank[var_val] = combineWblanks(var_out_wblank[var_val],
                                                 word[ti.getPos()]->getWblank());
      }
    }
    return word[ti.getPos()]->chunkPart(attr_items[ti.getContent()]);
  }
  break;

case ti_lu_count:
  return StringUtils::itoa(tmpword.size());

case ti_var:
  if (lword > 1) {
    out_wblank = combineWblanks(out_wblank, var_out_wblank[ti.getContent()]);
  }
  return variables[ti.getContent()];

case ti_lit_tag:
case ti_lit:
  return ti.getContent();

case ti_b:
  if (!blank_queue.empty()) {
    UString retblank = blank_queue.front();
    if (in_out) {
      blank_queue.pop();
    }
    return retblank;
  } else {
    return " "_u;
  }
  break;

case ti_get_case_from:
  if (checkIndex(element, ti.getPos()+1, lword)) {
    return copycase(word[ti.getPos()+1]->chunkPart(attr_items[ti.getContent()]),
                    evalString(ti.getPointer()));
  }
  break;

case ti_case_of_tl:
  if (checkIndex(element, ti.getPos(), lword)) {
    return StringUtils::getcase(word[ti.getPos()]->chunkPart(attr_items[ti.getContent()]));
  }
  break;

default:
  return ""_u;
}
return ""_u;
110}

112void
113Postchunk::processClip(xmlNode* element)
114{
int pos = 0;
UString part;
for(xmlAttr* i = element->properties; i != NULL__null; i = i->next) {
  if (!xmlStrcmp(i->name, (const xmlChar*) "part")) {
    part = to_ustring((const char*) i->children->content);
  } else if (!xmlStrcmp(i->name, (const xmlChar*) "pos")) {
    pos = atoi((const char *)i->children->content);
  }
}
evalStringCache[element] = TransferInstr(ti_clip_tl, part, pos, NULL__null);
125}

127void
128Postchunk::processBlank(xmlNode* element)
129{
if (element->properties == NULL__null) {
  evalStringCache[element] = TransferInstr(ti_b, " "_u, -1);
} else {
  int pos = atoi((const char *) element->properties->children->content) - 1;
  evalStringCache[element] = TransferInstr(ti_b, ""_u, pos);
}
136}

138void
139Postchunk::processLuCount(xmlNode* element)
140{
evalStringCache[element] = TransferInstr(ti_lu_count, ""_u, 0);
142}

144void
145Postchunk::processCaseOf(xmlNode* element)
146{
int pos = 0;
UString part;
for (xmlAttr* i = element->properties; i != NULL__null; i = i->next) {
  if (!xmlStrcmp(i->name, (const xmlChar*) "part")) {
    part = to_ustring((const char*) i->children->content);
  } else if(!xmlStrcmp(i->name, (const xmlChar*) "pos")) {
    pos = atoi((const char *) i->children->content);
  }
}
evalStringCache[element] = TransferInstr(ti_case_of_tl, part, pos);
157}

159UString
160Postchunk::processLu(xmlNode* element)
161{
in_lu = true;
out_wblank.clear();

UString myword;
for (auto i : children(element)) {
  myword.append(evalString(i));
}
in_lu = false;

if (lword == 1) {
  out_wblank = word[1]->getWblank();
}

if (myword.empty()) {
  return ""_u;
} else {
  return out_wblank+"^"_u+myword+"$"_u;
}
180}

182UString
183Postchunk::processMlu(xmlNode* element)
184{
UString value;

bool first_time = true;
out_wblank.clear();
in_lu = true;

for (auto i : children(element)) {
  UString myword;

  for (auto j : children(i)) {
    myword.append(evalString(j));
  }

if (!first_time) {
    if(!myword.empty() && myword[0] != '#') {  //'+#' problem
      value += '+';
    }
  } else {
    if (!myword.empty()) {
      first_time = false;
    }
  }

value.append(myword);
}

in_lu = false;

if (lword == 1) {
  out_wblank = word[1]->getWblank();
}

if (value.empty()) {
  return ""_u;
} else {
  return out_wblank+"^"_u+value+"$"_u;
}
222}

224UString
225Postchunk::processChunk(xmlNode* element)
226{
cerr << "Error: unexpected expression: '" << element->name << "'" << endl;
exit(EXIT_FAILURE1);
return ""_u; // make the type checker happy
230}

232void
233Postchunk::processOut(xmlNode *localroot)
234{
in_out = true;

for (auto i : children(localroot)) {
  if(!xmlStrcmp(i->name, (const xmlChar *) "lu")) {
    write(processLu(i), output);
  } else if(!xmlStrcmp(i->name, (const xmlChar *) "mlu")) {
    write(processMlu(i), output);
  } else { // 'b'
    write(evalString(i), output);
  }
}

in_out = false;
248}

250void
251Postchunk::processTags(xmlNode *localroot)
252{
for (auto i : children(localroot)) {
  if(!xmlStrcmp(i->name, (xmlChar const *) "tag")) {
    for (auto j : children(i)) {
      write(evalString(j), output);
    }
  }
}
260}

262void
263Postchunk::processLet(xmlNode *localroot)
264{
xmlNode *leftSide = NULL__null, *rightSide = NULL__null;

for (auto i : children(localroot)) {
  if(leftSide == NULL__null) {
    leftSide = i;
  } else {
    rightSide = i;
    break;
  }
}

map<xmlNode *, TransferInstr>::iterator it = evalStringCache.find(leftSide);
if(it != evalStringCache.end())
{
  TransferInstr &ti = it->second;
  switch(ti.getType())
  {
    case ti_var:
      in_let_var = true;
      var_val = ti.getContent();
      var_out_wblank[var_val].clear();

      variables[ti.getContent()] = evalString(rightSide);

      in_let_var = false;
      return;

    case ti_clip_tl:
    {
      bool match = word[ti.getPos()]->setChunkPart(attr_items[ti.getContent()], evalString(rightSide));
      if(!match && trace)
      {
        cerr << "apertium-postchunk warning: <let> on line " << localroot->line << " sometimes discards its value." << endl;
      }
    }
      return;

    default:
      return;
  }
}
if(!xmlStrcmp(leftSide->name, (const xmlChar *) "var"))
{
  in_let_var = true;

  UString const val = to_ustring((const char *) leftSide->properties->children->content);

  var_val = val;
  var_out_wblank[var_val].clear();

  variables[val] = evalString(rightSide);

  in_let_var = false;
  evalStringCache[leftSide] = TransferInstr(ti_var, val, 0);
}
else if(!xmlStrcmp(leftSide->name, (const xmlChar *) "clip"))
{
  int pos = 0;
  UString part;

  for(xmlAttr *i = leftSide->properties; i != NULL__null; i = i->next)
  {
    if(!xmlStrcmp(i->name, (const xmlChar *) "part"))
    {
      part = to_ustring((const char*)i->children->content);
    }
    else if(!xmlStrcmp(i->name, (const xmlChar *) "pos"))
    {
pos = atoi((const char *) i->children->content);
    }
  }


  bool match = word[pos]->setChunkPart(attr_items[part],
			 evalString(rightSide));
  if(!match && trace)
  {
    cerr << "apertium-postchunk warning: <let> on line " << localroot->line << " sometimes discards its value." << endl;
  }
  evalStringCache[leftSide] = TransferInstr(ti_clip_tl, part, pos, NULL__null);
}
346}

348void
349Postchunk::processModifyCase(xmlNode *localroot)
350{
if (dictionary_case) return;
1
Assuming field 'dictionary_case' is false→
2
←
Taking false branch→
xmlNode *leftSide = NULL__null, *rightSide = NULL__null;
3
←
'leftSide' initialized to a null pointer value→

for (auto i : children(localroot)) {
  if(leftSide == NULL__null) {
    leftSide = i;
  } else {
    rightSide = i;
    break;
  }
}

if(!xmlStrcmp(leftSide->name, (const xmlChar *) "clip"))
4
←
Access to field 'name' results in a dereference of a null pointer (loaded from variable 'leftSide')
{
  int pos = 0;
  UString part;

  for(xmlAttr *i = leftSide->properties; i != NULL__null; i = i->next)
  {
    if(!xmlStrcmp(i->name, (const xmlChar *) "part"))
    {
      part = to_ustring((const char*)i->children->content);
    }
    else if(!xmlStrcmp(i->name, (const xmlChar *) "pos"))
    {
      pos = atoi((const char *) i->children->content);
    }
  }

  UString const result = StringUtils::copycase(evalString(rightSide),
		   word[pos]->chunkPart(attr_items[part]));
  bool match = word[pos]->setChunkPart(attr_items[part], result);

  if(!match && trace)
  {
    cerr << "apertium-postchunk warning: <modify-case> on line " << localroot->line << " sometimes discards its value." << endl;
  }
}
else if(!xmlStrcmp(leftSide->name, (const xmlChar *) "var"))
{
  UString const val = to_ustring((const char *) leftSide->properties->children->content);
  variables[val] = StringUtils::copycase(evalString(rightSide), variables[val]);
}
394}

396void
397Postchunk::processCallMacro(xmlNode *localroot)
398{
UString n = to_ustring((const char *) localroot->properties->children->content);
int npar = 0;

xmlNode *macro = macro_map[macros[n]];

for(xmlAttr *i = macro->properties; i != NULL__null; i = i->next)
{
  if(!xmlStrcmp(i->name, (const xmlChar *) "npar"))
  {
    npar = atoi((const char *) i->children->content);
    break;
  }
}

if (npar <= 0)
{
  throw "Postchunk::processCallMacro() assumes npar > 0, but got npar <= 0";
}

InterchunkWord **myword = NULL__null;
if(npar > 0)
{
  myword = new InterchunkWord *[npar+1];
}

myword[0] = word[0];

bool indexesOK = true;
int idx = 1;
for (auto i : children(localroot)) {
  int pos = atoi((const char *) i->properties->children->content);
  if(!checkIndex(localroot, pos, lword)) {
    indexesOK = false;      // avoid segfaulting on empty chunks, e.g. ^x<x>{}$
    pos = 1;
  }
  myword[idx] = word[pos];
  idx++;
}

swap(myword, word);
swap(npar, lword);

if(indexesOK) {
  for (auto i : children(macro)) {
    processInstruction(i);
  }
}
else {
  cerr << "Warning: Not calling macro \"" << n << "\" from line " << localroot->line << " (empty word?)" << endl;
}

swap(myword, word);
swap(npar, lword);

delete[] myword;
454}

456TransferToken &
457Postchunk::readToken(InputFile& in)
458{
if(!input_buffer.isEmpty())
{
  return input_buffer.next();
}

UString content;
while(true)
{
  UChar32 val = in.get();
  if(in.eof() || (internal_null_flush && val == 0))
  {
    return input_buffer.add(TransferToken(content, tt_eof));
  }
  if(val == '\\')
  {
    content += '\\';
    content += in.get();
  }
  else if(val == '[')
  {
    content += '[';
    while(true)
    {
      UChar32 val2 = in.get();
      if(val2 == '\\') {
        content += '\\';
        content += in.get();
      } else if(val2 == ']') {
        content += ']';
        break;
      } else {
        content += val2;
      }
    }
  }
  else if(inword && val == '{')
  {
    content += '{';
    while(true) {
      UChar32 val2 = in.get();
      if(val2 == '\\') {
        content += '\\';
        content += in.get();
      } else if(val2 == '}') {
        UChar32 val3 = in.peek();
        content += '}';
        if(val3 == '$') {
          break;
        }
      } else {
        content += val2;
      }
    }
  }
  else if(inword && val == '$')
  {
    inword = false;
    return input_buffer.add(TransferToken(content, tt_word));
  }
  else if(val == '^')
  {
    inword = true;
    return input_buffer.add(TransferToken(content, tt_blank));
  }
  else
  {
    content += val;
  }
}
528}

530void
531Postchunk::postchunk_wrapper_null_flush(InputFile& in, UFILE* out)
532{
null_flush = false;
internal_null_flush = true;

while(!in.eof())
{
  postchunk(in, out);
  u_fputcu_fputc_72('\0', out);
  u_fflushu_fflush_72(out);
  variables = variable_defaults;
}

internal_null_flush = false;
null_flush = true;
546}

548void
549Postchunk::postchunk(InputFile& in, UFILE* out)
550{
if(getNullFlush())
{
  postchunk_wrapper_null_flush(in, out);
}

unsigned int last = input_buffer.getPos();
unsigned int prev_last = last;
int lastrule_id = -1;
set<int> banned_rules;

output = out;
ms.init(me->getInitial());

while(true)
{
  if(ms.size() == 0)
  {
    if(lastrule != NULL__null)
    {
      int num_words_to_consume = applyRule();

      //Consume all the words from the input which matched the rule.
      //This piece of code is executed unless the rule contains a "reject-current-rule" instruction
      if(num_words_to_consume < 0)
      {
        banned_rules.clear();
        input_buffer.setPos(last);
      }
      else if(num_words_to_consume > 0)
      {
        banned_rules.clear();
        if(prev_last >= input_buffer.getSize())
        {
          input_buffer.setPos(0);
        }
        else
        {
          input_buffer.setPos(prev_last+1);
        }
        int num_consumed_words = 0;
        while(num_consumed_words < num_words_to_consume && !input_buffer.isEmpty())
        {
          TransferToken& local_tt = input_buffer.next();
          if (local_tt.getType() == tt_word)
          {
            num_consumed_words++;
          }
        }
      }
      else
      {
        //Add rule to banned rules
        banned_rules.insert(lastrule_id);
        input_buffer.setPos(prev_last);
        input_buffer.next();
        last = input_buffer.getPos();
      }
      lastrule_id = -1;
    }
    else
    {
      if(tmpword.size() != 0) {
        unchunk(*tmpword[0], output);
        tmpword.clear();
        input_buffer.setPos(last);
        input_buffer.next();
        prev_last = last;
        banned_rules.clear();
        last = input_buffer.getPos();
        ms.init(me->getInitial());
      }
      else if(tmpblank.size() != 0) {
        write(*tmpblank[0], output);
        tmpblank.clear();
        prev_last = last;
        last = input_buffer.getPos();
        ms.init(me->getInitial());
      }
    }
  }
  int val = ms.classifyFinals(me->getFinals(), banned_rules);
  if(val != -1)
  {
    size_t lastrule_line = rule_lines[val-1];
    lastrule = rule_map[val-1];
    last = input_buffer.getPos();
    lastrule_id = val;

    if(trace)
    {
      cerr << endl << "apertium-postchunk: Rule " << val << " line " << lastrule_line;
      for (auto& it : tmpword) {
        cerr << " " << *it;
      }
      cerr << endl;
    }
  }

  TransferToken &current = readToken(in);

  switch(current.getType())
  {
  case tt_word:
    applyWord(current.getContent());
    tmpword.push_back(&current.getContent());
    break;

  case tt_blank:
    ms.step(' ');
    tmpblank.push_back(&current.getContent());
    break;

  case tt_eof:
    if(tmpword.size() != 0) {
      tmpblank.push_back(&current.getContent());
      ms.clear();
    }
    else {
      write(current.getContent(), output);
      return;
    }
    break;

  default:
    cerr << "Error: Unknown input token." << endl;
    return;
  }
}
679}

681int
682Postchunk::applyRule()
683{
UString const chunk = *tmpword[0];
tmpword.clear();
splitWordsAndBlanks(chunk, tmpword, tmpblank);

word = new InterchunkWord *[tmpword.size()+1];
lword = tmpword.size();
word[0] = new InterchunkWord(wordzero(chunk));

for(unsigned int i = 1, limit = tmpword.size()+1; i != limit; i++)
{
  if(i != 1) {
    blank_queue.push(*tmpblank[i-1]);
  }

  word[i] = new InterchunkWord(*tmpword[i-1]);
}

int words_to_consume = processRule(lastrule);
lastrule = NULL__null;

if(word)
{
  for(unsigned int i = 0, limit = tmpword.size() + 1; i != limit; i++)
  {
    delete word[i];
  }
  delete[] word;
}
word = NULL__null;

for(unsigned int i = 0, limit = tmpword.size(); i != limit; i++)
{
  if(i != 0)
  {
    delete tmpblank[i];
  }
  delete tmpword[i];
}
tmpword.clear();
tmpblank.clear();
ms.init(me->getInitial());
return words_to_consume;
726}

728void
729Postchunk::applyWord(UString const &word_str)
730{
ms.step('^');
for(unsigned int i = 0, limit = word_str.size(); i < limit; i++)
{
  switch(word_str[i])
  {
    case '\\':
      i++;
ms.step(u_toloweru_tolower_72(word_str[i]), any_char);
break;

    case '<':
742/*	for(unsigned int j = i+1; j != limit; j++)
{
 if(word_str[j] == '>')
 {
   int symbol = alphabet(word_str.substr(i, j-i+1));
   if(symbol)
   {
     ms.step(symbol, any_tag);
   }
   else
   {
     ms.step(any_tag);
   }
   i = j;
   break;
 }
}
break;*/

    case '{':  // ignore the unmodifiable part of the chunk
      ms.step('$');
      return;

    default:
ms.step(u_toloweru_tolower_72(word_str[i]), any_char);
break;
  }
}
ms.step('$');
771}

773vector<UString>
774Postchunk::getVecTags(UString const &chunk)
775{
vector<UString> vectags;

for(int i = 0, limit = chunk.size(); i != limit; i++)
{
  if(chunk[i] == '\\')
  {
    i++;
  }
  else if(chunk[i] == '<')
  {
    UString mytag;
    do
    {
      mytag += chunk[i++];
    }
    while(chunk[i] != '>');
    mytag += '>';
    vectags.push_back(mytag);
  }
  else if(chunk[i] == '{')
  {
    break;
  }
}
return vectags;
801}

803int
804Postchunk::beginChunk(UString const &chunk)
805{
for(int i = 0, limit = chunk.size(); i != limit; i++)
{
  if(chunk[i] == '\\')
  {
    i++;
  }
  else if(chunk[i] == '{')
  {
    return i + 1;
  }
}
return chunk.size();
818}

820int
821Postchunk::endChunk(UString const &chunk)
822{
return chunk.size()-2;
824}

826UString
827Postchunk::wordzero(UString const &chunk)
828{
for(unsigned int i = 0, limit = chunk.size(); i != limit ;i++)
{
  if(chunk[i] == '\\')
  {
    i++;
  }
  else if(chunk[i] == '{')
  {
    return chunk.substr(0, i);
  }
}

return ""_u;
842}

844UString
845Postchunk::pseudolemma(UString const &chunk)
846{
for(unsigned int i = 0, limit = chunk.size(); i != limit ;i++)
{
  if(chunk[i] == '\\')
  {
    i++;
  }
  else if(chunk[i] == '<' || chunk[i] == '{')
  {
    return chunk.substr(0, i);
  }
}

return ""_u;
860}

862void
863Postchunk::unchunk(UString const &chunk, UFILE* output)
864{
vector<UString> vectags = getVecTags(chunk);
UString case_info = StringUtils::getcase(pseudolemma(chunk));
bool uppercase_all = false;
bool uppercase_first = false;

if(!dictionary_case && case_info == "AA"_u)
{
  uppercase_all = true;
}
else if(!dictionary_case && case_info == "Aa"_u)
{
  uppercase_first = true;
}

for(int i = beginChunk(chunk), limit = endChunk(chunk); i < limit; i++)
{
  if(chunk[i] == '\\') {
    u_fputcu_fputc_72('\\', output);
    u_fputcu_fputc_72(chunk[++i], output);
  } else if(chunk[i] == '^') {
    u_fputcu_fputc_72('^', output);
    while(chunk[++i] != '$')
    {
      if(chunk[i] == '\\')
      {
        u_fputcu_fputc_72('\\', output);
        u_fputcu_fputc_72(chunk[++i], output);
      }
      else if(chunk[i] == '<')
      {
        if(u_isdigitu_isdigit_72(chunk[i+1]))
        {
          int j = ++i;
          while (chunk[++i] != '>');
          unsigned long value = StringUtils::stoi(chunk.substr(j, i-j)) - 1;
          if(vectags.size() > value)
          {
            write(vectags[value], output);
          }
        }
        else
        {
          u_fputcu_fputc_72('<', output);
          while(chunk[++i] != '>') u_fputcu_fputc_72(chunk[i], output);
          u_fputcu_fputc_72('>', output);
        }
      }
      else
      {
        if(uppercase_all)
        {
          // TODO
          u_fputcu_fputc_72(u_toupperu_toupper_72(chunk[i]), output);
        }
        else if(uppercase_first)
        {
          if(u_isalnumu_isalnum_72(chunk[i])) {
            // TODO
            u_fputcu_fputc_72(u_toupperu_toupper_72(chunk[i]), output);
            uppercase_first = false;
          } else {
            u_fputcu_fputc_72(chunk[i], output);
          }
        }
        else
        {
          u_fputcu_fputc_72(chunk[i], output);
        }
      }
    }
    u_fputcu_fputc_72('$', output);
  }
  else if(chunk[i] == '[')
  {
    u_fputcu_fputc_72('[', output);
    while(chunk[++i] != ']')
    {
      if(chunk[i] == '\\')
      {
        u_fputcu_fputc_72('\\', output);
        u_fputcu_fputc_72(chunk[++i], output);
      }
      else
      {
        u_fputcu_fputc_72(chunk[i], output);
      }
    }
    u_fputcu_fputc_72(']', output);
  }
  else
  {
    u_fputcu_fputc_72(chunk[i], output);
  }
}
959}


962void
963Postchunk::splitWordsAndBlanks(UString const &chunk, vector<UString *> &words,
                             vector<UString *> &blanks)
965{
vector<UString> vectags = getVecTags(chunk);
UString case_info = StringUtils::getcase(pseudolemma(chunk));
bool uppercase_all = false;
bool uppercase_first = false;
bool lastblank = true;

if(case_info == "AA"_u)
{
  uppercase_all = true;
}
else if(case_info == "Aa"_u)
{
  uppercase_first = true;
}

for(int i = beginChunk(chunk), limit = endChunk(chunk); i < limit; i++)
{
  if(chunk[i] == '^')
  {
    if(!lastblank)
    {
      blanks.push_back(new UString(""_u));
    }
    lastblank = false;
    UString *myword = new UString();
    UString &ref = *myword;

    while(chunk[++i] != '$')
    {
      if(chunk[i] == '\\')
      {
        ref += '\\';
        ref += chunk[++i];
      }
      else if(chunk[i] == '<')
      {
        if(u_isdigitu_isdigit_72(chunk[i+1]))
        {
          // replace tag
          int j = ++i;
          while (chunk[++i] != '>');
          unsigned long value = StringUtils::stoi(chunk.substr(j, i-j)) - 1;
          if(vectags.size() > value)
          {
            ref.append(vectags[value]);
          }
        }
        else
        {
          ref += '<';
          while(chunk[++i] != '>') ref += chunk[i];
          ref += '>';
        }
      }
      else
      {
        if(uppercase_all)
        {
          // TODO
          ref += u_toupperu_toupper_72(chunk[i]);
        }
        else if(uppercase_first)
        {
          if(u_isalnumu_isalnum_72(chunk[i]))
          {
            // TODO
            ref += u_toupperu_toupper_72(chunk[i]);
            uppercase_first = false;
          }
          else
          {
            ref += chunk[i];
          }
        }
        else
        {
          ref += chunk[i];
        }
      }
    }

    words.push_back(myword);
  }
  else if(chunk[i] == '[')
  {
    if(chunk[i+1] == '[') //wordbound blank
    {
      if(!lastblank)
      {
        blanks.push_back(new UString(""_u));
      }
      lastblank = false;
      UString *myword = new UString();
      UString &ref = *myword;

      while(true)
      {
        if(chunk[i] == '\\')
        {
          ref += '\\';
          ref += chunk[++i];
        }
        else if(chunk[i] == ']' && chunk[i-1] == ']')
        {
          ref += chunk[i];
          i++; //i->"^"_u
          break;
        }
        else
        {
          ref += chunk[i];
        }

        i++;
      }

      while(chunk[++i] != '$')
      {
        if(chunk[i] == '\\')
        {
          ref += '\\';
          ref += chunk[++i];
        }
        else if(chunk[i] == '<')
        {
          if(u_isdigitu_isdigit_72(chunk[i+1]))
          {
            // replace tag
            int j = ++i;
            while (chunk[++i] != '>');
            unsigned long value = StringUtils::stoi(chunk.substr(j, i-j)) - 1;
            if(vectags.size() > value)
            {
              ref.append(vectags[value]);
            }
          }
          else
          {
            ref += '<';
            while(chunk[++i] != '>') ref += chunk[i];
            ref += '>';
          }
        }
        else
        {
          if(uppercase_all)
          {
            // TODO
            ref += u_toupperu_toupper_72(chunk[i]);
          }
          else if(uppercase_first)
          {
            if(u_isalnumu_isalnum_72(chunk[i])) // TODO
            {
              ref += u_toupperu_toupper_72(chunk[i]); // TODO
              uppercase_first = false;
            }
            else
            {
              ref += chunk[i];
            }
          }
          else
          {
            ref += chunk[i];
          }
        }
      }

      words.push_back(myword);
    }
    else
    {
      if (!(lastblank && blanks.back()))
      {
        blanks.push_back(new UString());
      }
      UString &ref = *(blanks.back());
      ref += '[';
      while(chunk[++i] != ']')
      {
        if(chunk[i] == '\\')
        {
          ref += '\\';
          ref += chunk[++i];
        }
        else
        {
          ref += chunk[i];
        }
      }
      ref += chunk[i];

      lastblank = true;
    }
  }
  else
  {
    if (!lastblank)
    {
      UString *myblank = new UString(""_u);
      blanks.push_back(myblank);
    }
    UString &ref = *(blanks.back());
    if(chunk[i] == '\\')
    {
      ref += '\\';
      ref += chunk[++i];
    }
    else
    {
      ref += chunk[i];
    }
    lastblank = true;
  }
}
1182}