commit 8812f0e2add6118c547b2fb4336f2d835d955d1a Author: Daniel Swanson Date: Sat Jun 12 17:17:55 2021 -0500 use shared helper to read blanks diff --git a/apertium/apertium_cleanstream.cc b/apertium/apertium_cleanstream.cc index 0792947..38aa316 100644 --- a/apertium/apertium_cleanstream.cc +++ b/apertium/apertium_cleanstream.cc @@ -32,33 +32,6 @@ using namespace std; -UString -readFullBlock(InputFile& input, UChar32 const delim1, UChar32 const delim2) -{ - UString result; - result += delim1; - UChar32 c = delim1; - - while(!input.eof() && c != delim2) { - c = input.get(); - result += c; - if(c != '\\') { - continue; - } else { - result += '\\'; - c = input.get(); - result += c; - } - } - - if(c != delim2) - { - cerr << "Error: expected: " << delim2 << ", saw: " << c << endl; - } - - return result; -} - int main (int argc, char** argv) { @@ -114,7 +87,7 @@ main (int argc, char** argv) buf += c; } else if(!intoken && c == '[') { u_fputc(ws, output); - blanktmp = readFullBlock(input, L'[', L']'); + blanktmp = input.readBlock('[', ']'); if(keepblank) { write(blanktmp, output); } diff --git a/apertium/perceptron_spec.cc b/apertium/perceptron_spec.cc index 70b20c5..80304bb 100644 --- a/apertium/perceptron_spec.cc +++ b/apertium/perceptron_spec.cc @@ -95,7 +95,7 @@ PerceptronSpec::PerceptronSpec() { } untagged_sentinel = make_sentinel_wordoids("!UNTAGGED!"_u, "!UT!"_u); - token_wordoids_underflow = make_sentinel_token("!SURF_UNDERFLOW!"_u, "!TOKNDERFLOW!"_u, "!TUF!"_u); + token_wordoids_underflow = make_sentinel_token("!SURF_UNDERFLOW!"_u, "!TOK_UNDERFLOW!"_u, "!TUF!"_u); token_wordoids_overflow = make_sentinel_token("!SURF_OVERFLOW!"_u, "!TOK_OVERFLOW!"_u, "!TOF!"_u); static_constructed = true;