Index: branches/weighted-transfer/apertium-weights-learner/learner.py
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/learner.py	(revision 71845)
+++ branches/weighted-transfer/apertium-weights-learner/learner.py	(nonexistent)
@@ -1,203 +0,0 @@
-#! /usr/bin/python3
-
-import re, sys, os, pipes, math
-from optparse import OptionParser, OptionGroup
-from subprocess import call
-import coverage
-import kenlm
-
-# regular expression to cut out a sentence 
-sent_re = re.compile('.*?<sent>\$')
-
-weights_head = '<?xml version="1.0" encoding="UTF-8"?>\n<transfer-weights>\n'
-weights_tail = '</transfer-weights>'
-
-def get_options():
-    """
-    Parse commandline arguments
-    """
-    usage = "USAGE: ./%prog [<option>...] [INPUT_FILE]"
-    op = OptionParser(usage=usage)
-
-    op.add_option("-p", "--pair", dest="pfolder",
-                  help="language pair data folder. Mandatory.", metavar="PAIR_FOLDER")
-
-    op.add_option("-d", "--direction", dest="direction",
-                  help="transfer direction. Mandatory.", metavar="DIRECTION")
-
-    op.add_option("-l", "--lang-model", dest="model",
-                  help="path to language model. Mandatory.", metavar="LANG_MODEL")
-
-    op.add_option("-m", "--max-sent", dest="max_sent",
-                  help="maximum number of ambiguous sentences.", metavar="MAX_SENT")
-
-    op.add_option("-o", "--out", dest="ofname",
-                  help="output results to OUTPUT_FILE.", metavar="OUTPUT_FILE")
-
-    (opts, args) = op.parse_args()
-
-    if opts.pfolder is None:
-        op.error("please specify language pair data folder.")
-        op.print_help()
-        sys.exit(1)
-
-    if opts.direction is None:
-        op.error("please specify transfer direction.")
-        op.print_help()
-        sys.exit(1)
-
-    if opts.model is None:
-        op.error("please specify model.")
-        op.print_help()
-        sys.exit(1)
-
-    if opts.ofname is None:
-        op.error("please specify output file.")
-        op.print_help()
-        sys.exit(1)
-
-    opts.pfolder = opts.pfolder.rstrip('/\\')
-
-    if len(args) > 1:
-        op.error("too many arguments.")
-        op.print_help()
-        sys.exit(1)
-
-    return opts, args
-
-def pattern_to_xml(pattern, weight=1.):
-    pattern_line = '      <pattern weight="{}">\n'.format(weight)
-    for pattern_item in pattern:
-        parts = pattern_item.split('<', maxsplit=1)
-        lemma, tags = parts[0], parts[1].strip('>')
-        pattern_line += '        <pattern-item lemma="{}" tags="{}"/>\n'.format(lemma, '.'.join(tags.split('><')))
-    pattern_line += '      </pattern>\n'
-    return pattern_line
-
-def get_weights(rule_group, pattern, sent_line, tixfname, binfname):
-    total = 0.
-    weights_list = []
-    
-    pattern = pattern_to_xml(pattern)
-
-    for focus_rule in rule_group:
-        weights_line = weights_head + '  <rule-group>\n'
-        for rule in rule_group:
-            weights_line += '    <rule id="{}">\n'.format(rule[1])
-            if rule == focus_rule:
-                weights_line += pattern
-            weights_line += '    </rule>\n'
-        weights_line += '  </rule-group>\n' + weights_tail
-        with open('tmpweights.w1x', 'w', encoding='utf-8') as wfile:
-            wfile.write(weights_line)
-        translation = translate(sent_line, tixfname, binfname, 'tmpweights.w1x')
-        score = model.score(translation.lower(), bos = True, eos = True)
-        weights_list.append([translation, math.exp(score), focus_rule[1]])
-    for translation, score, rule_id in weights_list:
-        total += score
-    for score_item in weights_list:
-        score_item[1] = score_item[1] / total
-        #print(score_item[1], score_item[0])
-    return weights_list
-
-def output_final_weights(weights_dict, wfname):
-    with open(wfname, 'w', encoding='utf-8') as wfile:
-        wfile.write(weights_head)
-        for rule_group in weights_dict.values():
-            wfile.write('  <rule-group>\n')
-            for rule_id, pattern_dict in rule_group.items():
-                wfile.write('    <rule id="{}">\n'.format(rule_id))
-                for pattern, weight in pattern_dict.items():
-                    wfile.write(pattern_to_xml(pattern, weight))
-                wfile.write('    </rule>\n')
-            wfile.write('  </rule-group>\n')
-        wfile.write(weights_tail)            
-
-def translate(sent_line, tixfname, binfname, weightsfname):
-    pipe = pipes.Template()
-    pipe.append('lt-proc -b {}'.format('.'.join((binfname, 'autobil.bin'))), '--')
-    pipe.append('apertium-transfer -bw {} {} {}'.format(weightsfname, '.'.join((tixfname, 't1x')), '.'.join((binfname, 't1x.bin'))), '--')
-    pipe.append('apertium-interchunk {} {}'.format('.'.join((tixfname, 't2x')), '.'.join((binfname, 't2x.bin'))), '--')
-    pipe.append('apertium-postchunk {} {}'.format('.'.join((tixfname, 't3x')), '.'.join((binfname, 't3x.bin'))), '--')
-    pipe.append('lt-proc -g {}'.format('.'.join((binfname, 'autogen.bin'))), '--')
-    pipe.append('apertium-retxt', '--')
-
-    pipefile = pipe.open('pipefile', 'w')
-    pipefile.write(sent_line)
-    pipefile.close()
-    return open('pipefile').read()
-
-def search_ambiguous(ambiguous_rules, coverage_item):
-    """
-    Look for a pattern covered by one of ambiguous rules in ambiguous_rules.
-    If found, return the rule and the pattern.
-    """
-    for part in coverage_item:
-        if part[1][0] in ambiguous_rules:
-            return part[1][0], part[0]
-    return None, None
-
-if __name__ == "__main__":
-    opts, args = get_options()
-    tixfname = os.path.join(opts.pfolder, 
-                          '.'.join((os.path.basename(opts.pfolder), opts.direction)))
-    binfname = os.path.join(opts.pfolder, opts.direction)
-    rfname = '.'.join((tixfname, 't1x'))
-    cat_dict, pattern_FST, ambiguous_rules = coverage.prepare(rfname)
-    #print(ambiguous_rules)
-
-    weights_dict = {rule_num: {rule[1]: {} for rule in rule_list} for rule_num, rule_list in ambiguous_rules.items()}
-    #print(weights_dict)
-
-    if len(args) == 0:
-        input_stream = sys.stdin
-    elif len(args) == 1:
-        try:
-            input_stream = open(args[0], 'r', encoding='utf-8')
-        except FileNotFoundError:
-            print('Failed to locate input file \'{}\'. '
-                  'Have you misspelled the name?'.format(args[0]))
-            sys.exit(1)
-
-    print('Loading language model... ')
-    model = kenlm.Model(opts.model)
-    print('done.')
-
-    sentences = []
-    sent_count = 0
-    if opts.max_sent:
-        max_sent_count = int(opts.max_sent)
-    else:
-        max_sent_count = None
-
-    for line in input_stream:
-        line = line.strip()
-        for sent_match in sent_re.finditer(line):
-            sent_line = sent_match.group(0)
-            if '*' not in sent_line:
-                coverage_list, parsed_line = coverage.process_line(sent_line,
-                                        cat_dict, pattern_FST,
-                                        None, False, True, False)
-                # check if any rule in any coverage is ambiguous,
-                # take first coverage and first rule if any
-                for coverage_item in coverage_list:
-                    rule_number, pattern = search_ambiguous(ambiguous_rules, coverage_item)
-                    if rule_number is not None:
-                        #print('{}\n{}\n'.format(sent_count, sent_line))
-                        #for item in parsed_line:
-                        #    print(str(item) + '\n')
-                        #print(coverage.coverage_to_groups(coverage_item) + '\n\n')
-                        weights_list = get_weights(ambiguous_rules[rule_number], pattern, sent_line, tixfname, binfname)
-                        #print(weights_list)
-                        for translation, score, rule_id in weights_list:
-                            weights_dict[rule_number][rule_id].setdefault(tuple(pattern), 0.)
-                            weights_dict[rule_number][rule_id][tuple(pattern)] += score
-                        sent_count += 1
-                        if sent_count % 100 == 0:
-                            print(sent_count)
-                        break
-        if max_sent_count is not None and sent_count >= max_sent_count:
-            break
-
-    #print(weights_dict)
-    output_final_weights(weights_dict, opts.ofname)

Property changes on: branches/weighted-transfer/apertium-weights-learner/learner.py
___________________________________________________________________
Deleted: svn:executable
## -1 +0,0 ##
-*
\ No newline at end of property
Index: branches/weighted-transfer/apertium-weights-learner/README.md
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/README.md	(nonexistent)
+++ branches/weighted-transfer/apertium-weights-learner/README.md	(revision 71846)
@@ -0,0 +1,46 @@
+# apertium-weights-learner
+
+This is a python3 script that can be used for transfer weights training (see http://wiki.apertium.org/wiki/Ideas_for_Google_Summer_of_Code/Weighted_transfer_rules).
+
+## Prerequisites
+To run this version of transfer weights training for a given language pair, you need:
+* source and target language corpora (they don't have to be parallel to each other)
+* apertium with the language pair of interest
+* kenlm (https://kheafield.com/code/kenlm/)
+
+## Get the corpora
+The corpora can be obtained from http://www.statmt.org/wmt12/translation-task.html
+
+## Prepare language model
+You need to make a language model for your target language.
+
+* First, take a big corpus, tokenize and normalize it using tools/simpletok.py script:
+```
+cd tools
+./simpletok.py INPUT_FILE OUTPUT_FILE
+```
+* Alternatively, if you have your corpus in a number of separate files (which would be the case with. e.g., news crawl corpora from http://www.statmt.org/wmt12/translation-task.html), you can run tools/glue.py to simultaneously normalize and glue all the files together:
+```
+cd tools
+./glue.py INPUT_DIRECTORY OUTPUT_FILE
+```
+* After that, you are ready to train language model. Cd into build directory of your kenlm installation and run:
+```
+bin/lmplz -o 5 -T FOLDER_FOR_TMP_FILE <CORPUS_FILE >MODEL_NAME.arpa
+```
+Be advised that you might need disk space for your language model rougly 15 times of the corpus volume.
+* It is highly recommended that you compile a binary after that as it works significantly faster:
+```
+bin/build_binary -T FOLDER_FOR_TMP_FILE MODEL_NAME.arpa MODEL_NAME.mmap
+```
+Be advised that you might need disk space for your language model rougly half of the arpa file volume.
+
+## Run training
+* Edit configuration file twlconfig.py, which is self-explanatory.
+* Run training script:
+```
+./twlearner.py
+```
+
+## Known issues
+So far, learner script runs out of memory or something at approx. 7500 input lines while looking for ambiguous sentences and translating them and gets killed by the system.
Index: branches/weighted-transfer/apertium-weights-learner/coverage.py
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/coverage.py	(revision 71845)
+++ branches/weighted-transfer/apertium-weights-learner/coverage.py	(revision 71846)
@@ -2,6 +2,7 @@
 
 import re, sys
 from optparse import OptionParser, OptionGroup
+from time import clock
 
 try: # see if lxml is installed
     from lxml import etree as ET
@@ -274,7 +275,10 @@
     """
     if print_out:
         output_stream.write(line + '\n')
+    btime = clock()
     line = get_cats_by_line(line, cat_dict)
+    #print('{:f} cats'.format(clock() - btime))
+    btime = clock()
     coverage_list = calculate_coverage_r(pattern_FST, line, pattern_FST)
     parsed_coverages = []
     if coverage_list != []:
@@ -291,6 +295,7 @@
     else:
         if print_out:
             output_stream.write('No coverage found\n')
+    #print('{:f} coverages'.format(clock() - btime))
     return parsed_coverages, line
 
 def get_options():
Index: branches/weighted-transfer/apertium-weights-learner/tools/glue.py
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/tools/glue.py	(nonexistent)
+++ branches/weighted-transfer/apertium-weights-learner/tools/glue.py	(revision 71846)
@@ -0,0 +1,18 @@
+#! /usr/bin/python3
+
+import os, sys
+import simpletok
+
+if len(sys.argv) != 3:
+    print("Please specify input folder and output file name:")
+    print("./glue INPUT_FOLDER OUTPUT_FILE")
+else:
+    main_dir = sys.argv[1]
+    fnames = os.listdir(main_dir)
+    fnames.sort()
+    with open(sys.argv[2], 'w', encoding='utf-8') as ofile: 
+        for ifname in fnames:
+            with open(os.path.join(main_dir, ifname), 'r', encoding='utf-8') as ifile:
+                for line in ifile:
+                    ofile.write(simpletok.normalize(line))
+            ofile.write('\n')
Index: branches/weighted-transfer/apertium-weights-learner/tools/simpletok.py
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/tools/simpletok.py	(nonexistent)
+++ branches/weighted-transfer/apertium-weights-learner/tools/simpletok.py	(revision 71846)
@@ -0,0 +1,24 @@
+#! /usr/bin/python3
+
+import sys, re
+
+beforepunc_re = re.compile(r'([¿("/])(\w)')
+afterpunc_re = re.compile(r'(\w)([;:,.!?)"/—])')
+quot_re = re.compile("[«»']")
+numfix_re = re.compile('([0-9]) ([,.:][0-9])')
+beforedash_re = re.compile(r'(\W)-(\w)')
+afterdash_re = re.compile(r'(\w)-(\W)')
+
+def normalize(line):
+    line = line.lower().replace('--', '—').replace(' - ', ' — ')
+    line = quot_re.sub('"', line)
+    line = beforedash_re.sub(r'\1— \2', afterdash_re.sub(r'\1 —\2', line))
+    line = beforepunc_re.sub(r'\1 \2', afterpunc_re.sub(r'\1 \2', line))
+    line = numfix_re.sub(r'\1\2', line)
+    return line
+
+if __name__ == "__main__":
+    with open(sys.argv[1], 'r', encoding='utf-8') as ifile,\
+         open(sys.argv[2], 'w', encoding='utf-8') as ofile:
+        for line in ifile:
+            ofile.write(normalize(line))

Property changes on: branches/weighted-transfer/apertium-weights-learner/tools/simpletok.py
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Index: branches/weighted-transfer/apertium-weights-learner/twlconfig.py
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/twlconfig.py	(nonexistent)
+++ branches/weighted-transfer/apertium-weights-learner/twlconfig.py	(revision 71846)
@@ -0,0 +1,20 @@
+# full path to source corpus from which to learn the rules
+source_corpus = "/home/nm/source/apertium/weighted-transfer/apertium-weights-learner/data/2007-100000.txt"
+
+# name of apertium pair (not direction)
+apertium_pair_name = "en-es"
+
+# full path to apertium language pair data folder
+apertium_pair_data = "/home/nm/source/apertium/weighted-transfer/apertium-en-es"
+
+# translation direction
+source = "en"
+target = "es"
+
+# full path to kenlm language model
+# may be either arpa (text format) or mmap (binary) 
+# but mmap is strongly preferred as it loads and scores faster
+language_model = "/media/nm/storage/es-news-tokenized.mmap"
+
+# full path to a folder where to store intermediate data and results
+data_folder = "/home/nm/source/apertium/weighted-transfer/apertium-weights-learner/data/"
Index: branches/weighted-transfer/apertium-weights-learner/twlearner.py
===================================================================
--- branches/weighted-transfer/apertium-weights-learner/twlearner.py	(nonexistent)
+++ branches/weighted-transfer/apertium-weights-learner/twlearner.py	(revision 71846)
@@ -0,0 +1,352 @@
+#! /usr/bin/python3
+
+import re, sys, os, pipes
+from math import exp
+import coverage
+import kenlm
+from time import perf_counter as clock
+import twlconfig # a simple config in python file
+
+# regular expression to cut out a sentence 
+sent_re = re.compile('.*?<sent>\$')
+inter_re = re.compile(r'\$.*?\^')
+apertium_re = re.compile(r'[@#~]')
+apertium_token_re = re.compile(r'\^.*?\$')
+
+weights_head = '<?xml version="1.0" encoding="UTF-8"?>\n<transfer-weights>\n'
+weights_tail = '</transfer-weights>'
+
+beforepunc_re = re.compile(r'([¿("/])(\w)')
+afterpunc_re = re.compile(r'(\w)([;:,.!?)"/—])')
+quot_re = re.compile("[«»']")
+numfix_re = re.compile('([0-9]) ([,.:][0-9])')
+beforedash_re = re.compile(r'(\W)-(\w)')
+afterdash_re = re.compile(r'(\w)-(\W)')
+
+def normalize(line):
+    line = line.lower().replace('--', '—').replace(' - ', ' — ')
+    line = quot_re.sub('"', line)
+    line = beforedash_re.sub(r'\1— \2', afterdash_re.sub(r'\1 —\2', line))
+    line = beforepunc_re.sub(r'\1 \2', afterpunc_re.sub(r'\1 \2', line))
+    line = numfix_re.sub(r'\1\2', line)
+    return line
+
+def pattern_to_xml(pattern, weight=1.):
+    """
+    Create XML representation of the pattern for weigths file.
+    """
+    pattern_line = '      <pattern weight="{}">\n'.format(weight)
+    for pattern_item in pattern:
+        parts = pattern_item.strip('^$').split('<', maxsplit=1)
+        lemma, tags = parts[0], parts[1].strip('>')
+        pattern_line += '        <pattern-item lemma="{}" tags="{}"/>\n'.format(lemma, '.'.join(tags.split('><')))
+    pattern_line += '      </pattern>\n'
+    return pattern_line
+
+def load_rules(pair_data, source, target):
+    """
+    Load t1x transfer rules file from pair_data folder in source-target direction.
+    """
+    tixbasename = '{}.{}-{}'.format(os.path.basename(pair_data), source, target)
+    tixbasepath = os.path.join(pair_data, tixbasename)
+    binbasepath = os.path.join(pair_data, '{}-{}'.format(source, target))
+    tixfname = '.'.join((tixbasepath, 't1x'))
+    cat_dict, pattern_FST, ambiguous_rules = coverage.prepare(tixfname)
+
+    return tixbasepath, binbasepath, cat_dict, pattern_FST, ambiguous_rules
+
+def make_prefix(corpus, data_folder):
+    """
+    Make common prefix for all intermediate files.
+    """
+    trimmed_basename = os.path.splitext(os.path.basename(corpus))[0]
+    prefix = os.path.join(data_folder, trimmed_basename)
+    return prefix
+
+def tag_corpus(pair_data, source, target, corpus, prefix, data_folder):
+    """
+    Take source language corpus.
+    Tag it but do not translate.
+    """
+    print('Tagging source corpus.')
+    btime = clock()
+
+    # make output file name
+    ofname = '{}-tagged.txt'.format(prefix)
+
+    # create partial pipeline
+    pipe = pipes.Template()
+    pipe.append('apertium -d "{}" {}-{}-tagger'.format(pair_data, source, target), '--')
+    pipe.append('apertium-pretransfer', '--')
+    
+    # tag
+    pipe.copy(corpus, ofname)
+
+    print('Done in {:.2f}'.format(clock() - btime))    
+    return ofname
+
+def clean_tagged(corpus, prefix):
+    """
+    Tokenize, lowercase and clean up tagged corpus.
+    """
+    print('Cleaning up tagged source corpus.')
+    btime = clock()
+
+    # make output file name
+    ofname = '{}-tagged-clean.txt'.format(prefix)
+
+    with open(corpus, 'r', encoding='utf-8') as ifile, \
+         open(ofname, 'w', encoding='utf-8') as ofile:
+        for line in ifile:
+            ofile.write(inter_re.sub('$ ^', line.replace('"', '').lower()))
+
+    print('Done in {:.2f}'.format(clock() - btime))
+    return ofname
+
+def search_ambiguous(ambiguous_rules, coverage_item):
+    """
+    Look for patterns covered by one of the ambiguous rules in ambiguous_rules.
+    If found, return the rules and their patterns.
+    """
+    pattern_list = []
+    for i, part in enumerate(coverage_item):
+        if part[1][0] in ambiguous_rules:
+            pattern_list.append((i, part[1][0], tuple(part[0])))
+    return pattern_list
+
+def detect_ambiguous(corpus, prefix, cat_dict, pattern_FST, ambiguous_rules, tixfname, binfname):
+    """
+    """
+    print('Looking for ambiguous sentences and translating them.')
+    btime = clock()
+
+    # make output file name
+    ofname = '{}-ambiguous.txt'.format(prefix)
+    #sfname = '{}-sentences.txt'.format(prefix)
+
+    with open(corpus, 'r', encoding='utf-8') as ifile, \
+         open(ofname, 'w', encoding='utf-8') as ofile: #,\
+         #open(sfname, 'w', encoding='utf-8') as sfile:
+        for line in ifile:
+            # look at each sentence in line
+            for sent_match in sent_re.finditer(line.strip()):
+                # get coverages
+                coverage_list, parsed_line = coverage.process_line(sent_match.group(0),
+                                                        cat_dict, pattern_FST,
+                                                        None, False, True, False)
+                if coverage_list != []:
+                    coverage_item = coverage_list[0]
+                    # look for ambiguous chunks
+                    pattern_list = search_ambiguous(ambiguous_rules, coverage_item)
+                    if pattern_list != []:
+                        # segment the sentence into parts each containing one ambiguous chunk
+                        sentence_segments, prev = [], 0
+                        for i, rule_group_number, pattern in pattern_list:
+                            piece_of_line = '^' + '$ ^'.join(sum([chunk[0] for chunk in coverage_item[prev:i+1]], [])) + '$'
+                            sentence_segments.append([rule_group_number, pattern, piece_of_line])
+                            prev = i+1
+                        if sentence_segments != []:
+                            # add up the tail of the sentence
+                            if prev <= len(coverage_item):
+                                piece_of_line = ' ^' + '$ ^'.join(sum([chunk[0] for chunk in coverage_item[prev:]], [])) + '$'
+                                sentence_segments[-1][2] += piece_of_line
+
+                            # first, translate each segment with default rules
+                            for sentence_segment in sentence_segments:
+                                sentence_segment.append(translate(sentence_segment[2], tixfname, binfname))
+
+                            # second, translate each segment with all the rules
+                            # and make full sentence, where other segments are translated with default rules
+                            for j, sentence_segment in enumerate(sentence_segments):
+                                translation_list = translate_ambiguous(ambiguous_rules[sentence_segment[0]], 
+                                                                       sentence_segment[1], sentence_segment[2],
+                                                                       tixfname, binfname)
+                                output_list = []
+                                for rule, translation in translation_list:
+                                    translated_sentence = ' '.join(sentence_segment[3] for sentence_segment in sentence_segments[:j]) +\
+                                                          ' ' + translation + ' ' +\
+                                                          ' '.join(sentence_segment[3] for sentence_segment in sentence_segments[j+1:])
+                                    output_list.append('{}\t{}'.format(rule[0], translated_sentence.strip(' ')))
+                                # store results to a file
+                                # first, print rule group number, pattern, and number of rules in the group
+                                print('{}\t^{}$\t{}'.format(sentence_segment[0], '$ ^'.join(sentence_segment[1]), len(output_list)), file=ofile)
+                                # then, output all the translations in the following way: rule number, then translated sentence
+                                print('\n'.join(output_list), file=ofile)
+    print('Done in {:.2f}'.format(clock() - btime))
+    return ofname
+
+def translate(sent_line, tixfname, binfname, weightsfname=None):
+    """
+    Translate sent_line using given weights file.
+    """
+    # create pipeline
+    pipe = pipes.Template()
+    pipe.append('lt-proc -b {}'.format('.'.join((binfname, 'autobil.bin'))), '--')
+    # use weights file
+    if weightsfname is not None:
+        pipe.append('apertium-transfer -bw {} {} {}'.format(weightsfname, '.'.join((tixfname, 't1x')), '.'.join((binfname, 't1x.bin'))), '--')
+    # do not use weights file
+    else:
+        pipe.append('apertium-transfer -b {} {}'.format('.'.join((tixfname, 't1x')), '.'.join((binfname, 't1x.bin'))), '--')
+    pipe.append('apertium-interchunk {} {}'.format('.'.join((tixfname, 't2x')), '.'.join((binfname, 't2x.bin'))), '--')
+    pipe.append('apertium-postchunk {} {}'.format('.'.join((tixfname, 't3x')), '.'.join((binfname, 't3x.bin'))), '--')
+    pipe.append('lt-proc -g {}'.format('.'.join((binfname, 'autogen.bin'))), '--')
+    pipe.append('apertium-retxt', '--')
+
+    # translate
+    pipefile = pipe.open('pipefile', 'w')
+    pipefile.write(sent_line)
+    pipefile.close()
+    return apertium_re.sub('', open('pipefile').read().lower())
+
+def translate_ambiguous(rule_group, pattern, sent_line, tixfname, binfname):
+    """
+    Translate sent_line for each rule in rule_group.
+    """
+    translation_list = []
+    pattern = pattern_to_xml(pattern)
+
+    #for each rule
+    for focus_rule in rule_group:
+        # create weights file favoring that rule
+        weights_line = weights_head + '  <rule-group>\n'
+        for rule in rule_group:
+            weights_line += '    <rule id="{}">\n'.format(rule[1])
+            if rule == focus_rule:
+                weights_line += pattern
+            weights_line += '    </rule>\n'
+        weights_line += '  </rule-group>\n' + weights_tail
+        with open('tmpweights.w1x', 'w', encoding='utf-8') as wfile:
+            wfile.write(weights_line)
+
+        # translate using created file
+        translation = translate(sent_line, tixfname, binfname, 'tmpweights.w1x')
+        translation_list.append((focus_rule, translation))
+    return translation_list
+
+def score_sentences(ambig_sentences_fname, model, prefix):
+    """
+    Score translated sentences.
+    """
+    print('Scoring ambiguous sentences.')
+    btime, chunk_counter, sentence_counter = clock(), 0, 0
+
+    # make output file name
+    ofname = '{}-chunk-weights.txt'.format(prefix)
+
+    with open(ambig_sentences_fname, 'r', encoding='utf-8') as ifile, \
+         open(ofname, 'w', encoding='utf-8') as ofile:
+        reading = True
+        while reading:
+            try:
+                line = ifile.readline()
+                group_number, pattern, rulecount = line.rstrip('\n').split('\t')
+                weights_list = []
+                # score
+                total = 0.
+                for i in range(int(rulecount)):
+                    line = ifile.readline()
+                    rule_number, sentence = line.rstrip('\n').split('\t')
+                    score = exp(model.score(sentence, bos = True, eos = True))
+                    weights_list.append((rule_number, score))
+                    total += score
+                    sentence_counter += 1
+                # normalize and print out         
+                for rule_number, score in weights_list:
+                    print(group_number, rule_number, pattern, score / total, sep='\t', file=ofile)
+                chunk_counter += 1
+            except ValueError:
+                reading = False
+            except IndexError:
+                reading = False
+            except EOFError:
+                reading = False
+    print('Scored {} chunks, {} sentences in {:.2f}'.format(chunk_counter, sentence_counter, clock() - btime))
+    return ofname
+
+def make_xml_rules(scores_fname, prefix, rule_map):
+    """
+    Sum up the weights for each rule-pattern pair,
+    add the result to xml weights file.
+    """
+    print('Summing up the weights and making xml rules.')
+    btime = clock()
+
+    # make output file names
+    sorted_scores_fname = '{}-chunk-weights-sorted.txt'.format(prefix)
+    ofname = '{}-rule-weights.w1x'.format(prefix)
+
+    # create pipe
+    pipe = pipes.Template()
+    pipe.append('sort $IN > $OUT', 'ff')
+    pipe.copy(scores_fname, sorted_scores_fname)
+
+    with open(sorted_scores_fname, 'r', encoding='utf-8') as ifile,\
+         open(ofname, 'w', encoding='utf-8') as ofile:
+        prev_group_number, prev_rule_number, prev_pattern, weight = ifile.readline().rstrip('\n').split('\t')
+        total_pattern_weight = float(weight)
+        ofile.write(weights_head)
+        ofile.write('  <rule-group>\n    <rule id="{}">\n'.format(rule_map[prev_rule_number]))
+        for line in ifile:
+            group_number, rule_number, pattern, weight = line.rstrip('\n').split('\t')
+            if pattern != prev_pattern:
+                # pattern changed, flush previous
+                ofile.write(pattern_to_xml(apertium_token_re.findall(prev_pattern), total_pattern_weight))
+                total_pattern_weight = 0.
+            if group_number != prev_group_number:
+                # rule group changed, close previuos, open new
+                ofile.write('    </rule>\n  </rule-group>\n  <rule-group>\n    <rule id="{}">\n'.format(rule_map[rule_number]))
+            elif rule_number != prev_rule_number:
+                # rule changed, close previuos, open new
+                ofile.write('    </rule>\n    <rule id="{}">\n'.format(rule_map[rule_number]))
+            # add up rule-pattern weights
+            total_pattern_weight += float(weight)
+            prev_group_number, prev_rule_number, prev_pattern = group_number, rule_number, pattern
+        ofile.write('    </rule>\n  </rule-group>\n')
+        ofile.write(weights_tail)
+
+    print('Done in {:.2f}'.format(clock() - btime))
+    return ofname
+
+if __name__ == "__main__":
+    if not os.path.exists(twlconfig.data_folder):
+        os.makedirs(twlconfig.data_folder)
+    prefix = make_prefix(twlconfig.source_corpus, twlconfig.data_folder)
+
+    tbtime = clock()
+
+    # tag corpus
+    tagged_fname = tag_corpus(twlconfig.apertium_pair_data, 
+                              twlconfig.source, twlconfig.target, 
+                              twlconfig.source_corpus, prefix,
+                              twlconfig.data_folder)
+
+    # clean up tagged corpus
+    clean_fname = clean_tagged(tagged_fname, prefix)
+
+    # load rules, build rule FST
+    tixbasepath, binbasepath, cat_dict, pattern_FST, ambiguous_rules = \
+        load_rules(twlconfig.apertium_pair_data, twlconfig.source, twlconfig.target)
+    rule_map = {}
+    for rule_group in ambiguous_rules.values():
+        rule_map.update({rule[0]:rule[1] for rule in rule_group})
+
+    # detect and store sentences with ambiguity
+    ambig_sentences_fname = detect_ambiguous(clean_fname, prefix, 
+                                             cat_dict, pattern_FST,
+                                             ambiguous_rules,
+                                             tixbasepath, binbasepath)
+
+    # load language model
+    print('Loading language model.')
+    btime = clock()
+    model = kenlm.Model(twlconfig.language_model)
+    print('Done in {:.2f}'.format(clock() - btime))
+
+    # estimate rule weights for each ambiguous chunk
+    scores_fname = score_sentences(ambig_sentences_fname, model, prefix)
+
+    # sum up weigths for rule-pattern and make final xml
+    make_xml_rules(scores_fname, prefix, rule_map)
+
+    print('Performed in {:.2f}'.format(clock() - tbtime))

Property changes on: branches/weighted-transfer/apertium-weights-learner/twlearner.py
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property