Index: branches/apertium-tagger/experiments/experiments.py =================================================================== --- branches/apertium-tagger/experiments/experiments.py (revision 71521) +++ branches/apertium-tagger/experiments/experiments.py (revision 71522) @@ -1,3 +1,5 @@ +from collections import defaultdict +import re import functools from os.path import join as pjoin from statistics import mean, stdev @@ -7,35 +9,91 @@ tagger_train_sup, tagger_train_unsup, tagger_train_percep) + +NUM_KV_REGEX = re.compile(r'(\D+)(\d+)') +KV_REGEX = re.compile(r'([^-]+)-([^-]+)') +BOOL_OPTIONS = ['cg', 'sup'] + + # Experiment registry -experiments = {} -experiment_groups = {} +experiment_groups = defaultdict(list) +experiment_func_map = {} -def exp_name(name): +def reg(name): def reg(func): - func.name = name - return func + experiment_func_map[name] = func return reg -def reg(func): - experiments[func.name] = func - - def needs_tsx(func): func.needs_tsx = True return func -def group(name): - def reg(func): - experiment_groups.setdefault(name, {}) - experiment_groups[name][func.name] = func - return func - return reg +class BadSpecStrException(Exception): + pass +def has_needs_tsx(spec): + lspec, kvspec = spec + assert(len(lspec) >= 1) + name, *lspec = lspec + return getattr(experiment_func_map[name], 'needs_tsx', False) + + +def run_experiment(spec, lab): + lspec, kvspec = spec + assert(len(lspec) >= 1) + name, *lspec = lspec + experiment_func = experiment_func_map[name] + experiment_func(lab, *lspec, spec=spec, **kvspec) + + +def str_to_spec(s): + spec = [], {} + bits = s.split('_') + for option in BOOL_OPTIONS: + spec[1][option] = False + got_kv = False + for bit in bits: + match = NUM_KV_REGEX.match(bit) + if match: + spec[1][match.group(1)] = match.group(2) + got_kv = True + continue + match = KV_REGEX.match(bit) + if match: + spec[1][match.group(1)] = match.group(2) + got_kv = True + continue + if bit in BOOL_OPTIONS: + spec[1][bit] = True + got_kv = True + continue + if got_kv: + BadSpecStrException("Args must preceed kwargs") + spec[0].append(bit) + return spec + + +def spec_to_str(spec): + lspec, kvspec = spec + result_bits = lspec[:] + for k, v in kvspec.items(): + if not v: + continue + if k in BOOL_OPTIONS: + if v: + result_bits.append(k) + continue + if isinstance(v, int): + result_bits.append("{}{}".format(k, v)) + continue + result_bits.append("{}-{}".format(k, v)) + return "_".join(result_bits) + + # Statistical helpers def aggregates(data): return (min(data), max(data), @@ -44,13 +102,14 @@ def xval_experiment(func): @functools.wraps(func) - def wrapper(lab, *args, **kwargs): + def wrapper(lab, spec, *args, **kwargs): + name = spec_to_str(spec) recall = [] recall_available = [] for i, xval_fns in enumerate(lab.xval_fns): - xval_fns['test'] = xval_fns['prefix'] + 'test.' + func.name - xval_fns['model'] = xval_fns['prefix'] + 'model.' + func.name - func(lab, xval_fns) + xval_fns['test'] = xval_fns['prefix'] + 'test.' + name + xval_fns['model'] = xval_fns['prefix'] + 'model.' + name + func(lab, xval_fns, *args, spec=spec, **kwargs) evaluator = TaggerEvaluator( xval_fns['src'], xval_fns['ref'], xval_fns['test']) evaluator.run_analysis() @@ -66,18 +125,11 @@ return (evaluator.recall, evaluator.recall_available) # Experiments -for do_cg in [False, True]: - for unigram_type in range(1, 4): - unigram_model = 'unigram' + str(unigram_type) - name = ('cg_' if do_cg else '') + unigram_model - @reg - @group('default') - @xval_experiment - @exp_name(name) - def unigram_experiment(lab, xval_fns, - do_cg=do_cg, - unigram_model=unigram_model): + +@reg('unigram') +@xval_experiment +def unigram_experiment(lab, xval_fns, cg, model, **kwargs): tagger_train_sup(unigram_model, xval_fns['model'], xval_fns['train']) if do_cg: @@ -90,51 +142,46 @@ input=tagger_input, output=xval_fns['test']).check_returncode() for do_cg in [False, True]: - name = ('cg_' if do_cg else '') + '1st' + for unigram_type in range(1, 4): + unigram_model = 'unigram' + str(unigram_type) + name = ('cg_' if do_cg else '') + unigram_model - @reg - @group('default') - @xval_experiment - @exp_name(name) - def pick_first_experiment(lab, xval_fns, do_cg=do_cg): - if do_cg: + experiment_groups['default'].append(( + ['unigram'], + { + 'cg': do_cg, + 'model': unigram_type, + } + )) + + +@reg('1st') +@xval_experiment +def pick_first_experiment(lab, xval_fns, cg, **kwargs): + if cg: tagger_input = cg_proc(lab.cg_fn, input=xval_fns['src']) else: tagger_input = xval_fns['src'] extract_first_analysis(tagger_input, xval_fns['test']) -for cg_aug in [0, 1, 2, 3, (4, 0), (4, 5), (4, 10), (4, 20), (4, 30)]: - if isinstance(cg_aug, tuple): - cg_aug, cg_aug_t = cg_aug - else: - cg_aug_t = None - for do_cg in [None, 'in', 'dual', 'inv']: - for is_supervised, model in [(True, 'bigram'), - (False, 'bigram'), - (False, 'lwsw')]: - if is_supervised: - iterations_list = [0] - else: - iterations_list = [0, 50, 250] - for iterations in iterations_list: - name = ( - "{cgt}{cg}{sup}_{model}".format( - cgt='cgt{}_'.format(cg_aug) if cg_aug else '', - cg='cg{}_'.format(do_cg if do_cg != 'in' else '') - if do_cg else '', - sup='sup' if is_supervised else 'unsup', - model=model - ) + - ("_i{iterations}".format(iterations=iterations) - if len(iterations_list) > 1 else "") + - ("_j{iterations}".format(iterations=cg_aug_t) - if cg_aug_t else "")) - @needs_tsx - def model_experiment(lab, xval_fns, cg_aug=cg_aug, do_cg=do_cg, - is_supervised=is_supervised, model=model, - iterations=iterations): - if is_supervised: +for do_cg in [False, True]: + name = ('cg_' if do_cg else '') + '1st' + pick_first_experiment + experiment_groups['default'].append(( + ['1st'], + { + 'cg': do_cg + } + )) + + +@reg('gen') +@needs_tsx +@xval_experiment +def model_experiment(lab, xval_fns, cgt, cg, + sup, model, iters, ambgcls, **kwargs): + if sup: tagger_train_sup( model, xval_fns['model'], train_fn=xval_fns['train'], @@ -141,9 +188,9 @@ trainsrc_fn=xval_fns['trainsrc'], dic_fn=lab.dic_fn, tsx_fn=lab.tsx_fn, - iterations=iterations, - cg_aug=cg_aug, - ambg_classes=cg_aug_t, + iterations=iters, + cg_aug=cgt, + ambg_classes=ambgcls, cgtrain_fn=xval_fns['traincgtag']) else: tagger_train_unsup( @@ -151,40 +198,64 @@ trainsrc_fn=xval_fns['trainsrc'], dic_fn=lab.dic_fn, tsx_fn=lab.tsx_fn, - iterations=iterations, - cg_aug=cg_aug, - ambg_classes=cg_aug_t, + iterations=iters, + cg_aug=cgt, + ambg_classes=ambgcls, cgtrain_fn=xval_fns['traincgtag']) - if do_cg == 'in': + if cg == 'in': tagger_input = xval_fns['cgtag'] else: tagger_input = xval_fns['src'] - if do_cg in ['dual', 'inv']: + if cg in ['dual', 'inv']: cg_tagger_input = xval_fns['cgtag'] else: cg_tagger_input = None tagger_cg_aug = None - if do_cg == 'dual': + if cg == 'dual': tagger_cg_aug = 1 - elif do_cg == 'inv': + elif cg == 'inv': tagger_cg_aug = 2 tagger_tag( model, xval_fns['model'], cg_fn=cg_tagger_input, - cg_aug=tagger_cg_aug, input=tagger_input, + cgt=tagger_cg_aug, input=tagger_input, output=xval_fns['test'] ).check_returncode() - model_experiment = exp_name(name)(model_experiment) + + +for cg_aug in [0, 1, 2, 3, (4, 0), (4, 5), (4, 10), (4, 20), (4, 30)]: + if isinstance(cg_aug, tuple): + cg_aug, cg_aug_t = cg_aug + else: + cg_aug_t = None + for do_cg in [None, 'in', 'dual', 'inv']: + for is_supervised, model in [(True, 'bigram'), + (False, 'bigram'), + (False, 'lwsw')]: + if is_supervised: + iterations_list = [0] + else: + iterations_list = [0, 50, 250] + for iterations in iterations_list: + spec = ( + ['gen'], + { + 'cgt': cg_aug, + 'cg': do_cg, + 'sup': is_supervised, + 'iters': iterations, + 'ambgcls': cg_aug_t, + 'model': model, + } + ) + if not cg_aug and do_cg in [None, 'in']: - model_experiment = group('default')(model_experiment) + experiment_groups['default'].append(spec) else: - model_experiment = group('cg-extra')(model_experiment) - reg(xval_experiment(model_experiment)) + experiment_groups['cg-extra'].append(spec) -@reg -@group('default') -@exp_name('word_count') -def word_count(lab): +@reg('word_count') +def word_count(lab, **kwargs): count = 0 for line in open(lab.src_fn): if line.strip() != '': @@ -191,12 +262,12 @@ count += 1 return count +experiment_groups['default'].append((['word_count'], {})) -@reg -@group('meta') -@exp_name('new_cg_ambg') + +@reg('new_cg_ambg') @needs_tsx -def new_cg_ambg(lab): +def new_cg_ambg(lab, **kwargs): model_fn = pjoin(lab.work_dir, 'new_cg_ambg.model') tagger_train_sup('bigram', model_fn, train_fn=lab.ref_fn, @@ -217,24 +288,20 @@ ambg_classes.add(line) return count, len(ambg_classes) +experiment_groups['meta'].append((['new_cg_ambg'], {})) -for do_cg in [False, True]: - for mtx_basename in ['kaztags', 'unigram_model3', 'spacyflattags']: - mtx_fn = 'mtx/' + mtx_basename + '.mtx' - name = ('cg_' if do_cg else '') + mtx_basename + '_percep' - @reg - @group('percep') - @xval_experiment - @exp_name(name) - def percep_experiment(lab, xval_fns, - do_cg=do_cg, mtx_fn=mtx_fn): +@reg('percep') +@xval_experiment +def percep_experiment(lab, xval_fns, cg, mtx, **kwargs): + mtx_fn = 'mtx/' + mtx + '.mtx' + tagger_train_percep(xval_fns['model'], train_fn=xval_fns['train'], trainsrc_fn=xval_fns['trainsrc'], mtx_fn=mtx_fn, sent_seg=lab.sent_seg) - if do_cg: + if cg: tagger_input = xval_fns['cgtag'] else: tagger_input = xval_fns['src'] @@ -243,3 +310,14 @@ 'percep', xval_fns['model'], input=tagger_input, output=xval_fns['test'] ).check_returncode() + + +for do_cg in [False, True]: + for mtx_basename in ['kaztags', 'unigram_model3', 'spacyflattags']: + experiment_groups['percep'].append(( + ['percep'], + { + 'cg': do_cg, + 'mtx': mtx_basename, + } + )) Index: branches/apertium-tagger/experiments/run_experiment.py =================================================================== --- branches/apertium-tagger/experiments/run_experiment.py (revision 71521) +++ branches/apertium-tagger/experiments/run_experiment.py (revision 71522) @@ -12,7 +12,8 @@ from pprint import pformat import asyncio -from experiments import experiment_groups, experiments +from experiments import experiment_groups, str_to_spec, spec_to_str,\ + has_needs_tsx, run_experiment from shell_utils import cd, check_run from shell_wrappers import (cg_proc, copy_blanks, extract_src, fix_dix, cg_conv_clean, split_n_r, strip_blanks, @@ -84,9 +85,17 @@ def parse_args(): + tagger_info = [] + for group_name, group_taggers in experiment_groups.items(): + tagger_info.append("{}:".format(group_name)) + for tagger in group_taggers: + tagger_info.append(" {}".format(spec_to_str(tagger))) + tagger_info.append("") parser = argparse.ArgumentParser( description="Runs a series of experiments on different part of speech " - "taggers and different language data.") + "taggers and different language data.\n\nGROUPS:\n\n" + + "\n".join(tagger_info), + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( 'languagesdir', help="Path to the directory containing all the individaul language " @@ -343,8 +352,8 @@ xval_fn['cgtag'], self.folds, i, write_blanks=self.sent_seg) - def can_run_experiment(self, experiment_func): - if self.no_tsx and getattr(experiment_func, 'needs_tsx', False): + def can_run_experiment(self, spec): + if self.no_tsx and has_needs_tsx(spec): return False return True @@ -376,29 +385,29 @@ check_run(['make']) lab = mk_lab() - def run_tagger(tagger): - experiment = experiments[tagger] - if lab.can_run_experiment(experiment): + def run_spec(spec): + if lab.can_run_experiment(spec): if args.dry: - print("Running {}/{}".format(lang, tagger)) + print("Running {}/{}".format(lang, spec)) else: try: - languages_tagger_accuracies[lang][tagger] = \ - experiment(lab) + languages_tagger_accuracies[lang][spec_to_str(spec)] = \ + run_experiment(spec, lab) except: - languages_tagger_accuracies[lang][tagger] = None + languages_tagger_accuracies[lang][spec_to_str(spec)] = None traceback.print_exc() else: print("Skipping {}/{} since it needs a tsx" - .format(lang, tagger)) + .format(lang, spec_to_str(spec))) + languages_tagger_accuracies[lang] = {} if args.taggers: - for tagger in args.taggers: - run_tagger(tagger) + for spec_str in args.taggers: + run_spec(str_to_spec(spec_str)) else: for tagger_group in args.tagger_groups: - for tagger in experiment_groups[tagger_group]: - run_tagger(tagger) + for spec in experiment_groups[tagger_group]: + run_spec(spec) finally: result_pretty = pformat(languages_tagger_accuracies) print(result_pretty)