[hltdi-l3] push by onlysk...@gmail.com - L3Lite: Sentence, Node, Features classes. on 2014-04-20 07:07 GMT
0 views
Skip to first unread message
hltd...@googlecode.com
Apr 20, 2014, 3:07:27 AM4/20/14
to hltdi-...@googlegroups.com
Revision: edc7222fb039
Branch: default
Author: Michael Gasser <gas...@cs.indiana.edu>
Date: Sun Apr 20 07:07:10 2014 UTC
Log: L3Lite: Sentence, Node, Features classes.
http://code.google.com/p/hltdi-l3/source/detail?r=edc7222fb039
Added:
/l3lite/features.py
/l3lite/sentence.py
Modified:
/l3lite/__init__.py
/l3lite/constraint.py
/l3lite/entry.py
/l3lite/language.py
/l3lite/ui.py
/lite.py
=======================================
--- /dev/null
+++ /l3lite/features.py Sun Apr 20 07:07:10 2014 UTC
@@ -0,0 +1,84 @@
+#
+# L3Lite features (dicts).
+#
+########################################################################
+#
+# This file is part of the HLTDI L^3 project
+# for parsing, generation, translation, and computer-assisted
+# human translation.
+#
+# Copyright (C) 2014, HLTDI <gas...@cs.indiana.edu>
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =========================================================================
+
+# 2014.04.19
+# -- Created.
+
+class Features(dict):
+
+ def __init__(self, dct):
+ dict.__init__(self, dct)
+
+ def __repr__(self):
+ l = []
+ for f, v in self.items():
+ l.append("{}={}".format(f, v))
+ return "{{{0}}}".format(','.join(l))
+
+ def to_list(self):
+ """Convert features dict to a sorted list."""
+ l = list(self.items())
+ l.sort()
+ return l
+
+ @staticmethod
+ def simple_unify(x, y):
+ """Unify the expressions x and y, returning the result
or 'fail'."""
+ # If they're the same, return one.
+ if x == y:
+ return x
+# # If both are dicts, call unify_dict
+# elif isinstance(x, dict) and isinstance(y, dict):
+# x.unify(y)
+ # Otherwise fail
+ else:
+ return 'fail'
+
+ def unify(self, other):
+ """other is a Features object or a dict. Attempt to unify self
with other,
+ returning the result or 'fail'."""
+ result = Features({})
+ for k in set(self.keys()) | set(other.keys()):
+ # Check all of the keys of self and other
+ self_val, other_val = self.get(k, 'nil'), other.get(k, 'nil')
+ if self_val != 'nil':
+ if other_val != 'nil':
+ # If x and y both have a value for k, try to unify the
values
+ u = Features.simple_unify(self_val, other_val)
+ if u == 'fail':
+ return 'fail'
+ else:
+ result[k] = u
+ else:
+ # If self has a value for k but other doesn't, use
self's value
+ result[k] = self_val
+ elif other_val != 'nil':
+ # If other has a value for k but self doesn't, use other's
value
+ result[k] = other_val
+
+ return result
+
+
=======================================
--- /dev/null
+++ /l3lite/sentence.py Sun Apr 20 07:07:10 2014 UTC
@@ -0,0 +1,182 @@
+#
+# L3Lite sentences and how parse and translate them.
+#
+########################################################################
+#
+# This file is part of the HLTDI L^3 project
+# for parsing, generation, translation, and computer-assisted
+# human translation.
+#
+# Copyright (C) 2014, HLTDI <gas...@cs.indiana.edu>
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =========================================================================
+
+# 2014.04.15
+# -- Created.
+
+# ui.py loads language, etc.
+from .ui import *
+
+class Sentence:
+ """A sentence is a list of words (or other lexical tokens) that gets
+ assigned a set of variables and constraints that are run during
+ parsing or translation."""
+
+ def __init__(self, raw='', language=None,
+ tokens=None, analyses=None,
+ nodes=None, groups=None):
+ # A string representing the raw sentence
+ self.raw = raw
+# # A list of strings tokenized words (or morphemes)
+# self.tokens = tokens or []
+# # A list of analyses pairs (root, features) for particular tokens
+# self.analyses = analyses or []
+ # A language object
+ self.language = language
+ # A list of Node objects, one for each token
+ self.nodes = nodes or []
+ # A list of candidate groups found during lexicalization
+ self.groups = groups or []
+ # A list of constraints to run
+ self.constraints = []
+
+ def __repr__(self):
+ """Print name."""
+ if self.raw:
+ return '|| {} ||'.format(self.raw)
+ else:
+ return '|| {} sentence ||'.format(self.language)
+
+ def tokenize(self):
+ """Segment the sentence string into tokens, analyze them
morphologically,
+ and create a Node object for each."""
+ if not self.nodes:
+ # (Otherwise it's already done.)
+ # Split at spaces by default (later allow for dedicated
language-specific tokenizers).
+ tokens = self.raw.split()
+ self.nodes = []
+ index = 0
+ for token in tokens:
+ # Look up token in language.forms
+ if token not in self.language.forms:
+ # Not found, just use the raw string
+ self.nodes.append(Node(token, index, None, self))
+ index += 1
+ else:
+ # A dict, for unambiguous forms, or a list of dicts,
for ambiguous forms
+ formdict = self.language.forms[token]
+ if isinstance(formdict, dict):
+ # A single entry
+ if 'seg' in formdict:
+ segs = formdict['seg']
+ for seg in segs:
+ tok, analysis = seg
+ self.nodes.append(Node(tok, index,
analysis, self))
+ index += 1
+ else:
+ self.nodes.append(Node(token, index, formdict,
self))
+ index += 1
+ else:
+ # Multiple dicts: ambiguity; let node handle it
+ self.nodes.append(Node(token, index, formdict,
self))
+ index += 1
+
+ def activate_groups(self):
+ """Activate all groups that are compatible with the tokens in the
sentence."""
+ if not self.nodes:
+ print("Tokenization must precede group activation.")
+ return
+ candidates = []
+ for node in self.nodes:
+ keys = {node.token}
+ anal = node.analyses
+ if anal:
+ if isinstance(anal, list):
+ for a in anal:
+ keys.add(a.get('root'))
+ else:
+ keys.add(anal.get('root'))
+ for k in keys:
+ if k in self.language.groups:
+ for group in self.language.groups[k]:
+ candidates.append((node.index, group))
+ # Now filter candidates to see if all words are present in the
sentence
+ # For each group, save a list of list of sentence token indices
that correspond
+ # to the group's words
+ groups = []
+ for head_index, group in candidates:
+ indices = group.match_nodes(self.nodes, head_index)
+ if not indices:
+ # This group is out
+ continue
+ groups.append((head_index, indices, group))
+
+ self.groups = groups
+
+class Node:
+ """Sentence token and its associated analyses and variables."""
+
+ def __init__(self, token, index, analyses, sentence):
+ # Raw form in sentence (possibly result of segmentation)
+ self.token = token
+ # Position in sentence
+ self.index = index
+ # List of analyses
+ if analyses and not isinstance(analyses, list):
+ analyses = [analyses]
+ self.analyses = analyses
+ # Back pointer to sentence
+ self.sentence = sentence
+ # We'll need these for multiple matchings
+ self.cats = self.get_cats()
+
+ def __repr__(self):
+ """Print name."""
+ return "*{}".format(self.token)
+
+ def get_cats(self):
+ """The set of categories for the node's token, or None."""
+ if not self.analyses:
+ return None
+ cats = set()
+ for analysis in self.analyses:
+ if 'cats' in analysis:
+ cats.update(analysis['cats'])
+ return cats
+
+ def match(self, item, features):
+ """Does this node match the group item (word, lexeme, category) and
+ and any features associated with it?"""
+ # If item is a category, don't bother looking at token
+ if Entry.is_cat(item):
+ if self.cats and item in self.cats:
+ return True
+ elif self.token == item:
+ # item matches this node's token; features are irrelevant
+ return True
+ elif self.analyses:
+ # Check each combination of root and analysis features
+ for analysis in self.analyses:
+ root = analysis.get('root', '')
+ node_features = analysis.get('features')
+ if root == item:
+ if not features:
+ return True
+ elif not node_features:
+ return True
+ elif node_features.unify(features):
+ return True
+ return False
=======================================
--- /l3lite/__init__.py Fri Apr 4 06:26:55 2014 UTC
+++ /l3lite/__init__.py Sun Apr 20 07:07:10 2014 UTC
@@ -1,5 +1,5 @@
"""Do-it-yourself L3. Create simple bilingual lexicons and grammars for
language pairs."""
-__all__ = ['language', 'entry', 'ui', 'constraint', 'variable']
+__all__ =
['language', 'entry', 'ui', 'constraint', 'variable', 'sentence', 'features']
-from .ui import *
+from .sentence import *
=======================================
--- /l3lite/constraint.py Wed Apr 16 06:00:17 2014 UTC
+++ /l3lite/constraint.py Sun Apr 20 07:07:10 2014 UTC
@@ -32,6 +32,10 @@
# -- Created ComplexSetConvexity
# 2014.04.05
# -- Created ComplexUnionSelection
+# 2014.04.15
+# -- Constraint types used so far:
+# UnionSelection, PrecedenceSelection, ComplexUnionSelection,
+# ComplexSetConvexity, Union, Disjoint, Inclusion
from .variable import *
import itertools
=======================================
--- /l3lite/entry.py Thu Mar 27 17:19:15 2014 UTC
+++ /l3lite/entry.py Sun Apr 20 07:07:10 2014 UTC
@@ -41,16 +41,19 @@
# -- Lots of changes and additions to groups.
# 2014.03.24
# -- words attribute in Group is a list of [word, feat_dict] pairs.
+# 2014.04.16
+# -- Created simpler Group (with no dependency types), renamed old Group
to MWE.
import copy, itertools
import yaml
+from .features import *
+
LEXEME_PRE = '%'
CLASS_PRE = '$'
class Entry:
- """The central unit in L3Lite, containing lexical and grammatical
- constraints that are instantiated during processing."""
+ """Superclass for Group and possibly other lexical classes."""
ID = 1
dflt_dep = 'dflt'
@@ -71,6 +74,11 @@
"""Print name."""
return '<{}:{}>'.format(self.name, self.id)
+ @staticmethod
+ def is_cat(name):
+ """Is this the name of a category?"""
+ return name[0] == CLASS_PRE
+
## Serialization
def to_dict(self):
@@ -410,18 +418,18 @@
def inherit_gov(self, cls):
"""Inherit government constraints from class."""
-class Group(Entry):
+class MWE(Entry):
"""Multi-word expressions. Each group consists of a head and a set of
nodes,
possibly connected to other nodes through explicit dependencies and an
explicit
order of the nodes.
Variable slots have dedicated names that allow them to be
referenced in translations.
- Groups must be created *after* other lexical items.
+ MWEs must be created *after* other lexical items.
{index: [word_obj, {dep/position_feats}...}
"""
def __init__(self, name, language, head, head_feats=None,
head_order=None, head_lexeme=False):
- """name of a Group is something like acabar_de_V.
+ """name of a MWE is something like acabar_de_V.
head is the word that is the syntactic head of the group."""
Entry.__init__(self, name, language)
# A list of [word feats] pairs; index in the list is the word's
(node's) ID
@@ -473,15 +481,15 @@
@staticmethod
def from_dict(d, language):
- """Convert a dict (loaded from a yaml file) to a Lex object."""
+ """Convert a dict (loaded from a yaml file) to a MWE object."""
lexeme = d['head_lexeme']
- g = Group(d.get('name'), language, d.get('words')[0][0],
head_lexeme=lexeme)
+ m = MWE(d.get('name'), language, d.get('words')[0][0],
head_lexeme=lexeme)
# for id, word in d.get('words').items():
for id, word in enumerate(d.get('words')):
if id == 0:
# Just handle the dependencies for this case
deps = word[1]
- g.words[0][1] = copy.deepcopy(deps)
+ m.words[0][1] = copy.deepcopy(deps)
else:
name = word[0]
lex = language.get_words(name)[0]
@@ -490,9 +498,8 @@
lex_info = [lex.clone(), copy.deepcopy(deps)]
else:
lex_info = [lex.clone()]
- g.words.append(lex_info)
-# g.words[id] = lex_info
- return g
+ m.words.append(lex_info)
+ return m
## Getters
@@ -634,6 +641,91 @@
tdict['m'].append(trg_id)
# tdict['m'].append((-1, trg_id))
+class Group(Entry):
+ """Primitive multi-word expressions. Default is a head with unlabeled
dependencies
+ to all other words and translations, including alignments, to one or
more
+ other languages."""
+
+ def __init__(self, words, head_index=-1, head='', language=None,
name='',
+ features=None):
+ # words is a list of strings or (string, dict) tuples
+ # name may be specified explicitly or not
+ name = name or Group.make_name(words)
+ Entry.__init__(self, name, language)
+ self.words = words
+ if head:
+ self.head = head
+ else:
+ self.head = words[head_index]
+ # Either None or a list of feat-val dicts for words that require
them
+ # Convert dicts to Features objects
+ if isinstance(features, list):
+ features = [Features(d) for d in features]
+ self.features = features
+
+ def __repr__(self):
+ """Print name."""
+ return '<{}:{}>'.format(self.name, self.id)
+
+ @staticmethod
+ def make_name(words):
+ # Each word is either a string or a (string, feat_dict) pair
+# strings = []
+# for word in words:
+# if isinstance(word, str):
+# strings.append(word)
+# else:
+# form, feat_dict = word
+# fv = ['{}={}'.format(f, v) for f, v in feat_dict.items()]
+# fv = ','.join(fv)
+# strings.append("{}:{}".format(form, fv))
+ return '_'.join(words)
+
+ # Serialization
+
+ def to_dict(self):
+ """Convert the group to a dictionary to be serialized in a yaml
file."""
+ d = Entry.to_dict(self)
+ d['words'] = self.words
+ d['features'] = self.features
+ return d
+
+ @staticmethod
+ def from_dict(d, language, head):
+ """Convert a dict (loaded from a yaml file) to a Group object."""
+ words = d['words']
+ features = d.get('features')
+ p = Group(words, head=head, language=language, features=features)
+ return p
+
+ def match_nodes(self, nodes, head_index):
+ """Attempt to match the group words (and features) with nodes from
a sentence."""
+# print("Does {} match {}".format(self, nodes))
+ tok_indices = []
+ for index, word in enumerate(self.words):
+# print(" Attempting to match {}".format(word))
+ matched = False
+ for node in nodes:# for word, feats in zip(self.words,
self.features):
+
+# print(" Trying {}".format(node))
+ if index == node.index == head_index:
+ # This is the token corresponding to the group head
+ tok_indices.append(node.index)
+# print(" Head matched already".format(node))
+ matched = True
+ break
+ else:
+ feats = self.features[index] if self.features else None
+ if node.match(word, feats):
+ tok_indices.append(node.index)
+# print(" Matched node {}".format(node))
+ matched = True
+ break
+ if not matched:
+# print(" {} not matched; failed".format(word))
+ return False
+ return tok_indices
+
class EntryError(Exception):
'''Class for errors encountered when attempting to update an entry.'''
=======================================
--- /l3lite/language.py Fri Apr 4 06:26:55 2014 UTC
+++ /l3lite/language.py Sun Apr 20 07:07:10 2014 UTC
@@ -33,12 +33,16 @@
# 2014.03.24
# -- Words, lexemes, and classes are all in the same dictionary
(self.words).
# Lexemes start with %, classes with $.
+# 2014.04.17
+# -- Analysis and generation dicts for particular wordforms.
from .entry import *
from .constraint import *
import os, yaml
+LANGUAGE_DIR = os.path.join(os.path.dirname(__file__), 'languages')
+
class Language:
"""Dictionaries of words, lexemes, grammatical features, and
lexical classes."""
@@ -47,7 +51,9 @@
def __init__(self,
name, abbrev,
- words=None, lexemes=None, grams=None, classes=None,
groups=None):
+ words=None, lexemes=None, grams=None, classes=None,
+ mwes=None, groups=None, forms=None,
+ genforms=None):
"""Initialize dictionaries and names."""
self.name = name
self.abbrev = abbrev
@@ -57,11 +63,16 @@
# self.lexemes = lexemes or {}
self.grams = grams or {}
# self.classes = classes or {}
+ self.mwes = mwes or {}
+ self.forms = forms or {}
self.groups = groups or {}
# Record possibilities for dependency labels, feature values,
order constraints
self.possible = {}
# Record whether language has changed since last loaded
self.changed = False
+ # Dictionary of morphologically generated words:
+ # {lexeme: {(feat, val): {(feat, val): wordform,...}, ...}, ...}
+ self.genforms = genforms or {}
Language.languages.append(abbrev)
def __repr__(self):
@@ -97,11 +108,21 @@
# Words are lists
words[k] = [lex.to_dict() for lex in v]
d['words'] = words
+ if self.mwes:
+ mwes = {}
+ for k, v in self.mwes.items():
+ mwes[k] = [m.to_dict() for m in v]
+ d['mwes'] = mwes
if self.groups:
groups = {}
- for k, v in self.groups.items():
- groups[k] = [g.to_dict() for g in v]
+ for head, v in self.groups.items():
+ groups[head] = [g.to_dict() for g in v]
d['groups'] = groups
+ if self.forms:
+ forms = {}
+ for k, v in self.forms.items():
+ # v is an fv dict or a list of fv dicts
+ forms[k] = v
return d
def write(self, directory, filename=''):
@@ -136,11 +157,30 @@
l.words = {}
for k, v in words.items():
l.words[k] = [Lex.from_dict(lex, l) for lex in v]
+ mwes = d.get('mwes')
+ if mwes:
+ l.mwes = {}
+ for k, v in mwes.items():
+ l.mwes[k] = [MWE.from_dict(m, l) for m in v]
groups = d.get('groups')
if groups:
l.groups = {}
- for k, v in groups.items():
- l.groups[k] = [Group.from_dict(g, l) for g in v]
+ for head, v in groups.items():
+ l.groups[head] = [Group.from_dict(g, l, head) for g in v]
+ forms = d.get('forms')
+ if forms:
+ l.forms = {}
+ for k, v in forms.items():
+ # v should be a dict or a list of dicts
+ # Convert features value to a Features object
+ if isinstance(v, dict):
+ if 'features' in v:
+ v['features'] = Features(v['features'])
+ else:
+ for d in v:
+ if 'features' in d:
+ d['features'] = Features(d['features'])
+ l.forms[k] = v
return l
@staticmethod
@@ -151,11 +191,21 @@
dct = yaml.load(file)
return Language.from_dict(dct)
+ @staticmethod
+ def load(abbrev):
+ path = os.path.join(LANGUAGE_DIR, abbrev + '.lg')
+ try:
+ language = Language.read(path)
+ return language
+ except IOError:
+ print("That language doesn't seem to exist.")
+ return
+
### Basic setters. Create entries (dicts) for item. For debugging
purposes, include name
### in entry.
- def add_word(self, word, cls=None, group=False):
- entry = Lex(word, self, cls=cls, group=group)
+ def add_word(self, word, cls=None, mwe=False):
+ entry = Lex(word, self, cls=cls, mwe=mwe)
if word in self.words:
self.words[word].append(entry)
else:
@@ -173,6 +223,45 @@
self.changed = True
return entry
+ def add_form(self, form, dct, reverse=True):
+ """Form dict has root, features, cats.
+ If reverse it True, also add the form to the genforms dicdt."""
+ if form not in self.forms:
+ self.forms[form] = dct
+ else:
+ entry = self.forms[form]
+ if isinstance(entry, dict):
+ # Make this the second entry
+ self.forms[form] = [entry, dct]
+ else:
+ # There are already two or more entries in a list
+ entry.append(dct)
+ if reverse:
+ lexeme = dct['root']
+ features = dct['features']
+ self.add_genform(form, lexeme, features)
+
+ def add_genform(self, form, lexeme, features):
+ """Add the form to a lexeme- and feature-keyed dict."""
+ if lexeme not in self.genforms:
+ self.genforms[lexeme] = {}
+ featdict = self.genforms[lexeme]
+ # features is a Features object; convert it to a list of tuples
+ features = features.to_list()
+ feat = features.pop(0)
+ self.make_featdict(featdict, feat, features, form)
+
+ @staticmethod
+ def make_featdict(featdict, feat, features, form):
+ """Make a feat-value dict with the form as final value."""
+ if not features:
+ featdict[feat] = form
+ return
+ if feat not in featdict:
+ featdict[feat] = {}
+ new_feat = features.pop(0)
+ Language.make_featdict(featdict[feat], new_feat, features, form)
+
def add_class(self, cls):
if cls in self.words:
s = "Class {} already in dictionary"
@@ -183,14 +272,46 @@
self.changed = True
return entry
- def add_group(self, name, head, head_order=None, head_lexeme=False):
- entry = Group(name, self, head, head_order=head_order,
head_lexeme=head_lexeme)
- if head not in self.groups:
- self.groups[head] = []
- self.groups[head].append(entry)
+ def add_mwe(self, name, head, head_order=None, head_lexeme=False):
+ entry = MWE(name, self, head, head_order=head_order,
head_lexeme=head_lexeme)
+ if head not in self.mwes:
+ self.mwes[head] = []
+ self.mwes[head].append(entry)
self.changed = True
return entry
+ def add_group(self, words, head_index=-1, head='', name='',
features=None):
+ group = Group(words, head_index=head_index, head=head,
+ language=self, name=name, features=features)
+# print('Group {}, head {}'.format(group, group.head))
+ if features:
+ head_i = words.index(group.head)
+ head_feats = features[head_i]
+ else:
+ head_feats = None
+ self.add_group_to_lexicon(group.head, group, head_feats)
+ self.changed = True
+ return group
+
+ def add_group_to_lexicon(self, head, group, features):
+ if not features:
+ # Add the group to the list of groups for the head word/lexeme
+ if head not in self.groups:
+ self.groups[head] = {}
+ if () not in self.groups[head]:
+ self.groups[head][()] = []
+ self.groups[head][()].append(group)
+ else:
+ # Convert fv dict to an alphabetized tuple of fv pairs
+ fvs = list(features.items())
+ fvs.sort()
+ fvs = tuple(fvs)
+ if head not in self.groups:
+ self.groups[head] = {}
+ if fvs not in self.groups[head]:
+ self.groups[head][fvs] = []
+ self.groups[head][fvs].append(group)
+
def add_gram(self, gram, feature, count=1):
"""A gram, for example, 'plural', must have a feature, for example,
'number'."""
@@ -237,6 +358,8 @@
"""Returns a single lexeme entry."""
return self.words.get(lexeme)[0]
+ ### Analysis and generation of word forms.
+
## Dependencies (word, lexeme, class entries)
def record_label(self, label):
=======================================
--- /l3lite/ui.py Thu Mar 27 17:19:15 2014 UTC
+++ /l3lite/ui.py Sun Apr 20 07:07:10 2014 UTC
@@ -34,8 +34,6 @@
from .language import *
import os, sys
-LANGUAGE_DIR = os.path.join(os.path.dirname(__file__), 'languages')
-
class UI:
"""Normally only one of these so doesn't have to be a class. Later a
subclass
of tkinter Frame?"""
=======================================
--- /lite.py Wed Apr 16 06:00:17 2014 UTC
+++ /lite.py Sun Apr 20 07:07:10 2014 UTC
@@ -28,7 +28,7 @@
# 2014.02.09
# -- Created
-__version__ = 0.1
+__version__ = 0.3
import l3lite
@@ -39,70 +39,80 @@
def language(name, abbrev):
return l3lite.Language(name, abbrev)
-def eg():
- ### Spanish
- e = l3lite.Language('español', 'spa')
- ## Words
- mm = e.add_word('mujer', cls='$sus')
- la = e.add_word('la', cls='$det')
- ## Classes
- ss = e.add_class('$sus')
- ss.add_depin('sj', {'min': 0, 'max': 1, 'dflt': 0})
- ss.add_depin('oj', {'min': 0, 'max': 1, 'dflt': 0})
- ss.add_depout('det', {'min': 0, 'max': 1, 'dflt': 1})
- ss.add_order(['det', '^'])
- ss.set_gram('num', {'sing': 2, 'plur': 1})
- ss.set_gram('gen', {'masc': 1, 'fem': 1})
- ss.add_agr('det', 'gen', 'gen')
- dd = e.add_class('$det')
- dd.add_depin('det', {'min': 1, 'max': 1})
- # alguien
- e.add_class('$alg')
- # Some grammatical features
- e.add_gram('plur', 'num')
- e.add_gram('sing', 'num')
- e.add_gram('pres', 'tmp')
- e.add_gram('pret', 'tmp')
- ## Translations
- mm.add_trans('grn', 'kuña')
- mm.add_trans('grn', 'kuñakarai')
- mm.set_gram('num', 'sing')
- mm.set_gram('gen', 'fem')
- la.set_gram('num', 'sing')
- la.set_gram('gen', 'fem')
- # Inheritance (after class creation)
- la.inherit()
- mm.inherit()
- ### Guarani
- g = l3lite.Language('guarani', 'grn')
- ## Words
- kk = g.add_word('kuña')
- ### English
- E = l3lite.Language('English', 'eng')
- ## Words
- rr = E.add_word('read')
- tt = E.add_word('the')
-# rrr = E.add_word('riot')
-# rrr = E.add_word('act')
- nn = E.add_class('$sbd')
- ## Groups
- # English: read sbd. the riot act
- rra = E.add_group('read_sbd_the_riot_act', 'read')
- rra.add_word('$sbd')
- rra.add_word('the')
- rra.add_word('riot')
- rra.add_word('act')
- # Spanish: cantar (a) alg. las cuarenta
- clc = e.add_group('cantar_alg_las_cuarenta', '%cantar')
- clc.add_word('$alg')
- clc.add_word('las')
- clc.add_word('cuarenta')
- # Translation: Eng->Spa
- rra.add_trans('spa', 'cantar_alg_las_cuarenta')
- rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 0, 0)
- rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 1, 1)
- rra.add_trans_del('spa', 'cantar_alg_las_cuarenta', 3)
- return e, g, E
+def phrases():
+ eng = language('English', 'eng')
+ esp = language('español', 'spa')
+ amh = language('አማርኛ', 'amh')
+ rra = eng.add_phrase(['%read', '$sbd', 'the', 'riot', 'act'],
head='%read')
+# clc = esp.add_phrase(['%cantar', 'a', '$algn', 'las', 'cuarenta'], 0)
+ vo_awq = amh.add_phrase(['$vb', "%'wq"],
+ head="%'wq",
+ features=[{'tam': 'ger'},
{'tam': 'imf', 'pol': 'neg'}],
+ name="%%በልቶ_አያቅም")
+ return rra, vo_awq
+
+##def eg():
+## ### Spanish
+## e = l3lite.Language('español', 'spa')
+## ## Words
+## mm = e.add_word('mujer', cls='$sus')
+## la = e.add_word('la', cls='$det')
+## ## Classes
+## ss = e.add_class('$sus')
+## ss.add_depin('sj', {'min': 0, 'max': 1, 'dflt': 0})
+## ss.add_depin('oj', {'min': 0, 'max': 1, 'dflt': 0})
+## ss.add_depout('det', {'min': 0, 'max': 1, 'dflt': 1})
+## ss.add_order(['det', '^'])
+## ss.set_gram('num', {'sing': 2, 'plur': 1})
+## ss.set_gram('gen', {'masc': 1, 'fem': 1})
+## ss.add_agr('det', 'gen', 'gen')
+## dd = e.add_class('$det')
+## dd.add_depin('det', {'min': 1, 'max': 1})
+## # alguien
+## e.add_class('$alg')
+## # Some grammatical features
+## e.add_gram('plur', 'num')
+## e.add_gram('sing', 'num')
+## e.add_gram('pres', 'tmp')
+## e.add_gram('pret', 'tmp')
+## ## Translations
+## mm.add_trans('grn', 'kuña')
+## mm.add_trans('grn', 'kuñakarai')
+## mm.set_gram('num', 'sing')
+## mm.set_gram('gen', 'fem')
+## la.set_gram('num', 'sing')
+## la.set_gram('gen', 'fem')
+## # Inheritance (after class creation)
+## la.inherit()
+## mm.inherit()
+## ### Guarani
+## g = l3lite.Language('guarani', 'grn')
+## ## Words
+## kk = g.add_word('kuña')
+## ### English
+## E = l3lite.Language('English', 'eng')
+## ## Words
+## rr = E.add_word('read')
+## tt = E.add_word('the')
+## nn = E.add_class('$sbd')
+## ## Groups
+## # English: read sbd. the riot act
+## rra = E.add_group('read_sbd_the_riot_act', 'read')
+## rra.add_word('$sbd')
+## rra.add_word('the')
+## rra.add_word('riot')
+## rra.add_word('act')
+## # Spanish: cantar (a) alg. las cuarenta
+## clc = e.add_group('cantar_alg_las_cuarenta', '%cantar')
+## clc.add_word('$alg')
+## clc.add_word('las')
+## clc.add_word('cuarenta')
+## # Translation: Eng->Spa
+## rra.add_trans('spa', 'cantar_alg_las_cuarenta')
+## rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 0, 0)
+## rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 1, 1)
+## rra.add_trans_del('spa', 'cantar_alg_las_cuarenta', 3)
+## return e, g, E
def ui():
u = l3lite.UI()
@@ -191,20 +201,7 @@
g3snode = l3lite.DetIVar('g3snode', 2) # {2}
g4snode = l3lite.DetIVar('g4snode', 5) # {5}
g5snode = l3lite.Var('g5snode', set(), {0, 1, 2, 3, 4}, 3, 3) #
impossible
-## # Constrain gnode positions by individual gnode variables
-## gsnodeU0 = l3lite.UnionSelection(g0snode,
l3lite.Determined('gnpossel0', {0, 1, 2}),
-## [gno0, gno1, gno2])
-## gsnodeU1 = l3lite.UnionSelection(g1snode,
l3lite.Determined('gnpossel1', {0, 1}),
-## [gno3, gno4])
-## gsnodeU2 = l3lite.UnionSelection(g2snode,
l3lite.Determined('gnpossel2', {0, 1}),
-## [gno5, gno6])
-### gsnodeU5 = l3lite.UnionSelection(g5snode,
l3lite.Determined('gnpossel5', {0, 1, 2}),
-### [gno9, gno10, gno11])
-## # Set convexity constraint for each of these
-## setcon0 = l3lite.SetConvexity(g0snode)
-## setcon1 = l3lite.SetConvexity(g1snode)
-## setcon2 = l3lite.SetConvexity(g2snode)
-### setcon5 = l3lite.SetConvexity(g5snode)
+ # Set convexity constraint for each of these
setconv = l3lite.ComplexSetConvexity(groups, [g0snode, g1snode,
g2snode,
g3snode, g4snode,
g5snode])
## ComplexUnionSelection
@@ -212,13 +209,6 @@
## selvars: [g0snode, g1snode, ...]
## seqvars: w2gnodes
## selvar: groups
- # mainvars
-# gnodes0 = l3lite.Determined('gnodes0', {0, 1, 2})
-# gnodes1 = l3lite.Determined('gnodes1', {3, 4})
-# gnodes2 = l3lite.Determined('gnodes2', {5, 6})
-# gnodes3 = l3lite.DetIVar('gnodes3', 7)
-# gnodes4 = l3lite.DetIVar('gnodes4', 8)
-# gnodes5 = l3lite.Determined('gnodes5', {9, 10, 11})
complexU = l3lite.ComplexUnionSelection(selvar=groups,
selvars=[g0snode, g1snode,
g2snode, g3snode,
g4snode, g5snode],
Branch: default
Author: Michael Gasser <gas...@cs.indiana.edu>
Date: Sun Apr 20 07:07:10 2014 UTC
Log: L3Lite: Sentence, Node, Features classes.
http://code.google.com/p/hltdi-l3/source/detail?r=edc7222fb039
Added:
/l3lite/features.py
/l3lite/sentence.py
Modified:
/l3lite/__init__.py
/l3lite/constraint.py
/l3lite/entry.py
/l3lite/language.py
/l3lite/ui.py
/lite.py
=======================================
--- /dev/null
+++ /l3lite/features.py Sun Apr 20 07:07:10 2014 UTC
@@ -0,0 +1,84 @@
+#
+# L3Lite features (dicts).
+#
+########################################################################
+#
+# This file is part of the HLTDI L^3 project
+# for parsing, generation, translation, and computer-assisted
+# human translation.
+#
+# Copyright (C) 2014, HLTDI <gas...@cs.indiana.edu>
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =========================================================================
+
+# 2014.04.19
+# -- Created.
+
+class Features(dict):
+
+ def __init__(self, dct):
+ dict.__init__(self, dct)
+
+ def __repr__(self):
+ l = []
+ for f, v in self.items():
+ l.append("{}={}".format(f, v))
+ return "{{{0}}}".format(','.join(l))
+
+ def to_list(self):
+ """Convert features dict to a sorted list."""
+ l = list(self.items())
+ l.sort()
+ return l
+
+ @staticmethod
+ def simple_unify(x, y):
+ """Unify the expressions x and y, returning the result
or 'fail'."""
+ # If they're the same, return one.
+ if x == y:
+ return x
+# # If both are dicts, call unify_dict
+# elif isinstance(x, dict) and isinstance(y, dict):
+# x.unify(y)
+ # Otherwise fail
+ else:
+ return 'fail'
+
+ def unify(self, other):
+ """other is a Features object or a dict. Attempt to unify self
with other,
+ returning the result or 'fail'."""
+ result = Features({})
+ for k in set(self.keys()) | set(other.keys()):
+ # Check all of the keys of self and other
+ self_val, other_val = self.get(k, 'nil'), other.get(k, 'nil')
+ if self_val != 'nil':
+ if other_val != 'nil':
+ # If x and y both have a value for k, try to unify the
values
+ u = Features.simple_unify(self_val, other_val)
+ if u == 'fail':
+ return 'fail'
+ else:
+ result[k] = u
+ else:
+ # If self has a value for k but other doesn't, use
self's value
+ result[k] = self_val
+ elif other_val != 'nil':
+ # If other has a value for k but self doesn't, use other's
value
+ result[k] = other_val
+
+ return result
+
+
=======================================
--- /dev/null
+++ /l3lite/sentence.py Sun Apr 20 07:07:10 2014 UTC
@@ -0,0 +1,182 @@
+#
+# L3Lite sentences and how parse and translate them.
+#
+########################################################################
+#
+# This file is part of the HLTDI L^3 project
+# for parsing, generation, translation, and computer-assisted
+# human translation.
+#
+# Copyright (C) 2014, HLTDI <gas...@cs.indiana.edu>
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =========================================================================
+
+# 2014.04.15
+# -- Created.
+
+# ui.py loads language, etc.
+from .ui import *
+
+class Sentence:
+ """A sentence is a list of words (or other lexical tokens) that gets
+ assigned a set of variables and constraints that are run during
+ parsing or translation."""
+
+ def __init__(self, raw='', language=None,
+ tokens=None, analyses=None,
+ nodes=None, groups=None):
+ # A string representing the raw sentence
+ self.raw = raw
+# # A list of strings tokenized words (or morphemes)
+# self.tokens = tokens or []
+# # A list of analyses pairs (root, features) for particular tokens
+# self.analyses = analyses or []
+ # A language object
+ self.language = language
+ # A list of Node objects, one for each token
+ self.nodes = nodes or []
+ # A list of candidate groups found during lexicalization
+ self.groups = groups or []
+ # A list of constraints to run
+ self.constraints = []
+
+ def __repr__(self):
+ """Print name."""
+ if self.raw:
+ return '|| {} ||'.format(self.raw)
+ else:
+ return '|| {} sentence ||'.format(self.language)
+
+ def tokenize(self):
+ """Segment the sentence string into tokens, analyze them
morphologically,
+ and create a Node object for each."""
+ if not self.nodes:
+ # (Otherwise it's already done.)
+ # Split at spaces by default (later allow for dedicated
language-specific tokenizers).
+ tokens = self.raw.split()
+ self.nodes = []
+ index = 0
+ for token in tokens:
+ # Look up token in language.forms
+ if token not in self.language.forms:
+ # Not found, just use the raw string
+ self.nodes.append(Node(token, index, None, self))
+ index += 1
+ else:
+ # A dict, for unambiguous forms, or a list of dicts,
for ambiguous forms
+ formdict = self.language.forms[token]
+ if isinstance(formdict, dict):
+ # A single entry
+ if 'seg' in formdict:
+ segs = formdict['seg']
+ for seg in segs:
+ tok, analysis = seg
+ self.nodes.append(Node(tok, index,
analysis, self))
+ index += 1
+ else:
+ self.nodes.append(Node(token, index, formdict,
self))
+ index += 1
+ else:
+ # Multiple dicts: ambiguity; let node handle it
+ self.nodes.append(Node(token, index, formdict,
self))
+ index += 1
+
+ def activate_groups(self):
+ """Activate all groups that are compatible with the tokens in the
sentence."""
+ if not self.nodes:
+ print("Tokenization must precede group activation.")
+ return
+ candidates = []
+ for node in self.nodes:
+ keys = {node.token}
+ anal = node.analyses
+ if anal:
+ if isinstance(anal, list):
+ for a in anal:
+ keys.add(a.get('root'))
+ else:
+ keys.add(anal.get('root'))
+ for k in keys:
+ if k in self.language.groups:
+ for group in self.language.groups[k]:
+ candidates.append((node.index, group))
+ # Now filter candidates to see if all words are present in the
sentence
+ # For each group, save a list of list of sentence token indices
that correspond
+ # to the group's words
+ groups = []
+ for head_index, group in candidates:
+ indices = group.match_nodes(self.nodes, head_index)
+ if not indices:
+ # This group is out
+ continue
+ groups.append((head_index, indices, group))
+
+ self.groups = groups
+
+class Node:
+ """Sentence token and its associated analyses and variables."""
+
+ def __init__(self, token, index, analyses, sentence):
+ # Raw form in sentence (possibly result of segmentation)
+ self.token = token
+ # Position in sentence
+ self.index = index
+ # List of analyses
+ if analyses and not isinstance(analyses, list):
+ analyses = [analyses]
+ self.analyses = analyses
+ # Back pointer to sentence
+ self.sentence = sentence
+ # We'll need these for multiple matchings
+ self.cats = self.get_cats()
+
+ def __repr__(self):
+ """Print name."""
+ return "*{}".format(self.token)
+
+ def get_cats(self):
+ """The set of categories for the node's token, or None."""
+ if not self.analyses:
+ return None
+ cats = set()
+ for analysis in self.analyses:
+ if 'cats' in analysis:
+ cats.update(analysis['cats'])
+ return cats
+
+ def match(self, item, features):
+ """Does this node match the group item (word, lexeme, category) and
+ and any features associated with it?"""
+ # If item is a category, don't bother looking at token
+ if Entry.is_cat(item):
+ if self.cats and item in self.cats:
+ return True
+ elif self.token == item:
+ # item matches this node's token; features are irrelevant
+ return True
+ elif self.analyses:
+ # Check each combination of root and analysis features
+ for analysis in self.analyses:
+ root = analysis.get('root', '')
+ node_features = analysis.get('features')
+ if root == item:
+ if not features:
+ return True
+ elif not node_features:
+ return True
+ elif node_features.unify(features):
+ return True
+ return False
=======================================
--- /l3lite/__init__.py Fri Apr 4 06:26:55 2014 UTC
+++ /l3lite/__init__.py Sun Apr 20 07:07:10 2014 UTC
@@ -1,5 +1,5 @@
"""Do-it-yourself L3. Create simple bilingual lexicons and grammars for
language pairs."""
-__all__ = ['language', 'entry', 'ui', 'constraint', 'variable']
+__all__ =
['language', 'entry', 'ui', 'constraint', 'variable', 'sentence', 'features']
-from .ui import *
+from .sentence import *
=======================================
--- /l3lite/constraint.py Wed Apr 16 06:00:17 2014 UTC
+++ /l3lite/constraint.py Sun Apr 20 07:07:10 2014 UTC
@@ -32,6 +32,10 @@
# -- Created ComplexSetConvexity
# 2014.04.05
# -- Created ComplexUnionSelection
+# 2014.04.15
+# -- Constraint types used so far:
+# UnionSelection, PrecedenceSelection, ComplexUnionSelection,
+# ComplexSetConvexity, Union, Disjoint, Inclusion
from .variable import *
import itertools
=======================================
--- /l3lite/entry.py Thu Mar 27 17:19:15 2014 UTC
+++ /l3lite/entry.py Sun Apr 20 07:07:10 2014 UTC
@@ -41,16 +41,19 @@
# -- Lots of changes and additions to groups.
# 2014.03.24
# -- words attribute in Group is a list of [word, feat_dict] pairs.
+# 2014.04.16
+# -- Created simpler Group (with no dependency types), renamed old Group
to MWE.
import copy, itertools
import yaml
+from .features import *
+
LEXEME_PRE = '%'
CLASS_PRE = '$'
class Entry:
- """The central unit in L3Lite, containing lexical and grammatical
- constraints that are instantiated during processing."""
+ """Superclass for Group and possibly other lexical classes."""
ID = 1
dflt_dep = 'dflt'
@@ -71,6 +74,11 @@
"""Print name."""
return '<{}:{}>'.format(self.name, self.id)
+ @staticmethod
+ def is_cat(name):
+ """Is this the name of a category?"""
+ return name[0] == CLASS_PRE
+
## Serialization
def to_dict(self):
@@ -410,18 +418,18 @@
def inherit_gov(self, cls):
"""Inherit government constraints from class."""
-class Group(Entry):
+class MWE(Entry):
"""Multi-word expressions. Each group consists of a head and a set of
nodes,
possibly connected to other nodes through explicit dependencies and an
explicit
order of the nodes.
Variable slots have dedicated names that allow them to be
referenced in translations.
- Groups must be created *after* other lexical items.
+ MWEs must be created *after* other lexical items.
{index: [word_obj, {dep/position_feats}...}
"""
def __init__(self, name, language, head, head_feats=None,
head_order=None, head_lexeme=False):
- """name of a Group is something like acabar_de_V.
+ """name of a MWE is something like acabar_de_V.
head is the word that is the syntactic head of the group."""
Entry.__init__(self, name, language)
# A list of [word feats] pairs; index in the list is the word's
(node's) ID
@@ -473,15 +481,15 @@
@staticmethod
def from_dict(d, language):
- """Convert a dict (loaded from a yaml file) to a Lex object."""
+ """Convert a dict (loaded from a yaml file) to a MWE object."""
lexeme = d['head_lexeme']
- g = Group(d.get('name'), language, d.get('words')[0][0],
head_lexeme=lexeme)
+ m = MWE(d.get('name'), language, d.get('words')[0][0],
head_lexeme=lexeme)
# for id, word in d.get('words').items():
for id, word in enumerate(d.get('words')):
if id == 0:
# Just handle the dependencies for this case
deps = word[1]
- g.words[0][1] = copy.deepcopy(deps)
+ m.words[0][1] = copy.deepcopy(deps)
else:
name = word[0]
lex = language.get_words(name)[0]
@@ -490,9 +498,8 @@
lex_info = [lex.clone(), copy.deepcopy(deps)]
else:
lex_info = [lex.clone()]
- g.words.append(lex_info)
-# g.words[id] = lex_info
- return g
+ m.words.append(lex_info)
+ return m
## Getters
@@ -634,6 +641,91 @@
tdict['m'].append(trg_id)
# tdict['m'].append((-1, trg_id))
+class Group(Entry):
+ """Primitive multi-word expressions. Default is a head with unlabeled
dependencies
+ to all other words and translations, including alignments, to one or
more
+ other languages."""
+
+ def __init__(self, words, head_index=-1, head='', language=None,
name='',
+ features=None):
+ # words is a list of strings or (string, dict) tuples
+ # name may be specified explicitly or not
+ name = name or Group.make_name(words)
+ Entry.__init__(self, name, language)
+ self.words = words
+ if head:
+ self.head = head
+ else:
+ self.head = words[head_index]
+ # Either None or a list of feat-val dicts for words that require
them
+ # Convert dicts to Features objects
+ if isinstance(features, list):
+ features = [Features(d) for d in features]
+ self.features = features
+
+ def __repr__(self):
+ """Print name."""
+ return '<{}:{}>'.format(self.name, self.id)
+
+ @staticmethod
+ def make_name(words):
+ # Each word is either a string or a (string, feat_dict) pair
+# strings = []
+# for word in words:
+# if isinstance(word, str):
+# strings.append(word)
+# else:
+# form, feat_dict = word
+# fv = ['{}={}'.format(f, v) for f, v in feat_dict.items()]
+# fv = ','.join(fv)
+# strings.append("{}:{}".format(form, fv))
+ return '_'.join(words)
+
+ # Serialization
+
+ def to_dict(self):
+ """Convert the group to a dictionary to be serialized in a yaml
file."""
+ d = Entry.to_dict(self)
+ d['words'] = self.words
+ d['features'] = self.features
+ return d
+
+ @staticmethod
+ def from_dict(d, language, head):
+ """Convert a dict (loaded from a yaml file) to a Group object."""
+ words = d['words']
+ features = d.get('features')
+ p = Group(words, head=head, language=language, features=features)
+ return p
+
+ def match_nodes(self, nodes, head_index):
+ """Attempt to match the group words (and features) with nodes from
a sentence."""
+# print("Does {} match {}".format(self, nodes))
+ tok_indices = []
+ for index, word in enumerate(self.words):
+# print(" Attempting to match {}".format(word))
+ matched = False
+ for node in nodes:# for word, feats in zip(self.words,
self.features):
+
+# print(" Trying {}".format(node))
+ if index == node.index == head_index:
+ # This is the token corresponding to the group head
+ tok_indices.append(node.index)
+# print(" Head matched already".format(node))
+ matched = True
+ break
+ else:
+ feats = self.features[index] if self.features else None
+ if node.match(word, feats):
+ tok_indices.append(node.index)
+# print(" Matched node {}".format(node))
+ matched = True
+ break
+ if not matched:
+# print(" {} not matched; failed".format(word))
+ return False
+ return tok_indices
+
class EntryError(Exception):
'''Class for errors encountered when attempting to update an entry.'''
=======================================
--- /l3lite/language.py Fri Apr 4 06:26:55 2014 UTC
+++ /l3lite/language.py Sun Apr 20 07:07:10 2014 UTC
@@ -33,12 +33,16 @@
# 2014.03.24
# -- Words, lexemes, and classes are all in the same dictionary
(self.words).
# Lexemes start with %, classes with $.
+# 2014.04.17
+# -- Analysis and generation dicts for particular wordforms.
from .entry import *
from .constraint import *
import os, yaml
+LANGUAGE_DIR = os.path.join(os.path.dirname(__file__), 'languages')
+
class Language:
"""Dictionaries of words, lexemes, grammatical features, and
lexical classes."""
@@ -47,7 +51,9 @@
def __init__(self,
name, abbrev,
- words=None, lexemes=None, grams=None, classes=None,
groups=None):
+ words=None, lexemes=None, grams=None, classes=None,
+ mwes=None, groups=None, forms=None,
+ genforms=None):
"""Initialize dictionaries and names."""
self.name = name
self.abbrev = abbrev
@@ -57,11 +63,16 @@
# self.lexemes = lexemes or {}
self.grams = grams or {}
# self.classes = classes or {}
+ self.mwes = mwes or {}
+ self.forms = forms or {}
self.groups = groups or {}
# Record possibilities for dependency labels, feature values,
order constraints
self.possible = {}
# Record whether language has changed since last loaded
self.changed = False
+ # Dictionary of morphologically generated words:
+ # {lexeme: {(feat, val): {(feat, val): wordform,...}, ...}, ...}
+ self.genforms = genforms or {}
Language.languages.append(abbrev)
def __repr__(self):
@@ -97,11 +108,21 @@
# Words are lists
words[k] = [lex.to_dict() for lex in v]
d['words'] = words
+ if self.mwes:
+ mwes = {}
+ for k, v in self.mwes.items():
+ mwes[k] = [m.to_dict() for m in v]
+ d['mwes'] = mwes
if self.groups:
groups = {}
- for k, v in self.groups.items():
- groups[k] = [g.to_dict() for g in v]
+ for head, v in self.groups.items():
+ groups[head] = [g.to_dict() for g in v]
d['groups'] = groups
+ if self.forms:
+ forms = {}
+ for k, v in self.forms.items():
+ # v is an fv dict or a list of fv dicts
+ forms[k] = v
return d
def write(self, directory, filename=''):
@@ -136,11 +157,30 @@
l.words = {}
for k, v in words.items():
l.words[k] = [Lex.from_dict(lex, l) for lex in v]
+ mwes = d.get('mwes')
+ if mwes:
+ l.mwes = {}
+ for k, v in mwes.items():
+ l.mwes[k] = [MWE.from_dict(m, l) for m in v]
groups = d.get('groups')
if groups:
l.groups = {}
- for k, v in groups.items():
- l.groups[k] = [Group.from_dict(g, l) for g in v]
+ for head, v in groups.items():
+ l.groups[head] = [Group.from_dict(g, l, head) for g in v]
+ forms = d.get('forms')
+ if forms:
+ l.forms = {}
+ for k, v in forms.items():
+ # v should be a dict or a list of dicts
+ # Convert features value to a Features object
+ if isinstance(v, dict):
+ if 'features' in v:
+ v['features'] = Features(v['features'])
+ else:
+ for d in v:
+ if 'features' in d:
+ d['features'] = Features(d['features'])
+ l.forms[k] = v
return l
@staticmethod
@@ -151,11 +191,21 @@
dct = yaml.load(file)
return Language.from_dict(dct)
+ @staticmethod
+ def load(abbrev):
+ path = os.path.join(LANGUAGE_DIR, abbrev + '.lg')
+ try:
+ language = Language.read(path)
+ return language
+ except IOError:
+ print("That language doesn't seem to exist.")
+ return
+
### Basic setters. Create entries (dicts) for item. For debugging
purposes, include name
### in entry.
- def add_word(self, word, cls=None, group=False):
- entry = Lex(word, self, cls=cls, group=group)
+ def add_word(self, word, cls=None, mwe=False):
+ entry = Lex(word, self, cls=cls, mwe=mwe)
if word in self.words:
self.words[word].append(entry)
else:
@@ -173,6 +223,45 @@
self.changed = True
return entry
+ def add_form(self, form, dct, reverse=True):
+ """Form dict has root, features, cats.
+ If reverse it True, also add the form to the genforms dicdt."""
+ if form not in self.forms:
+ self.forms[form] = dct
+ else:
+ entry = self.forms[form]
+ if isinstance(entry, dict):
+ # Make this the second entry
+ self.forms[form] = [entry, dct]
+ else:
+ # There are already two or more entries in a list
+ entry.append(dct)
+ if reverse:
+ lexeme = dct['root']
+ features = dct['features']
+ self.add_genform(form, lexeme, features)
+
+ def add_genform(self, form, lexeme, features):
+ """Add the form to a lexeme- and feature-keyed dict."""
+ if lexeme not in self.genforms:
+ self.genforms[lexeme] = {}
+ featdict = self.genforms[lexeme]
+ # features is a Features object; convert it to a list of tuples
+ features = features.to_list()
+ feat = features.pop(0)
+ self.make_featdict(featdict, feat, features, form)
+
+ @staticmethod
+ def make_featdict(featdict, feat, features, form):
+ """Make a feat-value dict with the form as final value."""
+ if not features:
+ featdict[feat] = form
+ return
+ if feat not in featdict:
+ featdict[feat] = {}
+ new_feat = features.pop(0)
+ Language.make_featdict(featdict[feat], new_feat, features, form)
+
def add_class(self, cls):
if cls in self.words:
s = "Class {} already in dictionary"
@@ -183,14 +272,46 @@
self.changed = True
return entry
- def add_group(self, name, head, head_order=None, head_lexeme=False):
- entry = Group(name, self, head, head_order=head_order,
head_lexeme=head_lexeme)
- if head not in self.groups:
- self.groups[head] = []
- self.groups[head].append(entry)
+ def add_mwe(self, name, head, head_order=None, head_lexeme=False):
+ entry = MWE(name, self, head, head_order=head_order,
head_lexeme=head_lexeme)
+ if head not in self.mwes:
+ self.mwes[head] = []
+ self.mwes[head].append(entry)
self.changed = True
return entry
+ def add_group(self, words, head_index=-1, head='', name='',
features=None):
+ group = Group(words, head_index=head_index, head=head,
+ language=self, name=name, features=features)
+# print('Group {}, head {}'.format(group, group.head))
+ if features:
+ head_i = words.index(group.head)
+ head_feats = features[head_i]
+ else:
+ head_feats = None
+ self.add_group_to_lexicon(group.head, group, head_feats)
+ self.changed = True
+ return group
+
+ def add_group_to_lexicon(self, head, group, features):
+ if not features:
+ # Add the group to the list of groups for the head word/lexeme
+ if head not in self.groups:
+ self.groups[head] = {}
+ if () not in self.groups[head]:
+ self.groups[head][()] = []
+ self.groups[head][()].append(group)
+ else:
+ # Convert fv dict to an alphabetized tuple of fv pairs
+ fvs = list(features.items())
+ fvs.sort()
+ fvs = tuple(fvs)
+ if head not in self.groups:
+ self.groups[head] = {}
+ if fvs not in self.groups[head]:
+ self.groups[head][fvs] = []
+ self.groups[head][fvs].append(group)
+
def add_gram(self, gram, feature, count=1):
"""A gram, for example, 'plural', must have a feature, for example,
'number'."""
@@ -237,6 +358,8 @@
"""Returns a single lexeme entry."""
return self.words.get(lexeme)[0]
+ ### Analysis and generation of word forms.
+
## Dependencies (word, lexeme, class entries)
def record_label(self, label):
=======================================
--- /l3lite/ui.py Thu Mar 27 17:19:15 2014 UTC
+++ /l3lite/ui.py Sun Apr 20 07:07:10 2014 UTC
@@ -34,8 +34,6 @@
from .language import *
import os, sys
-LANGUAGE_DIR = os.path.join(os.path.dirname(__file__), 'languages')
-
class UI:
"""Normally only one of these so doesn't have to be a class. Later a
subclass
of tkinter Frame?"""
=======================================
--- /lite.py Wed Apr 16 06:00:17 2014 UTC
+++ /lite.py Sun Apr 20 07:07:10 2014 UTC
@@ -28,7 +28,7 @@
# 2014.02.09
# -- Created
-__version__ = 0.1
+__version__ = 0.3
import l3lite
@@ -39,70 +39,80 @@
def language(name, abbrev):
return l3lite.Language(name, abbrev)
-def eg():
- ### Spanish
- e = l3lite.Language('español', 'spa')
- ## Words
- mm = e.add_word('mujer', cls='$sus')
- la = e.add_word('la', cls='$det')
- ## Classes
- ss = e.add_class('$sus')
- ss.add_depin('sj', {'min': 0, 'max': 1, 'dflt': 0})
- ss.add_depin('oj', {'min': 0, 'max': 1, 'dflt': 0})
- ss.add_depout('det', {'min': 0, 'max': 1, 'dflt': 1})
- ss.add_order(['det', '^'])
- ss.set_gram('num', {'sing': 2, 'plur': 1})
- ss.set_gram('gen', {'masc': 1, 'fem': 1})
- ss.add_agr('det', 'gen', 'gen')
- dd = e.add_class('$det')
- dd.add_depin('det', {'min': 1, 'max': 1})
- # alguien
- e.add_class('$alg')
- # Some grammatical features
- e.add_gram('plur', 'num')
- e.add_gram('sing', 'num')
- e.add_gram('pres', 'tmp')
- e.add_gram('pret', 'tmp')
- ## Translations
- mm.add_trans('grn', 'kuña')
- mm.add_trans('grn', 'kuñakarai')
- mm.set_gram('num', 'sing')
- mm.set_gram('gen', 'fem')
- la.set_gram('num', 'sing')
- la.set_gram('gen', 'fem')
- # Inheritance (after class creation)
- la.inherit()
- mm.inherit()
- ### Guarani
- g = l3lite.Language('guarani', 'grn')
- ## Words
- kk = g.add_word('kuña')
- ### English
- E = l3lite.Language('English', 'eng')
- ## Words
- rr = E.add_word('read')
- tt = E.add_word('the')
-# rrr = E.add_word('riot')
-# rrr = E.add_word('act')
- nn = E.add_class('$sbd')
- ## Groups
- # English: read sbd. the riot act
- rra = E.add_group('read_sbd_the_riot_act', 'read')
- rra.add_word('$sbd')
- rra.add_word('the')
- rra.add_word('riot')
- rra.add_word('act')
- # Spanish: cantar (a) alg. las cuarenta
- clc = e.add_group('cantar_alg_las_cuarenta', '%cantar')
- clc.add_word('$alg')
- clc.add_word('las')
- clc.add_word('cuarenta')
- # Translation: Eng->Spa
- rra.add_trans('spa', 'cantar_alg_las_cuarenta')
- rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 0, 0)
- rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 1, 1)
- rra.add_trans_del('spa', 'cantar_alg_las_cuarenta', 3)
- return e, g, E
+def phrases():
+ eng = language('English', 'eng')
+ esp = language('español', 'spa')
+ amh = language('አማርኛ', 'amh')
+ rra = eng.add_phrase(['%read', '$sbd', 'the', 'riot', 'act'],
head='%read')
+# clc = esp.add_phrase(['%cantar', 'a', '$algn', 'las', 'cuarenta'], 0)
+ vo_awq = amh.add_phrase(['$vb', "%'wq"],
+ head="%'wq",
+ features=[{'tam': 'ger'},
{'tam': 'imf', 'pol': 'neg'}],
+ name="%%በልቶ_አያቅም")
+ return rra, vo_awq
+
+##def eg():
+## ### Spanish
+## e = l3lite.Language('español', 'spa')
+## ## Words
+## mm = e.add_word('mujer', cls='$sus')
+## la = e.add_word('la', cls='$det')
+## ## Classes
+## ss = e.add_class('$sus')
+## ss.add_depin('sj', {'min': 0, 'max': 1, 'dflt': 0})
+## ss.add_depin('oj', {'min': 0, 'max': 1, 'dflt': 0})
+## ss.add_depout('det', {'min': 0, 'max': 1, 'dflt': 1})
+## ss.add_order(['det', '^'])
+## ss.set_gram('num', {'sing': 2, 'plur': 1})
+## ss.set_gram('gen', {'masc': 1, 'fem': 1})
+## ss.add_agr('det', 'gen', 'gen')
+## dd = e.add_class('$det')
+## dd.add_depin('det', {'min': 1, 'max': 1})
+## # alguien
+## e.add_class('$alg')
+## # Some grammatical features
+## e.add_gram('plur', 'num')
+## e.add_gram('sing', 'num')
+## e.add_gram('pres', 'tmp')
+## e.add_gram('pret', 'tmp')
+## ## Translations
+## mm.add_trans('grn', 'kuña')
+## mm.add_trans('grn', 'kuñakarai')
+## mm.set_gram('num', 'sing')
+## mm.set_gram('gen', 'fem')
+## la.set_gram('num', 'sing')
+## la.set_gram('gen', 'fem')
+## # Inheritance (after class creation)
+## la.inherit()
+## mm.inherit()
+## ### Guarani
+## g = l3lite.Language('guarani', 'grn')
+## ## Words
+## kk = g.add_word('kuña')
+## ### English
+## E = l3lite.Language('English', 'eng')
+## ## Words
+## rr = E.add_word('read')
+## tt = E.add_word('the')
+## nn = E.add_class('$sbd')
+## ## Groups
+## # English: read sbd. the riot act
+## rra = E.add_group('read_sbd_the_riot_act', 'read')
+## rra.add_word('$sbd')
+## rra.add_word('the')
+## rra.add_word('riot')
+## rra.add_word('act')
+## # Spanish: cantar (a) alg. las cuarenta
+## clc = e.add_group('cantar_alg_las_cuarenta', '%cantar')
+## clc.add_word('$alg')
+## clc.add_word('las')
+## clc.add_word('cuarenta')
+## # Translation: Eng->Spa
+## rra.add_trans('spa', 'cantar_alg_las_cuarenta')
+## rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 0, 0)
+## rra.add_trans_map('spa', 'cantar_alg_las_cuarenta', 1, 1)
+## rra.add_trans_del('spa', 'cantar_alg_las_cuarenta', 3)
+## return e, g, E
def ui():
u = l3lite.UI()
@@ -191,20 +201,7 @@
g3snode = l3lite.DetIVar('g3snode', 2) # {2}
g4snode = l3lite.DetIVar('g4snode', 5) # {5}
g5snode = l3lite.Var('g5snode', set(), {0, 1, 2, 3, 4}, 3, 3) #
impossible
-## # Constrain gnode positions by individual gnode variables
-## gsnodeU0 = l3lite.UnionSelection(g0snode,
l3lite.Determined('gnpossel0', {0, 1, 2}),
-## [gno0, gno1, gno2])
-## gsnodeU1 = l3lite.UnionSelection(g1snode,
l3lite.Determined('gnpossel1', {0, 1}),
-## [gno3, gno4])
-## gsnodeU2 = l3lite.UnionSelection(g2snode,
l3lite.Determined('gnpossel2', {0, 1}),
-## [gno5, gno6])
-### gsnodeU5 = l3lite.UnionSelection(g5snode,
l3lite.Determined('gnpossel5', {0, 1, 2}),
-### [gno9, gno10, gno11])
-## # Set convexity constraint for each of these
-## setcon0 = l3lite.SetConvexity(g0snode)
-## setcon1 = l3lite.SetConvexity(g1snode)
-## setcon2 = l3lite.SetConvexity(g2snode)
-### setcon5 = l3lite.SetConvexity(g5snode)
+ # Set convexity constraint for each of these
setconv = l3lite.ComplexSetConvexity(groups, [g0snode, g1snode,
g2snode,
g3snode, g4snode,
g5snode])
## ComplexUnionSelection
@@ -212,13 +209,6 @@
## selvars: [g0snode, g1snode, ...]
## seqvars: w2gnodes
## selvar: groups
- # mainvars
-# gnodes0 = l3lite.Determined('gnodes0', {0, 1, 2})
-# gnodes1 = l3lite.Determined('gnodes1', {3, 4})
-# gnodes2 = l3lite.Determined('gnodes2', {5, 6})
-# gnodes3 = l3lite.DetIVar('gnodes3', 7)
-# gnodes4 = l3lite.DetIVar('gnodes4', 8)
-# gnodes5 = l3lite.Determined('gnodes5', {9, 10, 11})
complexU = l3lite.ComplexUnionSelection(selvar=groups,
selvars=[g0snode, g1snode,
g2snode, g3snode,
g4snode, g5snode],
Reply all
Reply to author
Forward
0 new messages