Scripts and updated data for tagger training

Author: marcriera

.gitattributes

@@ -1,202 +1,7 @@
 * text=auto !eol
-/AUTHORS -text
-/COPYING -text
-/ChangeLog -text
-/Makefile.am -text
-/NEWS -text
-/README -text
-/apertium-eng.eng.acx -text svneol=unset#application/xml
-/apertium-eng.eng.dix -text
-/apertium-eng.eng.mtx -text
-/apertium-eng.eng.rlx -text
-/apertium-eng.eng.tsx -text
-/apertium-eng.pc.in -text
-/apertium-eng.post-eng.dix -text
-/autogen.sh -text
-/commondefns.mtx -text
-/configure.ac -text
-dev/eng.freq.txt -text
-/eng.prob -text svneol=unset#unset
-/modes.xml -text svneol=unset#application/xml
-/tagger.make -text
-texts/MANIFEST -text
-texts/TRAINING -text
-texts/afghanistan1.handtagged.ilie.txt -text
-texts/afghanistan1.handtagged.yongloong.txt -text
-texts/afghanistan1.raw.txt -text
-texts/afghanistan1.tagged.txt -text
-texts/aizkolaritza1.handtagged.andrei.txt -text
-texts/aizkolaritza1.raw.txt -text
-texts/aizkolaritza1.tagged.txt -text
-texts/alabama1.handtagged.darkgaia.txt -text
-texts/alabama1.handtagged.ilie.txt -text
-texts/alabama1.raw.txt -text
-texts/alabama1.tagged.txt -text
-texts/albedo1.handtagged.andrei.txt -text
-texts/albedo1.handtagged.niroulapradeep.txt -text
-texts/albedo1.raw.txt -text
-texts/albedo1.tagged.txt -text
-texts/alfred1.handtagged.darkgaia.txt -text
-texts/alfred1.handtagged.yongloong.txt -text
-texts/alfred1.raw.txt -text
-texts/alfred1.tagged.txt -text
-texts/algeria1.handtagged.quicksilver.txt -text
-texts/algeria1.handtagged.rap.txt -text
-texts/algeria1.handtagged.yugin.txt -text
-texts/algeria1.raw.txt -text
-texts/algeria1.tagged.txt -text
-texts/aluminium1.handtagged.andrei.txt -text
-texts/aluminium1.raw.txt -text
-texts/aluminium1.tagged.txt -text
-texts/ambiorix1.handtagged.darkgaia1.txt -text
-texts/ambiorix1.handtagged.quicksilver.txt -text
-texts/ambiorix1.raw.txt -text
-texts/ambiorix1.tagged.txt -text
-texts/anarchism1.handtagged.darkgaia.txt -text
-texts/anarchism1.handtagged.ilie.txt -text
-texts/anarchism1.raw.txt -text
-texts/anarchism1.tagged.txt -text
-texts/anarchism1.tagged.txt.1 -text
-texts/animism1.handtagged.andrei.txt -text
-texts/animism1.handtagged.cmy.txt -text
-texts/animism1.raw.txt -text
-texts/animism1.tagged.txt -text
-texts/atoms1.handtagged.andrei.txt -text
-texts/atoms1.handtagged.kristian-hansen.txt -text
-texts/atoms1.raw.txt -text
-texts/atoms1.tagged.txt -text
-texts/belize1.handtagged._VN_.txt -text
-texts/belize1.handtagged.darkgaia.txt -text
-texts/belize1.raw.txt -text
-texts/belize1.tagged.txt -text
-texts/belize2.handtagged.andrei.txt -text
-texts/belize2.handtagged.darkgaia.txt -text
-texts/belize2.raw.txt -text
-texts/belize2.tagged.txt -text
-texts/bessarabia1.handtagged.andrei.txt -text
-texts/bessarabia1.handtagged.darkgaia.txt -text
-texts/bessarabia1.raw.txt -text
-texts/bessarabia1.tagged.txt -text
-texts/caucasus1.handtagged.bobsan.txt -text
-texts/caucasus1.handtagged.leongxuhua.txt -text
-texts/caucasus1.handtagged.rap.txt -text
-texts/caucasus1.raw.txt -text
-texts/caucasus1.tagged.txt -text
-texts/coil1.handtagged.andrei.txt -text
-texts/coil1.handtagged.kranzer.txt -text
-texts/coil1.raw.txt -text
-texts/coil1.tagged.txt -text
-texts/colombia1.handtagged.ilie.txt -text
-texts/colombia1.handtagged.quicksilver.txt -text
-texts/colombia1.raw.txt -text
-texts/colombia1.tagged.txt -text
-texts/consensus/afghanistan1.consensus.txt -text
-texts/consensus/aizkolaritza1.consensus.txt -text
-texts/consensus/alabama1.consensus.txt -text
-texts/consensus/albedo1.consensus.txt -text
-texts/consensus/alfred1.consensus.txt -text
-texts/consensus/algeria1.consensus.txt -text
-texts/consensus/aluminium1.consensus.txt -text
-texts/consensus/ambiorix1.consensus.txt -text
-texts/consensus/anarchism1.consensus.txt -text
-texts/consensus/animism1.consensus.txt -text
-texts/consensus/atoms1.consensus.txt -text
-texts/consensus/belize1.consensus.txt -text
-texts/consensus/belize2.consensus.txt -text
-texts/consensus/bessarabia2.consensus.txt -text
-texts/consensus/caucasus1.consensus.txt -text
-texts/consensus/coil1.consensus.txt -text
-texts/consensus/colombia1.consensus.txt -text
-texts/consensus/dakhma1.consensus.txt -text
-texts/consensus/derbent1.consensus.txt -text
-texts/consensus/eng.tagged -text
-texts/consensus/faust1.consensus.txt -text
-texts/consensus/karabakh1.consensus.txt -text
-texts/consensus/kazakhstan1.consensus.txt -text
-texts/consensus/khazars1.consensus.txt -text
-texts/consensus/kyrgyzstan.consensus.txt -text
-texts/consensus/monarchy1.consensus.txt -text
-texts/consensus/partisans1.consensus.txt -text
-texts/consensus/peru1.consensus.txt -text
-texts/consensus/pirates.consensus.txt -text
-texts/consensus/sahara1.consensus.txt -text
-texts/consensus/sunday1.consensus.txt -text
-texts/consensus/tokamak1.consensus.txt -text
-texts/consensus/transmission1.consensus.txt -text
-texts/consensus/turing1.consensus.txt -text
-texts/consensus/vietnam1.consensus.txt -text
-texts/dakhma1.handtagged.spectie.txt -text
-texts/dakhma1.raw.txt -text
-texts/dakhma1.tagged.txt -text
-texts/derbent1.handtagged.andrei.txt -text
-texts/derbent1.handtagged.ilie.txt -text
-texts/derbent1.raw.txt -text
-texts/derbent1.tagged.txt -text
-texts/faust1.handtagged.spectie.txt -text
-texts/faust1.raw.txt -text
-texts/faust1.tagged.txt -text
-texts/generate-manifest.sh -text
-texts/karabakh1.handtagged.darkgaia1.txt -text
-texts/karabakh1.handtagged.ilie.txt -text
-texts/karabakh1.raw.txt -text
-texts/karabakh1.tagged.txt -text
-texts/kazakhstan1.handtagged.andrei.txt -text
-texts/kazakhstan1.handtagged.ilie.txt -text
-texts/kazakhstan1.handtagged.kranzer.txt -text
-texts/kazakhstan1.raw.txt -text
-texts/kazakhstan1.tagged.txt -text
-texts/khazars1.handtagged.andrei.txt -text
-texts/khazars1.handtagged.kranzer.txt -text
-texts/khazars1.raw.txt -text
-texts/khazars1.tagged.txt -text
-texts/kurchatov1.raw.txt -text
-texts/kyrgyzstan1.handtagged.andrei.txt -text
-texts/kyrgyzstan1.handtagged.jxthng.txt -text
-texts/kyrgyzstan1.handtagged.kiril-kostadinov.txt -text
-texts/kyrgyzstan1.raw.txt -text
-texts/kyrgyzstan1.tagged.txt -text
-texts/monarchy1.handtagged.andrei.txt -text
-texts/monarchy1.handtagged.ivanstamboliev.txt -text
-texts/monarchy1.raw.txt -text
-texts/monarchy1.tagged.txt -text
-texts/partisans1.handtagged.andrei.txt -text
-texts/partisans1.handtagged.quicksilver.txt -text
-texts/partisans1.raw.txt -text
-texts/partisans1.tagged.txt -text
-texts/peru1.handtagged.andrei.txt -text
-texts/peru1.handtagged.darkgaia.txt -text
-texts/peru1.raw.txt -text
-texts/peru1.tagged.txt -text
-texts/pirates.handtagged.porsi98.txt -text
-texts/pirates.raw.txt -text
-texts/sahara1.handtagged.andrei.txt -text
-texts/sahara1.handtagged.darkgaia1.txt -text
-texts/sahara1.handtagged.niroulapradeep.txt -text
-texts/sahara1.raw.txt -text
-texts/sahara1.tagged.txt -text
-texts/sunday1.handtagged.alvin.txt -text
-texts/sunday1.handtagged.andrei.txt -text
-texts/sunday1.handtagged.kristian-hansen.txt -text
-texts/sunday1.handtagged.yoonglong.txt -text
-texts/sunday1.raw.txt -text
-texts/sunday1.tagged.txt -text
-texts/tin_soldier.handtagged.wenxuanteoh.txt -text
-texts/tin_soldier.raw.txt -text
-texts/tokamak1.handtagged.andrei.txt -text
-texts/tokamak1.handtagged.jorge-correia.txt -text
-texts/tokamak1.raw.txt -text
-texts/tokamak1.tagged.txt -text
-texts/transmission1.handtagged.spectie.txt -text
-texts/transmission1.raw.txt -text
-texts/transmission1.tagged.txt -text
-texts/turing1.handtagged.spectie.txt -text
-texts/turing1.raw.txt -text
-texts/turing1.tagged.txt -text
-texts/vietnam1.handtagged.andrei.txt -text
-texts/vietnam1.handtagged.yuang-siang-ng.txt -text
-texts/vietnam1.raw.txt -text
-texts/vietnam1.tagged.txt -text
+*.prob binary
 *.dix linguist-language=XML linguist-detectable=true
+*.metadix linguist-language=XML linguist-detectable=true
 *.lrx linguist-language=XML linguist-detectable=true
 *.lsx linguist-language=XML linguist-detectable=true
 *.tsx linguist-language=XML linguist-detectable=true

.gitignore

@@ -18,3 +18,10 @@
 /.deps
 /test/*-output.txt
 /dev/testvoc/duplicates.*.txt
+/tagger-data/eng.tagged
+/tagger-data/eng.tagged.txt
+/tagger-data/eng.untagged
+/tagger-data/eng.dic
+/tagger-data/eng.dic.expanded
+/tagger-data/eng.crp
+/tagger-data/eng.crp.txt

apertium-eng.eng.dix

@@ -111,7 +111,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <sdef n="obj" 	c="Object"/>
     <sdef n="subj" 	c="Subject"/>
     <sdef n="pers" 	c="Personal (pronoun)"/>
-    <sdef n="file" 	c="Filename"/>
   </sdefs>
   <pardefs>
 <pardef n="guionet">
@@ -251,10 +250,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 </pardef>
 
-<pardef n="filenames">
-  <e>       <re>[a-z\-_0-9]+.[a-z0-9]+</re><p><l></l><r><s n="n"/><s n="file"/></r></p></e>
-</pardef>
-
 <pardef n="Aa">
   <e>       <i>A</i></e>
   <e r="LR"><p><l>a</l>         <r>A</r></p></e>
@@ -8319,11 +8314,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 <e lm="northbound">      <i>northbound</i><par n="expensive__adj"/></e>
 <e lm="northeasterly">   <i>northeasterly</i><par n="expensive__adj"/></e>
 <e lm="northeastern">    <i>northeastern</i><par n="expensive__adj"/></e>
+<e r="LR" lm="north-eastern"><p><l>north-eastern</l><r>northeastern</r></p><par n="expensive__adj"/></e>
 <e lm="northerly">       <i>northerly</i><par n="expensive__adj"/></e>
 <e lm="northern">        <i>northern</i><par n="expensive__adj"/></e>
 <e lm="northernmost">    <i>northernmost</i><par n="expensive__adj"/></e>
 <e lm="northwesterly">   <i>northwesterly</i><par n="expensive__adj"/></e>
 <e lm="northwestern">    <i>northwestern</i><par n="expensive__adj"/></e>
+<e r="LR" lm="north-western"><p><l>north-western</l><r>northwestern</r></p><par n="expensive__adj"/></e>
 <e lm="nostalgic">       <i>nostalgic</i><par n="expensive__adj"/></e>
 <e lm="nosy">            <i>nosy</i><par n="expensive__adj"/></e>
 <e lm="not-for-profit">  <i>not-for-profit</i><par n="expensive__adj"/></e>
@@ -16390,6 +16387,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 <e lm="4-3-3 system">    <i>4-3-3<b/>system</i><par n="house__n"/></e>
 <e lm="4-4-2 system">    <i>4-4-2<b/>system</i><par n="house__n"/></e>
 <e lm="(x)th and (y)th centuries (AD|BC)"><par n="ordinals-century"/><i><b/>and<b/></i><par n="ordinals-century"/><i><b/>centuries</i><par n="BC"/><i></i><par n="personnel__n"/></e>
+<e lm="decades">         <re>[12][0-9][0-9]0s</re><p><l></l><r><s n="n"/><s n="pl"/></r></p></e>
 
 
     <!-- SECTION: Nouns -->
@@ -69352,6 +69350,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 <e lm="suck">            <i>suck</i><par n="accept__vblex"/></e>
 <e lm="suckle">          <i>suckl</i><par n="liv/e__vblex"/></e>
 <e lm="suffer">          <i>suffer</i><par n="accept__vblex"/></e>
+<e lm="suffer from">     <i>suffer</i><par n="accept__vblex"/><p><l><b/>from</l><r><g><b/>from</g></r></p></e>
 <e lm="suffer together"> <i>suffer</i><par n="accept__vblex"/><p><l><b/>together</l><r><g><b/>together</g></r></p></e>
 <e lm="suffice">         <i>suffic</i><par n="liv/e__vblex"/></e>
 <e lm="suffix">          <i>suffix</i><par n="accept__vblex"/></e>
@@ -69750,6 +69749,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 <e lm="use as a proverb"><i>us</i><par n="liv/e__vblex"/><p><l><b/>as<b/>a<b/>proverb</l><r><g><b/>as<b/>a<b/>proverb</g></r></p></e>
 <e lm="use bellows">     <i>us</i><par n="liv/e__vblex"/><p><l><b/>bellows</l><r><g><b/>bellows</g></r></p></e>
 <e lm="use to">          <i>us</i><par n="liv/e__vblex"/><p><l><b/>to</l><r><g><b/>to</g></r></p></e>
+<e lm="use to">          <i>us</i><par n="liv/e__vblex"/><p><l><b/></l><r><j/></r></p><i>to</i><par n="at__pr"/></e>
 <e lm="usher">           <i>usher</i><par n="accept__vblex"/></e>
 <e lm="usurp">           <i>usurp</i><par n="accept__vblex"/></e>
 <e lm="utilise">         <i>utili</i><par n="analy/se__vblex"/></e>
@@ -70110,7 +70110,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 <e>                      <par n="coma"/></e>
 <e>                      <par n="cometa"/></e>
 <e>                      <par n="emails"/></e>
-<e>                      <par n="filenames"/></e>
 <e>                      <par n="guionet"/></e>
 <e>                      <par n="lainausmerkit"/></e>
 <e>                      <par n="numeros"/></e>

eng.hmm.prob

@@ -25,6 +25,9 @@
       <program name="cg-proc -w">
         <file name="eng.rlx.bin"/>
       </program>
+      <program name="apertium-tagger -g">
+        <file name="eng.prob"/>
+      </program>
     </pipeline>
   </mode>
 

eng.perceptron.prob

@@ -0,0 +1,25 @@
+# Corpora for tagger training
+
+This folder contains the corpora required to train the statistical tagger. They are grouped in two folders, `tagged` and `crp`.
+
+## tagged
+
+Corpora for [supervised training](https://wiki.apertium.org/wiki/Supervised_tagger_training). Files must have the extension `.tagged` and each line must contain a disambiguated Apertium lexical unit, in [Apertium stream format](https://wiki.apertium.org/wiki/Apertium_stream_format). For example:
+
+```
+^Stars/star<n><pl>$
+^shine/shine<vblex><pres>$
+^brightly/brightly<adv>$
+^in/in<pr>$
+^the/the<det><def><sp>$
+^sky/sky<n><sg>$
+^./.<sent>$ 
+```
+
+Before training, files from the folder are merged into a file named `eng.tagged`.
+
+## crp
+
+Corpora for [unsupervised training](https://wiki.apertium.org/wiki/Unsupervised_tagger_training). Files must have the extension `.txt` and contain **plain (raw)** text.
+
+Before training, files from the folder are merged into a file named `eng.crp.txt`.

eng.prob

@@ -0,0 +1 @@
+This is an example text for unsupervised training.