Convert for test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Useful functions\n",
    "import regex\n",
    "\n",
    "def roman_number(inp: str) -> int:\n",
    "    \"\"\"\n",
    "    Source: https://stackoverflow.com/questions/19308177/converting-roman-numerals-to-integers-in-python\n",
    "    Author: https://stackoverflow.com/users/1201737/r366y\n",
    "    :param num:\n",
    "    :return:\n",
    "\n",
    "    >>> roman_number(\"XXIV\")\n",
    "    24\n",
    "    \"\"\"\n",
    "    roman_numerals = {'I': 1, 'V': 5, 'X': 10, 'L': 50, 'C': 100, 'D': 500, 'M': 1000}\n",
    "    result = 0\n",
    "    for i, c in enumerate(inp.upper()):\n",
    "        if (i+1) == len(inp) or roman_numerals[c] >= roman_numerals[inp[i+1].upper()]:\n",
    "            result += roman_numerals[c]\n",
    "        else:\n",
    "            result -= roman_numerals[c]\n",
    "    print(inp, result)\n",
    "    return result\n",
    "\n",
    "def uvji(val):\n",
    "    return val.replace(\"V\", \"U\").replace(\"v\", \"u\").replace(\"J\", \"I\").replace(\"j\", \"i\")\n",
    "\n",
    "def get_tasks(morph):\n",
    "    return dict([\n",
    "        m.split(\"=\") \n",
    "        for m in morph.split(\"|\")\n",
    "    ])\n",
    "\n",
    "# Useful constants\n",
    "\n",
    "WRONG_CLITICS = {\"quis1\"}\n",
    "DOTS_EXCEPT_APOSTROPHES = r\".?!\\\"“”\\\"«»…\\[\\]\\(\\)„“\"\n",
    "TASKS = \"form,lemma,Deg,Numb,Person,Mood_Tense_Voice,Case,Gend,Dis,pos\".split(\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_input(tsv_file):\n",
    "    clitics = []\n",
    "    files = {}\n",
    "\n",
    "    with open(tsv_file) as f:\n",
    "        header = []\n",
    "        cur_text = None\n",
    "        previous_anno = None\n",
    "        anno = None\n",
    "        for lineno, line in enumerate(f):\n",
    "            line = line.strip().split(\"\\t\")\n",
    "            if lineno == 0:\n",
    "                header = line\n",
    "                continue\n",
    "\n",
    "            previous_anno = anno\n",
    "            anno = dict(zip(header, line))\n",
    "            if anno[\"lemma\"] == \"[METADATA]\":\n",
    "                cur_text = anno[\"form\"]\n",
    "                files[anno[\"form\"]] = []\n",
    "                continue\n",
    "\n",
    "\n",
    "            if anno[\"form\"] in DOTS_EXCEPT_APOSTROPHES:\n",
    "                if files[cur_text][-1] != {}:\n",
    "                    files[cur_text].append({})\n",
    "                continue\n",
    "\n",
    "            anno[\"Dis\"] = \"_\"\n",
    "\n",
    "            anno.update(get_tasks(anno[\"morph\"]))\n",
    "            anno[\"Mood_Tense_Voice\"] = \"|\".join([\n",
    "                anno.get(part, \"_\")\n",
    "                for part in \"Mood_Tense_Voice\".split(\"_\")\n",
    "            ]).replace(\"_|_|_\", \"_\")\n",
    "\n",
    "\n",
    "            if anno[\"lemma\"].isnumeric():\n",
    "                if int(anno[\"lemma\"]) > 3:\n",
    "                    anno[\"lemma\"] = anno[\"form\"] = \"3\"\n",
    "\n",
    "            if anno[\"lemma\"][-1].isnumeric() and len(anno[\"lemma\"]) > 1:\n",
    "                anno[\"lemma\"], anno[\"Dis\"] = anno[\"lemma\"][:-1], anno[\"lemma\"][-1]\n",
    "\n",
    "            if anno[\"lemma\"] == \"[ROMAN_NUMBER]\":\n",
    "                anno[\"lemma\"] = anno[\"form\"] = roman_number(anno[\"form\"])\n",
    "                if anno[\"lemma\"] > 3:\n",
    "                    anno[\"lemma\"] = anno[\"form\"] = \"3\"\n",
    "                else:\n",
    "                    anno[\"lemma\"] = anno[\"form\"] = str(anno[\"form\"])\n",
    "\n",
    "            if anno[\"lemma\"] == \"[Greek]\":\n",
    "                continue\n",
    "\n",
    "            if anno[\"POS\"] == \"OUT\":\n",
    "                print(anno)\n",
    "\n",
    "            if anno[\"POS\"] == \"PUNC\":\n",
    "                if anno[\"form\"] in \"?.;!)(][\":\n",
    "                    files[cur_text].append(None)\n",
    "                continue\n",
    "\n",
    "            if anno[\"POS\"] == \"VERaux\":\n",
    "                anno[\"POS\"] = \"VER\"\n",
    "\n",
    "            if len(files[cur_text]) and files[cur_text][-1] and files[cur_text][-1] == previous_anno \\\n",
    "                and files[cur_text][-1][\"form\"] == anno[\"form\"]:\n",
    "                if anno[\"lemma\"] not in WRONG_CLITICS:\n",
    "                    clitics.append(anno[\"lemma\"])\n",
    "                    files[cur_text][-1][\"lemma\"] = files[cur_text][-1][\"lemma\"]+\"界\"+anno[\"lemma\"]\n",
    "                    continue\n",
    "            elif len(files[cur_text]) and files[cur_text][-1] and files[cur_text][-1][\"form\"] == anno[\"form\"][1:-1]:\n",
    "                    clitics.append(anno[\"lemma\"])\n",
    "                    files[cur_text][-1][\"lemma\"] = files[cur_text][-1][\"lemma\"]+\"界\"+anno[\"lemma\"]\n",
    "                    continue\n",
    "\n",
    "            anno[\"lemma\"] = uvji(anno[\"lemma\"])\n",
    "            anno[\"form\"] = uvji(anno[\"form\"])\n",
    "\n",
    "            if \".\" in anno[\"form\"]:\n",
    "                print(anno)\n",
    "\n",
    "            if False and anno[\"POS\"].startswith(\"NOM\"):\n",
    "                anno[\"POS\"] = \"NOM\"\n",
    "            anno[\"pos\"] = anno[\"POS\"]\n",
    "\n",
    "            files[cur_text].append(anno)\n",
    "    return files, clitics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "Titles = {\n",
    "    \"urn:cts:latinLit:stoa0275.stoa022.opp-lat1:3\" : \"Tertullien, *De pallio*\",\n",
    "    \"urn:cts:latinLit:stoa0275.stoa027.opp-lat2:9-10\" : \"Tertullien, *De spectaculis*\",\n",
    "    \"urn:cts:latinLit:stoa0040.stoa003.opp-lat4:17.4\" : \"Augustin, *De civitate Dei*\",\n",
    "    \"urn:cts:latinLit:stoa0040.stoa011.opp-lat4:262.1-262.4\" : \"Augustin, *Lettre CCLXII*\",\n",
    "    \"urn:cts:latinLit:stoa0270.stoa002.opp-lat2:9-10\" : \"Sulpice Sévère, *Vita Martini*\",\n",
    "    \"urn:cts:latinLit:stoa0238.stoa002.perseus-lat2:pr.1-1.20\" : \"Prudence, *Psychomachie*\",\n",
    "    \"urn:cts:latinLit:stoa0096.stoa003.opp-lat2:1.35-1.37\" : \"Commodien, *Instructiones*\",\n",
    "    \"urn:cts:latinLit:stoa0104a.stoa010.opp-lat1:6-8\" : \"Cyprien de Carthage, *De unitate Ecclesiae*\",\n",
    "    \"urn:cts:latinLit:stoa0249a.stoa002.opp-lat1:6.53-6.60\" : \"Salvien de Marseille, *De gubernatione Dei*\",\n",
    "    \"urn:cts:latinLit:stoa0076c.stoa002.opp-lat2:8.8-8.10\" : \"Jean Cassien, *Institutiones*\",\n",
    "    \"urn:cts:latinLit:stoa0022.stoa044.opp-lat1:1-8\" : \"Ambroise de Milan, *De Tobia*\",\n",
    "    \"urn:cts:latinLit:stoa0054.stoa001a.opp-lat1:1-2\" : \"Bède le Vénérable, *De locis sanctis*\",\n",
    "    \"urn:cts:latinLit:stoa0149b.stoa001.opp-lat1:2\" : \"Hilaire de Poitiers, *Tractatus super psalmos*\",\n",
    "    \"urn:cts:latinLit:stoa0171.stoa002.opp-lat1:26-27\" : \"Lactance, *De mortibus persecutoruma*\",\n",
    "    \"urn:cts:latinLit:stoa0162.stoa024.opp-lat1:1.1.1-1.2.3\" : \"Jérôme, *Commentaire sur Jérémie*\",\n",
    "    \"urn:cts:latinLit:stoa0058.stoa023.perseus-lat1:3\" : \"Boèce, *Contra Eutychen et Nestorium*\",\n",
    "    \"urn:cts:latinLit:stoa0143.stoa001:@.30-2.31\" : \"Grégoire de Tours, *Historia Francorum*\",\n",
    "    \"urn:cts:latinLit:stoa0261.stoa002:4.3\" : \"Sidoine Apollinaire, *Lettres*\",\n",
    "    \"urn:cts:latinLit:stoa0112.stoa001:1-3\" : \"Eginhard, *Vie de Charlemagne*\",\n",
    "    # Phase 2\n",
    "    \"[REF:urn:cts:latinLit:stoa0187b.stoa002:1-4]\": \"Mamertin, *Panegyricus dicto Juliano imperatori*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0287.stoa001:1.20]\": \"Végèce, *Epitome de re militari*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0186-stoa001:3.14]\": \"Macrobe, *Saturnales*\",\n",
    "    \"[REF:urn:cts:latinLit:phi2331.phi004.perseus-lat1:25-27]\": \"*Histoire Auguste*, Marc Aurèle\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0041a.stoa005:1.18-1.28]\": \"Caelius Aurelianus, *Gynaeciorum Sorani*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0110.stoa002a:47-63]\": \"Donat, *In Bucolicis Vergilii commentarium, praefatio*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0285c.stoa001]\": \"Vacca, *Vita M. Annaei Lucani*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0116c.stoa001:1-2]\": \"Euantius, *De comoedia uel de fabula*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0107.stoa001:38-40]\": \"Darès de Phrygie, *De excidio Troiae historia*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0023.stoa001.perseus-lat2:20.3.1-20.3.12]\": \"Ammien Marcellin, *Res gestae*\",\n",
    "    \"[REF:urn:cts:latinLit:stoa0146b.stoa001:5.2]\": \"Hégésippe, *Histoires*\",\n",
    "    # Phase 3\n",
    "    \"[REF:More]\": \"Thomas More, *Utopia*\",\n",
    "    \"[REF:LegendeAndre]\": \"Jacques de Voragine, *Saint André* (Legende Dorée)\",\n",
    "    \"[REF:LegendeAntoine]\": \"Jacques de Voragine, *Saint Antoine* (Legende Dorée)\",\n",
    "    \"[REF:LegendeAlexis]\": \"Jacques de Voragine, *Saint Alexis* (Legende Dorée)\",\n",
    "    \"[REF:LegendeSylvestre]\": \"Jacques de Voragine, *Saint Sylvestre* (Legende Dorée)\",\n",
    "    \"[REF:LegendeLucie]\": \"Jacques de Voragine, *Saint Lucie* (Legende Dorée)\",\n",
    "    \"[REF:LegendeMarieMadeleine]\": \"Jacques de Voragine, *Saint Marie-Madeleine* (Legende Dorée)\",\n",
    "    \"[REF:LegendeThomas]\": \"Jacques de Voragine, *Saint Thomas* (Legende Dorée)\",\n",
    "    \"[REF:LegendeFrancois]\": \"Jacques de Voragine, *Saint François* (Legende Dorée)\",\n",
    "}\n",
    "\n",
    "Titles = {\n",
    "    key: value.replace(\"*\", \"\").replace(\",\", \" -- \").replace(\"(\", \"\").replace(\")\", \"\")\n",
    "    for key, value in Titles.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'form': 'Tib.', 'lemma': 'Tiberius', 'POS': 'NOMpro', 'morph': 'Case=Gen|Numb=Sing', 'Dis': '_', 'Case': 'Gen', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "{'form': 'M.', 'lemma': 'Marcus', 'POS': 'NOMpro', 'morph': 'Case=Nom|Numb=Sing', 'Dis': '_', 'Case': 'Nom', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "{'form': 'M.', 'lemma': 'Marcus', 'POS': 'NOMpro', 'morph': 'Case=Dat|Numb=Sing', 'Dis': '_', 'Case': 'Dat', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "XL 40\n",
      "L 50\n",
      "XX 20\n",
      "XXX 30\n",
      "III 3\n",
      "{'form': 'C.', 'lemma': 'Caius', 'POS': 'NOMpro', 'morph': 'Case=Abl|Numb=Sing', 'Dis': '_', 'Case': 'Abl', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "{'form': 'M.', 'lemma': 'Marcus', 'POS': 'NOMpro', 'morph': 'Case=Nom|Numb=Sing', 'Dis': '_', 'Case': 'Nom', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "{'form': 'M.', 'lemma': 'Marcus', 'POS': 'NOMpro', 'morph': 'Case=Acc|Numb=Sing', 'Dis': '_', 'Case': 'Acc', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "III 3\n",
      "{'form': 'C.', 'lemma': 'Caius', 'POS': 'NOMpro', 'morph': 'Case=Abl|Numb=Sing', 'Dis': '_', 'Case': 'Abl', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "II 2\n",
      "{'form': 'L.', 'lemma': 'Lucius', 'POS': 'NOMpro', 'morph': 'Case=Abl|Numb=Sing', 'Dis': '_', 'Case': 'Abl', 'Numb': 'Sing', 'Mood_Tense_Voice': '_'}\n",
      "{'form': 'Kal.', 'lemma': 'kalendae', 'POS': 'NOMcom', 'morph': 'Case=Acc|Numb=Plur', 'Dis': '_', 'Case': 'Acc', 'Numb': 'Plur', 'Mood_Tense_Voice': '_'}\n",
      "XXVI 26\n",
      "VII 7\n",
      "X 10\n",
      "XIV 14\n",
      "XI 11\n",
      "LXXXIIII 84\n",
      "VII 7\n",
      "XII 12\n",
      "LXX 70\n",
      "XV 15\n",
      "XV 15\n",
      "xxiiii 24\n",
      "{'form': 'etc', 'lemma': 'etc', 'POS': 'OUT', 'morph': 'MORPH=empty', 'Dis': '_', 'MORPH': 'empty', 'Mood_Tense_Voice': '_'}\n",
      "{'form': 'etc', 'lemma': 'etc', 'POS': 'OUT', 'morph': 'MORPH=empty', 'Dis': '_', 'MORPH': 'empty', 'Mood_Tense_Voice': '_'}\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "files, all_clits = {}, []\n",
    "for file in glob.glob(\"./raw/*.tsv\"):\n",
    "    content, clits = read_input(file)\n",
    "    files.update(content)\n",
    "    all_clits.extend(clits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[REF:urn:cts:latinLit:stoa0187b.stoa002:1-4] has 695 tokens and 33 sentences\n",
      "[REF:urn:cts:latinLit:stoa0287.stoa001:1.20] has 555 tokens and 31 sentences\n",
      "[REF:urn:cts:latinLit:stoa0186-stoa001:3.14] has 715 tokens and 44 sentences\n",
      "[REF:urn:cts:latinLit:phi2331.phi004.perseus-lat1:25-27] has 555 tokens and 44 sentences\n",
      "[REF:urn:cts:latinLit:stoa0041a.stoa005:1.18-1.28] has 646 tokens and 50 sentences\n",
      "[REF:urn:cts:latinLit:stoa0110.stoa002a:47-63] has 688 tokens and 29 sentences\n",
      "[REF:urn:cts:latinLit:stoa0285c.stoa001] has 481 tokens and 22 sentences\n",
      "[REF:urn:cts:latinLit:stoa0116c.stoa001:1-2] has 725 tokens and 26 sentences\n",
      "[REF:urn:cts:latinLit:stoa0107.stoa001:38-40] has 474 tokens and 20 sentences\n",
      "[REF:urn:cts:latinLit:stoa0023.stoa001.perseus-lat2:20.3.1-20.3.12] has 762 tokens and 26 sentences\n",
      "[REF:urn:cts:latinLit:stoa0146b.stoa001:5.2] has 377 tokens and 19 sentences\n",
      "urn:cts:latinLit:stoa0275.stoa022.opp-lat1:3 has 558 tokens and 42 sentences\n",
      "urn:cts:latinLit:stoa0275.stoa027.opp-lat2:9-10 has 621 tokens and 38 sentences\n",
      "urn:cts:latinLit:stoa0040.stoa003.opp-lat4:17.4 has 2467 tokens and 164 sentences\n",
      "urn:cts:latinLit:stoa0040.stoa011.opp-lat4:262.1-262.4 has 571 tokens and 21 sentences\n",
      "urn:cts:latinLit:stoa0270.stoa002.opp-lat2:9-10 has 503 tokens and 27 sentences\n",
      "urn:cts:latinLit:stoa0238.stoa002.perseus-lat2:pr.1-1.20 has 464 tokens and 17 sentences\n",
      "urn:cts:latinLit:stoa0096.stoa003.opp-lat2:1.35-1.37 has 411 tokens and 49 sentences\n",
      "urn:cts:latinLit:stoa0104a.stoa010.opp-lat1:6-8 has 654 tokens and 31 sentences\n",
      "urn:cts:latinLit:stoa0249a.stoa002.opp-lat1:6.53-6.60 has 527 tokens and 26 sentences\n",
      "urn:cts:latinLit:stoa0076c.stoa002.opp-lat2:8.8-8.10 has 540 tokens and 17 sentences\n",
      "urn:cts:latinLit:stoa0022.stoa044.opp-lat1:1-8 has 555 tokens and 42 sentences\n",
      "urn:cts:latinLit:stoa0054.stoa001a.opp-lat1:1-2 has 928 tokens and 43 sentences\n",
      "urn:cts:latinLit:stoa0149b.stoa001.opp-lat1:2 has 8340 tokens and 444 sentences\n",
      "urn:cts:latinLit:stoa0171.stoa002.opp-lat1:26-27 has 561 tokens and 31 sentences\n",
      "urn:cts:latinLit:stoa0162.stoa024.opp-lat1:1.1.1-1.2.3 has 502 tokens and 25 sentences\n",
      "urn:cts:latinLit:stoa0058.stoa023.perseus-lat1:3 has 625 tokens and 41 sentences\n",
      "urn:cts:latinLit:stoa0143.stoa001:@.30-2.31 has 551 tokens and 33 sentences\n",
      "urn:cts:latinLit:stoa0261.stoa002:4.3 has 726 tokens and 30 sentences\n",
      "urn:cts:latinLit:stoa0112.stoa001:1-3 has 628 tokens and 22 sentences\n",
      "[REF:More] has 9165 tokens and 450 sentences\n",
      "[REF:LegendeAndre] has 2923 tokens and 220 sentences\n",
      "[REF:LegendeAntoine] has 1239 tokens and 128 sentences\n",
      "[REF:LegendeAlexis] has 1253 tokens and 87 sentences\n",
      "[REF:LegendeSylvestre] has 1365 tokens and 98 sentences\n",
      "[REF:LegendeLucie] has 940 tokens and 95 sentences\n",
      "[REF:LegendeMarieMadeleine] has 989 tokens and 78 sentences\n",
      "[REF:LegendeThomas] has 1459 tokens and 116 sentences\n",
      "[REF:LegendeFrancois] has 653 tokens and 45 sentences\n",
      "['que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'ue', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'ne', 'ne', 'que', 'que', 'que', 'que', 'cum', 'que', 'cum', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'cum', 'cum', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'ue', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'cum', 'que', 'que', 'as', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'quis', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'ue', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'ne', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'ue', 'que', 'que', 'ne', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'ue', 'ne', 'que', 'que', 'que', 'ne', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'ne', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'cum', 'cum', 'que', 'que', 'que', 'que', 'cum', 'que', 'cum', 'que', 'que', 'cum', 'que', 'que', 'cum', 'que', 'cum', 'cum', 'que', 'cum', 'que', 'que', 'que', 'que', 'cum', 'ne', 'que', 'cum', 'que', 'ne', 'que', 'que', 'que', 'cum', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que', 'que']\n"
     ]
    }
   ],
   "source": [
    "# Write\n",
    "        \n",
    "with open(\"lasla-model-ready.tsv\", \"w\") as full:\n",
    "    full.write(\"\\t\".join(TASKS).replace(\"form\", \"token\")+\"\\n\")\n",
    "    for file in files:\n",
    "        print(f\"{file} has {len([an for an in files[file] if an])} tokens \"\n",
    "              f\"and {len([an for an in files[file] if not an])} sentences\")\n",
    "        with open(f\"lasla-model-ready/{Titles.get(file, file)}.tsv\", \"w\") as f:\n",
    "            f.write(\"\\t\".join(TASKS).replace(\"form\", \"token\")+\"\\n\")\n",
    "            for annot in files[file]:\n",
    "                if not annot:\n",
    "                    f.write(\"\\n\")\n",
    "                    full.write(\"\\n\")\n",
    "                    continue\n",
    "                f.write(\"\\t\".join([annot.get(h, \"_\") for h in TASKS])+\"\\n\")\n",
    "                full.write(\"\\t\".join([annot.get(h, \"_\") for h in TASKS])+\"\\n\")\n",
    "    full.write(\"\\n\")\n",
    "print(all_clits)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}