From 6a995e4c2baaee2e6d96ce28965bb5806dc91e19 Mon Sep 17 00:00:00 2001
From: Andy Lulham <a.lulham@gmail.com>
Date: Sat, 18 May 2024 14:20:42 +0100
Subject: [PATCH] feat: improve search filters
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

 * Show full names of languages
 * Group together weird twitter language codes
 * Show unknown language tweets by default
 * Attempt to clarify ‘Deleted?’ filter a bit
---
 output/index.md | 35 ++++++++++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/output/index.md b/output/index.md
index 09b3232b..a9c44d5e 100644
--- a/output/index.md
+++ b/output/index.md
@@ -14,7 +14,7 @@ Proposed [Twitter community notes](https://twitter.com/i/communitynotes/download
         <th>Note</th>
         <th>Reasons</th>
         <th>Tweet language</th>
-        <th>Deleted?</th>
+        <th>Tweet status</th>
         <th>Username</th>
         <th>Tweet content</th>
       </tr>
@@ -25,6 +25,15 @@ Proposed [Twitter community notes](https://twitter.com/i/communitynotes/download
 </div>
 
 <script>
+  /*
+  This might not be quite right.
+  It’s mostly BCP-47, but with some idiosyncracies.
+  E.g.:
+    * Hebrew is `iw` instead of `he`
+    * Indonesian is `in` instead of `id`
+    * Haitian Creole is included (`ht`)
+  */
+  const langLookup = {'af': 'Afrikaans', 'am': 'Amharic', 'ar': 'Arabic', 'arn': 'Mapudungun', 'as': 'Assamese', 'az': 'Azerbaijani', 'ba': 'Bashkir', 'be': 'Belarusian', 'bg': 'Bulgarian', 'bn': 'Bengali', 'bo': 'Tibetan', 'br': 'Breton', 'bs': 'Bosnian', 'ca': 'Catalan', 'co': 'Corsican', 'cs': 'Czech', 'cy': 'Welsh', 'da': 'Danish', 'de': 'German', 'dsb': 'Lower Sorbian', 'dv': 'Divehi', 'el': 'Greek', 'en': 'English', 'es': 'Spanish', 'et': 'Estonian', 'eu': 'Basque', 'fa': 'Persian', 'fi': 'Finnish', 'fil': 'Filipino', 'fo': 'Faroese', 'fr': 'French', 'fy': 'Frisian', 'ga': 'Irish', 'gd': 'Scottish Gaelic', 'gl': 'Galician', 'gsw': 'Swiss German', 'gu': 'Gujarati', 'ha': 'Hausa', 'hi': 'Hindi', 'hr': 'Croatian', 'hrv': 'Serbo-Croatian', 'hsb': 'Upper Sorbian', 'ht': 'Haitian Creole', 'hu': 'Hungarian', 'hy': 'Armenian', 'ig': 'Igbo', 'ii': 'Yi', 'in': 'Indonesian', 'is': 'Icelandic', 'it': 'Italian', 'iu': 'Inuktitut', 'iw': 'Hebrew', 'ja': 'Japanese', 'ka': 'Georgian', 'kk': 'Kazakh', 'kl': 'Greenlandic', 'km': 'Khmer', 'kn': 'Kannada', 'ko': 'Korean', 'kok': 'Konkani', 'kb': 'Kurdi', 'ky': 'Kyrgyz', 'lb': 'Luxembourgish', 'lo': 'Lao', 'lt': 'Lithuanian', 'lv': 'Latvian', 'mi': 'Maori', 'mk': 'Macedonian', 'ml': 'Malayalam', 'mn': 'Mongolian', 'moh': 'Mohawk', 'mr': 'Marathi', 'ms': 'Malay', 'mt': 'Maltese', 'my': 'Burmese', 'nb': 'Norwegian (Bokmål)', 'ne': 'Nepali', 'nl': 'Dutch', 'nn': 'Norwegian (Nynorsk)', 'no': 'Norwegian', 'oc': 'Occitan', 'or': 'Odia', 'pa': 'Punjabi', 'pl': 'Polish', 'prs': 'Dari', 'ps': 'Pashto', 'pt': 'Portuguese', 'quc': 'K\'iche', 'qu': 'Quechua', 'rm': 'Romansh', 'ro': 'Romanian', 'ru': 'Russian', 'rw': 'Kinyarwanda', 'sa': 'Sanskrit', 'sah': 'Yakut', 'se': 'Sami (Northern)', 'si': 'Sinhala', 'sk': 'Slovak', 'sl': 'Slovenian', 'sma': 'Sami (Southern)', 'smj': 'Sami (Lule)', 'smn': 'Sami (Inari)', 'sms': 'Sami (Skolt)', 'sq': 'Albanian', 'sr': 'Serbian', 'st': 'Sesotho', 'sv': 'Swedish', 'sw': 'Kiswahili', 'syc': 'Syriac', 'ta': 'Tamil', 'te': 'Telugu', 'tg': 'Tajik', 'th': 'Thai', 'tk': 'Turkmen', 'tl': 'Tagalog', 'tn': 'Tswana', 'tr': 'Turkish', 'tt': 'Tatar', 'tzm': 'Tamazight', 'ug': 'Uyghur', 'uk': 'Ukrainian', 'ur': 'Urdu', 'uz': 'Uzbek', 'vi': 'Vietnamese', 'wo': 'Wolof', 'xh': 'Xhosa', 'yo': 'Yoruba', 'zh': 'Chinese', 'zu': 'Zulu', 'art': 'X', 'qam': 'X', 'qct': 'X', 'qht': 'X', 'qme': 'X', 'qst': 'X', 'und': 'X', 'zxx': 'X'}
   let table = new DataTable('table', {
     layout: {
       top2Start: 'search',
@@ -89,14 +98,30 @@ Proposed [Twitter community notes](https://twitter.com/i/communitynotes/download
         data: 'lang',
         visible: false,
         defaultContent: '',
-        searchPanes: {
-          emptyMessage: 'Unknown (deleted tweet)'
+        render: function (data, type, row, meta) {
+          if (!data) {
+            return 'X unknown (deleted tweet)';
+          }
+          const niceName = langLookup[data];
+          if (niceName === 'X') {
+            // there are a handful of language codes that are used for
+            // esoteric twitter things, including emoji-only tweets (`art`)
+            // and hashtag-only tweets (`qht`). We lump these all together
+            return 'X special';
+          }
+          return niceName + ' (' + data + ')';
         }
       },
       {
         data: 'deleted',
         visible: false,
-        defaultContent: 0
+        defaultContent: 0,
+        render: function (data, type, row, meta) {
+          if (type === 'display') {
+            return (data === 1) ? 'Deleted' : 'Published';
+          }
+          return data;
+        }
       },
       {
         data: 'user',
@@ -120,7 +145,7 @@ Proposed [Twitter community notes](https://twitter.com/i/communitynotes/download
       preSelect: [
         {
           column: 5,
-          rows: ['en', 'zxx', 'qam', 'qme', 'und', '']
+          rows: ['English (en)', 'X special', 'X unknown (deleted tweet)']
         },
         {
           column: 6,