diff --git a/notebooks/analysis.ipynb b/notebooks/analysis.ipynb index b377bf5..1c32d1a 100644 --- a/notebooks/analysis.ipynb +++ b/notebooks/analysis.ipynb @@ -184,6 +184,19 @@ "cell_type": "code", "execution_count": 7, "outputs": [], + "source": [ + "def calculate_diversity_score(tp_mean_sd, joins_mean_sd, jvd_mean_sd):\n", + " diversity = 1 / 3 * (tp_mean_sd[1]/tp_mean_sd[0] + joins_mean_sd[1]/joins_mean_sd[0] + jvd_mean_sd[1]/jvd_mean_sd[0])\n", + " return diversity" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 8, + "outputs": [], "source": [ "# get frequencies of each modifier obtained from the sparql queries\n", "count_pattern = re.compile('count|COUNT')\n", @@ -224,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -242,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "outputs": [ { "name": "stdout", @@ -262,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "outputs": [ { "name": "stdout", @@ -282,7 +295,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "outputs": [ { "name": "stdout", @@ -302,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "outputs": [ { "name": "stdout", @@ -322,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "outputs": [ { "name": "stdout", @@ -342,7 +355,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "outputs": [ { "name": "stdout", @@ -362,10 +375,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "outputs": [], "source": [ - "qald10_wd_test = json.load(open(\"./data/qald_10/qald_10.json\"))\n", + "qald10_wd_test = json.load(open(\"../data/qald_10/qald_10.json\"))\n", "qald10_wd_test_sparqls = [question[\"query\"][\"sparql\"] for question in qald10_wd_test[\"questions\"]]" ], "metadata": { @@ -374,7 +387,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "outputs": [ { "name": "stdout", @@ -405,6 +418,27 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 18, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Diversity Score of QALD-10 Test set: 1.275097671454253\n" + ] + } + ], + "source": [ + "# Calculate Diversity Score\n", + "diversity_qald10_wd_test = calculate_diversity_score([mean_tp_qald10_wd_test, std_tp_qald10_wd_test], [mean_joins_qald10_wd_test, std_joins_qald10_wd_test], [mean_jvd_qald10_wd_test, std_jvd_qald10_wd_test])\n", + "print(\"Diversity Score of QALD-10 Test set: \", diversity_qald10_wd_test)" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "metadata": { @@ -416,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -434,7 +468,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "outputs": [ { "name": "stdout", @@ -454,7 +488,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "outputs": [ { "name": "stdout", @@ -474,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 22, "outputs": [ { "name": "stdout", @@ -494,7 +528,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 23, "outputs": [ { "name": "stdout", @@ -514,7 +548,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 24, "outputs": [ { "name": "stdout", @@ -534,7 +568,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "outputs": [ { "name": "stdout", @@ -554,7 +588,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "outputs": [], "source": [ "qald9_dbpedia_train = json.load(open(\"/Users/longquan/Documents/OSS/QALD_9_plus/data/qald_9_plus_train_dbpedia.json\"))\n", @@ -566,7 +600,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "outputs": [ { "name": "stdout", @@ -597,6 +631,27 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 28, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Diversity Score of QALD-9-Plus DBpedia Train set: 1.010026202120737\n" + ] + } + ], + "source": [ + "# Calculate Diversity Score\n", + "diversity_qald9_dbpedia_train = calculate_diversity_score([mean_tp_qald9_dbpedia_train, std_tp_qald9_dbpedia_train], [mean_joins_qald9_dbpedia_train, std_joins_qald9_dbpedia_train], [mean_jvd_qald9_dbpedia_train, std_jvd_qald9_dbpedia_train])\n", + "print(\"Diversity Score of QALD-9-Plus DBpedia Train set: \", diversity_qald9_dbpedia_train)" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "metadata": { @@ -608,7 +663,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -626,7 +681,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 30, "outputs": [ { "name": "stdout", @@ -646,7 +701,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 31, "outputs": [ { "name": "stdout", @@ -666,7 +721,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 32, "outputs": [ { "name": "stdout", @@ -686,7 +741,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 33, "outputs": [ { "name": "stdout", @@ -706,7 +761,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 34, "outputs": [ { "name": "stdout", @@ -726,7 +781,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 35, "outputs": [ { "name": "stdout", @@ -746,7 +801,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 36, "outputs": [], "source": [ "qald9_dbpedia_test = json.load(open(\"/Users/longquan/Documents/OSS/QALD_9_plus/data/qald_9_plus_test_dbpedia.json\"))\n", @@ -758,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 37, "outputs": [ { "name": "stdout", @@ -789,6 +844,27 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 38, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Diversity Score of QALD-9-Plus DBpedia Test set: 0.9439482740620054\n" + ] + } + ], + "source": [ + "# Calculate Diversity Score\n", + "diversity_qald9_dbpedia_test = calculate_diversity_score([mean_tp_qald9_dbpedia_test, std_tp_qald9_dbpedia_test], [mean_joins_qald9_dbpedia_test, std_joins_qald9_dbpedia_test], [mean_jvd_qald9_dbpedia_test, std_jvd_qald9_dbpedia_test])\n", + "print(\"Diversity Score of QALD-9-Plus DBpedia Test set: \", diversity_qald9_dbpedia_test)" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "metadata": { @@ -800,7 +876,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -818,7 +894,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 40, "outputs": [ { "name": "stdout", @@ -838,7 +914,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 41, "outputs": [ { "name": "stdout", @@ -858,7 +934,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 42, "outputs": [ { "name": "stdout", @@ -878,7 +954,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 43, "outputs": [ { "name": "stdout", @@ -898,7 +974,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 44, "outputs": [ { "name": "stdout", @@ -918,7 +994,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 45, "outputs": [ { "name": "stdout", @@ -938,7 +1014,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 46, "outputs": [], "source": [ "qald9_wd_train = json.load(open(\"/Users/longquan/Documents/OSS/QALD_9_plus/data/qald_9_plus_train_wikidata.json\"))\n", @@ -950,7 +1026,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 47, "outputs": [ { "name": "stdout", @@ -981,6 +1057,27 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 48, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Diversity Score of QALD-9-Plus Wikidata Train set: 1.178244171151099\n" + ] + } + ], + "source": [ + "# Calculate Diversity Score\n", + "diversity_qald9_wd_train = calculate_diversity_score([mean_tp_qald9_wd_train, std_tp_qald9_wd_train], [mean_joins_qald9_wd_train, std_joins_qald9_wd_train], [mean_jvd_qald9_wd_train, std_jvd_qald9_wd_train])\n", + "print(\"Diversity Score of QALD-9-Plus Wikidata Train set: \", diversity_qald9_wd_train)" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "metadata": { @@ -992,7 +1089,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -1010,7 +1107,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 50, "outputs": [ { "name": "stdout", @@ -1030,7 +1127,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 51, "outputs": [ { "name": "stdout", @@ -1050,7 +1147,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 52, "outputs": [ { "name": "stdout", @@ -1070,7 +1167,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 53, "outputs": [ { "name": "stdout", @@ -1090,7 +1187,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 54, "outputs": [ { "name": "stdout", @@ -1110,7 +1207,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 55, "outputs": [ { "name": "stdout", @@ -1130,7 +1227,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 56, "outputs": [], "source": [ "qald9_wd_test = json.load(open(\"/Users/longquan/Documents/OSS/QALD_9_plus/data/qald_9_plus_test_wikidata.json\"))\n", @@ -1142,7 +1239,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 57, "outputs": [ { "name": "stdout", @@ -1173,6 +1270,27 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 58, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Diversity Score of QALD-9-Plus Wikidata Test set: 1.075475906064686\n" + ] + } + ], + "source": [ + "# Calculate Diversity Score\n", + "diversity_qald9_wd_test = calculate_diversity_score([mean_tp_qald9_wd_test, std_tp_qald9_wd_test], [mean_joins_qald9_wd_test, std_joins_qald9_wd_test], [mean_jvd_qald9_wd_test, std_jvd_qald9_wd_test])\n", + "print(\"Diversity Score of QALD-9-Plus Wikidata Test set: \", diversity_qald9_wd_test)" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "source": [ @@ -1184,7 +1302,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 59, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -1202,7 +1320,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 60, "outputs": [], "source": [ "xticks = [\"QALD10-wd-test\", \"QALD9-plus-dbpedia-train\", \"QALD9-plus-dbpedia-test\", \"QALD9-plus-wd-train\", \"QALD9-plus-wd-test\"]" @@ -1213,7 +1331,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 61, "outputs": [ { "data": { @@ -1235,7 +1353,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 62, "outputs": [ { "data": { @@ -1257,7 +1375,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 63, "outputs": [ { "data": {