02.process-pubmed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract a dataframe of PubMed articles from efetch & esummary XML results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import itertools\n",
    "import pathlib\n",
    "import pandas\n",
    "from pubmedpy.xml import iter_extract_elems\n",
    "from pubmedpy.efetch import extract_all\n",
    "from pubmedpy.esummary import parse_esummary_article_info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load incoming citation counts from esummary records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179656"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docsums = list()\n",
    "# generator of XML PubmedArticle elements\n",
    "docsum_elems = iter_extract_elems('data/pubmed/esummary/compbio-english.xml.xz', tag='DocSum')\n",
    "for elem in docsum_elems:\n",
    "    # Example esummary XML for <DocSum> at https://github.com/dhimmel/pubmedpy/blob/f554a06e13e24d661dc5ff93ad07179fb3d7f0af/pubmedpy/data/esummary.xml\n",
    "    docsums.append(parse_esummary_article_info(elem))\n",
    "len(docsums)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid  pmc_cited_by_count\n",
       "0  7477412                   2\n",
       "1  7479891                   4"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "citation_count_df = (\n",
    "    pandas.DataFrame(docsums)\n",
    "    .rename(columns={\"pubmed_id\": \"pmid\"})\n",
    "    [[\"pmid\", \"pmc_cited_by_count\"]]\n",
    ")\n",
    "citation_count_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load article information from efetch records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179656"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles = list()\n",
    "# generator of XML PubmedArticle elements\n",
    "article_elems = iter_extract_elems('data/pubmed/efetch/compbio-english.xml.xz', tag='PubmedArticle')\n",
    "for elem in article_elems:\n",
    "    # Example efetch XML for <PubmedArticle> at https://github.com/dhimmel/pubmedpy/blob/f554a06e13e24d661dc5ff93ad07179fb3d7f0af/pubmedpy/data/efetch.xml\n",
    "    articles.append(extract_all(elem))\n",
    "len(articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>doi</th>\n",
       "      <th>journal</th>\n",
       "      <th>journal_nlm_id</th>\n",
       "      <th>title</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>authors</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2211</th>\n",
       "      <td>11448876</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1093/bioinformatics/17.7.581</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>9808944</td>\n",
       "      <td>LDB2000: sequence-based integrated maps of the...</td>\n",
       "      <td>2001-07</td>\n",
       "      <td>[{'fore_name': 'X', 'last_name': 'Ke', 'affili...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9983</th>\n",
       "      <td>15266247</td>\n",
       "      <td>PMC1395763</td>\n",
       "      <td>None</td>\n",
       "      <td>MedGenMed</td>\n",
       "      <td>100894134</td>\n",
       "      <td>Conference report--genomics: the promise of SN...</td>\n",
       "      <td>2004-04-19</td>\n",
       "      <td>[{'fore_name': 'Sara M', 'last_name': 'Mariani...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15488</th>\n",
       "      <td>16094372</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1038/nature03869</td>\n",
       "      <td>Nature</td>\n",
       "      <td>0410462</td>\n",
       "      <td>Identification of JAK/STAT signalling componen...</td>\n",
       "      <td>2005-08-11</td>\n",
       "      <td>[{'fore_name': 'Patrick', 'last_name': 'Müller...</td>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19362</th>\n",
       "      <td>16615769</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1021/ac051639u</td>\n",
       "      <td>Anal Chem</td>\n",
       "      <td>0370536</td>\n",
       "      <td>Correlation and convolution analysis of peptid...</td>\n",
       "      <td>2006-04-15</td>\n",
       "      <td>[{'fore_name': 'Matthew J', 'last_name': 'Snia...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51674</th>\n",
       "      <td>20566863</td>\n",
       "      <td>PMC2901460</td>\n",
       "      <td>10.1073/pnas.1003379107</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>7505876</td>\n",
       "      <td>Genome sequences of the human body louse and i...</td>\n",
       "      <td>2010-06-21</td>\n",
       "      <td>[{'fore_name': 'Ewen F', 'last_name': 'Kirknes...</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61206</th>\n",
       "      <td>21569439</td>\n",
       "      <td>PMC3107185</td>\n",
       "      <td>10.1186/1471-2164-12-237</td>\n",
       "      <td>BMC Genomics</td>\n",
       "      <td>100965258</td>\n",
       "      <td>Genome-level homology and phylogeny of Shewane...</td>\n",
       "      <td>2011-05-12</td>\n",
       "      <td>[{'fore_name': 'Rebecca B', 'last_name': 'Diko...</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73946</th>\n",
       "      <td>22761568</td>\n",
       "      <td>PMC3386159</td>\n",
       "      <td>10.1371/journal.pcbi.1002590</td>\n",
       "      <td>PLoS Comput Biol</td>\n",
       "      <td>101238922</td>\n",
       "      <td>Learning with slight forgetting optimizes sens...</td>\n",
       "      <td>2012-06-28</td>\n",
       "      <td>[{'fore_name': 'Masaya', 'last_name': 'Hirashi...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83154</th>\n",
       "      <td>23576030</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1002/elps.201200710</td>\n",
       "      <td>Electrophoresis</td>\n",
       "      <td>8204476</td>\n",
       "      <td>Phosphoproteomics--more than meets the eye.</td>\n",
       "      <td>2013-05-14</td>\n",
       "      <td>[{'fore_name': 'Stefan', 'last_name': 'Loroch'...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89965</th>\n",
       "      <td>24187136</td>\n",
       "      <td>PMC3861660</td>\n",
       "      <td>10.1074/jbc.m113.507285</td>\n",
       "      <td>J Biol Chem</td>\n",
       "      <td>2985121R</td>\n",
       "      <td>Inhibition of mitochondrial pyruvate transport...</td>\n",
       "      <td>2013-11-01</td>\n",
       "      <td>[{'fore_name': 'Jianhai', 'last_name': 'Du', '...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92226</th>\n",
       "      <td>24405844</td>\n",
       "      <td>PMC3890554</td>\n",
       "      <td>10.1186/1471-2350-15-6</td>\n",
       "      <td>BMC Med Genet</td>\n",
       "      <td>100968552</td>\n",
       "      <td>A novel MIP gene mutation associated with auto...</td>\n",
       "      <td>2014-01-09</td>\n",
       "      <td>[{'fore_name': 'Yibo', 'last_name': 'Yu', 'aff...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93470</th>\n",
       "      <td>24524683</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1021/ja410678y</td>\n",
       "      <td>J Am Chem Soc</td>\n",
       "      <td>7503056</td>\n",
       "      <td>Mass spectrometric proteomics reveals that nuc...</td>\n",
       "      <td>2014-02-14</td>\n",
       "      <td>[{'fore_name': 'Zhifeng', 'last_name': 'Du', '...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96377</th>\n",
       "      <td>24798176</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1177/1062860614532517</td>\n",
       "      <td>Am J Med Qual</td>\n",
       "      <td>9300756</td>\n",
       "      <td>Enhanced assessment of chest pain and related ...</td>\n",
       "      <td>2014-05-05</td>\n",
       "      <td>[{'fore_name': 'Joseph A', 'last_name': 'Ladap...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100183</th>\n",
       "      <td>25146848</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1007/s12035-014-8867-6</td>\n",
       "      <td>Mol Neurobiol</td>\n",
       "      <td>8900963</td>\n",
       "      <td>Induced Pluripotent Stem Cells for Disease Mod...</td>\n",
       "      <td>2014-08-23</td>\n",
       "      <td>[{'fore_name': 'Lei', 'last_name': 'Cao', 'aff...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107414</th>\n",
       "      <td>25765008</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1016/j.artmed.2014.12.012</td>\n",
       "      <td>Artif Intell Med</td>\n",
       "      <td>8915031</td>\n",
       "      <td>Predicting protein complexes from weighted pro...</td>\n",
       "      <td>2015-02-18</td>\n",
       "      <td>[{'fore_name': 'Konstantinos', 'last_name': 'T...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119498</th>\n",
       "      <td>26797767</td>\n",
       "      <td>PMC4761715</td>\n",
       "      <td>10.18632/aging.100874</td>\n",
       "      <td>Aging (Albany NY)</td>\n",
       "      <td>101508617</td>\n",
       "      <td>Metabolomics profiling reveals novel markers f...</td>\n",
       "      <td>2016-01</td>\n",
       "      <td>[{'fore_name': 'Jonas', 'last_name': 'Zierer',...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130529</th>\n",
       "      <td>27801294</td>\n",
       "      <td>PMC5088516</td>\n",
       "      <td>10.1186/s12864-016-3064-9</td>\n",
       "      <td>BMC Genomics</td>\n",
       "      <td>100965258</td>\n",
       "      <td>Combined genomic and structural analyses of a ...</td>\n",
       "      <td>2016-10-25</td>\n",
       "      <td>[{'fore_name': 'Ana Carolina Vieira', 'last_na...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139992</th>\n",
       "      <td>28595569</td>\n",
       "      <td>PMC5465541</td>\n",
       "      <td>10.1186/s12864-017-3841-0</td>\n",
       "      <td>BMC Genomics</td>\n",
       "      <td>100965258</td>\n",
       "      <td>Genome-wide comparative transcriptome analysis...</td>\n",
       "      <td>2017-06-08</td>\n",
       "      <td>[{'fore_name': 'Jianyong', 'last_name': 'Wu', ...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140387</th>\n",
       "      <td>28627464</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1016/j.jprot.2017.06.004</td>\n",
       "      <td>J Proteomics</td>\n",
       "      <td>101475056</td>\n",
       "      <td>Analysis of protein complexes in Arabidopsis l...</td>\n",
       "      <td>2017-06-13</td>\n",
       "      <td>[{'fore_name': 'Uma K', 'last_name': 'Aryal', ...</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146917</th>\n",
       "      <td>29149287</td>\n",
       "      <td>PMC5909424</td>\n",
       "      <td>10.1093/dnares/dsx046</td>\n",
       "      <td>DNA Res</td>\n",
       "      <td>9423827</td>\n",
       "      <td>Genomic and epigenomic immunity in common bean...</td>\n",
       "      <td>2018-04-01</td>\n",
       "      <td>[{'fore_name': 'Manon M S', 'last_name': 'Rich...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160837</th>\n",
       "      <td>30311980</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1113/ep087159</td>\n",
       "      <td>Exp Physiol</td>\n",
       "      <td>9002940</td>\n",
       "      <td>Characterizing the plasma metabolome during 14...</td>\n",
       "      <td>2018-11-01</td>\n",
       "      <td>[{'fore_name': 'Nathan G', 'last_name': 'Lawle...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            pmid       pmcid                              doi  \\\n",
       "2211    11448876        None  10.1093/bioinformatics/17.7.581   \n",
       "9983    15266247  PMC1395763                             None   \n",
       "15488   16094372        None              10.1038/nature03869   \n",
       "19362   16615769        None                10.1021/ac051639u   \n",
       "51674   20566863  PMC2901460          10.1073/pnas.1003379107   \n",
       "61206   21569439  PMC3107185         10.1186/1471-2164-12-237   \n",
       "73946   22761568  PMC3386159     10.1371/journal.pcbi.1002590   \n",
       "83154   23576030        None           10.1002/elps.201200710   \n",
       "89965   24187136  PMC3861660          10.1074/jbc.m113.507285   \n",
       "92226   24405844  PMC3890554           10.1186/1471-2350-15-6   \n",
       "93470   24524683        None                10.1021/ja410678y   \n",
       "96377   24798176        None         10.1177/1062860614532517   \n",
       "100183  25146848        None        10.1007/s12035-014-8867-6   \n",
       "107414  25765008        None     10.1016/j.artmed.2014.12.012   \n",
       "119498  26797767  PMC4761715            10.18632/aging.100874   \n",
       "130529  27801294  PMC5088516        10.1186/s12864-016-3064-9   \n",
       "139992  28595569  PMC5465541        10.1186/s12864-017-3841-0   \n",
       "140387  28627464        None      10.1016/j.jprot.2017.06.004   \n",
       "146917  29149287  PMC5909424            10.1093/dnares/dsx046   \n",
       "160837  30311980        None                 10.1113/ep087159   \n",
       "\n",
       "                         journal journal_nlm_id  \\\n",
       "2211              Bioinformatics        9808944   \n",
       "9983                   MedGenMed      100894134   \n",
       "15488                     Nature        0410462   \n",
       "19362                  Anal Chem        0370536   \n",
       "51674   Proc Natl Acad Sci U S A        7505876   \n",
       "61206               BMC Genomics      100965258   \n",
       "73946           PLoS Comput Biol      101238922   \n",
       "83154            Electrophoresis        8204476   \n",
       "89965                J Biol Chem       2985121R   \n",
       "92226              BMC Med Genet      100968552   \n",
       "93470              J Am Chem Soc        7503056   \n",
       "96377              Am J Med Qual        9300756   \n",
       "100183             Mol Neurobiol        8900963   \n",
       "107414          Artif Intell Med        8915031   \n",
       "119498         Aging (Albany NY)      101508617   \n",
       "130529              BMC Genomics      100965258   \n",
       "139992              BMC Genomics      100965258   \n",
       "140387              J Proteomics      101475056   \n",
       "146917                   DNA Res        9423827   \n",
       "160837               Exp Physiol        9002940   \n",
       "\n",
       "                                                    title publication_date  \\\n",
       "2211    LDB2000: sequence-based integrated maps of the...          2001-07   \n",
       "9983    Conference report--genomics: the promise of SN...       2004-04-19   \n",
       "15488   Identification of JAK/STAT signalling componen...       2005-08-11   \n",
       "19362   Correlation and convolution analysis of peptid...       2006-04-15   \n",
       "51674   Genome sequences of the human body louse and i...       2010-06-21   \n",
       "61206   Genome-level homology and phylogeny of Shewane...       2011-05-12   \n",
       "73946   Learning with slight forgetting optimizes sens...       2012-06-28   \n",
       "83154         Phosphoproteomics--more than meets the eye.       2013-05-14   \n",
       "89965   Inhibition of mitochondrial pyruvate transport...       2013-11-01   \n",
       "92226   A novel MIP gene mutation associated with auto...       2014-01-09   \n",
       "93470   Mass spectrometric proteomics reveals that nuc...       2014-02-14   \n",
       "96377   Enhanced assessment of chest pain and related ...       2014-05-05   \n",
       "100183  Induced Pluripotent Stem Cells for Disease Mod...       2014-08-23   \n",
       "107414  Predicting protein complexes from weighted pro...       2015-02-18   \n",
       "119498  Metabolomics profiling reveals novel markers f...          2016-01   \n",
       "130529  Combined genomic and structural analyses of a ...       2016-10-25   \n",
       "139992  Genome-wide comparative transcriptome analysis...       2017-06-08   \n",
       "140387  Analysis of protein complexes in Arabidopsis l...       2017-06-13   \n",
       "146917  Genomic and epigenomic immunity in common bean...       2018-04-01   \n",
       "160837  Characterizing the plasma metabolome during 14...       2018-11-01   \n",
       "\n",
       "                                                  authors  pmc_cited_by_count  \n",
       "2211    [{'fore_name': 'X', 'last_name': 'Ke', 'affili...                   6  \n",
       "9983    [{'fore_name': 'Sara M', 'last_name': 'Mariani...                   0  \n",
       "15488   [{'fore_name': 'Patrick', 'last_name': 'Müller...                 133  \n",
       "19362   [{'fore_name': 'Matthew J', 'last_name': 'Snia...                   1  \n",
       "51674   [{'fore_name': 'Ewen F', 'last_name': 'Kirknes...                 171  \n",
       "61206   [{'fore_name': 'Rebecca B', 'last_name': 'Diko...                  16  \n",
       "73946   [{'fore_name': 'Masaya', 'last_name': 'Hirashi...                   4  \n",
       "83154   [{'fore_name': 'Stefan', 'last_name': 'Loroch'...                   8  \n",
       "89965   [{'fore_name': 'Jianhai', 'last_name': 'Du', '...                  29  \n",
       "92226   [{'fore_name': 'Yibo', 'last_name': 'Yu', 'aff...                  13  \n",
       "93470   [{'fore_name': 'Zhifeng', 'last_name': 'Du', '...                   6  \n",
       "96377   [{'fore_name': 'Joseph A', 'last_name': 'Ladap...                   0  \n",
       "100183  [{'fore_name': 'Lei', 'last_name': 'Cao', 'aff...                  10  \n",
       "107414  [{'fore_name': 'Konstantinos', 'last_name': 'T...                   5  \n",
       "119498  [{'fore_name': 'Jonas', 'last_name': 'Zierer',...                   8  \n",
       "130529  [{'fore_name': 'Ana Carolina Vieira', 'last_na...                   5  \n",
       "139992  [{'fore_name': 'Jianyong', 'last_name': 'Wu', ...                   9  \n",
       "140387  [{'fore_name': 'Uma K', 'last_name': 'Aryal', ...                  16  \n",
       "146917  [{'fore_name': 'Manon M S', 'last_name': 'Rich...                   6  \n",
       "160837  [{'fore_name': 'Nathan G', 'last_name': 'Lawle...                   1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article_df = pandas.DataFrame(articles)\n",
    "article_df.pmid = article_df.pmid.astype(int)\n",
    "article_df = article_df.merge(citation_count_df, how=\"left\")\n",
    "article_df = article_df.sort_values('pmid')\n",
    "article_df.sample(n=20, random_state=0).sort_values('pmid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>position</th>\n",
       "      <th>fore_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>reverse_position</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>1</td>\n",
       "      <td>A H</td>\n",
       "      <td>Samad</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7477412</td>\n",
       "      <td>2</td>\n",
       "      <td>W W</td>\n",
       "      <td>Cai</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7477412</td>\n",
       "      <td>3</td>\n",
       "      <td>X</td>\n",
       "      <td>Hu</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7477412</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>Irvin</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7477412</td>\n",
       "      <td>5</td>\n",
       "      <td>J</td>\n",
       "      <td>Jing</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid  position fore_name last_name  reverse_position\n",
       "0  7477412         1       A H     Samad                10\n",
       "1  7477412         2       W W       Cai                 9\n",
       "2  7477412         3         X        Hu                 8\n",
       "3  7477412         4         B     Irvin                 7\n",
       "4  7477412         5         J      Jing                 6"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_rows = list()\n",
    "affiliation_rows = list()\n",
    "for pmid, authors in zip(article_df.pmid, article_df.authors):\n",
    "    for i, author in enumerate(authors):\n",
    "        row_template = {'pmid': pmid, 'position': i + 1}\n",
    "        row = row_template.copy()\n",
    "        row.update(author)\n",
    "        row['reverse_position'] = len(authors) - i\n",
    "        for affiliation in row.pop('affiliations'):\n",
    "            affiliation_rows.append(\n",
    "                {**row_template, \"affiliation\": affiliation}\n",
    "            )\n",
    "        author_rows.append(row)\n",
    "author_df = pandas.DataFrame(author_rows)\n",
    "author_df = author_df.sort_values(['pmid', 'position'])\n",
    "author_df.head()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>position</th>\n",
       "      <th>affiliation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>1</td>\n",
       "      <td>Dept. of Pathology, Cornell Medical College, N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>1</td>\n",
       "      <td>National Center for Human Genome Research, Nat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7479895</td>\n",
       "      <td>1</td>\n",
       "      <td>National Center for Human Genome Research, Nat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7497116</td>\n",
       "      <td>1</td>\n",
       "      <td>Sandia National Labs, Albuquerque, New Mexico ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7497128</td>\n",
       "      <td>1</td>\n",
       "      <td>Sandia National Laboratories, Albuquerque, NM ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid  position                                        affiliation\n",
       "0  7477412         1  Dept. of Pathology, Cornell Medical College, N...\n",
       "1  7479891         1  National Center for Human Genome Research, Nat...\n",
       "2  7479895         1  National Center for Human Genome Research, Nat...\n",
       "3  7497116         1  Sandia National Labs, Albuquerque, New Mexico ...\n",
       "4  7497128         1  Sandia National Laboratories, Albuquerque, NM ..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "affiliation_df = pandas.DataFrame(affiliation_rows)\n",
    "affiliation_df = affiliation_df.sort_values(['pmid', 'position'])\n",
    "affiliation_df.head()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "393612"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of unique affiliations\n",
    "affiliation_df.affiliation.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write article dataframe to TSV\n",
    "(\n",
    "    article_df\n",
    "    [['pmid', 'pmcid', 'doi', 'journal', 'publication_date', 'pmc_cited_by_count', 'title']]\n",
    "    .to_csv('data/pubmed/articles.tsv.xz', sep='\\t', index=False)\n",
    ")\n",
    "# Write author dataframe to TSV\n",
    "author_df.to_csv('data/pubmed/authors.tsv.xz', sep='\\t', index=False)\n",
    "\n",
    "# Write affiliation dataframe to TSV\n",
    "affiliation_df.to_csv('data/pubmed/affiliations.tsv.xz', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PLoS One                           7755\n",
       "Methods Mol Biol                   5440\n",
       "Bioinformatics                     5073\n",
       "J Proteome Res                     4433\n",
       "BMC Genomics                       4086\n",
       "                                   ... \n",
       "Int J Drug Policy                     1\n",
       "J Sex Res                             1\n",
       "Int J Nurs Stud                       1\n",
       "United European Gastroenterol J       1\n",
       "Gerodontology                         1\n",
       "Name: journal, Length: 4300, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# PubMed articles in PMC by journal\n",
    "article_df.journal.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>journal</th>\n",
       "      <th>A A Case Rep</th>\n",
       "      <th>AACN Adv Crit Care</th>\n",
       "      <th>AACN Clin Issues</th>\n",
       "      <th>AAOHN J</th>\n",
       "      <th>AAPS J</th>\n",
       "      <th>AAPS PharmSci</th>\n",
       "      <th>ABNF J</th>\n",
       "      <th>ACS Appl Mater Interfaces</th>\n",
       "      <th>ACS Chem Biol</th>\n",
       "      <th>ACS Chem Neurosci</th>\n",
       "      <th>...</th>\n",
       "      <th>Zoo Biol</th>\n",
       "      <th>Zool Res</th>\n",
       "      <th>Zoolog Sci</th>\n",
       "      <th>Zoology (Jena)</th>\n",
       "      <th>Zoonoses Public Health</th>\n",
       "      <th>Zootaxa</th>\n",
       "      <th>Zygote</th>\n",
       "      <th>eNeuro</th>\n",
       "      <th>mBio</th>\n",
       "      <th>mSphere</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pmcid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>89</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>84</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>204</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 4300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "journal  A A Case Rep  AACN Adv Crit Care  AACN Clin Issues  AAOHN J  AAPS J  \\\n",
       "pmcid                                                                          \n",
       "False               1                   6                 1        1       5   \n",
       "True                0                   0                 1        0      24   \n",
       "\n",
       "journal  AAPS PharmSci  ABNF J  ACS Appl Mater Interfaces  ACS Chem Biol  \\\n",
       "pmcid                                                                      \n",
       "False                1       0                         11             89   \n",
       "True                 2       1                          1             84   \n",
       "\n",
       "journal  ACS Chem Neurosci  ...  Zoo Biol  Zool Res  Zoolog Sci  \\\n",
       "pmcid                       ...                                   \n",
       "False                    8  ...         2         0          15   \n",
       "True                    17  ...         0         4           0   \n",
       "\n",
       "journal  Zoology (Jena)  Zoonoses Public Health  Zootaxa  Zygote  eNeuro  \\\n",
       "pmcid                                                                      \n",
       "False                 1                       2        9       2       0   \n",
       "True                  0                       0        0       0       8   \n",
       "\n",
       "journal  mBio  mSphere  \n",
       "pmcid                   \n",
       "False       0        0  \n",
       "True      204       37  \n",
       "\n",
       "[2 rows x 4300 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# PubMed articles in PMC by journal\n",
    "pandas.crosstab(article_df.pmcid.notna(), article_df.journal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>doi</th>\n",
       "      <th>journal</th>\n",
       "      <th>journal_nlm_id</th>\n",
       "      <th>title</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>authors</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7513103</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Turk J Pediatr</td>\n",
       "      <td>0417505</td>\n",
       "      <td>Child health, the genome project and phenylket...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'C R', 'last_name': 'Scriver', ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>7764291</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Australas Biotechnol</td>\n",
       "      <td>9113681</td>\n",
       "      <td>Some ethical issues in genetic medicine.</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'N A', 'last_name': 'Tonti-Fili...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>7764293</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Australas Biotechnol</td>\n",
       "      <td>9113681</td>\n",
       "      <td>Legal rights and genetic engineering.</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'N', 'last_name': 'Stoianoff', ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>7856962</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Am Nurse</td>\n",
       "      <td>7506499</td>\n",
       "      <td>Survey assesses RN management of genetic infor...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'C', 'last_name': 'Scanlon', 'a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>7965253</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1111/j.1552-6909.1994.tb01909.x</td>\n",
       "      <td>J Obstet Gynecol Neonatal Nurs</td>\n",
       "      <td>8503123</td>\n",
       "      <td>The genome project.</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'B S', 'last_name': 'Raff', 'af...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178813</th>\n",
       "      <td>31808361</td>\n",
       "      <td>PMC6900616</td>\n",
       "      <td>10.1177/1533033819892260</td>\n",
       "      <td>Technol Cancer Res Treat</td>\n",
       "      <td>101140941</td>\n",
       "      <td>Bioinformatics Analysis of Expression and Alte...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'Yong-Zi', 'last_name': 'Chen',...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179500</th>\n",
       "      <td>31898667</td>\n",
       "      <td>None</td>\n",
       "      <td>10.4103/jcrt.jcrt_866_18</td>\n",
       "      <td>J Cancer Res Ther</td>\n",
       "      <td>101249598</td>\n",
       "      <td>Regulation of HMGA2 and KRAS genes in epitheli...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'Tuba', 'last_name': 'Gunel', '...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179614</th>\n",
       "      <td>32053766</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Discov Med</td>\n",
       "      <td>101250006</td>\n",
       "      <td>Translating cancer genomics for precision onco...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'Philipp K', 'last_name': 'Habe...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179647</th>\n",
       "      <td>32421967</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1615/critrevimmunol.2019033126</td>\n",
       "      <td>Crit Rev Immunol</td>\n",
       "      <td>8914819</td>\n",
       "      <td>Pathophysiology, Etiology, Epidemiology of Typ...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'Begum', 'last_name': 'Dariya',...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179648</th>\n",
       "      <td>32422010</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1615/critreveukaryotgeneexpr.2019030785</td>\n",
       "      <td>Crit Rev Eukaryot Gene Expr</td>\n",
       "      <td>9007261</td>\n",
       "      <td>Bioinformatics Approaches to Explore the Phylo...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'fore_name': 'Farwa', 'last_name': 'Jabbir',...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2868 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            pmid       pmcid                                         doi  \\\n",
       "7        7513103        None                                        None   \n",
       "34       7764291        None                                        None   \n",
       "35       7764293        None                                        None   \n",
       "54       7856962        None                                        None   \n",
       "62       7965253        None          10.1111/j.1552-6909.1994.tb01909.x   \n",
       "...          ...         ...                                         ...   \n",
       "178813  31808361  PMC6900616                    10.1177/1533033819892260   \n",
       "179500  31898667        None                    10.4103/jcrt.jcrt_866_18   \n",
       "179614  32053766        None                                        None   \n",
       "179647  32421967        None           10.1615/critrevimmunol.2019033126   \n",
       "179648  32422010        None  10.1615/critreveukaryotgeneexpr.2019030785   \n",
       "\n",
       "                               journal journal_nlm_id  \\\n",
       "7                       Turk J Pediatr        0417505   \n",
       "34                Australas Biotechnol        9113681   \n",
       "35                Australas Biotechnol        9113681   \n",
       "54                            Am Nurse        7506499   \n",
       "62      J Obstet Gynecol Neonatal Nurs        8503123   \n",
       "...                                ...            ...   \n",
       "178813        Technol Cancer Res Treat      101140941   \n",
       "179500               J Cancer Res Ther      101249598   \n",
       "179614                      Discov Med      101250006   \n",
       "179647                Crit Rev Immunol        8914819   \n",
       "179648     Crit Rev Eukaryot Gene Expr        9007261   \n",
       "\n",
       "                                                    title publication_date  \\\n",
       "7       Child health, the genome project and phenylket...             None   \n",
       "34               Some ethical issues in genetic medicine.             None   \n",
       "35                  Legal rights and genetic engineering.             None   \n",
       "54      Survey assesses RN management of genetic infor...             None   \n",
       "62                                    The genome project.             None   \n",
       "...                                                   ...              ...   \n",
       "178813  Bioinformatics Analysis of Expression and Alte...             None   \n",
       "179500  Regulation of HMGA2 and KRAS genes in epitheli...             None   \n",
       "179614  Translating cancer genomics for precision onco...             None   \n",
       "179647  Pathophysiology, Etiology, Epidemiology of Typ...             None   \n",
       "179648  Bioinformatics Approaches to Explore the Phylo...             None   \n",
       "\n",
       "                                                  authors  pmc_cited_by_count  \n",
       "7       [{'fore_name': 'C R', 'last_name': 'Scriver', ...                   0  \n",
       "34      [{'fore_name': 'N A', 'last_name': 'Tonti-Fili...                   0  \n",
       "35      [{'fore_name': 'N', 'last_name': 'Stoianoff', ...                   0  \n",
       "54      [{'fore_name': 'C', 'last_name': 'Scanlon', 'a...                   0  \n",
       "62      [{'fore_name': 'B S', 'last_name': 'Raff', 'af...                   0  \n",
       "...                                                   ...                 ...  \n",
       "178813  [{'fore_name': 'Yong-Zi', 'last_name': 'Chen',...                   0  \n",
       "179500  [{'fore_name': 'Tuba', 'last_name': 'Gunel', '...                   1  \n",
       "179614  [{'fore_name': 'Philipp K', 'last_name': 'Habe...                   0  \n",
       "179647  [{'fore_name': 'Begum', 'last_name': 'Dariya',...                   0  \n",
       "179648  [{'fore_name': 'Farwa', 'last_name': 'Jabbir',...                   0  \n",
       "\n",
       "[2868 rows x 9 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# articles without a publication date\n",
    "article_df[article_df.publication_date.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.0    138089\n",
       "7.0      24915\n",
       "4.0      13784\n",
       "Name: publication_date, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# article proportions by date precision\n",
    "article_df.publication_date.str.len().value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}