Theta_and_Stampede.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsl version: 0.7.2-a0\n",
      "Parsl file source: /home/ubuntu/anaconda3/envs/remote_ix_py3.6/lib/python3.6/site-packages/parsl/__init__.py\n"
     ]
    }
   ],
   "source": [
    "import parsl\n",
    "from parsl.app.app import python_app, bash_app\n",
    "import time\n",
    "import os\n",
    "\n",
    "print(\"Parsl version: {}\".format(parsl.__version__))\n",
    "print(\"Parsl file source: {}\".format(parsl.__file__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Enable only for logging. Please remove prior to demo time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parsl.set_stream_logger()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configurations\n",
    "\n",
    "Here we configure Parsl to use the Theta Supercomputer at Argonne National Lab, and Cori at NERSC.\n",
    "\n",
    "##### Authentication\n",
    "Both systems use two factor authentication, which Parsl supports via an interactive SSH channel that\n",
    "requires the user to enter the 2FA token string as per the facilities setup when prompted.\n",
    "\n",
    "##### Execution\n",
    "\n",
    "\n",
    "##### Data Transport\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/remote_ix_py3.6/lib/python3.6/site-packages/paramiko/ecdsakey.py:164: CryptographyDeprecationWarning: Support for unsafe construction of public numbers from encoded data will be removed in a future version. Please use EllipticCurvePublicKey.from_encoded_point\n",
      "  self.ecdsa_curve.curve_class(), pointinfo\n",
      "/home/ubuntu/anaconda3/envs/remote_ix_py3.6/lib/python3.6/site-packages/paramiko/kex_ecdh_nist.py:39: CryptographyDeprecationWarning: encode_point has been deprecated on EllipticCurvePublicNumbers and will be removed in a future version. Please use EllipticCurvePublicKey.public_bytes to obtain both compressed and uncompressed point encoding.\n",
      "  m.add_string(self.Q_C.public_numbers().encode_point())\n",
      "/home/ubuntu/anaconda3/envs/remote_ix_py3.6/lib/python3.6/site-packages/paramiko/kex_ecdh_nist.py:96: CryptographyDeprecationWarning: Support for unsafe construction of public numbers from encoded data will be removed in a future version. Please use EllipticCurvePublicKey.from_encoded_point\n",
      "  self.curve, Q_S_bytes\n",
      "/home/ubuntu/anaconda3/envs/remote_ix_py3.6/lib/python3.6/site-packages/paramiko/kex_ecdh_nist.py:111: CryptographyDeprecationWarning: encode_point has been deprecated on EllipticCurvePublicNumbers and will be removed in a future version. Please use EllipticCurvePublicKey.public_bytes to obtain both compressed and uncompressed point encoding.\n",
      "  hm.add_string(self.Q_C.public_numbers().encode_point())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter thetalogin6.alcf.anl.gov Logon password :········\n",
      "Enter cori03-224.nersc.gov Logon password :········\n"
     ]
    }
   ],
   "source": [
    "from parsl.config import Config\n",
    "\n",
    "from parsl.providers import SlurmProvider, CobaltProvider\n",
    "from parsl.executors import HighThroughputExecutor\n",
    "from parsl.channels import SSHInteractiveLoginChannel\n",
    "\n",
    "config = Config(\n",
    "    executors=[\n",
    "        HighThroughputExecutor(\n",
    "            label='theta',\n",
    "            max_workers=4,\n",
    "            address='try.parsl-project.org',\n",
    "            interchange_address='thetalogin6',            \n",
    "            provider=CobaltProvider(                \n",
    "                channel=SSHInteractiveLoginChannel(\n",
    "                    hostname=\"thetalogin6.alcf.anl.gov\",\n",
    "                    username=\"yadunand\",                         # MUST SET PER USER\n",
    "                    script_dir=\"/home/yadunand/parsl_scripts\"                    \n",
    "                ),\n",
    "                queue=\"debug-flat-quad\",\n",
    "                init_blocks=1,                \n",
    "                worker_init='source /home/yadunand/setup_parsl_0.7.2.sh',\n",
    "                account='CSC249ADCD01',                          # MUST SET PER USER\n",
    "                cmd_timeout=120\n",
    "            ),            \n",
    "        ),        \n",
    "        HighThroughputExecutor(\n",
    "            label=\"cori\",\n",
    "            worker_debug=False,\n",
    "            address='try.parsl-project.org',\n",
    "            interchange_address='cori03-224.nersc.gov',            \n",
    "            provider=SlurmProvider(\n",
    "                partition='debug',  # Replace with partition name\n",
    "                # channel=SSHChannel(\n",
    "                channel=SSHInteractiveLoginChannel(\n",
    "                    hostname='cori03-224.nersc.gov',\n",
    "                    username='yadunand',                         # MUST SET PER USER\n",
    "                    script_dir='/global/homes/y/yadunand/parsl_scripts',\n",
    "                ),                \n",
    "                init_blocks=1,                \n",
    "                scheduler_options=\"#SBATCH --constraint=knl,quad,cache\",\n",
    "                # scheduler_options=\"#SBATCH --constraint=haswell\",\n",
    "                worker_init='source ~/setup_parsl_0.7.2.sh',                \n",
    "            ),\n",
    "        )\n",
    "    ],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<parsl.dataflow.dflow.DataFlowKernel at 0x7f3288b69780>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parsl.load(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@python_app(executors=['theta'])\n",
    "def platform_theta(sleep=10, stdout=None):\n",
    "    import platform\n",
    "    import time\n",
    "    time.sleep(sleep)\n",
    "    return platform.uname()\n",
    "\n",
    "@python_app(executors=['cori'])\n",
    "def platform_cori(sleep=10, stdout=None):\n",
    "    import platform\n",
    "    import time\n",
    "    time.sleep(sleep)\n",
    "    return platform.uname()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta_p = platform_theta()\n",
    "cori_p = platform_cori()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From theta:  uname_result(system='Linux', node='nid00001', release='4.4.103-6.38_4.0.104-cray_ari_c', version='#1 SMP Thu Jan 17 19:44:04 UTC 2019 (5395680)', machine='x86_64', processor='x86_64')\n",
      "From cori:  uname_result(system='Linux', node='nid02517', release='4.4.103-6.38_4.0.153-cray_ari_c', version='#1 SMP Thu Nov 1 16:05:05 UTC 2018 (6ef8fef)', machine='x86_64', processor='x86_64')\n"
     ]
    }
   ],
   "source": [
    "# print(\"Got result: {}\".format(x.result())\n",
    "print(\"From theta: \", theta_p.result())\n",
    "print(\"From cori: \", cori_p.result())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "time.sleep(300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "parsl.dfk().cleanup()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (remote_ix_py3.6)",
   "language": "python",
   "name": "remote_ix_py3.6"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}