RailScanner_01_generate_metadata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1. Inventorize the Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os # Import the os module, for the os.walk function\n",
    "import re\n",
    "from shutil import copy2\n",
    "import datetime\n",
    "import json\n",
    "\n",
    "rootDir = 'P:/ProRail'       # Set the directory you want to start from - in this case a USB drive\n",
    "destDir = 'Y:/susanb/ProRail_iRODS' # Set the iRODS directory you want to write to - used davrods to see iRODS as netork drive\n",
    "\n",
    "full_file_list = []\n",
    "\n",
    "for dirName, subdirList, fileList in os.walk(rootDir):\n",
    "    #print('Found directory: %s' % dirName)\n",
    "    for fname in fileList:\n",
    "        print('\\t%s' % fname)\n",
    "        if (\"ProRail\" in dirName):\n",
    "            full_file = dirName+\"/\"+fname\n",
    "            full_file_list.append(full_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(full_file_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2. Sort by Color, Infrared and Frame Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "color_list = []\n",
    "infra_list = []\n",
    "frame_metadata_list = []\n",
    "ext_list = []\n",
    "stub_list = []\n",
    "\n",
    "for file in full_file_list:\n",
    " \n",
    "    stub, ext = file.split(\".\")\n",
    "    #if stub not in stub_list:\n",
    "        #stub_list.append(stub)\n",
    "    #ext_list.append(ext)\n",
    "    \n",
    "    if re.compile(\"C_DEV\").search(stub):\n",
    "        color_list.append(file)\n",
    "    if re.compile(\"NIR_DEV\").search(stub):\n",
    "        infra_list.append(file)\n",
    "    if re.compile(\"txt\").search(ext):\n",
    "        if re.compile(\"timestamp\").search(stub):\n",
    "            next\n",
    "        else:\n",
    "            frame_metadata_list.append(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(frame_metadata_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(infra_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(color_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3. Expand out the Frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normally we would write out the frames but there is not enough room"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 4. Assign Event IDs and Create Collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "timestamp_list = []\n",
    "\n",
    "# This saves the timestamp for each filename, saving us work \n",
    "colorfile_stamp = {}\n",
    "infrafile_stamp = {}\n",
    "\n",
    "# create the color stamps and add to list\n",
    "for file in color_list:\n",
    "    drive, root, src = file.split(\"/\")\n",
    "    year, month, day, hour, minute, second, color, dev, stamp = src.split(\"_\")\n",
    "    d = datetime.datetime(int(year), int(month), int(day), int(hour), int(minute), int(second))\n",
    "    timestamp_list.append(d)\n",
    "    colorfile_stamp[src]=d\n",
    "\n",
    "# create the infra stamps and add to list\n",
    "for file in infra_list:\n",
    "    drive, root, src = file.split(\"/\")\n",
    "    year, month, day, hour, minute, second, color, dev, stamp = src.split(\"_\")\n",
    "    d = datetime.datetime(int(year), int(month), int(day), int(hour), int(minute), int(second))\n",
    "    timestamp_list.append(d)\n",
    "    infrafile_stamp[src]=d\n",
    "    #print(src,d)\n",
    "\n",
    "#sort\n",
    "timestamp_list.sort()\n",
    "\n",
    "# save this for collection identification\n",
    "collection_stamp = timestamp_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(timestamp_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in future we will get this out of the iRODS database\n",
    "# SEB need to work on this\n",
    "init_event = 0\n",
    "\n",
    "if not os.path.exists(destDir):\n",
    "    os.makedirs(destDir)\n",
    "\n",
    "# Create a collection for the the json files\n",
    "collectionDir = destDir+\"/\"+str(collection_stamp.strftime(\"%Y%m%d%H%M%S\"))+\"_collection_initial_event_\"+str(init_event+1)\n",
    "if not os.path.exists(collectionDir):\n",
    "    os.makedirs(collectionDir)\n",
    "    \n",
    "# This is a fixed delta for sensitivity purposes\n",
    "fixed_delta = datetime.timedelta(seconds=10)\n",
    "\n",
    "# We need to hash the timestamps with their associated events\n",
    "timestamp_event_dict = {}\n",
    "# We also need the first recorded timestamp on that event\n",
    "event_initiate = {}\n",
    "event_collection = {}\n",
    "\n",
    "# This assigns common event numbers\n",
    "event_no = init_event\n",
    "event_stamp = datetime.datetime(1900, 1, 1, 0, 0, 0) # initialise\n",
    "for n,stamp in enumerate(timestamp_list):\n",
    "    delta = stamp-event_stamp\n",
    "\n",
    "    if (delta > fixed_delta ):\n",
    "        event_stamp = stamp\n",
    "        event_no += 1\n",
    "        #print(\"New event!\", str(event_no))\n",
    "        #print(stamp)\n",
    "        newDir = destDir+\"/\"+str(stamp.strftime(\"%Y%m%d%H%M%S\"))+\"_event_\"+str(event_no)\n",
    "        event_collection[event_no]=newDir\n",
    "        if not os.path.exists(newDir):\n",
    "            os.makedirs(newDir)\n",
    "        timestamp_event_dict[stamp]=event_no\n",
    "        event_initiate[event_no]=stamp\n",
    "    else:\n",
    "        #print(stamp)\n",
    "        timestamp_event_dict[stamp]=event_no\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(timestamp_event_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(event_collection)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 5. Copy Over the Files to the Correct Place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matchstamp = re.compile('...._.._.._.._.._..')\n",
    "for file in full_file_list:\n",
    "    \n",
    "    startloc = re.search(matchstamp, file)\n",
    "    if startloc and os.stat(file).st_size > 0: # if filename has a datestamp and is not empty\n",
    "        stamp = file[startloc.span()[0]:startloc.span()[1]]\n",
    "        year, month, day, hour, minute, second = stamp.split(\"_\")\n",
    "        d = datetime.datetime(int(year), int(month), int(day), int(hour), int(minute), int(second))\n",
    "        if d in timestamp_event_dict.keys(): #ignore files if their timestamp is not in the timestamplist\n",
    "            event_id=timestamp_event_dict[d]\n",
    "            #print(file, event_id, event_collection[event_id])\n",
    "            drive, root, src = file.split(\"/\")\n",
    "            if not os.path.exists(event_collection[event_id]+\"/\"+src): #do not overwrite\n",
    "                copy2(file, event_collection[event_id])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 7. Make the Manifest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "location_schema = {'fields': [{'name': 'location_id', 'type': 'integer'}, {'name': 'location', 'type': 'geopoint'}, \\\n",
    "                              {'name': 'location_name', 'type': 'string'}], \"primaryKey\":[\"location_id\"] }\n",
    "camera_schema = {'fields': [{'name': 'camera_id', 'type': 'integer'}, {'name': 'manufacturer', 'type': 'string'}, \\\n",
    "                              {'name': 'spectrum', 'type': 'string'}], \"primaryKey\":[\"camera_schema\"] }\n",
    "settings_schema = {'fields': [{'name': 'camera_id', 'type': 'integer'}, \\\n",
    "                              {'name': 'time_stamp', 'type': 'datetime'}, \\\n",
    "                              {'name': 'settings', 'type': 'string'}, \\\n",
    "                              {'name': 'height', 'type': 'integer'}, \\\n",
    "                              {'name': 'width', 'type': 'integer'}], \\\n",
    "                               \"primaryKey\":[\"camera_id\",\"time_stamp\"]} \n",
    "\n",
    "collection_schema = {'fields': [{'name': 'collection_id', 'type': 'integer'}, \\\n",
    "                                {'name': 'location_id', 'type': 'integer'}, \\\n",
    "                                {'name': 'time_stamp', 'type': 'datetime'}, \\\n",
    "                                { 'name': 'source', 'type': 'integer'}], \\\n",
    "                                 \"primaryKey\": [\"collection_id\", \"location_id\",\"time_stamp\"] }\n",
    "event_schema = {'fields': [{'name': 'event_id', 'type': 'integer'}, {'name': 'collection_id', 'type': 'integer'}, \\\n",
    "                           {'name': 'illumination', 'type': 'integer'}, {'name': 'precipation', 'type': 'integer'}, \\\n",
    "                              {'name': 'time_stamp', 'type': 'datetime'}], \"primaryKey\":[\"event_id\"]  }\n",
    "conversion_schema = {'fields': [{'name': 'event_id', 'type': 'integer'}, \\\n",
    "                                {'name': 'camera_id', 'type': 'integer'}, \\\n",
    "                                {'name': 'conversion_id', 'type': 'integer'}, \\\n",
    "                                {'name': 'format', 'type': 'string'}, \\\n",
    "                                {'name': 'speed', 'type': 'integer'}, \\\n",
    "                                {'name': 'description', 'type': 'string'}, \\\n",
    "                                {'name': 'file_location', 'type': 'string'},\\\n",
    "                                {'name': 'conversion_date', 'type': 'datetime'}], \\\n",
    "                                \"primaryKey\":[\"event_id\",\"camera_id\",\"conversion_id\"] }\n",
    "frame_schema = {'fields': [{'name': 'event_id', 'type': 'integer'}, {'name': 'camera_id', 'type': 'integer'}, \\\n",
    "                              {'name': 'frame_no', 'type': 'integer'}, {'name': 'capture_time', 'type': 'time'}, \\\n",
    "                              {'name': 'file_location', 'type': 'string'} ], \"primaryKey\":[\"event_id\",\"camera_id\", \\\n",
    "                              \"frame_no\"] }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "manifest={}\n",
    "manifest[\"location\"]= location_schema\n",
    "manifest[\"camera\"]= camera_schema\n",
    "manifest[\"settings\"] = settings_schema\n",
    "manifest[\"collection\"]=collection_schema\n",
    "manifest[\"event\"]= event_schema\n",
    "manifest[\"conversion\"]=conversion_schema\n",
    "manifest[\"frame\"]=frame_schema"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 8. Write the Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# location source\n",
    "location_source= [{\"location_id\": 0, \"location\": [51.8707149,4.2901064], \"location_name\": \"Botlek\"}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# camera source\n",
    "# not sure about the dimensions of the infrared frame\n",
    "camera_source= [ {\"camera_id\":0, \"manufacturer\": \"Battenburg\", \"spectrum\": \"optical\", \"height\":500, \"width\":1080},\\\n",
    "               {\"camera_id\":1, \"manufacturer\": \"Battenburg\", \"spectrum\": \"infrared\", \"height\":500, \"width\":1080}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# settings source\n",
    "# We skip over the settings file \n",
    "settings_source=[{\"camera_id\":0, \"time_stamp\":str(collection_stamp), \"settings\":None}, \\\n",
    "                 {\"camera_id\":1, \"time_stamp\":str(collection_stamp), \"settings\":None} ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# event source\n",
    "event_source = []\n",
    "\n",
    "# create the event table\n",
    "for event in event_initiate:\n",
    "    d= event_initiate[event]\n",
    "    events_record = {}\n",
    "    events_record[\"event_id\"]=event\n",
    "    events_record[\"location_id\"]=0  \n",
    "    events_record[\"time_stamp\"]=str(d)\n",
    "    events_record[\"precipation\"]=0\n",
    "    events_record[\"illumination\"]=0\n",
    "    event_source.append(events_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(event_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# collection source\n",
    "collection_source=[{\"collection_id\":0, \"location_id\":0, \"time_stamp\":str(collection_stamp), \"source\":1}]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conversion source \n",
    "for file_name in colorfile_stamp:\n",
    "    stamp = colorfile_stamp[file_name]\n",
    "    event_id=timestamp_event_dict[stamp]\n",
    "    conversion_source=[{\"event_id\":event_id, \"camera_id\":0, \"conversion_id\":0, \\\n",
    "                        \"format\":\"avi\", \"speed\":\"full\", \"file_location\":file_name}]\n",
    "    \n",
    "for file_name in infrafile_stamp:\n",
    "    stamp = infrafile_stamp[file_name]\n",
    "    event_id=timestamp_event_dict[stamp]\n",
    "    conversion_source=[{\"event_id\":event_id, \"camera_id\":1, \"conversion_id\":0, \\\n",
    "                        \"format\":\"avi\", \"speed\":\"full\", \"file_location\":file_name}]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame_source = []\n",
    "# frame schema\n",
    "for n, file in enumerate(frame_metadata_list):\n",
    "        drive, root, src = file.split('/')\n",
    "        stub, ext = src.split(\".\")\n",
    "        #print(stub)\n",
    "        assoc_file = stub+\".avi\"\n",
    "\n",
    "        if (assoc_file in colorfile_stamp):\n",
    "            stamp = colorfile_stamp[assoc_file]\n",
    "            event_no=timestamp_event_dict[stamp]\n",
    "            camera_id = 0\n",
    "        elif (assoc_file in infrafile_stamp):\n",
    "            stamp = infrafile_stamp[assoc_file]\n",
    "            event_no=timestamp_event_dict[stamp]\n",
    "            camera_id = 1\n",
    "        with open(file) as f:\n",
    "            for line in f:\n",
    "                line=line.strip()\n",
    "                frame_no, timedate, ms, mms = line.split('\\t')\n",
    "                timedate_string = timedate+\" \"+ms+mms\n",
    "                frame_dict = {'event_id':event_no,'camera_id':camera_id,\"frame_no\":frame_no,\"capture_time\":timedate_string,\"file_location\":assoc_file}\n",
    "                frame_source.append(frame_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 9. Dump to File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(collectionDir+\"/manifest.json\", 'w') as out:\n",
    "    json.dump(manifest,out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(collectionDir+\"/camera_table.json\", 'w') as out:\n",
    "    json.dump(camera_source,out)\n",
    "\n",
    "with open(collectionDir+\"/settings_table.json\", 'w') as out:\n",
    "    json.dump(settings_source,out)\n",
    "\n",
    "with open(collectionDir+\"/collection_table.json\", 'w') as out:\n",
    "    json.dump(collection_source,out)\n",
    "\n",
    "with open(collectionDir+\"/event_table.json\", 'w') as out:\n",
    "    json.dump(event_source,out)    \n",
    "\n",
    "with open(collectionDir+\"/conversion_table.json\", 'w') as out:\n",
    "    json.dump(conversion_source,out)  \n",
    "\n",
    "with open(collectionDir+\"/frame_table.json\", 'w') as out:\n",
    "    json.dump(frame_source,out)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 6. Write the Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#iRODS 'imeta' commands are generated in a text file, to be read into iRODS from the command line\n",
    "metadata_file = open(collectionDir+\"/metadata.txt\", \"w\")\n",
    "for event_id in event_collection:\n",
    "    location_id = event_source[event_id-1]['location_id']\n",
    "    metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                        \" event_id \"+str(event_id)+\"\\n\")\n",
    "    metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                        \" collection_stamp '\"+str(collection_stamp)+\"'\\n\")\n",
    "    metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                        \" precipitation \"+str(event_source[event_id-1]['precipation'])+\"\\n\")\n",
    "    metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                        \" illumination \"+str(event_source[event_id-1]['illumination'])+\"\\n\")\n",
    "    metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                        \" event_time_stamp '\"+str(event_source[event_id-1]['time_stamp'])+\"'\\n\")\n",
    "    metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                        \" location_name '\"+location_source[location_id]['location_name']+\"'\\n\")\n",
    "\n",
    "for file_name in colorfile_stamp:\n",
    "    stub, ext = file_name.split(\".\")\n",
    "    if ext == \"avi\":\n",
    "        stamp = colorfile_stamp[file_name]\n",
    "        event_id=timestamp_event_dict[stamp]\n",
    "        metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                            \" color_movie yes \"+\"\\n\")\n",
    "    \n",
    "for file_name in infrafile_stamp:\n",
    "    stub, ext = file_name.split(\".\")\n",
    "    if ext == \"avi\":\n",
    "        stamp = infrafile_stamp[file_name]\n",
    "        event_id=timestamp_event_dict[stamp]\n",
    "        metadata_file.write(\"imeta add -C \"+str(event_collection[event_id])+\\\n",
    "                            \" IR_movie yes \"+\"\\n\")\n",
    "metadata_file.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}