Support summary written to HDFS

python/mxboard/__init__.py

@@ -20,4 +20,4 @@
 from __future__ import absolute_import
 from .writer import SummaryWriter
 
-__version__ = '0.1.0'
+__version__ = '0.1.1'

python/mxboard/event_file_writer.py

@@ -130,8 +130,15 @@ def __init__(self, logdir, max_queue=10, flush_secs=120, filename_suffix=None, v
         the event file:
         """
         self._logdir = logdir
-        if not os.path.exists(self._logdir):
-            os.makedirs(self._logdir)
+        parse_result = six.moves.urllib.parse.urlparse(self._logdir)
+        if parse_result.scheme == '':
+            if not os.path.exists(self._logdir):
+                os.makedirs(self._logdir)
+        elif parse_result.scheme in ('hdfs', 'viewfs'):
+            import pyarrow.fs
+            hdfs = pyarrow.fs.HadoopFileSystem(host='default', port=0)
+            hdfs.create_dir(parse_result.path)
+
         self._event_queue = six.moves.queue.Queue(max_queue)
         self._ev_writer = EventsWriter(os.path.join(self._logdir, "events"), verbose=verbose)
         self._flush_secs = flush_secs

python/mxboard/record_writer.py

@@ -18,6 +18,7 @@
 """Writer for writing events to the event file."""
 
 from __future__ import absolute_import
+import six
 import struct
 from ._crc32c import crc32c
 
@@ -37,7 +38,14 @@ class RecordWriter(object):
     def __init__(self, path):
         self._writer = None
         try:
-            self._writer = open(path, 'wb')
+            parse_result = six.moves.urllib.parse.urlparse(path)
+            if parse_result.scheme == '':
+                self._writer = open(path, 'wb')
+            elif parse_result.scheme in ('hdfs', 'viewfs'):
+                import pyarrow.fs
+                # Use fs.defaultFS from core-site.xml
+                hdfs = pyarrow.fs.HadoopFileSystem(host='default', port=0)
+                self._writer = hdfs.open_output_stream(parse_result.path)
         except (OSError, IOError) as err:
             raise ValueError('failed to open file {}: {}'.format(path, str(err)))
 

python/setup.py

@@ -50,7 +50,7 @@ def compile_summary_protobuf():
 
 setup(
     name='mxboard',
-    version='0.1.0',
+    version='0.1.1',
     description='A logging tool for visualizing MXNet data in TensorBoard',
     long_description='MXBoard is a logging tool that enables visualization of MXNet data in TensorBoard.',
     author='Amazon Web Services',