Support summary written to HDFS

python/mxboard/__init__.py

@@ -20,4 +20,4 @@
 from __future__ import absolute_import
 from .writer import SummaryWriter
 
-__version__ = '0.1.0'
+__version__ = '0.1.1'

python/mxboard/record_writer.py

@@ -18,6 +18,7 @@
 """Writer for writing events to the event file."""
 
 from __future__ import absolute_import
+import six
 import struct
 from ._crc32c import crc32c
 
@@ -37,7 +38,14 @@ class RecordWriter(object):
     def __init__(self, path):
         self._writer = None
         try:
-            self._writer = open(path, 'wb')
+            parse_result = six.moves.urllib.parse.urlparse(path)
+            if parse_result.scheme == '':
+                self._writer = open(path, 'wb')
+            elif parse_result.scheme in ('hdfs', 'viewfs'):
+                import pyarrow.fs
+                # Use fs.defaultFS from core-site.xml
+                hdfs = pyarrow.fs.HadoopFileSystem(host='default', port=0)
+                self._writer = hdfs.open_output_stream(parse_result.path)
         except (OSError, IOError) as err:
             raise ValueError('failed to open file {}: {}'.format(path, str(err)))
 

python/setup.py

@@ -50,7 +50,7 @@ def compile_summary_protobuf():
 
 setup(
     name='mxboard',
-    version='0.1.0',
+    version='0.1.1',
     description='A logging tool for visualizing MXNet data in TensorBoard',
     long_description='MXBoard is a logging tool that enables visualization of MXNet data in TensorBoard.',
     author='Amazon Web Services',