Streamreader

include/zim/blob.h

@@ -29,22 +29,25 @@
 
 namespace zim
 {
-  class Buffer;
   class Blob
   {
-      const char* _data;
-      size_type _size;
-      std::shared_ptr<const Buffer> _buffer;
+    public: // types
+      using DataPtr = std::shared_ptr<const char>;
 
-    public:
+    public: // functions
       Blob();
       Blob(const char* data, size_type size);
-      Blob(std::shared_ptr<const Buffer> buffer);
+      Blob(const DataPtr& buffer, size_type size);
 
-      operator std::string() const { return std::string(_data, _size); }
-      const char* data() const  { return _data; }
-      const char* end() const   { return _data + _size; }
+      operator std::string() const { return std::string(_data.get(), _size); }
+      const char* data() const  { return _data.get(); }
+      const char* end() const   { return _data.get() + _size; }
       size_type size() const     { return _size; }
+
+   private:
+     DataPtr _data;
+     size_type _size;
+
   };
 
   inline std::ostream& operator<< (std::ostream& out, const Blob& blob)

include/zim/fileheader.h

@@ -72,7 +72,7 @@ namespace zim
       {}
 
       void write(int out_fd) const;
-      void read(std::shared_ptr<const Buffer> buffer);
+      void read(const Buffer& buffer);
 
       // Do some sanity check, raise a ZimFileFormateError is
       // something is wrong.

src/blob.cpp

@@ -24,23 +24,33 @@
 
 namespace zim {
 
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
+};
+
+} // unnamed namespace
+
+
 Blob::Blob()
  : _data(0),
    _size(0)
 {}
 
 Blob::Blob(const char* data, size_type size)
- : _data(data),
+ : _data(DataPtr(data, NoDelete())),
    _size(size)
 {
   ASSERT(size, <, SIZE_MAX);
   ASSERT(data, <, (void*)(SIZE_MAX-size));
 }
 
-Blob::Blob(std::shared_ptr<const Buffer> buffer)
- : _data(buffer->data()),
-   _size(size_type(buffer->size())),
-   _buffer(buffer)
+Blob::Blob(const DataPtr& buffer, size_type size)
+ : _data(buffer),
+   _size(size)
 {}
 
 

src/buffer.cpp

@@ -34,111 +34,59 @@
 
 namespace zim {
 
-namespace {
-
-class SubBuffer : public Buffer {
-  public:
-    SubBuffer(const std::shared_ptr<const Buffer> src, offset_t offset, zsize_t size)
-      : Buffer(size),
-        _data(src, src->data(offset))
-    {
-      ASSERT(offset.v, <=, src->size().v);
-      ASSERT(offset.v+size.v, <=, src->size().v);
-    }
-
-  const char* dataImpl(offset_t offset) const {
-        return _data.get() + offset.v;
-    }
-
-  private:
-    const std::shared_ptr<const char> _data;
+namespace
+{
+
+struct NoDelete
+{
+  template<class T> void operator()(T*) {}
 };
 
 } // unnamed namespace
 
-std::shared_ptr<const Buffer> Buffer::sub_buffer(offset_t offset, zsize_t size) const
+const Buffer Buffer::sub_buffer(offset_t offset, zsize_t size) const
 {
-  return std::make_shared<SubBuffer>(shared_from_this(), offset, size);
+  ASSERT(offset.v, <=, m_size.v);
+  ASSERT(offset.v+size.v, <=, m_size.v);
+  auto sub_data = DataPtr(m_data, data(offset));
+  return Buffer(sub_data, size);
 }
 
-////////////////////////////////////////////////////////////////////////////////
-// MemoryViewBuffer
-////////////////////////////////////////////////////////////////////////////////
-
-MemoryViewBuffer::MemoryViewBuffer(const char* buffer, zsize_t size)
-  : Buffer(size)
-  , _data(buffer)
-{}
-
-const char*
-MemoryViewBuffer::dataImpl(offset_t offset) const {
-    return _data + offset.v;
+const Buffer Buffer::makeBuffer(const DataPtr& data, zsize_t size)
+{
+  return Buffer(data, size);
 }
 
-////////////////////////////////////////////////////////////////////////////////
-// MemoryBuffer
-////////////////////////////////////////////////////////////////////////////////
-
-MemoryBuffer::MemoryBuffer(zsize_t size)
-  : Buffer(size)
-  , _data(new char[size.v])
-{}
-
-MemoryBuffer::MemoryBuffer(std::unique_ptr<char[]> buffer, zsize_t size)
-  : Buffer(size)
-  , _data(std::move(buffer))
-{}
-
-const char*
-MemoryBuffer::dataImpl(offset_t offset) const {
-    return _data.get() + offset.v;
+const Buffer Buffer::makeBuffer(const char* data, zsize_t size)
+{
+  return Buffer(DataPtr(data, NoDelete()), size);
 }
 
-////////////////////////////////////////////////////////////////////////////////
-// MMapBuffer
-////////////////////////////////////////////////////////////////////////////////
-
-#ifdef ENABLE_USE_MMAP
-MMapBuffer::MMapBuffer(int fd, offset_t offset, zsize_t size):
-  Buffer(size),
-  _offset(0)
+Buffer Buffer::makeBuffer(zsize_t size)
 {
-  offset_t pa_offset(offset.v & ~(sysconf(_SC_PAGE_SIZE) - 1));
-  _offset = offset-pa_offset;
-#if defined(__APPLE__) || defined(__OpenBSD__)
-  #define MAP_FLAGS MAP_PRIVATE
-#elif defined(__FreeBSD__)
-  #define MAP_FLAGS MAP_PRIVATE|MAP_PREFAULT_READ
-#else
-  #define MAP_FLAGS MAP_PRIVATE|MAP_POPULATE
-#endif
-#if !MMAP_SUPPORT_64
-  if(pa_offset.v >= INT32_MAX) {
-    throw MMapException();
-  }
-#endif
-  _data = (char*)mmap(NULL, size.v + _offset.v, PROT_READ, MAP_FLAGS, fd, pa_offset.v);
-  if (_data == MAP_FAILED )
-  {
-    std::ostringstream s;
-    s << "Cannot mmap size " << size.v << " at off " << offset.v << " : " << strerror(errno);
-    throw std::runtime_error(s.str());
-  }
-#undef MAP_FLAGS
+  return Buffer(DataPtr(new char[size.v], std::default_delete<char[]>()), size);
 }
 
-MMapBuffer::~MMapBuffer()
+Buffer::Buffer(const DataPtr& data, zsize_t size)
+  : m_size(size),
+    m_data(data)
 {
-  munmap(_data, size_.v + _offset.v);
+  ASSERT(m_size.v, <, SIZE_MAX);
 }
 
 const char*
-MMapBuffer::dataImpl(offset_t offset) const
-{
-  offset += _offset;
-  return _data + offset.v;
+Buffer::data(offset_t offset) const {
+  ASSERT(offset.v, <=, m_size.v);
+  return m_data.get() + offset.v;
+}
+
+char*
+Buffer::data(offset_t offset)  {
+  ASSERT(offset.v, <=, m_size.v);
+  // We know we can do this cast as the only way to get a non const Buffer is
+  // to use the factory allocating the memory for us.
+  return const_cast<char*>(m_data.get() + offset.v);
 }
 
-#endif // ENABLE_USE_MMAP
 
 } //zim

src/buffer.h

@@ -29,89 +29,45 @@
 #include "zim_types.h"
 #include "endian_tools.h"
 #include "debug.h"
+#include <zim/blob.h>
 
 namespace zim {
 
-class Buffer : public std::enable_shared_from_this<Buffer> {
-  public:
-    explicit Buffer(zsize_t size)
-      : size_(size)
-    {
-      ASSERT(size_.v, <, SIZE_MAX);
-    };
+class Buffer {
+  public: // types
+    typedef std::shared_ptr<const char> DataPtr;
 
-    Buffer(const Buffer& ) = delete;
-    void operator=(const Buffer& ) = delete;
+  public: // functions
+    static const Buffer makeBuffer(const char* data, zsize_t size);
+    static const Buffer makeBuffer(const DataPtr& data, zsize_t size);
+    static Buffer makeBuffer(zsize_t size);
 
-    virtual ~Buffer() {};
-    const char* data(offset_t offset=offset_t(0)) const {
-      ASSERT(offset.v, <=, size_.v);
-      return dataImpl(offset);
-    }
+    const char* data(offset_t offset=offset_t(0)) const;
+    char* data(offset_t offset=offset_t(0));
 
     char at(offset_t offset) const {
-        return *(data(offset));
+      return *(data(offset));
     }
-    zsize_t size() const { return size_; }
-    std::shared_ptr<const Buffer> sub_buffer(offset_t offset, zsize_t size) const;
+    zsize_t size() const { return m_size; }
+    const Buffer sub_buffer(offset_t offset, zsize_t size) const;
 
     template<typename T>
     T as(offset_t offset) const {
-      ASSERT(offset.v, <, size_.v);
-      ASSERT(offset.v+sizeof(T), <=, size_.v);
+      ASSERT(offset.v, <, m_size.v);
+      ASSERT(offset.v+sizeof(T), <=, m_size.v);
       return fromLittleEndian<T>(data(offset));
     }
 
-  protected:
-    virtual const char* dataImpl(offset_t offset) const = 0;
-
-  protected:
-    const zsize_t size_;
-};
-
-
-class MemoryViewBuffer : public Buffer {
-  public:
-    MemoryViewBuffer(const char* buffer, zsize_t size);
+    operator Blob() const { return Blob(m_data, m_size.v); }
 
-  protected:
-    const char* dataImpl(offset_t offset) const;
+  private: // functions
+    Buffer(const DataPtr& data, zsize_t size);
 
-  protected:
-    const char* const _data;
+  private: // data
+    zsize_t m_size;
+    DataPtr m_data;
 };
 
-class MemoryBuffer : public Buffer {
-  public:
-    explicit MemoryBuffer(zsize_t size);
-    MemoryBuffer(std::unique_ptr<char[]> buffer, zsize_t size);
-
-    char* buf() { return _data.get(); }
-
-  protected:
-    const char* dataImpl(offset_t offset) const;
-
-  private:
-    const std::unique_ptr<char[]> _data;
-};
-
-
-#ifdef ENABLE_USE_MMAP
-class MMapException : std::exception {};
-
-class MMapBuffer : public Buffer {
-  public:
-    MMapBuffer(int fd, offset_t offset, zsize_t size);
-    ~MMapBuffer();
-
-    const char* dataImpl(offset_t offset) const;
-
-  private:
-    offset_t _offset;
-    char* _data;
-};
-#endif
-
-};
+} // zim namespace
 
 #endif //ZIM_BUFFER_H_

src/cluster.cpp

@@ -58,7 +58,7 @@ zsize_t read_size(const Reader* reader, bool isExtended, offset_t offset)
     return _read_size<uint32_t>(reader, offset);
 }
 
-std::shared_ptr<const Buffer>
+const Buffer
 getClusterBuffer(const Reader& zimReader, offset_t offset, CompressionType comp)
 {
   zsize_t uncompressed_size(0);
@@ -80,7 +80,8 @@ getClusterBuffer(const Reader& zimReader, offset_t offset, CompressionType comp)
     default:
       throw std::logic_error("compressions should not be something else than zimcompLzma, zimComZip or zimcompZstd.");
   }
-  return std::make_shared<MemoryBuffer>(std::move(uncompressed_data), uncompressed_size);
+  auto shared_data = std::shared_ptr<const char>(uncompressed_data.release(), std::default_delete<char[]>());
+  return Buffer::makeBuffer(shared_data, uncompressed_size);
 }
 
 std::unique_ptr<const Reader>
@@ -161,7 +162,7 @@ getClusterReader(const Reader& zimReader, offset_t offset, CompressionType* comp
     offset_t current = offset_t(sizeof(OFFSET_TYPE));
     while (--n_offset)
     {
-      OFFSET_TYPE new_offset = buffer->as<OFFSET_TYPE>(current);
+      OFFSET_TYPE new_offset = buffer.as<OFFSET_TYPE>(current);
       ASSERT(new_offset, >=, offset);
       ASSERT(new_offset, <=, reader->size().v);
 
@@ -187,7 +188,7 @@ getClusterReader(const Reader& zimReader, offset_t offset, CompressionType* comp
         return Blob();
       }
       auto buffer = reader->get_buffer(offsets[blob_index_type(n)], blobSize);
-      return Blob(buffer);
+      return buffer;
     } else {
       return Blob();
     }
@@ -206,7 +207,7 @@ getClusterReader(const Reader& zimReader, offset_t offset, CompressionType* comp
       }
       offset += offsets[blob_index_type(n)];
       auto buffer = reader->get_buffer(offset, size);
-      return Blob(buffer);
+      return buffer;
     } else {
       return Blob();
     }