36 #ifndef VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX 37 #define VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX 41 #include "multi_array_chunked.hxx" 42 #include "hdf5impex.hxx" 45 #ifdef VIGRA_CHECK_BOUNDS 46 #define VIGRA_ASSERT_INSIDE(diff) \ 47 vigra_precondition(this->isInside(diff), "Index out of bounds") 49 #define VIGRA_ASSERT_INSIDE(diff) 67 template <
unsigned int N,
class T,
class Alloc = std::allocator<T> >
83 :
public ChunkBase<N, T>
88 typedef value_type * pointer;
89 typedef value_type & reference;
91 Chunk(shape_type
const &
shape, shape_type
const & start,
93 : ChunkBase<N, T>(detail::defaultStride(shape))
105 std::size_t
size()
const 110 void write(
bool deallocate =
true)
112 if(this->pointer_ != 0)
114 if(!array_->file_.isReadOnly())
116 herr_t status = array_->file_.writeBlock(array_->dataset_, start_,
118 vigra_postcondition(status >= 0,
119 "ChunkedArrayHDF5: write to dataset failed.");
123 alloc_.deallocate(this->pointer_, this->
size());
131 if(this->pointer_ == 0)
133 this->pointer_ = alloc_.allocate(this->
size());
134 herr_t status = array_->file_.readBlock(array_->dataset_, start_, shape_,
136 vigra_postcondition(status >= 0,
137 "ChunkedArrayHDF5: read from dataset failed.");
139 return this->pointer_;
142 shape_type shape_, start_;
147 Chunk & operator=(Chunk
const &);
153 typedef T value_type;
154 typedef value_type * pointer;
155 typedef value_type & reference;
185 shape_type
const &
shape,
186 shape_type
const & chunk_shape=shape_type(),
188 Alloc
const & alloc = Alloc())
191 dataset_name_(dataset),
193 compression_(options.compression_method),
224 Alloc
const & alloc = Alloc())
225 :
ChunkedArray<N, T>(shape_type(), shape_type(), options),
227 dataset_name_(dataset),
229 compression_(options.compression_method),
237 bool exists = file_.existsDataset(dataset_name_);
239 if(mode == HDF5File::Replace)
241 mode = HDF5File::New;
243 else if(mode == HDF5File::Default)
246 mode = HDF5File::ReadOnly;
248 mode = HDF5File::New;
251 if(mode == HDF5File::ReadOnly)
254 vigra_precondition(!file_.isReadOnly(),
255 "ChunkedArrayHDF5(): 'mode' is incompatible with read-only file.");
257 vigra_precondition(exists || !file_.isReadOnly(),
258 "ChunkedArrayHDF5(): dataset does not exist, but file is read-only.");
260 if(!exists || mode == HDF5File::New)
279 if(compression_ == DEFAULT_COMPRESSION)
280 compression_ = ZLIB_FAST;
281 vigra_precondition(compression_ != LZ4,
282 "ChunkedArrayHDF5(): HDF5 does not support LZ4 compression.");
284 vigra_precondition(this->
size() > 0,
285 "ChunkedArrayHDF5(): invalid shape.");
286 typename detail::HDF5TypeTraits<T>::value_type init(this->fill_scalar_);
287 dataset_ = file_.createDataset<N, T>(dataset_name_,
295 dataset_ = file_.getDatasetHandleShared(dataset_name_);
299 typedef detail::HDF5TypeTraits<T> TypeTraits;
300 if(TypeTraits::numberOfBands() > 1)
302 vigra_precondition(fileShape.size() == N+1,
303 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
304 vigra_precondition(fileShape[0] == TypeTraits::numberOfBands(),
305 "ChunkedArrayHDF5(file, dataset): dataset has wrong number of bands.");
306 shape_type
shape(fileShape.begin()+1);
309 vigra_precondition(shape == this->shape_,
310 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
314 this->shape_ =
shape;
319 vigra_precondition(fileShape.size() == N,
320 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
321 shape_type
shape(fileShape.begin());
324 vigra_precondition(shape == this->shape_,
325 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
329 this->shape_ =
shape;
330 ChunkStorage(detail::computeChunkArrayShape(shape, this->bits_, this->mask_)).swap(this->handle_array_);
334 end = this->handle_array_.end();
337 i->chunk_state_.store(base_type::chunk_asleep);
352 void closeImpl(
bool force_destroy)
354 flushToDiskImpl(
true, force_destroy);
360 flushToDiskImpl(
false,
false);
363 void flushToDiskImpl(
bool destroy,
bool force_destroy)
365 if(file_.isReadOnly())
368 threading::lock_guard<threading::mutex> guard(*this->chunk_lock_);
370 end = this->handle_array_.end();
371 if(destroy && !force_destroy)
375 vigra_precondition(i->chunk_state_.load() <= 0,
376 "ChunkedArrayHDF5::close(): cannot close file because there are active chunks.");
378 i = this->handle_array_.begin();
382 Chunk * chunk =
static_cast<Chunk*
>(i->pointer_);
398 virtual bool isReadOnly()
const 400 return file_.isReadOnly();
403 virtual pointer loadChunk(ChunkBase<N, T> ** p, shape_type
const & index)
405 vigra_precondition(file_.isOpen(),
406 "ChunkedArrayHDF5::loadChunk(): file was already closed.");
409 *p =
new Chunk(this->
chunkShape(index), index*this->chunk_shape_,
this, alloc_);
410 this->overhead_bytes_ +=
sizeof(Chunk);
412 return static_cast<Chunk *
>(*p)->read();
415 virtual bool unloadChunk(ChunkBase<N, T> * chunk,
bool )
419 static_cast<Chunk *
>(chunk)->write();
423 virtual std::string backend()
const 425 return "ChunkedArrayHDF5<'" + file_.filename() +
"/" + dataset_name_ +
"'>";
428 virtual std::size_t
dataBytes(ChunkBase<N,T> * c)
const 430 return c->pointer_ == 0
432 :
static_cast<Chunk*
>(c)->
size()*
sizeof(T);
437 return sizeof(Chunk) +
sizeof(SharedChunkHandle<N, T>);
440 std::string fileName()
const 442 return file_.filename();
445 std::string datasetName()
const 447 return dataset_name_;
451 std::string dataset_name_;
453 CompressionMethod compression_;
461 #undef VIGRA_ASSERT_INSIDE Sequential iterator for MultiArrayView.
Definition: multi_fwd.hxx:161
Option object for ChunkedArray construction.
Definition: multi_array_chunked.hxx:1274
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode=HDF5File::ReadOnly, ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct for an already existing dataset with given 'options', using 'alloc' to manage the in-memory...
Definition: multi_array_chunked_hdf5.hxx:221
Main MultiArray class containing the memory management.
Definition: multi_array.hxx:2422
Interface and base class for chunked arrays.
Definition: multi_array_chunked.hxx:470
virtual std::size_t overheadBytesPerChunk() const
Bytes of main memory needed to manage a single chunk.
Definition: multi_array_chunked_hdf5.hxx:435
Definition: accessor.hxx:43
view_type::difference_type difference_type
Definition: multi_array.hxx:2470
shape_type const & chunkShape() const
Return the global chunk shape.
Wrapper for shared hid_t objects.
Definition: hdf5impex.hxx:410
NumericTraits< V >::Promote prod(TinyVectorBase< V, SIZE, D1, D2 > const &l)
product of the vector's elements
Definition: tinyvector.hxx:2097
Definition: multi_array_chunked_hdf5.hxx:68
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode, shape_type const &shape, shape_type const &chunk_shape=shape_type(), ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct with given 'shape', 'chunk_shape' and 'options', using 'alloc' to manage the in-memory vers...
Definition: multi_array_chunked_hdf5.hxx:183
iterator end()
Create the end iterator for scan-order iteration over the entire chunked array.
Definition: multi_array_chunked.hxx:2391
shape_type const & shape() const
Return the shape in this array.
OpenMode
Set how a file is opened.
Definition: hdf5impex.hxx:1013
Class for fixed size vectors.This class contains an array of size SIZE of the specified VALUETYPE...
Definition: accessor.hxx:940
std::size_t dataBytes() const
Bytes of main memory occupied by the array's data.
Definition: multi_array_chunked.hxx:1677
Base class for, and view to, vigra::MultiArray.
Definition: multi_array.hxx:652
MultiArrayIndex size() const
Return the number of elements in this array.
Access to HDF5 files.
Definition: hdf5impex.hxx:956