xrootd
XrdClZipArchive.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Michal Simon <michal.simon@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #ifndef SRC_XRDZIP_XRDZIPARCHIVE_HH_
26 #define SRC_XRDZIP_XRDZIPARCHIVE_HH_
27 
28 #include "XrdCl/XrdClFile.hh"
30 #include "XrdCl/XrdClJobManager.hh"
31 #include "XrdCl/XrdClDefaultEnv.hh"
32 #include "XrdCl/XrdClPostMaster.hh"
33 #include "XrdZip/XrdZipEOCD.hh"
34 #include "XrdZip/XrdZipCDFH.hh"
36 #include "XrdZip/XrdZipLFH.hh"
37 #include "XrdCl/XrdClZipCache.hh"
38 
39 #include <memory>
40 #include <unordered_map>
41 
42 //-----------------------------------------------------------------------------
43 // Forward declaration needed for friendship
44 //-----------------------------------------------------------------------------
45 namespace XrdEc{ class StrmWriter; class Reader; template<bool> class OpenOnlyImpl; };
46 class MicroTest;
47 
48 namespace XrdCl
49 {
50  using namespace XrdZip;
51 
52  //---------------------------------------------------------------------------
53  // ZipArchive provides following functionalities:
54  // - parsing of existing ZIP archive
55  // - reading data from existing ZIP archive
56  // - appending data to existing ZIP archive
57  // - querying stat info and checksum for given file in ZIP archive
58  //---------------------------------------------------------------------------
59  class ZipArchive
60  {
61  friend class XrdEc::StrmWriter;
62  friend class XrdEc::Reader;
63  template<bool>
64  friend class XrdEc::OpenOnlyImpl;
65  friend class ::MicroTest;
66 
67  template<typename RSP>
68  friend XRootDStatus ReadFromImpl( ZipArchive&, const std::string&, uint64_t, uint32_t, void*, ResponseHandler*, uint16_t );
69 
70  public:
71  //-----------------------------------------------------------------------
73  //-----------------------------------------------------------------------
74  ZipArchive( bool enablePlugIns = true );
75 
76  //-----------------------------------------------------------------------
78  //-----------------------------------------------------------------------
79  virtual ~ZipArchive();
80 
81  //-----------------------------------------------------------------------
89  //-----------------------------------------------------------------------
90  XRootDStatus OpenArchive( const std::string &url,
91  OpenFlags::Flags flags,
92  ResponseHandler *handler,
93  uint16_t timeout = 0 );
94 
95  //-----------------------------------------------------------------------
103  //-----------------------------------------------------------------------
104  XRootDStatus OpenFile( const std::string &fn,
106  uint64_t size = 0,
107  uint32_t crc32 = 0 );
108 
109  //-----------------------------------------------------------------------
118  //-----------------------------------------------------------------------
119  inline
120  XRootDStatus Read( uint64_t offset,
121  uint32_t size,
122  void *buffer,
123  ResponseHandler *handler,
124  uint16_t timeout = 0 )
125  {
126  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
127  return ReadFrom( openfn, offset, size, buffer, handler, timeout );
128  }
129 
130  //-----------------------------------------------------------------------
139  //-----------------------------------------------------------------------
140  inline
141  XRootDStatus PgRead( uint64_t offset,
142  uint32_t size,
143  void *buffer,
144  ResponseHandler *handler,
145  uint16_t timeout = 0 )
146  {
147  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
148  return PgReadFrom( openfn, offset, size, buffer, handler, timeout );
149  }
150 
151  //-----------------------------------------------------------------------
161  //-----------------------------------------------------------------------
162  XRootDStatus ReadFrom( const std::string &fn,
163  uint64_t offset,
164  uint32_t size,
165  void *buffer,
166  ResponseHandler *handler,
167  uint16_t timeout = 0 );
168 
169  //-----------------------------------------------------------------------
179  //-----------------------------------------------------------------------
180  XRootDStatus PgReadFrom( const std::string &fn,
181  uint64_t offset,
182  uint32_t size,
183  void *buffer,
184  ResponseHandler *handler,
185  uint16_t timeout = 0 );
186 
187  //-----------------------------------------------------------------------
195  //-----------------------------------------------------------------------
196  inline XRootDStatus Write( uint32_t size,
197  const void *buffer,
198  ResponseHandler *handler,
199  uint16_t timeout = 0 )
200  {
201  if( openstage != Done || openfn.empty() )
202  return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
203 
204  return WriteImpl( size, buffer, handler, timeout );
205  }
206 
207  //-----------------------------------------------------------------------
212  //-----------------------------------------------------------------------
213  XRootDStatus UpdateMetadata( uint32_t crc32 );
214 
215  //-----------------------------------------------------------------------
225  //-----------------------------------------------------------------------
226  XRootDStatus AppendFile( const std::string &fn,
227  uint32_t crc32,
228  uint32_t size,
229  const void *buffer,
230  ResponseHandler *handler,
231  uint16_t timeout = 0 );
232 
233  //-----------------------------------------------------------------------
239  //-----------------------------------------------------------------------
240  inline XRootDStatus Stat( const std::string &fn, StatInfo *&info )
241  { // make sure archive has been opened and CD has been parsed
242  if( openstage != Done )
243  return XRootDStatus( stError, errInvalidOp );
244  // make sure the file is part of the archive
245  auto cditr = cdmap.find( fn );
246  if( cditr == cdmap.end() )
247  return XRootDStatus( stError, errNotFound );
248  // create the result
249  info = make_stat( fn );
250  return XRootDStatus();
251  }
252 
253  //-----------------------------------------------------------------------
258  //-----------------------------------------------------------------------
259  inline XRootDStatus Stat( StatInfo *&info )
260  {
261  if( openfn.empty() )
262  return XRootDStatus( stError, errInvalidOp );
263  return Stat( openfn, info );
264  }
265 
266  //-----------------------------------------------------------------------
272  //-----------------------------------------------------------------------
273  inline XRootDStatus GetCRC32( const std::string &fn, uint32_t &cksum )
274  { // make sure archive has been opened and CD has been parsed
275  if( openstage != Done )
276  return XRootDStatus( stError, errInvalidOp );
277  // make sure the file is part of the archive
278  auto cditr = cdmap.find( fn );
279  if( cditr == cdmap.end() )
280  return XRootDStatus( stError, errNotFound );
281  cksum = cdvec[cditr->second]->ZCRC32;
282  return XRootDStatus();
283  }
284 
285  //-----------------------------------------------------------------------
287  //
291  //-----------------------------------------------------------------------
293  uint16_t timeout = 0 );
294 
295  //-----------------------------------------------------------------------
298  //-----------------------------------------------------------------------
300  {
301  if( openstage != Done || openfn.empty() )
303  0, "Archive not opened." );
304  openfn.clear();
305  lfh.reset();
306  return XRootDStatus();
307  }
308 
309  //-----------------------------------------------------------------------
312  //-----------------------------------------------------------------------
314 
315  //-----------------------------------------------------------------------
317  //-----------------------------------------------------------------------
318  inline bool IsOpen()
319  {
320  return openstage == Done;
321  }
322 
323  //------------------------------------------------------------------------
325  //------------------------------------------------------------------------
326  inline bool IsSecure()
327  {
328  return archive.IsSecure();
329  }
330 
331  //-----------------------------------------------------------------------
333  //-----------------------------------------------------------------------
334  inline bool SetProperty( const std::string &name, const std::string &value )
335  {
336  return archive.SetProperty( name, value );
337  }
338 
339  //-----------------------------------------------------------------------
341  //-----------------------------------------------------------------------
342  inline bool GetProperty( const std::string &name, std::string &value )
343  {
344  return archive.GetProperty( name, value );
345  }
346 
347  //-----------------------------------------------------------------------
349  //-----------------------------------------------------------------------
350  inline File& GetFile()
351  {
352  return archive;
353  }
354 
355  private:
356 
357  //-----------------------------------------------------------------------
366  //-----------------------------------------------------------------------
367  XRootDStatus WriteImpl( uint32_t size,
368  const void *buffer,
369  ResponseHandler *handler,
370  uint16_t timeout );
371 
372  //-----------------------------------------------------------------------
380  //-----------------------------------------------------------------------
381  XRootDStatus OpenOnly( const std::string &url,
382  bool update,
383  ResponseHandler *handler,
384  uint16_t timeout = 0 );
385 
386  //-----------------------------------------------------------------------
390  //-----------------------------------------------------------------------
392 
393  //-----------------------------------------------------------------------
397  //-----------------------------------------------------------------------
398  void SetCD( const buffer_t &buffer );
399 
400  //-----------------------------------------------------------------------
405  //-----------------------------------------------------------------------
406  template<typename Response>
407  inline static AnyObject* PkgRsp( Response *rsp )
408  {
409  if( !rsp ) return nullptr;
410  AnyObject *pkg = new AnyObject();
411  pkg->Set( rsp );
412  return pkg;
413  }
414 
415  //-----------------------------------------------------------------------
417  //-----------------------------------------------------------------------
418  template<typename Response>
419  inline static void Free( XRootDStatus *st, Response *rsp )
420  {
421  delete st;
422  delete rsp;
423  }
424 
425  //-----------------------------------------------------------------------
432  //-----------------------------------------------------------------------
433  template<typename Response>
434  inline static void Schedule( ResponseHandler *handler, XRootDStatus *st, Response *rsp = nullptr )
435  {
436  if( !handler ) return Free( st, rsp );
437  ResponseJob *job = new ResponseJob( handler, st, PkgRsp( rsp ), 0 );
439  }
440 
441  //-----------------------------------------------------------------------
447  //-----------------------------------------------------------------------
448  inline static StatInfo* make_stat( const StatInfo &starch, uint64_t size )
449  {
450  StatInfo *info = new StatInfo( starch );
451  uint32_t flags = info->GetFlags();
452  info->SetFlags( flags & ( ~StatInfo::IsWritable ) ); // make sure it is not listed as writable
453  info->SetSize( size );
454  return info;
455  }
456 
457  //-----------------------------------------------------------------------
462  //-----------------------------------------------------------------------
463  inline StatInfo* make_stat( const std::string &fn )
464  {
465  StatInfo *infoptr = 0;
466  XRootDStatus st = archive.Stat( false, infoptr );
467  std::unique_ptr<StatInfo> stinfo( infoptr );
468  auto itr = cdmap.find( fn );
469  if( itr == cdmap.end() ) return nullptr;
470  size_t index = itr->second;
471  return make_stat( *stinfo, cdvec[index]->uncompressedSize );
472  }
473 
474  //-----------------------------------------------------------------------
476  //-----------------------------------------------------------------------
477  inline static XRootDStatus* make_status( const XRootDStatus &status = XRootDStatus() )
478  {
479  return new XRootDStatus( status );
480  }
481 
482  //-----------------------------------------------------------------------
484  //-----------------------------------------------------------------------
485  inline void Clear()
486  {
487  buffer.reset();
488  eocd.reset();
489  cdvec.clear();
490  cdmap.clear();
491  zip64eocd.reset();
492  openstage = None;
493  }
494 
495  //-----------------------------------------------------------------------
497  //-----------------------------------------------------------------------
499  {
500  None = 0, //< opening/parsing not started
501  HaveEocdBlk, //< we have the End of Central Directory record
502  HaveZip64EocdlBlk, //< we have the ZIP64 End of Central Directory locator record
503  HaveZip64EocdBlk, //< we have the ZIP64 End of Central Directory record
504  HaveCdRecords, //< we have Central Directory records
505  Done, //< we are done parsing the Central Directory
506  Error, //< opening/parsing failed
507  NotParsed //< the ZIP archive has been opened but Central Directory is not parsed
508  };
509 
510  //-----------------------------------------------------------------------
512  //-----------------------------------------------------------------------
513  struct NewFile
514  {
515  NewFile( uint64_t offset, std::unique_ptr<LFH> lfh ) : offset( offset ),
516  lfh( std::move( lfh ) ),
517  overwrt( false )
518  {
519  }
520 
521  NewFile( NewFile && nf ) : offset( nf.offset ),
522  lfh( std::move( nf.lfh ) ),
523  overwrt( nf.overwrt )
524  {
525  }
526 
527  uint64_t offset; // the offset of the LFH of the file
528  std::unique_ptr<LFH> lfh; // LFH of the file
529  bool overwrt; // if true the LFH needs to be overwritten on close
530  };
531 
532  //-----------------------------------------------------------------------
534  //-----------------------------------------------------------------------
535  typedef std::unordered_map<std::string, ZipCache> zipcache_t;
536  typedef std::unordered_map<std::string, NewFile> new_files_t;
537 
538  File archive; //> File object for handling the ZIP archive
539  uint64_t archsize; //> size of the ZIP archive
540  bool cdexists; //> true if Central Directory exists, false otherwise
541  bool updated; //> true if the ZIP archive has been updated, false otherwise
542  std::unique_ptr<char[]> buffer; //> buffer for keeping the data to be parsed or raw data
543  std::unique_ptr<EOCD> eocd; //> End of Central Directory record
544  cdvec_t cdvec; //> vector of Central Directory File Headers
545  cdmap_t cdmap; //> mapping of file name to CDFH index
546  uint64_t cdoff; //> Central Directory offset
547  uint32_t orgcdsz; //> original CD size
548  uint32_t orgcdcnt; //> original number CDFH records
549  buffer_t orgcdbuf; //> buffer with the original CDFH records
550  std::unique_ptr<ZIP64_EOCD> zip64eocd; //> ZIP64 End of Central Directory record
551  OpenStages openstage; //> stage of opening / parsing a ZIP archive
552  std::string openfn; //> file name of opened file
553  zipcache_t zipcache; //> cache for inflating compressed data
554  std::unique_ptr<LFH> lfh; //> Local File Header record for the newly appended file
555  bool ckpinit; //> a flag indicating whether a checkpoint has been initialized
556  new_files_t newfiles; //> all newly appended files
557  };
558 
559 } /* namespace XrdZip */
560 
561 #endif /* SRC_XRDZIP_XRDZIPARCHIVE_HH_ */
Definition: XrdClAnyObject.hh:33
void Set(Type object, bool own=true)
Definition: XrdClAnyObject.hh:59
static PostMaster * GetPostMaster()
Get default post master.
Directory list.
Definition: XrdClXRootDResponses.hh:650
A file.
Definition: XrdClFile.hh:46
void QueueJob(Job *job, void *arg=0)
Add a job to be run.
Definition: XrdClJobManager.hh:92
JobManager * GetJobManager()
Get the job manager object user by the post master.
Handle an async response.
Definition: XrdClXRootDResponses.hh:1117
Call the user callback.
Definition: XrdClResponseJob.hh:31
Object stat info.
Definition: XrdClXRootDResponses.hh:400
@ IsWritable
Write access is allowed.
Definition: XrdClXRootDResponses.hh:414
uint32_t GetFlags() const
Get flags.
void SetSize(uint64_t size)
Set size.
void SetFlags(uint32_t flags)
Set flags.
Write operation (.
Definition: XrdClFileOperations.hh:457
Request status.
Definition: XrdClXRootDResponses.hh:219
Definition: XrdClZipArchive.hh:60
OpenStages
Stages of opening and parsing a ZIP archive.
Definition: XrdClZipArchive.hh:499
@ HaveCdRecords
Definition: XrdClZipArchive.hh:504
@ HaveZip64EocdlBlk
Definition: XrdClZipArchive.hh:502
@ Done
Definition: XrdClZipArchive.hh:505
@ HaveEocdBlk
Definition: XrdClZipArchive.hh:501
@ HaveZip64EocdBlk
Definition: XrdClZipArchive.hh:503
@ Error
Definition: XrdClZipArchive.hh:506
File & GetFile()
Get the underlying File object.
Definition: XrdClZipArchive.hh:350
XRootDStatus ReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
buffer_t orgcdbuf
Definition: XrdClZipArchive.hh:549
XRootDStatus PgRead(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:141
bool IsSecure()
Check if the underlying file is using an encrypted connection.
Definition: XrdClZipArchive.hh:326
bool cdexists
Definition: XrdClZipArchive.hh:540
static XRootDStatus * make_status(const XRootDStatus &status=XRootDStatus())
Allocate new XRootDStatus object.
Definition: XrdClZipArchive.hh:477
XRootDStatus Stat(const std::string &fn, StatInfo *&info)
Definition: XrdClZipArchive.hh:240
XRootDStatus Write(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:196
XRootDStatus UpdateMetadata(uint32_t crc32)
uint64_t archsize
Definition: XrdClZipArchive.hh:539
bool GetProperty(const std::string &name, std::string &value)
Get property on the underlying File object.
Definition: XrdClZipArchive.hh:342
XRootDStatus WriteImpl(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout)
static void Schedule(ResponseHandler *handler, XRootDStatus *st, Response *rsp=nullptr)
Definition: XrdClZipArchive.hh:434
std::unique_ptr< char[]> buffer
Definition: XrdClZipArchive.hh:542
static AnyObject * PkgRsp(Response *rsp)
Definition: XrdClZipArchive.hh:407
XRootDStatus OpenArchive(const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus OpenOnly(const std::string &url, bool update, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus List(DirectoryList *&list)
bool ckpinit
Definition: XrdClZipArchive.hh:555
XRootDStatus AppendFile(const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
bool SetProperty(const std::string &name, const std::string &value)
Set property on the underlying File object.
Definition: XrdClZipArchive.hh:334
new_files_t newfiles
Definition: XrdClZipArchive.hh:556
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:554
XRootDStatus Stat(StatInfo *&info)
Definition: XrdClZipArchive.hh:259
void Clear()
Clear internal ZipArchive objects.
Definition: XrdClZipArchive.hh:485
cdvec_t cdvec
Definition: XrdClZipArchive.hh:544
uint32_t orgcdsz
Definition: XrdClZipArchive.hh:547
StatInfo * make_stat(const std::string &fn)
Definition: XrdClZipArchive.hh:463
OpenStages openstage
Definition: XrdClZipArchive.hh:551
zipcache_t zipcache
Definition: XrdClZipArchive.hh:553
XRootDStatus CloseArchive(ResponseHandler *handler, uint16_t timeout=0)
Create the central directory at the end of ZIP archive and close it.
std::unique_ptr< EOCD > eocd
Definition: XrdClZipArchive.hh:543
std::unordered_map< std::string, NewFile > new_files_t
Definition: XrdClZipArchive.hh:536
bool IsOpen()
Definition: XrdClZipArchive.hh:318
static StatInfo * make_stat(const StatInfo &starch, uint64_t size)
Definition: XrdClZipArchive.hh:448
XRootDStatus PgReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
virtual ~ZipArchive()
Destructor.
XRootDStatus CloseFile()
Definition: XrdClZipArchive.hh:299
File archive
Definition: XrdClZipArchive.hh:538
friend XRootDStatus ReadFromImpl(ZipArchive &, const std::string &, uint64_t, uint32_t, void *, ResponseHandler *, uint16_t)
uint32_t orgcdcnt
Definition: XrdClZipArchive.hh:548
std::unique_ptr< ZIP64_EOCD > zip64eocd
Definition: XrdClZipArchive.hh:550
std::unordered_map< std::string, ZipCache > zipcache_t
Type that maps file name to its cache.
Definition: XrdClZipArchive.hh:535
bool updated
Definition: XrdClZipArchive.hh:541
XRootDStatus GetCRC32(const std::string &fn, uint32_t &cksum)
Definition: XrdClZipArchive.hh:273
ZipArchive(bool enablePlugIns=true)
Constructor.
buffer_t GetCD()
cdmap_t cdmap
Definition: XrdClZipArchive.hh:545
XRootDStatus OpenFile(const std::string &fn, OpenFlags::Flags flags=OpenFlags::None, uint64_t size=0, uint32_t crc32=0)
XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:120
std::string openfn
Definition: XrdClZipArchive.hh:552
uint64_t cdoff
Definition: XrdClZipArchive.hh:546
static void Free(XRootDStatus *st, Response *rsp)
Free status and response.
Definition: XrdClZipArchive.hh:419
void SetCD(const buffer_t &buffer)
Definition: XrdClZipArchive.hh:45
Definition: XrdEcReader.hh:58
Definition: XrdEcStrmWriter.hh:53
Definition: XrdClAnyObject.hh:26
StatImpl< false > Stat(Ctx< File > file, Arg< bool > force, uint16_t timeout=0)
Definition: XrdClFileOperations.hh:446
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errNotFound
Definition: XrdClStatus.hh:100
ZipReadFromImpl< false > ReadFrom(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:302
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:51
Definition: XrdClZipArchive.hh:45
Definition: XrdZipCDFH.hh:40
std::vector< std::unique_ptr< CDFH > > cdvec_t
Definition: XrdZipCDFH.hh:44
std::vector< char > buffer_t
Definition: XrdZipUtils.hh:54
std::unordered_map< std::string, size_t > cdmap_t
Definition: XrdZipCDFH.hh:54
Definition: XrdOucJson.hh:4517
none object for initializing empty Optional
Definition: XrdClOptional.hh:35
Flags
Open flags, may be or'd when appropriate.
Definition: XrdClFileSystem.hh:76
@ None
Nothing.
Definition: XrdClFileSystem.hh:77
LFH of a newly appended file (in case it needs to be overwritten)
Definition: XrdClZipArchive.hh:514
uint64_t offset
Definition: XrdClZipArchive.hh:527
bool overwrt
Definition: XrdClZipArchive.hh:529
NewFile(NewFile &&nf)
Definition: XrdClZipArchive.hh:521
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:528
NewFile(uint64_t offset, std::unique_ptr< LFH > lfh)
Definition: XrdClZipArchive.hh:515