SeqAn3 3.4.0-rc.1
The Modern C++ library for sequence analysis.
Loading...
Searching...
No Matches
gz_ostream.hpp
1// zipstream Library License:
2// --------------------------
3//
4// The zlib/libpng License Copyright (c) 2003 Jonathan de Halleux.
5//
6// This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.
7//
8// Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
9//
10// 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
11//
12// 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
13//
14// 3. This notice may not be removed or altered from any source distribution
15//
16// Altered zipstream library header
17// Author: Jonathan de Halleux, dehalleux@pelikhan.com, 2003
18// Author: David Weese <david.weese@fu-berlin.de>
19// Author: Enrico Siragusa <enrico.siragusa@fu-berlin.de>
20// Author: Hannes Hauswedell <hannes.hauswedell@fu-berlin.de>
21
22#pragma once
23
24#include <iostream>
25#include <cstring>
26#include <vector>
27
28#if !defined(SEQAN3_HAS_ZLIB) && !defined(SEQAN3_HEADER_TEST)
29#error "This file cannot be used when building without ZLIB-support."
30#endif // !defined(SEQAN3_HAS_ZLIB) && !defined(SEQAN3_HEADER_TEST)
31
32#if defined(SEQAN3_HAS_ZLIB)
33
34#include <zlib.h>
35
36namespace seqan3::contrib
37{
38
39// Default gzip buffer size, change this to suite your needs.
40const size_t GZ_OUTPUT_DEFAULT_BUFFER_SIZE = 921600;
41
42// --------------------------------------------------------------------------
43// Enum EStrategy
44// --------------------------------------------------------------------------
45// Compression strategy, see zlib doc.
46
47enum EStrategy
48{
49 StrategyFiltered = 1,
50 StrategyHuffmanOnly = 2,
51 DefaultStrategy = 0
52};
53
54// --------------------------------------------------------------------------
55// Class basic_gz_ostreambuf
56// --------------------------------------------------------------------------
57// A stream decorator that takes raw input and zips it to a ostream.
58// The class wraps up the inflate method of the zlib library 1.1.4 https://www.zlib.net
59
60template <typename Elem,
61 typename Tr = std::char_traits<Elem>,
62 typename ElemA = std::allocator<Elem>,
63 typename ByteT = unsigned char,
64 typename ByteAT = std::allocator<ByteT>
65 >
66class basic_gz_ostreambuf :
67 public std::basic_streambuf<Elem, Tr>
68{
69public:
70 typedef std::basic_ostream<Elem, Tr> & ostream_reference;
71 typedef ElemA char_allocator_type;
72 typedef ByteT byte_type;
73 typedef ByteAT byte_allocator_type;
74 typedef byte_type * byte_buffer_type;
75 typedef Tr traits_type;
76 typedef typename Tr::char_type char_type;
77 typedef typename Tr::int_type int_type;
78 typedef std::vector<byte_type, byte_allocator_type> byte_vector_type;
79 typedef std::vector<char_type, char_allocator_type> char_vector_type;
80
81 // Construct a zip stream
82 // More info on the following parameters can be found in the zlib documentation.
83 basic_gz_ostreambuf(ostream_reference ostream_,
84 size_t level_,
85 EStrategy strategy_,
86 size_t window_size_,
87 size_t memory_level_,
88 size_t buffer_size_);
89
90 ~basic_gz_ostreambuf();
91
92 int sync();
93 int_type overflow(int_type c);
94
95 // flushes the zip buffer and output buffer.
96 // This method should be called at the end of the compression.
97 // Calling flush multiple times, will lower the compression ratio.
99
100 // flushes the zip buffer and output buffer and finalize the zip stream
101 // This method should be called at the end of the compression.
102 std::streamsize flush_finalize();
103
104
105private:
106 bool zip_to_stream(char_type *, std::streamsize);
107 size_t fill_input_buffer();
108 // flush the zip buffer using a particular mode and flush output buffer
109 std::streamsize flush(int flush_mode);
110
111 ostream_reference m_ostream;
112 z_stream m_zip_stream;
113 int m_err;
114 byte_vector_type m_output_buffer;
115 char_vector_type m_buffer;
116};
117
118// --------------------------------------------------------------------------
119// Class basic_gz_ostreambuf implementation
120// --------------------------------------------------------------------------
121
122template <typename Elem,
123 typename Tr,
124 typename ElemA,
125 typename ByteT,
126 typename ByteAT>
127basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::basic_gz_ostreambuf(
128 ostream_reference ostream_,
129 size_t level_,
130 EStrategy strategy_,
131 size_t window_size_,
132 size_t memory_level_,
133 size_t buffer_size_
134 ) :
135 m_ostream(ostream_),
136 m_output_buffer(buffer_size_, 0),
137 m_buffer(buffer_size_, 0)
138{
139 m_zip_stream.zalloc = (alloc_func)0;
140 m_zip_stream.zfree = (free_func)0;
141
142 m_zip_stream.next_in = NULL;
143 m_zip_stream.avail_in = 0;
144 m_zip_stream.avail_out = 0;
145 m_zip_stream.next_out = NULL;
146
147 m_err = deflateInit2(
148 &m_zip_stream,
149 std::min(9, static_cast<int>(level_)),
150 Z_DEFLATED,
151 static_cast<int>(window_size_),
152 std::min(9, static_cast<int>(memory_level_)),
153 static_cast<int>(strategy_)
154 );
155
156 this->setp(&(m_buffer[0]), &(m_buffer[m_buffer.size() - 1]));
157}
158
159template <typename Elem,
160 typename Tr,
161 typename ElemA,
162 typename ByteT,
163 typename ByteAT>
164basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::~basic_gz_ostreambuf()
165{
166 flush_finalize();
167 m_ostream.flush();
168 m_err = deflateEnd(&m_zip_stream);
169}
170
171template <typename Elem,
172 typename Tr,
173 typename ElemA,
174 typename ByteT,
175 typename ByteAT>
176int basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::sync()
177{
178 if (this->pptr() && this->pptr() > this->pbase())
179 {
180 if (traits_type::eq_int_type(overflow(traits_type::eof()), traits_type::eof()))
181 return -1;
182 }
183
184 return 0;
185}
186
187template <typename Elem,
188 typename Tr,
189 typename ElemA,
190 typename ByteT,
191 typename ByteAT>
192typename basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::int_type
193basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::overflow(
194 typename basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::int_type c)
195{
196 int w = static_cast<int>(this->pptr() - this->pbase());
197
198 if (!traits_type::eq_int_type(c, traits_type::eof()))
199 {
200 *this->pptr() = c;
201 ++w;
202 }
203
204 if (zip_to_stream(this->pbase(), w))
205 {
206 this->setp(this->pbase(), this->epptr() - 1);
207 return c;
208 }
209 else
210 {
211 return traits_type::eof();
212 }
213}
214
215template <typename Elem,
216 typename Tr,
217 typename ElemA,
218 typename ByteT,
219 typename ByteAT>
220bool basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::zip_to_stream(
221 typename basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::char_type * buffer_,
222 std::streamsize buffer_size_)
223{
224 std::streamsize written_byte_size = 0;
225
226 m_zip_stream.next_in = (byte_buffer_type)buffer_;
227 m_zip_stream.avail_in = static_cast<uInt>(buffer_size_ * sizeof(char_type));
228 m_zip_stream.avail_out = static_cast<uInt>(m_output_buffer.size());
229 m_zip_stream.next_out = &(m_output_buffer[0]);
230 size_t remainder = 0;
231
232 do
233 {
234 m_err = deflate(&m_zip_stream, 0);
235
236 if (m_err == Z_OK || m_err == Z_STREAM_END)
237 {
238 written_byte_size = static_cast<std::streamsize>(m_output_buffer.size()) - m_zip_stream.avail_out;
239
240 // output buffer is full, dumping to ostream
241 m_ostream.write((const char_type *) &(m_output_buffer[0]),
242 static_cast<std::streamsize>(written_byte_size / sizeof(char_type)));
243
244 // checking if some bytes were not written.
245 if ((remainder = written_byte_size % sizeof(char_type)) != 0)
246 {
247 // copy to the beginning of the stream
248 std::memmove(&(m_output_buffer[0]),
249 &(m_output_buffer[written_byte_size - remainder]),
250 remainder);
251 }
252
253 m_zip_stream.avail_out = static_cast<uInt>(m_output_buffer.size() - remainder);
254 m_zip_stream.next_out = &m_output_buffer[remainder];
255 }
256 }
257 while (m_zip_stream.avail_in != 0 && m_err == Z_OK);
258
259 return m_err == Z_OK;
260}
261
262template <typename Elem,
263 typename Tr,
264 typename ElemA,
265 typename ByteT,
266 typename ByteAT>
267std::streamsize basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::flush(int flush_mode)
268{
269 int const buffer_size = static_cast<int>(this->pptr() - this->pbase()); // amount of data currently in buffer
270
271 std::streamsize written_byte_size = 0, total_written_byte_size = 0;
272
273 m_zip_stream.next_in = (byte_buffer_type) this->pbase();
274 m_zip_stream.avail_in = static_cast<uInt>(buffer_size * sizeof(char_type));
275 m_zip_stream.avail_out = static_cast<uInt>(m_output_buffer.size());
276 m_zip_stream.next_out = &(m_output_buffer[0]);
277 size_t remainder = 0;
278
279 do
280 {
281 m_err = deflate(&m_zip_stream, flush_mode);
282 if (m_err == Z_OK || m_err == Z_STREAM_END)
283 {
284 written_byte_size = static_cast<std::streamsize>(m_output_buffer.size()) - m_zip_stream.avail_out;
285 total_written_byte_size += written_byte_size;
286
287 // output buffer is full, dumping to ostream
288 m_ostream.write((const char_type *) &(m_output_buffer[0]),
289 static_cast<std::streamsize>(written_byte_size / sizeof(char_type) * sizeof(byte_type)));
290
291 // checking if some bytes were not written.
292 if ((remainder = written_byte_size % sizeof(char_type)) != 0)
293 {
294 // copy to the beginning of the stream
295 std::memmove(&(m_output_buffer[0]),
296 &(m_output_buffer[written_byte_size - remainder]),
297 remainder);
298 }
299
300 m_zip_stream.avail_out = static_cast<uInt>(m_output_buffer.size() - remainder);
301 m_zip_stream.next_out = &m_output_buffer[remainder];
302 }
303 }
304 while (m_err == Z_OK);
305
306 m_ostream.flush();
307
308 return total_written_byte_size;
309}
310
311template <typename Elem,
312 typename Tr,
313 typename ElemA,
314 typename ByteT,
315 typename ByteAT>
316std::streamsize basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::flush()
317{
318 return flush(Z_SYNC_FLUSH);
319}
320
321template <typename Elem,
322 typename Tr,
323 typename ElemA,
324 typename ByteT,
325 typename ByteAT>
326std::streamsize basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT>::flush_finalize()
327{
328 return flush(Z_FINISH);
329}
330
331// --------------------------------------------------------------------------
332// Class basic_gz_ostreambase
333// --------------------------------------------------------------------------
334// Base class for zip ostreams.
335// Contains a basic_gz_ostreambuf.
336
337template <typename Elem,
338 typename Tr = std::char_traits<Elem>,
339 typename ElemA = std::allocator<Elem>,
340 typename ByteT = unsigned char,
341 typename ByteAT = std::allocator<ByteT>
342 >
343class basic_gz_ostreambase :
344 virtual public std::basic_ios<Elem, Tr>
345{
346public:
347 typedef std::basic_ostream<Elem, Tr> & ostream_reference;
348 typedef basic_gz_ostreambuf<Elem, Tr, ElemA, ByteT, ByteAT> zip_streambuf_type;
349
350 // Construct a zip stream
351 // More info on the following parameters can be found in the zlib documentation.
352 basic_gz_ostreambase(ostream_reference ostream_,
353 size_t level_,
354 EStrategy strategy_,
355 size_t window_size_,
356 size_t memory_level_,
357 size_t buffer_size_) :
358 m_buf(ostream_, level_, strategy_, window_size_, memory_level_, buffer_size_)
359 {
360 this->init(&m_buf);
361 }
362
363 // returns the underlying zip ostream object
364 zip_streambuf_type * rdbuf() { return &m_buf; }
365
366private:
367 zip_streambuf_type m_buf;
368};
369
370// --------------------------------------------------------------------------
371// Class basic_gz_ostream
372// --------------------------------------------------------------------------
373// A zipper ostream
374//
375// This class is a ostream decorator that behaves 'almost' like any other ostream.
376// At construction, it takes any ostream that shall be used to output of the compressed data.
377// When finished, you need to call the special method zflush or call the destructor
378// to flush all the intermidiate streams.
379//
380// Example:
381//
382// // creating the target zip string, could be a fstream
383// ostringstream ostringstream_;
384// // creating the zip layer
385// zip_ostream zipper(ostringstream_);
386// // writing data
387// zipper<<f<<" "<<d<<" "<<ui<<" "<<ul<<" "<<us<<" "<<c<<" "<<dum;
388// // zip ostream needs special flushing...
389// zipper.zflush();
390
391template <typename Elem,
392 typename Tr = std::char_traits<Elem>,
393 typename ElemA = std::allocator<Elem>,
394 typename ByteT = unsigned char,
395 typename ByteAT = std::allocator<ByteT>
396 >
397class basic_gz_ostream :
398 public basic_gz_ostreambase<Elem, Tr, ElemA, ByteT, ByteAT>,
399 public std::basic_ostream<Elem, Tr>
400{
401public:
402 typedef basic_gz_ostreambase<Elem, Tr, ElemA, ByteT, ByteAT> zip_ostreambase_type;
403 typedef std::basic_ostream<Elem, Tr> ostream_type;
404 typedef ostream_type & ostream_reference;
405
406 // Constructs a zipper ostream decorator
407 //
408 // ostream_ ostream where the compressed output is written
409 // is_gzip_ true if gzip header and footer have to be added
410 // level_ level of compression 0, bad and fast, 9, good and slower,
411 // strategy_ compression strategy
412 // window_size_ see zlib doc
413 // memory_level_ see zlib doc
414 // buffer_size_ the buffer size used to zip data
415
416 basic_gz_ostream(ostream_reference ostream_,
417 size_t level_ = Z_DEFAULT_COMPRESSION,
418 EStrategy strategy_ = DefaultStrategy,
419 size_t window_size_ = 31, // 15 (size) + 16 (gzip header)
420 size_t memory_level_ = 8,
421 size_t buffer_size_ = GZ_OUTPUT_DEFAULT_BUFFER_SIZE) :
422 zip_ostreambase_type(ostream_, level_, strategy_, window_size_, memory_level_, buffer_size_),
423 ostream_type(this->rdbuf())
424 {}
425
426 ~basic_gz_ostream()
427 {
428 ostream_type::flush(); this->rdbuf()->flush_finalize();
429 }
430
431 // flush inner buffer and zipper buffer
432 basic_gz_ostream<Elem, Tr> & flush()
433 {
434 ostream_type::flush(); this->rdbuf()->flush(); return *this;
435 }
436
437#ifdef _WIN32
438private:
439 void _Add_vtordisp1() {} // Required to avoid VC++ warning C4250
440 void _Add_vtordisp2() {} // Required to avoid VC++ warning C4250
441#endif
442};
443
444// ===========================================================================
445// Typedefs
446// ===========================================================================
447
448// A typedef for basic_gz_ostream<char>
449typedef basic_gz_ostream<char> gz_ostream;
450// A typedef for basic_gz_ostream<wchar_t>
451typedef basic_gz_ostream<wchar_t> gz_wostream;
452
453} // namespace seqan3::contrib
454
455#endif // defined(SEQAN3_HAS_ZLIB)
T flush(T... args)
T init(T... args)
T memmove(T... args)
T min(T... args)
T rdbuf(T... args)
T remainder(T... args)