input.hpp

Go to the documentation of this file.

// -----------------------------------------------------------------------------------------------------
// Copyright (c) 2006-2021, Knut Reinert & Freie Universität Berlin
// Copyright (c) 2016-2021, Knut Reinert & MPI für molekulare Genetik
// This file may be used, modified and/or redistributed under the terms of the 3-clause BSD-License
// shipped with this file and also available at: https://github.com/seqan/seqan3/blob/master/LICENSE.md
// -----------------------------------------------------------------------------------------------------
 
#pragma once
 
#include <cassert>
#include <seqan3/std/filesystem>
#include <fstream>
#include <string>
#include <variant>
#include <vector>
 
#include <seqan3/alphabet/adaptation/char.hpp>
#include <seqan3/alphabet/aminoacid/aa27.hpp>
#include <seqan3/alphabet/nucleotide/dna15.hpp>
#include <seqan3/alphabet/nucleotide/dna5.hpp>
#include <seqan3/alphabet/quality/phred42.hpp>
#include <seqan3/alphabet/quality/qualified.hpp>
#include <seqan3/core/detail/pack_algorithm.hpp>
#include <seqan3/io/detail/in_file_iterator.hpp>
#include <seqan3/io/detail/misc_input.hpp>
#include <seqan3/io/detail/record.hpp>
#include <seqan3/io/exception.hpp>
#include <seqan3/io/sam_file/format_sam.hpp>
#include <seqan3/io/sequence_file/format_embl.hpp>
#include <seqan3/io/sequence_file/format_fasta.hpp>
#include <seqan3/io/sequence_file/format_fastq.hpp>
#include <seqan3/io/sequence_file/format_genbank.hpp>
#include <seqan3/io/sequence_file/input_format_concept.hpp>
#include <seqan3/io/sequence_file/record.hpp>
#include <seqan3/io/stream/concept.hpp>
#include <seqan3/utility/type_list/traits.hpp>
 
namespace seqan3
{
 
// ----------------------------------------------------------------------------
// sequence_file_input_traits
// ----------------------------------------------------------------------------
 
template <typename t>
SEQAN3_CONCEPT sequence_file_input_traits = requires (t v)
{
    requires writable_alphabet<typename t::sequence_alphabet>;
    requires writable_alphabet<typename t::sequence_legal_alphabet>;
    requires explicitly_convertible_to<typename t::sequence_legal_alphabet, typename t::sequence_alphabet>;
    requires sequence_container<typename t::template sequence_container<typename t::sequence_alphabet>>;
 
    requires writable_alphabet<typename t::id_alphabet>;
    requires sequence_container<typename t::template id_container<typename t::id_alphabet>>;
 
    requires writable_quality_alphabet<typename t::quality_alphabet>;
    requires sequence_container<typename t::template quality_container<typename t::quality_alphabet>>;
};
 
// ----------------------------------------------------------------------------
// sequence_file_input_default_traits
// ----------------------------------------------------------------------------
 
struct sequence_file_input_default_traits_dna
{
    using sequence_alphabet                 = dna5;
 
    using sequence_legal_alphabet           = dna15;
 
    template <typename _sequence_alphabet>
    using sequence_container                = std::vector<_sequence_alphabet>;
 
    using id_alphabet                       = char;
 
    template <typename _id_alphabet>
    using id_container                      = std::basic_string<_id_alphabet>;
 
    using quality_alphabet                  = phred42;
 
    template <typename _quality_alphabet>
    using quality_container                 = std::vector<_quality_alphabet>;
 
};
 
struct sequence_file_input_default_traits_aa : sequence_file_input_default_traits_dna
{
    using sequence_alphabet = aa27;
 
    using sequence_legal_alphabet = aa27;
};
 
// ----------------------------------------------------------------------------
// sequence_file_input
// ----------------------------------------------------------------------------
 
template <
    sequence_file_input_traits traits_type_ = sequence_file_input_default_traits_dna,
    detail::fields_specialisation selected_field_ids_ = fields<field::seq, field::id, field::qual>,
    detail::type_list_of_sequence_file_input_formats valid_formats_ = type_list<format_embl,
                                                                                format_fasta,
                                                                                format_fastq,
                                                                                format_genbank,
                                                                                format_sam>>
class sequence_file_input
{
public:
    using traits_type           = traits_type_;
    using selected_field_ids    = selected_field_ids_;
    using valid_formats         = valid_formats_;
    using stream_char_type      = char;
 
#ifdef SEQAN3_DEPRECATED_310
    using field_ids            = fields<field::seq, field::id, field::qual, field::_seq_qual_deprecated>;
#else // ^^^ before seqan 3.1 / after seqan 3.1 vvv
    using field_ids            = fields<field::seq, field::id, field::qual>;
#endif // SEQAN3_DEPRECATED_310
 
    static_assert([] () constexpr
                  {
                      for (field f : selected_field_ids::as_array)
                          if (!field_ids::contains(f))
                              return false;
                      return true;
                  }(),
                  "You selected a field that is not valid for sequence files, please refer to the documentation "
                  "of sequence_file_input::field_ids for the accepted values.");
 
#ifdef SEQAN3_DEPRECATED_310
    static_assert([] () constexpr
                  {
                      return !(selected_field_ids::contains(field::_seq_qual_deprecated) &&
                               (selected_field_ids::contains(field::seq) ||
                               (selected_field_ids::contains(field::qual))));
                  }(),
                  "You may not select field::seq_qual and either of field::seq and field::qual at the same time.");
#endif // SEQAN3_DEPRECATED_310
 
    using sequence_type         = typename traits_type::template sequence_container<
                                    typename traits_type::sequence_alphabet>;
    using id_type               = typename traits_type::template id_container<
                                    typename traits_type::id_alphabet>;
    using quality_type          = typename traits_type::template quality_container<
                                    typename traits_type::quality_alphabet>;
#ifdef SEQAN3_DEPRECATED_310
    using sequence_quality_type = typename traits_type::
                                    template sequence_container<qualified<typename traits_type::sequence_alphabet,
                                                                          typename traits_type::quality_alphabet>>;
 
    using field_types           = type_list<sequence_type, id_type, quality_type, sequence_quality_type>;
#else // ^^^ before seqan 3.1 / after seqan 3.1 vvv
    using field_types           = type_list<sequence_type, id_type, quality_type>;
#endif // SEQAN3_DEPRECATED_310
 
    using record_type           = sequence_record<detail::select_types_with_ids_t<field_types,
                                                                                  field_ids,
                                                                                  selected_field_ids>,
                                                  selected_field_ids>;
 
    using value_type        = record_type;
    using reference         = record_type &;
    using const_reference   = void;
    using size_type         = size_t;
    using difference_type   = std::make_signed_t<size_t>;
    using iterator          = detail::in_file_iterator<sequence_file_input>;
    using const_iterator    = void;
    using sentinel          = std::default_sentinel_t;
 
    sequence_file_input() = delete;
    sequence_file_input(sequence_file_input const &) = delete;
    sequence_file_input & operator=(sequence_file_input const &) = delete;
    sequence_file_input(sequence_file_input &&) = default;
    sequence_file_input & operator=(sequence_file_input &&) = default;
    ~sequence_file_input() = default;
 
    sequence_file_input(std::filesystem::path filename,
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new std::ifstream{}, stream_deleter_default}
    {
        primary_stream->rdbuf()->pubsetbuf(stream_buffer.data(), stream_buffer.size());
        static_cast<std::basic_ifstream<char> *>(primary_stream.get())->open(filename,
                                                                             std::ios_base::in | std::ios::binary);
 
        if (!primary_stream->good())
            throw file_open_error{"Could not open file " + filename.string() + " for reading."};
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream, filename);
 
        // initialise format handler or throw if format is not found
        using format_variant_t = typename detail::variant_from_tags<valid_formats,
                                                                    detail::sequence_file_input_format_exposer>::type;
        format_variant_t format_variant{};
        detail::set_format(format_variant, filename);
 
        std::visit([&] (auto && selected_format)
        {
            using format_t = std::remove_cvref_t<decltype(selected_format)>;
            format = std::make_unique<selected_sequence_format<format_t>>();
        }, format_variant);
    }
    /* NOTE(h-2): Curiously we do not need a user-defined deduction guide for the above constructor.
     * A combination of default template parameters and auto-deduction guides works as expected,
     * independent of whether the second/optional parameter is specified or not, i.e. it is possible
     * to auto-deduct and overwrite a single template parameter out of the four if the optional parameter
     * is specified and use the default otherwise.
     */
 
    template <input_stream stream_t,
              sequence_file_input_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_t>::char_type, stream_char_type>
    sequence_file_input(stream_t                 & stream,
                        file_format        const & SEQAN3_DOXYGEN_ONLY(format_tag),
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{&stream, stream_deleter_noop},
        format{std::make_unique<selected_sequence_format<file_format>>()}
    {
        static_assert(list_traits::contains<file_format, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream);
    }
 
    template <input_stream stream_t,
              sequence_file_input_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_t>::char_type, stream_char_type>
    sequence_file_input(stream_t                && stream,
                        file_format        const & SEQAN3_DOXYGEN_ONLY(format_tag),
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new stream_t{std::move(stream)}, stream_deleter_default},
        format{std::make_unique<selected_sequence_format<file_format>>()}
    {
        static_assert(list_traits::contains<file_format, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream);
    }
 
    iterator begin()
    {
        // buffer first record
        if (!first_record_was_read)
        {
            read_next_record();
            first_record_was_read = true;
        }
 
        return {*this};
    }
 
    sentinel end() noexcept
    {
        return {};
    }
 
    reference front() noexcept
    {
        return *begin();
    }
 
#ifdef SEQAN3_DEPRECATED_310
    using sequence_file_input_options_type
        = sequence_file_input_options<typename traits_type::sequence_legal_alphabet,
                                      selected_field_ids::contains(field::_seq_qual_deprecated)>;
#else // ^^^ before seqan 3.1 / after seqan 3.1 vvv
    using sequence_file_input_options_type = sequence_file_input_options<typename traits_type::sequence_legal_alphabet>;
#endif // SEQAN3_DEPRECATED_310
    sequence_file_input_options_type options;
 
protected:
 
    record_type record_buffer;
    std::vector<char> stream_buffer{std::vector<char>(1'000'000)};
 
    using stream_ptr_t = std::unique_ptr<std::basic_istream<stream_char_type>,
                                         std::function<void(std::basic_istream<stream_char_type>*)>>;
    static void stream_deleter_noop(std::basic_istream<stream_char_type> *) {}
    static void stream_deleter_default(std::basic_istream<stream_char_type> * ptr) { delete ptr; }
 
    stream_ptr_t primary_stream{nullptr, stream_deleter_noop};
    stream_ptr_t secondary_stream{nullptr, stream_deleter_noop};
 
    bool first_record_was_read{false};
    bool at_end{false};
 
private:
    void read_next_record()
    {
        // clear the record
        record_buffer.clear();
 
        // at end if we could not read further
        if ((std::istreambuf_iterator<stream_char_type>{*secondary_stream} ==
             std::istreambuf_iterator<stream_char_type>{}))
        {
            at_end = true;
            return;
        }
 
        format->read_sequence_record(*secondary_stream, record_buffer, options);
    }
 
    struct sequence_format_base
    {
        sequence_format_base() = default; 
        sequence_format_base(sequence_format_base const &) = default; 
        sequence_format_base(sequence_format_base &&) = default; 
        sequence_format_base & operator=(sequence_format_base const &) = default; 
        sequence_format_base & operator=(sequence_format_base &&) = default; 
        virtual ~sequence_format_base() = default; 
 
        virtual void read_sequence_record(std::istream & instream,
                                          record_type & record_buffer,
                                          sequence_file_input_options_type const & options) = 0;
    };
 
    template <typename format_t>
    struct selected_sequence_format final : public sequence_format_base
    {
        selected_sequence_format() = default; 
        selected_sequence_format(selected_sequence_format const &) = default; 
        selected_sequence_format(selected_sequence_format &&) = default; 
        selected_sequence_format & operator=(selected_sequence_format const &) = default; 
        selected_sequence_format & operator=(selected_sequence_format &&) = default; 
        ~selected_sequence_format() = default; 
 
        void read_sequence_record(std::istream & instream,
                                  record_type & record_buffer,
                                  sequence_file_input_options_type const & options) override
        {
            // read new record
#ifdef SEQAN3_DEPRECATED_310
            if constexpr (selected_field_ids::contains(field::_seq_qual_deprecated))
            {
                _format.read_sequence_record(instream,
                                             options,
                                             detail::get_or_ignore<field::_seq_qual_deprecated>(record_buffer),
                                             detail::get_or_ignore<field::id>(record_buffer),
                                             detail::get_or_ignore<field::_seq_qual_deprecated>(record_buffer));
            }
            else
#endif // SEQAN3_DEPRECATED_310
            {
                _format.read_sequence_record(instream,
                                             options,
                                             detail::get_or_ignore<field::seq>(record_buffer),
                                             detail::get_or_ignore<field::id>(record_buffer),
                                             detail::get_or_ignore<field::qual>(record_buffer));
            }
        };
 
        detail::sequence_file_input_format_exposer<format_t> _format{};
    };
 
    std::unique_ptr<sequence_format_base> format{};
 
    friend iterator;
};
 
template <input_stream stream_type,
          sequence_file_input_format file_format>
sequence_file_input(stream_type & stream,
                    file_format const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,         // actually use the default
                           typename sequence_file_input<>::selected_field_ids,  // default field ids.
                           type_list<file_format>>;
 
template <input_stream stream_type,
          sequence_file_input_format file_format>
sequence_file_input(stream_type && stream,
                    file_format const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,         // actually use the default
                           typename sequence_file_input<>::selected_field_ids,  // default field ids.
                           type_list<file_format>>;
 
template <input_stream stream_type,
          sequence_file_input_format file_format,
          detail::fields_specialisation selected_field_ids>
sequence_file_input(stream_type && stream,
                    file_format const &,
                    selected_field_ids const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,       // actually use the default
                           selected_field_ids,
                           type_list<file_format>>;
 
template <input_stream stream_type,
          sequence_file_input_format file_format,
          detail::fields_specialisation selected_field_ids>
sequence_file_input(stream_type & stream,
                    file_format const &,
                    selected_field_ids const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,       // actually use the default
                           selected_field_ids,
                           type_list<file_format>>;
 
} // namespace seqan3