input.hpp

Go to the documentation of this file.

// -----------------------------------------------------------------------------------------------------
// Copyright (c) 2006-2019, Knut Reinert & Freie Universität Berlin
// Copyright (c) 2016-2019, Knut Reinert & MPI für molekulare Genetik
// This file may be used, modified and/or redistributed under the terms of the 3-clause BSD-License
// shipped with this file and also available at: https://github.com/seqan/seqan3/blob/master/LICENSE.md
// -----------------------------------------------------------------------------------------------------

#pragma once

#include <cassert>
#include <fstream>
#include <string>
#include <variant>
#include <vector>

#include <range/v3/algorithm/equal.hpp>

#include <seqan3/alphabet/adaptation/char.hpp>
#include <seqan3/alphabet/aminoacid/aa27.hpp>
#include <seqan3/alphabet/nucleotide/all.hpp>
#include <seqan3/alphabet/quality/phred42.hpp>
#include <seqan3/alphabet/quality/qualified.hpp>
#include <seqan3/core/type_traits/basic.hpp>
#include <seqan3/io/stream/concept.hpp>
#include <seqan3/io/exception.hpp>
#include <seqan3/std/filesystem>
#include <seqan3/io/record.hpp>
#include <seqan3/io/detail/in_file_iterator.hpp>
#include <seqan3/io/detail/misc_input.hpp>
#include <seqan3/io/detail/record.hpp>
#include <seqan3/io/sequence_file/input_format_concept.hpp>
#include <seqan3/io/sequence_file/format_embl.hpp>
#include <seqan3/io/sequence_file/format_fasta.hpp>
#include <seqan3/io/sequence_file/format_fastq.hpp>
#include <seqan3/io/sequence_file/format_genbank.hpp>
#include <seqan3/io/sequence_file/format_sam.hpp>
#include <seqan3/range/container/concatenated_sequences.hpp>

namespace seqan3
{

// ----------------------------------------------------------------------------
// SequenceFileInputTraits
// ----------------------------------------------------------------------------

template <typename t>
SEQAN3_CONCEPT SequenceFileInputTraits = requires (t v)
{
    requires WritableAlphabet<typename t::sequence_alphabet>;
    requires WritableAlphabet<typename t::sequence_legal_alphabet>;
    requires ExplicitlyConvertibleTo<typename t::sequence_legal_alphabet, typename t::sequence_alphabet>;
    requires SequenceContainer<typename t::template sequence_container<typename t::sequence_alphabet>>;
    requires SequenceContainer<typename t::template sequence_container_container<
        typename t::template sequence_container<typename t::sequence_alphabet>>>;

    requires WritableAlphabet<typename t::id_alphabet>;
    requires SequenceContainer<typename t::template id_container<typename t::id_alphabet>>;
    requires SequenceContainer<typename t::template id_container_container<typename t::template id_container<
        typename t::id_alphabet>>>;

    requires WritableQualityAlphabet<typename t::quality_alphabet>;
    requires SequenceContainer<typename t::template quality_container<typename t::quality_alphabet>>;
    requires SequenceContainer<typename t::template quality_container_container<
        typename t::template quality_container<typename t::quality_alphabet>>>;
};

// ----------------------------------------------------------------------------
// sequence_file_input_default_traits
// ----------------------------------------------------------------------------

struct sequence_file_input_default_traits_dna
{
    using sequence_alphabet                 = dna5;

    using sequence_legal_alphabet           = dna15;

    template <typename _sequence_alphabet>
    using sequence_container                = std::vector<_sequence_alphabet>;

    template <typename _sequence_container>
    using sequence_container_container      = concatenated_sequences<_sequence_container>;

    using id_alphabet                       = char;

    template <typename _id_alphabet>
    using id_container                      = std::basic_string<_id_alphabet>;

    template <typename _id_container>
    using id_container_container            = concatenated_sequences<_id_container>;

    using quality_alphabet                  = phred42;

    template <typename _quality_alphabet>
    using quality_container                 = std::vector<_quality_alphabet>;

    template <typename _quality_container>
    using quality_container_container       = concatenated_sequences<_quality_container>;
};

struct sequence_file_input_default_traits_aa : sequence_file_input_default_traits_dna
{
    using sequence_alphabet = aa27;

    using sequence_legal_alphabet = aa27;
};

// ----------------------------------------------------------------------------
// sequence_file_input
// ----------------------------------------------------------------------------

template <
    SequenceFileInputTraits                    traits_type_        = sequence_file_input_default_traits_dna,
    detail::Fields                             selected_field_ids_ = fields<field::SEQ,
                                                                            field::ID,
                                                                            field::QUAL>,
    detail::TypeListOfSequenceFileInputFormats valid_formats_      = type_list<format_embl,
                                                                               format_fasta,
                                                                               format_fastq,
                                                                               format_genbank,
                                                                               format_sam>,
    Char                                       stream_char_type_   = char>
class sequence_file_input
{
public:
    using traits_type           = traits_type_;
    using selected_field_ids    = selected_field_ids_;
    using valid_formats         = valid_formats_;
    using stream_char_type      = stream_char_type_;

    using field_ids            = fields<field::SEQ, field::ID, field::QUAL, field::SEQ_QUAL>;

    static_assert([] () constexpr
                  {
                      for (field f : selected_field_ids::as_array)
                          if (!field_ids::contains(f))
                              return false;
                      return true;
                  }(),
                  "You selected a field that is not valid for sequence files, please refer to the documentation "
                  "of sequence_file_input::field_ids for the accepted values.");

    static_assert([] () constexpr
                  {
                      return !(selected_field_ids::contains(field::SEQ_QUAL) &&
                               (selected_field_ids::contains(field::SEQ) ||
                               (selected_field_ids::contains(field::QUAL))));
                  }(),
                  "You may not select field::SEQ_QUAL and either of field::SEQ and field::QUAL at the same time.");

    using sequence_type         = typename traits_type::template sequence_container<
                                    typename traits_type::sequence_alphabet>;
    using id_type               = typename traits_type::template id_container<
                                    typename traits_type::id_alphabet>;
    using quality_type          = typename traits_type::template quality_container<
                                    typename traits_type::quality_alphabet>;
    using sequence_quality_type = typename traits_type::
                                    template sequence_container<qualified<typename traits_type::sequence_alphabet,
                                                                          typename traits_type::quality_alphabet>>;

    using field_types           = type_list<sequence_type, id_type, quality_type, sequence_quality_type>;

    using record_type           = record<detail::select_types_with_ids_t<field_types, field_ids, selected_field_ids>,
                                         selected_field_ids>;

    using sequence_column_type          = typename traits_type::template sequence_container_container<sequence_type>;
    using id_column_type                = typename traits_type::template id_container_container<id_type>;
    using quality_column_type           = typename traits_type::template quality_container_container<quality_type>;
    using sequence_quality_column_type  = typename traits_type::template sequence_container_container<sequence_quality_type>;
    using field_column_types            = type_list<sequence_column_type,
                                                    id_column_type,
                                                    quality_column_type,
                                                    sequence_quality_column_type>;
    using file_as_tuple_type            = record<detail::select_types_with_ids_t<field_column_types,
                                                                                 field_ids,
                                                                                 selected_field_ids>,
                                                 selected_field_ids>;

    using value_type        = record_type;
    using reference         = record_type &;
    using const_reference   = void;
    using size_type         = size_t;
    using difference_type   = std::make_signed_t<size_t>;
    using iterator          = detail::in_file_iterator<sequence_file_input>;
    using const_iterator    = void;
    using sentinel          = std::ranges::default_sentinel_t;

    sequence_file_input() = delete;
    sequence_file_input(sequence_file_input const &) = delete;
    sequence_file_input & operator=(sequence_file_input const &) = delete;
    sequence_file_input(sequence_file_input &&) = default;
    sequence_file_input & operator=(sequence_file_input &&) = default;
    ~sequence_file_input() = default;

    sequence_file_input(std::filesystem::path filename,
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new std::ifstream{filename, std::ios_base::in | std::ios::binary}, stream_deleter_default}
    {
        if (!primary_stream->good())
            throw file_open_error{"Could not open file " + filename.string() + " for reading."};

        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream, filename);

        // initialise format handler or throw if format is not found
        detail::set_format(format, filename);

        // buffer first record
        read_next_record();
    }
    /* NOTE(h-2): Curiously we do not need a user-defined deduction guide for the above constructor.
     * A combination of default template parameters and auto-deduction guides works as expected,
     * independent of whether the second/optional parameter is specified or not, i.e. it is possible
     * to auto-deduct and overwrite a single template parameter out of the four if the optional parameter
     * is specified and use the default otherwise.
     */

    template <IStream2 stream_t,
              SequenceFileInputFormat file_format>
    sequence_file_input(stream_t                 & stream,
                        file_format        const & SEQAN3_DOXYGEN_ONLY(format_tag),
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{&stream, stream_deleter_noop},
        format{detail::sequence_file_input_format<file_format>{}}
    {
        static_assert(meta::in<valid_formats, file_format>::value,
                      "You selected a format that is not in the valid_formats of this file.");

        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream);

        // buffer first record
        read_next_record();
    }

    template <IStream2 stream_t,
              SequenceFileInputFormat file_format>
    sequence_file_input(stream_t                && stream,
                        file_format        const & SEQAN3_DOXYGEN_ONLY(format_tag),
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new stream_t{std::move(stream)}, stream_deleter_default},
        format{detail::sequence_file_input_format<file_format>{}}
    {
        static_assert(meta::in<valid_formats, file_format>::value,
                      "You selected a format that is not in the valid_formats of this file.");

        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream);

        // buffer first record
        read_next_record();
    }

    iterator begin() noexcept
    {
        return {*this};
    }

    sentinel end() noexcept
    {
        return {};
    }

    reference front() noexcept
    {
        return record_buffer;
    }

    template <field f>
    friend auto & get(sequence_file_input & file)
    {
        static_assert(sequence_file_input::selected_field_ids::contains(f),
                      "You requested a field via get that was not selected for the file.");

        file.read_columns();

        return seqan3::get<f>(file.columns_buffer);
    }

    template <field f>
    friend auto && get(sequence_file_input && file)
    {
        return std::move(get<f>(file));
    }

    template <size_t i>
    friend auto & get(sequence_file_input & file)
    {
        static_assert(i < sequence_file_input::selected_field_ids::as_array.size(),
                      "You requested a field number larger than the number of selected fields for the file.");
        file.read_columns();

        return std::get<i>(file.columns_buffer);
    }

    template <size_t i>
    friend auto && get(sequence_file_input && file)
    {
        return std::move(get<i>(file));
    }

    template <typename t>
    friend auto & get(sequence_file_input & file)
    {
        file.read_columns();

        return std::get<t>(file.columns_buffer);
    }

    template <typename t>
    friend auto && get(sequence_file_input && file)
    {
        return std::move(get<t>(file));
    }

    sequence_file_input_options<typename traits_type::sequence_legal_alphabet,
                             selected_field_ids::contains(field::SEQ_QUAL)> options;

protected:

    record_type record_buffer;
    file_as_tuple_type columns_buffer;

    using stream_ptr_t = std::unique_ptr<std::basic_istream<stream_char_type>,
                                         std::function<void(std::basic_istream<stream_char_type>*)>>;
    static void stream_deleter_noop(std::basic_istream<stream_char_type> *) {}
    static void stream_deleter_default(std::basic_istream<stream_char_type> * ptr) { delete ptr; }

    stream_ptr_t primary_stream{nullptr, stream_deleter_noop};
    stream_ptr_t secondary_stream{nullptr, stream_deleter_noop};

    bool at_end{false};

    using format_type = typename detail::variant_from_tags<valid_formats, detail::sequence_file_input_format>::type;
    format_type format;

    void read_next_record()
    {
        // clear the record
        record_buffer.clear();

        // at end if we could not read further
        if ((std::istreambuf_iterator<stream_char_type>{*secondary_stream} ==
             std::istreambuf_iterator<stream_char_type>{}))
        {
            at_end = true;
            return;
        }

        assert(!format.valueless_by_exception());
        std::visit([&] (auto & f)
        {
            // read new record
            if constexpr (selected_field_ids::contains(field::SEQ_QUAL))
            {
                f.read(*secondary_stream,
                       options,
                       detail::get_or_ignore<field::SEQ_QUAL>(record_buffer),
                       detail::get_or_ignore<field::ID>(record_buffer),
                       detail::get_or_ignore<field::SEQ_QUAL>(record_buffer));
            }
            else
            {
                f.read(*secondary_stream,
                       options,
                       detail::get_or_ignore<field::SEQ>(record_buffer),
                       detail::get_or_ignore<field::ID>(record_buffer),
                       detail::get_or_ignore<field::QUAL>(record_buffer));
            }
        }, format);
    }

    void read_columns()
    {
        //TODO don't do multiple visits
        //TODO create specialised version for concatenated_sequences where we append on the concat
        auto & sequence_column_buffer = detail::get_or_ignore<field::SEQ>(columns_buffer);
        auto &       id_column_buffer = detail::get_or_ignore<field::ID>(columns_buffer);
        auto &     qual_column_buffer = detail::get_or_ignore<field::QUAL>(columns_buffer);
        auto & seq_qual_column_buffer = detail::get_or_ignore<field::SEQ_QUAL>(columns_buffer);

        // read the remaining records and split into column buffers
        for (auto & rec : *this)
        {
            if constexpr (selected_field_ids::contains(field::SEQ))
                sequence_column_buffer.push_back(std::move(seqan3::get<field::SEQ>(rec)));
            if constexpr (selected_field_ids::contains(field::ID))
                id_column_buffer.push_back(std::move(seqan3::get<field::ID>(rec)));
            if constexpr (selected_field_ids::contains(field::QUAL))
                qual_column_buffer.push_back(std::move(seqan3::get<field::QUAL>(rec)));
            if constexpr (selected_field_ids::contains(field::SEQ_QUAL))
                seq_qual_column_buffer.push_back(std::move(seqan3::get<field::SEQ_QUAL>(rec)));
        }
    }

    friend iterator;
};

template <IStream2                           stream_type,
          SequenceFileInputFormat            file_format,
          detail::Fields                     selected_field_ids>
sequence_file_input(stream_type && stream,
                    file_format const &,
                    selected_field_ids const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,       // actually use the default
                           selected_field_ids,
                           type_list<file_format>,
                           typename std::remove_reference_t<stream_type>::char_type>;

template <IStream2                           stream_type,
          SequenceFileInputFormat            file_format,
          detail::Fields                     selected_field_ids>
sequence_file_input(stream_type & stream,
                    file_format const &,
                    selected_field_ids const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,       // actually use the default
                           selected_field_ids,
                           type_list<file_format>,
                           typename std::remove_reference_t<stream_type>::char_type>;

} // namespace seqan3

// ------------------------------------------------------------------
// std-overloads for the tuple-like interface
// ------------------------------------------------------------------

namespace std
{
template <seqan3::SequenceFileInputTraits                    traits_type,
          seqan3::detail::Fields                             selected_field_ids,
          seqan3::detail::TypeListOfSequenceFileInputFormats valid_formats,
          seqan3::Char                                       stream_char_t>
struct tuple_size<seqan3::sequence_file_input<traits_type, selected_field_ids, valid_formats, stream_char_t>>
{
    static constexpr size_t value = selected_field_ids::as_array.size();
};

template <size_t                                             elem_no,
          seqan3::SequenceFileInputTraits                    traits_type,
          seqan3::detail::Fields                             selected_field_ids,
          seqan3::detail::TypeListOfSequenceFileInputFormats valid_formats,
          seqan3::Char                                       stream_char_t>
struct tuple_element<elem_no, seqan3::sequence_file_input<traits_type, selected_field_ids, valid_formats, stream_char_t>>
    : tuple_element<elem_no, typename seqan3::sequence_file_input<traits_type,
                                                               selected_field_ids,
                                                               valid_formats,
                                                               stream_char_t>::file_as_tuple_type>
{};

} // namespace std