io/sam_file/output.hpp

Go to the documentation of this file.

// -----------------------------------------------------------------------------------------------------
// Copyright (c) 2006-2023, Knut Reinert & Freie Universität Berlin
// Copyright (c) 2016-2023, Knut Reinert & MPI für molekulare Genetik
// This file may be used, modified and/or redistributed under the terms of the 3-clause BSD-License
// shipped with this file and also available at: https://github.com/seqan/seqan3/blob/master/LICENSE.md
// -----------------------------------------------------------------------------------------------------
 
#pragma once
 
#include <cassert>
#include <filesystem>
#include <fstream>
#include <ranges>
#include <string>
#include <string_view>
#include <variant>
#include <vector>
 
#include <seqan3/io/detail/misc_output.hpp>
#include <seqan3/io/detail/out_file_iterator.hpp>
#include <seqan3/io/detail/record.hpp>
#include <seqan3/io/detail/record_like.hpp>
#include <seqan3/io/exception.hpp>
#include <seqan3/io/record.hpp>
#include <seqan3/io/sam_file/format_bam.hpp>
#include <seqan3/io/sam_file/format_sam.hpp>
#include <seqan3/io/sam_file/header.hpp>
#include <seqan3/io/sam_file/output_format_concept.hpp>
#include <seqan3/io/sam_file/output_options.hpp>
#include <seqan3/io/sam_file/sam_flag.hpp>
#include <seqan3/io/stream/concept.hpp>
#include <seqan3/utility/tuple/concept.hpp>
#include <seqan3/utility/type_list/traits.hpp>
 
namespace seqan3
{
 
// ----------------------------------------------------------------------------
// sam_file_output
// ----------------------------------------------------------------------------
 
template <detail::fields_specialisation selected_field_ids_ = fields<field::seq,
                                                                     field::id,
                                                                     field::ref_id,
                                                                     field::ref_offset,
                                                                     field::cigar,
                                                                     field::mapq,
                                                                     field::qual,
                                                                     field::flag,
                                                                     field::mate,
                                                                     field::tags,
                                                                     field::header_ptr>,
          detail::type_list_of_sam_file_output_formats valid_formats_ = type_list<format_sam, format_bam>,
          typename ref_ids_type = ref_info_not_given>
class sam_file_output
{
public:
    using selected_field_ids = selected_field_ids_;
    using valid_formats = valid_formats_;
    using stream_char_type = char;
 
    using field_ids = fields<field::seq,
                             field::id,
                             field::ref_id,
                             field::ref_offset,
                             field::cigar,
                             field::mapq,
                             field::flag,
                             field::qual,
                             field::mate,
                             field::tags,
                             field::header_ptr>;
 
    static_assert(!selected_field_ids::contains(field::offset),
                  "The field::offset is deprecated. It is already stored in the field::cigar as soft clipping (S) "
                  "at the front and not needed otherwise.");
 
    static_assert(!selected_field_ids::contains(field::alignment),
                  "The seqan3::field::alignment was removed from the allowed fields for seqan3::sam_file_output. "
                  "Only seqan3::field::cigar is supported. seqan3::cigar_from_alignment on how to get a CIGAR string "
                  "from an alignment.");
 
    static_assert(
        []() constexpr
        {
            for (field f : selected_field_ids::as_array)
                if (!field_ids::contains(f))
                    return false;
            return true;
        }(),
        "You selected a field that is not valid for SAM files, "
        "please refer to the documentation of "
        "seqan3::sam_file_output::field_ids for the accepted values.");
 
    using value_type = void;
    using reference = void;
    using const_reference = void;
    using size_type = void;
    using difference_type = std::ptrdiff_t;
    using iterator = detail::out_file_iterator<sam_file_output>;
    using const_iterator = void;
    using sentinel = std::default_sentinel_t;
 
    sam_file_output() = delete;
    sam_file_output(sam_file_output const &) = delete;
    sam_file_output & operator=(sam_file_output const &) = delete;
    sam_file_output(sam_file_output &&) = default;
    sam_file_output & operator=(sam_file_output &&) = default;
    ~sam_file_output()
    {
        if (header_has_been_written)
            return;
 
        assert(!format.valueless_by_exception());
 
        std::visit(
            [&](auto & f)
            {
                if constexpr (std::same_as<ref_ids_type, ref_info_not_given>)
                    f.write_header(*secondary_stream, options, std::ignore);
                else
                    f.write_header(*secondary_stream, options, *header_ptr);
            },
            format);
    }
 
    sam_file_output(std::filesystem::path filename,
                    selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new std::ofstream{}, stream_deleter_default}
    {
        primary_stream->rdbuf()->pubsetbuf(stream_buffer.data(), stream_buffer.size());
        static_cast<std::basic_ofstream<char> *>(primary_stream.get())
            ->open(filename, std::ios_base::out | std::ios::binary);
 
        // open stream
        if (!primary_stream->good())
            throw file_open_error{"Could not open file " + filename.string() + " for writing."};
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_ostream(*primary_stream, filename);
 
        // initialise format handler or throw if format is not found
        detail::set_format(format, filename);
    }
 
    template <output_stream stream_type, sam_file_output_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_type>::char_type, stream_char_type>
    sam_file_output(stream_type & stream,
                    file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                    selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{&stream, stream_deleter_noop},
        secondary_stream{&stream, stream_deleter_noop},
        format{detail::sam_file_output_format_exposer<file_format>{}}
    {
        static_assert(list_traits::contains<file_format, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
    }
 
    template <output_stream stream_type, sam_file_output_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_type>::char_type, stream_char_type>
    sam_file_output(stream_type && stream,
                    file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                    selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new stream_type{std::move(stream)}, stream_deleter_default},
        secondary_stream{&*primary_stream, stream_deleter_noop},
        format{detail::sam_file_output_format_exposer<file_format>{}}
    {
        static_assert(list_traits::contains<file_format, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
    }
 
    template <typename ref_ids_type_, std::ranges::forward_range ref_lengths_type>
        requires std::same_as<std::remove_reference_t<ref_ids_type_>, ref_ids_type>
    sam_file_output(std::filesystem::path const & filename,
                    ref_ids_type_ && ref_ids,
                    ref_lengths_type && ref_lengths,
                    selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        sam_file_output{filename, selected_field_ids{}}
 
    {
        initialise_header_information(std::forward<ref_ids_type_>(ref_ids),
                                      std::forward<ref_lengths_type>(ref_lengths));
    }
 
    template <output_stream stream_type,
              sam_file_output_format file_format,
              typename ref_ids_type_, // generic type to capture lvalue references
              std::ranges::forward_range ref_lengths_type>
        requires std::same_as<std::remove_reference_t<ref_ids_type_>, ref_ids_type>
    sam_file_output(stream_type && stream,
                    ref_ids_type_ && ref_ids,
                    ref_lengths_type && ref_lengths,
                    file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                    selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        sam_file_output{std::forward<stream_type>(stream), file_format{}, selected_field_ids{}}
    {
        initialise_header_information(std::forward<ref_ids_type_>(ref_ids),
                                      std::forward<ref_lengths_type>(ref_lengths));
    }
 
    iterator begin() noexcept
    {
        return {*this};
    }
 
    sentinel end() noexcept
    {
        return {};
    }
 
    template <typename record_t>
    void push_back(record_t && r)
        requires detail::record_like<record_t>
    {
        using default_mate_t = std::tuple<std::string_view, std::optional<int32_t>, int32_t>;
 
        write_record(detail::get_or<field::header_ptr>(r, nullptr),
                     detail::get_or<field::seq>(r, std::string_view{}),
                     detail::get_or<field::qual>(r, std::string_view{}),
                     detail::get_or<field::id>(r, std::string_view{}),
                     detail::get_or<field::ref_seq>(r, std::string_view{}),
                     detail::get_or<field::ref_id>(r, std::ignore),
                     detail::get_or<field::ref_offset>(r, std::optional<int32_t>{}),
                     detail::get_or<field::cigar>(r, std::vector<cigar>{}),
                     detail::get_or<field::flag>(r, sam_flag::none),
                     detail::get_or<field::mapq>(r, 0u),
                     detail::get_or<field::mate>(r, default_mate_t{}),
                     detail::get_or<field::tags>(r, sam_tag_dictionary{}),
                     detail::get_or<field::evalue>(r, 0u),
                     detail::get_or<field::bit_score>(r, 0u));
    }
 
    template <typename tuple_t>
    void push_back(tuple_t && t)
        requires tuple_like<tuple_t> && (!detail::record_like<tuple_t>)
    {
        using default_mate_t = std::tuple<std::string_view, std::optional<int32_t>, int32_t>;
 
        // index_of might return npos, but this will be handled well by get_or_ignore (and just return ignore)
        write_record(detail::get_or<selected_field_ids::index_of(field::header_ptr)>(t, nullptr),
                     detail::get_or<selected_field_ids::index_of(field::seq)>(t, std::string_view{}),
                     detail::get_or<selected_field_ids::index_of(field::qual)>(t, std::string_view{}),
                     detail::get_or<selected_field_ids::index_of(field::id)>(t, std::string_view{}),
                     detail::get_or<selected_field_ids::index_of(field::ref_seq)>(t, std::string_view{}),
                     detail::get_or<selected_field_ids::index_of(field::ref_id)>(t, std::ignore),
                     detail::get_or<selected_field_ids::index_of(field::ref_offset)>(t, std::optional<int32_t>{}),
                     detail::get_or<selected_field_ids::index_of(field::cigar)>(t, std::vector<cigar>{}),
                     detail::get_or<selected_field_ids::index_of(field::flag)>(t, sam_flag::none),
                     detail::get_or<selected_field_ids::index_of(field::mapq)>(t, 0u),
                     detail::get_or<selected_field_ids::index_of(field::mate)>(t, default_mate_t{}),
                     detail::get_or<selected_field_ids::index_of(field::tags)>(t, sam_tag_dictionary{}),
                     detail::get_or<selected_field_ids::index_of(field::evalue)>(t, 0u),
                     detail::get_or<selected_field_ids::index_of(field::bit_score)>(t, 0u));
    }
 
    template <typename arg_t, typename... arg_types>
        requires (sizeof...(arg_types) + 1 <= selected_field_ids::size)
    void emplace_back(arg_t && arg, arg_types &&... args)
    {
        push_back(std::tie(arg, args...));
    }
 
    template <typename rng_t>
    sam_file_output & operator=(rng_t && range)
        requires std::ranges::input_range<rng_t> && tuple_like<std::ranges::range_reference_t<rng_t>>
    {
        for (auto && record : range)
            push_back(std::forward<decltype(record)>(record));
        return *this;
    }
 
    template <typename rng_t>
    friend sam_file_output & operator|(rng_t && range, sam_file_output & f)
        requires std::ranges::input_range<rng_t> && tuple_like<std::ranges::range_reference_t<rng_t>>
    {
        f = range;
        return f;
    }
 
    template <typename rng_t>
    friend sam_file_output operator|(rng_t && range, sam_file_output && f)
        requires std::ranges::input_range<rng_t> && tuple_like<std::ranges::range_reference_t<rng_t>>
    {
        f = range;
        return std::move(f);
    }
 
    sam_file_output_options options;
 
    std::basic_ostream<stream_char_type> & get_stream()
    {
        return *secondary_stream;
    }
 
    auto & header()
    {
        if constexpr (std::same_as<ref_ids_type, ref_info_not_given>)
            throw std::logic_error{"Please construct your file with reference id and length information in order "
                                   "to properly initialise the header before accessing it."};
 
        return *header_ptr;
    }
 
protected:
    bool header_has_been_written{false};
 
    std::vector<char> stream_buffer{std::vector<char>(1'000'000)};
 
    using stream_ptr_t = std::unique_ptr<std::basic_ostream<stream_char_type>,
                                         std::function<void(std::basic_ostream<stream_char_type> *)>>;
    static void stream_deleter_noop(std::basic_ostream<stream_char_type> *)
    {}
    static void stream_deleter_default(std::basic_ostream<stream_char_type> * ptr)
    {
        delete ptr;
    }
 
    stream_ptr_t primary_stream{nullptr, stream_deleter_noop};
    stream_ptr_t secondary_stream{nullptr, stream_deleter_noop};
 
    using format_type = typename detail::variant_from_tags<valid_formats, detail::sam_file_output_format_exposer>::type;
 
    format_type format;
 
    using header_type = sam_file_header<
        std::conditional_t<std::same_as<ref_ids_type, ref_info_not_given>, std::vector<std::string>, ref_ids_type>>;
 
    std::unique_ptr<header_type> header_ptr;
 
    template <typename ref_ids_type_, typename ref_lengths_type>
    void initialise_header_information(ref_ids_type_ && ref_ids, ref_lengths_type && ref_lengths)
    {
        assert(std::ranges::size(ref_ids) == std::ranges::size(ref_lengths));
 
        header_ptr = std::make_unique<sam_file_header<ref_ids_type>>(std::forward<ref_ids_type_>(ref_ids));
 
        for (int32_t idx = 0; idx < std::ranges::distance(header_ptr->ref_ids()); ++idx)
        {
            header_ptr->ref_id_info.emplace_back(ref_lengths[idx], "");
 
            if constexpr (std::ranges::contiguous_range<std::ranges::range_reference_t<ref_ids_type_>>
                          && std::ranges::sized_range<std::ranges::range_reference_t<ref_ids_type_>>
                          && std::ranges::borrowed_range<std::ranges::range_reference_t<ref_ids_type_>>)
            {
                auto && id = header_ptr->ref_ids()[idx];
                header_ptr->ref_dict[std::span{std::ranges::data(id), std::ranges::size(id)}] = idx;
            }
            else
            {
                header_ptr->ref_dict[header_ptr->ref_ids()[idx]] = idx;
            }
        }
    }
 
    template <typename record_header_ptr_t, typename... pack_type>
    void write_record(record_header_ptr_t && record_header_ptr, pack_type &&... remainder)
    {
        static_assert((sizeof...(pack_type) == 13), "Wrong parameter list passed to write_record.");
 
        assert(!format.valueless_by_exception());
 
        std::visit(
            [&](auto & f)
            {
                // use header from record if explicitly given, e.g. file_output = file_input
                if constexpr (!std::same_as<record_header_ptr_t, std::nullptr_t>)
                {
                    f.write_alignment_record(*secondary_stream,
                                             options,
                                             *record_header_ptr,
                                             std::forward<pack_type>(remainder)...);
                }
                else if constexpr (std::same_as<ref_ids_type, ref_info_not_given>)
                {
                    f.write_alignment_record(*secondary_stream,
                                             options,
                                             std::ignore,
                                             std::forward<pack_type>(remainder)...);
                }
                else
                {
                    f.write_alignment_record(*secondary_stream,
                                             options,
                                             *header_ptr,
                                             std::forward<pack_type>(remainder)...);
                }
            },
            format);
 
        header_has_been_written = true; // when writing a record, the header is written automatically
    }
 
    friend iterator;
};
 
template <detail::fields_specialisation selected_field_ids>
sam_file_output(std::filesystem::path, selected_field_ids const &)
    -> sam_file_output<selected_field_ids, typename sam_file_output<>::valid_formats, ref_info_not_given>;
 
template <output_stream stream_type,
          sam_file_output_format file_format,
          detail::fields_specialisation selected_field_ids>
sam_file_output(stream_type &&, file_format const &, selected_field_ids const &)
    -> sam_file_output<selected_field_ids, type_list<file_format>, ref_info_not_given>;
 
template <output_stream stream_type,
          sam_file_output_format file_format,
          detail::fields_specialisation selected_field_ids>
sam_file_output(stream_type &, file_format const &, selected_field_ids const &)
    -> sam_file_output<selected_field_ids, type_list<file_format>, ref_info_not_given>;
 
template <output_stream stream_type, sam_file_output_format file_format>
sam_file_output(stream_type &&, file_format const &)
    -> sam_file_output<typename sam_file_output<>::selected_field_ids, type_list<file_format>, ref_info_not_given>;
 
template <output_stream stream_type, sam_file_output_format file_format>
sam_file_output(stream_type &, file_format const &)
    -> sam_file_output<typename sam_file_output<>::selected_field_ids, type_list<file_format>, ref_info_not_given>;
 
template <detail::fields_specialisation selected_field_ids,
          std::ranges::forward_range ref_ids_type,
          std::ranges::forward_range ref_lengths_type>
sam_file_output(std::filesystem::path const &, ref_ids_type &&, ref_lengths_type &&, selected_field_ids const &)
    -> sam_file_output<selected_field_ids,
                       typename sam_file_output<>::valid_formats,
                       std::remove_reference_t<ref_ids_type>>;
 
template <std::ranges::forward_range ref_ids_type, std::ranges::forward_range ref_lengths_type>
sam_file_output(std::filesystem::path const &, ref_ids_type &&, ref_lengths_type &&)
    -> sam_file_output<typename sam_file_output<>::selected_field_ids,
                       typename sam_file_output<>::valid_formats,
                       std::remove_reference_t<ref_ids_type>>;
 
template <output_stream stream_type,
          std::ranges::forward_range ref_ids_type,
          std::ranges::forward_range ref_lengths_type,
          sam_file_output_format file_format,
          detail::fields_specialisation selected_field_ids>
sam_file_output(stream_type &&, ref_ids_type &&, ref_lengths_type &&, file_format const &, selected_field_ids const &)
    -> sam_file_output<selected_field_ids, type_list<file_format>, std::remove_reference_t<ref_ids_type>>;
 
template <output_stream stream_type,
          std::ranges::forward_range ref_ids_type,
          std::ranges::forward_range ref_lengths_type,
          sam_file_output_format file_format,
          detail::fields_specialisation selected_field_ids>
sam_file_output(stream_type &, ref_ids_type &&, ref_lengths_type &&, file_format const &, selected_field_ids const &)
    -> sam_file_output<selected_field_ids, type_list<file_format>, std::remove_reference_t<ref_ids_type>>;
 
template <output_stream stream_type,
          std::ranges::forward_range ref_ids_type,
          std::ranges::forward_range ref_lengths_type,
          sam_file_output_format file_format>
sam_file_output(stream_type &&, ref_ids_type &&, ref_lengths_type &&, file_format const &)
    -> sam_file_output<typename sam_file_output<>::selected_field_ids,
                       type_list<file_format>,
                       std::remove_reference_t<ref_ids_type>>;
 
template <output_stream stream_type,
          std::ranges::forward_range ref_ids_type,
          std::ranges::forward_range ref_lengths_type,
          sam_file_output_format file_format>
sam_file_output(stream_type &, ref_ids_type &&, ref_lengths_type &&, file_format const &)
    -> sam_file_output<typename sam_file_output<>::selected_field_ids,
                       type_list<file_format>,
                       std::remove_reference_t<ref_ids_type>>;
 
} // namespace seqan3