Squashed commit of the following:

commit 70409de0e2af0a16a7136bab31086ef7eab04d9b
Author: Benjamin Buchfink <[email protected]>
Date:   Tue Jun 20 09:25:18 2023 +0200

    Updated version.

commit 4099a6d4e2d61fa0ca32e1ea2935bf7fda871601
Author: Benjamin Buchfink <[email protected]>
Date:   Mon Jun 19 17:03:26 2023 +0200

    Added alias taxid.

commit 38b77f968b2013944480e4c855be7ee8f6fe6c57
Author: Benjamin Buchfink <[email protected]>
Date:   Mon Jun 19 16:10:41 2023 +0200

    Added qlen for sam format.

commit 9db6be8ccd7293d501f4640cf317c72c0532c926
Author: Benjamin Buchfink <[email protected]>
Date:   Tue Jun 13 16:57:07 2023 +0200

    Delete iostream

commit f91883fac79b43c29989b3091dec6d981457164b
Author: Benjamin Buchfink <[email protected]>
Date:   Tue Jun 13 12:53:17 2023 +0200

    Use array for root.

commit 2d9667af766c8379b1f734fd6263ab8ece2efd2a
Author: Benjamin Buchfink <[email protected]>
Date:   Tue Jun 13 12:53:10 2023 +0200

    Use array as root.

commit 95023132611bf4dfa2667ea172a2aa5542a7cabe
Author: emile151 <[email protected]>
Date:   Tue Jun 13 12:25:34 2023 +0200

    Dev (#15)

    * json format reworked in blast_tab

    * json format reworked in blast_tab

    * json format reworked in blast_tab

    * json format reworked in blast_tab

    * json format reworked in blast_tab

    * json format reworked in blast_tab

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * json format reworked in blast_tab (comments fixed)

    * Merge conflcit resolved

    * json format reworked in blast_tab (comments fixed)

commit 608bc06a082c528eecdb2642205208906f1a756d
Author: Benjamin Buchfink <[email protected]>
Date:   Tue Jun 13 11:23:31 2023 +0200

    Added seed masking for --algo 1.

commit aac15b8ba2489c76c8810714ae572f3e3278a1d8
Author: Benjamin Buchfink <[email protected]>
Date:   Mon Jun 12 17:23:31 2023 +0200

    Filter low complex seeds in enum_seeds.

commit ddca6fa9cce1eb65d17893930d969b8bd6228e1c
Merge: 5c10a7dd 14f3550
Author: Benjamin Buchfink <[email protected]>
Date:   Wed Jun 7 09:33:45 2023 +0200

    Merge branch 'master' into dev

Author: bbuchfink

.gitignore

@@ -54,5 +54,5 @@ Makefile
 diamond
 src/extra/
 .vs/
-CMakeSettings.json
 .unison
+/CMakeSettings.json

src/ChangeLog

@@ -1,3 +1,9 @@
+[2.1.8]
+- Fixed an issue that could cause reduced performance when running in
+  query-indexed mode.
+- Added support for the JSON output format (option `-f json-flat`).
+- Added the option `--sam-query-len` to output query length in SAM format.
+
 [2.1.7]
 - Fixed a bug that caused taxonomy names not to be loaded correctly for the
   `makedb` workflow.

src/align/align.cpp

@@ -326,7 +326,7 @@ void align_queries(Consumer* output_file, Search::Config& cfg)
 
 		timer.go("Computing alignments");
 		HitIterator hit_it(query_range.first, query_range.second, hit_buf->data(), hit_buf->data() + hit_buf->size());
-		OutputWriter writer{ output_file };
+        OutputWriter writer{output_file, (*cfg.output_format == OutputFormat::json) ? ',' : char(0)};
 		output_sink.reset(new ReorderQueue<TextBuffer*, OutputWriter>(query_range.first, writer));
 		unique_ptr<thread> heartbeat;
 		if (config.verbosity >= 3 && config.load_balancing == Config::query_parallel && !config.swipe_all && config.heartbeat)

src/align/output.cpp

@@ -26,6 +26,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include "../output/daa/daa_write.h"
 #include "../util/sequence/sequence.h"
 
+
 using std::vector;
 
 namespace Extension {
@@ -51,7 +52,7 @@ TextBuffer* generate_output(vector<Match> &targets, const Extension::Stats& stat
 	}
 	else if (aligned || config.report_unaligned)
 		f->print_query_intro(info);
-			
+
 	for (int i = 0; i < (int)targets.size(); ++i) {
 
 		if (targets[i].hsp.empty())

src/basic/basic.cpp

@@ -29,7 +29,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include "../util/util.h"
 #include "../stats/standard_matrix.h"
 
-const char* Const::version_string = "2.1.7";
+const char* Const::version_string = "2.1.8";
 using std::string;
 using std::vector;
 using std::count;

src/basic/config.cpp

@@ -315,8 +315,9 @@ Config::Config(int argc, const char **argv, bool check_io, CommandLineParser& pa
 \t100 = DIAMOND alignment archive (DAA)\n\
 \t101 = SAM\n\
 \t102 = Taxonomic classification\n\
-\t103 = PAF\n\n\
-\tValue 6 may be followed by a space-separated list of these keywords:\n\n\
+\t103 = PAF\n\
+\t104 = JSON (flat)\n\n\
+\tValues 6 and 104 may be followed by a space-separated list of these keywords:\n\n\
 \tqseqid means Query Seq - id\n\
 \tqlen means Query sequence length\n\
 \tsseqid means Subject Seq - id\n\
@@ -422,11 +423,12 @@ Config::Config(int argc, const char **argv, bool check_io, CommandLineParser& pa
 		("multiprocessing", 0, "enable distributed-memory parallel processing", multiprocessing)
 		("mp-init", 0, "initialize multiprocessing run", mp_init)
 		("mp-recover", 0, "enable continuation of interrupted multiprocessing run", mp_recover)
-		("mp-query-chunk", 0, "process only a single query chunk as specified", mp_query_chunk, -1)		
+		("mp-query-chunk", 0, "process only a single query chunk as specified", mp_query_chunk, -1)
 		("culling-overlap", 0, "minimum range overlap with higher scoring hit to delete a hit (default=50%)", inner_culling_overlap, 50.0)
 		("taxon-k", 0, "maximum number of targets to report per species", taxon_k, (uint64_t)0)
-		("range-cover", 0, "percentage of query range to be covered for range culling (default=50%)", query_range_cover, 50.0)		
+		("range-cover", 0, "percentage of query range to be covered for range culling (default=50%)", query_range_cover, 50.0)
 		("xml-blord-format", 0, "Use gnl|BL_ORD_ID| style format in XML output", xml_blord_format)
+		("sam-query-len", 0, "add the query length to the SAM format (tag ZQ)", sam_qlen_field)
 		("stop-match-score", 0, "Set the match score of stop codons against each other.", stop_match_score, 1)		
 		("target-indexed", 0, "Enable target-indexed mode", target_indexed)
 		("unaligned-targets", 0, "", unaligned_targets)
@@ -669,7 +671,7 @@ Config::Config(int argc, const char **argv, bool check_io, CommandLineParser& pa
 
 	if (verbosity >= 1 || command == regression_test) {
 		ostream& header_out = command == Config::help ? cout : cerr;
-		header_out << Const::program_name << " v" << Const::version_string << "." << (unsigned)Const::build_version << " (C) Max Planck Society for the Advancement of Science" << endl;
+		header_out << Const::program_name << " v" << Const::version_string << "." << (unsigned)Const::build_version << " (C) Max Planck Society for the Advancement of Science, Benjamin Buchfink, University of Tuebingen" << endl;
 		header_out << "Documentation, support and updates available at http://www.diamondsearch.org" << endl;
 		header_out << "Please cite: http://dx.doi.org/10.1038/s41592-021-01101-x Nature Methods (2021)" << endl << endl;
 	}

src/basic/config.h

@@ -339,6 +339,7 @@ struct Config
     int zdrop;
 	bool heartbeat;
 	bool no_parse_seqids;
+	bool sam_qlen_field;
 
     SequenceType dbtype;
 

src/basic/const.h

@@ -25,7 +25,7 @@ struct Const
 {
 
 	enum {
-		build_version = 161,
+		build_version = 162,
 #ifdef SINGLE_THREADED
 		seedp_bits = 0,
 #else

src/basic/seed_iterator.h

@@ -159,31 +159,39 @@ struct SketchIterator
 template<uint64_t B>
 struct HashedSeedIterator
 {
-	HashedSeedIterator(const Sequence &seq, const Shape &sh):
-		ptr_(seq.data()),
-		end_(ptr_ + seq.length()),
+	HashedSeedIterator(Letter* seq, Loc len, const Shape &sh):
+		long_mask(sh.long_mask()),
+		ptr_(seq),
+		end_(ptr_ + len),
 		last_(0)
 	{
-		for (int i = 0; (i < sh.length_ - 1) && ptr_ < end_; ++i) {
+		for (int i = 0; i < sh.length_ && ptr_ < end_; ++i)
 			last_ = (last_ << B) | Reduction::reduction(letter_mask(*(ptr_++)));
-		}
 	}
 	bool good() const
 	{
-		return ptr_ < end_;
+		return ptr_ <= end_;
 	}
-	bool get(uint64_t &seed, uint64_t mask)
-	{
-		last_ <<= B;
-		const Letter l = letter_mask(*(ptr_++));
-		if (!is_amino_acid(l))
-			return false;
-		last_ |= Reduction::reduction(l);
-		seed = MurmurHash()(last_ & mask);
-		return true;
+	uint64_t operator*() const {
+		return MurmurHash()(last_ & long_mask);
+	}
+	HashedSeedIterator& operator++() {
+		while (ptr_ < end_) {
+			last_ <<= B;
+			const Letter l = letter_mask(*(ptr_++));
+			if (!is_amino_acid(l))
+				continue;
+			last_ |= Reduction::reduction(l);
+			return *this;
+		}
+		++ptr_;
+	}
+	Letter* seq_ptr(const Shape& sh) const {
+		return ptr_ - sh.length_;
 	}
 private:
-	const Letter *ptr_, *end_;
+	const uint64_t long_mask;
+	Letter *ptr_, *end_;
 	uint64_t last_;
 };
 

src/data/enum_seeds.h

@@ -71,7 +71,6 @@ Search::SeedStats enum_seeds_minimizer(SequenceSet* seqs, F* f, unsigned begin,
 template<typename F, uint64_t BITS, typename Filter>
 void enum_seeds_hashed(SequenceSet* seqs, F* f, unsigned begin, unsigned end, const Filter* filter, const EnumCfg& cfg)
 {
-	uint64_t key;
 	for (unsigned i = begin; i < end; ++i) {
 		if (cfg.skip && (*cfg.skip)[i / align_mode.query_contexts])
 			continue;
@@ -80,16 +79,16 @@ void enum_seeds_hashed(SequenceSet* seqs, F* f, unsigned begin, unsigned end, co
 		for (size_t shape_id = cfg.shape_begin; shape_id < cfg.shape_end; ++shape_id) {
 			const Shape& sh = shapes[shape_id];
 			if (seq.length() < sh.length_) continue;
-			const uint64_t shape_mask = sh.long_mask();
 			//const __m128i shape_mask = sh.long_mask_sse_;
-			HashedSeedIterator<BITS> it(seq, sh);
-			Loc j = 0;
+			HashedSeedIterator<BITS> it(seqs->ptr(i), seqs->length(i), sh);
 			while (it.good()) {
-				if (it.get(key, shape_mask)) {
-					if (filter->contains(key, shape_id))
-						(*f)(key, seqs->position(i, j), i, shape_id);
-				}
-				++j;
+				const uint64_t key = *it;
+				if (filter->contains(key, shape_id))
+					if (!cfg.filter_low_complexity_seeds || Search::seed_is_complex(it.seq_ptr(sh), sh, cfg.seed_cut))
+						(*f)(key, seqs->position(i, it.seq_ptr(sh) - seq.data()), i, shape_id);
+					else if (cfg.mask_low_complexity_seeds)
+						*it.seq_ptr(sh) |= SEED_MASK;
+				++it;
 			}
 		}
 	}