Experimental plasmid support

Author: lmrodriguezr

lib/miga/cli/objects_helper.rb

@@ -60,6 +60,9 @@ def load_and_filter_datasets(silent = false)
       o &&= (d.ref? == self[:ref]) unless self[:ref].nil?
       o &&= (d.active? == self[:active]) unless self[:active].nil?
       o &&= (self[:multi] ? d.multi? : d.nonmulti?) unless self[:multi].nil?
+      unless self[:markers].nil?
+        o &&= (self[:markers] ? d.markers? : !d.markers?)
+      end
       unless self[:taxonomy].nil?
         o &&= !d.metadata[:tax].nil? && d.metadata[:tax].in?(self[:taxonomy])
       end

lib/miga/cli/opt_helper.rb

@@ -120,10 +120,11 @@ def opt_object(opt, what = %i[project dataset])
   # as determined by +what+ an Array with any combination of:
   # - :ref To filter by reference (--ref) or query (--no-ref)
   # - :multi To filter by multiple (--multi) or single (--no-multi) species
+  # - :markers To filter by with (--markers) or without markers (--no-markers)
   # - :active To filter by active (--active) or inactive (--no-active)
   # - :taxonomy To filter by taxonomy (--taxonomy)
   # The "k-th" filter (--dataset-k) is always included
-  def opt_filter_datasets(opt, what = %i[ref multi active taxonomy])
+  def opt_filter_datasets(opt, what = %i[ref multi markers active taxonomy])
     what.each do |w|
       case w
       when :ref
@@ -136,6 +137,11 @@ def opt_filter_datasets(opt, what = %i[ref multi active taxonomy])
           '--[no-]multi',
           'Use only multi-species (or only single-species) datasets'
         ) { |v| self[:multi] = v }
+      when :markers
+        opt.on(
+          '--[no-]markers',
+          'Use only datasets with (or without) markers'
+        ) { |v| self[:markers] = v }
       when :active
         opt.on(
           '--[no-]active',

lib/miga/dataset.rb

@@ -6,6 +6,7 @@
 require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
+require 'miga/dataset/type'
 require 'miga/dataset/hooks'
 
 # This library is only required by +#closest_relatives+, so it is now
@@ -18,6 +19,7 @@
 class MiGA::Dataset < MiGA::MiGA
   include MiGA::Dataset::Result
   include MiGA::Dataset::Status
+  include MiGA::Dataset::Type
   include MiGA::Dataset::Hooks
 
   # Class-level
@@ -56,6 +58,7 @@ def initialize(project, name, is_ref = true, metadata = {})
             name.to_s
     @project, @name, @metadata = project, name, nil
     metadata[:ref] = is_ref
+    metadata[:type] ||= :empty
     @metadata_future = [
       File.join(project.path, 'metadata', "#{name}.json"),
       metadata
@@ -89,12 +92,6 @@ def save
   # +Project+ interface
   alias :save! :save
 
-  ##
-  # Get the type of dataset as Symbol
-  def type
-    metadata[:type]
-  end
-
   ##
   # Delete the dataset with all it's contents (including results) and returns
   # nil
@@ -146,22 +143,6 @@ def query?
     !metadata[:ref]
   end
 
-  ##
-  # Is this dataset known to be multi-organism?
-  def multi?
-    return false if metadata[:type].nil? || @@KNOWN_TYPES[type].nil?
-
-    @@KNOWN_TYPES[type][:multi]
-  end
-
-  ##
-  # Is this dataset known to be single-organism?
-  def nonmulti?
-    return false if metadata[:type].nil? || @@KNOWN_TYPES[type].nil?
-
-    !@@KNOWN_TYPES[type][:multi]
-  end
-
   ##
   # Is this dataset active?
   def active?

lib/miga/dataset/base.rb

@@ -32,6 +32,12 @@ def EXCLUDE_NOREF_TASKS
       @@EXCLUDE_NOREF_TASKS
     end
 
+    ##
+    # Tasks to be excluded from datasets without markers
+    def EXCLUDE_NOMARKER_TASKS
+      @@EXCLUDE_NOMARKER_TASKS
+    end
+
     ##
     # Tasks to be executed only in datasets that are single-organism. These
     # tasks are ignored for multi-organism datasets or for unknown types
@@ -81,45 +87,67 @@ module MiGA::Dataset::Base
   # Supported dataset types
   @@KNOWN_TYPES = {
     genome: {
-      description: 'The genome from an isolate', multi: false
+      description: 'The genome from an isolate',
+      multi: false, markers: true,
+      project_types: %i[mixed genomes clade]
     },
     scgenome: {
-      description: 'A Single-cell Amplified Genome (SAG)', multi: false
+      description: 'A Single-cell Amplified Genome (SAG)',
+      multi: false, markers: true,
+      project_types: %i[mixed genomes clade]
     },
     popgenome: {
-      description: 'A Metagenome-Assembled Genome (MAG)', multi: false
+      description: 'A Metagenome-Assembled Genome (MAG)',
+      multi: false, markers: true,
+      project_types: %i[mixed genomes clade]
     },
     metagenome: {
-      description: 'A metagenome (excluding viromes)', multi: true
+      description: 'A metagenome (excluding viromes)',
+      multi: true, markers: true,
+      project_types: %i[mixed metagenomes]
     },
     virome: {
-      description: 'A viral metagenome', multi: true
+      description: 'A viral metagenome',
+      multi: true,
+      markers: true, # <- We don't expect, but can be useful for contamination
+      project_types: %i[mixed metagenomes]
+    },
+    plasmid: {
+      description: 'An individual plasmid',
+      multi: false, markers: false,
+      project_types: %i[mixed plasmids]
     }
   }
 
   ##
   # Returns an Array of tasks (Symbols) to be executed before project-wide tasks
-  @@PREPROCESSING_TASKS = [
-    :raw_reads, :trimmed_reads, :read_quality, :trimmed_fasta,
-    :assembly, :cds, :essential_genes, :mytaxa, :mytaxa_scan,
-    :taxonomy, :distances, :ssu, :stats
+  @@PREPROCESSING_TASKS = %i[
+    raw_reads trimmed_reads read_quality trimmed_fasta
+    assembly cds essential_genes mytaxa mytaxa_scan
+    taxonomy distances ssu stats
   ]
 
   ##
   # Tasks to be excluded from query datasets
-  @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan, :taxonomy]
+  @@EXCLUDE_NOREF_TASKS = %i[mytaxa_scan taxonomy]
   @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map { |i| [i, true] }]
 
+  ##
+  # Tasks to be excluded from datasets without markers
+  @@EXCLUDE_NOMARKER_TASKS = %i[essential_genes ssu]
+  @@_EXCLUDE_NOMARKER_TASKS_H =
+    Hash[@@EXCLUDE_NOMARKER_TASKS.map { |i| [i, true] }]
+
   ##
   # Tasks to be executed only in datasets that are single-organism. These
   # tasks are ignored for multi-organism datasets or for unknown types
-  @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :taxonomy, :distances]
+  @@ONLY_NONMULTI_TASKS = %i[mytaxa_scan taxonomy distances]
   @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map { |i| [i, true] }]
 
   ##
   # Tasks to be executed only in datasets that are multi-organism. These
   # tasks are ignored for single-organism datasets or for unknwon types
-  @@ONLY_MULTI_TASKS = [:mytaxa]
+  @@ONLY_MULTI_TASKS = %i[mytaxa]
   @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map { |i| [i, true] }]
 
   ##

lib/miga/dataset/hooks.rb

@@ -15,6 +15,7 @@
 # Supported hooks:
 # - run_lambda(lambda, args...)
 # - recalculate_status()
+# - check_type()
 # - clear_run_counts()
 # - run_cmd(cmd)
 # Internal hooks:
@@ -27,6 +28,7 @@ module MiGA::Dataset::Hooks
   def default_hooks
     {
       on_create: [[:recalculate_status]],
+      on_save: [[:check_type]],
       on_activate: [[:clear_run_counts], [:recalculate_status]],
       on_inactivate: [[:recalculate_status]],
       on_result_ready: [[:_pull_result_hooks]],
@@ -51,6 +53,12 @@ def hook_recalculate_status(_hook_args, _event_args)
     recalculate_status
   end
 
+  ##
+  # Ensure that the dataset type exists and is compatible with the project type
+  def hook_check_type(_hook_args, _event_args)
+    check_type
+  end
+
   ##
   # Run +cmd+ in the command-line with {{variables}}:
   # dataset, project, project_name, miga, object (if defined for the event)

lib/miga/dataset/result/ignore.rb

@@ -17,10 +17,14 @@ def ignore_task?(task)
   # - project: incompatible project
   # - noref: incompatible dataset, only for reference
   # - multi: incompatible dataset, only for multi
+  # - nomarkers: incompatible dataset, only for markers
   # - nonmulti: incompatible dataset, only for nonmulti
   # - complete: the task is already complete
   def ignore_reasons
-    %i[empty inactive upstream force project noref multi nonmulti complete]
+    %i[
+      empty inactive upstream force project
+      noref multi nonmulti nomarkers complete
+    ]
   end
 
   ##
@@ -91,9 +95,15 @@ def ignore_nonmulti?(task)
     ignore_by_type?(task, :nonmulti)
   end
 
+  ##
+  # Ignore +task+ because it's not a markers dataset
+  def ignore_nomarkers?(task)
+    ignore_by_type?(task, :nomarkers)
+  end
+
   ##
   # Ignore +task+ by +type+ of dataset, one of: +:noref+, +:multi+, or
-  # +:nonmulti+
+  # +:nonmulti+, +:nomarkers+
   def ignore_by_type?(task, type)
     return false if force_task?(task)
 
@@ -105,6 +115,8 @@ def ignore_by_type?(task, type)
         [:multi?, self.class.ONLY_MULTI_TASKS]
       when :nonmulti
         [:nonmulti?, self.class.ONLY_NONMULTI_TASKS]
+      when :nomarkers
+        [:markers?, self.class.EXCLUDE_NOMARKER_TASKS]
       else
         raise "Unexpected error, unknown type reason: #{type}"
       end

lib/miga/dataset/type.rb

@@ -0,0 +1,51 @@
+##
+# Helper module including specific functions for dataset type
+module MiGA::Dataset::Type
+  ##
+  # Get the type of dataset as Symbol
+  def type
+    metadata[:type]
+  end
+
+  ##
+  # Is this dataset known to be multi-organism?
+  def multi?
+    self.class.KNOWN_TYPES.dig(type, :multi)
+  end
+
+  ##
+  # Is this dataset known to be single-organism?
+  def nonmulti?
+    y = self.class.KNOWN_TYPES.dig(type, :multi)
+    y.nil? ? nil : !y
+  end
+
+  ##
+  # Are universal marker genes expected to be found in this dataset?
+  def markers?
+    self.class.KNOWN_TYPES.dig(type, :markers)
+  end
+
+  ##
+  # Check that the dataset type is defined, known, and compatible with the
+  # project type and raise an exception if any of these checks fail
+  #
+  # If the dataset type is +:empty+, it returns +false+ without raising an
+  # exception, and true otherwise (and no tests are failed)
+  def check_type
+    raise MiGA::Error.new('Undefined dataset type') unless type
+    return false if type == :empty
+
+    unless self.class.KNOWN_TYPES[type]
+      raise MiGA::Error.new("Unknown dataset type: #{type}")
+    end
+    unless self.class.KNOWN_TYPES[type][:project_types].include? project.type
+      raise MiGA::Error.new(
+        "Dataset type (#{type}) incompatible with project (#{project.type})"
+      )
+    end
+
+    true
+  end
+
+end

lib/miga/project.rb

@@ -98,7 +98,7 @@ def type
   ##
   # Is this a clade project?
   def clade?
-    type == :clade
+    %i[clade plasmids].include? type
   end
 
   ##
@@ -115,6 +115,12 @@ def multi?
   # Same as multi? For backward compatibility
   alias is_multi? multi?
 
+  ##
+  # Does the project support the use of universal markers?
+  def markers?
+    @@KNOWN_TYPES[type][:markers]
+  end
+
   ##
   # Is this project active? Currently a dummy function, returns
   # always true.

lib/miga/project/base.rb

@@ -89,32 +89,36 @@ module MiGA::Project::Base
   @@KNOWN_TYPES = {
     mixed: {
       description: 'Mixed collection of genomes, metagenomes, and viromes',
-      single: true, multi: true
+      single: true, multi: true, markers: true
     },
     genomes: {
       description: 'Collection of genomes',
-      single: true, multi: false
+      single: true, multi: false, markers: true
     },
     clade: {
       description: 'Collection of closely-related genomes (ANI >= 90%)',
-      single: true, multi: false
+      single: true, multi: false, markers: true
     },
     metagenomes: {
       description: 'Collection of metagenomes and/or viromes',
-      single: false, multi: true
+      single: false, multi: true, markers: true
+    },
+    plasmids: {
+      description: 'Collection of plasmids',
+      single: true, multi: false, markers: false
     }
   }
 
   ##
   # Project-wide distance estimations
-  @@DISTANCE_TASKS = [
-    :project_stats, :haai_distances, :aai_distances, :ani_distances,
-    :clade_finding
+  @@DISTANCE_TASKS = %i[
+    project_stats haai_distances aai_distances ani_distances
+    clade_finding
   ]
 
   ##
   # Project-wide tasks for :clade projects
-  @@INCLADE_TASKS = [:subclades, :ogs]
+  @@INCLADE_TASKS = %i[subclades ogs]
 
   ##
   # Options supported by projects
@@ -131,7 +135,9 @@ module MiGA::Project::Base
     },
     haai_p: {
       desc: 'Value of aai.rb -p on hAAI', type: String,
-      default: proc { |project| project.clade? ? 'no' : 'fastaai' },
+      default: proc { |project|
+        project.clade? || !project.markers? ? 'no' : 'fastaai'
+      },
       in: %w[blast+ blast blat diamond fastaai no]
     },
     aai_p: {