Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Re nest parallelize #27

Merged
merged 5 commits into from
Nov 7, 2024
Merged

Re nest parallelize #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8caec48
Re-nest stuff for parallelization
Oddant1 Aug 22, 2024
18ff90f
Style
Oddant1 Aug 22, 2024
aba84b8
Shouldn't get rid of that lol
Oddant1 Aug 22, 2024
5129e25
lint
Oddant1 Aug 22, 2024
70c7a83
More modifications
Oddant1 Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 25 additions & 35 deletions q2_boots/_alpha.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,8 @@

import pandas as pd

from q2_diversity import (alpha as q2div_alpha,
alpha_phylogenetic as q2div_alpha_phylogenetic)
from q2_diversity_lib.alpha import METRICS

from q2_boots._resample import resample


def alpha_average(data: pd.Series, average_method: str) -> pd.Series:
if average_method == "median":
Expand All @@ -35,32 +31,28 @@ def alpha_collection(ctx, table, sampling_depth, metric, n,
replacement, phylogeny=None):
_validate_alpha_metric(metric, phylogeny)

resample_function = resample
alpha_metric_function = _get_alpha_metric_function(ctx, metric, phylogeny)

tables = resample_function(ctx=ctx,
table=table,
sampling_depth=sampling_depth,
n=n,
replacement=replacement)
resample_action = ctx.get_action("boots", "resample")
alpha_metric_action = _get_alpha_metric_action(ctx, metric, phylogeny)

results = _alpha_collection_from_tables(tables, alpha_metric_function)
tables, = resample_action(table=table,
sampling_depth=sampling_depth,
n=n,
replacement=replacement)

results = _alpha_collection_from_tables(tables, alpha_metric_action)
return results


def alpha(ctx, table, sampling_depth, metric, n, replacement, phylogeny=None,
average_method='median'):

alpha_collection_function = alpha_collection
alpha_collection_action = ctx.get_action("boots", "alpha_collection")
alpha_average_action = ctx.get_action('boots', 'alpha_average')
sample_data = alpha_collection_function(ctx=ctx,
table=table,
sampling_depth=sampling_depth,
phylogeny=phylogeny,
metric=metric,
n=n,
replacement=replacement)
sample_data, = alpha_collection_action(table=table,
sampling_depth=sampling_depth,
phylogeny=phylogeny,
metric=metric,
n=n,
replacement=replacement)

result, = alpha_average_action(sample_data, average_method)
return result
Expand All @@ -71,27 +63,25 @@ def _validate_alpha_metric(metric, phylogeny):
raise ValueError(f'Metric {metric} requires a phylogenetic tree.')


def _get_alpha_metric_function(ctx, metric, phylogeny):
def _get_alpha_metric_action(ctx, metric, phylogeny):
if _is_phylogenetic_alpha_metric(metric):
alpha_metric_function = q2div_alpha_phylogenetic
alpha_metric_function = functools.partial(alpha_metric_function,
ctx=ctx,
phylogeny=phylogeny,
metric=metric)
alpha_metric_action = ctx.get_action("diversity", "alpha_phylogenetic")
alpha_metric_action = functools.partial(alpha_metric_action,
phylogeny=phylogeny,
metric=metric)
else:
alpha_metric_function = q2div_alpha
alpha_metric_function = functools.partial(alpha_metric_function,
ctx=ctx,
metric=metric)
return alpha_metric_function
alpha_metric_action = ctx.get_action("diversity", "alpha")
alpha_metric_action = functools.partial(alpha_metric_action,
metric=metric)
return alpha_metric_action


def _is_phylogenetic_alpha_metric(metric):
return metric in (METRICS['PHYLO']['IMPL'] | METRICS['PHYLO']['UNIMPL'])


def _alpha_collection_from_tables(tables, alpha_metric_function):
def _alpha_collection_from_tables(tables, alpha_metric_action):
results = []
for table in tables.values():
results.append(alpha_metric_function(table=table))
results.append(alpha_metric_action(table=table)[0])
return results
69 changes: 33 additions & 36 deletions q2_boots/_beta.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,8 @@
import skbio
from hdmedians import medoid

from q2_diversity import (beta as q2div_beta,
beta_phylogenetic as q2div_beta_phylogenetic)
from q2_diversity_lib.beta import METRICS

from q2_boots._resample import resample

_METRIC_MOD_DEFAULTS = {
'bypass_tips': False,
'pseudocount': 1,
Expand Down Expand Up @@ -58,17 +54,16 @@ def beta_collection(
variance_adjusted=_METRIC_MOD_DEFAULTS['variance_adjusted']):
_validate_beta_metric(metric, phylogeny)

resample_function = resample
beta_metric_function = _get_beta_metric_function(
resample_action = ctx.get_action("boots", "resample")
beta_metric_action = _get_beta_metric_action(
ctx, metric, phylogeny, bypass_tips, pseudocount, alpha,
variance_adjusted)

tables = resample_function(ctx=ctx,
table=table,
sampling_depth=sampling_depth,
n=n,
replacement=replacement)
results = _beta_collection_from_tables(tables, beta_metric_function)
tables, = resample_action(table=table,
sampling_depth=sampling_depth,
n=n,
replacement=replacement)
results = _beta_collection_from_tables(tables, beta_metric_action)

return results

Expand All @@ -79,19 +74,18 @@ def beta(ctx, table, metric, sampling_depth, n, replacement,
pseudocount=_METRIC_MOD_DEFAULTS['pseudocount'],
alpha=_METRIC_MOD_DEFAULTS['alpha'],
variance_adjusted=_METRIC_MOD_DEFAULTS['variance_adjusted']):
beta_collection_function = beta_collection
beta_collection_action = ctx.get_action('boots', 'beta_collection')
beta_average_action = ctx.get_action('boots', 'beta_average')
dms = beta_collection_function(ctx=ctx,
table=table,
phylogeny=phylogeny,
metric=metric,
sampling_depth=sampling_depth,
n=n,
pseudocount=pseudocount,
replacement=replacement,
variance_adjusted=variance_adjusted,
alpha=alpha,
bypass_tips=bypass_tips)
dms, = beta_collection_action(table=table,
phylogeny=phylogeny,
metric=metric,
sampling_depth=sampling_depth,
n=n,
pseudocount=pseudocount,
replacement=replacement,
variance_adjusted=variance_adjusted,
alpha=alpha,
bypass_tips=bypass_tips)

result, = beta_average_action(dms, average_method)
return result
Expand Down Expand Up @@ -129,32 +123,35 @@ def _validate_beta_metric(metric, phylogeny):
raise ValueError(f'Metric {metric} requires a phylogenetic tree.')


def _get_beta_metric_function(
def _get_beta_metric_action(
ctx, metric, phylogeny,
bypass_tips=_METRIC_MOD_DEFAULTS['bypass_tips'],
pseudocount=_METRIC_MOD_DEFAULTS['pseudocount'],
alpha=_METRIC_MOD_DEFAULTS['alpha'],
variance_adjusted=_METRIC_MOD_DEFAULTS['variance_adjusted']):
if _is_phylogenetic_beta_metric(metric):
beta_metric_function = q2div_beta_phylogenetic
beta_metric_function = functools.partial(
beta_metric_function, ctx=ctx, phylogeny=phylogeny, metric=metric,
bypass_tips=bypass_tips, alpha=alpha,
beta_metric_action = ctx.get_action("diversity", "beta_phylogenetic")
beta_metric_action = functools.partial(
beta_metric_action,
phylogeny=phylogeny,
metric=metric,
bypass_tips=bypass_tips,
alpha=alpha,
variance_adjusted=variance_adjusted)
else:
beta_metric_function = q2div_beta
beta_metric_function = functools.partial(
beta_metric_function, ctx=ctx, metric=metric,
pseudocount=pseudocount)
return beta_metric_function
beta_metric_action = ctx.get_action("diversity", "beta")
beta_metric_action = functools.partial(beta_metric_action,
metric=metric,
pseudocount=pseudocount)
return beta_metric_action


def _is_phylogenetic_beta_metric(metric):
return metric in METRICS['PHYLO']['IMPL'] | METRICS['PHYLO']['UNIMPL']


def _beta_collection_from_tables(tables, beta_metric_function):
def _beta_collection_from_tables(tables, beta_metric_action):
results = []
for table in tables.values():
results.append(beta_metric_function(table=table))
results.append(beta_metric_action(table=table)[0])
return results
31 changes: 14 additions & 17 deletions q2_boots/_core_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,17 @@
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

from q2_boots._resample import resample
from q2_boots._alpha import (_validate_alpha_metric,
_get_alpha_metric_function,
from q2_boots._alpha import (_validate_alpha_metric, _get_alpha_metric_action,
_alpha_collection_from_tables)
from q2_boots._beta import (_validate_beta_metric,
_get_beta_metric_function,
from q2_boots._beta import (_validate_beta_metric, _get_beta_metric_action,
_beta_collection_from_tables)


def core_metrics(ctx, table, sampling_depth, metadata, n, replacement,
phylogeny=None, alpha_average_method='median',
beta_average_method='non-metric-median'):

resample_function = resample
resample_action = ctx.get_action('boots', 'resample')
alpha_average_action = ctx.get_action('boots', 'alpha_average')
beta_average_action = ctx.get_action('boots', 'beta_average')
pcoa_action = ctx.get_action('diversity', 'pcoa')
Expand All @@ -37,28 +34,27 @@ def core_metrics(ctx, table, sampling_depth, metadata, n, replacement,
for beta_metric in beta_metrics:
_validate_beta_metric(beta_metric, phylogeny)

resampled_tables = resample_function(ctx=ctx,
table=table,
sampling_depth=sampling_depth,
n=n,
replacement=replacement)
resampled_tables, = resample_action(table=table,
sampling_depth=sampling_depth,
n=n,
replacement=replacement)

alpha_vectors = {}
for alpha_metric in alpha_metrics:
alpha_metric_function = _get_alpha_metric_function(
alpha_metric_action = _get_alpha_metric_action(
ctx, alpha_metric, phylogeny)
alpha_collection = _alpha_collection_from_tables(
resampled_tables, alpha_metric_function)
resampled_tables, alpha_metric_action)
avg_alpha_vector, = alpha_average_action(
alpha_collection, alpha_average_method)
alpha_vectors[alpha_metric] = avg_alpha_vector

beta_dms = {}
for beta_metric in beta_metrics:
beta_metric_function = _get_beta_metric_function(
beta_metric_action = _get_beta_metric_action(
ctx, beta_metric, phylogeny)
beta_collection = _beta_collection_from_tables(
resampled_tables, beta_metric_function)
resampled_tables, beta_metric_action)
avg_beta_dm, = beta_average_action(
beta_collection, beta_average_method)
beta_dms[beta_metric] = avg_beta_dm
Expand All @@ -68,7 +64,8 @@ def core_metrics(ctx, table, sampling_depth, metadata, n, replacement,
for key, dm in beta_dms.items():
pcoa_results, = pcoa_action(dm)
pcoas[key] = pcoa_results
emperor_plots[key] = emperor_plot_action(pcoa=pcoa_results,
metadata=metadata)[0]
emperor_plot, = emperor_plot_action(pcoa=pcoa_results,
metadata=metadata)
emperor_plots[key] = emperor_plot

return resampled_tables, alpha_vectors, beta_dms, pcoas, emperor_plots
6 changes: 3 additions & 3 deletions q2_boots/_resample.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ def resample(ctx, table, sampling_depth, n, replacement):
resampled_tables = []

for i in range(n):
resampled_table = rarefy_action(table=table,
sampling_depth=sampling_depth,
with_replacement=replacement)[0]
resampled_table, = rarefy_action(table=table,
sampling_depth=sampling_depth,
with_replacement=replacement)
resampled_tables.append(resampled_table)

return {f'resampled-table-{i}': t for i, t in enumerate(resampled_tables)}
Loading