Skip to content

Commit

Permalink
Merge pull request #93 from elixir-europe/multi-auth-provider-credent…
Browse files Browse the repository at this point in the history
…ials

Multi auth provider credentials
  • Loading branch information
kdp-cloud authored Jan 2, 2025
2 parents 08cc1ba + dc51e2d commit 8ee3540
Show file tree
Hide file tree
Showing 12 changed files with 256 additions and 59 deletions.
7 changes: 5 additions & 2 deletions .github/workflows/test-mars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
python-version: ["3.9", "3.10", "3.11", "3.12"]
python-version: ["3.9", "3.13"]
runs-on: ${{ matrix.os }}
env:
working-directory: ./mars-cli
Expand Down Expand Up @@ -48,6 +48,9 @@ jobs:
run: ruff check mars_lib/
working-directory: ${{ env.working-directory }}

- name: Create mypy cache directory
run: mkdir -p /tmp/mypy_cache

- name: Type checking
run: mypy --install-types --non-interactive mars_lib/
run: mypy --install-types --non-interactive --cache-dir /tmp/mypy_cache mars_lib/
working-directory: ${{ env.working-directory }}
46 changes: 32 additions & 14 deletions mars-cli/mars_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,22 @@ def cli(ctx, development):

@cli.command()
@click.option(
"--credential-service-name", type=click.STRING, help="service name from the keyring"
"--webin-username",
type=click.STRING,
help="Username for webin authentication",
envvar="WEBIN_USERNAME",
)
@click.option(
"--username-credentials", type=click.STRING, help="Username from the keyring"
"--metabolights-username",
type=click.STRING,
help="Username for MetaboLights metadata submission",
envvar="METABOLIGHTS_USERNAME",
)
@click.option(
"--metabolights-ftp-username",
type=click.STRING,
help="Username for MetaboLights data submission",
envvar="METABOLIGHTS_FTP_USERNAME",
)
@click.option(
"--credentials-file",
Expand All @@ -218,8 +230,10 @@ def cli(ctx, development):
@click.option("--submit-to-ena", type=click.BOOL, default=True, help="Submit to ENA.")
@click.option(
"--file-transfer",
type=click.STRING,
help="provide the name of a file transfer solution, like ftp or aspera",
type=click.Choice(["ftp", "aspera"], case_sensitive=False),
required=True,
default="ftp",
help="provide the name of a file transfer solution, like ftp or aspera. The default is ftp.",
)
@click.option(
"--data-files",
Expand Down Expand Up @@ -247,8 +261,9 @@ def cli(ctx, development):
@click.pass_context
def submit(
ctx,
credential_service_name,
username_credentials,
webin_username,
metabolights_username,
metabolights_ftp_username,
credentials_file,
isa_json_file,
submit_to_biosamples,
Expand Down Expand Up @@ -280,8 +295,9 @@ def submit(
data_file_paths = [f.name for f in data_files] if file_transfer else []

submission(
credential_service_name,
username_credentials,
webin_username,
metabolights_username,
metabolights_ftp_username,
credentials_file,
isa_json_file.name,
target_repositories,
Expand Down Expand Up @@ -361,12 +377,14 @@ def validate_isa_json(isa_json_file, investigation_is_root, validation_schema):

@cli.command()
@click.option(
"--service-name",
type=click.STRING,
"--auth-provider",
type=click.Choice(
["webin", "metabolights_metadata", "metabolights_data"], case_sensitive=False
),
is_flag=False,
flag_value="value",
default=f"mars-cli_{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}",
help='You are advised to include service name to match the credentials to. If empty, it defaults to "mars-cli_{DATESTAMP}"',
required=True,
help="",
)
@click.argument(
"username",
Expand All @@ -380,9 +398,9 @@ def validate_isa_json(isa_json_file, investigation_is_root, validation_schema):
confirmation_prompt=True,
help="The password to store. Note: You are required to confirm the password.",
)
def set_password(service_name, username, password):
def set_password(auth_provider, username, password):
"""Store a password in the keyring."""
CredentialManager(service_name).set_password_keyring(username, password)
CredentialManager(auth_provider).set_password_keyring(username, password)


if __name__ == "__main__":
Expand Down
73 changes: 67 additions & 6 deletions mars-cli/mars_lib/authentication.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,68 @@
from typing import Optional
import io
from typing import Optional, Union
import requests
import json
from enum import Enum


class AuthProvider(Enum):
"""
Holds constants, tied to the repository authentication providers.
"""

WEBIN = "webin"
METABOLIGHTS_METADATA = "metabolights_metadata"
METABOLIGHTS_DATA = "metabolights_data"

@classmethod
def available_providers(cls):
return {item.value for item in cls}

@classmethod
def is_valid_provider(cls, provider: str):
return provider in cls.available_providers()


def load_credentials(
credentials_file: Union[io.TextIOWrapper, str]
) -> dict[str, dict[str, str]]:
"""
Validate the credentials.
Args:
credentials_file (_): The credentials in file formate.
Raises:
ValueError: If the credentials are not valid.
Returns:
dict: The credentials.
"""
if isinstance(credentials_file, str):
with open(credentials_file, "r") as file:
credentials = json.load(file)
elif isinstance(credentials_file, io.TextIOWrapper):
with open(credentials_file.name, "r") as file:
credentials = json.load(file)
else:
raise TypeError("Credentials file must be of type str or io.TextIOWrapper.")

if not all(
repo in AuthProvider.available_providers() for repo in credentials.keys()
):
raise ValueError(
f"Credentials dictionary must have valid keys. Valid keys are:\n{AuthProvider.available_providers()}"
)

if not all(
key in ["username", "password"]
for repo, creds in credentials.items()
for key in creds.keys()
):
raise ValueError(
"Credentials dictionary must contain 'username' and 'password' keys."
)
return credentials


def get_webin_auth_token(
Expand Down Expand Up @@ -45,7 +107,10 @@ def get_webin_auth_token(

def get_metabolights_auth_token(
credentials_dict: dict[str, str],
headers: dict[str, str] = {"Content-Type": "application/x-www-form-urlencoded"},
headers: dict[str, str] = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "application/json",
},
auth_url: str = "https://www-test.ebi.ac.uk/metabolights/mars/ws3/auth/token",
) -> Optional[str]:
"""
Expand All @@ -59,10 +124,6 @@ def get_metabolights_auth_token(
Returns:
str: The obtained token.
"""
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "application/json",
}
form_data = f'grant_type=password&username={credentials_dict["username"]}&password={credentials_dict["password"]}'
try:
response = requests.post(
Expand Down
9 changes: 7 additions & 2 deletions mars-cli/mars_lib/credential.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import getpass
import keyring.util.platform_ as keyring_platform

from mars_lib.authentication import AuthProvider

"""
Credential Manager Module
=========================
Expand Down Expand Up @@ -52,8 +54,11 @@


class CredentialManager:
def __init__(self, service_name: str) -> None:
self.service_name = service_name
def __init__(self, auth_provider: str) -> None:
if not AuthProvider.is_valid_provider(auth_provider):
raise ValueError(f"Invalid authentication provider: {auth_provider}")

self.service_name = auth_provider

def get_credential_env(self, username: str) -> str:
"""
Expand Down
4 changes: 2 additions & 2 deletions mars-cli/mars_lib/models/isa_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ class MaterialAttribute(IsaBase):


class Study(CommentedIsaBase):
id: str = Field(alias="@id", default=None)
id: Optional[str] = Field(alias="@id", default=None)
assays: List[Assay] = []
characteristicCategories: List[MaterialAttribute] = []
description: Optional[str] = None
Expand Down Expand Up @@ -284,7 +284,7 @@ def validate_filename(cls, v: str) -> Union[str, None]:


class Investigation(CommentedIsaBase):
id: str = Field(alias="@id", default=None)
id: Optional[str] = Field(alias="@id", default=None)
description: Optional[str] = None
filename: Optional[str] = None
identifier: Optional[str] = None
Expand Down
50 changes: 31 additions & 19 deletions mars-cli/mars_lib/submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@
import requests
import json
from typing import Any
from mars_lib.authentication import get_metabolights_auth_token, get_webin_auth_token
from mars_lib.authentication import (
get_metabolights_auth_token,
get_webin_auth_token,
load_credentials,
AuthProvider,
)
from mars_lib.biosamples_external_references import (
get_header,
biosamples_endpoints,
Expand Down Expand Up @@ -44,8 +49,9 @@ def save_step_to_file(time_stamp: float, filename: str, isa_json: IsaJson):


def submission(
credential_service_name: str,
username_credentials: str,
webin_username: str,
metabolights_username: str,
metabolights_ftp_username: str,
credentials_file: TextIOWrapper,
isa_json_file: str,
target_repositories: list[str],
Expand All @@ -59,17 +65,24 @@ def submission(
# Get password from the credential manager
# Else:
# read credentials from file
if not (credential_service_name is None or username_credentials is None):
cm = CredentialManager(credential_service_name)
if all([webin_username, metabolights_username, metabolights_ftp_username]):
user_credentials = {
"username": username_credentials,
"password": cm.get_password_keyring(username_credentials),
cred_pair[0]: {
"username": cred_pair[1],
"password": CredentialManager(cred_pair[0]).get_password_keyring(
cred_pair[1]
),
}
for cred_pair in zip(
AuthProvider.available_providers(),
[webin_username, metabolights_username, metabolights_ftp_username],
)
}
else:
if credentials_file == "":
raise ValueError("No credentials found")

user_credentials = json.load(credentials_file)
user_credentials = load_credentials(credentials_file)

isa_json = load_isa_json(isa_json_file, investigation_is_root)

Expand Down Expand Up @@ -101,7 +114,7 @@ def submission(
# Submit to Biosamples
biosamples_result = submit_to_biosamples(
isa_json=isa_json,
biosamples_credentials=user_credentials,
biosamples_credentials=user_credentials[AuthProvider.WEBIN.value],
biosamples_url=urls["BIOSAMPLES"]["SUBMISSION"],
webin_token_url=urls["WEBIN"]["TOKEN"],
)
Expand All @@ -124,7 +137,7 @@ def submission(
file_paths=[
Path(df) for df in data_file_map[TargetRepository.ENA.value]
],
user_credentials=user_credentials,
user_credentials=user_credentials[AuthProvider.WEBIN.value],
submission_url=urls["ENA"]["DATA-SUBMISSION"],
file_transfer=file_transfer,
)
Expand All @@ -135,7 +148,7 @@ def submission(
# Step 2 : submit isa-json to ena
ena_result = submit_to_ena(
isa_json=isa_json,
user_credentials=user_credentials,
user_credentials=user_credentials[AuthProvider.WEBIN.value],
submission_url=urls["ENA"]["SUBMISSION"],
)
print_and_log(
Expand All @@ -159,7 +172,9 @@ def submission(
file_paths=data_file_map[TargetRepository.METABOLIGHTS.value],
file_transfer=file_transfer,
isa_json=isa_json,
metabolights_credentials=user_credentials,
metabolights_credentials=user_credentials[
AuthProvider.METABOLIGHTS_METADATA.value
],
metabolights_url=urls["METABOLIGHTS"]["SUBMISSION"],
metabolights_token_url=urls["METABOLIGHTS"]["TOKEN"],
)
Expand Down Expand Up @@ -252,9 +267,9 @@ def upload_to_metabolights(
"accept": "application/json",
"Authorization": f"Bearer {token}",
}
isa_json_str = isa_json.investigation.model_dump_json(
by_alias=True, exclude_none=True
)
isa_json_str = reduce_isa_json_for_target_repo(
isa_json, TargetRepository.METABOLIGHTS
).investigation.model_dump_json(by_alias=True, exclude_none=True)
json_file = io.StringIO(isa_json_str)

files = {"isa_json_file": ("isa_json.json", json_file)}
Expand Down Expand Up @@ -360,7 +375,7 @@ def submit_to_ena(
else result.request.body or ""
)
raise requests.HTTPError(
f"Request towards ENA failed!\nRequest:\nMethod:{result.request.method}\nStatus:{result.status_code}\nURL:{result.request.url}\nHeaders:{result.request.headers}\nBody:{body}"
f"Request towards ENA failed!\nRequest:\nMethod:{result.request.method}\nStatus:{result.status_code}\nURL:{submission_url}\nParams: ['webinUserName': {params.get('webinUserName')}, 'webinPassword': ****]\nHeaders:{result.request.headers}\nBody:{body}"
)

return result
Expand All @@ -372,11 +387,8 @@ def upload_to_ena(
submission_url: str,
file_transfer: str,
):
ALLOWED_FILE_TRANSFER_SOLUTIONS = {"ftp", "aspera"}
file_transfer = file_transfer.lower()

if file_transfer not in ALLOWED_FILE_TRANSFER_SOLUTIONS:
raise ValueError(f"Unsupported transfer protocol: {file_transfer}")
if file_transfer == "ftp":
uploader = FTPUploader(
submission_url,
Expand Down
18 changes: 18 additions & 0 deletions mars-cli/tests/fixtures/bad_credentials_file.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"webin": {
"username": "put-your-username-here",
"password": "put-your-password-here"
},
"metabolights_metadata": {
"username": "put-your-username-here",
"password": "put-your-password-here"
},
"metabolights_data": {
"username": "put-your-username-here",
"password": "put-your-password-here"
},
"blahblahblah_repo": {
"username": "put-your-username",
"password": "put-your-password"
}
}
Loading

0 comments on commit 8ee3540

Please sign in to comment.