Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ansible vars file #2115

Merged
merged 41 commits into from
Sep 20, 2024
Merged
Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
1e1d7b5
:wrench: Add URLs for Gitea and Hedgedoc
jemrobinson Aug 6, 2024
d4593ed
:sparkles: Add desktop files and icons for gitea and hedgedoc
jemrobinson Aug 6, 2024
9fc781b
:sparkles: Add desktop icons for input, output and shared folders
jemrobinson Aug 6, 2024
6d0da02
Move icons to /usr/local/share
JimMadge Aug 7, 2024
484d68c
WIP: write vars from pulumi to desired state
JimMadge Aug 8, 2024
1f6c0b9
Move desired state to new component
JimMadge Aug 8, 2024
3f285ea
Correct hostname arguments
JimMadge Aug 8, 2024
384e955
Correct subnet variable names
JimMadge Aug 8, 2024
d5d0768
Update NSG names
JimMadge Aug 8, 2024
3766790
Correct IP range name
JimMadge Aug 9, 2024
dbccff7
Fix IP ranges test
JimMadge Aug 9, 2024
7a41029
Move ansible vars file function
JimMadge Aug 9, 2024
7686234
Merge remote-tracking branch 'origin/develop' into ansible_vars_file
JimMadge Aug 13, 2024
5c193fe
Use string asset for blob
JimMadge Aug 13, 2024
d5920c8
Apply StringAsset to str not Output[str]
JimMadge Aug 13, 2024
b0febc7
Add StringAsset to typings
JimMadge Aug 13, 2024
9b48edb
Correct cloudinit template argument name
JimMadge Aug 13, 2024
56d4e8a
:bug: Ensure that icons directory is created
jemrobinson Aug 12, 2024
913ccd0
:recycle: Simplify ansible setup by copying all files in a directory …
jemrobinson Aug 12, 2024
7eb8317
:wrench: Do not create xrdp directories separately
jemrobinson Aug 12, 2024
d5fc6b7
Tidy ansible
JimMadge Aug 13, 2024
57b8d14
Remove unused notify
JimMadge Aug 13, 2024
3e1eb30
Correct template argument
JimMadge Aug 13, 2024
c67fe5b
Use vars file in desired state play
JimMadge Aug 13, 2024
9427541
Correct desktop file name
JimMadge Aug 13, 2024
546764e
Wait for clamd.ctl to be created
JimMadge Aug 8, 2024
3fcc1ae
Restart clamd when configuration is updated
JimMadge Aug 8, 2024
3edb214
Correct socket file name
JimMadge Aug 8, 2024
626be80
Move cloud init template to ansible
JimMadge Aug 13, 2024
8720a3e
Update data_safe_haven/infrastructure/common/ip_ranges.py
JimMadge Sep 2, 2024
f23a8c8
Merge remote-tracking branch 'origin/develop' into ansible_vars_file
JimMadge Sep 2, 2024
96fb551
Correct ip range test
JimMadge Sep 2, 2024
908f608
Merge remote-tracking branch 'origin/develop' into ansible_vars_file
JimMadge Sep 17, 2024
ab5e710
Merge branch 'mount_points' into ansible_vars_file
JimMadge Sep 17, 2024
22c754f
Add NFSV3StorageAccount component
JimMadge Sep 17, 2024
435697b
Add composite resource for NFS blob containers
JimMadge Sep 17, 2024
2bae206
Correct path to install_deb script
JimMadge Sep 17, 2024
cb89697
Correct paths in comments
JimMadge Sep 17, 2024
aa46d23
Update pulumi component properties
JimMadge Sep 17, 2024
d62c0cc
Correct template names
JimMadge Sep 18, 2024
9e6b1b9
Rename class
JimMadge Sep 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion data_safe_haven/infrastructure/common/ip_ranges.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class SREIpRanges:
apt_proxy_server = vnet.next_subnet(8)
clamav_mirror = vnet.next_subnet(8)
data_configuration = vnet.next_subnet(8)
data_desired_state = vnet.next_subnet(8)
desired_state = vnet.next_subnet(8)
data_private = vnet.next_subnet(8)
JimMadge marked this conversation as resolved.
Show resolved Hide resolved
firewall = vnet.next_subnet(64) # 64 address minimum
firewall_management = vnet.next_subnet(64) # 64 address minimum
Expand Down
20 changes: 18 additions & 2 deletions data_safe_haven/infrastructure/programs/declarative_sre.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from .sre.backup import SREBackupComponent, SREBackupProps
from .sre.clamav_mirror import SREClamAVMirrorComponent, SREClamAVMirrorProps
from .sre.data import SREDataComponent, SREDataProps
from .sre.desired_state import SREDesiredStateComponent, SREDesiredStateProps
from .sre.dns_server import SREDnsServerComponent, SREDnsServerProps
from .sre.firewall import SREFirewallComponent, SREFirewallProps
from .sre.identity import SREIdentityComponent, SREIdentityProps
Expand Down Expand Up @@ -182,7 +183,6 @@ def __call__(self) -> None:
resource_group=resource_group,
sre_fqdn=networking.sre_fqdn,
subnet_data_configuration=networking.subnet_data_configuration,
subnet_data_desired_state=networking.subnet_data_desired_state,
subnet_data_private=networking.subnet_data_private,
subscription_id=self.config.azure.subscription_id,
subscription_name=self.context.subscription_name,
Expand Down Expand Up @@ -336,6 +336,22 @@ def __call__(self) -> None:
tags=self.tags,
)

# Deploy desired state
desired_state = SREDesiredStateComponent(
"sre_desired_state",
self.stack_name,
SREDesiredStateProps(
admin_ip_addresses=self.config.sre.admin_ip_addresses,
dns_private_zones=dns.private_zones,
gitea_hostname=user_services.gitea_server.hostname,
hedgedoc_hostname=user_services.hedgedoc_server.hostname,
location=self.config.azure.location,
resource_group=resource_group,
subnet_desired_state=networking.subnet_desired_state,
subscription_name=self.context.subscription_name,
),
)

# Deploy workspaces
workspaces = SREWorkspacesComponent(
"sre_workspaces",
Expand All @@ -358,7 +374,7 @@ def __call__(self) -> None:
resource_group_name=resource_group.name,
software_repository_hostname=user_services.software_repositories.hostname,
sre_name=self.config.name,
storage_account_data_desired_state_name=data.storage_account_data_desired_state_name,
storage_account_desired_state_name=desired_state.storage_account_name,
storage_account_data_private_user_name=data.storage_account_data_private_user_name,
storage_account_data_private_sensitive_name=data.storage_account_data_private_sensitive_name,
subnet_workspaces=networking.subnet_workspaces,
Expand Down
158 changes: 1 addition & 157 deletions data_safe_haven/infrastructure/programs/sre/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from typing import ClassVar

import pulumi_random
from pulumi import ComponentResource, FileAsset, Input, Output, ResourceOptions
from pulumi import ComponentResource, Input, Output, ResourceOptions
from pulumi_azure_native import (
authorization,
keyvault,
Expand Down Expand Up @@ -34,7 +34,6 @@
SSLCertificate,
SSLCertificateProps,
)
from data_safe_haven.resources import resources_path
from data_safe_haven.types import AzureDnsZoneNames


Expand All @@ -54,7 +53,6 @@ def __init__(
resource_group: Input[resources.ResourceGroup],
sre_fqdn: Input[str],
subnet_data_configuration: Input[network.GetSubnetResult],
subnet_data_desired_state: Input[network.GetSubnetResult],
subnet_data_private: Input[network.GetSubnetResult],
subscription_id: Input[str],
subscription_name: Input[str],
Expand Down Expand Up @@ -82,9 +80,6 @@ def __init__(
self.subnet_data_configuration_id = Output.from_input(
subnet_data_configuration
).apply(get_id_from_subnet)
self.subnet_data_desired_state_id = Output.from_input(
subnet_data_desired_state
).apply(get_id_from_subnet)
self.subnet_data_private_id = Output.from_input(subnet_data_private).apply(
get_id_from_subnet
)
Expand Down Expand Up @@ -456,154 +451,6 @@ def __init__(
),
)

# Deploy desired state storage account
# - This holds the /desired_state container that is mounted by workspaces
# - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer
storage_account_data_desired_state = storage.StorageAccount(
f"{self._name}_storage_account_data_desired_state",
# Storage account names have a maximum of 24 characters
account_name=alphanumeric(
f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}"
)[:24],
enable_https_traffic_only=True,
enable_nfs_v3=True,
encryption=storage.EncryptionArgs(
key_source=storage.KeySource.MICROSOFT_STORAGE,
services=storage.EncryptionServicesArgs(
blob=storage.EncryptionServiceArgs(
enabled=True, key_type=storage.KeyType.ACCOUNT
),
file=storage.EncryptionServiceArgs(
enabled=True, key_type=storage.KeyType.ACCOUNT
),
),
),
kind=storage.Kind.BLOCK_BLOB_STORAGE,
is_hns_enabled=True,
location=props.location,
network_rule_set=storage.NetworkRuleSetArgs(
bypass=storage.Bypass.AZURE_SERVICES,
default_action=storage.DefaultAction.DENY,
ip_rules=Output.from_input(props.data_configuration_ip_addresses).apply(
lambda ip_ranges: [
storage.IPRuleArgs(
action=storage.Action.ALLOW,
i_p_address_or_range=str(ip_address),
)
for ip_range in sorted(ip_ranges)
for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips()
]
),
virtual_network_rules=[
storage.VirtualNetworkRuleArgs(
virtual_network_resource_id=props.subnet_data_desired_state_id,
)
],
),
resource_group_name=props.resource_group_name,
sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS),
opts=child_opts,
tags=child_tags,
)
# Deploy desired state share
container_desired_state = storage.BlobContainer(
f"{self._name}_blob_desired_state",
account_name=storage_account_data_desired_state.name,
container_name="desiredstate",
default_encryption_scope="$account-encryption-key",
deny_encryption_scope_override=False,
public_access=storage.PublicAccess.NONE,
resource_group_name=props.resource_group_name,
opts=ResourceOptions.merge(
child_opts,
ResourceOptions(parent=storage_account_data_desired_state),
),
)
# Set storage container ACLs
BlobContainerAcl(
f"{container_desired_state._name}_acl",
BlobContainerAclProps(
acl_user="r-x",
acl_group="r-x",
acl_other="r-x",
# ensure that the above permissions are also set on any newly created
# files (eg. with Azure Storage Explorer)
apply_default_permissions=True,
container_name=container_desired_state.name,
resource_group_name=props.resource_group_name,
storage_account_name=storage_account_data_desired_state.name,
subscription_name=props.subscription_name,
),
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=container_desired_state)
),
)
# Create file assets to upload
desired_state_directory = (resources_path / "workspace" / "ansible").absolute()
files_desired_state = [
(
FileAsset(str(file_path)),
file_path.name,
str(file_path.relative_to(desired_state_directory)),
)
for file_path in sorted(desired_state_directory.rglob("*"))
if file_path.is_file() and not file_path.name.startswith(".")
]
# Upload file assets to desired state container
for file_asset, file_name, file_path in files_desired_state:
storage.Blob(
f"{container_desired_state._name}_blob_{file_name}",
account_name=storage_account_data_desired_state.name,
blob_name=file_path,
container_name=container_desired_state.name,
resource_group_name=props.resource_group_name,
source=file_asset,
)
# Set up a private endpoint for the desired state storage account
storage_account_data_desired_state_endpoint = network.PrivateEndpoint(
f"{storage_account_data_desired_state._name}_private_endpoint",
location=props.location,
private_endpoint_name=f"{stack_name}-pep-storage-account-data-desired-state",
private_link_service_connections=[
network.PrivateLinkServiceConnectionArgs(
group_ids=["blob"],
name=f"{stack_name}-cnxn-pep-storage-account-data-private-sensitive",
private_link_service_id=storage_account_data_desired_state.id,
)
],
resource_group_name=props.resource_group_name,
subnet=network.SubnetArgs(id=props.subnet_data_desired_state_id),
opts=ResourceOptions.merge(
child_opts,
ResourceOptions(
ignore_changes=["custom_dns_configs"],
parent=storage_account_data_desired_state,
),
),
tags=child_tags,
)
# Add a private DNS record for each desired state endpoint custom DNS config
network.PrivateDnsZoneGroup(
f"{storage_account_data_desired_state._name}_private_dns_zone_group",
private_dns_zone_configs=[
network.PrivateDnsZoneConfigArgs(
name=replace_separators(
f"{stack_name}-storage-account-data-desired-state-to-{dns_zone_name}",
"-",
),
private_dns_zone_id=props.dns_private_zones[dns_zone_name].id,
)
for dns_zone_name in AzureDnsZoneNames.STORAGE_ACCOUNT
],
private_dns_zone_group_name=f"{stack_name}-dzg-storage-account-data-desired-state",
private_endpoint_name=storage_account_data_desired_state_endpoint.name,
resource_group_name=props.resource_group_name,
opts=ResourceOptions.merge(
child_opts,
ResourceOptions(parent=storage_account_data_desired_state),
),
)

# Deploy sensitive data blob storage account
# - This holds the /data and /output containers that are mounted by workspaces
# - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer
Expand Down Expand Up @@ -911,9 +758,6 @@ def __init__(
self.storage_account_data_configuration_name = (
storage_account_data_configuration.name
)
self.storage_account_data_desired_state_name = (
storage_account_data_desired_state.name
)
self.managed_identity = identity_key_vault_reader
self.password_nexus_admin = Output.secret(password_nexus_admin.result)
self.password_database_service_admin = Output.secret(
Expand Down
Loading
Loading