Iterating.

common/src/api/internal/shared.rs

@@ -6,13 +6,13 @@
 
 use crate::{
  address::NUM_SOURCE_NAT_PORTS,
- api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name},
+ api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name, Vni},
 };
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::{
- collections::HashMap,
+ collections::{HashMap, HashSet},
  fmt,
  net::{IpAddr, Ipv4Addr, Ipv6Addr},
  str::FromStr,
@@ -590,6 +590,43 @@ impl TryFrom<&[IpNetwork]> for IpAllowList {
  }
 }
 
+/// A VPC route resolved into a concrete target.
+#[derive(
+ Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct ReifiedVpcRoute {
+ pub dest: IpNet,
+ pub target: RouterTarget,
+}
+
+/// The target for a given router entry.
+#[derive(
+ Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+#[serde(tag = "type", rename_all = "snake_case", content = "value")]
+pub enum RouterTarget {
+ Drop,
+ InternetGateway,
+ Ip(IpAddr),
+ VpcSubnet(IpNet),
+}
+
+/// XX
+#[derive(
+ Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterId {
+ pub vni: Vni,
+ pub subnet: Option<IpNet>,
+}
+
+/// XX
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ReifiedVpcRouteSet {
+ pub id: RouterId,
+ pub routes: HashSet<ReifiedVpcRoute>,
+}
+
 #[cfg(test)]
 mod tests {
  use crate::api::{

illumos-utils/src/opte/port_manager.rs

@@ -16,6 +16,10 @@ use ipnetwork::IpNetwork;
 use omicron_common::api::external;
 use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
+use omicron_common::api::internal::shared::ReifiedVpcRoute;
+use omicron_common::api::internal::shared::ReifiedVpcRouteSet;
+use omicron_common::api::internal::shared::RouterId;
+use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget;
 use omicron_common::api::internal::shared::SourceNatConfig;
 use oxide_vpc::api::AddRouterEntryReq;
 use oxide_vpc::api::DhcpCfg;
@@ -36,6 +40,8 @@ use slog::error;
 use slog::info;
 use slog::Logger;
 use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::collections::HashSet;
 use std::net::IpAddr;
 use std::net::Ipv6Addr;
 use std::sync::atomic::AtomicU64;
@@ -60,6 +66,13 @@ struct PortManagerInner {
  // Map of all ports, keyed on the interface Uuid and its kind
  // (which includes the Uuid of the parent instance or service)
  ports: Mutex<BTreeMap<(Uuid, NetworkInterfaceKind), Port>>,
+
+ // XX: vs. Hashmap?
+ // XX: Should this be the UUID of the VPC? The rulesets are
+ // arguably shared v4+v6, although today we don't yet
+ // allow dual-stack, let alone v6.
+ // Map of all current resolved routes
+ routes: Mutex<HashMap<RouterId, HashSet<ReifiedVpcRoute>>>,
 }
 
 impl PortManagerInner {
@@ -86,6 +99,7 @@ impl PortManager {
  next_port_id: AtomicU64::new(0),
  underlay_ip,
  ports: Mutex::new(BTreeMap::new()),
+ routes: Mutex::new(Default::default()),
  });
 
  Self { inner }
@@ -400,6 +414,24 @@ impl PortManager {
  "route" => ?route,
  );
 
+ // XX: this is probably not the right initialisation here...
+ // XX: VPC rules should probably come from ctl plane.
+ let mut routes = self.inner.routes.lock().unwrap();
+ routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with(
+ || {
+ let mut out = HashSet::new();
+ out.insert(ReifiedVpcRoute {
+ dest: "0.0.0.0/0".parse().unwrap(),
+ target: ApiRouterTarget::InternetGateway,
+ });
+ out.insert(ReifiedVpcRoute {
+ dest: "::/0".parse().unwrap(),
+ target: ApiRouterTarget::InternetGateway,
+ });
+ out
+ },
+ );
+
  info!(
  self.inner.log,
  "Created OPTE port";
@@ -408,6 +440,23 @@ impl PortManager {
  Ok((port, ticket))
  }
 
+ pub fn vpc_routes_list(&self) -> Vec<ReifiedVpcRouteSet> {
+ let routes = self.inner.routes.lock().unwrap();
+ routes
+ .iter()
+ .map(|(k, v)| ReifiedVpcRouteSet { id: *k, routes: v.clone() })
+ .collect()
+ }
+
+ pub fn vpc_routes_ensure(&self, new_routes: Vec<ReifiedVpcRouteSet>) {
+ let mut routes = self.inner.routes.lock().unwrap();
+ // *routes = new_routes;
+ drop(routes);
+
+ // XX: compute deltas.
+ // XX: push down to OPTE.
+ }
+
  /// Ensure external IPs for an OPTE port are up to date.
  #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))]
  pub fn external_ips_ensure(

nexus/src/app/background/init.rs

@@ -22,6 +22,7 @@ use super::region_replacement;
 use super::service_firewall_rules;
 use super::sync_service_zone_nat::ServiceZoneNatTracker;
 use super::sync_switch_configuration::SwitchPortSettingsManager;
+use super::vpc_routes;
 use crate::app::oximeter::PRODUCER_LEASE_DURATION;
 use crate::app::sagas::SagaRequest;
 use nexus_config::BackgroundTaskConfig;
@@ -100,6 +101,9 @@ pub struct BackgroundTasks {
  /// task handle for propagation of VPC firewall rules for Omicron services
  /// with external network connectivity,
  pub task_service_firewall_propagation: common::TaskHandle,
+
+ /// task handle for propagation of VPC router rules to all OPTE ports
+ pub task_vpc_route_manager: common::TaskHandle,
 }
 
 impl BackgroundTasks {
@@ -368,6 +372,7 @@ impl BackgroundTasks {
  vec![],
  )
  };
+
  // Background task: service firewall rule propagation
  let task_service_firewall_propagation = driver.register(
  String::from("service_firewall_rule_propagation"),
@@ -377,12 +382,24 @@ impl BackgroundTasks {
  ),
  config.service_firewall_propagation.period_secs,
  Box::new(service_firewall_rules::ServiceRulePropagator::new(
- datastore,
+ datastore.clone(),
  )),
  opctx.child(BTreeMap::new()),
  vec![],
  );
 
+ let task_vpc_route_manager = {
+ let watcher = vpc_routes::VpcRouteManager::new(datastore);
+ driver.register(
+ "vpc_route_manager".to_string(),
+ "propagates updated VPC routes to all OPTE ports".into(),
+ config.switch_port_settings_manager.period_secs,
+ Box::new(watcher),
+ opctx.child(BTreeMap::new()),
+ vec![],
+ )
+ };
+
  BackgroundTasks {
  driver,
  task_internal_dns_config,
@@ -404,6 +421,7 @@ impl BackgroundTasks {
  task_region_replacement,
  task_instance_watcher,
  task_service_firewall_propagation,
+ task_vpc_route_manager,
  }
  }
 

nexus/src/app/background/mod.rs

@@ -25,5 +25,6 @@ mod service_firewall_rules;
 mod status;
 mod sync_service_zone_nat;
 mod sync_switch_configuration;
+mod vpc_routes;
 
 pub use init::BackgroundTasks;

nexus/src/app/background/vpc_routes.rs

@@ -0,0 +1,98 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Background task for propagating VPC routes (system and custom) to sleds.
+
+use super::common::BackgroundTask;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use nexus_db_model::{Sled, SledState};
+use nexus_db_queries::{context::OpContext, db::DataStore};
+use nexus_networking::sled_client_from_address;
+use nexus_types::{
+ deployment::SledFilter, external_api::views::SledPolicy, identity::Asset,
+};
+use omicron_common::api::{external::Vni, internal::shared::RouterId};
+use serde_json::json;
+use sled_agent_client::types::ReifiedVpcRoute;
+use std::{
+ collections::{HashMap, HashSet},
+ sync::Arc,
+};
+
+pub struct VpcRouteManager {
+ datastore: Arc<DataStore>,
+}
+
+impl VpcRouteManager {
+ pub fn new(datastore: Arc<DataStore>) -> Self {
+ Self { datastore }
+ }
+}
+
+impl BackgroundTask for VpcRouteManager {
+ fn activate<'a>(
+ &'a mut self,
+ opctx: &'a OpContext,
+ ) -> BoxFuture<'a, serde_json::Value> {
+ async {
+ let log = &opctx.log;
+
+ // XX: copied from omicron#5566
+ let sleds = match self
+ .datastore
+ .sled_list_all_batched(opctx, SledFilter::InService)
+ .await
+ {
+ Ok(v) => v,
+ Err(e) => {
+ let msg = format!("failed to enumerate sleds: {:#}", e);
+ error!(&log, "{msg}");
+ return json!({"error": msg});
+ }
+ }
+ .into_iter()
+ .filter(|sled| {
+ matches!(sled.state(), SledState::Active)
+ && matches!(sled.policy(), SledPolicy::InService { .. })
+ });
+
+ // Map sled db records to sled-agent clients
+ let sled_clients: Vec<(Sled, sled_agent_client::Client)> = sleds
+ .map(|sled| {
+ let client = sled_client_from_address(
+ sled.id(),
+ sled.address(),
+ &log,
+ );
+ (sled, client)
+ })
+ .collect();
+
+ // XX: actually reify rules.
+ let mut known_rules: HashMap<RouterId, HashSet<ReifiedVpcRoute>> =
+ HashMap::new();
+
+ for (sled, client) in sled_clients {
+ let Ok(a) = client.list_vpc_routes().await else {
+ warn!(
+ log,
+ "failed to fetch current VPC route state from sled";
+ "sled" => sled.serial_number(),
+ );
+ continue;
+ };
+
+ // XX: Who decides what we want? Do we figure out the NICs
+ // here? Or take the sled at their word for what subnets
+ // they want?
+
+ todo!()
+ }
+
+ json!({})
+ }
+ .boxed()
+ }
+}