support task & sandboxer instrument

Signed-off-by: Ziy1-Tan <[email protected]>
kuasar-io · Sep 8, 2024 · 452927a · 452927a
1 parent ca1da2d
commit 452927a
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 106 deletions.
diff --git a/vmm/common/src/tracer.rs b/vmm/common/src/tracer.rs
@@ -14,27 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-use std::str::FromStr;
-
 use opentelemetry::sdk::trace::Tracer;
 use opentelemetry::sdk::{trace, Resource};
-use opentelemetry_otlp::WithExportConfig;
-use tracing::level_filters::LevelFilter;
 use tracing_subscriber::EnvFilter;
 
-const DEFAULT_JAEGER_ENDPOINT: &str = "http://localhost:14268/api/traces";
-
-pub fn create_otlp_tracer(
-    otlp_service_name: &str,
-    otlp_endpoint: Option<String>,
-) -> anyhow::Result<Tracer> {
+pub fn init_otlp_tracer(otlp_service_name: &str) -> anyhow::Result<Tracer> {
     let tracer = opentelemetry_otlp::new_pipeline()
         .tracing()
-        .with_exporter(
-            opentelemetry_otlp::new_exporter()
-                .tonic()
-                .with_endpoint(otlp_endpoint.unwrap_or(DEFAULT_JAEGER_ENDPOINT.to_string())),
-        )
+        .with_exporter(opentelemetry_otlp::new_exporter().tonic())
         .with_trace_config(trace::config().with_resource(Resource::new(vec![
             opentelemetry::KeyValue::new("service.name", otlp_service_name.to_string()),
         ])))
@@ -43,10 +30,9 @@ pub fn create_otlp_tracer(
     Ok(tracer)
 }
 
-pub fn create_logger_filter(level: &str) -> anyhow::Result<EnvFilter> {
-    let log_level = LevelFilter::from_str(&level)?;
+pub fn init_logger_filter(log_level: &str) -> anyhow::Result<EnvFilter> {
     let filter = EnvFilter::from_default_env()
-        .add_directive(format!("containerd_sandbox={:?}", log_level).parse()?)
-        .add_directive(format!("vmm_sandboxer={:?}", log_level).parse()?);
+        .add_directive(format!("containerd_sandbox={}", log_level).parse()?)
+        .add_directive(format!("vmm_sandboxer={}", log_level).parse()?);
     Ok(filter)
 }
diff --git a/vmm/sandbox/config_clh.toml b/vmm/sandbox/config_clh.toml
@@ -1,5 +1,6 @@
 [sandbox]
 log_level = "info"
+enable_tracing = false
 
 [hypervisor]
 path = "/usr/local/bin/cloud-hypervisor"
@@ -15,6 +16,7 @@ debug = false
 
 [hypervisor.task]
 debug = false
+enable_tracing = false
 
 [hypervisor.virtiofsd]
 path = "/usr/local/bin/virtiofsd"

diff --git a/vmm/sandbox/config_stratovirt_aarch64.toml b/vmm/sandbox/config_stratovirt_aarch64.toml
@@ -1,5 +1,6 @@
 [sandbox]
 log_level = "info"
+enable_tracing = false
 
 [hypervisor]
 path = "/usr/bin/stratovirt"
@@ -15,4 +16,4 @@ debug = true
 enable_mem_prealloc = false
 
 [hypervisor.virtiofsd_conf]
-path = "/usr/bin/vhost_user_fs"
+path = "/usr/bin/vhost_user_fs"
diff --git a/vmm/sandbox/config_stratovirt_x86_64.toml b/vmm/sandbox/config_stratovirt_x86_64.toml
@@ -1,5 +1,6 @@
 [sandbox]
 log_level = "info"
+enable_tracing = false
 
 [hypervisor]
 path = "/usr/bin/stratovirt"

diff --git a/vmm/sandbox/src/bin/cloud_hypervisor/main.rs b/vmm/sandbox/src/bin/cloud_hypervisor/main.rs
@@ -16,11 +16,10 @@ limitations under the License.
 
 use clap::Parser;
 use opentelemetry::global;
-use tracing::{error, info_span};
-use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::util::SubscriberInitExt;
+use tracing::{info, info_span};
 use tracing_subscriber::Layer;
-use vmm_common::tracer::{create_logger_filter, create_otlp_tracer};
+use tracing_subscriber::{layer::SubscriberExt, Registry};
+use vmm_common::tracer::{init_logger_filter, init_otlp_tracer};
 use vmm_sandboxer::{
     args,
     cloud_hypervisor::{factory::CloudHypervisorVMFactory, hooks::CloudHypervisorHooks},
@@ -40,31 +39,19 @@ async fn main() {
     let config = Config::load_config(&args.config).await.unwrap();
 
     // Update args log level if it not presents args but in config.
-    let env_filter =
-        match create_logger_filter(&args.log_level.unwrap_or(config.sandbox.log_level())) {
-            Ok(filter) => filter,
-            Err(e) => {
-                error!("failed to init logger filter: {:?}", e);
-                return;
-            }
-        };
+    let env_filter = init_logger_filter(&args.log_level.unwrap_or(config.sandbox.log_level()))
+        .expect("failed to init logger filter");
 
     let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
     if config.sandbox.enable_tracing {
-        let tracer = match create_otlp_tracer("kuasar-vmm-sandboxer-clh-tracing-service", None) {
-            Ok(tracer) => tracer,
-            Err(e) => {
-                error!("failed to init otlp tracer: {:?}", e);
-                return;
-            }
-        };
+        let tracer = init_otlp_tracer("kuasar-vmm-sandboxer-clh-tracing-service")
+            .expect("failed to init otlp tracer");
+
         layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
     }
 
-    tracing_subscriber::registry()
-        .with(env_filter)
-        .with(layers)
-        .init();
+    let subscriber = Registry::default().with(env_filter).with(layers);
+    tracing::subscriber::set_global_default(subscriber).expect("unable to set global subscriber");
 
     let root_span = info_span!("kuasar-vmm-sandboxer-clh-root").entered();
 
@@ -78,6 +65,8 @@ async fn main() {
     // Do recovery job
     sandboxer.recover(&args.dir).await;
 
+    info!("Kuasar vmm sandboxer clh is started");
+
     // Run the sandboxer
     containerd_sandbox::run(
         "kuasar-vmm-sandboxer-clh",
@@ -88,6 +77,8 @@ async fn main() {
     .await
     .unwrap();
 
+    info!("Kuasar vmm sandboxer clh is exited");
+
     root_span.exit();
     global::shutdown_tracer_provider();
 }
diff --git a/vmm/sandbox/src/bin/qemu/main.rs b/vmm/sandbox/src/bin/qemu/main.rs
@@ -16,11 +16,10 @@ limitations under the License.
 
 use clap::Parser;
 use opentelemetry::global;
-use tracing::{error, info_span};
-use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::util::SubscriberInitExt;
+use tracing::info_span;
 use tracing_subscriber::Layer;
-use vmm_common::tracer::{create_logger_filter, create_otlp_tracer};
+use tracing_subscriber::{layer::SubscriberExt, Registry};
+use vmm_common::tracer::{init_logger_filter, init_otlp_tracer};
 use vmm_sandboxer::{
     args,
     config::Config,
@@ -57,30 +56,18 @@ async fn main() {
     };
 
     // Initialize log filter
-    let env_filter = match create_logger_filter(&config.sandbox.log_level()) {
-        Ok(filter) => filter,
-        Err(e) => {
-            error!("failed to init logger filter: {:?}", e);
-            return;
-        }
-    };
+    let env_filter =
+        init_logger_filter(&config.sandbox.log_level()).expect("failed to init logger filter");
 
     let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
     if config.sandbox.enable_tracing {
-        let tracer = match create_otlp_tracer("kuasar-vmm-sandboxer-qemu-tracing-service", None) {
-            Ok(tracer) => tracer,
-            Err(e) => {
-                error!("failed to init otlp tracer: {:?}", e);
-                return;
-            }
-        };
+        let tracer = init_otlp_tracer("kuasar-vmm-sandboxer-qemu-tracing-service")
+            .expect("failed to init otlp tracer");
         layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
     }
 
-    tracing_subscriber::registry()
-        .with(env_filter)
-        .with(layers)
-        .init();
+    let subscriber = Registry::default().with(env_filter).with(layers);
+    tracing::subscriber::set_global_default(subscriber).expect("unable to set global subscriber");
 
     let root_span = info_span!("kuasar-vmm-sandboxer-qemu-root").entered();
 

diff --git a/vmm/sandbox/src/bin/stratovirt/main.rs b/vmm/sandbox/src/bin/stratovirt/main.rs
@@ -16,11 +16,10 @@ limitations under the License.
 
 use clap::Parser;
 use opentelemetry::global;
-use tracing::{error, info_span};
-use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::util::SubscriberInitExt;
+use tracing::info_span;
+use tracing_subscriber::{layer::SubscriberExt, Registry};
 use tracing_subscriber::Layer;
-use vmm_common::tracer::{create_logger_filter, create_otlp_tracer};
+use vmm_common::tracer::{init_logger_filter, init_otlp_tracer};
 use vmm_sandboxer::{
     args,
     config::Config,
@@ -42,32 +41,18 @@ async fn main() {
     let config: Config<StratoVirtVMConfig> = Config::load_config(&args.config).await.unwrap();
 
     // Update args log level if it not presents args but in config.
-    let env_filter =
-        match create_logger_filter(&args.log_level.unwrap_or(config.sandbox.log_level())) {
-            Ok(filter) => filter,
-            Err(e) => {
-                error!("failed to init logger filter: {:?}", e);
-                return;
-            }
-        };
+    let env_filter = init_logger_filter(&args.log_level.unwrap_or(config.sandbox.log_level()))
+        .expect("failed to init logger filter");
 
     let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
     if config.sandbox.enable_tracing {
-        let tracer = match create_otlp_tracer("kuasar-vmm-sandboxer-stratovirt-otlp-service", None)
-        {
-            Ok(tracer) => tracer,
-            Err(e) => {
-                error!("failed to init otlp tracer: {:?}", e);
-                return;
-            }
-        };
+        let tracer = init_otlp_tracer("kuasar-vmm-sandboxer-stratovirt-otlp-service")
+            .expect("failed to init otlp tracer");
         layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
     }
 
-    tracing_subscriber::registry()
-        .with(env_filter)
-        .with(layers)
-        .init();
+    let subscriber = Registry::default().with(env_filter).with(layers);
+    tracing::subscriber::set_global_default(subscriber).expect("unable to set global subscriber");
 
     let root_span = info_span!("kuasar-vmm-sandboxer-stratovirt-root").entered();
 

diff --git a/vmm/sandbox/src/cloud_hypervisor/mod.rs b/vmm/sandbox/src/cloud_hypervisor/mod.rs
@@ -28,7 +28,7 @@ use tokio::{
     sync::watch::{channel, Receiver, Sender},
     task::JoinHandle,
 };
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info, instrument, warn};
 use vmm_common::SHARED_DIR_SUFFIX;
 
 use crate::{
@@ -156,6 +156,7 @@ impl CloudHypervisorVM {
 
 #[async_trait]
 impl VM for CloudHypervisorVM {
+    #[instrument(skip(self))]
     async fn start(&mut self) -> Result<u32> {
         create_dir_all(&self.base_dir).await?;
         let virtiofsd_pid = self.start_virtiofsd().await?;
@@ -209,6 +210,7 @@ impl VM for CloudHypervisorVM {
         Ok(pid.unwrap_or_default())
     }
 
+    #[instrument(skip(self))]
     async fn stop(&mut self, force: bool) -> Result<()> {
         let signal = if force {
             signal::SIGKILL
@@ -240,6 +242,7 @@ impl VM for CloudHypervisorVM {
         Ok(())
     }
 
+    #[instrument(skip(self))]
     async fn attach(&mut self, device_info: DeviceInfo) -> Result<()> {
         match device_info {
             DeviceInfo::Block(blk_info) => {
@@ -274,31 +277,37 @@ impl VM for CloudHypervisorVM {
         Ok(())
     }
 
+    #[instrument(skip(self))]
     async fn hot_attach(&mut self, device_info: DeviceInfo) -> Result<(BusType, String)> {
         let client = self.get_client()?;
         let addr = client.hot_attach(device_info)?;
         Ok((BusType::PCI, addr))
     }
 
+    #[instrument(skip(self))]
     async fn hot_detach(&mut self, id: &str) -> Result<()> {
         let client = self.get_client()?;
         client.hot_detach(id)?;
         Ok(())
     }
 
+    #[instrument(skip(self))]
     async fn ping(&self) -> Result<()> {
         // TODO
         Ok(())
     }
 
+    #[instrument(skip(self))]
     fn socket_address(&self) -> String {
         self.agent_socket.to_string()
     }
 
+    #[instrument(skip(self))]
     async fn wait_channel(&self) -> Option<Receiver<(u32, i128)>> {
         self.wait_chan.clone()
     }
 
+    #[instrument(skip(self))]
     async fn vcpus(&self) -> Result<VcpuThreads> {
         // Refer to https://github.com/firecracker-microvm/firecracker/issues/718
         Ok(VcpuThreads {
@@ -320,13 +329,15 @@ impl VM for CloudHypervisorVM {
         })
     }
 
+    #[instrument(skip(self))]
     fn pids(&self) -> Pids {
         self.pids.clone()
     }
 }
 
 #[async_trait]
 impl crate::vm::Recoverable for CloudHypervisorVM {
+    #[instrument(skip(self))]
     async fn recover(&mut self) -> Result<()> {
         self.client = Some(self.create_client().await?);
         let pid = self.pid()?;