Add microphone waveform in editor

apps/desktop/src-tauri/src/lib.rs

@@ -1620,6 +1620,52 @@ async fn seek_to(editor_instance: WindowEditorInstance, frame_number: u32) -> Re
     Ok(())
 }
 
+#[tauri::command]
+#[specta::specta]
+async fn get_mic_waveforms(editor_instance: WindowEditorInstance) -> Result<Vec<Vec<f32>>, String> {
+    const CHUNK_SIZE: usize = (cap_audio::AudioData::SAMPLE_RATE as usize) / 10; // ~100ms
+
+    let mut out = Vec::new();
+
+    for segment in editor_instance.segments.iter() {
+        if let Some(audio) = &segment.audio {
+            let channels = audio.channels() as usize;
+            let samples = audio.samples();
+            let mut waveform = Vec::new();
+
+            let mut i = 0;
+            while i < samples.len() {
+                let end = (i + CHUNK_SIZE * channels).min(samples.len());
+                let mut sum = 0.0f32;
+                for s in &samples[i..end] {
+                    sum += s.abs();
+                }
+                let avg = if end > i { sum / (end - i) as f32 } else { 0.0 };
+                waveform.push(avg);
+                i += CHUNK_SIZE * channels;
+            }
+
+            if let Some(max) = waveform
+                .iter()
+                .cloned()
+                .fold(None, |m, v| Some(m.map_or(v, |m: f32| m.max(v))))
+            {
+                if max > 0.0 {
+                    for v in waveform.iter_mut() {
+                        *v /= max;
+                    }
+                }
+            }
+
+            out.push(waveform);
+        } else {
+            out.push(Vec::new());
+        }
+    }
+
+    Ok(out)
+}
+
 // keep this async otherwise opening windows may hang on windows
 #[tauri::command]
 #[specta::specta]
@@ -1750,6 +1796,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) {
             open_file_path,
             get_video_metadata,
             create_editor_instance,
+            get_mic_waveforms,
             start_playback,
             stop_playback,
             set_playhead_position,

apps/desktop/src/App.tsx

@@ -103,6 +103,16 @@ function Inner() {
                   Reload
                 </Button>
               </div>
+
+              {import.meta.env.DEV && (
+                <div class="h-0 text-sm">
+                  <pre class="text-left mt-8">{`${e.toString()}\n\n${e.stack
+                    ?.toString()
+                    .split("\n")
+                    .slice(0, 10)
+                    .join("\n")}`}</pre>
+                </div>
+              )}
             </div>
           );
         }}

apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx

@@ -29,6 +29,84 @@ import {
   useSegmentWidth,
 } from "./Track";
 
+function WaveformCanvas(props: {
+  waveform: number[];
+  segment: { start: number; end: number };
+  secsPerPixel: number;
+}) {
+  let canvas: HTMLCanvasElement | undefined;
+  const { width } = useSegmentContext();
+  const { secsPerPixel } = useTimelineContext();
+
+  const render = () => {
+    if (!canvas) return;
+    const ctx = canvas.getContext("2d");
+    if (!ctx) return;
+
+    const w = width();
+    if (w <= 0) return;
+
+    const h = canvas.height;
+    canvas.width = w;
+    ctx.clearRect(0, 0, w, h);
+
+    const maxAmplitude = h;
+
+    ctx.fillStyle = "rgba(255,255,255,0.3)";
+    ctx.beginPath();
+
+    const step = 0.05 / secsPerPixel();
+
+    ctx.moveTo(0, h);
+
+    for (
+      let segmentTime = props.segment.start;
+      segmentTime <= props.segment.end + 0.1;
+      segmentTime += 0.1
+    ) {
+      const index = Math.floor(segmentTime * 10);
+      const xTime = index / 10;
+
+      const amplitude = props.waveform[index] * maxAmplitude;
+
+      const x = (xTime - props.segment.start) / secsPerPixel();
+      const y = h - amplitude;
+
+      const prevX = (xTime - 0.1 - props.segment.start) / secsPerPixel();
+      const prevAmplitude = props.waveform[index - 1] * maxAmplitude;
+      const prevY = h - prevAmplitude;
+
+      const cpX1 = prevX + step / 2;
+      const cpX2 = x - step / 2;
+
+      ctx.bezierCurveTo(cpX1, prevY, cpX2, y, x, y);
+    }
+
+    ctx.lineTo(
+      (props.segment.end + 0.3 - props.segment.start) / secsPerPixel(),
+      h
+    );
+
+    ctx.closePath();
+    ctx.fill();
+  };
+
+  createEffect(() => {
+    render();
+  });
+
+  return (
+    <canvas
+      ref={(el) => {
+        canvas = el;
+        render();
+      }}
+      class="absolute inset-0 w-full h-full pointer-events-none"
+      height={52}
+    />
+  );
+}
+
 export function ClipTrack(
   props: Pick<ComponentProps<"div">, "ref"> & {
     handleUpdatePlayhead: (e: MouseEvent) => void;
@@ -42,6 +120,8 @@ export function ClipTrack(
     editorState,
     setEditorState,
     totalDuration,
+    micWaveforms,
+    metaQuery,
   } = useEditorContext();
 
   const { secsPerPixel, duration } = useTimelineContext();
@@ -105,6 +185,11 @@ export function ClipTrack(
             return segmentIndex === selection.index;
           });
 
+          const waveform = () => {
+            const idx = segment.recordingSegment ?? i();
+            return micWaveforms()?.[idx] ?? [];
+          };
+
           return (
             <>
               <Show when={marker()}>
@@ -126,11 +211,7 @@ export function ClipTrack(
                         })()}
                       >
                         {(marker) => (
-                          <div
-                            class={cx(
-                              "h-7 -top-8 overflow-hidden rounded-full -translate-x-1/2"
-                            )}
-                          >
+                          <div class="h-7 -top-8 overflow-hidden rounded-full -translate-x-1/2 z-10">
                             <CutOffsetButton
                               value={(() => {
                                 const m = marker();
@@ -157,53 +238,38 @@ export function ClipTrack(
                       <Match
                         when={(() => {
                           const m = marker();
-                          if (m.type === "dual") return m;
+                          if (
+                            m.type === "dual" &&
+                            m.right &&
+                            m.right.type === "time"
+                          )
+                            return m.right;
                         })()}
                       >
-                        {(marker) => (
-                          <div class="h-7 w-0 absolute -top-8 flex flex-row rounded-full">
-                            <Show when={marker().left}>
-                              {(marker) => (
-                                <CutOffsetButton
-                                  value={(() => {
-                                    const m = marker();
-                                    return m.type === "reset" ? 0 : m.time;
-                                  })()}
-                                  class="-right-px absolute rounded-l-full !pr-1.5 rounded-tr-full"
-                                  onClick={() => {
-                                    setProject(
-                                      "timeline",
-                                      "segments",
-                                      i() - 1,
-                                      "end",
-                                      segmentRecording(i() - 1).display.duration
-                                    );
-                                  }}
-                                />
-                              )}
-                            </Show>
-                            <Show when={marker().right}>
-                              {(marker) => (
-                                <CutOffsetButton
-                                  value={(() => {
-                                    const m = marker();
-                                    return m.type === "reset" ? 0 : m.time;
-                                  })()}
-                                  class="-left-px absolute rounded-r-full !pl-1.5 rounded-tl-full"
-                                  onClick={() => {
-                                    setProject(
-                                      "timeline",
-                                      "segments",
-                                      i(),
-                                      "start",
-                                      0
-                                    );
-                                  }}
-                                />
-                              )}
-                            </Show>
-                          </div>
-                        )}
+                        {(marker) => {
+                          const markerValue = marker();
+                          return (
+                            <div class="h-7 w-0 absolute -top-8 flex flex-row rounded-full">
+                              <CutOffsetButton
+                                value={
+                                  markerValue.type === "time"
+                                    ? markerValue.time
+                                    : 0
+                                }
+                                class="-left-px absolute rounded-r-full !pl-1.5 rounded-tl-full"
+                                onClick={() => {
+                                  setProject(
+                                    "timeline",
+                                    "segments",
+                                    i(),
+                                    "start",
+                                    0
+                                  );
+                                }}
+                              />
+                            </div>
+                          );
+                        }}
                       </Match>
                     </Switch>
                   </div>
@@ -257,6 +323,16 @@ export function ClipTrack(
                   }
                 }}
               >
+                <Show
+                  when={metaQuery.data?.hasMicrophone && waveform().length > 0}
+                >
+                  <WaveformCanvas
+                    waveform={waveform()}
+                    segment={segment}
+                    secsPerPixel={secsPerPixel()}
+                  />
+                </Show>
+
                 <Markings segment={segment} prevDuration={prevDuration()} />
 
                 <SegmentHandle
@@ -433,7 +509,10 @@ export function ClipTrack(
                     <div class="w-[2px] bottom-0 -top-2 rounded-full from-red-300 to-transparent bg-gradient-to-b -translate-x-1/2" />
                     <div class="h-7 w-0 absolute -top-8 flex flex-row rounded-full">
                       <CutOffsetButton
-                        value={marker().time}
+                        value={(() => {
+                          const m = marker();
+                          return m.type === "time" ? m.time : 0;
+                        })()}
                         class="-right-px absolute rounded-l-full !pr-1.5 rounded-tr-full"
                         onClick={() => {
                           setProject(

apps/desktop/src/routes/editor/Timeline/index.tsx

@@ -225,21 +225,24 @@ export function Timeline() {
             </div>
           )}
         </Show>
-        <Show when={!split()}>
-          <div
-            class="absolute bottom-0 top-4 h-full rounded-full z-10 w-px pointer-events-none bg-gradient-to-b to-[120%] from-[rgb(226,64,64)]"
-            style={{
-              left: `${TIMELINE_PADDING}px`,
-              transform: `translateX(${Math.min(
-                (editorState.playbackTime - transform().position) /
-                  secsPerPixel(),
-                timelineBounds.width ?? 0
-              )}px)`,
-            }}
-          >
-            <div class="size-3 bg-[rgb(226,64,64)] rounded-full -mt-2 -ml-[calc(0.37rem-0.5px)]" />
-          </div>
-        </Show>
+        {/* <Show when={!split()}> */}
+        <div
+          class={cx(
+            "absolute bottom-0 top-4 h-full rounded-full z-10 w-px pointer-events-none bg-gradient-to-b to-[120%] from-[rgb(226,64,64)]",
+            split() && "opacity-70"
+          )}
+          style={{
+            left: `${TIMELINE_PADDING}px`,
+            transform: `translateX(${Math.min(
+              (editorState.playbackTime - transform().position) /
+                secsPerPixel(),
+              timelineBounds.width ?? 0
+            )}px)`,
+          }}
+        >
+          <div class="size-3 bg-[rgb(226,64,64)] rounded-full -mt-2 -ml-[calc(0.37rem-0.5px)]" />
+        </div>
+        {/* </Show> */}
         <ClipTrack
           ref={setTimelineRef}
           handleUpdatePlayhead={handleUpdatePlayhead}

apps/desktop/src/routes/editor/context.ts

@@ -198,6 +198,8 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider(
       },
     });
 
+    const [micWaveforms] = createResource(() => commands.getMicWaveforms());
+
     return {
       ...editorInstanceContext,
       meta() {
@@ -216,6 +218,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider(
       zoomOutLimit,
       exportState,
       setExportState,
+      micWaveforms,
     };
   },
   // biome-ignore lint/style/noNonNullAssertion: it's ok

apps/desktop/src/utils/tauri.ts

@@ -77,6 +77,9 @@ async getVideoMetadata(path: string) : Promise<VideoRecordingMetadata> {
 async createEditorInstance() : Promise<SerializedEditorInstance> {
     return await TAURI_INVOKE("create_editor_instance");
 },
+async getMicWaveforms() : Promise<number[][]> {
+    return await TAURI_INVOKE("get_mic_waveforms");
+},
 async startPlayback(fps: number, resolutionBase: XY<number>) : Promise<null> {
     return await TAURI_INVOKE("start_playback", { fps, resolutionBase });
 },