DNM DEBUG DISK USAGE

Author: mmlb

ansible/tasks/stage2-setup-postgres.yml

@@ -62,9 +62,18 @@
   when: stage2
   become: true
   block:
+    - name: Check disk usage before nix based installs
+      ansible.builtin.command: df -h /
+      register: df_before_nix_install
+      changed_when: false
+
+    - name: Show disk usage before nix based installs
+      ansible.builtin.debug:
+        var: df_before_nix_install.stdout_lines
+
     - name: Install packages from nix binary cache
       ansible.builtin.shell: |
-        sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{ nix_item }}"
+        sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{ nix_item }}; rc=$?; df -h / >&2; exit $rc"
       loop:
         - "{{ psql_version }}/bin"
         - pg_prove
@@ -73,10 +82,24 @@
         - "{{ postgresql_version }}_src"
       loop_control:
         loop_var: 'nix_item'
+      register: nix_install_result
+      ignore_errors: true
+
+    - name: Show df output for each nix install attempt
+      ansible.builtin.debug:
+        msg: "{{ item.stderr_lines }}"
+      loop: "{{ nix_install_result.results }}"
+      loop_control:
+        label: "{{ item.nix_item }}"
+
+    - name: Fail if any nix install failed
+      ansible.builtin.fail:
+        msg: "nix profile install failed for: {{ nix_install_result.results | selectattr('failed', 'defined') | selectattr('failed') | map(attribute='nix_item') | list }}"
+      when: nix_install_result is failed
 
     - name: Install supascan for baseline validation
       ansible.builtin.shell: |
-        sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supascan"
+        sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supascan && df -h / >&2"
 
     - name: nix collect garbage after supascan install
       ansible.builtin.shell:

ebssurrogate/scripts/nix-provision.sh

@@ -71,11 +71,27 @@ function cleanup_packages {
 	apt-get remove --purge --yes ansible
 }
 
+function show_disk_usage {
+	echo "disk usage post $1:"
+	df /
+	df -h /
+	du -x -h --max-depth=2 / | sort -rh | head -30
+}
+
+# Snapshot disk usage even when a step below fails; errexit would otherwise skip
+# the remaining show_disk_usage calls and we'd lose the state we want to debug.
+trap 'show_disk_usage EXIT' EXIT
+
 setup_apt
 update_and_upgrade_apt
+show_disk_usage update_and_upgrade_apt
 install_packages
+show_disk_usage install_packages
 install_nix
+show_disk_usage install_nix
 execute_stage2_playbook
+show_disk_usage ansible
 cleanup_packages
 update_and_upgrade_apt
 cleanup_apt
+show_disk_usage update_and_upgrade

ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -347,7 +347,13 @@ device_partition_mappings
 format_and_mount_rootfs
 format_build_partition
 setup_chroot_environment
+df /mnt
+df -h /mnt
+du -x -h --max-depth=2 /mnt | sort -rh | head -30
 execute_playbook
 update_systemd_services
 clean_system
+df /mnt
+df -h /mnt
+du -x -h --max-depth=2 /mnt | sort -rh | head -30
 umount_reset_mappings