Covmat script, della compat, command-line arguments for serial jobs

.gitignore

@@ -106,3 +106,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# GUI
+*.code-workspace

project/ana_cov_comp/README.md

@@ -0,0 +1,70 @@
+
+
+On Della, it's better to submit SLURM jobs via script instead of interactively. For 
+convenience, let's define a commonly used job script,
+
+```bash
+export sbatch1pn="/home/zatkins/repos/simonsobs/PSpipe/project/ana_cov_comp/paramfiles/1physicsnode.slurm"
+```
+
+We need windows. Note, this will generate "PSpipe" style windows. A particular dict file may load other, pre-generated windows.
+
+```bash
+for i in {0..5}; do \
+    sbatch --cpus-per-task 1 --mem 48G --time 20:00 --job-name get_window_dr6 ${sbatch1pn} "python -u python/get_window_dr6.py paramfiles/global_dr6_v4.dict $i $((i+1))"
+done
+```
+
+We need to generate some signal spectra for the covariance matrix,
+
+```bash
+sbatch --cpus-per-task 1 --mem 2G --time 2:00 --job-name get_best_fit_mflike ${sbatch1pn} "python -u python/get_best_fit_mflike.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict"
+```
+
+Next we'll need to compute the mode-coupling matrices,
+
+```bash
+for i in {0..20}; do \
+    sbatch --cpus-per-task 10 --mem 28G --time 5:00 --job-name get_mcm_and_bbl ${sbatch1pn} "python -u python/get_mcm_and_bbl.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict $i $((i+1))"
+done
+```
+
+Next, we compute the alms of the maps.
+
+```bash
+sbatch --cpus-per-task 10 --mem 28G --time 20:00 --job-name get_alms ${sbatch1pn} "python -u python/get_alms.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict"
+```
+
+Now spectra,
+
+```bash
+sbatch --cpus-per-task 1 --mem 28G --time 10:00 --job-name get_spectra_from_alms ${sbatch1pn} "python -u python/get_spectra_from_alms.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict"
+```
+
+```bash
+sbatch --cpus-per-task 1 --mem 1G --time 2:00 --job-name get_noise_model ${sbatch1pn} "python -u python/get_noise_model.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict"
+```
+
+```bash
+sbatch --cpus-per-task 1 --mem 1G --time 2:00 --job-name get_per_split_noise ${sbatch1pn} "python -u python/get_per_split_noise.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict"
+```
+
+Now we compute the analytic covariances blocks assuming an isotropic noise model.
+
+```bash
+sbatch --cpus-per-task 4 --mem 10G --time 15:00 --job-name get_sq_windows_alms ${sbatch1pn} "python -u python/get_sq_windows_alms.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict"
+```
+
+```bash
+for i in {0..230}; do \
+    sbatch --cpus-per-task 10 --mem 32G --time 5:00 --job-name get_covariance_blocks ${sbatch1pn} "python -u python/get_covariance_blocks.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict $i $((i+1))"
+done
+```
+
+For anisotropic covariances,
+
+```bash
+for i in {0..230}; do \
+    sbatch --cpus-per-task 4 --mem 10G --time 15:00 --job-name get_split_covariance_aniso ${sbatch1pn} "python -u python/get_split_covariance_aniso.py paramfiles/sims_dr6_v4_lmax5400_20230816.dict $i $((i+1))"
+done
+```

project/ana_cov_comp/paramfiles/1physicsnode.slurm

@@ -0,0 +1,23 @@
+#! /bin/bash
+#
+#SBATCH --nodes=1                # node count
+#SBATCH --ntasks-per-node=1      # total number of tasks per node
+#SBATCH -p physics
+#SBATCH --output=slurmoutput/R-%j.out
+#SBATCH --mail-type=begin        # send email when job begins
+#SBATCH --mail-type=end          # send email when job ends
+#SBATCH [email protected]
+
+cd /home/zatkins/repos/simonsobs/PSpipe/project/ana_cov_comp
+
+### set environment ###
+module purge
+module load cmb-della8/220426
+
+# activate conda
+conda activate pspy-della8
+
+export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
+export JULIA_NUM_THREADS=$SLURM_CPUS_PER_TASK
+
+srun $1

project/ana_cov_comp/paramfiles/cov_dr6_v4_20231128.dict

@@ -0,0 +1,192 @@
+surveys = ["dr6"]
+
+arrays_dr6 = ["pa4_f150", "pa4_f220", "pa5_f090", "pa5_f150", "pa6_f090", "pa6_f150"]
+
+data_dir = '/scratch/gpfs/zatkins/data/simonsobs/PSpipe/project/ana_cov_comp/'
+
+# orphans
+use_beam_covariance = True
+multistep_path = data_dir
+
+# mcms
+mcms_dir = data_dir + 'mcms'
+
+binned_mcm = False
+lmax = 5400
+use_toeplitz_mcm = False
+binning_file = data_dir + "binning/BIN_ACTPOL_50_4_SC_large_bin_at_low_ell"
+niter = 0
+type = 'Dl'
+
+# alms 
+alms_dir = data_dir + 'alms'
+apply_kspace_filter = False
+k_filter_dr6 = {"type": "binary_cross", "vk_mask": [-90, 90], "hk_mask": [-50, 50], "weighted": False}
+deconvolve_pixwin = True
+pixwin_dr6 = {"pix": 'CAR', "order": 0}
+remove_mean = False
+
+# spectra
+spectra_dir = data_dir + 'spectra'
+noise_model_dir = data_dir + 'noise_model'
+split_noise_dir = data_dir + 'split_noise'
+
+write_splits_spectra = True
+cov_T_E_only = False
+kspace_tf_path = "analytical"
+deconvolve_map_maker_tf_dr6 = False
+mm_tf_dr6_pa4_f150 = data_dir + "transfer_fcns/tf_unity.dat"
+mm_tf_dr6_pa4_f220 = data_dir + "transfer_fcns/tf_unity.dat"
+mm_tf_dr6_pa5_f090 = data_dir + "transfer_fcns/tf_unity.dat"
+mm_tf_dr6_pa5_f150 = data_dir + "transfer_fcns/tf_unity.dat"
+mm_tf_dr6_pa6_f090 = data_dir + "transfer_fcns/tf_unity.dat"
+mm_tf_dr6_pa6_f150 = data_dir + "transfer_fcns/tf_unity.dat"
+
+# cov 
+sq_win_alms_dir = data_dir + 'sq_win_alms'
+covariances_dir = data_dir + 'covariances'
+split_covariances_dir = data_dir + 'split_covariances'
+
+use_toeplitz_cov = True
+
+# window parameters
+ps_mask_dr6_pa4_f150 = data_dir + 'masks/act_pts_mask_fluxcut_15.0mJy_at150Ghz_rad_5.0_monster_dust_proj.fits'
+ps_mask_dr6_pa4_f220 = data_dir + 'masks/act_pts_mask_fluxcut_15.0mJy_at150Ghz_rad_5.0_monster_dust_proj.fits'
+ps_mask_dr6_pa5_f090 = data_dir + 'masks/act_pts_mask_fluxcut_15.0mJy_at150Ghz_rad_5.0_monster_dust_proj.fits'
+ps_mask_dr6_pa5_f150 = data_dir + 'masks/act_pts_mask_fluxcut_15.0mJy_at150Ghz_rad_5.0_monster_dust_proj.fits'
+ps_mask_dr6_pa6_f090 = data_dir + 'masks/act_pts_mask_fluxcut_15.0mJy_at150Ghz_rad_5.0_monster_dust_proj.fits'
+ps_mask_dr6_pa6_f150 = data_dir + 'masks/act_pts_mask_fluxcut_15.0mJy_at150Ghz_rad_5.0_monster_dust_proj.fits'
+
+gal_mask_dr6_pa4_f150 = data_dir + "masks/mask_galactic_equatorial_car_halfarcmin_pixelmatch_proj.fits"
+gal_mask_dr6_pa4_f220 = data_dir + "masks/mask_galactic_equatorial_car_halfarcmin_pixelmatch_proj.fits"
+gal_mask_dr6_pa5_f090 = data_dir + "masks/mask_galactic_equatorial_car_halfarcmin_pixelmatch_proj.fits"
+gal_mask_dr6_pa5_f150 = data_dir + "masks/mask_galactic_equatorial_car_halfarcmin_pixelmatch_proj.fits"
+gal_mask_dr6_pa6_f090 = data_dir + "masks/mask_galactic_equatorial_car_halfarcmin_pixelmatch_proj.fits"
+gal_mask_dr6_pa6_f150 = data_dir + "masks/mask_galactic_equatorial_car_halfarcmin_pixelmatch_proj.fits"
+
+coord_mask_dr6_pa4_f150 = None
+coord_mask_dr6_pa4_f220 = None
+coord_mask_dr6_pa5_f090 = None
+coord_mask_dr6_pa5_f150 = None
+coord_mask_dr6_pa6_f090 = None
+coord_mask_dr6_pa6_f150 = None
+
+apod_pts_source_degree = 0.3
+apod_survey_degree = 2
+edge_skip_rescale = 1
+cross_link_threshold = 0.97
+n_med_ivar = 3
+
+window_dir = '/scratch/gpfs/zatkins/data/simonsobs/mnms/masks/effective_est/'
+
+window_T_dr6_pa4_f150 = window_dir + "pa4_baseline_effective_est_20230816.fits"
+window_pol_dr6_pa4_f150 = window_dir + "pa4_baseline_effective_est_20230816.fits"
+window_kspace_dr6_pa4_f150 = None
+
+window_T_dr6_pa4_f220 = window_dir + "pa4_baseline_effective_est_20230816.fits"
+window_pol_dr6_pa4_f220 = window_dir + "pa4_baseline_effective_est_20230816.fits"
+window_kspace_dr6_pa4_f220 = None
+
+window_T_dr6_pa5_f090 = window_dir + "pa5_baseline_effective_est_20230816.fits"
+window_pol_dr6_pa5_f090 = window_dir + "pa5_baseline_effective_est_20230816.fits"
+window_kspace_dr6_pa5_f090 = None
+
+window_T_dr6_pa5_f150 = window_dir + "pa5_baseline_effective_est_20230816.fits"
+window_pol_dr6_pa5_f150 = window_dir + "pa5_baseline_effective_est_20230816.fits"
+window_kspace_dr6_pa5_f150 = None
+
+window_T_dr6_pa6_f090 = window_dir + "pa6_baseline_effective_est_20230816.fits"
+window_pol_dr6_pa6_f090 = window_dir + "pa6_baseline_effective_est_20230816.fits"
+window_kspace_dr6_pa6_f090 = None
+
+window_T_dr6_pa6_f150 = window_dir + "pa6_baseline_effective_est_20230816.fits"
+window_pol_dr6_pa6_f150 = window_dir + "pa6_baseline_effective_est_20230816.fits"
+window_kspace_dr6_pa6_f150 = None
+
+# maps
+map_dir = '/scratch/gpfs/ACT/dr6v4/maps/dr6v4_20230316/release/'
+
+n_splits_dr6 = 4
+src_free_maps_dr6 = True
+
+maps_dr6_pa4_f150 = [map_dir + 'cmb_night_pa4_f150_3pass_4way_set%d_map_srcfree.fits' % (i) for i in range(4)]
+maps_dr6_pa4_f220 = [map_dir + 'cmb_night_pa4_f220_3pass_4way_set%d_map_srcfree.fits' % (i) for i in range(4)]
+maps_dr6_pa5_f090 = [map_dir + 'cmb_night_pa5_f090_3pass_4way_set%d_map_srcfree.fits' % (i) for i in range(4)]
+maps_dr6_pa5_f150 = [map_dir + 'cmb_night_pa5_f150_3pass_4way_set%d_map_srcfree.fits' % (i) for i in range(4)]
+maps_dr6_pa6_f090 = [map_dir + 'cmb_night_pa6_f090_3pass_4way_set%d_map_srcfree.fits' % (i) for i in range(4)]
+maps_dr6_pa6_f150 = [map_dir + 'cmb_night_pa6_f150_3pass_4way_set%d_map_srcfree.fits' % (i) for i in range(4)]
+
+ivar_dr6_pa4_f150 = [map_dir + 'cmb_night_pa4_f150_3pass_4way_set%d_ivar.fits' % (i) for i in range(4)]
+ivar_dr6_pa4_f220 = [map_dir + 'cmb_night_pa4_f220_3pass_4way_set%d_ivar.fits' % (i) for i in range(4)]
+ivar_dr6_pa5_f090 = [map_dir + 'cmb_night_pa5_f090_3pass_4way_set%d_ivar.fits' % (i) for i in range(4)]
+ivar_dr6_pa5_f150 = [map_dir + 'cmb_night_pa5_f150_3pass_4way_set%d_ivar.fits' % (i) for i in range(4)]
+ivar_dr6_pa6_f090 = [map_dir + 'cmb_night_pa6_f090_3pass_4way_set%d_ivar.fits' % (i) for i in range(4)]
+ivar_dr6_pa6_f150 = [map_dir + 'cmb_night_pa6_f150_3pass_4way_set%d_ivar.fits' % (i) for i in range(4)]
+
+cal_dr6_pa4_f150 = 1.0072
+cal_dr6_pa4_f220 = 1.0340
+cal_dr6_pa5_f090 = 1.0219
+cal_dr6_pa5_f150 = 0.9877
+cal_dr6_pa6_f090 = 1.0182
+cal_dr6_pa6_f150 = 0.9699
+
+pol_eff_dr6_pa4_f150 = 0.9584
+pol_eff_dr6_pa4_f220 = 1.0
+pol_eff_dr6_pa5_f090 = 0.9646
+pol_eff_dr6_pa5_f150 = 0.9488
+pol_eff_dr6_pa6_f090 = 0.9789
+pol_eff_dr6_pa6_f150 = 0.9656
+
+# passbands
+passband_dir = data_dir + "passbands/"
+do_bandpass_integration = True
+freq_info_dr6_pa4_f150 = {"freq_tag": 150, "passband": passband_dir + "passband_dr6_pa4_f150.dat"}
+freq_info_dr6_pa4_f220 = {"freq_tag": 220, "passband": passband_dir + "passband_dr6_pa4_f220.dat"}
+freq_info_dr6_pa5_f090 = {"freq_tag": 90, "passband": passband_dir + "passband_dr6_pa5_f090.dat"}
+freq_info_dr6_pa5_f150 = {"freq_tag": 150, "passband": passband_dir + "passband_dr6_pa5_f150.dat"}
+freq_info_dr6_pa6_f090 = {"freq_tag": 90, "passband": passband_dir + "passband_dr6_pa6_f090.dat"}
+freq_info_dr6_pa6_f150 = {"freq_tag": 150, "passband": passband_dir + "passband_dr6_pa6_f150.dat"}
+
+# beams
+beam_dr6_pa4_f150 = data_dir + 'beams/20230130_beams/coadd_pa4_f150_night_beam_tform_jitter_cmb.txt'
+beam_dr6_pa4_f220 = data_dir + 'beams/20230130_beams/coadd_pa4_f220_night_beam_tform_jitter_cmb.txt'
+beam_dr6_pa5_f090 = data_dir + 'beams/20230130_beams/coadd_pa5_f090_night_beam_tform_jitter_cmb.txt'
+beam_dr6_pa5_f150 = data_dir + 'beams/20230130_beams/coadd_pa5_f150_night_beam_tform_jitter_cmb.txt'
+beam_dr6_pa6_f090 = data_dir + 'beams/20230130_beams/coadd_pa6_f090_night_beam_tform_jitter_cmb.txt'
+beam_dr6_pa6_f150 = data_dir + 'beams/20230130_beams/coadd_pa6_f150_night_beam_tform_jitter_cmb.txt'
+
+leakage_file_dir = data_dir + 'beams/20230130_beams/'
+
+leakage_beam_dr6_pa4_f150 = 'gamma_mp_uranus_pa4_f150.txt'
+leakage_beam_dr6_pa4_f220 = 'gamma_mp_uranus_pa4_f220.txt'
+leakage_beam_dr6_pa5_f090 = 'gamma_mp_uranus_pa5_f090.txt'
+leakage_beam_dr6_pa5_f150 = 'gamma_mp_uranus_pa5_f150.txt'
+leakage_beam_dr6_pa6_f090 = 'gamma_mp_uranus_pa6_f090.txt'
+leakage_beam_dr6_pa6_f150 = 'gamma_mp_uranus_pa6_f150.txt'
+
+# best fit params (only used for sim generation and covariances computation)
+best_fits_dir = data_dir + 'best_fits'
+
+cosmo_params = {"cosmomc_theta":0.0104085, "logA": 3.044, "ombh2": 0.02237, "omch2": 0.1200, "ns": 0.9649, "Alens": 1.0, "tau": 0.0544}
+fg_norm = {"nu_0": 150.0, "ell_0": 3000, "T_CMB": 2.725}
+fg_components = {'tt': ['tSZ_and_CIB', 'cibp', 'kSZ', 'radio', 'dust'], 'te': ['radio', 'dust'], 'ee': ['radio', 'dust'], 'bb': ['radio', 'dust'], 'tb': ['radio', 'dust'], 'eb': []} 
+fg_params = {"a_tSZ": 3.30, "a_kSZ": 1.60, "a_p": 6.90, "beta_p": 2.08, "a_c": 4.90, "beta_c": 2.20, "a_s": 3.10, "a_gtt": 8.7, "xi": 0.1, "T_d": 9.60, "a_gte": 0, "a_gtb": 0, "a_gee": 0, "a_gbb": 0, "a_pste": 0, "a_pstb": 0, "a_psee": 0, "a_psbb": 0}
+
+# sim
+iStart = 0
+iStop = 39
+sim_alm_dtype = "complex64"
+read_noise_alms_from_disk = False
+noise_sim_type = "fdw"
+noise_model_parameters = {"downgrade": 4, "mask_est_name": "dr6v3_20220316_baseline_union_mask", "mask_obs_name": "dr6v3_xlink_union_mask_0.001", "union_sources": "regular_20220316", "notes": "20220619"}
+
+# plot
+plot_dir = data_dir + 'plots'
+
+range_TT = [10, 8000]
+range_TE = [-150, 150]
+range_ET = [-150, 150]
+range_EE = [-20, 50]
+
+# planck
+planck_data_dir = data_dir + "planck_data/"