docker minus credentials

Author: tathey1

Dockerfile

@@ -19,7 +19,7 @@ RUN pip install -e .
 RUN chmod +x ./.aws.sh
 RUN ./.aws.sh
 
-
+CMD python experiments/BrainLine/scripts/soma_detect_image.py
 
 # Old version
 # FROM python:3.8-slim
@@ -28,7 +28,7 @@ RUN ./.aws.sh
 # WORKDIR /usr/src/app
 
 # #RUN apt-get update \
-# #    && apt-get install -y --no-install-recommends git \
+# #    && apt-get install -y --no-install-recommends git 
 # #    && apt-get purge -y --auto-remove \
 # RUN apt-get update
 # RUN apt-get install -y --no-install-recommends \

brainlit/BrainLine/apply_ilastik.py

@@ -495,8 +495,9 @@ def apply_ilastik_parallel(
         corners_chunks = [corners[i : i + 100] for i in range(0, len(corners), 100)]
 
         for corners_chunk in tqdm(corners_chunks, desc="corner chunks"):
-            Parallel(n_jobs=self.ncpu)(
-                delayed(self._process_chunk)(
+            if self.ncpu == 1:
+                for corner in tqdm(corners_chunk, leave=False):
+                    self._process_chunk(
                     corner[0],
                     corner[1],
                     volume_base_dir,
@@ -506,8 +507,20 @@ def apply_ilastik_parallel(
                     self.object_type,
                     results_dir,
                 )
-                for corner in tqdm(corners_chunk, leave=False)
-            )
+            else:
+                Parallel(n_jobs=self.ncpu)(
+                    delayed(self._process_chunk)(
+                        corner[0],
+                        corner[1],
+                        volume_base_dir,
+                        layer_names,
+                        threshold,
+                        data_dir,
+                        self.object_type,
+                        results_dir,
+                    )
+                    for corner in tqdm(corners_chunk, leave=False)
+                )
             for f in os.listdir(data_dir):
                 os.remove(os.path.join(data_dir, f))
 
@@ -566,50 +579,58 @@ def _process_chunk(
         fname = f"image_{c1[0]}_{c1[1]}_{c1[2]}.h5"
         fname = data_dir / fname
 
-        with h5py.File(fname, "w") as f:
-            dset = f.create_dataset("image_3channel", data=image_3channel)
-
-        subprocess.run(
-            [
-                f"{self.ilastik_path}",
-                "--headless",
-                f"--project={self.ilastik_project}",
-                fname,
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-        # subprocess.run(["/Applications/ilastik-1.3.3post3-OSX.app/Contents/ilastik-release/run_ilastik.sh", "--headless", "--project=/Users/thomasathey/Documents/mimlab/mouselight/ailey/benchmark_formal/brain3/matt_benchmark_formal_brain3.ilp", fname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-        fname_prob = str(fname).split(".")[0] + "_Probabilities.h5"
-        with h5py.File(fname_prob, "r") as f:
-            pred = f.get("exported_data")
-            if object_type == "soma":
-                fname_results = f"image_{c1[0]}_{c1[1]}_{c1[2]}_somas.txt"
-                fname_results = results_dir / fname_results
-                pred = pred[0, :, :, :]
-                mask = pred > threshold
-                labels = measure.label(mask)
-                props = measure.regionprops(labels)
+        for attempt in range(3):
+            with h5py.File(fname, "w") as f:
+                dset = f.create_dataset("image_3channel", data=image_3channel)
 
-                results = []
-                for prop in props:
-                    if prop["area"] > area_threshold:
-                        location = list(np.add(c1, prop["centroid"]))
-                        results.append(location)
-                if len(results) > 0:
-                    with open(fname_results, "w") as f2:
-                        for location in results:
-                            f2.write(str(location))
-                            f2.write("\n")
-            elif object_type == "axon":
-                dir_mask = volume_base_dir + "axon_mask"
-                vol_mask = CloudVolume(
-                    dir_mask, parallel=1, mip=mip, fill_missing=True, compress=False
-                )
-                pred = pred[1, :, :, :]
-                mask = np.array(pred > threshold).astype("uint64")
-                vol_mask[c1[0] : c2[0], c1[1] : c2[1], c1[2] : c2[2]] = mask
+            subprocess.run(
+                [
+                    f"{self.ilastik_path}",
+                    "--headless",
+                    f"--project={self.ilastik_project}",
+                    fname,
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+
+            fname_prob = str(fname).split(".")[0] + "_Probabilities.h5"
+            try:
+                with h5py.File(fname_prob, "r") as f:
+                    pred = f.get("exported_data")
+            except:
+                if attempt >= 2:
+                    raise ValueError(f"Tried to evaluate thrice and failed")
+                if os.path.isfile(fname_prob):
+                    os.remove(fname_prob)
+                continue
+            
+        if object_type == "soma":
+            fname_results = f"image_{c1[0]}_{c1[1]}_{c1[2]}_somas.txt"
+            fname_results = results_dir / fname_results
+            pred = pred[0, :, :, :]
+            mask = pred > threshold
+            labels = measure.label(mask)
+            props = measure.regionprops(labels)
+
+            results = []
+            for prop in props:
+                if prop["area"] > area_threshold:
+                    location = list(np.add(c1, prop["centroid"]))
+                    results.append(location)
+            if len(results) > 0:
+                with open(fname_results, "w") as f2:
+                    for location in results:
+                        f2.write(str(location))
+                        f2.write("\n")
+        elif object_type == "axon":
+            dir_mask = volume_base_dir + "axon_mask"
+            vol_mask = CloudVolume(
+                dir_mask, parallel=1, mip=mip, fill_missing=True, compress=False
+            )
+            pred = pred[1, :, :, :]
+            mask = np.array(pred > threshold).astype("uint64")
+            vol_mask[c1[0] : c2[0], c1[1] : c2[1], c1[2] : c2[2]] = mask
 
     def collect_soma_results(self, brain_id: str):
         """Combine all soma detections and post to neuroglancer. Intended for use after apply_ilastik_parallel.

experiments/BrainLine/scripts/soma_detect_image.py

@@ -37,9 +37,6 @@
 # Ilastik will run in "headless mode", and the following paths are needed to do so:
 ilastik_path = "/brainlit_dir/ilastik-1.4.0-Linux/run_ilastik.sh"  # "/Applications/ilastik-1.4.0b21-OSX.app/Contents/ilastik-release/run_ilastik.sh"  # "/data/tathey1/matt_wright/ilastik/ilastik-1.4.0rc5-Linux/run_ilastik.sh"  # path to ilastik executable
 ilastik_project = "/brainlit_dir/experiments/BrainLine/data/models/soma/matt_soma_rabies_pix_3ch.ilp"  # "/Users/thomasathey/Documents/mimlab/mouselight/ailey/detection_soma/matt_soma_rabies_pix_3ch.ilp"  # "/data/tathey1/matt_wright/ilastik/soma_model/matt_soma_rabies_pix_3ch.ilp"  # path to ilastik project
-ilastik_path = "/Applications/ilastik-1.4.0b21-OSX.app/Contents/ilastik-release/run_ilastik.sh"  # "/data/tathey1/matt_wright/ilastik/ilastik-1.4.0rc5-Linux/run_ilastik.sh"  # path to ilastik executable
-ilastik_project = "/Users/thomasathey/Documents/mimlab/mouselight/brainlit_parent/brainlit/experiments/BrainLine/data/models/soma/matt_soma_rabies_pix_3ch.ilp"  # "/data/tathey1/matt_wright/ilastik/soma_model/matt_soma_rabies_pix_3ch.ilp"  # path to ilastik project
-
 
 
 min_coords = [544, 1660, -1]

experiments/BrainLine/soma_analysis.ipynb

@@ -704,7 +704,7 @@
    "metadata": {},
    "source": [
     "```\n",
-    "python -m cloudreg.scripts.registration -input_s3_path precomputed://s3://smartspim-precomputed-volumes/2023_04_12/MS37/Ch_561 --output_s3_path precomputed://s3://smartspim-precomputed-volumes/2023_04_12/MS37/atlas_to_target --atlas_s3_path https://open-neurodata.s3.amazonaws.com/ara_2016/sagittal_50um/average_50um --parcellation_s3_path https://open-neurodata.s3.amazonaws.com/ara_2016/sagittal_10um/annotation_10um_2017 --atlas_orientation PIR -orientation RAI --rotation 0 0 0 --translation 0 0 0 --fixed_scale 1.07 -log_s3_path precomputed://s3://smartspim-precomputed-volumes/2023_04_12/MS37/atlas_to_target --missing_data_correction True --grid_correction False --bias_correction True --regularization 5000.0 --iterations 3000 --registration_resolution 100\n",
+    "python -m cloudreg.scripts.registration -input_s3_path precomputed://s3://smartspim-precomputed-volumes/2023_04_14/MS25/Ch_561 --output_s3_path precomputed://s3://smartspim-precomputed-volumes/2023_04_14/MS25/atlas_to_target --atlas_s3_path https://open-neurodata.s3.amazonaws.com/ara_2016/sagittal_50um/average_50um --parcellation_s3_path https://open-neurodata.s3.amazonaws.com/ara_2016/sagittal_10um/annotation_10um_2017 --atlas_orientation PIR -orientation RPI --rotation 0 0 0 --translation 0 0 0 --fixed_scale 1. -log_s3_path precomputed://s3://smartspim-precomputed-volumes/2023_04_14/MS25/atlas_to_target --missing_data_correction True --grid_correction False --bias_correction True --regularization 5000.0 --iterations 3000 --registration_resolution 100\n",
     "```"
    ]
   },