pipeline basics

README.md

@@ -2,11 +2,11 @@
 Link to mouselight article [here](https://www.hhmi.org/news/mouselight-project-maps-1000-neurons-and-counting-in-the-mouse-brain).
 
 [![Python](https://img.shields.io/badge/python-3.6-blue.svg)]()
-[![Build Status](https://travis-ci.com/neurodata/brainlight.svg?branch=master)](https://travis-ci.com/neurodata/brainlight)
-[![Documentation Status](https://readthedocs.org/projects/brainlight/badge/?version=latest)](https://brainlight.readthedocs.io/en/latest/?badge=latest)
-[![codecov](https://codecov.io/gh/neurodata/brainlight/branch/master/graph/badge.svg)](https://codecov.io/gh/neurodata/brainlight)
+[![Build Status](https://travis-ci.com/neurodata/brainlit.svg?branch=master)](https://travis-ci.com/neurodata/brainlit)
+[![Documentation Status](https://readthedocs.org/projects/brainlit/badge/?version=latest)](https://brainlit.readthedocs.io/en/latest/?badge=latest)
+[![codecov](https://codecov.io/gh/neurodata/brainlit/branch/master/graph/badge.svg)](https://codecov.io/gh/neurodata/brainlit)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 # Overview
-Tools to read and analyze brainlight data sets.
+Tools to read and analyze brainlit data sets.
 

brainlit/algorithms/__init__.py

@@ -1,5 +1,5 @@
-import brainlight.algorithms.connect_fragments
-import brainlight.algorithms.generate_fragments
+import brainlit.algorithms.connect_fragments
+import brainlit.algorithms.generate_fragments
 
-from brainlight.algorithms.connect_fragments import *
-from brainlight.algorithms.generate_fragments import *
+from brainlit.algorithms.connect_fragments import *
+from brainlit.algorithms.generate_fragments import *

brainlit/algorithms/connect_fragments/__init__.py

@@ -1,2 +1,2 @@
-from brainlight.algorithms.connect_fragments.connect_fragments import *
-from brainlight.algorithms.connect_fragments.make_connections import *
+from brainlit.algorithms.connect_fragments.connect_fragments import *
+from brainlit.algorithms.connect_fragments.make_connections import *

brainlit/algorithms/generate_fragments/__init__.py

@@ -1 +1 @@
-from brainlight.algorithms.generate_fragments.adaptive_thresh import *
+from brainlit.algorithms.generate_fragments.adaptive_thresh import *

brainlit/algorithms/generate_fragments/adaptive_thresh.py

@@ -1,39 +1,28 @@
 # Reference: http://insightsoftwareconsortium.github.io/SimpleITK-Notebooks/Python_html/300_Segmentation_Overview.html
 
-import brainlight
-from brainlight.utils.ngl_pipeline import NeuroglancerSession
+import brainlit
+from brainlit.utils.ngl_pipeline import NeuroglancerSession
 import SimpleITK as sitk
 from sklearn.mixture import GaussianMixture
 import numpy as np
 
 
-def get_seed(ngl_session):
+def get_seed(voxel):
     """ 
     Get a seed point for the center of a brain volume.
   
     Parameters
     -------
-    ngl_session : NeuroglancerSession: 
-        The NeuroglancerSession for which to return the seed.
+    voxel : tuple: 
+        The seed coordinates in x y z.
   
     Returns
     -------
     tuple
         A tuple containing the (z, y, x)-coordinates of the seed.
   
     """
-    if ngl_session.sz == None:
-        raise ValueError("ngl_session.sz may not be None")
-    if ngl_session.sy == None:
-        raise ValueError("ngl_session.sy may not be None")
-    if ngl_session.sx == None:
-        raise ValueError("ngl_session.sx may not be None")
-
-    return (
-        int(ngl_session.chunk_size[2] * ngl_session.sz / 2),
-        int(ngl_session.chunk_size[1] * ngl_session.sy / 2),
-        int(ngl_session.chunk_size[0] * ngl_session.sx / 2),
-    )
+    return (int(voxel[2]), int(voxel[1]), int(voxel[0]))
 
 
 def get_img_T1(img):
@@ -137,6 +126,7 @@ def fast_marching_seg(img, seed, stopping_value=150, sigma=0.5):
     fm_img = sitk.Cast(sitk.RescaleIntensity(fm_img), sitk.sitkUInt8)
     labels = sitk.GetArrayFromImage(fm_img)
     labels = (~labels.astype(bool)).astype(int)
+    print(labels)
     return labels
 
 

brainlit/algorithms/regression/__init__.py

@@ -1,2 +1,2 @@
-from brainlight.algorithms.regression.log_regression import *
-from brainlight.algorithms.regression.log_regression_segment import *
+from brainlit.algorithms.regression.log_regression import *
+from brainlit.algorithms.regression.log_regression_segment import *

brainlit/algorithms/regression/log_regression.py

@@ -1,12 +1,12 @@
 import numpy as np
-import brainlight
-from brainlight.utils import read_octree as octree
-from brainlight.utils import combine_swc_img
-from brainlight.utils import read_swc
-from brainlight.plot import visualize
-from brainlight.utils import swc2voxel
-from brainlight.preprocessing import image_process
-from brainlight.preprocessing import preprocess
+import brainlit
+from brainlit.utils import read_octree as octree
+from brainlit.utils import combine_swc_img
+from brainlit.utils import read_swc
+from brainlit.plot import visualize
+from brainlit.utils import swc2voxel
+from brainlit.preprocessing import image_process
+from brainlit.preprocessing import preprocess
 from scipy import ndimage as ndi
 from matplotlib import pyplot as plt
 from pathlib import Path

brainlit/algorithms/regression/log_regression_segment.py

@@ -1,12 +1,12 @@
 import numpy as np
-import brainlight
-from brainlight.utils import read_octree as octree
-from brainlight.utils import combine_swc_img
-from brainlight.utils import read_swc
-from brainlight.plot import visualize
-from brainlight.utils import swc2voxel
-from brainlight.preprocessing import image_process
-from brainlight.preprocessing import preprocess
+import brainlit
+from brainlit.utils import read_octree as octree
+from brainlit.utils import combine_swc_img
+from brainlit.utils import read_swc
+from brainlit.plot import visualize
+from brainlit.utils import swc2voxel
+from brainlit.preprocessing import image_process
+from brainlit.preprocessing import preprocess
 from scipy import ndimage as ndi
 from matplotlib import pyplot as plt
 from pathlib import Path

brainlit/plot/__init__.py

@@ -1 +1 @@
-from brainlight.plot.visualize import *
+from brainlit.plot.visualize import *

brainlit/preprocessing/__init__.py

@@ -1,2 +1,2 @@
-from brainlight.preprocessing.image_process import *
-from brainlight.preprocessing.preprocess import *
+from brainlit.preprocessing.image_process import *
+from brainlit.preprocessing.preprocess import *

brainlit/preprocessing/features/__init__.py

@@ -1,2 +1,2 @@
-from brainlight.preprocessing.features.linear_features import *
-from brainlight.preprocessing.features.neighborhood import *
+from brainlit.preprocessing.features.linear_features import *
+from brainlit.preprocessing.features.neighborhood import *

brainlit/preprocessing/features/linear_features.py

@@ -1,10 +1,10 @@
 import numpy as np
-import brainlight
-from brainlight.utils import read_octree as octree, image_getters
-from brainlight.utils import combine_swc_img
-from brainlight.utils import read_swc
-from brainlight.utils import swc2voxel
-from brainlight.preprocessing import preprocess, image_process
+import brainlit
+from brainlit.utils import read_octree as octree, image_getters
+from brainlit.utils import combine_swc_img
+from brainlit.utils import read_swc
+from brainlit.utils import swc2voxel
+from brainlit.preprocessing import preprocess, image_process
 from scipy import ndimage as ndi
 from pathlib import Path
 import pandas as pd

brainlit/preprocessing/features/neighborhood.py

@@ -1,10 +1,10 @@
 import numpy as np
-import brainlight
-from brainlight.utils import read_octree as octree, image_getters
-from brainlight.utils import combine_swc_img
-from brainlight.utils import read_swc
-from brainlight.utils import swc2voxel
-from brainlight.preprocessing import preprocess, image_process
+import brainlit
+from brainlit.utils import read_octree as octree, image_getters
+from brainlit.utils import combine_swc_img
+from brainlit.utils import read_swc
+from brainlit.utils import swc2voxel
+from brainlit.preprocessing import preprocess, image_process
 from scipy import ndimage as ndi
 from pathlib import Path
 import pandas as pd

brainlit/utils/ngl_pipeline.py

@@ -4,130 +4,42 @@
 import matplotlib.pyplot as plt
 import SimpleITK as sitk
 from cloudvolume import CloudVolume, view
-from sklearn.mixture import GaussianMixture
-import napari
-
+from cloudvolume.lib import Bbox
 
 class NeuroglancerSession:
     def __init__(self, url="s3://mouse-light-viz/precomputed_volumes/brain1", mip=1):
         self.seed = 1111
         self.url = url
+        self.cv = CloudVolume(self.url, parallel=True)
         self.mip = mip
-        cv = CloudVolume(self.url, progress=True, mip=0, cache=False, parallel=True)
-        self.cv = cv
-        cv_skel = CloudVolume(
-            self.url + "_segments", mip=0, cache=False, parallel=False
-        )
-        self.cv_skel = cv_skel
-        self.img = None
-        self.labels = None
-        self.chunk_size = (50, 50, 50)
-        self.sx = None
-        self.sy = None
-        self.sz = None
-
-    def _scale(self, verts):
-        scales = self.cv.scales[self.mip]["resolution"]
-        scaled = verts / scales
-        int_scaled = [int(i) for i in scaled]
-        return np.array(int_scaled).reshape(-1, 1)
-
-    def get_img(self, seg_id, v_id, sx=5, sy=5, sz=5):
-        """Gets image to the right (in each dimension) of the (v_id)-th vertex 
-        in the neuron skeleton.
-        Returns the image and skeleton
-        """
-        SEGID = seg_id
-        skel = self.cv_skel.skeleton.get(SEGID)
-        img = self.cv.download_point(
-            self._scale(skel.vertices[v_id]),
-            mip=self.mip,
-            size=(
-                self.chunk_size[0] * sx,
-                self.chunk_size[1] * sy,
-                self.chunk_size[2] * sz,
-            ),
-            coord_resolution=self.cv.scales[self.mip]["resolution"],
-        )
-        self.img = img
-        self.sx = sx
-        self.sy = sy
-        self.sz = sz
-
-        ## TODO: convert skel.vertices into voxel coordinates relative to img
-        return img, skel
-
-    def view_img(self, img=None, title="View"):
-        if img is None:
-            img = self.img
-        if img is None:
-            raise ValueError(f"Need an image to view it, not {type(img)}")
-        else:
-            with napari.gui_qt():
-                viewer = napari.view_image(np.squeeze(np.array(self.img)))
-                if self.labels is not None:
-                    viewer.add_labels(self.labels, name="segmentation")
-        return
-
-    def _img_to_labels(self, img=None, low=None, up=255):
-        if img is None:
-            img = self.img
-        img_T1 = sitk.GetImageFromArray(np.squeeze(img), isVector=False)
-        img_T1_255 = sitk.Cast(sitk.RescaleIntensity(img_T1), sitk.sitkUInt8)
-        seed = (
-            int(self.chunk_size[2] * self.sz / 2),
-            int(self.chunk_size[1] * self.sy / 2),
-            int(self.chunk_size[0] * self.sx / 2),
-        )
-        seg = sitk.Image(img_T1.GetSize(), sitk.sitkUInt8)
-        seg.CopyInformation(img_T1)
-        seg[seed] = 1
-        seg = sitk.BinaryDilate(seg, 1)
-        if low is None:
-            v = sitk.GetArrayFromImage(img_T1_255)
-            flat = v.flatten().reshape(-1, 1)  # img
-            gmm = GaussianMixture(n_components=2, random_state=self.seed)
-            gmm.fit(flat)
-            low = np.divide(
-                gmm.means_[0][0] * gmm.covariances_[0][0]
-                + gmm.means_[1][0] * gmm.covariances_[1][0],
-                gmm.covariances_[0][0] + gmm.covariances_[1][0],
-            )
-            print("result:", low)
-        seg_con = sitk.ConnectedThreshold(
-            img_T1_255, seedList=[seed], lower=int(np.round(low)), upper=up
-        )
-        vectorRadius = (1, 1, 1)
-        kernel = sitk.sitkBall
-        seg_clean = sitk.BinaryMorphologicalClosing(seg_con, vectorRadius, kernel)
-        labels = sitk.GetArrayFromImage(seg_clean)
-        return labels
-
-    def add_labels(self, labels=None, low=None):
-        if labels is None:
-            labels = self._img_to_labels(low=low)
-        print("ok")
-        if not np.any(labels):
-            print("Matrix is all 0!")
-        self.labels = labels
-        return self.labels
-
-    def push_img(self, img=None, url=None):
-        if url is None:
-            url = self.url + "_seg"
-        vol = CloudVolume(
-            url,
-            mip=self.mip,
-            non_aligned_writes=True,
-            fill_missing=True,
-            delete_black_uploads=False,
-        )
-        if img is None:
-            img = self.labels
-        if img is None:
-            raise Exception("you are pushing None")
-        bounds = self.img.bounds.to_list()
-        vol[
-            bounds[0] : bounds[3], bounds[1] : bounds[4], bounds[2] : bounds[5]
-        ] = img.astype("uint64")
-        return
+        self.chunk_size = self.cv.info['scales'][self.mip]['chunk_sizes'][0]
+        self.scales = self.cv.scales[self.mip]["resolution"]
+
+    def get_voxel(self, seg_id, v_id):
+        skeleton_url = "s3://mouse-light-viz/precomputed_volumes/brain1_segments"
+        cv_skel = CloudVolume(skeleton_url, mip=self.mip)
+        skel = cv_skel.skeleton.get(seg_id)
+        vertex = skel.vertices[v_id]
+        voxel = np.round(np.divide(vertex, self.scales)).astype(int)
+        return voxel
+
+    def pull_voxel(self, seg_id, v_id, nx=0, ny=0, nz=0, expand=True):
+        voxel = self.get_voxel(seg_id, v_id)
+        bounds = Bbox(voxel, voxel).expand_to_chunk_size(self.chunk_size)
+        seed = bounds.to_list()
+        shape = [self.chunk_size[0]*nx, self.chunk_size[1]*ny, self.chunk_size[2]*nz]
+        bounds = Bbox(np.subtract(seed[:3], shape), np.add(seed[3:], shape))
+        img = self.cv.download(bounds, mip=self.mip)
+        vox_in_img = voxel - np.array(bounds.to_list()[:3])
+        return np.squeeze(np.array(img)), bounds, vox_in_img
+
+    def pull_bounds_img(self, bounds):
+        img = self.cv.download(bounds, mip=self.mip)
+        return np.squeeze(np.array(img))
+
+    def pull_bounds_seg(self, bounds):
+        img = self.cv[bounds]
+        return np.squeeze(np.array(img))
+
+    def push(self, img, bounds):
+        self.cv[bounds] = img.astype("uint64")

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --cov=brainlit --doctest-modules
+#addopts = --cov=brainlit --doctest-modules
 
 
 filterwarnings = 

setup.py

@@ -4,7 +4,7 @@
 from sys import platform
 
 PACKAGE_NAME = "brainlit"
-DESCRIPTION = "Code to process and analyze brainlight data"
+DESCRIPTION = "Code to process and analyze brainlit data"
 with open("README.md", "r") as f:
     LONG_DESCRIPTION = f.read()
 AUTHOR = (