Skip to content

Commit

Permalink
Add PointCloud sample
Browse files Browse the repository at this point in the history
  • Loading branch information
shu223 committed Sep 15, 2018
1 parent a20e59e commit 23eccc3
Show file tree
Hide file tree
Showing 10 changed files with 462 additions and 15 deletions.
9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,16 @@ Available in iOS 12 or later.

### ARKit Depth

Depth visualization on ARKit. The depth on ARKit is available only when using `ARFaceTrackingConfiguration`.

![](README_resources/arkit-depth.gif)

Depth visualization on ARKit. The depth on ARKit is available only when using `ARFaceTrackingConfiguration`.
### 2D image in 3D space",

A demo to render a 2D image in 3D space.

### 3D rendering
![](README_resources/3d.gif)

[WIP] Render a 2D image in 3D space using depth.

### AR occlusion

Expand Down
Binary file added README_resources/3d.gif
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
30 changes: 27 additions & 3 deletions iOS-Depth-Sampler.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
8A7F88042148199000170436 /* ARKitDepth.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A7F88022148199000170436 /* ARKitDepth.storyboard */; };
8A7F88052148199000170436 /* ARKitDepthViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A7F88032148199000170436 /* ARKitDepthViewController.swift */; };
8A7F8807214819FF00170436 /* TrackingState+Description.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A7F8806214819FF00170436 /* TrackingState+Description.swift */; };
8A9A09DA214BA92E00432B1A /* PointCloud.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A9A09D8214BA92E00432B1A /* PointCloud.swift */; };
8AAF370F21480CAC006A551B /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8AAF370E21480CAC006A551B /* Main.storyboard */; };
8AAF371321480D32006A551B /* RootViewCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AAF371021480D31006A551B /* RootViewCell.swift */; };
8AAF371421480D32006A551B /* RootViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AAF371121480D32006A551B /* RootViewController.swift */; };
Expand All @@ -39,6 +40,9 @@
8AB7602B213631AD0051C3D1 /* PassThrough.metal in Sources */ = {isa = PBXBuildFile; fileRef = 8AB7602A213631AD0051C3D1 /* PassThrough.metal */; };
8AFD6510214AFA0B0085F997 /* image-with-depth.jpg in Resources */ = {isa = PBXBuildFile; fileRef = 8AFD650F214AFA0B0085F997 /* image-with-depth.jpg */; };
8AFD6512214AFB1F0085F997 /* image-with-matte.jpg in Resources */ = {isa = PBXBuildFile; fileRef = 8AFD6511214AFB1F0085F997 /* image-with-matte.jpg */; };
8AFD6516214B08EB0085F997 /* PointCloudViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AFD6514214B08EB0085F997 /* PointCloudViewController.swift */; };
8AFD6517214B08EB0085F997 /* PointCloud.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8AFD6515214B08EB0085F997 /* PointCloud.storyboard */; };
8AFD6519214B0CC80085F997 /* CIImage+Utils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AFD6518214B0CC80085F997 /* CIImage+Utils.swift */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
Expand All @@ -63,6 +67,7 @@
8A7F88022148199000170436 /* ARKitDepth.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = ARKitDepth.storyboard; sourceTree = "<group>"; };
8A7F88032148199000170436 /* ARKitDepthViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ARKitDepthViewController.swift; sourceTree = "<group>"; };
8A7F8806214819FF00170436 /* TrackingState+Description.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "TrackingState+Description.swift"; sourceTree = "<group>"; };
8A9A09D8214BA92E00432B1A /* PointCloud.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PointCloud.swift; sourceTree = "<group>"; };
8AABBCCC2136463900C938EE /* iOS-Depth-Sampler-Bridging-Header.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "iOS-Depth-Sampler-Bridging-Header.h"; sourceTree = "<group>"; };
8AAF370E21480CAC006A551B /* Main.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = Main.storyboard; sourceTree = "<group>"; };
8AAF371021480D31006A551B /* RootViewCell.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RootViewCell.swift; sourceTree = "<group>"; };
Expand All @@ -76,6 +81,9 @@
8AB7602A213631AD0051C3D1 /* PassThrough.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = PassThrough.metal; sourceTree = "<group>"; };
8AFD650F214AFA0B0085F997 /* image-with-depth.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = "image-with-depth.jpg"; sourceTree = "<group>"; };
8AFD6511214AFB1F0085F997 /* image-with-matte.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = "image-with-matte.jpg"; sourceTree = "<group>"; };
8AFD6514214B08EB0085F997 /* PointCloudViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PointCloudViewController.swift; sourceTree = "<group>"; };
8AFD6515214B08EB0085F997 /* PointCloud.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = PointCloud.storyboard; sourceTree = "<group>"; };
8AFD6518214B0CC80085F997 /* CIImage+Utils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CIImage+Utils.swift"; sourceTree = "<group>"; };
8FA83CA5E0F8F1F5F2E5DF63 /* Pods-iOS-Depth-Sampler.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-iOS-Depth-Sampler.release.xcconfig"; path = "Pods/Target Support Files/Pods-iOS-Depth-Sampler/Pods-iOS-Depth-Sampler.release.xcconfig"; sourceTree = "<group>"; };
DDC3399A5A9E8877FB64BCD8 /* Pods-iOS-Depth-Sampler.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-iOS-Depth-Sampler.debug.xcconfig"; path = "Pods/Target Support Files/Pods-iOS-Depth-Sampler/Pods-iOS-Depth-Sampler.debug.xcconfig"; sourceTree = "<group>"; };
FDB82C9008285D580EA95E40 /* Pods_iOS_Depth_Sampler.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_iOS_Depth_Sampler.framework; sourceTree = BUILT_PRODUCTS_DIR; };
Expand Down Expand Up @@ -128,6 +136,7 @@
8A171F14214928BB00121E80 /* AVDepthData+Utils.swift */,
8A603D892149460E00F848A4 /* PhotosUtils.swift */,
8A603D8B2149590F00F848A4 /* UIAlertController+Utils.swift */,
8AFD6518214B0CC80085F997 /* CIImage+Utils.swift */,
);
path = Utils;
sourceTree = "<group>";
Expand All @@ -150,13 +159,13 @@
path = Renderer;
sourceTree = "<group>";
};
8A603D84214945A900F848A4 /* Portrait Matte */ = {
8A603D84214945A900F848A4 /* Portrait-Matte */ = {
isa = PBXGroup;
children = (
8A603D85214945A900F848A4 /* PortraitMatteViewController.swift */,
8A603D86214945A900F848A4 /* PortraitMatte.storyboard */,
);
path = "Portrait Matte";
path = "Portrait-Matte";
sourceTree = "<group>";
};
8A603D8D21495B8A00F848A4 /* Samples */ = {
Expand All @@ -165,8 +174,9 @@
8A171F052148FE4100121E80 /* Realtime-Depth */,
8A171F16214930D100121E80 /* Realtime-Mask */,
8AAF371721481369006A551B /* Depth-from-Camera-Roll */,
8A603D84214945A900F848A4 /* Portrait Matte */,
8A603D84214945A900F848A4 /* Portrait-Matte */,
8A7F88012148199000170436 /* ARKit */,
8AFD6513214B08EB0085F997 /* PointCloud */,
);
path = Samples;
sourceTree = "<group>";
Expand Down Expand Up @@ -248,6 +258,16 @@
path = Resources;
sourceTree = "<group>";
};
8AFD6513214B08EB0085F997 /* PointCloud */ = {
isa = PBXGroup;
children = (
8A9A09D8214BA92E00432B1A /* PointCloud.swift */,
8AFD6514214B08EB0085F997 /* PointCloudViewController.swift */,
8AFD6515214B08EB0085F997 /* PointCloud.storyboard */,
);
path = PointCloud;
sourceTree = "<group>";
};
/* End PBXGroup section */

/* Begin PBXNativeTarget section */
Expand Down Expand Up @@ -314,6 +334,7 @@
8AFD6510214AFA0B0085F997 /* image-with-depth.jpg in Resources */,
8A702FA2212A3BD5004039F7 /* LaunchScreen.storyboard in Resources */,
8A171F0C2148FE4100121E80 /* RealtimeDepth.storyboard in Resources */,
8AFD6517214B08EB0085F997 /* PointCloud.storyboard in Resources */,
8A171F1B214930D100121E80 /* RealtimeDepthMask.storyboard in Resources */,
8A702F9F212A3BD5004039F7 /* Assets.xcassets in Resources */,
8AAF370F21480CAC006A551B /* Main.storyboard in Resources */,
Expand Down Expand Up @@ -376,10 +397,12 @@
8AB7602B213631AD0051C3D1 /* PassThrough.metal in Sources */,
8A7F88052148199000170436 /* ARKitDepthViewController.swift in Sources */,
8AAF371421480D32006A551B /* RootViewController.swift in Sources */,
8A9A09DA214BA92E00432B1A /* PointCloud.swift in Sources */,
8AAF371C21481369006A551B /* DepthFromCameraRollViewController.swift in Sources */,
8A171F13214905D400121E80 /* CVPixelBuffer+CIImage.swift in Sources */,
8AAF371B21481369006A551B /* UIImage+CVPixelBuffer.swift in Sources */,
8AAF371521480D32006A551B /* SampleDataSource.swift in Sources */,
8AFD6516214B08EB0085F997 /* PointCloudViewController.swift in Sources */,
8AB76029213628440051C3D1 /* MetalRenderer.swift in Sources */,
8A7F8807214819FF00170436 /* TrackingState+Description.swift in Sources */,
8A603D8A2149460E00F848A4 /* PhotosUtils.swift in Sources */,
Expand All @@ -393,6 +416,7 @@
8A603D87214945A900F848A4 /* PortraitMatteViewController.swift in Sources */,
8A702FAF212A3CDE004039F7 /* AVCaptureDevice+Extension.swift in Sources */,
8A702FAD212A3CDE004039F7 /* VideoCameraType.swift in Sources */,
8AFD6519214B0CC80085F997 /* CIImage+Utils.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
5 changes: 5 additions & 0 deletions iOS-Depth-Sampler/SampleDataSource.swift
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,10 @@ struct SampleDataSource {
detail: "Depth visualization on ARKit",
classPrefix: "ARKitDepth"
),
Sample(
title: "2D image in 3D space",
detail: "A demo to render a 2D image in 3D space",
classPrefix: "PointCloud"
),
]
}
73 changes: 73 additions & 0 deletions iOS-Depth-Sampler/Samples/PointCloud/PointCloud.storyboard
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
<?xml version="1.0" encoding="UTF-8"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
<device id="retina4_7" orientation="portrait">
<adaptation id="fullscreen"/>
</device>
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--Point Cloudl View Controller-->
<scene sceneID="tne-QT-ifu">
<objects>
<viewController id="BYZ-38-t0r" customClass="PointCloudlViewController" customModule="iOS_Depth_Sampler" customModuleProvider="target" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<subviews>
<imageView userInteractionEnabled="NO" contentMode="scaleAspectFit" horizontalHuggingPriority="251" verticalHuggingPriority="251" translatesAutoresizingMaskIntoConstraints="NO" id="MhN-6A-Pa1">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
</imageView>
<sceneKitView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="OtL-GN-ZSn">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
</sceneKitView>
<segmentedControl opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="plain" selectedSegmentIndex="0" translatesAutoresizingMaskIntoConstraints="NO" id="7Ie-V0-YgW">
<rect key="frame" x="16" y="40" width="343" height="29"/>
<segments>
<segment title="2D"/>
<segment title="3D"/>
</segments>
<connections>
<action selector="typeSegmentChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="KiL-Mt-1H8"/>
</connections>
</segmentedControl>
<button opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="roundedRect" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="PcE-WA-x3Y">
<rect key="frame" x="142" y="615" width="91" height="32"/>
<fontDescription key="fontDescription" type="system" weight="medium" pointSize="16"/>
<state key="normal" title="Open Picker"/>
<connections>
<action selector="pickerBtnTapped" destination="BYZ-38-t0r" eventType="touchUpInside" id="q2R-GC-lZp"/>
</connections>
</button>
</subviews>
<color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<constraints>
<constraint firstItem="7Ie-V0-YgW" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="16" id="7E3-LK-McK"/>
<constraint firstItem="6Tk-OE-BBY" firstAttribute="trailing" secondItem="7Ie-V0-YgW" secondAttribute="trailing" constant="16" id="EiH-MV-Jgg"/>
<constraint firstAttribute="bottom" secondItem="OtL-GN-ZSn" secondAttribute="bottom" id="FgZ-vE-qvg"/>
<constraint firstItem="PcE-WA-x3Y" firstAttribute="centerX" secondItem="8bC-Xf-vdC" secondAttribute="centerX" id="Goj-mB-VZ5"/>
<constraint firstItem="OtL-GN-ZSn" firstAttribute="top" secondItem="8bC-Xf-vdC" secondAttribute="top" id="HuF-pH-y9K"/>
<constraint firstItem="6Tk-OE-BBY" firstAttribute="bottom" secondItem="PcE-WA-x3Y" secondAttribute="bottom" constant="20" id="In9-1W-mGa"/>
<constraint firstItem="MhN-6A-Pa1" firstAttribute="top" secondItem="8bC-Xf-vdC" secondAttribute="top" id="NSG-27-s6R"/>
<constraint firstAttribute="trailing" secondItem="OtL-GN-ZSn" secondAttribute="trailing" id="caT-bo-eFr"/>
<constraint firstAttribute="bottom" secondItem="MhN-6A-Pa1" secondAttribute="bottom" id="eKA-ec-E0N"/>
<constraint firstAttribute="trailing" secondItem="MhN-6A-Pa1" secondAttribute="trailing" id="eMU-gE-H2p"/>
<constraint firstItem="MhN-6A-Pa1" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="idX-Vv-xGB"/>
<constraint firstItem="7Ie-V0-YgW" firstAttribute="top" secondItem="6Tk-OE-BBY" secondAttribute="top" constant="20" id="kFg-81-8NM"/>
<constraint firstItem="OtL-GN-ZSn" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="rhV-Uk-jG4"/>
</constraints>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
<connections>
<outlet property="imageView" destination="MhN-6A-Pa1" id="zwX-Y4-Ifp"/>
<outlet property="scnView" destination="OtL-GN-ZSn" id="bHL-jy-Juc"/>
<outlet property="typeSegmentedCtl" destination="7Ie-V0-YgW" id="Sqn-pn-RBU"/>
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
</objects>
</scene>
</scenes>
</document>
81 changes: 81 additions & 0 deletions iOS-Depth-Sampler/Samples/PointCloud/PointCloud.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
//
// PointCloud.swift
// iOS-Depth-Sampler
//
// Created by Shuichi Tsutsumi on 2018/09/14.
// Copyright © 2018 Shuichi Tsutsumi. All rights reserved.
//
// Reference: https://github.com/eugeneu/PoindCloudRenderer

import SceneKit

struct PointCloudVertex {
var x: Float, y: Float, z: Float
var r: Float, g: Float, b: Float
}

@objc class PointCloud: NSObject {

var pointCloud : [SCNVector3] = []
var colors: [UInt8] = []

public func pointCloudNode() -> SCNNode {
let points = self.pointCloud
var vertices = Array(repeating: PointCloudVertex(x: 0,y: 0,z: 0,r: 0,g: 0,b: 0), count: points.count)

for i in 0...(points.count-1) {
let p = points[i]
vertices[i].x = Float(p.x)
vertices[i].y = Float(p.y)
vertices[i].z = Float(p.z)
vertices[i].r = Float(colors[i * 4]) / 255.0
vertices[i].g = Float(colors[i * 4 + 1]) / 255.0
vertices[i].b = Float(colors[i * 4 + 2]) / 255.0
}

let node = buildNode(points: vertices)
return node
}

private func buildNode(points: [PointCloudVertex]) -> SCNNode {
let vertexData = NSData(
bytes: points,
length: MemoryLayout<PointCloudVertex>.size * points.count
)
let positionSource = SCNGeometrySource(
data: vertexData as Data,
semantic: SCNGeometrySource.Semantic.vertex,
vectorCount: points.count,
usesFloatComponents: true,
componentsPerVector: 3,
bytesPerComponent: MemoryLayout<Float>.size,
dataOffset: 0,
dataStride: MemoryLayout<PointCloudVertex>.size
)
let colorSource = SCNGeometrySource(
data: vertexData as Data,
semantic: SCNGeometrySource.Semantic.color,
vectorCount: points.count,
usesFloatComponents: true,
componentsPerVector: 3,
bytesPerComponent: MemoryLayout<Float>.size,
dataOffset: MemoryLayout<Float>.size * 3,
dataStride: MemoryLayout<PointCloudVertex>.size
)
let element = SCNGeometryElement(
data: nil,
primitiveType: .point,
primitiveCount: points.count,
bytesPerIndex: MemoryLayout<Int>.size
)

// for bigger dots
element.pointSize = 1
element.minimumPointScreenSpaceRadius = 1
element.maximumPointScreenSpaceRadius = 5

let pointsGeometry = SCNGeometry(sources: [positionSource, colorSource], elements: [element])

return SCNNode(geometry: pointsGeometry)
}
}
Loading

0 comments on commit 23eccc3

Please sign in to comment.