Skip to content

Commit 6235644

Browse files
alyssaxuualyssaxuu
committedJan 8, 2023
Test running image classification client-side
1 parent 5bf52e2 commit 6235644

File tree

6 files changed

+289
-141
lines changed

6 files changed

+289
-141
lines changed
 

‎.env

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ VERCEL_GIT_COMMIT_MESSAGE=""
1313
VERCEL_GIT_COMMIT_AUTHOR_LOGIN=""
1414
VERCEL_GIT_COMMIT_AUTHOR_NAME=""
1515
VERCEL_GIT_PULL_REQUEST_ID=""
16-
MODEL_URL="YOUR_MODEL_URL"
16+
NEXT_PUBLIC_MODEL_URL="YOUR_MODEL_URL"

‎components/ML.js

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import React from "react";
2+
import { Readable } from "stream";
3+
import * as tf from "@tensorflow/tfjs";
4+
import { model } from "@tensorflow/tfjs";
5+
6+
// Create a function to classify the image with a Readable stream
7+
const classifyImage = async (image) => {
8+
//const readStream = Readable.from(image);
9+
//const tf = await loadTf(readStream);
10+
11+
let Model;
12+
13+
function indexOfMax(arr) {
14+
if (arr.length === 0) {
15+
return -1;
16+
}
17+
18+
var max = arr[0];
19+
var maxIndex = 0;
20+
21+
for (var i = 1; i < arr.length; i++) {
22+
if (arr[i] > max) {
23+
maxIndex = i;
24+
max = arr[i];
25+
}
26+
}
27+
28+
return maxIndex;
29+
}
30+
31+
if (!Model) {
32+
console.log(tf);
33+
// Make sure to update the MODEL_URL environment variable on Vercel (or .env file) to point to the model.json on /public. Does not work on localhost or with a local file path.
34+
Model = await tf.loadGraphModel(process.env.NEXT_PUBLIC_MODEL_URL);
35+
}
36+
37+
const b = Buffer.from(
38+
image.replace(/^data:image\/(png|jpeg);base64,/, ""),
39+
"base64"
40+
);
41+
//const input = await tf.node.decodeImage(b);
42+
var img = new Image();
43+
img.src = image;
44+
var input = tf.browser.fromPixels(img);
45+
const result = await Model.predict(tf.expandDims(input.cast("float32"), 0));
46+
const index = await result.data();
47+
48+
const final = {
49+
number: indexOfMax(index) + 1,
50+
tensor: JSON.stringify(input.arraySync()),
51+
};
52+
53+
return final;
54+
};
55+
56+
export default classifyImage;

‎components/Viewer.js

+160-137
Original file line numberDiff line numberDiff line change
@@ -1,167 +1,190 @@
11
import React, { useState, useRef, useEffect } from "react";
2-
import styles from "./../styles/Viewer.module.css"
2+
import styles from "./../styles/Viewer.module.css";
33
import Overlay from "./Overlay";
4-
import {Camera} from "react-camera-pro";
4+
import { Camera } from "react-camera-pro";
5+
import classifyImage from "./ML";
56

67
const Viewer = (props) => {
78
const camera = useRef(null);
8-
const [numberOfCameras, setNumberOfCameras] = useState(0);
9+
const [numberOfCameras, setNumberOfCameras] = useState(0);
910
const [image, setImage] = useState(null);
10-
const [scanning, setScanning] = useState(false);
11-
const [loading, setLoading] = useState(false);
12-
const [recyclable, setRecyclable] = useState(false);
13-
const [next, setNext] = useState(false);
14-
const [plastic, setPlastic] = useState(1);
15-
const inputRef = useRef(null);
16-
const canvas = useRef(null);
17-
const [backImage, setBackImage] = useState(null);
11+
const [scanning, setScanning] = useState(false);
12+
const [loading, setLoading] = useState(false);
13+
const [recyclable, setRecyclable] = useState(false);
14+
const [next, setNext] = useState(false);
15+
const [plastic, setPlastic] = useState(1);
16+
const inputRef = useRef(null);
17+
const canvas = useRef(null);
18+
const [backImage, setBackImage] = useState(null);
1819

19-
const handleUploadClick = () => {
20+
const handleUploadClick = () => {
2021
inputRef.current?.click();
2122
};
2223

23-
const handleReturn = () => {
24-
setScanning(false);
25-
setLoading(false);
26-
setNext(false);
27-
}
24+
const handleReturn = () => {
25+
setScanning(false);
26+
setLoading(false);
27+
setNext(false);
28+
};
2829

29-
const fetchData = async() => {
30-
try {
31-
const res = await fetch("/api/ml", {
32-
method: 'POST',
33-
headers: {
34-
'Accept': 'application/json',
35-
'Content-Type': 'application/json'
36-
},
37-
body: JSON.stringify({image:image})
38-
});
39-
const data = await res.json();
40-
return data;
41-
} catch (err) {
42-
console.log(err);
43-
}
44-
}
30+
const fetchData = async () => {
31+
/*
32+
try {
33+
const res = await fetch("/api/ml", {
34+
method: "POST",
35+
headers: {
36+
Accept: "application/json",
37+
"Content-Type": "application/json",
38+
},
39+
body: JSON.stringify({ image: image }),
40+
});
41+
const data = await res.json();
42+
return data;
43+
} catch (err) {
44+
console.log(err);
45+
}
46+
*/
47+
return classifyImage(image);
48+
};
4549

46-
useEffect(() => {
47-
const run = async() => {
48-
const data = await fetchData();
49-
props.setTensor(data.tensor);
50-
props.setPred(parseInt(data.number))
51-
setPlastic(data.number);
52-
if (plastic === 1 || plastic === 2 || plastic === 5) {
53-
setRecyclable(true);
54-
} else {
55-
setRecyclable(false);
56-
}
57-
setLoading(false);
58-
setNext(true);
59-
}
60-
if (image) {
61-
scan()
62-
run();
63-
}
64-
}, [image]);
50+
useEffect(() => {
51+
const run = async () => {
52+
const data = await fetchData();
53+
props.setTensor(data.tensor);
54+
props.setPred(parseInt(data.number));
55+
setPlastic(data.number);
56+
if (plastic === 1 || plastic === 2 || plastic === 5) {
57+
setRecyclable(true);
58+
} else {
59+
setRecyclable(false);
60+
}
61+
setLoading(false);
62+
setNext(true);
63+
};
64+
if (image) {
65+
scan();
66+
run();
67+
}
68+
}, [image]);
6569

66-
function cropImage(img) {
67-
setBackImage(img)
70+
function cropImage(img) {
71+
setBackImage(img);
6872
const originalImage = new Image();
69-
originalImage.src = img;
73+
originalImage.src = img;
7074
const ctx = canvas.current.getContext("2d");
71-
72-
originalImage.addEventListener('load', function() {
73-
const originalWidth = originalImage.naturalWidth;
74-
const originalHeight = originalImage.naturalHeight;
75-
const aspectRatio = originalWidth/originalHeight;
76-
let newHeight = Math.floor(224/aspectRatio);
77-
let y = (newHeight/2)-112;
78-
79-
canvas.width = 224;
80-
canvas.height = 224;
81-
82-
ctx.drawImage(originalImage, 0, -y, 224, newHeight);
83-
setImage(canvas.current.toDataURL("image/jpeg"));
75+
76+
originalImage.addEventListener("load", function () {
77+
const originalWidth = originalImage.naturalWidth;
78+
const originalHeight = originalImage.naturalHeight;
79+
const aspectRatio = originalWidth / originalHeight;
80+
let newHeight = Math.floor(224 / aspectRatio);
81+
let y = newHeight / 2 - 112;
82+
83+
canvas.width = 224;
84+
canvas.height = 224;
85+
86+
ctx.drawImage(originalImage, 0, -y, 224, newHeight);
87+
setImage(canvas.current.toDataURL("image/jpeg"));
8488
});
85-
}
89+
}
8690

87-
const handleFileChange = (event) => {
91+
const handleFileChange = (event) => {
8892
if (!event.target.files) {
8993
return;
9094
}
91-
var reader = new FileReader();
92-
reader.readAsDataURL(event.target.files[0]);
93-
reader.onloadend = function() {
94-
var base64data = reader.result;
95-
cropImage(base64data);
96-
}
95+
var reader = new FileReader();
96+
reader.readAsDataURL(event.target.files[0]);
97+
reader.onloadend = function () {
98+
var base64data = reader.result;
99+
cropImage(base64data);
100+
};
97101
};
98102

99-
const scan = () => {
100-
setScanning(true);
101-
setLoading(true);
102-
}
103+
const scan = () => {
104+
setScanning(true);
105+
setLoading(true);
106+
};
103107

104-
const takePhoto = async () => {
105-
const photo = camera.current.takePhoto();
106-
cropImage(photo);
107-
}
108+
const takePhoto = async () => {
109+
const photo = camera.current.takePhoto();
110+
cropImage(photo);
111+
};
108112

109113
return (
110-
<div className={styles.camera}>
111-
<canvas className={styles.canvas} width={224} height={224} ref={canvas}></canvas>
112-
{!next &&
113-
<img src="example.svg" className={styles.example}/>
114-
}
115-
{!next &&
116-
<div className={styles.title}>Scan</div>
117-
}
118-
{scanning &&
119-
<img className={styles.preview} src={backImage}/>
120-
}
121-
<img className={styles.goback} src="close.svg" onClick={() => props.setView(false)}/>
122-
<div className={styles.overlay}></div>
123-
{
124-
loading &&
125-
<div className={styles.scanline}></div>
126-
}
127-
{!next &&
128-
<img className={styles.scanarea} src="scanarea.svg"/>
129-
}
130-
{!scanning &&
131-
<Camera ref={camera} numberOfCamerasCallback={setNumberOfCameras} facingMode='environment' />
132-
}
133-
<img src={image} alt='Image preview' className={styles.image} />
134-
{!scanning &&
135-
<button
136-
className={styles.takephoto}
137-
onClick={takePhoto}
138-
>
139-
<img src="cam.svg"/>
140-
</button>
141-
}
142-
{!scanning &&
143-
<button
144-
className={styles.switchcamera}
145-
hidden={numberOfCameras <= 1}
146-
onClick={() => {
147-
camera.current.switchCamera();
148-
}}
149-
>
150-
<img src="rotate.svg"/>
151-
</button>
152-
}
153-
{!scanning &&
154-
<img src="upload.svg" className={styles.upload} onClick={handleUploadClick}/>
155-
}
156-
<Overlay pred={props.pred} setPred={props.setPred} tensor={props.tensor} setTensor={props.setTensor} setNum={props.setNum} num={props.num} loading={loading} region={props.region} setPlastic={setPlastic} setRecyclable={setRecyclable} scanning={scanning} ready={next} plastic={plastic} recyclable={recyclable} handleReturn={handleReturn} />
157-
<input
114+
<div className={styles.camera}>
115+
<canvas
116+
className={styles.canvas}
117+
width={224}
118+
height={224}
119+
ref={canvas}
120+
></canvas>
121+
{!next && <img src="example.svg" className={styles.example} />}
122+
{!next && <div className={styles.title}>Scan</div>}
123+
{scanning && <img className={styles.preview} src={backImage} />}
124+
<img
125+
className={styles.goback}
126+
src="close.svg"
127+
onClick={() => props.setView(false)}
128+
/>
129+
<div className={styles.overlay}></div>
130+
{loading && <div className={styles.scanline}></div>}
131+
{!next && <img className={styles.scanarea} src="scanarea.svg" />}
132+
{!scanning && (
133+
<Camera
134+
ref={camera}
135+
numberOfCamerasCallback={setNumberOfCameras}
136+
facingMode="environment"
137+
/>
138+
)}
139+
<img src={image} alt="Image preview" className={styles.image} />
140+
{!scanning && (
141+
<button className={styles.takephoto} onClick={takePhoto}>
142+
<img src="cam.svg" />
143+
</button>
144+
)}
145+
{!scanning && (
146+
<button
147+
className={styles.switchcamera}
148+
hidden={numberOfCameras <= 1}
149+
onClick={() => {
150+
camera.current.switchCamera();
151+
}}
152+
>
153+
<img src="rotate.svg" />
154+
</button>
155+
)}
156+
{!scanning && (
157+
<img
158+
src="upload.svg"
159+
className={styles.upload}
160+
onClick={handleUploadClick}
161+
/>
162+
)}
163+
<Overlay
164+
pred={props.pred}
165+
setPred={props.setPred}
166+
tensor={props.tensor}
167+
setTensor={props.setTensor}
168+
setNum={props.setNum}
169+
num={props.num}
170+
loading={loading}
171+
region={props.region}
172+
setPlastic={setPlastic}
173+
setRecyclable={setRecyclable}
174+
scanning={scanning}
175+
ready={next}
176+
plastic={plastic}
177+
recyclable={recyclable}
178+
handleReturn={handleReturn}
179+
/>
180+
<input
158181
type="file"
159182
ref={inputRef}
160183
onChange={handleFileChange}
161-
style={{ display: 'none' }}
184+
style={{ display: "none" }}
162185
/>
163-
</div>
164-
)
165-
}
186+
</div>
187+
);
188+
};
166189

167-
export default Viewer;
190+
export default Viewer;

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"lint": "next lint"
1010
},
1111
"dependencies": {
12-
"@tensorflow/tfjs": "^4.1.0",
12+
"@tensorflow/tfjs": "^4.2.0",
1313
"@tensorflow/tfjs-node": "^4.1.0",
1414
"eslint": "8.29.0",
1515
"eslint-config-next": "13.0.6",

‎public/sw.js.map

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎yarn.lock

+70-1
Original file line numberDiff line numberDiff line change
@@ -1676,6 +1676,14 @@
16761676
"@types/seedrandom" "^2.4.28"
16771677
seedrandom "^3.0.5"
16781678

1679+
"@tensorflow/tfjs-backend-cpu@4.2.0":
1680+
version "4.2.0"
1681+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.2.0.tgz#24912f8ef20b04ca4056365d83e5cc5a1671581f"
1682+
integrity sha512-8HWg9J69m0Ovc6w8TVhhixMOcwA3t/NPXLblOA/sgJ+/JD5gsbpLWJk4QISQyb1RnpSVzw6PX3BSMTJU7hWVOg==
1683+
dependencies:
1684+
"@types/seedrandom" "^2.4.28"
1685+
seedrandom "^3.0.5"
1686+
16791687
"@tensorflow/tfjs-backend-webgl@2.8.6":
16801688
version "2.8.6"
16811689
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-2.8.6.tgz#b88b4276a2ff4e23b05470c506b5c720bf6eb8c3"
@@ -1699,6 +1707,17 @@
16991707
"@types/webgl-ext" "0.0.30"
17001708
seedrandom "^3.0.5"
17011709

1710+
"@tensorflow/tfjs-backend-webgl@4.2.0":
1711+
version "4.2.0"
1712+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.2.0.tgz#684368f9a2605511d6d6753bf6fe9a08c73f6791"
1713+
integrity sha512-Qvf+hD5pSh+xi48kChSGzcDKJemkc4EKfoVVjuxl4k25ZUPwuEd7zZUAtinkLu1dzgHNyvePZY8k+9rVm59HJA==
1714+
dependencies:
1715+
"@tensorflow/tfjs-backend-cpu" "4.2.0"
1716+
"@types/offscreencanvas" "~2019.3.0"
1717+
"@types/seedrandom" "^2.4.28"
1718+
"@types/webgl-ext" "0.0.30"
1719+
seedrandom "^3.0.5"
1720+
17021721
"@tensorflow/tfjs-converter@2.8.6":
17031722
version "2.8.6"
17041723
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-2.8.6.tgz#6182d302ae883e0c45f47674a78bdd33e23db3a6"
@@ -1709,6 +1728,11 @@
17091728
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.1.0.tgz#c4c4ee70936e5f1d52adf6e1fcf5dd1eb8a984a4"
17101729
integrity sha512-pR4TSUI949a/5uUWjHga8xzxD7Y9AbobHJtCmg86Bldfl2GV8aibz87cNjpoO+iUhH8WZo1TOJy8GpM+MPT2DA==
17111730

1731+
"@tensorflow/tfjs-converter@4.2.0":
1732+
version "4.2.0"
1733+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.2.0.tgz#3a51d446dc0d6194f31407345891966e0a77937f"
1734+
integrity sha512-m+E2KJM6yGQdi8ElzWpChdD/JaqhWMCi9yK70v/ndkOaCL2q2UN48nYP2T5S15vkDvMIgzAQyZfh7hxQsMuvRQ==
1735+
17121736
"@tensorflow/tfjs-core@2.8.6":
17131737
version "2.8.6"
17141738
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-2.8.6.tgz#d5e9d5fc1d1a83e3fbf80942f3154300a9f82494"
@@ -1734,6 +1758,20 @@
17341758
node-fetch "~2.6.1"
17351759
seedrandom "^3.0.5"
17361760

1761+
"@tensorflow/tfjs-core@4.2.0":
1762+
version "4.2.0"
1763+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-4.2.0.tgz#34dc455c0a00feac12015caec34a54414a404230"
1764+
integrity sha512-uuHkiWVC8b00ngFbHvAV7J7haRlN/9PEdeenCi0CzBjgKd7aN25wPWaoN0TSQcU+GT4FJ8mofMZ9VBYZ/s/WLg==
1765+
dependencies:
1766+
"@types/long" "^4.0.1"
1767+
"@types/offscreencanvas" "~2019.7.0"
1768+
"@types/seedrandom" "^2.4.28"
1769+
"@types/webgl-ext" "0.0.30"
1770+
"@webgpu/types" "0.1.21"
1771+
long "4.0.0"
1772+
node-fetch "~2.6.1"
1773+
seedrandom "^3.0.5"
1774+
17371775
"@tensorflow/tfjs-data@2.8.6":
17381776
version "2.8.6"
17391777
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-data/-/tfjs-data-2.8.6.tgz#5888ad0f7b7f8db2b7a5cf4af38e3c04d65efe32"
@@ -1751,6 +1789,15 @@
17511789
node-fetch "~2.6.1"
17521790
string_decoder "^1.3.0"
17531791

1792+
"@tensorflow/tfjs-data@4.2.0":
1793+
version "4.2.0"
1794+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-data/-/tfjs-data-4.2.0.tgz#48bb6654494f23f26953beed2b6640cd6e4641f3"
1795+
integrity sha512-11t7Q+ikseduJgkd9iSeRrtor1aA3o5PVCFhC5yYvR3JLO55ic1+4Ryo0EJfhRoismS6zBUJrpzX4K0zlLbIfw==
1796+
dependencies:
1797+
"@types/node-fetch" "^2.1.2"
1798+
node-fetch "~2.6.1"
1799+
string_decoder "^1.3.0"
1800+
17541801
"@tensorflow/tfjs-layers@2.8.6":
17551802
version "2.8.6"
17561803
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-layers/-/tfjs-layers-2.8.6.tgz#51dec5422fddde289e7915f318676fedeeb6a226"
@@ -1761,6 +1808,11 @@
17611808
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-layers/-/tfjs-layers-4.1.0.tgz#fafc93127a22ed79067b07f4ef58a34fa6332329"
17621809
integrity sha512-lzHNTZu1GwKl7hW5tt2COSpflE0m7xrsOf8AzRzpTDVJYYRx/x5ScMt/y//5jbRuaDOnb3EjT1FxWxwkD44/sg==
17631810

1811+
"@tensorflow/tfjs-layers@4.2.0":
1812+
version "4.2.0"
1813+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-layers/-/tfjs-layers-4.2.0.tgz#ff7d2d897cd8338a4894bd073a36e7f7ab9da742"
1814+
integrity sha512-SO0KTmCFOjrW+PlP9nKYXz07XGFq6uE7am9yH2bRaRPWpEeaKT/+k0C9vFMxI/GzRwY8AK4sLe4U+jE1mhYxGw==
1815+
17641816
"@tensorflow/tfjs-node@^2.7.0":
17651817
version "2.8.6"
17661818
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-node/-/tfjs-node-2.8.6.tgz#6cc442f953f5d6598885c31b585bbf9afb59dfdf"
@@ -1807,7 +1859,7 @@
18071859
regenerator-runtime "^0.13.5"
18081860
yargs "^16.0.3"
18091861

1810-
"@tensorflow/tfjs@4.1.0", "@tensorflow/tfjs@^4.1.0":
1862+
"@tensorflow/tfjs@4.1.0":
18111863
version "4.1.0"
18121864
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs/-/tfjs-4.1.0.tgz#de4f9151f19eb2ec4991c0a2a53e1a8c542bfbcf"
18131865
integrity sha512-jlrJ6MIBos8NkmF+NHIWBnKVBGYJTG06QmW/A0vgyXwkp+3PgzX8TJ4MWIv/7oZr7g27zfY6dXA1OFzrrgvklA==
@@ -1824,6 +1876,23 @@
18241876
regenerator-runtime "^0.13.5"
18251877
yargs "^16.0.3"
18261878

1879+
"@tensorflow/tfjs@^4.2.0":
1880+
version "4.2.0"
1881+
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs/-/tfjs-4.2.0.tgz#764a56d66aec771694295474e35dd2418fc12d6b"
1882+
integrity sha512-iZmtyGC9IJkx+TpFnkgDol8BHv2BU3zJ01HyNcuvnm1w1EqoNe+1n8bwvLzI/sxHMcHTqzuu7VugMaphryxE+A==
1883+
dependencies:
1884+
"@tensorflow/tfjs-backend-cpu" "4.2.0"
1885+
"@tensorflow/tfjs-backend-webgl" "4.2.0"
1886+
"@tensorflow/tfjs-converter" "4.2.0"
1887+
"@tensorflow/tfjs-core" "4.2.0"
1888+
"@tensorflow/tfjs-data" "4.2.0"
1889+
"@tensorflow/tfjs-layers" "4.2.0"
1890+
argparse "^1.0.10"
1891+
chalk "^4.1.0"
1892+
core-js "3"
1893+
regenerator-runtime "^0.13.5"
1894+
yargs "^16.0.3"
1895+
18271896
"@types/estree@0.0.39":
18281897
version "0.0.39"
18291898
resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"

0 commit comments

Comments
 (0)
Please sign in to comment.