Skip to content

Commit d9ea8d1

Browse files
CodingBeardCodingBeard
committed
Add tensorboard style web interface
Add Web docker container Update examples save dirs to point at ./logs/... Add callback.RecordStats to examples
1 parent b0396d1 commit d9ea8d1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+9017
-241
lines changed

‎Makefile

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ SHELL:=/bin/bash
22
PROJECT_NAME=tfkg
33
GO_FILES=./...
44

5-
.SILENT: examples-iris
6-
.PHONY: examples-iris
5+
.SILENT: web
6+
.PHONY: web
77

88
init-docker-m1:
99
docker pull tensorflow/tensorflow:devel-gpu@sha256:1452331ddc5c1995b508114ec9bae0812e17ce14342bd67e08244a07c0d5a5cb
@@ -16,38 +16,81 @@ init-docker:
1616
examples-iris:
1717
go generate ./...
1818
docker-compose up -d tf-jupyter-golang
19-
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg && go run examples/iris/main.go"
19+
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg/examples/iris && go run main.go"
2020

2121
examples-iris-gpu:
2222
go generate ./...
2323
docker-compose up -d tf-jupyter-golang-gpu
24-
docker-compose exec tf-jupyter-golang-gpu sh -c "cd /go/src/tfkg && go run examples/iris/main.go"
24+
docker-compose exec tf-jupyter-golang-gpu sh -c "cd /go/src/tfkg/examples/iris && go run main.go"
2525

2626
examples-multiple-inputs:
2727
go generate ./...
2828
docker-compose up -d tf-jupyter-golang
29-
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg && go run examples/multiple_inputs/main.go"
29+
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg/examples/multiple_inputs && go run main.go"
3030

3131
examples-multiple-inputs-gpu:
3232
go generate ./...
3333
docker-compose up -d tf-jupyter-golang-gpu
34-
docker-compose exec tf-jupyter-golang-gpu sh -c "cd /go/src/tfkg && go run examples/multiple_inputs/main.go"
34+
docker-compose exec tf-jupyter-golang-gpu sh -c "cd /go/src/tfkg/examples/multiple_inputs && go run main.go"
3535

3636
examples-jobs:
3737
go generate ./...
3838
docker-compose up -d tf-jupyter-golang
39-
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg && go run examples/jobs/main.go"
39+
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg/examples/jobs && go run main.go"
4040

4141
examples-jobs-gpu:
4242
go generate ./...
4343
docker-compose up -d tf-jupyter-golang-gpu
44-
docker-compose exec tf-jupyter-golang-gpu sh -c "cd /go/src/tfkg && go run examples/jobs/main.go"
44+
docker-compose exec tf-jupyter-golang-gpu sh -c "cd /go/src/tfkg/examples/jobs && go run main.go"
4545

4646
examples-class-weights:
4747
go generate ./...
4848
docker-compose up -d tf-jupyter-golang
49-
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg && go run examples/class_weights/main.go"
49+
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg/examples/class_weights && go run main.go"
5050

5151
test-python:
5252
docker-compose up -d tf-jupyter-golang
53-
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg && python test.py"
53+
docker-compose exec tf-jupyter-golang sh -c "cd /go/src/tfkg && python test.py"
54+
55+
web: frontend-build
56+
docker-compose up -d web
57+
echo Started web service on port 8082 reading model subdirs in dir: ./logs
58+
59+
dev-web-gin:
60+
docker-compose up -d web-development
61+
echo Starting web-development on port 8082
62+
docker-compose exec web-development sh -c "pkill -f gin" || true
63+
docker-compose exec web-development sh -c "cd /go/src/tfkg/web && rm -f tfkg && MODE=web gin --immediate --bin tfkg --port 80 --appPort 3010 run main.go"
64+
65+
dev-web-refresh:
66+
$(MAKE) -j2 dev-web-gin frontend-reload
67+
68+
#build the frontend
69+
frontend-build:
70+
echo Building frontend
71+
$(MAKE) -j1 npm-update
72+
echo installing gulp if needed, and running gulp
73+
docker-compose exec frontend sh -c "pkill -f gulp" || true
74+
docker-compose exec frontend sh -c "cd /app && npm update && if which gulp; then echo Found gulp; else npm install -g gulp-cli; fi && gulp --env=prod"
75+
76+
npm-update:
77+
echo Starting docker frontend build container
78+
docker-compose up -d frontend
79+
echo Running npm update
80+
docker-compose exec frontend sh -c "cd /app && npm update"
81+
82+
#build the frontend and watch for changes, rebuilding when detected
83+
frontend-watch:
84+
echo Starting docker frontend build container
85+
docker-compose up -d frontend
86+
echo installing gulp if needed, and running gulp
87+
docker-compose exec frontend sh -c "pkill -f gulp" || true
88+
docker-compose exec frontend sh -c "cd /app && if which gulp; then echo Found gulp; else npm install -g gulp-cli; fi && gulp watch --env=dev"
89+
90+
#build the frontend and watch for changes, rebuilding code and reloading the browser when detected
91+
frontend-reload:
92+
echo Starting docker frontend build container
93+
docker-compose up -d frontend
94+
echo installing gulp if needed, and running gulp
95+
docker-compose exec frontend sh -c "pkill -f gulp" || true
96+
docker-compose exec frontend sh -c "cd /app && if which gulp; then echo Found gulp; else npm --no-color install -g gulp-cli; fi && gulp reload --env=dev"

‎docker-compose.yml

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,28 @@ services:
1717
count: 1
1818
capabilities: [ gpu ]
1919
volumes:
20-
- .:/go/src/tfkg:delegated
20+
- .:/go/src/tfkg:delegated
21+
web:
22+
image: golang:1.17
23+
working_dir: /go/src/tfkg/web
24+
command: go run main.go
25+
volumes:
26+
- .:/go/src/tfkg:delegated
27+
ports:
28+
- 8082:80
29+
web-development:
30+
build:
31+
context: docker/web
32+
dockerfile: Dockerfile
33+
command: sleep infinity
34+
volumes:
35+
- .:/go/src/tfkg:delegated
36+
ports:
37+
- 8082:80
38+
frontend:
39+
image: node:13.8
40+
command: sleep infinity
41+
volumes:
42+
- .:/app
43+
ports:
44+
- 3000:3000

‎docker/tf-jupyter-golang-m1/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM tensorflow/tensorflow:devel-gpu@sha256:1452331ddc5c1995b508114ec9bae0812e17ce14342bd67e08244a07c0d5a5cb
22

3-
ENV PATH=$PATH:/usr/local/go/bin
3+
ENV PATH=$PATH:/usr/local/go/bin:/root/go/bin
44

55
RUN apt update && \
66
apt -y install wget && \

‎docker/tf-jupyter-golang/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM tensorflow/tensorflow:2.6.0-gpu-jupyter
22

3-
ENV PATH=$PATH:/usr/local/go/bin
3+
ENV PATH=$PATH:/usr/local/go/bin:/root/go/bin
44

55
RUN apt update && \
66
apt-get -y install wget && \

‎docker/web/Dockerfile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM golang:1.17
2+
3+
RUN go install github.com/codegangsta/gin@latest

‎examples/class_weights/main.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
tf "github.com/galeone/tensorflow/tensorflow/go"
1414
"math/rand"
1515
"os"
16-
"path/filepath"
1716
)
1817

1918
func main() {
@@ -22,7 +21,7 @@ func main() {
2221
LogLevel: cblog.DebugLevel,
2322
Format: "%{time:2006-01-02 15:04:05.000} : %{file}:%{line} : %{message}",
2423
LogToFile: true,
25-
FilePath: filepath.Join("examples/class_weights", "training.log"),
24+
FilePath: "training.log",
2625
FilePerm: os.ModePerm,
2726
LogToStdOut: true,
2827
SetAsDefaultLogger: true,

‎examples/iris/main.go

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"fmt"
45
"github.com/codingbeard/cberrors"
56
"github.com/codingbeard/cberrors/iowriterprovider"
67
"github.com/codingbeard/cblog"
@@ -13,11 +14,12 @@ import (
1314
tf "github.com/galeone/tensorflow/tensorflow/go"
1415
"os"
1516
"path/filepath"
17+
"time"
1618
)
1719

1820
func main() {
1921
// This is where the trained model will be saved
20-
saveDir := "examples/iris/saved_models/trained_model"
22+
saveDir := filepath.Join("../../logs", fmt.Sprintf("iris-%d", time.Now().Unix()))
2123
e := os.MkdirAll(saveDir, os.ModePerm)
2224
if e != nil {
2325
panic(e)
@@ -41,7 +43,7 @@ func main() {
4143
errorHandler := cberrors.NewErrorContainer(iowriterprovider.New(logger))
4244

4345
// Where the cached tokenizers and divisors will go, if you change your data you'll need to clear this
44-
cacheDir := "examples/iris/training-cache"
46+
cacheDir := "training-cache"
4547

4648
// Create a dataset for training and evaluation. iris.data is in the format: float32, float32, float32, float32, className
4749
// This means our categoryOffset is 4. The dataset will automatically pass this value in as the label Tensor when training and evaluating
@@ -58,7 +60,7 @@ func main() {
5860
logger,
5961
errorHandler,
6062
data.SingleFileDatasetConfig{
61-
FilePath: "examples/iris/data/iris.data",
63+
FilePath: "data/iris.data",
6264
CacheDir: cacheDir,
6365
CategoryOffset: 4,
6466
TrainPercent: 0.8,
@@ -152,6 +154,18 @@ func main() {
152154
Compare: callback.CheckpointCompareMax,
153155
SaveDir: saveDir,
154156
},
157+
&callback.RecordStats{
158+
OnEvent: callback.EventEnd,
159+
OnMode: callback.ModeVal,
160+
RecordDir: saveDir,
161+
RecordFileName: "train_stats.csv",
162+
},
163+
&callback.RecordStats{
164+
OnEvent: callback.EventSave,
165+
OnMode: callback.ModeVal,
166+
RecordDir: saveDir,
167+
RecordFileName: "saved_stats.csv",
168+
},
155169
},
156170
},
157171
)
@@ -215,13 +229,13 @@ func main() {
215229
Tracing predict
216230
Saving model
217231
Completed model base
218-
2021-12-07 06:16:37.111 : single_file_dataset.go:66 : Initialising single file dataset at: examples/iris/data/iris.data
232+
2021-12-07 06:16:37.111 : single_file_dataset.go:66 : Initialising single file dataset at: data/iris.data
219233
2021-12-07 06:16:37.115 : single_file_dataset.go:140 : Loading line offsets and stats from cache file
220234
2021-12-07 06:16:37.116 : single_file_dataset.go:146 : Found 151 rows. Got class counts: map[int]int{0:50, 1:50, 2:50}
221235
2021-12-07 06:16:37.117 : single_file_dataset.go:253 : Loaded Pre-Processor: petal_sizes
222236
2021-12-07 06:16:37.118 : single_file_dataset.go:261 : Loaded All Pre-Processors
223237
2021-12-07 06:16:37.118 : main.go:101 : Shuffling dataset
224-
2021-12-07 06:16:37.119 : main.go:105 : Training model: examples/iris/saved_models/trained_model
238+
2021-12-07 06:16:37.119 : main.go:105 : Training model: ../../logs/iris-
225239
2021-12-07 06:16:37.301 : logger.go:102 : End 1 5/5 (0s/0s) loss: 1.0304 acc: 0.0000 val_loss: 0.9951 val_acc: 0.0000
226240
2021-12-07 06:16:37.365 : logger.go:102 : End 2 5/5 (0s/0s) loss: 0.8511 acc: 0.2348 val_loss: 0.7440 val_acc: 0.6000
227241
2021-12-07 06:16:37.423 : logger.go:79 : Saved

‎examples/jobs/main.go

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"fmt"
45
"github.com/codingbeard/cberrors"
56
"github.com/codingbeard/cberrors/iowriterprovider"
67
"github.com/codingbeard/cblog"
@@ -13,11 +14,12 @@ import (
1314
tf "github.com/galeone/tensorflow/tensorflow/go"
1415
"os"
1516
"path/filepath"
17+
"time"
1618
)
1719

1820
func main() {
1921
// This is where the trained model will be saved
20-
saveDir := "examples/jobs/saved_models/trained_model"
22+
saveDir := filepath.Join("../../logs", fmt.Sprintf("jobs-%d", time.Now().Unix()))
2123
e := os.MkdirAll(saveDir, os.ModePerm)
2224
if e != nil {
2325
panic(e)
@@ -41,7 +43,7 @@ func main() {
4143
errorHandler := cberrors.NewErrorContainer(iowriterprovider.New(logger))
4244

4345
// Where the cached tokenizers and divisors will go, if you change your data you'll need to clear this
44-
cacheDir := "examples/jobs/training-cache"
46+
cacheDir := "training-cache"
4547

4648
// We define data processors for the title, location, department, company_profile, description, and requirements. These names will be used for the tokenizer or divisor cache file
4749
// The lineOffset is the offset in the data file
@@ -136,7 +138,7 @@ func main() {
136138
logger,
137139
errorHandler,
138140
data.SingleFileDatasetConfig{
139-
FilePath: "examples/jobs/data/fake_job_postings.csv",
141+
FilePath: "data/fake_job_postings.csv",
140142
CacheDir: cacheDir,
141143
CategoryOffset: 17,
142144
TrainPercent: 0.8,
@@ -329,6 +331,18 @@ func main() {
329331
Compare: callback.CheckpointCompareMax,
330332
SaveDir: saveDir,
331333
},
334+
&callback.RecordStats{
335+
OnEvent: callback.EventEnd,
336+
OnMode: callback.ModeVal,
337+
RecordDir: saveDir,
338+
RecordFileName: "train_stats.csv",
339+
},
340+
&callback.RecordStats{
341+
OnEvent: callback.EventSave,
342+
OnMode: callback.ModeVal,
343+
RecordDir: saveDir,
344+
RecordFileName: "saved_stats.csv",
345+
},
332346
},
333347
},
334348
)
@@ -439,36 +453,31 @@ func main() {
439453

440454
/*
441455
Example output:
442-
2021-12-11 12:15:29.145 : single_file_dataset.go:166 : Reading line offsets and counting stats
443-
2021-12-11 12:15:29.233 : single_file_dataset.go:267 : Found 17879 rows. Got class counts: map[int]int{0:17013, 1:866}
444-
2021-12-11 12:15:29.233 : single_file_dataset.go:298 : Fitting Pre-Processors
445-
Fitting preprocessors: 17476 17476/s2021-12-11 12:15:30.025 : single_file_dataset.go:155 : Loading line offsets and stats from cache file
446-
2021-12-11 12:15:30.025 : single_file_dataset.go:161 : Found 17879 rows. Got class counts: map[int]int{0:17013, 1:866}
447-
2021-12-11 12:15:30.025 : single_file_dataset.go:398 : Fit tokenizers
448-
2021-12-11 12:15:30.026 : main.go:168 : Shuffling dataset
449-
2021-12-11 12:15:30.027 : model.go:715 : Compiling and loading model. If anything goes wrong python error messages will be printed out.
450-
Initialising model
451-
Tracing learn
452-
Tracing evaluate
453-
Tracing predict
454-
Saving model
455-
Completed model base
456-
2021-12-11 12:15:57.609 : main.go:294 : Training model: examples/jobs/saved_models/trained_model
457-
2021-12-11 12:16:11.290 : logger.go:102 : End 1 8/8 (10s/10s) loss: 0.0650 acc: 0.7875 val_loss: 0.0729 val_acc: 0.9537
458-
2021-12-11 12:16:11.713 : logger.go:79 : Saved
459-
2021-12-11 12:16:20.775 : logger.go:102 : End 2 8/8 (9s/9s) loss: 0.0339 acc: 0.8660 val_loss: 0.0277 val_acc: 0.9156
460-
2021-12-11 12:16:29.844 : logger.go:102 : End 3 8/8 (9s/9s) loss: 0.0170 acc: 0.9183 val_loss: 0.0288 val_acc: 0.9444
461-
2021-12-11 12:16:38.930 : logger.go:102 : End 4 8/8 (9s/9s) loss: 0.0097 acc: 0.9541 val_loss: 0.0331 val_acc: 0.9425
462-
2021-12-11 12:16:47.940 : logger.go:102 : End 5 8/8 (8s/8s) loss: 0.0071 acc: 0.9661 val_loss: 0.0329 val_acc: 0.9150
463-
2021-12-11 12:16:57.036 : logger.go:102 : End 6 8/8 (9s/9s) loss: 0.0059 acc: 0.9661 val_loss: 0.0711 val_acc: 0.9675
464-
2021-12-11 12:16:57.044 : logger.go:79 : Saved
465-
2021-12-11 12:17:06.176 : logger.go:102 : End 7 8/8 (9s/9s) loss: 0.0057 acc: 0.9752 val_loss: 0.0643 val_acc: 0.9550
466-
2021-12-11 12:17:15.332 : logger.go:102 : End 8 8/8 (9s/9s) loss: 0.0036 acc: 0.9819 val_loss: 0.0736 val_acc: 0.9462
467-
2021-12-11 12:17:24.489 : logger.go:102 : End 9 8/8 (9s/9s) loss: 0.0046 acc: 0.9754 val_loss: 0.0471 val_acc: 0.9250
468-
2021-12-11 12:17:33.080 : logger.go : Train 10 68/71 (9s/9s) loss: 0.0041 acc: 0.9755 | Prefetched 32021-12-11 12:17:33.629857: I tensorflow/cc/saved_model/reader.cc:38] Reading SavedModel from: examples/jobs/saved_models/trained_model
469-
2021-12-11 12:17:33.629 : logger.go:102 : End 10 8/8 (9s/9s) loss: 0.0042 acc: 0.9753 val_loss: 0.0551 val_acc: 0.9244
470-
2021-12-11 12:17:33.629 : main.go:336 : Finished training
471-
2021-12-11 12:17:35.103 : inference.go:26 : Initialising inference provider with processors loaded from: examples/jobs/saved_models/trained_model
472-
2021-12-11 12:17:36.200 : main.go:435 : Predicted classes: [1 7.840193e-11]
456+
2021-12-16 18:31:13.908 : log.go:147 : Logger initialised
457+
2021-12-16 18:31:13.913 : single_file_dataset.go:75 : Initialising single file dataset at: data/iris.data
458+
2021-12-16 18:31:13.928 : single_file_dataset.go:158 : Loading line offsets and stats from cache file
459+
2021-12-16 18:31:13.930 : single_file_dataset.go:165 : Found 150 rows. Got class counts: map[int]int{0:50, 1:50, 2:50} Got class weights: map[int]float32{0:1, 1:1, 2:1}
460+
2021-12-16 18:31:13.936 : single_file_dataset.go:301 : Loaded Pre-Processor: petal_sizes
461+
2021-12-16 18:31:13.938 : single_file_dataset.go:309 : Loaded All Pre-Processors
462+
2021-12-16 18:31:13.946 : main.go:96 : Shuffling dataset
463+
2021-12-16 18:31:13.947 : model.go:705 : Compiling and loading model. If anything goes wrong python error messages will be printed out.
464+
2021-12-16 18:31:27.189 : main.go:119 : Training model: ../../logs/iris-1639679473
465+
2021-12-16 18:31:28.141 : logger.go:102 : End 1 5/5 (1s/1s) loss: 1.0205 acc: 0.0256 val_loss: 1.0300 val_acc: 0.0667
466+
2021-12-16 18:31:28.377 : logger.go:79 : Saved
467+
2021-12-16 18:31:28.537 : logger.go:102 : End 2 5/5 (0s/0s) loss: 0.8546 acc: 0.2955 val_loss: 0.7449 val_acc: 0.6667
468+
2021-12-16 18:31:28.578 : logger.go:79 : Saved
469+
2021-12-16 18:31:28.743 : logger.go:102 : End 3 5/5 (0s/0s) loss: 0.6240 acc: 0.6742 val_loss: 0.4640 val_acc: 0.7333
470+
2021-12-16 18:31:28.788 : logger.go:79 : Saved
471+
2021-12-16 18:31:28.951 : logger.go:102 : End 4 5/5 (0s/0s) loss: 0.4676 acc: 0.6818 val_loss: 0.3371 val_acc: 0.7333
472+
2021-12-16 18:31:29.114 : logger.go:102 : End 5 5/5 (1s/1s) loss: 0.3801 acc: 0.7803 val_loss: 0.2649 val_acc: 1.0000
473+
2021-12-16 18:31:29.154 : logger.go:79 : Saved
474+
2021-12-16 18:31:29.322 : logger.go:102 : End 6 5/5 (0s/0s) loss: 0.3067 acc: 0.9242 val_loss: 0.2028 val_acc: 1.0000
475+
2021-12-16 18:31:29.481 : logger.go:102 : End 7 5/5 (0s/0s) loss: 0.2399 acc: 0.9470 val_loss: 0.1583 val_acc: 1.0000
476+
2021-12-16 18:31:29.643 : logger.go:102 : End 8 5/5 (0s/0s) loss: 0.1906 acc: 0.9545 val_loss: 0.1324 val_acc: 0.9333
477+
2021-12-16 18:31:29.810 : logger.go:102 : End 9 5/5 (0s/0s) loss: 0.1601 acc: 0.9470 val_loss: 0.1143 val_acc: 0.9333
478+
2021-12-16 18:31:29.969 : logger.go:102 : End 10 5/5 (0s/0s) loss: 0.1416 acc: 0.9621 val_loss: 0.1006 val_acc: 0.9333
479+
2021-12-16 18:31:29.972 : main.go:173 : Finished training
480+
2021-12-16 18:31:29.973 : inference.go:26 : Initialising inference provider with processors loaded from: ../../logs/iris-1639679473
481+
2021-12-16 18:31:30.101 : main.go:212 : Predicted classes: Iris-setosa: 0.000068, Iris-versicolor: 0.320184, Iris-virginica: 0.679748
473482
*/
474483
}

0 commit comments

Comments
 (0)