forked from logsearch/logsearch-boshrelease
-
Notifications
You must be signed in to change notification settings - Fork 0
/
bosh-lite.yml
365 lines (347 loc) · 10.2 KB
/
bosh-lite.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
---
name: logsearch
director_uuid: <%= `bosh status --uuid` %>
releases:
- name: logsearch
version: latest
compilation:
workers: 2
network: default
reuse_compilation_vms: true
cloud_properties: {}
update:
serial: true
canaries: 1
canary_watch_time: 30000
update_watch_time: 30000
max_in_flight: 1
max_errors: 1
resource_pools:
- name: warden
network: default
stemcell:
name: bosh-warden-boshlite-ubuntu-trusty-go_agent
version: latest
cloud_properties: {}
jobs:
- name: api
release: logsearch
templates:
- name: elasticsearch
- name: api
- name: kibana
update:
serial: true
instances: 1
resource_pool: warden
networks:
- name: default
static_ips:
- 10.244.2.2
properties:
elasticsearch:
node:
allow_data: false
- name: ingestor
release: logsearch
templates:
- name: ingestor_lumberjack
- name: ingestor_syslog
- name: ingestor_relp
instances: 1
resource_pool: warden
networks:
- name: default
static_ips:
- 10.244.2.14
- name: queue
release: logsearch
templates:
- name: queue
instances: 1
resource_pool: warden
networks:
- name: default
static_ips:
- 10.244.2.10
- name: parser
release: logsearch
templates:
- name: parser
instances: 1
resource_pool: warden
networks:
- name: default
default:
- dns
- gateway
- name: elasticsearch_persistent
release: logsearch
templates:
- name: elasticsearch
update:
serial: true
instances: 1
resource_pool: warden
networks:
- name: default
static_ips:
- 10.244.2.6
persistent_disk: 1024
properties:
elasticsearch:
node:
allow_master: false
- name: elasticsearch_autoscale
release: logsearch
templates:
- name: elasticsearch
update:
serial: true
instances: 1
resource_pool: warden
networks:
- name: default
default:
- dns
- gateway
properties:
elasticsearch:
node:
allow_master: false
- name: test_e2e_errand
lifecycle: errand
release: logsearch
templates:
- name: test_e2e_errand
instances: 1
resource_pool: warden
networks:
- name: default
properties:
test_e2e_errand:
syslog_host: "10.244.2.14"
syslog_port: 5514
elasticsearch_hostport: "10.244.2.2:9200"
# upload your backfill data
# $ sshpass -pc1oudc0w scp -i "$HOME/.ssh/vagrant" 2014102721.20141027220003-api-0.log.xz [email protected]:/tmp/backfill.xz
# restart ingestion of it all
# $ sshpass -pc1oudc0w ssh -i "$HOME/.ssh/vagrant" [email protected] "echo c1oudc0w | sudo -p '' -S /var/vcap/jobs/ingestor_archiver/bin/reload /tmp/backfill.xz"
# utility if you want to block until complete
# $ sshpass -pc1oudc0w ssh -i "$HOME/.ssh/vagrant" [email protected] "echo c1oudc0w | sudo -p '' -S /var/vcap/jobs/ingestor_archiver/bin/wait-until-loaded"
#- name: restorer
# release: logsearch
# templates:
# - name: ingestor_archiver
# - name: parser
# instances: 1
# resource_pool: warden
# networks:
# - name: default
# default:
# - dns
# - gateway
# properties:
# redis:
# key: "logstash-restorer"
properties:
api:
port: 80
kibana:
elasticsearch: 127.0.0.1:9200
port: 5601
elasticsearch:
host: 10.244.2.2
cluster_name: logsearch--bosh-lite
drain: true
indices:
ttl_interval: "5m" # We want documents from test runs to be cleaned up frequently
exec:
environment:
ES_HEAP_SIZE: 256M
redis:
host: 10.244.2.10
logstash_parser:
debug: true
filters: |
<% filtersconf = File.join(File.dirname(File.expand_path(__FILE__)), 'examples/filters.conf') %>
<%= File.read(filtersconf).gsub(/^/, ' ').strip if File.exists? filtersconf %>
# src/sample-logs/repo-commits/logstash-filters.conf
<%= File.read("#{File.dirname(File.expand_path(__FILE__))}/src/sample-logs/repo-commits/logstash-filters.conf").gsub(/^/, ' ').strip %>
logstash_ingestor:
debug: true
relp:
port: 5515
syslog:
port: 5514
syslog_tls:
port: 443
# ingestor-20241214a
ssl_cert: |
-----BEGIN CERTIFICATE-----
MIICWjCCAcOgAwIBAgIJAN57Q1O2B6k0MA0GCSqGSIb3DQEBBQUAMG0xFTATBgNV
BAoMDGxvZ3NlYXJjaC5pbzEeMBwGA1UECwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5
MREwDwYDVQQDDAhpbmdlc3RvcjEhMB8GCSqGSIb3DQEJARYSbm9ib2R5QGV4YW1w
bGUuY29tMB4XDTE0MTIxNzIyMDkyNloXDTI0MTIxNDIyMDkyNlowXDEVMBMGA1UE
CgwMbG9nc2VhcmNoLmlvMQ0wCwYDVQQLDARURVNUMREwDwYDVQQDDAhpbmdlc3Rv
cjEhMB8GCSqGSIb3DQEJARYSbm9ib2R5QGV4YW1wbGUuY29tMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQClKtJSXPwsPWPIhSFqPDcyQwvsIY/nF+vNjetX1xjC
ATC6F6ZaKCcddF1JaomTiPR9+qVeNedGwvtcrCLyOYpqBWs6KuCq4dE/7QxEwova
yLEQRGMvZW8OClpjY0PdrpX+ekqllD/7CTMYvabc3Kq0Q6WPZAGLo01YoW2KKr13
GQIDAQABoxMwETAPBgNVHREECDAGhwQK9AIOMA0GCSqGSIb3DQEBBQUAA4GBAFLV
Z59nPFO1+W407lDCm4tXV3RpUCt4drAqNp1xum6U9fkcgzU0ZhrfmVCqGdFGWxxM
Jhe1iAbho/Jb85V9hB9txr2Uh5Xvr0r0faeJYqFF+NCc9G+PKiNVwbId0rH2bfwr
YLxYAgzDDozZ1bUSUQKbkiX8jSQHB63MPZXSAMWb
-----END CERTIFICATE-----
ssl_key: |
-----BEGIN RSA PRIVATE KEY-----
MIICXAIBAAKBgQClKtJSXPwsPWPIhSFqPDcyQwvsIY/nF+vNjetX1xjCATC6F6Za
KCcddF1JaomTiPR9+qVeNedGwvtcrCLyOYpqBWs6KuCq4dE/7QxEwovayLEQRGMv
ZW8OClpjY0PdrpX+ekqllD/7CTMYvabc3Kq0Q6WPZAGLo01YoW2KKr13GQIDAQAB
AoGAHX57Flgic+f2hJ05bVYZaTFN1Lndj5/W7Nr19rajZil+QQzuGNVovrrD2dNb
g+wF9OUoWJ15kkpJRrA6gVTDIYgZwoWjpo1X6NKfiMGbtITSePfwB17egeaVvoKW
nxfhd5absxOAC13hXNGEIyklzBMzyikTbrlemeo07gLq+H0CQQDWzxVTIXesjQ2S
Um9vvsUdlxqU5AjvsVPDZP3/d3NYF0ZALyyeqg1mE+pc4gEXw3T5Q5IMyxxFZjqQ
m2mpuZzDAkEAxNbXko2mSopYZ5/LR6KJFvze8t70b0TUvlvPvM3M6pMeViqSrfyt
QvHMNPRxbgoniNwm7W9x6Jp2UAUzH2kO8wJBAJc3KkTeH3fpx+8EdwwMGIkPERhV
OvE5PMUlOCT5usn9gGe4jcmX3lzIkkgWlTxcTOEYLx0wclNsdrfLn+NqFa8CQByU
xYB2KOsx41xIi4+/PgCkfwrs7LkrWWi6lBNqHpMBAaqpS9sPkWjjCy+1PrMnrk3l
CZH4WKXZp8w+tQmei5kCQCxQ6Se1d1fSYQlLsCoTKmbcOjFoT5joj1nMIwSwn/Sy
eevXlJLkedUMlEntH3b+dQjaRSWwWWtjzxY/Ou0CLKI=
-----END RSA PRIVATE KEY-----
lumberjack:
# ingestor-20241214a
ssl_certificate: |
-----BEGIN CERTIFICATE-----
MIICWjCCAcOgAwIBAgIJAN57Q1O2B6k0MA0GCSqGSIb3DQEBBQUAMG0xFTATBgNV
BAoMDGxvZ3NlYXJjaC5pbzEeMBwGA1UECwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5
MREwDwYDVQQDDAhpbmdlc3RvcjEhMB8GCSqGSIb3DQEJARYSbm9ib2R5QGV4YW1w
bGUuY29tMB4XDTE0MTIxNzIyMDkyNloXDTI0MTIxNDIyMDkyNlowXDEVMBMGA1UE
CgwMbG9nc2VhcmNoLmlvMQ0wCwYDVQQLDARURVNUMREwDwYDVQQDDAhpbmdlc3Rv
cjEhMB8GCSqGSIb3DQEJARYSbm9ib2R5QGV4YW1wbGUuY29tMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQClKtJSXPwsPWPIhSFqPDcyQwvsIY/nF+vNjetX1xjC
ATC6F6ZaKCcddF1JaomTiPR9+qVeNedGwvtcrCLyOYpqBWs6KuCq4dE/7QxEwova
yLEQRGMvZW8OClpjY0PdrpX+ekqllD/7CTMYvabc3Kq0Q6WPZAGLo01YoW2KKr13
GQIDAQABoxMwETAPBgNVHREECDAGhwQK9AIOMA0GCSqGSIb3DQEBBQUAA4GBAFLV
Z59nPFO1+W407lDCm4tXV3RpUCt4drAqNp1xum6U9fkcgzU0ZhrfmVCqGdFGWxxM
Jhe1iAbho/Jb85V9hB9txr2Uh5Xvr0r0faeJYqFF+NCc9G+PKiNVwbId0rH2bfwr
YLxYAgzDDozZ1bUSUQKbkiX8jSQHB63MPZXSAMWb
-----END CERTIFICATE-----
ssl_key: |
-----BEGIN RSA PRIVATE KEY-----
MIICXAIBAAKBgQClKtJSXPwsPWPIhSFqPDcyQwvsIY/nF+vNjetX1xjCATC6F6Za
KCcddF1JaomTiPR9+qVeNedGwvtcrCLyOYpqBWs6KuCq4dE/7QxEwovayLEQRGMv
ZW8OClpjY0PdrpX+ekqllD/7CTMYvabc3Kq0Q6WPZAGLo01YoW2KKr13GQIDAQAB
AoGAHX57Flgic+f2hJ05bVYZaTFN1Lndj5/W7Nr19rajZil+QQzuGNVovrrD2dNb
g+wF9OUoWJ15kkpJRrA6gVTDIYgZwoWjpo1X6NKfiMGbtITSePfwB17egeaVvoKW
nxfhd5absxOAC13hXNGEIyklzBMzyikTbrlemeo07gLq+H0CQQDWzxVTIXesjQ2S
Um9vvsUdlxqU5AjvsVPDZP3/d3NYF0ZALyyeqg1mE+pc4gEXw3T5Q5IMyxxFZjqQ
m2mpuZzDAkEAxNbXko2mSopYZ5/LR6KJFvze8t70b0TUvlvPvM3M6pMeViqSrfyt
QvHMNPRxbgoniNwm7W9x6Jp2UAUzH2kO8wJBAJc3KkTeH3fpx+8EdwwMGIkPERhV
OvE5PMUlOCT5usn9gGe4jcmX3lzIkkgWlTxcTOEYLx0wclNsdrfLn+NqFa8CQByU
xYB2KOsx41xIi4+/PgCkfwrs7LkrWWi6lBNqHpMBAaqpS9sPkWjjCy+1PrMnrk3l
CZH4WKXZp8w+tQmei5kCQCxQ6Se1d1fSYQlLsCoTKmbcOjFoT5joj1nMIwSwn/Sy
eevXlJLkedUMlEntH3b+dQjaRSWwWWtjzxY/Ou0CLKI=
-----END RSA PRIVATE KEY-----
# Warden CPI requires crappy network settings; so "hide" them away here
networks:
- name: default
# Assumes up to 7 VMs, including 4 static and 3 dynamic.
# Plus 7 (double the size) unused IPs, due to BOSH bug/quirk.
subnets:
- cloud_properties:
name: random
range: 10.244.2.0/30
reserved:
- 10.244.2.1
static:
- 10.244.2.2
- cloud_properties:
name: random
range: 10.244.2.4/30
reserved:
- 10.244.2.5
static:
- 10.244.2.6
- cloud_properties:
name: random
range: 10.244.2.8/30
reserved:
- 10.244.2.9
static:
- 10.244.2.10
- cloud_properties:
name: random
range: 10.244.2.12/30
reserved:
- 10.244.2.13
static:
- 10.244.2.14
- cloud_properties:
name: random
range: 10.244.2.16/30
reserved:
- 10.244.2.17
static: []
- cloud_properties:
name: random
range: 10.244.2.20/30
reserved:
- 10.244.2.21
static: []
- cloud_properties:
name: random
range: 10.244.2.24/30
reserved:
- 10.244.2.25
static: []
# Bonus double-sized network required due to BOSH oddity
- cloud_properties:
name: random
range: 10.244.2.28/30
reserved:
- 10.244.2.29
static: []
- cloud_properties:
name: random
range: 10.244.2.32/30
reserved:
- 10.244.2.33
static: []
- cloud_properties:
name: random
range: 10.244.2.36/30
reserved:
- 10.244.2.37
static: []
- cloud_properties:
name: random
range: 10.244.2.40/30
reserved:
- 10.244.2.41
static: []
- cloud_properties:
name: random
range: 10.244.2.44/30
reserved:
- 10.244.2.45
static: []
- cloud_properties:
name: random
range: 10.244.2.48/30
reserved:
- 10.244.2.49
static: []
- cloud_properties:
name: random
range: 10.244.2.52/30
reserved:
- 10.244.2.53
static: []
apply_spec:
properties:
ntp:
- 0.europe.pool.ntp.org
- 1.europe.pool.ntp.org
- 2.europe.pool.ntp.org
- 3.europe.pool.ntp.org