docs: improve cdn migration info (#5719)

docker-compose.yml

@@ -53,7 +53,7 @@ services:
  sh -c "
  /app/cds-engine-linux-amd64 config new > /app/conf/conf.toml;
  mkdir -p /app/artifacts /app/repositories /app/hatchery-local;
- chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local;
+ chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local /app/cdn-buffer /app/cdn-storage;
  /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \
  api.artifact.local.baseDirectory=/app/artifacts \
  api.cache.redis.host=cds-cache:6379 \
@@ -68,8 +68,13 @@ services:
  api.smtp.host=smtpmocksrv \
  cdn.cache.redis.host=cds-cache:6379 \
  cdn.cache.redis.password=cds \
+ cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379 \
+ cdn.storageUnits.buffers.redis.redis.password=cds \
+ cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer \
+ cdn.storageUnits.storages.local.local.path=/app/cdn-storage \
+ cdn.storageUnits.storages.cds.cds.host=http://cds-api:8081 \
  cdn.database.host=cds-db \
- cdn.publicTCP=cds-cdn:8090 \
+ cdn.publicTCP=${HOSTNAME}:8090 \
  cdn.api.http.url=http://cds-api:8081 \
  cdn.url=http://cds-cdn:8089 \
  hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086 \
@@ -101,7 +106,8 @@ services:
  elasticsearch.elasticsearch.url=http://elasticsearch:9200 \
  ui.url=http://${HOSTNAME}:8080 \
  ui.api.http.url=http://cds-api:8081 \
- ui.hooksURL=http://cds-hooks:8083;
+ ui.hooksURL=http://cds-hooks:8083 \
+ ui.cdnURL=http://cds-cdn:8089;
  echo '';
  echo '/!\ Please run the export command BELOW, this will be useful to create admin user';
  echo '';
@@ -116,6 +122,8 @@ services:
  - cds-artifacts-volume:/app/artifacts
  - cds-repositories-volume:/app/repositories
  - cds-hatchery-local-volume:/app/hatchery-local
+ - cds-cdn-buffer-volume:/app/cdn-buffer
+ - cds-cdn-storage-volume:/app/cdn-storage
 
  cds-api:
  image: ${CDS_DOCKER_IMAGE:-ovhcom/cds-engine:latest}
@@ -144,6 +152,7 @@ services:
  links:
  - cds-api
  - cds-hooks
+ - cds-cdn
  healthcheck:
  test: ["CMD", "curl", "-f", "http://127.0.0.1:8080/mon/version"]
  interval: 30s
@@ -166,12 +175,15 @@ services:
  command: sh -c "/app/cds-engine-linux-amd64 start cdn --config /app/conf/conf.toml"
  volumes:
  - cds-conf-volume:/app/conf
+ - cds-cdn-buffer-volume:/app/cdn-buffer
+ - cds-cdn-storage-volume:/app/cdn-storage
  ports:
  - "8089:8089"
  - "8090:8090"
  links:
  - cds-db
  - cds-cache
+ - cds-api
  healthcheck:
  test: ["CMD", "curl", "-f", "http://127.0.0.1:8089/mon/version"]
  interval: 30s
@@ -290,3 +302,5 @@ volumes:
  cds-hatchery-local-volume:
  cds-conf-volume:
  cds-elasticsearch-volume:
+ cds-cdn-buffer-volume:
+ cds-cdn-storage-volume:

docs/assets/sass/_base.sass

@@ -226,7 +226,7 @@ header
  transform: translate(-50%, -50%)
  display: block
  width: 45px
- height: 44px 
+ height: 44px
 
 #mainNav
  h5
@@ -299,7 +299,7 @@ ul.global-nav
  left: 10px;
  top: 10px;
  font-size: 2.25rem
- 
+
 
 #hero
  background-color: $dark-grey
@@ -352,7 +352,7 @@ section
  margin-right: 30px
 
 .fa-search
- cursor: pointer 
+ cursor: pointer
 
 #search-query
  background-color: transparent
@@ -369,7 +369,7 @@ section
  &:focus
  background-color: $light-grey
  color: $dark-grey
- 
+
 #hero
  background-color: $dark-grey
 
@@ -411,7 +411,7 @@ section
 
 #page-content
  .asciinema-terminal .line span
- font-family: Consolas, Menlo, 'Bitstream Vera Sans Mono', monospace, 'Powerline Symbols' 
+ font-family: Consolas, Menlo, 'Bitstream Vera Sans Mono', monospace, 'Powerline Symbols'
 
 #page-content-with-menu
  position: relative
@@ -564,8 +564,6 @@ dd
  code
  display: inline-block
  box-sizing: border-box
- background-color: $light-grey
- color: $dark-grey
  font-family: 'Open Sans Mono', monospace
  vertical-align: baseline
  font-size: 14px
@@ -583,9 +581,6 @@ dd
  .highlight code span, code, pre code
  font-family: "Open Sans Mono", monospace
 
- code, pre code
- color: #303030
-
  pre code
  padding: 0
 
@@ -597,7 +592,6 @@ dd
  background-color: #121314!important
 
  pre
- background-color: #f7f7f7
  display: block
  margin: 20px 0
  padding: 15px
@@ -783,14 +777,14 @@ $feature-box-div-margin-bottom: 40px
  background-color: #92c455
  padding-top: 70px
  padding-bottom: 100px
- 
+
  div
  display: inline-block
  color: #fff
  vertical-align: top
  width: 100%
  text-align: center
- 
+
  h2
  color: black
  text-align: center
@@ -801,7 +795,7 @@ $feature-box-div-margin-bottom: 40px
  background-color: #fff
  padding-bottom: 20px
  text-align: center
- 
+
  .section-text
  width: 100%
  padding-right: 10%
@@ -810,11 +804,11 @@ $feature-box-div-margin-bottom: 40px
  color: black
  text-align: center
  padding-bottom: 0px
- 
+
  .player
  width: 100%
  max-width: 100%
- 
+
 
  div
  display: grid
@@ -843,7 +837,7 @@ $feature-box-div-margin-bottom: 40px
 
  tr:nth-child(odd)
  background-color: #e882a8
- 
+
  tr:nth-child(even)
  background-color: #e882a8
 
@@ -853,7 +847,7 @@ $feature-box-div-margin-bottom: 40px
  text-align: center
  border-bottom: 1px solid
  border-bottom-color: #de3e7b
- 
+
  .fa-times
  font-size: 1rem
  color: #eee
@@ -873,12 +867,12 @@ $feature-box-div-margin-bottom: 40px
  margin-left: 5px
  a
  text-decoration: none
- 
+
  a
  font-size: 0.5rem
  color: #eee
  text-decoration: underline
- 
+
 
 .page
  display: flex

docs/content/development/contribute/documentation.md

@@ -5,7 +5,7 @@ card:
  name: contribute
 ---
 
-Documentation https://ovh.github.io/cds is generated with Hugo. Source are under https://github.com/ovh/cds/tree/{{< param "version" >}}/docs/content
+Documentation https://ovh.github.io/cds is generated with Hugo. Source are under https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/docs/content
 
 Write / Generate / Test documentation:
 

docs/content/development/contribute/hatchery.md

@@ -9,9 +9,9 @@ card:
 
 * First of all you need to create a new package like the other into the hatchery package. Let's call this package vSphere for our example.
 
-* You have to implement the Service interface (see [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/types.go)) in order to configure launch this new hatchery mode via CDS engine CLI.
+* You have to implement the Service interface (see [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/types.go)) in order to configure launch this new hatchery mode via CDS engine CLI.
 
-* Your have to create a Configuration structure composed of the [hatchery.CommonConfiguration](https://godoc.org/github.com/ovh/cds/sdk/hatchery#CommonConfiguration) and the variables you need to access to vSphere API. You finally have to update the [engine main.go file](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/main.go) to manage this new service and add and manage the configuration structure as part of the [global configuration](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/types.go).
+* Your have to create a Configuration structure composed of the [hatchery.CommonConfiguration](https://godoc.org/github.com/ovh/cds/sdk/hatchery#CommonConfiguration) and the variables you need to access to vSphere API. You finally have to update the [engine main.go file](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/main.go) to manage this new service and add and manage the configuration structure as part of the [global configuration](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/types.go).
 
 * You need to implement the hatchery interface (see [here](https://godoc.org/github.com/ovh/cds/sdk/hatchery#Interface))
 

docs/content/development/contribute/plugin.md

@@ -9,14 +9,14 @@ A CDS worker executes job, and job is composed of steps.
 
 A step is an [action]({{< relref "/docs/actions/_index.md" >}})
 
-A Plugin is simply an executable which expose a GRPC server corresponding to the right [proto file](https://github.com/ovh/cds/tree/{{< param "version" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto). You can use the programming language of your choice. The CDS worker will simply query the GRPC server of the plugin.
+A Plugin is simply an executable which expose a GRPC server corresponding to the right [proto file](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto). You can use the programming language of your choice. The CDS worker will simply query the GRPC server of the plugin.
 
 In order to communicate with a CDS worker, a plugin MUST fill the following requirements:
 
 + Expose a GRPC server
-+ Implement methods and messages coming from this [proto file](https://github.com/ovh/cds/tree/{{< param "version" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto)
++ Implement methods and messages coming from this [proto file](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto)
 + Display this message at the launch of your plugin XXX is ready to accept new connection where XXX is your ip address with port or your Unix socket (example: `127.0.0.1:55939 is ready to accept new connection` or for a Unix socket `XXX.sock is ready to accept new connection`). Note that your plugin can use any Unix socket or tcp port as long as it informs the worker using the log line above.
 
-More resources that may help you in developing a CDS plugin are available: [SDK in this directory](https://github.com/ovh/cds/tree/{{< param "version" >}}/sdk/grpcplugin/actionplugin) with some examples [here](https://github.com/ovh/cds/tree/{{< param "version" >}}/contrib/grpcplugins/action/examples).
+More resources that may help you in developing a CDS plugin are available: [SDK in this directory](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/sdk/grpcplugin/actionplugin) with some examples [here](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/contrib/grpcplugins/action/examples).
 
-Contribute on https://github.com/ovh/cds/tree/master/contrib/grpcplugins/action
+Contribute on https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/contrib/grpcplugins/action

docs/content/docs/components/_index.md

@@ -15,16 +15,16 @@ cards:
  description: "The most powerful Command Line for a CI/CD Platform. cdsctl is the CDS Command Line - you can script everything with it, cdsctl also provide some cool commands such as cdsctl shell to browse your projects and workflows without the need to open a browser."
  button: "cdsctl reference"
  button_path: "/docs/components/cdsctl"
-- name: worker
- title: "CDS Worker"
- description: "A pipeline is structured in sequential stages containing one or multiple concurrent jobs. A Job will be executed by a worker. The binary worker is available from CDS Job"
- button: "worker reference"
- button_path: "/docs/components/worker"
 - name: hatchery
  title: "Hatchery"
  description: "Hatchery is a service dedicated to spawn and kill worker in accordance with build queue needs."
  button: "Discover the Hatchery component"
  button_path: "/docs/components/hatchery"
+- name: worker
+ title: "CDS Worker"
+ description: "A pipeline is structured in sequential stages containing one or multiple concurrent jobs. A Job will be executed by a worker. The binary worker is available from CDS Job"
+ button: "worker reference"
+ button_path: "/docs/components/worker"
 - name: Engine
  title: "Engine"
  description: "Engine is the core component of CDS"

docs/content/docs/components/cdn/_index.md → docs/content/docs/components/cdn.md

@@ -30,8 +30,10 @@ You must have at least one storage unit, one file buffer and one log buffer to b
 
 ## Use case
 
-Workers and hatcheries communicate with CDN, sending step logs and service log
-![CDN_RECEIVE](/images/cdn_receive.png)
+Workers and hatcheries communicate with CDN, sending step logs and service log.
 
-CDS UI and CLI communicate with CDN to get entire logs, or stream them
-![CDN_GET](/images/cdn_get.png)
+![CDN_RECEIVE](/images/cdn_logs_receive.png?width=600px)
+
+CDS UI and CLI communicate with CDN to get entire logs, or stream them.
+
+![CDN_GET](/images/cdn_logs_get.png?width=600px)

docs/content/docs/concepts/files/action-syntax.md

@@ -120,5 +120,5 @@ cdsctl worker model import ./cds-docker-package.yml
 Or with a remote file:
 
 ```bash
-cdsctl action import https://raw.githubusercontent.com/ovh/cds/master/contrib/actions/cds-docker-package.yml
+cdsctl action import https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/contrib/actions/cds-docker-package.yml
 ```

docs/content/docs/concepts/files/worker_model-syntax.md

@@ -25,7 +25,7 @@ cdsctl worker model import ./go-official-1.13.yml
 or with a remote file:
 
 ```bash
-cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/master/contrib/worker-models/go-official-1.13.yml
+cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/contrib/worker-models/go-official-1.13.yml
 ```
 
 {{< note >}}

docs/content/docs/concepts/template.md

@@ -64,7 +64,7 @@ cdsctl template bulk -f instances.yml
 With cdsctl you can import/export a template from/to yaml files, you can also create a template in the UI from the **settings** menu:
 ```sh
 cdsctl template push ./my-template/*.yml #from local files
-cdsctl template push https://raw.githubusercontent.com/ovh/cds/master/tests/fixtures/template/simple/example-simple.yml #from remote files
+cdsctl template push https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/tests/fixtures/template/simple/example-simple.yml #from remote files
 
 cdsctl template pull shared.infra/my-template --output-dir ./my-template
 ```

docs/content/docs/concepts/variables.md

@@ -221,5 +221,5 @@ You can use many helpers:
 
 ### Deep in code
 
-Are you a Go developer? See all helpers on https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/interpolate/interpolate_helper.go#L23
-and some unit tests on https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/interpolate/interpolate_test.go#L72 
+Are you a Go developer? See all helpers on https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/interpolate/interpolate_helper.go#L23
+and some unit tests on https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/interpolate/interpolate_test.go#L72 

docs/content/docs/concepts/workflow/notifications.md

@@ -28,7 +28,7 @@ For the go templating you have few variables you can use/iterate over.
  - `.TotalKO`: total number of KO tests
  - `.TotalSkipped`: total number of skipped tests
 
-If you need to know about other variable you can check `WorkflowNodeRun` data structure [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/workflow_run.go).
+If you need to know about other variable you can check `WorkflowNodeRun` data structure [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/workflow_run.go).
 
 For example by default the template of pull-request comment is:
 
@@ -72,4 +72,4 @@ And displayed on GitHub:
 ![example_pr_comment.png](../images/example_pr_comment.png?height=200px)
 ## Events
 
-If you need to trigger some specific actions on the technical side, like for example use a microservice which listens to all events in your workflow (updates, launch, stop, etc.), you can add an event integration like, for example, [Kafka]({{< relref "/docs/integrations/kafka/kafka_events.md">}}) and listen to the kafka topic to trigger some actions on your side. Events are more like sending notifications to machines instead of user notifications which are made for users. The see structure of sent events, you can look [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/event.go) and [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/event_workflow.go).
+If you need to trigger some specific actions on the technical side, like for example use a microservice which listens to all events in your workflow (updates, launch, stop, etc.), you can add an event integration like, for example, [Kafka]({{< relref "/docs/integrations/kafka/kafka_events.md">}}) and listen to the kafka topic to trigger some actions on your side. Events are more like sending notifications to machines instead of user notifications which are made for users. The see structure of sent events, you can look [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/event.go) and [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/event_workflow.go).

docs/content/docs/integrations/kafka/kafka_events.md

@@ -10,7 +10,7 @@ If you are a CDS Administrator, you can configue this integration to be availabl
 
 An example of use case could be that you want to generate some svg badge. Then you can just add an event
 integration for your workflow and plug the
-[badge microservice](https://github.com/ovh/cds/tree/{{< param "version" >}}/contrib/uservices/badge) on this kafka topic.
+[badge microservice](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/contrib/uservices/badge) on this kafka topic.
 
 Notice that Kafka communication is done using SASL and TLS enable only.
 

docs/content/hosting/database.md

@@ -31,4 +31,4 @@ $ $PATH_TO_CDS/engine database upgrade --db-host <host> --db-port <port> --db-us
 
 ## More details
 
-[Read more about CDS Database Management](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/sql/README.md)
+[Read more about CDS Database Management](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/sql/README.md)