docs: improve cdn migration info (#5719)

docker-compose.yml

@@ -53,7 +53,7 @@ services:
         sh -c "
         /app/cds-engine-linux-amd64 config new > /app/conf/conf.toml;
         mkdir -p /app/artifacts /app/repositories /app/hatchery-local;
-        chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local;
+        chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local /app/cdn-buffer /app/cdn-storage;
         /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \
             api.artifact.local.baseDirectory=/app/artifacts \
             api.cache.redis.host=cds-cache:6379 \
@@ -68,8 +68,13 @@ services:
             api.smtp.host=smtpmocksrv \
             cdn.cache.redis.host=cds-cache:6379 \
             cdn.cache.redis.password=cds \
+            cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379 \
+            cdn.storageUnits.buffers.redis.redis.password=cds \
+            cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer \
+            cdn.storageUnits.storages.local.local.path=/app/cdn-storage \
+            cdn.storageUnits.storages.cds.cds.host=http://cds-api:8081 \
             cdn.database.host=cds-db \
-            cdn.publicTCP=cds-cdn:8090 \
+            cdn.publicTCP=${HOSTNAME}:8090 \
             cdn.api.http.url=http://cds-api:8081 \
             cdn.url=http://cds-cdn:8089 \
             hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086 \
@@ -101,7 +106,8 @@ services:
             elasticsearch.elasticsearch.url=http://elasticsearch:9200 \
             ui.url=http://${HOSTNAME}:8080 \
             ui.api.http.url=http://cds-api:8081 \
-            ui.hooksURL=http://cds-hooks:8083;
+            ui.hooksURL=http://cds-hooks:8083 \
+            ui.cdnURL=http://cds-cdn:8089;
             echo '';
             echo '/!\ Please run the export command BELOW, this will be useful to create admin user';
             echo '';
@@ -116,6 +122,8 @@ services:
       - cds-artifacts-volume:/app/artifacts
       - cds-repositories-volume:/app/repositories
       - cds-hatchery-local-volume:/app/hatchery-local
+      - cds-cdn-buffer-volume:/app/cdn-buffer
+      - cds-cdn-storage-volume:/app/cdn-storage
 
   cds-api:
     image: ${CDS_DOCKER_IMAGE:-ovhcom/cds-engine:latest}
@@ -144,6 +152,7 @@ services:
     links:
        - cds-api
        - cds-hooks
+       - cds-cdn
     healthcheck:
       test: ["CMD", "curl", "-f", "http://127.0.0.1:8080/mon/version"]
       interval: 30s
@@ -166,12 +175,15 @@ services:
     command: sh -c "/app/cds-engine-linux-amd64 start cdn --config /app/conf/conf.toml"
     volumes:
       - cds-conf-volume:/app/conf
+      - cds-cdn-buffer-volume:/app/cdn-buffer
+      - cds-cdn-storage-volume:/app/cdn-storage
     ports:
       - "8089:8089"
       - "8090:8090"
     links:
        - cds-db
        - cds-cache
+       - cds-api
     healthcheck:
       test: ["CMD", "curl", "-f", "http://127.0.0.1:8089/mon/version"]
       interval: 30s
@@ -290,3 +302,5 @@ volumes:
   cds-hatchery-local-volume:
   cds-conf-volume:
   cds-elasticsearch-volume:
+  cds-cdn-buffer-volume:
+  cds-cdn-storage-volume:

docs/assets/sass/_base.sass

@@ -226,7 +226,7 @@ header
     transform: translate(-50%, -50%)
     display: block
     width: 45px
-    height: 44px  
+    height: 44px
 
 #mainNav
   h5
@@ -299,7 +299,7 @@ ul.global-nav
   left: 10px;
   top: 10px;
   font-size: 2.25rem
-  
+
 
 #hero
   background-color: $dark-grey
@@ -352,7 +352,7 @@ section
   margin-right: 30px
 
 .fa-search
-  cursor: pointer    
+  cursor: pointer
 
 #search-query
   background-color: transparent
@@ -369,7 +369,7 @@ section
   &:focus
     background-color: $light-grey
     color: $dark-grey
-    
+
 #hero
   background-color: $dark-grey
 
@@ -411,7 +411,7 @@ section
 
 #page-content
   .asciinema-terminal .line span
-    font-family: Consolas, Menlo, 'Bitstream Vera Sans Mono', monospace, 'Powerline Symbols'  
+    font-family: Consolas, Menlo, 'Bitstream Vera Sans Mono', monospace, 'Powerline Symbols'
 
 #page-content-with-menu
   position: relative
@@ -564,8 +564,6 @@ dd
   code
     display: inline-block
     box-sizing: border-box
-    background-color: $light-grey
-    color: $dark-grey
     font-family: 'Open Sans Mono', monospace
     vertical-align: baseline
     font-size: 14px
@@ -583,9 +581,6 @@ dd
   .highlight code span, code, pre code
     font-family: "Open Sans Mono", monospace
 
-  code, pre code
-    color: #303030
-
   pre code
     padding: 0
 
@@ -597,7 +592,6 @@ dd
       background-color: #121314!important
 
   pre
-    background-color: #f7f7f7
     display: block
     margin: 20px 0
     padding: 15px
@@ -783,14 +777,14 @@ $feature-box-div-margin-bottom: 40px
   background-color: #92c455
   padding-top: 70px
   padding-bottom: 100px
-  
+
   div
     display: inline-block
     color: #fff
     vertical-align: top
     width: 100%
     text-align: center
-    
+
     h2
       color: black
       text-align: center
@@ -801,7 +795,7 @@ $feature-box-div-margin-bottom: 40px
   background-color: #fff
   padding-bottom: 20px
   text-align: center
-  
+
   .section-text
     width: 100%
     padding-right: 10%
@@ -810,11 +804,11 @@ $feature-box-div-margin-bottom: 40px
       color: black
       text-align: center
       padding-bottom: 0px
-  
+
   .player
     width: 100%
     max-width: 100%
-    
+
 
   div
     display: grid
@@ -843,7 +837,7 @@ $feature-box-div-margin-bottom: 40px
 
     tr:nth-child(odd)
       background-color: #e882a8
-      
+
     tr:nth-child(even)
       background-color: #e882a8
 
@@ -853,7 +847,7 @@ $feature-box-div-margin-bottom: 40px
         text-align: center
         border-bottom: 1px solid
         border-bottom-color: #de3e7b
-    
+
     .fa-times
       font-size: 1rem
       color: #eee
@@ -873,12 +867,12 @@ $feature-box-div-margin-bottom: 40px
         margin-left: 5px
         a
           text-decoration: none
-        
+
       a
         font-size: 0.5rem
         color: #eee
         text-decoration: underline
-        
+
 
 .page
   display: flex

docs/content/development/contribute/documentation.md

@@ -5,7 +5,7 @@ card:
   name: contribute
 ---
 
-Documentation https://ovh.github.io/cds is generated with Hugo. Source are under https://github.com/ovh/cds/tree/{{< param "version" >}}/docs/content
+Documentation https://ovh.github.io/cds is generated with Hugo. Source are under https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/docs/content
 
 Write / Generate / Test documentation:
 

docs/content/development/contribute/hatchery.md

@@ -9,9 +9,9 @@ card:
 
 * First of all you need to create a new package like the other into the hatchery package. Let's call this package vSphere for our example.
 
-* You have to implement the Service interface (see [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/types.go)) in order to configure launch this new hatchery mode via CDS engine CLI.
+* You have to implement the Service interface (see [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/types.go)) in order to configure launch this new hatchery mode via CDS engine CLI.
 
-* Your have to create a Configuration structure composed of the [hatchery.CommonConfiguration](https://godoc.org/github.com/ovh/cds/sdk/hatchery#CommonConfiguration) and the variables you need to access to vSphere API. You finally have to update the [engine main.go file](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/main.go) to manage this new service and add and manage the configuration structure as part of the [global configuration](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/types.go).
+* Your have to create a Configuration structure composed of the [hatchery.CommonConfiguration](https://godoc.org/github.com/ovh/cds/sdk/hatchery#CommonConfiguration) and the variables you need to access to vSphere API. You finally have to update the [engine main.go file](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/main.go) to manage this new service and add and manage the configuration structure as part of the [global configuration](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/types.go).
 
 * You need to implement the hatchery interface (see [here](https://godoc.org/github.com/ovh/cds/sdk/hatchery#Interface))
 

docs/content/development/contribute/plugin.md

@@ -9,14 +9,14 @@ A CDS worker executes job, and job is composed of steps.
 
 A step is an [action]({{< relref "/docs/actions/_index.md" >}})
 
-A Plugin is simply an executable which expose a GRPC server corresponding to the right [proto file](https://github.com/ovh/cds/tree/{{< param "version" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto). You can use the programming language of your choice. The CDS worker will simply query the GRPC server of the plugin.
+A Plugin is simply an executable which expose a GRPC server corresponding to the right [proto file](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto). You can use the programming language of your choice. The CDS worker will simply query the GRPC server of the plugin.
 
 In order to communicate with a CDS worker, a plugin MUST fill the following requirements:
 
 + Expose a GRPC server
-+ Implement methods and messages coming from this [proto file](https://github.com/ovh/cds/tree/{{< param "version" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto)
++ Implement methods and messages coming from this [proto file](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/sdk/grpcplugin/actionplugin/actionplugin.proto)
 + Display this message at the launch of your plugin XXX is ready to accept new connection where XXX is your ip address with port or your Unix socket (example: `127.0.0.1:55939 is ready to accept new connection` or for a Unix socket `XXX.sock is ready to accept new connection`). Note that your plugin can use any Unix socket or tcp port as long as it informs the worker using the log line above.
 
-More resources that may help you in developing a CDS plugin are available: [SDK in this directory](https://github.com/ovh/cds/tree/{{< param "version" >}}/sdk/grpcplugin/actionplugin) with some examples [here](https://github.com/ovh/cds/tree/{{< param "version" >}}/contrib/grpcplugins/action/examples).
+More resources that may help you in developing a CDS plugin are available: [SDK in this directory](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/sdk/grpcplugin/actionplugin) with some examples [here](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/contrib/grpcplugins/action/examples).
 
-Contribute on https://github.com/ovh/cds/tree/master/contrib/grpcplugins/action
+Contribute on https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/contrib/grpcplugins/action

docs/content/docs/components/_index.md

@@ -15,16 +15,16 @@ cards:
   description: "The most powerful Command Line for a CI/CD Platform. cdsctl is the CDS Command Line - you can script everything with it, cdsctl also provide some cool commands such as cdsctl shell to browse your projects and workflows without the need to open a browser."
   button: "cdsctl reference"
   button_path: "/docs/components/cdsctl"
-- name: worker
-  title: "CDS Worker"
-  description: "A pipeline is structured in sequential stages containing one or multiple concurrent jobs. A Job will be executed by a worker. The binary worker is available from CDS Job"
-  button: "worker reference"
-  button_path: "/docs/components/worker"
 - name: hatchery
   title: "Hatchery"
   description: "Hatchery is a service dedicated to spawn and kill worker in accordance with build queue needs."
   button: "Discover the Hatchery component"
   button_path: "/docs/components/hatchery"
+- name: worker
+  title: "CDS Worker"
+  description: "A pipeline is structured in sequential stages containing one or multiple concurrent jobs. A Job will be executed by a worker. The binary worker is available from CDS Job"
+  button: "worker reference"
+  button_path: "/docs/components/worker"
 - name: Engine
   title: "Engine"
   description: "Engine is the core component of CDS"

docs/content/docs/components/cdn/_index.md → docs/content/docs/components/cdn.md

@@ -30,8 +30,10 @@ You must have at least one storage unit, one file buffer and one log buffer to b
 
 ## Use case
 
-Workers and hatcheries communicate with CDN, sending step logs and service log
-![CDN_RECEIVE](/images/cdn_receive.png)
+Workers and hatcheries communicate with CDN, sending step logs and service log.
 
-CDS UI and CLI communicate with CDN to get entire logs, or stream them
-![CDN_GET](/images/cdn_get.png)
+![CDN_RECEIVE](/images/cdn_logs_receive.png?width=600px)
+
+CDS UI and CLI communicate with CDN to get entire logs, or stream them.
+
+![CDN_GET](/images/cdn_logs_get.png?width=600px)

docs/content/docs/concepts/files/action-syntax.md

@@ -120,5 +120,5 @@ cdsctl worker model import ./cds-docker-package.yml
 Or with a remote file:
 
 ```bash
-cdsctl action import https://raw.githubusercontent.com/ovh/cds/master/contrib/actions/cds-docker-package.yml
+cdsctl action import https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/contrib/actions/cds-docker-package.yml
 ```

docs/content/docs/concepts/files/worker_model-syntax.md

@@ -25,7 +25,7 @@ cdsctl worker model import ./go-official-1.13.yml
 or with a remote file:
 
 ```bash
-cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/master/contrib/worker-models/go-official-1.13.yml
+cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/contrib/worker-models/go-official-1.13.yml
 ```
 
 {{< note >}}

docs/content/docs/concepts/template.md

@@ -64,7 +64,7 @@ cdsctl template bulk -f instances.yml
 With cdsctl you can import/export a template from/to yaml files, you can also create a template in the UI from the **settings** menu:
 ```sh
 cdsctl template push ./my-template/*.yml #from local files
-cdsctl template push https://raw.githubusercontent.com/ovh/cds/master/tests/fixtures/template/simple/example-simple.yml #from remote files
+cdsctl template push https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/tests/fixtures/template/simple/example-simple.yml #from remote files
 
 cdsctl template pull shared.infra/my-template --output-dir ./my-template
 ```

docs/content/docs/concepts/variables.md

@@ -221,5 +221,5 @@ You can use many helpers:
 
 ### Deep in code
 
-Are you a Go developer? See all helpers on https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/interpolate/interpolate_helper.go#L23
-and some unit tests on https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/interpolate/interpolate_test.go#L72 
+Are you a Go developer? See all helpers on https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/interpolate/interpolate_helper.go#L23
+and some unit tests on https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/interpolate/interpolate_test.go#L72 

docs/content/docs/concepts/workflow/notifications.md

@@ -28,7 +28,7 @@ For the go templating you have few variables you can use/iterate over.
     - `.TotalKO`: total number of KO tests
     - `.TotalSkipped`: total number of skipped tests
 
-If you need to know about other variable you can check `WorkflowNodeRun` data structure [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/workflow_run.go).
+If you need to know about other variable you can check `WorkflowNodeRun` data structure [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/workflow_run.go).
 
 For example by default the template of pull-request comment is:
 
@@ -72,4 +72,4 @@ And displayed on GitHub:
 ![example_pr_comment.png](../images/example_pr_comment.png?height=200px)
 ## Events
 
-If you need to trigger some specific actions on the technical side, like for example use a microservice which listens to all events in your workflow (updates, launch, stop, etc.), you can add an event integration like, for example, [Kafka]({{< relref "/docs/integrations/kafka/kafka_events.md">}}) and listen to the kafka topic to trigger some actions on your side. Events are more like sending notifications to machines instead of user notifications which are made for users. The see structure of sent events, you can look [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/event.go) and [here](https://github.com/ovh/cds/blob/{{< param "version" >}}/sdk/event_workflow.go).
+If you need to trigger some specific actions on the technical side, like for example use a microservice which listens to all events in your workflow (updates, launch, stop, etc.), you can add an event integration like, for example, [Kafka]({{< relref "/docs/integrations/kafka/kafka_events.md">}}) and listen to the kafka topic to trigger some actions on your side. Events are more like sending notifications to machines instead of user notifications which are made for users. The see structure of sent events, you can look [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/event.go) and [here](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/sdk/event_workflow.go).

docs/content/docs/integrations/kafka/kafka_events.md

@@ -10,7 +10,7 @@ If you are a CDS Administrator, you can configue this integration to be availabl
 
 An example of use case could be that you want to generate some svg badge. Then you can just add an event
 integration for your workflow and plug the
-[badge microservice](https://github.com/ovh/cds/tree/{{< param "version" >}}/contrib/uservices/badge) on this kafka topic.
+[badge microservice](https://github.com/ovh/cds/tree/{{< param "version" "master" >}}/contrib/uservices/badge) on this kafka topic.
 
 Notice that Kafka communication is done using SASL and TLS enable only.
 

docs/content/hosting/database.md

@@ -31,4 +31,4 @@ $ $PATH_TO_CDS/engine database upgrade --db-host <host> --db-port <port> --db-us
 
 ## More details
 
-[Read more about CDS Database Management](https://github.com/ovh/cds/blob/{{< param "version" >}}/engine/sql/README.md)
+[Read more about CDS Database Management](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/engine/sql/README.md)