CPU masks specification support (pre-collections) (#34)

* Initial support for setting cpu masks in pre-collections playbooks * Support CPU affinity specification for static nodes * Disable improper block trimming * Proper affinity item specification for dynnodes
ydb-platform · Jul 10, 2024 · 37324b7 · 37324b7
1 parent 398ad37
commit 37324b7
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 20 deletions.
diff --git a/group_vars/all.example b/group_vars/all.example
@@ -19,10 +19,10 @@ ydb_dir: /opt/ydb
 ydb_tls_dir: ./tls/CA/certs/2023-03-09_16-49-39
 
 # YDB server software archive
-ydb_archive: ydbd-23.1.26-linux-amd64.tar.gz
+ydb_archive: yasubd-24.1.18.1-linux-amd64.tar.gz
 
 # YDB server software release
-ydb_version: 23.1.26
+ydb_version: 24.1.18.1
 
 # Extra options passed to tar for unpack, typically "--strip-component=1"
 ydb_unpack_options: "--strip-component=1"
@@ -41,8 +41,8 @@ ydb_default_groups: 9
 ydb_config: config.yaml
 
 # Number of cores for static and dynamic nodes
-ydb_cores_static: 7
-ydb_cores_dynamic: 7
+ydb_cores_static: 31
+ydb_cores_dynamic: 14
 
 # YDB physical disk device names and labels for static nodes
 ydb_disks:
@@ -57,11 +57,20 @@ ydb_brokers:
   - ycydb-s3
 
 # The list of dynnodes to run on each host
+#  instance is the service suffix
+#  offset is the integer added to base port numbers
+#  affinity is optional, and is the vCPU mask to run the service
+# To check: for pid in `pgrep ydbd`; do sudo taskset -cp $pid; done
 ydb_dynnodes:
-  - { instance: 'alfa', offset: 0 }
-  - { instance: 'bravo', offset: 1 }
-  - { instance: 'charlie', offset: 2 }
-  - { instance: 'delta', offset: 3 }
+  - { instance: 'a', offset: 0, affinity: '32-47' }
+  - { instance: 'b', offset: 1, affinity: '48-63' }
+  - { instance: 'c', offset: 2, affinity: '64-79' }
+  - { instance: 'd', offset: 3, affinity: '80-95' }
+  - { instance: 'e', offset: 4, affinity: '96-111' }
+  - { instance: 'f', offset: 5, affinity: '112-127' }
+
+# Optional parameter defining the CPU affinity of static nodes' services
+ydb_affinity_static: '0-31'
 
 # Extra sleep time to allow dynamic nodes to settle up
 dynnode_restart_sleep_seconds: 30
diff --git a/roles/ydbd_dynamic/templates/dynnode-service.j2 b/roles/ydbd_dynamic/templates/dynnode-service.j2
@@ -18,22 +18,26 @@ SyslogFacility=daemon
 SyslogLevel=err
 Environment=LD_LIBRARY_PATH={{ ydb_dir }}/lib
 ExecStart={{ ydb_dir }}/bin/ydbd server --cluster-name "{{ ydb_cluster }}" \
-    --yaml-config  {{ ydb_dir }}/cfg/ydbd-dynamic.yaml \
-    --grpcs-port {{ 2136 + item.offset }} --grpc-ca {{ ydb_dir }}/certs/ca.crt \
-    --ic-port {{ 19002 + item.offset }} --ca {{ ydb_dir }}/certs/ca.crt \
-    --mon-port {{ 8766 + item.offset }} --mon-cert {{ ydb_dir }}/certs/web.pem \
-    --tenant /{{ ydb_domain }}/{{ ydb_dbname }} \
-    --node-broker grpcs://{{ ydb_brokers[0] }}:2135 \
-{% if ydb_brokers|length > 1 %}
-    --node-broker grpcs://{{ ydb_brokers[1] }}:2135 \
-{% endif %}
-{% if ydb_brokers|length > 2 %}
-    --node-broker grpcs://{{ ydb_brokers[2] }}:2135 \
-{% endif %}
+    --yaml-config  {{ ydb_dir }}/cfg/ydbd-dynamic.yaml --ca {{ ydb_dir }}/certs/ca.crt \
+    --grpc-ca {{ ydb_dir }}/certs/ca.crt --mon-cert {{ ydb_dir }}/certs/web.pem \
+{%- if 'offset' in item %}
+    --grpcs-port {{ 2136 + item.offset }}  --ic-port {{ 19002 + item.offset }} --mon-port {{ 8766 + item.offset }}  \
+{%- else %}
+    --grpcs-port 2136 --ic-port 19002 --mon-port 8766  \
+{%- endif %}
+{%- for brk in ydb_brokers %}
+    --node-broker grpcs://{{ brk }}:2135 \
+{%- endfor %}
+{%- if ydb_node_dc is defined %}
     --data-center {{ ydb_node_dc }} --rack {{ ydb_node_rack }} --body {{ ydb_node_body }}
+{%- endif %}
+    --tenant /{{ ydb_domain }}/{{ ydb_dbname }}
 LimitNOFILE=65536
 LimitCORE=0
 LimitMEMLOCK=32212254720
+{%- if 'affinity' in item %}
+CPUAffinity={{ item.affinity }}
+{%- endif %}
 
 [Install]
 WantedBy=multi-user.target
diff --git a/roles/ydbd_static/templates/static-service.j2 b/roles/ydbd_static/templates/static-service.j2
@@ -1,3 +1,4 @@
+#jinja2: trim_blocks:False
 [Unit]
 Description=YDB storage node
 After=network-online.target rc-local.service
@@ -21,6 +22,9 @@ LimitNOFILE=65536
 LimitCORE=0
 #LimitCORE=infinity
 LimitMEMLOCK=3221225472
+{%- if ydb_affinity_static is defined %}
+CPUAffinity={{ ydb_affinity_static }}
+{%- endif %}
 
 [Install]
 WantedBy=multi-user.target