diff --git a/.dmtlint.yaml b/.dmtlint.yaml index 9ec052bb20..9e4375a6a7 100644 --- a/.dmtlint.yaml +++ b/.dmtlint.yaml @@ -13,25 +13,15 @@ linters-settings: - "spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.contentType" rbac: exclude-rules: - # We exclude RBAC rules for virt-operator and cdi-operator because they create ClusterRoles and ClusterRoleBindings with wildcards. - # If we remove wildcard, virt-operator and cdi-operator will be unable to create them, as they do not have wildcard permissions themselves. + # We exclude RBAC rules for virt-operator because it creates ClusterRoles and ClusterRoleBindings with wildcards. + # If we remove wildcard, virt-operator will be unable to create them, as it does not have wildcard permissions itself. wildcards: - kind: ClusterRole name: d8:virtualization:kubevirt-operator - - kind: ClusterRole - name: d8:containerized-data-importer:cdi-operator - # We exclude RBAC rules for CDI and Kubevirt resources because they are used by upstream deployments. + # We exclude RBAC rules for Kubevirt resources because they are used by upstream deployments. # Changing these rules will require patching upstream code. placement: - - kind: ClusterRoleBinding - name: d8:containerized-data-importer:cdi-operator - - kind: ServiceAccount - name: cdi-operator - - kind: Role - name: cdi-operator - - kind: RoleBinding - name: cdi-operator - kind: ServiceAccount name: kubevirt-operator - kind: Role @@ -40,10 +30,7 @@ linters-settings: name: kubevirt-operator - kind: RoleBinding name: virt-operator - - kind: ClusterRole - name: d8:containerized-data-importer:cdi-operator binding-subject: - - cdi-sa - kubevirt-internal-virtualization-controller - kubevirt-internal-virtualization-handler module: @@ -54,6 +41,7 @@ linters-settings: - test/performance/ssh - test/e2e/legacy/testdata/sshkeys - images/dvcr-artifact/staging + - images/pvc-artifact/staging container: exclude-rules: readiness-probe: @@ -64,3 +52,61 @@ linters-settings: - kind: Deployment name: dvcr container: dvcr-garbage-collection + templates: + exclude-rules: + # These mount-points.yaml directories pre-create mount points for containerd strict mode. + # They are consumed by pods that are NOT rendered from this module's Helm templates: + # virt-launcher (created per-VM by KubeVirt at runtime), dvcr importer/uploader (created on + # demand by the controllers), and the virt-* components deployed by virt-operator. + # The mount-points linter only inspects Helm pod controllers, so it cannot see these + # mountPaths and reports false positives. + mount-points: + - /auth + - /certs + - /data + - /dev/bus/usb + - /dvcr-auth + - /dvcr-src-auth + - /etc/docker/registry + - /etc/libvirt + - /etc/podinfo + - /etc/ssl/docker + - /etc/virt-api/certificates + - /etc/virt-controller/certificates + - /etc/virt-controller/exportca + - /etc/virt-handler/clientcertificates + - /etc/virt-handler/servercertificates + - /etc/virt-operator/certificates + - /etc/virtualization-api-proxy/certificates + - /etc/virtualization-api/certificates + - /etc/virtualization-audit/certificates + - /init/usr/bin + - /kubeconfig.local + - /opt + - /path + - /pods + - /profile-data + - /proxycerts + - /run/cilium + - /run/kubevirt + - /run/kubevirt-libvirt-runtimes + - /run/kubevirt-private + - /scratch + - /shared + - /tmp/k8s-webhook-server/serving-certs + - /usr/lib/modules + - /var/cache/libvirt + - /var/lib/kubelet/device-plugins + - /var/lib/kubelet/plugins + - /var/lib/kubelet/plugins_registry + - /var/lib/kubelet/pods + - /var/lib/kubevirt + - /var/lib/kubevirt-node-labeller + - /var/lib/libvirt + - /var/lib/libvirt/qemu + - /var/lib/libvirt/qemu/nvram + - /var/lib/libvirt/swtpm + - /var/lib/registry + - /var/lib/swtpm-localca + - /var/log/libvirt + - /var/run/cdi diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2279a971e8..26bd65b486 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,7 +8,7 @@ /tests/ @hardcoretime @danilrwx /images/ @universal-itengineer @nevermarine -/images/cdi-artifact/ @Isteb4k @yaroslavborbat @diafour +/images/pvc-artifact/ @Isteb4k @yaroslavborbat @diafour /images/virt-artifact/ @Isteb4k @yaroslavborbat @diafour /images/dvcr-artifact/ @Isteb4k @yaroslavborbat @diafour /images/vm-route-forge/ @yaroslavborbat @diafour diff --git a/.github/actions/milestone-changelog/action.yml b/.github/actions/milestone-changelog/action.yml index 4ba0c6bef9..d825b1705a 100644 --- a/.github/actions/milestone-changelog/action.yml +++ b/.github/actions/milestone-changelog/action.yml @@ -36,6 +36,7 @@ runs: vdsnapshot vmsnapshot vmrestore + vmpool disks vd images diff --git a/.github/scripts/bash/e2e/cleanup-nightly-resources.sh b/.github/scripts/bash/e2e/cleanup-nightly-resources.sh index 60ba28546b..3f612f788d 100644 --- a/.github/scripts/bash/e2e/cleanup-nightly-resources.sh +++ b/.github/scripts/bash/e2e/cleanup-nightly-resources.sh @@ -23,6 +23,10 @@ source "${SCRIPT_DIR}/common.sh" LABEL_SELECTOR="${LABEL_SELECTOR:-test=nightly-e2e}" KEEP_HOURS="${KEEP_HOURS:-47}" FRIDAY_KEEP_HOURS="${FRIDAY_KEEP_HOURS:-71}" +# sds-elastic (Ceph) nested clusters are heavy, so they are torn down sooner (~1 day) +# and are not granted the Friday extension. Matched by storage type in the resource name. +ELASTIC_KEEP_HOURS="${ELASTIC_KEEP_HOURS:-23}" +ELASTIC_NAME_PATTERN="${ELASTIC_NAME_PATTERN:-sds-elastic}" current_date_seconds="$(date -u +%s)" @@ -35,20 +39,28 @@ collect_items_json() { should_keep() { local created_at="$1" + local name="$2" local resource_created_at_seconds local age_seconds local weekday_of_day + local keep_hours="${KEEP_HOURS}" + local friday_keep_hours="${FRIDAY_KEEP_HOURS}" + + if [[ "${name}" == *"${ELASTIC_NAME_PATTERN}"* ]]; then + keep_hours="${ELASTIC_KEEP_HOURS}" + friday_keep_hours="${ELASTIC_KEEP_HOURS}" + fi resource_created_at_seconds="$(date -d "${created_at}" -u +%s)" age_seconds="$(( current_date_seconds - resource_created_at_seconds ))" weekday_of_day="$(date -d "${created_at}" -u +%u)" - if [ "${age_seconds}" -lt "$(( KEEP_HOURS * 3600 ))" ]; then + if [ "${age_seconds}" -lt "$(( keep_hours * 3600 ))" ]; then echo "keep" return 0 fi - if [ "${weekday_of_day}" -eq 5 ] && [ "${age_seconds}" -lt "$(( FRIDAY_KEEP_HOURS * 3600 ))" ]; then + if [ "${weekday_of_day}" -eq 5 ] && [ "${age_seconds}" -lt "$(( friday_keep_hours * 3600 ))" ]; then echo "keep" return 0 fi @@ -69,7 +81,7 @@ cleanup_kind() { created_at="$(echo "${item}" | jq -r '.created_at')" [ -z "${name}" ] && continue - decision="$(should_keep "${created_at}")" + decision="$(should_keep "${created_at}" "${name}")" if [ "${decision}" = "keep" ]; then printf "%-63s %22s\n" "[INFO] Keep ${kind}/${name}:" "created_at ${created_at}" continue diff --git a/.github/scripts/bash/e2e/configure-sds-elastic.sh b/.github/scripts/bash/e2e/configure-sds-elastic.sh new file mode 100755 index 0000000000..d850accc1c --- /dev/null +++ b/.github/scripts/bash/e2e/configure-sds-elastic.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/wait-sds-elastic.sh +source "${SCRIPT_DIR}/wait-sds-elastic.sh" + +ELASTIC_CLUSTER_NAME=elastic +ELASTIC_STORAGE_CLASS=nested-ceph-rbd +# StorageClassMigration needs a second RBD storage class as a migration target. +ELASTIC_STORAGE_CLASSES=(nested-ceph-rbd nested-ceph-rbd-r3) + +# sds-elastic (Ceph via Rook) is Experimental, so enable it and its dependencies +# (sds-node-configurator, csi-ceph). On the stage profile the modules are absent in +# the stage registry, so pull them from the deckhouse-prod ModuleSource created by +# enable-sdn.sh; otherwise use the default deckhouse source. +apply_module_configs() { + local source_field=" source: deckhouse" + + if [ -n "${MODULE_SOURCE_REGISTRY_CFG:-}" ]; then + source_field=" source: deckhouse-prod" + fi + + kubectl apply -f - </dev/null 2>&1; then + echo "[SUCCESS] StorageClass ${sc} is present" + break + fi + echo "[INFO] Wait 10s for StorageClass ${sc} (attempt ${i}/30)" + sleep 10 + done +done + +echo "[INFO] Set default cluster storage class to ${ELASTIC_STORAGE_CLASS}" +kubectl patch mc global --type='json' \ + -p='[{"op": "replace", "path": "/spec/settings/defaultClusterStorageClass", "value": "'"${ELASTIC_STORAGE_CLASS}"'"}]' + +echo "[INFO] Show existing storageclasses and volumesnapshotclasses" +kubectl get storageclass +kubectl get volumesnapshotclass || echo "[WARNING] No volumesnapshotclasses found" diff --git a/.github/scripts/bash/e2e/power-off-nested-vms.sh b/.github/scripts/bash/e2e/power-off-nested-vms.sh index b27585effa..cfcffca192 100755 --- a/.github/scripts/bash/e2e/power-off-nested-vms.sh +++ b/.github/scripts/bash/e2e/power-off-nested-vms.sh @@ -20,9 +20,9 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=.github/scripts/bash/e2e/common.sh source "${SCRIPT_DIR}/common.sh" -# Constants (nested cluster: 1 master + 3 workers x2) -REQUIRED_MEM_GI=86 -REQUIRED_CPU=26 +# Constants ((nested cluster: 1 master + 3 workers) x3: replicated + nfs + sds-elastic) +REQUIRED_MEM_GI=129 +REQUIRED_CPU=39 MIN_MEM_GI_PER_NODE=12 MIN_CPU_PER_NODE=4 MIN_NODES_FOR_PLACEMENT=3 diff --git a/.github/scripts/bash/e2e/render-dvp-static-values.sh b/.github/scripts/bash/e2e/render-dvp-static-values.sh index 6b6c7a03bc..9720e17d94 100644 --- a/.github/scripts/bash/e2e/render-dvp-static-values.sh +++ b/.github/scripts/bash/e2e/render-dvp-static-values.sh @@ -57,6 +57,19 @@ envsubst_variables="$(grep -oE '\$\{[A-Z0-9_]+\}' values.yaml.tmpl | sort -u | t envsubst "${envsubst_variables}" \ < values.yaml.tmpl > values.yaml +# The template defines one additional worker disk. When ADDITIONAL_DISK_COUNT > 1 +# (e.g. sds-elastic runs several Ceph OSDs per node), append the extra disks, each +# of ADDITIONAL_DISK_SIZE, to every worker node group. +additional_disk_count="${ADDITIONAL_DISK_COUNT:-1}" +if (( additional_disk_count > 1 )); then + for (( d = 2; d <= additional_disk_count; d++ )); do + # ADDITIONAL_DISK_SIZE is exported by the workflow step env and read by yq's env(). + yq eval --inplace \ + '(.instances.additionalNodes[] | select(.name == "worker") | .cfg.additionalDisks) += [{"size": env(ADDITIONAL_DISK_SIZE)}]' \ + values.yaml + done +fi + mkdir -p tmp touch tmp/discovered-values.yaml diff --git a/.github/scripts/bash/e2e/wait-sds-elastic.sh b/.github/scripts/bash/e2e/wait-sds-elastic.sh new file mode 100755 index 0000000000..1224574cfd --- /dev/null +++ b/.github/scripts/bash/e2e/wait-sds-elastic.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" +# shellcheck source=.github/scripts/bash/e2e/deckhouse.sh +source "${SCRIPT_DIR}/deckhouse.sh" + +# Waits until the raw additional disks that back the Ceph OSDs are discovered by +# sds-node-configurator. Expects ELASTIC_OSD_DISKS_PER_NODE consumable BlockDevices +# per worker node (one OSD per additional disk). +elastic_blockdevices_ready() { + local count=60 + local workers + local blockdevices + local disks_per_node="${ELASTIC_OSD_DISKS_PER_NODE:-1}" + local expected + + workers="$(kubectl get nodes -o name | grep -c worker || true)" + workers=$((workers)) + + if [[ "$workers" -eq 0 ]]; then + echo "[ERROR] No worker nodes found" + return 1 + fi + + expected=$(( workers * disks_per_node )) + + for i in $(seq 1 "$count"); do + blockdevices="$(kubectl get blockdevices.storage.deckhouse.io -o json | jq '[.items[] | select(.status.consumable == true)] | length' || echo 0)" + blockdevices=$((blockdevices)) + if [[ "$blockdevices" -ge "$expected" ]]; then + echo "[SUCCESS] Consumable blockdevices (${blockdevices}) is greater or equal to expected (${expected} = ${workers} workers x ${disks_per_node} disks)" + kubectl get blockdevices.storage.deckhouse.io -o wide + return 0 + fi + + echo "[INFO] Wait 10s until consumable blockdevices >= ${expected} (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show blockdevices" + kubectl get blockdevices.storage.deckhouse.io -o wide || true + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + sleep 10 + done + + echo "[ERROR] Consumable blockdevices did not reach ${expected} in time" + echo "[DEBUG] Show cluster nodes" + kubectl get nodes || true + echo "[DEBUG] Show blockdevices" + kubectl get blockdevices.storage.deckhouse.io -o wide || true + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 || true + echo "::endgroup::" + return 1 +} + +# Waits until the ElasticCluster reaches phase Ready and Ceph reports HEALTH_OK. +# Rook cluster bring-up (mon/mgr/osd) on nested VMs is slow: with several OSDs per node +# plus occasional sds-node-configurator restarts a full bring-up can take ~50 min, so the +# timeout is deliberately generous (240 x 15s = 60 min). +elastic_cluster_ready() { + local ec_name="$1" + local count=240 + local phase + local health + + for i in $(seq 1 "$count"); do + phase="$(kubectl get ec "$ec_name" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")" + health="$(kubectl get ec "$ec_name" -o jsonpath='{.status.health.status}' 2>/dev/null || echo "")" + + if [[ "$phase" == "Ready" && "$health" == "HEALTH_OK" ]]; then + echo "[SUCCESS] ElasticCluster ${ec_name} is Ready (${health})" + kubectl get ec "$ec_name" -o wide + return 0 + fi + + echo "[INFO] Wait 15s for ElasticCluster ${ec_name} (phase=${phase:-}, health=${health:-}) (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] ElasticCluster status" + kubectl get ec "$ec_name" -o wide || true + echo "[DEBUG] CephCluster status" + kubectl get cephcluster -A -o wide 2>/dev/null || true + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + sleep 15 + done + + echo "[ERROR] ElasticCluster ${ec_name} did not become Ready/HEALTH_OK in time" + echo "::group::ElasticCluster" + kubectl get ec "$ec_name" -o yaml || true + echo "::endgroup::" + echo "::group::LVMVolumeGroups" + kubectl get lvmvolumegroup -o wide || true + echo "::endgroup::" + echo "::group::CephCluster" + kubectl get cephcluster -A -o yaml 2>/dev/null || true + echo "::endgroup::" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 || true + echo "::endgroup::" + return 1 +} diff --git a/.github/scripts/bash/e2e/wait-virtualization-ready.sh b/.github/scripts/bash/e2e/wait-virtualization-ready.sh index 3685a16ac1..4083ce1da8 100644 --- a/.github/scripts/bash/e2e/wait-virtualization-ready.sh +++ b/.github/scripts/bash/e2e/wait-virtualization-ready.sh @@ -180,6 +180,11 @@ enable_maintenance_mode() { echo "[INFO] Switch csi-nfs module to maintenance mode" kubectl patch mc csi-nfs --type merge --patch '{"spec":{"maintenance":"NoResourceReconciliation"}}' ;; + sds-elastic) + echo "[INFO] Switch sds-elastic and csi-ceph modules to maintenance mode" + kubectl patch mc sds-elastic --type merge --patch '{"spec":{"maintenance":"NoResourceReconciliation"}}' + kubectl patch mc csi-ceph --type merge --patch '{"spec":{"maintenance":"NoResourceReconciliation"}}' + ;; local) echo "[INFO] Switch sds-local-volume module to maintenance mode" kubectl patch mc sds-local-volume --type merge --patch '{"spec":{"maintenance":"NoResourceReconciliation"}}' diff --git a/.github/workflows/check-changelog-entry.yml b/.github/workflows/check-changelog-entry.yml index b459953e33..49e5b9e47e 100644 --- a/.github/workflows/check-changelog-entry.yml +++ b/.github/workflows/check-changelog-entry.yml @@ -46,6 +46,7 @@ jobs: vdsnapshot vmsnapshot vmrestore + vmpool disks vd images diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml index f3ee59e7c9..c4bc698b0a 100644 --- a/.github/workflows/dev_module_build.yml +++ b/.github/workflows/dev_module_build.yml @@ -248,7 +248,6 @@ jobs: # Find directories containing .golangci.yaml (excluding symlinks) mapfile -t config_dirs < <( find . \ - -path ./images/cdi-cloner/cloner-startup -prune -o \ -path ./images/dvcr-artifact -prune -o \ -path ./test/performance/shatal -prune -o \ -type f -name '.golangci.yaml' -not -type l -printf '%h\0' | \ @@ -332,6 +331,10 @@ jobs: task virtualization-controller:init task virtualization-controller:test:unit + - name: Run unit test pvc-artifact + run: | + task pvc-artifact:test:unit + test_scripts_js: runs-on: ubuntu-22.04 name: Run JS unit tests diff --git a/.github/workflows/dev_validation.yaml b/.github/workflows/dev_validation.yaml index 6a1fcc173f..c24ae9b278 100644 --- a/.github/workflows/dev_validation.yaml +++ b/.github/workflows/dev_validation.yaml @@ -126,7 +126,7 @@ jobs: severity: warning ignore_paths: >- vendor - images/cdi-artifact + images/pvc-artifact images/virt-api/__virt images/virt-controller/__virt images/virt-handler/__virt diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index c6d2b78d49..f1912e425a 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -40,10 +40,15 @@ on: type: string default: "50Gi" description: "Set additional disk size for workers node in cluster config" + cluster_config_additional_disk_count: + required: false + type: string + default: "1" + description: "Number of additional disks per worker node (each of cluster_config_additional_disk_size). sds-elastic uses more than one to run multiple Ceph OSDs per node." storage_type: required: true type: string - description: "Storage type (ceph or replicated or etc.)" + description: "Storage type (replicated, nfs, sds-elastic, etc.)" nested_storageclass_name: required: true type: string @@ -257,6 +262,7 @@ jobs: APT_MIRROR_URL: ${{ inputs.apt_mirror_url }} CLUSTER_CONFIG_WORKERS_MEMORY: ${{ inputs.cluster_config_workers_memory }} ADDITIONAL_DISK_SIZE: ${{ inputs.cluster_config_additional_disk_size }} + ADDITIONAL_DISK_COUNT: ${{ inputs.cluster_config_additional_disk_count }} ENABLED_MODULES: "" NESTED_CLUSTER_NETWORK_NAME: ${{ inputs.nested_cluster_network_name }} run: bash "${E2E_SCRIPT_DIR}/render-dvp-static-values.sh" @@ -490,6 +496,20 @@ jobs: MODULE_SOURCE_REGISTRY_CFG: ${{ inputs.registry_profile == 'stage' && secrets.PROD_IO_REGISTRY_DOCKER_CFG || '' }} run: bash "${E2E_SCRIPT_DIR}/configure-csi-nfs.sh" + - name: Configure sds-elastic storage + if: ${{ inputs.storage_type == 'sds-elastic' }} + id: storage-sds-elastic-setup + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-elastic + env: + NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} + # sds-elastic, csi-ceph and sds-node-configurator are absent in the stage + # registry; pull them from prod via the same ModuleSource deckhouse-prod that + # enable-sdn.sh creates for the stage profile. + MODULE_SOURCE_REGISTRY_CFG: ${{ inputs.registry_profile == 'stage' && secrets.PROD_IO_REGISTRY_DOCKER_CFG || '' }} + # One OSD per additional disk per worker; the wait expects workers x this count. + ELASTIC_OSD_DISKS_PER_NODE: ${{ inputs.cluster_config_additional_disk_count }} + run: bash "${E2E_SCRIPT_DIR}/configure-sds-elastic.sh" + configure-virtualization: name: Configure Virtualization runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 91a28789d0..92cf355082 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -48,6 +48,8 @@ jobs: LABEL_SELECTOR: test=nightly-e2e KEEP_HOURS: "47" FRIDAY_KEEP_HOURS: "71" + # Ceph (sds-elastic) nested clusters are torn down after ~1 day. + ELASTIC_KEEP_HOURS: "23" run: bash .github/scripts/bash/e2e/cleanup-nightly-resources.sh power-off-vms-for-nested: @@ -160,12 +162,49 @@ jobs: BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} E2E_ARTIFACTS_GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + e2e-sds-elastic: + name: E2E Pipeline (Elastic) + needs: + - set-vars + uses: ./.github/workflows/e2e-nightly-reusable-pipeline.yml + with: + storage_type: sds-elastic + pipeline_job_name: "E2E Pipeline (Elastic)" + nested_storageclass_name: nested-ceph-rbd + nested_cluster_network_name: cn-4006-for-e2e-test + branch: main + virtualization_tag: main + deckhouse_channel: ${{ needs.set-vars.outputs.deckhouse_channel }} + deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }} + registry_profile: ${{ needs.set-vars.outputs.registry_profile }} + default_user: cloud + go_version: "1.24.13" + e2e_timeout: "3.5h" + e2e_focus_tests: "VirtualDiskProvisioning|VirtualDiskSnapshots|VirtualImageCreation|VirtualDiskResizing|DiskAttachment|BlockDeviceHotplug|Migration|StorageClassMigration|RWOVirtualDiskMigration|VMSOP|Restore|DataExports" + e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }} + date_start: ${{ needs.set-vars.outputs.date_start }} + randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} + cluster_config_workers_memory: "9Gi" + cluster_config_additional_disk_size: "50Gi" + # Two 50Gi disks per worker -> two Ceph OSDs per node for more throughput. + cluster_config_additional_disk_count: "2" + cluster_config_k8s_version: "Automatic" + apt_mirror_enabled: true + secrets: + DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} + VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + REGISTRY_DOCKER_CFG: ${{ needs.set-vars.outputs.registry_profile == 'stage' && secrets.STAGE_IO_REGISTRY_DOCKER_CFG || secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} + E2E_ARTIFACTS_GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + report-to-channel: runs-on: ubuntu-latest name: End-to-End tests report needs: - e2e-replicated - e2e-nfs + - e2e-sds-elastic if: ${{ always()}} steps: - uses: actions/checkout@v6 @@ -200,7 +239,7 @@ jobs: id: render-report uses: actions/github-script@v7 env: - EXPECTED_STORAGE_TYPES: '["replicated","nfs"]' + EXPECTED_STORAGE_TYPES: '["replicated","nfs","sds-elastic"]' LOOP_API_BASE_URL: ${{ secrets.LOOP_API_BASE_URL }} LOOP_CHANNEL_ID: ${{ secrets.LOOP_CHANNEL_ID }} LOOP_TOKEN: ${{ secrets.LOOP_TOKEN }} diff --git a/.github/workflows/release_module_release-channels.yml b/.github/workflows/release_module_release-channels.yml index 02c9528a1d..c8d81a7366 100644 --- a/.github/workflows/release_module_release-channels.yml +++ b/.github/workflows/release_module_release-channels.yml @@ -449,12 +449,11 @@ jobs: with: go-version: "${{ env.GO_VERSION }}" - - name: Login to PROD_REGISTRY - uses: deckhouse/modules-actions/setup@v2 + - name: Log in to private registry + id: registry-login + uses: ./.github/actions/registry-login with: - registry: ${{ vars.PROD_READ_REGISTRY }} - registry_login: ${{ secrets.PROD_READ_REGISTRY_USER }} - registry_password: ${{ secrets.PROD_READ_REGISTRY_PASSWORD }} + docker_cfg: ${{ secrets.PROD_RU_REGISTRY_DOCKER_CFG }} - uses: actions/checkout@v4 with: diff --git a/.prettierignore b/.prettierignore index 40ab026e2e..35abbbcff8 100644 --- a/.prettierignore +++ b/.prettierignore @@ -10,7 +10,6 @@ images/**/mount-points.yaml CHANGELOG/*.yml CHANGELOG/*.yaml crds/embedded -images/cdi-artifact/__containerized-data-importer* images/virt-artifact/__kubevirt* .gitlab-ci.yml tools/kubeconform diff --git a/CHANGELOG/CHANGELOG-v1.9.3.yml b/CHANGELOG/CHANGELOG-v1.9.3.yml new file mode 100644 index 0000000000..1fe6a7345a --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.9.3.yml @@ -0,0 +1,17 @@ +module: + fixes: + - summary: >- + Pre-create missing container mount points so virt-handler and virtualization-dra pods start + under containerd strict mode. + pull_request: https://github.com/deckhouse/virtualization/pull/2584 + - summary: >- + Fixed slow downloading of images from DVCR to the node when attaching them to a virtual + machine. + pull_request: https://github.com/deckhouse/virtualization/pull/2576 +vm: + fixes: + - summary: >- + Fixed a volume mount leak that could leave a VM with hotplugged images stuck in the + Terminating state during deletion. + pull_request: https://github.com/deckhouse/virtualization/pull/2581 + diff --git a/CHANGELOG/CHANGELOG-v1.9.md b/CHANGELOG/CHANGELOG-v1.9.md index 69f5a2a837..b8e4aae60a 100644 --- a/CHANGELOG/CHANGELOG-v1.9.md +++ b/CHANGELOG/CHANGELOG-v1.9.md @@ -21,6 +21,7 @@ - **[core]** Fixed reduced throughput during live migration of running VMs compared to v1.8.3. [#2541](https://github.com/deckhouse/virtualization/pull/2541) - **[core]** Remove excess empty labels with unused tsc frequencies on nodes. [#2351](https://github.com/deckhouse/virtualization/pull/2351) - **[core]** Better handling Windows guests: start and migration should work in clusters with frequent CPU frequencies drifts [#2345](https://github.com/deckhouse/virtualization/pull/2345) + - **[module]** Fixed slow downloading of images from DVCR to the node when attaching them to a virtual machine. [#2576](https://github.com/deckhouse/virtualization/pull/2576) - **[module]** Fixed slow import and upload of images to DVCR when network bandwidth was not the bottleneck. [#2552](https://github.com/deckhouse/virtualization/pull/2552) - **[module]** Fixed an issue where invalid `virtualization` module ModuleConfig settings could block the Deckhouse queue. [#2246](https://github.com/deckhouse/virtualization/pull/2246) - **[module]** Fixed duplicate series on the `Virtualization / Overview` dashboard. [#2189](https://github.com/deckhouse/virtualization/pull/2189) @@ -28,6 +29,7 @@ - **[vd]** Fixed cancellation of virtual disk storage class changes and cancellation of local disk migration. [#2502](https://github.com/deckhouse/virtualization/pull/2502) - **[vd]** Time spent in the `WaitForFirstConsumer` phase is no longer included in `.status.stats.creationDuration.totalProvisioning` of virtual disks. [#2379](https://github.com/deckhouse/virtualization/pull/2379) - **[vd]** Allow ingress from virtualization namespace to importer pods [#2356](https://github.com/deckhouse/virtualization/pull/2356) + - **[vm]** Fixed a volume mount leak that could leave a VM with hotplugged images stuck in the Terminating state during deletion. [#2581](https://github.com/deckhouse/virtualization/pull/2581) - **[vm]** Fixed an issue that prevented a VM from starting after a failed migration of a disk on local storage. [#2509](https://github.com/deckhouse/virtualization/pull/2509) - **[vm]** Fixed live migration of VMs with disks on local storage attached via VirtualMachineBlockDeviceAttachment (hotplug). The target node no longer matches the source node. [#2508](https://github.com/deckhouse/virtualization/pull/2508) - **[vm]** Fixed a false reboot requirement for VMs with only the Main network after upgrading to v1.9.1. Such VMs now do not receive the RestartRequired status if their configuration has not actually changed. [#2475](https://github.com/deckhouse/virtualization/pull/2475) diff --git a/Taskfile.yaml b/Taskfile.yaml index cc83d74680..4062e3951b 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -15,6 +15,9 @@ includes: gohooks: taskfile: ./images/hooks dir: ./images/hooks + pvc-artifact: + taskfile: ./images/pvc-artifact + dir: ./images/pvc-artifact e2e: taskfile: ./test/e2e dir: ./test/e2e @@ -162,7 +165,6 @@ tasks: TARGETS=( "images/hooks" "images/dvcr-artifact" - "images/cdi-cloner/cloner-startup" "images/virtualization-dra" "src/cli" "test/e2e" diff --git a/api/client/generated/clientset/versioned/typed/core/v1alpha2/core_client.go b/api/client/generated/clientset/versioned/typed/core/v1alpha2/core_client.go index 95d9f09723..9bde1c921a 100644 --- a/api/client/generated/clientset/versioned/typed/core/v1alpha2/core_client.go +++ b/api/client/generated/clientset/versioned/typed/core/v1alpha2/core_client.go @@ -42,6 +42,7 @@ type VirtualizationV1alpha2Interface interface { VirtualMachineMACAddressesGetter VirtualMachineMACAddressLeasesGetter VirtualMachineOperationsGetter + VirtualMachinePoolsGetter VirtualMachineSnapshotsGetter VirtualMachineSnapshotOperationsGetter } @@ -107,6 +108,10 @@ func (c *VirtualizationV1alpha2Client) VirtualMachineOperations(namespace string return newVirtualMachineOperations(c, namespace) } +func (c *VirtualizationV1alpha2Client) VirtualMachinePools(namespace string) VirtualMachinePoolInterface { + return newVirtualMachinePools(c, namespace) +} + func (c *VirtualizationV1alpha2Client) VirtualMachineSnapshots(namespace string) VirtualMachineSnapshotInterface { return newVirtualMachineSnapshots(c, namespace) } diff --git a/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_core_client.go b/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_core_client.go index 3e4d10a08e..b4498e50fd 100644 --- a/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_core_client.go +++ b/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_core_client.go @@ -84,6 +84,10 @@ func (c *FakeVirtualizationV1alpha2) VirtualMachineOperations(namespace string) return newFakeVirtualMachineOperations(c, namespace) } +func (c *FakeVirtualizationV1alpha2) VirtualMachinePools(namespace string) v1alpha2.VirtualMachinePoolInterface { + return newFakeVirtualMachinePools(c, namespace) +} + func (c *FakeVirtualizationV1alpha2) VirtualMachineSnapshots(namespace string) v1alpha2.VirtualMachineSnapshotInterface { return newFakeVirtualMachineSnapshots(c, namespace) } diff --git a/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_virtualmachinepool.go b/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_virtualmachinepool.go new file mode 100644 index 0000000000..66f8670201 --- /dev/null +++ b/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_virtualmachinepool.go @@ -0,0 +1,52 @@ +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + corev1alpha2 "github.com/deckhouse/virtualization/api/client/generated/clientset/versioned/typed/core/v1alpha2" + v1alpha2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + gentype "k8s.io/client-go/gentype" +) + +// fakeVirtualMachinePools implements VirtualMachinePoolInterface +type fakeVirtualMachinePools struct { + *gentype.FakeClientWithList[*v1alpha2.VirtualMachinePool, *v1alpha2.VirtualMachinePoolList] + Fake *FakeVirtualizationV1alpha2 +} + +func newFakeVirtualMachinePools(fake *FakeVirtualizationV1alpha2, namespace string) corev1alpha2.VirtualMachinePoolInterface { + return &fakeVirtualMachinePools{ + gentype.NewFakeClientWithList[*v1alpha2.VirtualMachinePool, *v1alpha2.VirtualMachinePoolList]( + fake.Fake, + namespace, + v1alpha2.SchemeGroupVersion.WithResource("virtualmachinepools"), + v1alpha2.SchemeGroupVersion.WithKind("VirtualMachinePool"), + func() *v1alpha2.VirtualMachinePool { return &v1alpha2.VirtualMachinePool{} }, + func() *v1alpha2.VirtualMachinePoolList { return &v1alpha2.VirtualMachinePoolList{} }, + func(dst, src *v1alpha2.VirtualMachinePoolList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha2.VirtualMachinePoolList) []*v1alpha2.VirtualMachinePool { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha2.VirtualMachinePoolList, items []*v1alpha2.VirtualMachinePool) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/images/cdi-cloner/cloner-startup/internal/helpers/env_process.go b/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_virtualmachinepool_expansion.go similarity index 58% rename from images/cdi-cloner/cloner-startup/internal/helpers/env_process.go rename to api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_virtualmachinepool_expansion.go index efdc88101f..5cd648e768 100644 --- a/images/cdi-cloner/cloner-startup/internal/helpers/env_process.go +++ b/api/client/generated/clientset/versioned/typed/core/v1alpha2/fake/fake_virtualmachinepool_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2025 Flant JSC +Copyright 2026 Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,26 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helpers +package fake import ( - "fmt" - "os" - "strconv" -) + "context" -func GetEnv(key string) (string, error) { - value, ok := os.LookupEnv(key) - if !ok { - return "", fmt.Errorf("missing required environment variable: %s", key) - } - return value, nil -} + "github.com/deckhouse/virtualization/api/subresources/v1alpha2" +) -func GetBoolEnv(key string) (bool, error) { - value, err := GetEnv(key) - if err != nil { - return false, err - } - return strconv.ParseBool(value) +func (c *fakeVirtualMachinePools) ScaleDownWith(ctx context.Context, name string, opts v1alpha2.VirtualMachinePoolScaleDownWith) error { + return nil } diff --git a/api/client/generated/clientset/versioned/typed/core/v1alpha2/virtualmachinepool.go b/api/client/generated/clientset/versioned/typed/core/v1alpha2/virtualmachinepool.go new file mode 100644 index 0000000000..fb0fc5839b --- /dev/null +++ b/api/client/generated/clientset/versioned/typed/core/v1alpha2/virtualmachinepool.go @@ -0,0 +1,70 @@ +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + context "context" + + scheme "github.com/deckhouse/virtualization/api/client/generated/clientset/versioned/scheme" + corev1alpha2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// VirtualMachinePoolsGetter has a method to return a VirtualMachinePoolInterface. +// A group's client should implement this interface. +type VirtualMachinePoolsGetter interface { + VirtualMachinePools(namespace string) VirtualMachinePoolInterface +} + +// VirtualMachinePoolInterface has methods to work with VirtualMachinePool resources. +type VirtualMachinePoolInterface interface { + Create(ctx context.Context, virtualMachinePool *corev1alpha2.VirtualMachinePool, opts v1.CreateOptions) (*corev1alpha2.VirtualMachinePool, error) + Update(ctx context.Context, virtualMachinePool *corev1alpha2.VirtualMachinePool, opts v1.UpdateOptions) (*corev1alpha2.VirtualMachinePool, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, virtualMachinePool *corev1alpha2.VirtualMachinePool, opts v1.UpdateOptions) (*corev1alpha2.VirtualMachinePool, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*corev1alpha2.VirtualMachinePool, error) + List(ctx context.Context, opts v1.ListOptions) (*corev1alpha2.VirtualMachinePoolList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *corev1alpha2.VirtualMachinePool, err error) + VirtualMachinePoolExpansion +} + +// virtualMachinePools implements VirtualMachinePoolInterface +type virtualMachinePools struct { + *gentype.ClientWithList[*corev1alpha2.VirtualMachinePool, *corev1alpha2.VirtualMachinePoolList] +} + +// newVirtualMachinePools returns a VirtualMachinePools +func newVirtualMachinePools(c *VirtualizationV1alpha2Client, namespace string) *virtualMachinePools { + return &virtualMachinePools{ + gentype.NewClientWithList[*corev1alpha2.VirtualMachinePool, *corev1alpha2.VirtualMachinePoolList]( + "virtualmachinepools", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *corev1alpha2.VirtualMachinePool { return &corev1alpha2.VirtualMachinePool{} }, + func() *corev1alpha2.VirtualMachinePoolList { return &corev1alpha2.VirtualMachinePoolList{} }, + ), + } +} diff --git a/images/cdi-cloner/cloner-startup/internal/helpers/cloner.go b/api/client/generated/clientset/versioned/typed/core/v1alpha2/virtualmachinepool_expansion.go similarity index 54% rename from images/cdi-cloner/cloner-startup/internal/helpers/cloner.go rename to api/client/generated/clientset/versioned/typed/core/v1alpha2/virtualmachinepool_expansion.go index eae15e6cae..992d2dc4ac 100644 --- a/images/cdi-cloner/cloner-startup/internal/helpers/cloner.go +++ b/api/client/generated/clientset/versioned/typed/core/v1alpha2/virtualmachinepool_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2025 Flant JSC +Copyright 2026 Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,26 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helpers +package v1alpha2 import ( - "os" - "os/exec" - "strconv" + "context" + "fmt" + + "github.com/deckhouse/virtualization/api/subresources/v1alpha2" ) -func RunCloner(contentType string, uploadBytes uint64, mountPoint string) error { - cmd := exec.Command("/usr/bin/cdi-cloner", - "-v=3", - "-alsologtostderr", - "-content-type="+contentType, - "-upload-bytes="+strconv.FormatUint(uploadBytes, 10), - "-mount="+mountPoint, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return err - } - return nil +type VirtualMachinePoolExpansion interface { + ScaleDownWith(ctx context.Context, name string, opts v1alpha2.VirtualMachinePoolScaleDownWith) error +} + +func (c *virtualMachinePools) ScaleDownWith(ctx context.Context, name string, opts v1alpha2.VirtualMachinePoolScaleDownWith) error { + return fmt.Errorf("not implemented") } diff --git a/api/client/generated/informers/externalversions/core/v1alpha2/interface.go b/api/client/generated/informers/externalversions/core/v1alpha2/interface.go index ed97a49b94..c98d319363 100644 --- a/api/client/generated/informers/externalversions/core/v1alpha2/interface.go +++ b/api/client/generated/informers/externalversions/core/v1alpha2/interface.go @@ -52,6 +52,8 @@ type Interface interface { VirtualMachineMACAddressLeases() VirtualMachineMACAddressLeaseInformer // VirtualMachineOperations returns a VirtualMachineOperationInformer. VirtualMachineOperations() VirtualMachineOperationInformer + // VirtualMachinePools returns a VirtualMachinePoolInformer. + VirtualMachinePools() VirtualMachinePoolInformer // VirtualMachineSnapshots returns a VirtualMachineSnapshotInformer. VirtualMachineSnapshots() VirtualMachineSnapshotInformer // VirtualMachineSnapshotOperations returns a VirtualMachineSnapshotOperationInformer. @@ -139,6 +141,11 @@ func (v *version) VirtualMachineOperations() VirtualMachineOperationInformer { return &virtualMachineOperationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// VirtualMachinePools returns a VirtualMachinePoolInformer. +func (v *version) VirtualMachinePools() VirtualMachinePoolInformer { + return &virtualMachinePoolInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // VirtualMachineSnapshots returns a VirtualMachineSnapshotInformer. func (v *version) VirtualMachineSnapshots() VirtualMachineSnapshotInformer { return &virtualMachineSnapshotInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/api/client/generated/informers/externalversions/core/v1alpha2/virtualmachinepool.go b/api/client/generated/informers/externalversions/core/v1alpha2/virtualmachinepool.go new file mode 100644 index 0000000000..de1f406bac --- /dev/null +++ b/api/client/generated/informers/externalversions/core/v1alpha2/virtualmachinepool.go @@ -0,0 +1,102 @@ +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + context "context" + time "time" + + versioned "github.com/deckhouse/virtualization/api/client/generated/clientset/versioned" + internalinterfaces "github.com/deckhouse/virtualization/api/client/generated/informers/externalversions/internalinterfaces" + corev1alpha2 "github.com/deckhouse/virtualization/api/client/generated/listers/core/v1alpha2" + apicorev1alpha2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// VirtualMachinePoolInformer provides access to a shared informer and lister for +// VirtualMachinePools. +type VirtualMachinePoolInformer interface { + Informer() cache.SharedIndexInformer + Lister() corev1alpha2.VirtualMachinePoolLister +} + +type virtualMachinePoolInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewVirtualMachinePoolInformer constructs a new informer for VirtualMachinePool type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewVirtualMachinePoolInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredVirtualMachinePoolInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredVirtualMachinePoolInformer constructs a new informer for VirtualMachinePool type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredVirtualMachinePoolInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtualizationV1alpha2().VirtualMachinePools(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtualizationV1alpha2().VirtualMachinePools(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtualizationV1alpha2().VirtualMachinePools(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtualizationV1alpha2().VirtualMachinePools(namespace).Watch(ctx, options) + }, + }, + &apicorev1alpha2.VirtualMachinePool{}, + resyncPeriod, + indexers, + ) +} + +func (f *virtualMachinePoolInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredVirtualMachinePoolInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *virtualMachinePoolInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apicorev1alpha2.VirtualMachinePool{}, f.defaultInformer) +} + +func (f *virtualMachinePoolInformer) Lister() corev1alpha2.VirtualMachinePoolLister { + return corev1alpha2.NewVirtualMachinePoolLister(f.Informer().GetIndexer()) +} diff --git a/api/client/generated/informers/externalversions/generic.go b/api/client/generated/informers/externalversions/generic.go index 8b16f3d58a..093bdb6a29 100644 --- a/api/client/generated/informers/externalversions/generic.go +++ b/api/client/generated/informers/externalversions/generic.go @@ -82,6 +82,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Virtualization().V1alpha2().VirtualMachineMACAddressLeases().Informer()}, nil case v1alpha2.SchemeGroupVersion.WithResource("virtualmachineoperations"): return &genericInformer{resource: resource.GroupResource(), informer: f.Virtualization().V1alpha2().VirtualMachineOperations().Informer()}, nil + case v1alpha2.SchemeGroupVersion.WithResource("virtualmachinepools"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Virtualization().V1alpha2().VirtualMachinePools().Informer()}, nil case v1alpha2.SchemeGroupVersion.WithResource("virtualmachinesnapshots"): return &genericInformer{resource: resource.GroupResource(), informer: f.Virtualization().V1alpha2().VirtualMachineSnapshots().Informer()}, nil case v1alpha2.SchemeGroupVersion.WithResource("virtualmachinesnapshotoperations"): diff --git a/api/client/generated/listers/core/v1alpha2/expansion_generated.go b/api/client/generated/listers/core/v1alpha2/expansion_generated.go index e47e2ae835..f7da265496 100644 --- a/api/client/generated/listers/core/v1alpha2/expansion_generated.go +++ b/api/client/generated/listers/core/v1alpha2/expansion_generated.go @@ -110,6 +110,14 @@ type VirtualMachineOperationListerExpansion interface{} // VirtualMachineOperationNamespaceLister. type VirtualMachineOperationNamespaceListerExpansion interface{} +// VirtualMachinePoolListerExpansion allows custom methods to be added to +// VirtualMachinePoolLister. +type VirtualMachinePoolListerExpansion interface{} + +// VirtualMachinePoolNamespaceListerExpansion allows custom methods to be added to +// VirtualMachinePoolNamespaceLister. +type VirtualMachinePoolNamespaceListerExpansion interface{} + // VirtualMachineSnapshotListerExpansion allows custom methods to be added to // VirtualMachineSnapshotLister. type VirtualMachineSnapshotListerExpansion interface{} diff --git a/api/client/generated/listers/core/v1alpha2/virtualmachinepool.go b/api/client/generated/listers/core/v1alpha2/virtualmachinepool.go new file mode 100644 index 0000000000..2bc93b5adb --- /dev/null +++ b/api/client/generated/listers/core/v1alpha2/virtualmachinepool.go @@ -0,0 +1,70 @@ +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + corev1alpha2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// VirtualMachinePoolLister helps list VirtualMachinePools. +// All objects returned here must be treated as read-only. +type VirtualMachinePoolLister interface { + // List lists all VirtualMachinePools in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*corev1alpha2.VirtualMachinePool, err error) + // VirtualMachinePools returns an object that can list and get VirtualMachinePools. + VirtualMachinePools(namespace string) VirtualMachinePoolNamespaceLister + VirtualMachinePoolListerExpansion +} + +// virtualMachinePoolLister implements the VirtualMachinePoolLister interface. +type virtualMachinePoolLister struct { + listers.ResourceIndexer[*corev1alpha2.VirtualMachinePool] +} + +// NewVirtualMachinePoolLister returns a new VirtualMachinePoolLister. +func NewVirtualMachinePoolLister(indexer cache.Indexer) VirtualMachinePoolLister { + return &virtualMachinePoolLister{listers.New[*corev1alpha2.VirtualMachinePool](indexer, corev1alpha2.Resource("virtualmachinepool"))} +} + +// VirtualMachinePools returns an object that can list and get VirtualMachinePools. +func (s *virtualMachinePoolLister) VirtualMachinePools(namespace string) VirtualMachinePoolNamespaceLister { + return virtualMachinePoolNamespaceLister{listers.NewNamespaced[*corev1alpha2.VirtualMachinePool](s.ResourceIndexer, namespace)} +} + +// VirtualMachinePoolNamespaceLister helps list and get VirtualMachinePools. +// All objects returned here must be treated as read-only. +type VirtualMachinePoolNamespaceLister interface { + // List lists all VirtualMachinePools in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*corev1alpha2.VirtualMachinePool, err error) + // Get retrieves the VirtualMachinePool from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*corev1alpha2.VirtualMachinePool, error) + VirtualMachinePoolNamespaceListerExpansion +} + +// virtualMachinePoolNamespaceLister implements the VirtualMachinePoolNamespaceLister +// interface. +type virtualMachinePoolNamespaceLister struct { + listers.ResourceIndexer[*corev1alpha2.VirtualMachinePool] +} diff --git a/api/client/kubeclient/client.go b/api/client/kubeclient/client.go index e0063001f0..864597e40e 100644 --- a/api/client/kubeclient/client.go +++ b/api/client/kubeclient/client.go @@ -54,8 +54,10 @@ type Client interface { kubernetes.Interface ClusterVirtualImages() virtualizationv1alpha2.ClusterVirtualImageInterface VirtualMachines(namespace string) virtualizationv1alpha2.VirtualMachineInterface + VirtualMachinePools(namespace string) virtualizationv1alpha2.VirtualMachinePoolInterface VirtualImages(namespace string) virtualizationv1alpha2.VirtualImageInterface VirtualDisks(namespace string) virtualizationv1alpha2.VirtualDiskInterface + VirtualDiskSnapshots(namespace string) virtualizationv1alpha2.VirtualDiskSnapshotInterface VirtualMachineBlockDeviceAttachments(namespace string) virtualizationv1alpha2.VirtualMachineBlockDeviceAttachmentInterface VirtualMachineIPAddresses(namespace string) virtualizationv1alpha2.VirtualMachineIPAddressInterface VirtualMachineIPAddressLeases() virtualizationv1alpha2.VirtualMachineIPAddressLeaseInterface @@ -84,6 +86,15 @@ func (c client) VirtualMachines(namespace string) virtualizationv1alpha2.Virtual } } +func (c client) VirtualMachinePools(namespace string) virtualizationv1alpha2.VirtualMachinePoolInterface { + return &vmpool{ + VirtualMachinePoolInterface: c.virtClient.VirtualizationV1alpha2().VirtualMachinePools(namespace), + restClient: c.restClient, + namespace: namespace, + resource: "virtualmachinepools", + } +} + func (c client) ClusterVirtualImages() virtualizationv1alpha2.ClusterVirtualImageInterface { return c.virtClient.VirtualizationV1alpha2().ClusterVirtualImages() } @@ -96,6 +107,10 @@ func (c client) VirtualDisks(namespace string) virtualizationv1alpha2.VirtualDis return c.virtClient.VirtualizationV1alpha2().VirtualDisks(namespace) } +func (c client) VirtualDiskSnapshots(namespace string) virtualizationv1alpha2.VirtualDiskSnapshotInterface { + return c.virtClient.VirtualizationV1alpha2().VirtualDiskSnapshots(namespace) +} + func (c client) VirtualMachineBlockDeviceAttachments(namespace string) virtualizationv1alpha2.VirtualMachineBlockDeviceAttachmentInterface { return c.virtClient.VirtualizationV1alpha2().VirtualMachineBlockDeviceAttachments(namespace) } diff --git a/api/client/kubeclient/vmpool.go b/api/client/kubeclient/vmpool.go new file mode 100644 index 0000000000..630f6db574 --- /dev/null +++ b/api/client/kubeclient/vmpool.go @@ -0,0 +1,47 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeclient + +import ( + "context" + "encoding/json" + "fmt" + + "k8s.io/client-go/rest" + + virtualizationv1alpha2 "github.com/deckhouse/virtualization/api/client/generated/clientset/versioned/typed/core/v1alpha2" + subv1alpha2 "github.com/deckhouse/virtualization/api/subresources/v1alpha2" +) + +type vmpool struct { + virtualizationv1alpha2.VirtualMachinePoolInterface + restClient *rest.RESTClient + namespace string + resource string +} + +// ScaleDownWith removes the named pool members through the aggregated-apiserver +// scaleDownWith subresource, which deletes them and decrements the pool's +// replicas atomically, bypassing the anonymous /scale guard. +func (v vmpool) ScaleDownWith(ctx context.Context, name string, opts subv1alpha2.VirtualMachinePoolScaleDownWith) error { + path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "scaledownwith") + body, err := json.Marshal(&opts) + if err != nil { + return err + } + return v.restClient.Post().AbsPath(path).Body(body).SetHeader("Content-Type", "application/json").Do(ctx).Error() +} diff --git a/api/core/v1alpha2/finalizers.go b/api/core/v1alpha2/finalizers.go index bf25c0eaeb..fdb3b1859f 100644 --- a/api/core/v1alpha2/finalizers.go +++ b/api/core/v1alpha2/finalizers.go @@ -23,9 +23,14 @@ const ( // FinalizerKVVMProtection protects KVVMs from deletion. // // Deprecated: FinalizerKVVMProtection is deprecated and should be deleted in the next versions. - FinalizerKVVMProtection = "virtualization.deckhouse.io/kvvm-protection" - FinalizerIPAddressProtection = "virtualization.deckhouse.io/vmip-protection" - FinalizerPodProtection = "virtualization.deckhouse.io/pod-protection" + FinalizerKVVMProtection = "virtualization.deckhouse.io/kvvm-protection" + FinalizerIPAddressProtection = "virtualization.deckhouse.io/vmip-protection" + FinalizerPodProtection = "virtualization.deckhouse.io/pod-protection" + // FinalizerPVCProtection keeps PVCs backing block device volumes of a + // virtual machine while those volumes may be mounted on a node. It is + // released only after the KVVM, KVVMI and virt-launcher pods are gone, + // so the storage backend cannot destroy data under a live mount. + FinalizerPVCProtection = "virtualization.deckhouse.io/pvc-protection" FinalizerVDSnapshotProtection = "virtualization.deckhouse.io/vdsnapshot-protection" FinalizerVMSnapshotProtection = "virtualization.deckhouse.io/vmsnapshot-protection" FinalizerVMOPProtectionByEvacuationController = "virtualization.deckhouse.io/vmop-protection-by-evacuation-controller" diff --git a/api/core/v1alpha2/register.go b/api/core/v1alpha2/register.go index 821755d18f..ec9c569262 100644 --- a/api/core/v1alpha2/register.go +++ b/api/core/v1alpha2/register.go @@ -38,6 +38,9 @@ var VirtualImageGVK = schema.GroupVersionKind{Group: SchemeGroupVersion.Group, V // VirtualDiskGVK is group version kind for VirtualDisk var VirtualDiskGVK = schema.GroupVersionKind{Group: SchemeGroupVersion.Group, Version: SchemeGroupVersion.Version, Kind: VirtualDiskKind} +// VirtualMachinePoolGVK is group version kind for VirtualMachinePool +var VirtualMachinePoolGVK = schema.GroupVersionKind{Group: SchemeGroupVersion.Group, Version: SchemeGroupVersion.Version, Kind: VirtualMachinePoolKind} + // Kind takes an unqualified kind and returns back a Group qualified GroupKind func Kind(kind string) schema.GroupKind { return SchemeGroupVersion.WithKind(kind).GroupKind() @@ -70,6 +73,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VirtualDiskList{}, &VirtualMachine{}, &VirtualMachineList{}, + &VirtualMachinePool{}, + &VirtualMachinePoolList{}, &VirtualMachineBlockDeviceAttachment{}, &VirtualMachineBlockDeviceAttachmentList{}, &VirtualMachineClass{}, diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index d591d0e8bd..5cd8be48d1 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -290,7 +290,7 @@ type VirtualMachineStatus struct { Phase MachinePhase `json:"phase"` // The name of the node on which the VM is currently running. Node string `json:"nodeName"` - // Name of `virtualMachineIPAddressName` holding the ip address of the VirtualMachine. + // Name of the VirtualMachineIPAddress resource with an IP address of the VM. VirtualMachineIPAddress string `json:"virtualMachineIPAddressName"` // IP address of VM. IPAddress string `json:"ipAddress"` diff --git a/api/core/v1alpha2/virtual_machine_pool.go b/api/core/v1alpha2/virtual_machine_pool.go new file mode 100644 index 0000000000..b546f79216 --- /dev/null +++ b/api/core/v1alpha2/virtual_machine_pool.go @@ -0,0 +1,269 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + VirtualMachinePoolKind = "VirtualMachinePool" + VirtualMachinePoolResource = "virtualmachinepools" +) + +// VirtualMachinePool declaratively manages a group of identical virtual machines: +// it keeps the requested number of replicas, scales via the standard `scale` +// subresource, and reuses "heavy" disks across replica generations. +// +// The resource is available only in paid editions (EE/SE+) and is gated behind +// the `VirtualMachinePool` module feature gate. +// +// +kubebuilder:object:root=true +// +kubebuilder:metadata:labels={heritage=deckhouse,module=virtualization} +// +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector +// +kubebuilder:resource:categories={virtualization},scope=Namespaced,shortName={vmpool,vmpools},singular=virtualmachinepool +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".status.replicas",description="Current number of pool members (including Terminating)." +// +kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.readyReplicas",description="Number of members ready to serve." +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time of resource creation." +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type VirtualMachinePool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VirtualMachinePoolSpec `json:"spec"` + Status VirtualMachinePoolStatus `json:"status,omitempty"` +} + +// VirtualMachinePoolList contains a list of VirtualMachinePool resources. +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type VirtualMachinePoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []VirtualMachinePool `json:"items"` +} + +// VirtualMachinePoolSpec is the desired state of a VirtualMachinePool. +// +// The disks a replica gets are declared in two places that must stay in sync: +// virtualDiskTemplates describes each per-replica disk, and the template's +// blockDeviceRefs references those disks (by name, kind VirtualDisk) to set the +// boot order and interleave shared images. The three rules below enforce a +// bijection between them so neither list can carry a dangling entry. +// +// Every virtualDiskTemplates entry must be referenced by a VirtualDisk in the template. +// +kubebuilder:validation:XValidation:rule="has(self.virtualMachineTemplate.spec) && has(self.virtualMachineTemplate.spec.blockDeviceRefs) && self.virtualDiskTemplates.all(t, self.virtualMachineTemplate.spec.blockDeviceRefs.exists(r, r.kind == 'VirtualDisk' && r.name == t.name))",message="each virtualDiskTemplates entry must be referenced by a VirtualDisk entry in virtualMachineTemplate.spec.blockDeviceRefs" +// Every VirtualDisk reference in the template must name a virtualDiskTemplates entry. +// +kubebuilder:validation:XValidation:rule="has(self.virtualMachineTemplate.spec) && has(self.virtualMachineTemplate.spec.blockDeviceRefs) && self.virtualMachineTemplate.spec.blockDeviceRefs.filter(r, r.kind == 'VirtualDisk').all(r, self.virtualDiskTemplates.exists(t, t.name == r.name))",message="each VirtualDisk reference in virtualMachineTemplate.spec.blockDeviceRefs must name a virtualDiskTemplates entry" +// The reference is one-to-one: no virtualDiskTemplates entry is referenced twice. +// +kubebuilder:validation:XValidation:rule="has(self.virtualMachineTemplate.spec) && has(self.virtualMachineTemplate.spec.blockDeviceRefs) && self.virtualMachineTemplate.spec.blockDeviceRefs.filter(r, r.kind == 'VirtualDisk').size() == self.virtualDiskTemplates.size()",message="each virtualDiskTemplates entry must be referenced exactly once (no duplicate VirtualDisk references)" +type VirtualMachinePoolSpec struct { + // Replicas is the desired number of virtual machines in the pool. + // + // The field is written only by its owner — an autoscaler or a human via the + // `scale` subresource, or by the addressed scale-down handler. The controller + // never writes it. Bounds are held by the autoscaler; the hard ceiling is the + // namespace ResourceQuota. + // + // +kubebuilder:validation:Minimum=0 + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // ScaleDownPolicy chooses how a replica is picked when the pool is scaled down + // anonymously through the `scale` subresource. It is required and has no + // default, forcing a conscious choice between "any replica may be killed" and + // "only addressed removal is allowed". + // + // - `NewestFirst` — anonymous scale-down is allowed; the youngest replicas + // (least accumulated state) are removed first. + // - `OldestFirst` — anonymous scale-down is allowed; the oldest replicas are + // removed first (faster rotation). + // - `Explicit` — anonymous scale-down through `scale` is rejected by a + // webhook; replicas can be removed only by address. For "busy" workloads + // such as CI runners and VDI. + // + // +kubebuilder:validation:Enum=NewestFirst;OldestFirst;Explicit + ScaleDownPolicy ScaleDownPolicy `json:"scaleDownPolicy"` + + // VirtualMachineTemplate is the template every replica is stamped from. Its + // `spec` is an ordinary VirtualMachineSpec, so a replica is no different from a + // manually created virtual machine. + VirtualMachineTemplate VirtualMachineTemplateSpec `json:"virtualMachineTemplate"` + + // VirtualDiskTemplates describes each per-replica disk (reclaim policy, size, + // data source). Names are unique within the pool (list-map key) and every + // template must be referenced by a VirtualDisk entry in + // virtualMachineTemplate.spec.blockDeviceRefs, which sets the boot order (see + // the bijection rule on the spec). A disk with reclaim Delete belongs to its + // VirtualMachine and is removed with it; a disk with reclaim Retain belongs to + // the pool, outlives the replica and is reused on a later scale-up. + // + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + // +listType=map + // +listMapKey=name + VirtualDiskTemplates []VirtualDiskTemplateSpec `json:"virtualDiskTemplates"` +} + +// VirtualDiskTemplateSpec describes a per-replica disk. +type VirtualDiskTemplateSpec struct { + // Name identifies the disk template within the pool. It is a DNS-1123 label + // (no dots), because it is embedded into VirtualDisk names. + // + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // Reclaim controls what happens to the disk when its replica is removed. + // + // +optional + Reclaim VirtualDiskReclaim `json:"reclaim,omitempty"` + + // Spec is the desired state of the disk (an ordinary VirtualDiskSpec). + Spec VirtualDiskSpec `json:"spec"` +} + +// VirtualDiskReclaimPolicy selects the fate of a per-replica disk on scale-down. +type VirtualDiskReclaimPolicy string + +const ( + // VirtualDiskReclaimDelete removes the disk together with its replica (owner + // is the VirtualMachine). This is the default. + VirtualDiskReclaimDelete VirtualDiskReclaimPolicy = "Delete" + // VirtualDiskReclaimRetain keeps the disk (owner is the pool); it is reused on + // the next scale-up. + VirtualDiskReclaimRetain VirtualDiskReclaimPolicy = "Retain" +) + +// VirtualDiskReclaim is the reclaim policy and warm-buffer settings of a disk +// template. +// +// +kubebuilder:validation:XValidation:rule="self.onScaleDown == 'Retain' || (self.keep == 0 && !has(self.ttl))",message="keep and ttl are only valid with onScaleDown: Retain" +// +kubebuilder:validation:XValidation:rule="self.keep == 0 || has(self.ttl)",message="keep requires ttl; without ttl free disks are never garbage-collected, so keep would have no effect" +type VirtualDiskReclaim struct { + // OnScaleDown is Delete (default) or Retain. + // + // +kubebuilder:validation:Enum=Delete;Retain + // +kubebuilder:default=Delete + // +optional + OnScaleDown VirtualDiskReclaimPolicy `json:"onScaleDown,omitempty"` + + // Keep is the number of free (Retain) disks always kept warm for fast + // scale-up; these are immune to the ttl. Only meaningful with Retain. + // + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + // +optional + Keep int32 `json:"keep,omitempty"` + + // TTL is how long a free disk lives beyond the warm buffer before it is + // garbage-collected. Only meaningful with Retain. + // + // +optional + TTL *metav1.Duration `json:"ttl,omitempty"` +} + +// ScaleDownPolicy selects which replica is removed on anonymous scale-down. +type ScaleDownPolicy string + +const ( + ScaleDownPolicyNewestFirst ScaleDownPolicy = "NewestFirst" + ScaleDownPolicyOldestFirst ScaleDownPolicy = "OldestFirst" + ScaleDownPolicyExplicit ScaleDownPolicy = "Explicit" +) + +// VirtualMachineTemplateSpec describes the metadata and spec a pool replica is +// created with. +type VirtualMachineTemplateSpec struct { + // Metadata applied to every replica. Arbitrary user labels and annotations are + // allowed; the controller adds its managed pool labels on top. A curated + // struct (not the full ObjectMeta) so the CRD schema exposes labels and + // annotations instead of an opaque object. + // + // +optional + Metadata VirtualMachineTemplateMetadata `json:"metadata,omitempty"` + + // Spec of the virtual machine that backs each replica. + // + // +optional + Spec VirtualMachineSpec `json:"spec,omitempty"` +} + +// VirtualMachineTemplateMetadata is the subset of object metadata a pool +// stamps onto each replica. +type VirtualMachineTemplateMetadata struct { + // +optional + Labels map[string]string `json:"labels,omitempty"` + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// VirtualMachinePoolStatus is the observed state of a VirtualMachinePool. +type VirtualMachinePoolStatus struct { + // ObservedGeneration is the generation of the spec the controller has processed. + // + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Replicas is the number of existing members, including those in Terminating: + // such a machine still occupies resources, so it is real capacity, not a phantom. + // + // +optional + Replicas int32 `json:"replicas,omitempty"` + + // ReadyReplicas is the number of members ready to serve (Terminating excluded). + // + // +optional + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + + // DesiredTemplateHash is the hash of the current virtualMachineTemplate — the + // revision the controller is converging replicas to (cf. updateRevision on a + // StatefulSet). + // + // +optional + DesiredTemplateHash string `json:"desiredTemplateHash,omitempty"` + + // UpdatedReplicas is the number of replicas effectively on DesiredTemplateHash + // (fully synced). + // + // +optional + UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` + + // RestartPendingReplicas is the number of replicas patched to the new template + // whose disruptive part still awaits a restart. + // + // +optional + RestartPendingReplicas int32 `json:"restartPendingReplicas,omitempty"` + + // Selector is the label selector the controller publishes for the `scale` + // subresource; HPA/KEDA read it themselves. + // + // +optional + Selector string `json:"selector,omitempty"` + + // Conditions describe the current state of the pool. + // + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 2320fd5da9..56c4094ed0 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -92,10 +92,11 @@ func (r IpAddressReadyReason) String() string { } const ( - ReasonIPAddressReady IpAddressReadyReason = "VirtualMachineIPAddressReady" - ReasonIPAddressNotReady IpAddressReadyReason = "VirtualMachineIPAddressNotReady" - ReasonIPAddressNotAssigned IpAddressReadyReason = "VirtualMachineIPAddressNotAssigned" - ReasonIPAddressNotAvailable IpAddressReadyReason = "VirtualMachineIPAddressNotAvailable" + ReasonIPAddressReady IpAddressReadyReason = "VirtualMachineIPAddressReady" + ReasonIPAddressNotReady IpAddressReadyReason = "VirtualMachineIPAddressNotReady" + ReasonIPAddressNotAssigned IpAddressReadyReason = "VirtualMachineIPAddressNotAssigned" + ReasonIPAddressNotAvailable IpAddressReadyReason = "VirtualMachineIPAddressNotAvailable" + ReasonIPAddressNonAllocatable IpAddressReadyReason = "VirtualMachineIPAddressNonAllocatable" ) type MacAddressReadyReason string diff --git a/api/core/v1alpha2/vmopcondition/condition.go b/api/core/v1alpha2/vmopcondition/condition.go index 2ab75e9c65..542d2a7099 100644 --- a/api/core/v1alpha2/vmopcondition/condition.go +++ b/api/core/v1alpha2/vmopcondition/condition.go @@ -139,6 +139,9 @@ const ( // ReasonOperationCompleted is a ReasonCompleted indicating that operation is completed. ReasonOperationCompleted ReasonCompleted = "OperationCompleted" + + // ReasonMigrationNetworkUnavailable indicates that liveMigration.network.systemNetwork is configured in the ModuleConfig but the source node has no resolved migration interface. + ReasonMigrationNetworkUnavailable ReasonCompleted = "MigrationNetworkUnavailable" ) // ReasonCompleted represents specific reasons for the 'SignalSent' condition type. diff --git a/api/core/v1alpha2/vmpoolcondition/condition.go b/api/core/v1alpha2/vmpoolcondition/condition.go new file mode 100644 index 0000000000..ded30ba7a5 --- /dev/null +++ b/api/core/v1alpha2/vmpoolcondition/condition.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmpoolcondition + +// Type is a type of VirtualMachinePool condition. +type Type string + +func (t Type) String() string { + return string(t) +} + +const ( + // TypeAvailable indicates whether the pool has enough ready replicas. + TypeAvailable Type = "Available" + // TypeProgressing indicates that a self-converging rollout is in progress + // (scaling, creation, migration). + TypeProgressing Type = "Progressing" + // TypeSynced indicates whether every live replica is effectively on the + // current virtualMachineTemplate. + TypeSynced Type = "Synced" +) + +// AvailableReason is a reason for the Available condition. +type AvailableReason string + +func (r AvailableReason) String() string { + return string(r) +} + +const ( + // The pool has no minReplicas/maxUnavailable, so Available means every desired + // replica is ready — hence "all", not "minimum". + ReasonAllReplicasReady AvailableReason = "AllReplicasReady" + ReasonInsufficientReadyReplicas AvailableReason = "InsufficientReadyReplicas" +) + +// ProgressingReason is a reason for the Progressing condition. +type ProgressingReason string + +func (r ProgressingReason) String() string { + return string(r) +} + +const ( + ReasonPoolStable ProgressingReason = "PoolStable" + // ReplicasProgressing covers any convergence of the replica count — scaling + // as well as replacing a replica that disappeared — not only scaling. + ReasonReplicasProgressing ProgressingReason = "ReplicasProgressing" +) + +// SyncedReason is a reason for the Synced condition. +type SyncedReason string + +func (r SyncedReason) String() string { + return string(r) +} + +const ( + ReasonPoolSynced SyncedReason = "PoolSynced" + ReasonRolloutInProgress SyncedReason = "RolloutInProgress" + ReasonRestartPendingApproval SyncedReason = "RestartPendingApproval" +) diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go index 58d00cb82b..8ee5d37f0a 100644 --- a/api/core/v1alpha2/zz_generated.deepcopy.go +++ b/api/core/v1alpha2/zz_generated.deepcopy.go @@ -1472,6 +1472,27 @@ func (in *VirtualDiskPersistentVolumeClaim) DeepCopy() *VirtualDiskPersistentVol return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualDiskReclaim) DeepCopyInto(out *VirtualDiskReclaim) { + *out = *in + if in.TTL != nil { + in, out := &in.TTL, &out.TTL + *out = new(v1.Duration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualDiskReclaim. +func (in *VirtualDiskReclaim) DeepCopy() *VirtualDiskReclaim { + if in == nil { + return nil + } + out := new(VirtualDiskReclaim) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualDiskSnapshot) DeepCopyInto(out *VirtualDiskSnapshot) { *out = *in @@ -1698,6 +1719,24 @@ func (in *VirtualDiskStatus) DeepCopy() *VirtualDiskStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualDiskTemplateSpec) DeepCopyInto(out *VirtualDiskTemplateSpec) { + *out = *in + in.Reclaim.DeepCopyInto(&out.Reclaim) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualDiskTemplateSpec. +func (in *VirtualDiskTemplateSpec) DeepCopy() *VirtualDiskTemplateSpec { + if in == nil { + return nil + } + out := new(VirtualDiskTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualImage) DeepCopyInto(out *VirtualImage) { *out = *in @@ -3069,6 +3108,119 @@ func (in *VirtualMachinePod) DeepCopy() *VirtualMachinePod { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePool) DeepCopyInto(out *VirtualMachinePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePool. +func (in *VirtualMachinePool) DeepCopy() *VirtualMachinePool { + if in == nil { + return nil + } + out := new(VirtualMachinePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachinePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePoolList) DeepCopyInto(out *VirtualMachinePoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VirtualMachinePool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePoolList. +func (in *VirtualMachinePoolList) DeepCopy() *VirtualMachinePoolList { + if in == nil { + return nil + } + out := new(VirtualMachinePoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachinePoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePoolSpec) DeepCopyInto(out *VirtualMachinePoolSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.VirtualMachineTemplate.DeepCopyInto(&out.VirtualMachineTemplate) + if in.VirtualDiskTemplates != nil { + in, out := &in.VirtualDiskTemplates, &out.VirtualDiskTemplates + *out = make([]VirtualDiskTemplateSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePoolSpec. +func (in *VirtualMachinePoolSpec) DeepCopy() *VirtualMachinePoolSpec { + if in == nil { + return nil + } + out := new(VirtualMachinePoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePoolStatus) DeepCopyInto(out *VirtualMachinePoolStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePoolStatus. +func (in *VirtualMachinePoolStatus) DeepCopy() *VirtualMachinePoolStatus { + if in == nil { + return nil + } + out := new(VirtualMachinePoolStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineSnapshot) DeepCopyInto(out *VirtualMachineSnapshot) { *out = *in @@ -3472,6 +3624,54 @@ func (in *VirtualMachineStatus) DeepCopy() *VirtualMachineStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachineTemplateMetadata) DeepCopyInto(out *VirtualMachineTemplateMetadata) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineTemplateMetadata. +func (in *VirtualMachineTemplateMetadata) DeepCopy() *VirtualMachineTemplateMetadata { + if in == nil { + return nil + } + out := new(VirtualMachineTemplateMetadata) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachineTemplateSpec) DeepCopyInto(out *VirtualMachineTemplateSpec) { + *out = *in + in.Metadata.DeepCopyInto(&out.Metadata) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineTemplateSpec. +func (in *VirtualMachineTemplateSpec) DeepCopy() *VirtualMachineTemplateSpec { + if in == nil { + return nil + } + out := new(VirtualMachineTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeightedVirtualMachineAndPodAffinityTerm) DeepCopyInto(out *WeightedVirtualMachineAndPodAffinityTerm) { *out = *in diff --git a/api/scripts/update-codegen.sh b/api/scripts/update-codegen.sh index b07d310fc5..7139c7d674 100755 --- a/api/scripts/update-codegen.sh +++ b/api/scripts/update-codegen.sh @@ -41,7 +41,8 @@ function source::settings { "VirtualImage" "ClusterVirtualImage" "NodeUSBDevice" - "USBDevice") + "USBDevice" + "VirtualMachinePool") # shellcheck source=/dev/null source "${CODEGEN_PKG}/kube_codegen.sh" diff --git a/api/subresources/register.go b/api/subresources/register.go index 872ad19a8f..e839a66c17 100644 --- a/api/subresources/register.go +++ b/api/subresources/register.go @@ -59,6 +59,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VirtualMachineCancelEvacuation{}, &VirtualMachineAddResourceClaim{}, &VirtualMachineRemoveResourceClaim{}, + &VirtualMachinePool{}, + &VirtualMachinePoolScaleDownWith{}, ) return nil } diff --git a/api/subresources/types.go b/api/subresources/types.go index 90ce98abaf..7b416efd3a 100644 --- a/api/subresources/types.go +++ b/api/subresources/types.go @@ -109,3 +109,19 @@ type VirtualMachineRemoveResourceClaim struct { Name string DryRun []string } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type VirtualMachinePool struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type VirtualMachinePoolScaleDownWith struct { + metav1.TypeMeta + + Targets []string + DryRun []string +} diff --git a/api/subresources/v1alpha2/register.go b/api/subresources/v1alpha2/register.go index d978d3f4f3..39da230e19 100644 --- a/api/subresources/v1alpha2/register.go +++ b/api/subresources/v1alpha2/register.go @@ -61,6 +61,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VirtualMachineCancelEvacuation{}, &VirtualMachineAddResourceClaim{}, &VirtualMachineRemoveResourceClaim{}, + &VirtualMachinePool{}, + &VirtualMachinePoolScaleDownWith{}, ) return nil } diff --git a/api/subresources/v1alpha2/types.go b/api/subresources/v1alpha2/types.go index 2bfcbcedbc..7339f7a5c3 100644 --- a/api/subresources/v1alpha2/types.go +++ b/api/subresources/v1alpha2/types.go @@ -117,3 +117,22 @@ type VirtualMachineRemoveResourceClaim struct { Name string `json:"name"` DryRun []string `json:"dryRun,omitempty"` } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type VirtualMachinePool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:conversion-gen:explicit-from=net/url.Values + +type VirtualMachinePoolScaleDownWith struct { + metav1.TypeMeta `json:",inline"` + + // Targets are the names of the pool member VirtualMachines to remove. + Targets []string `json:"targets"` + + DryRun []string `json:"dryRun,omitempty"` +} diff --git a/api/subresources/v1alpha2/zz_generated.conversion.go b/api/subresources/v1alpha2/zz_generated.conversion.go index 7955b3f836..7f21f8d085 100644 --- a/api/subresources/v1alpha2/zz_generated.conversion.go +++ b/api/subresources/v1alpha2/zz_generated.conversion.go @@ -98,6 +98,26 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*VirtualMachinePool)(nil), (*subresources.VirtualMachinePool)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_VirtualMachinePool_To_subresources_VirtualMachinePool(a.(*VirtualMachinePool), b.(*subresources.VirtualMachinePool), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*subresources.VirtualMachinePool)(nil), (*VirtualMachinePool)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_subresources_VirtualMachinePool_To_v1alpha2_VirtualMachinePool(a.(*subresources.VirtualMachinePool), b.(*VirtualMachinePool), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*VirtualMachinePoolScaleDownWith)(nil), (*subresources.VirtualMachinePoolScaleDownWith)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_VirtualMachinePoolScaleDownWith_To_subresources_VirtualMachinePoolScaleDownWith(a.(*VirtualMachinePoolScaleDownWith), b.(*subresources.VirtualMachinePoolScaleDownWith), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*subresources.VirtualMachinePoolScaleDownWith)(nil), (*VirtualMachinePoolScaleDownWith)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_subresources_VirtualMachinePoolScaleDownWith_To_v1alpha2_VirtualMachinePoolScaleDownWith(a.(*subresources.VirtualMachinePoolScaleDownWith), b.(*VirtualMachinePoolScaleDownWith), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*VirtualMachinePortForward)(nil), (*subresources.VirtualMachinePortForward)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_VirtualMachinePortForward_To_subresources_VirtualMachinePortForward(a.(*VirtualMachinePortForward), b.(*subresources.VirtualMachinePortForward), scope) }); err != nil { @@ -173,6 +193,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*url.Values)(nil), (*VirtualMachinePoolScaleDownWith)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_url_Values_To_v1alpha2_VirtualMachinePoolScaleDownWith(a.(*url.Values), b.(*VirtualMachinePoolScaleDownWith), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*url.Values)(nil), (*VirtualMachinePortForward)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_url_Values_To_v1alpha2_VirtualMachinePortForward(a.(*url.Values), b.(*VirtualMachinePortForward), scope) }); err != nil { @@ -468,6 +493,69 @@ func Convert_url_Values_To_v1alpha2_VirtualMachineFreeze(in *url.Values, out *Vi return autoConvert_url_Values_To_v1alpha2_VirtualMachineFreeze(in, out, s) } +func autoConvert_v1alpha2_VirtualMachinePool_To_subresources_VirtualMachinePool(in *VirtualMachinePool, out *subresources.VirtualMachinePool, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + return nil +} + +// Convert_v1alpha2_VirtualMachinePool_To_subresources_VirtualMachinePool is an autogenerated conversion function. +func Convert_v1alpha2_VirtualMachinePool_To_subresources_VirtualMachinePool(in *VirtualMachinePool, out *subresources.VirtualMachinePool, s conversion.Scope) error { + return autoConvert_v1alpha2_VirtualMachinePool_To_subresources_VirtualMachinePool(in, out, s) +} + +func autoConvert_subresources_VirtualMachinePool_To_v1alpha2_VirtualMachinePool(in *subresources.VirtualMachinePool, out *VirtualMachinePool, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + return nil +} + +// Convert_subresources_VirtualMachinePool_To_v1alpha2_VirtualMachinePool is an autogenerated conversion function. +func Convert_subresources_VirtualMachinePool_To_v1alpha2_VirtualMachinePool(in *subresources.VirtualMachinePool, out *VirtualMachinePool, s conversion.Scope) error { + return autoConvert_subresources_VirtualMachinePool_To_v1alpha2_VirtualMachinePool(in, out, s) +} + +func autoConvert_v1alpha2_VirtualMachinePoolScaleDownWith_To_subresources_VirtualMachinePoolScaleDownWith(in *VirtualMachinePoolScaleDownWith, out *subresources.VirtualMachinePoolScaleDownWith, s conversion.Scope) error { + out.Targets = *(*[]string)(unsafe.Pointer(&in.Targets)) + out.DryRun = *(*[]string)(unsafe.Pointer(&in.DryRun)) + return nil +} + +// Convert_v1alpha2_VirtualMachinePoolScaleDownWith_To_subresources_VirtualMachinePoolScaleDownWith is an autogenerated conversion function. +func Convert_v1alpha2_VirtualMachinePoolScaleDownWith_To_subresources_VirtualMachinePoolScaleDownWith(in *VirtualMachinePoolScaleDownWith, out *subresources.VirtualMachinePoolScaleDownWith, s conversion.Scope) error { + return autoConvert_v1alpha2_VirtualMachinePoolScaleDownWith_To_subresources_VirtualMachinePoolScaleDownWith(in, out, s) +} + +func autoConvert_subresources_VirtualMachinePoolScaleDownWith_To_v1alpha2_VirtualMachinePoolScaleDownWith(in *subresources.VirtualMachinePoolScaleDownWith, out *VirtualMachinePoolScaleDownWith, s conversion.Scope) error { + out.Targets = *(*[]string)(unsafe.Pointer(&in.Targets)) + out.DryRun = *(*[]string)(unsafe.Pointer(&in.DryRun)) + return nil +} + +// Convert_subresources_VirtualMachinePoolScaleDownWith_To_v1alpha2_VirtualMachinePoolScaleDownWith is an autogenerated conversion function. +func Convert_subresources_VirtualMachinePoolScaleDownWith_To_v1alpha2_VirtualMachinePoolScaleDownWith(in *subresources.VirtualMachinePoolScaleDownWith, out *VirtualMachinePoolScaleDownWith, s conversion.Scope) error { + return autoConvert_subresources_VirtualMachinePoolScaleDownWith_To_v1alpha2_VirtualMachinePoolScaleDownWith(in, out, s) +} + +func autoConvert_url_Values_To_v1alpha2_VirtualMachinePoolScaleDownWith(in *url.Values, out *VirtualMachinePoolScaleDownWith, s conversion.Scope) error { + // WARNING: Field TypeMeta does not have json tag, skipping. + + if values, ok := map[string][]string(*in)["targets"]; ok && len(values) > 0 { + out.Targets = *(*[]string)(unsafe.Pointer(&values)) + } else { + out.Targets = nil + } + if values, ok := map[string][]string(*in)["dryRun"]; ok && len(values) > 0 { + out.DryRun = *(*[]string)(unsafe.Pointer(&values)) + } else { + out.DryRun = nil + } + return nil +} + +// Convert_url_Values_To_v1alpha2_VirtualMachinePoolScaleDownWith is an autogenerated conversion function. +func Convert_url_Values_To_v1alpha2_VirtualMachinePoolScaleDownWith(in *url.Values, out *VirtualMachinePoolScaleDownWith, s conversion.Scope) error { + return autoConvert_url_Values_To_v1alpha2_VirtualMachinePoolScaleDownWith(in, out, s) +} + func autoConvert_v1alpha2_VirtualMachinePortForward_To_subresources_VirtualMachinePortForward(in *VirtualMachinePortForward, out *subresources.VirtualMachinePortForward, s conversion.Scope) error { out.Protocol = in.Protocol out.Port = in.Port diff --git a/api/subresources/v1alpha2/zz_generated.deepcopy.go b/api/subresources/v1alpha2/zz_generated.deepcopy.go index 6554e97509..e83a39f0f3 100644 --- a/api/subresources/v1alpha2/zz_generated.deepcopy.go +++ b/api/subresources/v1alpha2/zz_generated.deepcopy.go @@ -192,6 +192,67 @@ func (in *VirtualMachineFreeze) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePool) DeepCopyInto(out *VirtualMachinePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePool. +func (in *VirtualMachinePool) DeepCopy() *VirtualMachinePool { + if in == nil { + return nil + } + out := new(VirtualMachinePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachinePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePoolScaleDownWith) DeepCopyInto(out *VirtualMachinePoolScaleDownWith) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DryRun != nil { + in, out := &in.DryRun, &out.DryRun + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePoolScaleDownWith. +func (in *VirtualMachinePoolScaleDownWith) DeepCopy() *VirtualMachinePoolScaleDownWith { + if in == nil { + return nil + } + out := new(VirtualMachinePoolScaleDownWith) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachinePoolScaleDownWith) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachinePortForward) DeepCopyInto(out *VirtualMachinePortForward) { *out = *in diff --git a/api/subresources/zz_generated.deepcopy.go b/api/subresources/zz_generated.deepcopy.go index 8268bde57f..dc9fa4f7a8 100644 --- a/api/subresources/zz_generated.deepcopy.go +++ b/api/subresources/zz_generated.deepcopy.go @@ -192,6 +192,67 @@ func (in *VirtualMachineFreeze) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePool) DeepCopyInto(out *VirtualMachinePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePool. +func (in *VirtualMachinePool) DeepCopy() *VirtualMachinePool { + if in == nil { + return nil + } + out := new(VirtualMachinePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachinePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachinePoolScaleDownWith) DeepCopyInto(out *VirtualMachinePoolScaleDownWith) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DryRun != nil { + in, out := &in.DryRun, &out.DryRun + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachinePoolScaleDownWith. +func (in *VirtualMachinePoolScaleDownWith) DeepCopy() *VirtualMachinePoolScaleDownWith { + if in == nil { + return nil + } + out := new(VirtualMachinePoolScaleDownWith) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachinePoolScaleDownWith) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachinePortForward) DeepCopyInto(out *VirtualMachinePortForward) { *out = *in diff --git a/build/components/versions.yml b/build/components/versions.yml index b466a63b68..b9033ee69c 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,8 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.53 - 3p-containerized-data-importer: v1.60.3-v12n.21 + 3p-kubevirt: v1.6.2-v12n.57 distribution: 3.1.1 package: acl: v2.3.1 diff --git a/crds/doc-ru-virtualmachinepools.yaml b/crds/doc-ru-virtualmachinepools.yaml new file mode 100644 index 0000000000..ff9a16233f --- /dev/null +++ b/crds/doc-ru-virtualmachinepools.yaml @@ -0,0 +1,1140 @@ +spec: + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: | + VirtualMachinePool декларативно управляет группой одинаковых виртуальных машин: поддерживает заданное число реплик, масштабируется через стандартный субресурс `scale` и переиспользует «тяжёлые» диски между поколениями реплик. + + Ресурс доступен только в платных редакциях (EE/SE+) и требует включения feature gate `VirtualMachinePool`. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + spec: + description: | + Желаемое состояние пула виртуальных машин. + properties: + replicas: + description: | + Желаемое число виртуальных машин в пуле. + + Поле пишет только его владелец — автоскейлер или человек через субресурс `scale`, либо обработчик адресного сжатия. Контроллер его не изменяет. Границы удерживает автоскейлер; жёсткий потолок — ResourceQuota неймспейса. + scaleDownPolicy: + description: | + Определяет, какая реплика удаляется при безадресном сжатии пула через субресурс `scale`. Поле обязательно и не имеет значения по умолчанию — выбор делается осознанно. + + * `NewestFirst` — безадресное сжатие разрешено; первыми удаляются самые молодые реплики (с наименьшим накопленным состоянием). + * `OldestFirst` — безадресное сжатие разрешено; первыми удаляются самые старые реплики (быстрая ротация). + * `Explicit` — безадресное сжатие через `scale` отклоняется validating-вебхуком; реплики убираются только по имени. Для «занятых» нагрузок, например раннеров CI и VDI. + virtualDiskTemplates: + description: | + Описание дисков на каждую реплику и, по их порядку, блочных устройств реплики — первый шаблон является загрузочным устройством. Это единственный источник дисков реплики: в `virtualMachineTemplate` поля `blockDeviceRefs` нет, контроллер строит его сам из этих шаблонов. Диск с политикой возврата `Delete` принадлежит своей виртуальной машине и удаляется вместе с ней; диск с политикой `Retain` принадлежит пулу, переживает реплику и переиспользуется при последующем масштабировании вверх. + items: + description: | + Описание диска на одну реплику. + properties: + name: + description: | + Имя шаблона диска в пуле. Лейбл DNS-1123 (без точек), так как встраивается в имена VirtualDisk. + reclaim: + description: | + Управляет судьбой диска при удалении его реплики. + properties: + keep: + description: | + Сколько свободных (`Retain`) дисков всегда держать в резерве для быстрого масштабирования вверх; на них не распространяется `ttl`. Имеет смысл только с `Retain`. + onScaleDown: + description: | + `Delete` (по умолчанию) — диск удаляется вместе с репликой; `Retain` — диск сохраняется и переиспользуется. + ttl: + description: | + Время жизни свободного диска сверх резервного буфера, после которого он удаляется сборщиком мусора. Имеет смысл только с `Retain`. + spec: + description: | + Спецификация диска (см. [VirtualDisk](cr.html#virtualdisk)). + properties: + dataSource: + properties: + containerImage: + description: | + Использование образа, который хранится во внешнем реестре контейнеров. + Поддерживаются только реестры контейнеров с включённым протоколом TLS. + Чтобы предоставить собственную цепочку центров сертификации, используйте поле `caBundle`. + properties: + caBundle: + description: | + Цепочка сертификатов в формате Base64 для проверки подключения к реестру контейнеров. + image: + description: | + Путь к образу в реестре контейнеров. + imagePullSecret: + properties: + name: + description: | + Имя секрета, содержащего учётные данные для подключения к реестру контейнеров. + Секрет должен находиться в том же неймспейсе. + http: + description: | + Создание диска из файла, размещённого на указанном URL-адресе. Поддерживаемые схемы: + + * HTTP; + * HTTPS. + + Для схемы HTTPS есть возможность пропустить проверку TLS. + properties: + caBundle: + description: | + Цепочка сертификатов в формате Base64 для проверки TLS-сертификата сервера, на котором размещается файл. + checksum: + description: | + Контрольная сумма файла для проверки целостности и отсутствия изменений в загруженных данных. + Файл должен соответствовать всем указанным контрольным суммам. + url: + description: | + URL-адрес, указывающий на файл для создания образа. Допустимые форматы файла: + + * qcow2; + * vmdk; + * vdi; + * iso; + * raw. + + Файл может быть сжат в архив в одном из следующих форматов: + + * gz; + * xz. + objectRef: + description: | + Использование существующего ресурса VirtualImage, ClusterVirtualImage или VirtualDiskSnapshot для создания диска. + properties: + kind: + description: | + Ссылка на существующий ресурс VirtualImage, ClusterVirtualImage или VirtualDiskSnapshot. + name: + description: | + Имя существующего ресурса VirtualImage, ClusterVirtualImage или VirtualDiskSnapshot. + type: + description: | + Доступные типы источников для создания диска: + + * `HTTP` — из файла, опубликованного на HTTP/HTTPS-сервере; + * `ContainerImage` — из образа в реестре контейнеров; + * `ObjectRef` — из существующего ресурса; + * `Upload` — загрузить образ диска вручную. + persistentVolumeClaim: + description: | + Настройки для создания PersistentVolumeClaim (PVC) для хранения диска. + properties: + size: + description: | + Желаемый размер PVC для хранения диска. Если диск создаётся из образа, размер PVC должен быть не меньше размера исходного образа в распакованном состоянии. + + Данный параметр можно опустить, если заполнен блок `.spec.dataSource`. В этом случае контроллер определит размер диска автоматически на основе размера распакованного образа из источника, указанного в `.spec.dataSource`. + storageClassName: + description: | + Имя StorageClass, необходимого для PVC. Подробнее об использовании StorageClass для PVC: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1. + + При создании дисков можно явно указать необходимый StorageClass. Если этого не сделать, будет использован StorageClass, доступный по умолчанию. + + Особенности диска и поведение виртуальной машины зависят от выбранного StorageClass. + + Параметр `VolumeBindingMode` в StorageClass влияет на процесс создания дисков. Допустимые значения: + + - `Immediate` — диск будет создан и доступен для использования сразу после создания; + - `WaitForFirstConsumer` — диск будет создан при первом использовании на узле, где будет запущена виртуальная машина. + + StorageClass поддерживает различные настройки хранения: + + - создание блочного устройства (`Block`) или файловой системы (`FileSystem`); + - множественный доступ (`ReadWriteMany`) или единичный доступ (`ReadWriteOnce`). `ReadWriteMany`-диски поддерживают множественный доступ, что позволяет выполнять «живую» миграцию виртуальных машин. В отличие от них, `ReadWriteOnce`-диски, ограниченные доступом только с одного узла, не могут обеспечить такую возможность. + + Для известных типов хранилищ Deckhouse самостоятельно определит наиболее эффективные настройки при создании дисков (в порядке убывания приоритетов): + + 1. `Block` + `ReadWriteMany`; + 2. `FileSystem` + `ReadWriteMany`; + 3. `Block` + `ReadWriteOnce`; + 4. `FileSystem` + `ReadWriteOnce`. + virtualMachineTemplate: + description: | + Шаблон, из которого создаётся каждая реплика. Поле `spec` совпадает со спецификацией VirtualMachine, поэтому реплика ничем не отличается от вручную созданной виртуальной машины. + properties: + metadata: + description: | + Метаданные, применяемые к каждой реплике. Произвольные пользовательские лейблы и аннотации разрешены; управляемые лейблы пула контроллер добавляет поверх них. + spec: + description: | + Спецификация виртуальной машины, из которой создаётся каждая реплика (обычный VirtualMachineSpec, см. [VirtualMachine](cr.html#virtualmachine)). + properties: + affinity: + description: | + [По аналогии](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity), как и в параметре подов `spec.affinity` в Kubernetes. + + Настройка `affinity` полностью аналогична приведённой выше документации, за исключением названий некоторых параметров. Используются следующие аналоги: + + * `podAffinity` -> `virtualMachineAndPodAffinity`; + * `podAffinityTerm` -> `virtualMachineAndPodAffinityTerm`. + properties: + nodeAffinity: + description: Описывает affinity-правила узлов для ВМ. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: + "Планировщик предпочтёт размещать поды + на узлах, удовлетворяющих affinity-выражениям из + этого поля, но может выбрать и узел, нарушающий + одно или несколько выражений. Наиболее предпочтителен + узел с наибольшей суммой весов: для каждого узла, + удовлетворяющего всем требованиям планирования (запрос + ресурсов, обязательные affinity-выражения и т. п.), + сумма вычисляется перебором элементов этого поля + с прибавлением «веса», если узел соответствует соответствующим + matchExpressions; узлы с наибольшей суммой наиболее + предпочтительны." + items: + description: + Пустой предпочитаемый терм планирования + соответствует всем объектам с неявным весом 0 + (то есть ничего не делает). Терм со значением + null не соответствует ни одному объекту (также + ничего не делает). + properties: + preference: + description: + Терм селектора узлов, связанный + с соответствующим весом. + properties: + matchExpressions: + description: + Список требований селектора + узлов по меткам узла. + items: + description: + Требование селектора узлов + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + Ключ метки, к которому + применяется селектор. + operator: + description: + "Задаёт отношение ключа + к набору значений. Допустимые операторы: + In, NotIn, Exists, DoesNotExist, + Gt и Lt." + values: + description: + Массив строковых значений. + Для операторов In и NotIn массив + должен быть непустым; для Exists + и DoesNotExist — пустым; для Gt + и Lt — содержать ровно один элемент, + интерпретируемый как целое число. + При strategic merge patch массив + заменяется целиком. + matchFields: + description: + Список требований селектора + узлов по полям узла. + items: + description: + Требование селектора узлов + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + Ключ метки, к которому + применяется селектор. + operator: + description: + "Задаёт отношение ключа + к набору значений. Допустимые операторы: + In, NotIn, Exists, DoesNotExist, + Gt и Lt." + values: + description: + Массив строковых значений. + Для операторов In и NotIn массив + должен быть непустым; для Exists + и DoesNotExist — пустым; для Gt + и Lt — содержать ровно один элемент, + интерпретируемый как целое число. + При strategic merge patch массив + заменяется целиком. + weight: + description: + Вес, связанный с соответствием + данному nodeSelectorTerm, в диапазоне 1–100. + requiredDuringSchedulingIgnoredDuringExecution: + description: + Если заданные этим полем требования affinity + не выполнены на момент планирования, под не будет + размещён на узле. Если они перестают выполняться + во время работы пода (например, из-за обновления), + система может как попытаться в итоге вытеснить под + с узла, так и не делать этого. + properties: + nodeSelectorTerms: + description: + Обязательное поле. Список термов + селектора узлов. Термы объединяются по ИЛИ. + items: + description: + Пустой терм селектора узлов или + терм со значением null не соответствует ни + одному объекту. Требования внутри терма объединяются + по И. Тип TopologySelectorTerm реализует подмножество + NodeSelectorTerm. + properties: + matchExpressions: + description: + Список требований селектора + узлов по меткам узла. + items: + description: + Требование селектора узлов + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + Ключ метки, к которому + применяется селектор. + operator: + description: + "Задаёт отношение ключа + к набору значений. Допустимые операторы: + In, NotIn, Exists, DoesNotExist, + Gt и Lt." + values: + description: + Массив строковых значений. + Для операторов In и NotIn массив + должен быть непустым; для Exists + и DoesNotExist — пустым; для Gt + и Lt — содержать ровно один элемент, + интерпретируемый как целое число. + При strategic merge patch массив + заменяется целиком. + matchFields: + description: + Список требований селектора + узлов по полям узла. + items: + description: + Требование селектора узлов + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + Ключ метки, к которому + применяется селектор. + operator: + description: + "Задаёт отношение ключа + к набору значений. Допустимые операторы: + In, NotIn, Exists, DoesNotExist, + Gt и Lt." + values: + description: + Массив строковых значений. + Для операторов In и NotIn массив + должен быть непустым; для Exists + и DoesNotExist — пустым; для Gt + и Lt — содержать ровно один элемент, + интерпретируемый как целое число. + При strategic merge patch массив + заменяется целиком. + virtualMachineAndPodAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + virtualMachineAndPodAffinityTerm: + description: + Обязательное поле. Терм affinity + виртуальных машин, связанный с соответствующим + весом. + properties: + labelSelector: + description: + Селектор меток — это запрос + по меткам к набору ресурсов. Результаты + matchLabels и matchExpressions объединяются + по И. Пустой селектор соответствует всем + объектам, селектор со значением null — + ни одному. + properties: + matchExpressions: + description: + matchExpressions — список + требований селектора меток. Требования + объединяются по И. + items: + description: + Требование селектора + меток — это селектор, содержащий + ключ, набор значений и оператор, + связывающий ключ со значениями. + properties: + key: + description: + key — ключ метки, + к которому применяется селектор. + operator: + description: + "operator задаёт + отношение ключа к набору значений. + Допустимые операторы: In, NotIn, + Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In + и NotIn массив должен быть непустым; + для Exists и DoesNotExist — + пустым. При strategic merge + patch массив заменяется целиком. + matchLabels: + description: + matchLabels — набор пар + {ключ, значение}. Одна пара {ключ, + значение} эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In + и массивом values, содержащим только + «значение». Требования объединяются + по И. + namespaceSelector: + description: + Селектор меток — это запрос + по меткам к набору ресурсов. Результаты + matchLabels и matchExpressions объединяются + по И. Пустой селектор соответствует всем + объектам, селектор со значением null — + ни одному. + properties: + matchExpressions: + description: + matchExpressions — список + требований селектора меток. Требования + объединяются по И. + items: + description: + Требование селектора + меток — это селектор, содержащий + ключ, набор значений и оператор, + связывающий ключ со значениями. + properties: + key: + description: + key — ключ метки, + к которому применяется селектор. + operator: + description: + "operator задаёт + отношение ключа к набору значений. + Допустимые операторы: In, NotIn, + Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In + и NotIn массив должен быть непустым; + для Exists и DoesNotExist — + пустым. При strategic merge + patch массив заменяется целиком. + matchLabels: + description: + matchLabels — набор пар + {ключ, значение}. Одна пара {ключ, + значение} эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In + и массивом values, содержащим только + «значение». Требования объединяются + по И. + weight: + description: + Вес, связанный с соответствием + данному vmAndPodAffinityTerm, в диапазоне + 1–100. + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + description: + Селектор меток — это запрос по + меткам к набору ресурсов. Результаты matchLabels + и matchExpressions объединяются по И. Пустой + селектор соответствует всем объектам, селектор + со значением null — ни одному. + properties: + matchExpressions: + description: + matchExpressions — список требований + селектора меток. Требования объединяются + по И. + items: + description: + Требование селектора меток + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + key — ключ метки, к которому + применяется селектор. + operator: + description: + "operator задаёт отношение + ключа к набору значений. Допустимые + операторы: In, NotIn, Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In и NotIn + массив должен быть непустым; для + Exists и DoesNotExist — пустым. + При strategic merge patch массив + заменяется целиком. + matchLabels: + description: + matchLabels — набор пар {ключ, + значение}. Одна пара {ключ, значение} + эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In и + массивом values, содержащим только «значение». + Требования объединяются по И. + namespaceSelector: + description: + Селектор меток — это запрос по + меткам к набору ресурсов. Результаты matchLabels + и matchExpressions объединяются по И. Пустой + селектор соответствует всем объектам, селектор + со значением null — ни одному. + properties: + matchExpressions: + description: + matchExpressions — список требований + селектора меток. Требования объединяются + по И. + items: + description: + Требование селектора меток + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + key — ключ метки, к которому + применяется селектор. + operator: + description: + "operator задаёт отношение + ключа к набору значений. Допустимые + операторы: In, NotIn, Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In и NotIn + массив должен быть непустым; для + Exists и DoesNotExist — пустым. + При strategic merge patch массив + заменяется целиком. + matchLabels: + description: + matchLabels — набор пар {ключ, + значение}. Одна пара {ключ, значение} + эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In и + массивом values, содержащим только «значение». + Требования объединяются по И. + virtualMachineAndPodAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + virtualMachineAndPodAffinityTerm: + description: + Обязательное поле. Терм affinity + виртуальных машин, связанный с соответствующим + весом. + properties: + labelSelector: + description: + Селектор меток — это запрос + по меткам к набору ресурсов. Результаты + matchLabels и matchExpressions объединяются + по И. Пустой селектор соответствует всем + объектам, селектор со значением null — + ни одному. + properties: + matchExpressions: + description: + matchExpressions — список + требований селектора меток. Требования + объединяются по И. + items: + description: + Требование селектора + меток — это селектор, содержащий + ключ, набор значений и оператор, + связывающий ключ со значениями. + properties: + key: + description: + key — ключ метки, + к которому применяется селектор. + operator: + description: + "operator задаёт + отношение ключа к набору значений. + Допустимые операторы: In, NotIn, + Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In + и NotIn массив должен быть непустым; + для Exists и DoesNotExist — + пустым. При strategic merge + patch массив заменяется целиком. + matchLabels: + description: + matchLabels — набор пар + {ключ, значение}. Одна пара {ключ, + значение} эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In + и массивом values, содержащим только + «значение». Требования объединяются + по И. + namespaceSelector: + description: + Селектор меток — это запрос + по меткам к набору ресурсов. Результаты + matchLabels и matchExpressions объединяются + по И. Пустой селектор соответствует всем + объектам, селектор со значением null — + ни одному. + properties: + matchExpressions: + description: + matchExpressions — список + требований селектора меток. Требования + объединяются по И. + items: + description: + Требование селектора + меток — это селектор, содержащий + ключ, набор значений и оператор, + связывающий ключ со значениями. + properties: + key: + description: + key — ключ метки, + к которому применяется селектор. + operator: + description: + "operator задаёт + отношение ключа к набору значений. + Допустимые операторы: In, NotIn, + Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In + и NotIn массив должен быть непустым; + для Exists и DoesNotExist — + пустым. При strategic merge + patch массив заменяется целиком. + matchLabels: + description: + matchLabels — набор пар + {ключ, значение}. Одна пара {ключ, + значение} эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In + и массивом values, содержащим только + «значение». Требования объединяются + по И. + weight: + description: + Вес, связанный с соответствием + данному vmAndPodAffinityTerm, в диапазоне + 1–100. + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + description: + Селектор меток — это запрос по + меткам к набору ресурсов. Результаты matchLabels + и matchExpressions объединяются по И. Пустой + селектор соответствует всем объектам, селектор + со значением null — ни одному. + properties: + matchExpressions: + description: + matchExpressions — список требований + селектора меток. Требования объединяются + по И. + items: + description: + Требование селектора меток + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + key — ключ метки, к которому + применяется селектор. + operator: + description: + "operator задаёт отношение + ключа к набору значений. Допустимые + операторы: In, NotIn, Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In и NotIn + массив должен быть непустым; для + Exists и DoesNotExist — пустым. + При strategic merge patch массив + заменяется целиком. + matchLabels: + description: + matchLabels — набор пар {ключ, + значение}. Одна пара {ключ, значение} + эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In и + массивом values, содержащим только «значение». + Требования объединяются по И. + namespaceSelector: + description: + Селектор меток — это запрос по + меткам к набору ресурсов. Результаты matchLabels + и matchExpressions объединяются по И. Пустой + селектор соответствует всем объектам, селектор + со значением null — ни одному. + properties: + matchExpressions: + description: + matchExpressions — список требований + селектора меток. Требования объединяются + по И. + items: + description: + Требование селектора меток + — это селектор, содержащий ключ, набор + значений и оператор, связывающий ключ + со значениями. + properties: + key: + description: + key — ключ метки, к которому + применяется селектор. + operator: + description: + "operator задаёт отношение + ключа к набору значений. Допустимые + операторы: In, NotIn, Exists и DoesNotExist." + values: + description: + values — массив строковых + значений. Для операторов In и NotIn + массив должен быть непустым; для + Exists и DoesNotExist — пустым. + При strategic merge patch массив + заменяется целиком. + matchLabels: + description: + matchLabels — набор пар {ключ, + значение}. Одна пара {ключ, значение} + эквивалентна элементу matchExpressions + с полем key = «ключ», оператором In и + массивом values, содержащим только «значение». + Требования объединяются по И. + blockDeviceRefs: + description: | + Список блочных устройств, которые могут быть смонтированы в ВМ. + + Порядок загрузки определяется порядком в списке. + items: + properties: + bootOrder: + description: | + Порядок загрузки блочного устройства. Меньшее значение означает более высокий приоритет. + Если параметр не задан ни для одного устройства, порядок загрузки определяется позицией устройства в списке (начиная с 1). + Если параметр задан хотя бы для одного устройства, порядок загрузки определяется указанными значениями. + kind: + description: | + Поддерживаемые типы устройств: + + * `ClusterVirtualImage` — использовать ClusterVirtualImage в качестве диска. Данный тип всегда монтируется в режиме для чтения (`ReadOnly`). ISO-образ будет смонтирован как устройство CD-ROM; + * `VirtualImage` — использовать VirtualImage в качестве диска. Данный тип всегда монтируется в режиме для чтения (`ReadOnly`). ISO-образ будет смонтирован как устройство CD-ROM; + * `VirtualDisk` — использовать VirtualDisk в качестве диска. Данный тип всегда монтируется в режиме для чтения и записи (`ReadWrite`). + name: + description: | + Имя ресурса заданного типа. + bootloader: + description: | + Загрузчик для ВМ: + + * `BIOS` — использовать BIOS; + * `EFI` — использовать Unified Extensible Firmware (EFI/UEFI); + * `EFIWithSecureBoot` — использовать UEFI/EFI с поддержкой функции Secure Boot. + cpu: + description: | + Блок определяет настройки CPU для виртуальной машины. + properties: + coreFraction: + description: | + Гарантированная доля времени CPU, выделяемая ВМ. Указывается в процентах. + Допустимый диапазон значений определяется параметром `sizePolicy` в используемом VirtualMachineClass. Если `sizePolicy` не задан, используйте значения в диапазоне `1–100%`. + Если значение параметра не указано, применяется значение по умолчанию из `sizePolicy` используемого VirtualMachineClass. + Если значение по умолчанию в классе не задано, применяется `100%`. + cores: + description: | + Количество ядер. + disruptions: + description: | + Описание политики применения изменений, требующих перезагрузки ВМ. + + Для применения изменений в некоторых параметрах конфигурации ВМ потребуется перезагрузка. Данная политика позволяет задать поведение, определяющее как ВМ будет реагировать на такие изменения. + properties: + restartApprovalMode: + description: | + Режим одобрения для изменений, требующих перезагрузки ВМ: + + - `Manual` — изменения не будут применены до тех пор, пока пользователь самостоятельно не осуществит перезагрузку ВМ; + - `Automatic` — ВМ будет перезагружена сразу после сохранения параметров, требующих перезагрузки. + enableParavirtualization: + description: | + Использовать шину `virtio` для подключения виртуальных устройств ВМ. Чтобы отключить `virtio` для ВМ, установите значение `False`. + + > **Внимание**: Для использования режима паравиртуализации некоторые ОС требуют установки соответствующих драйверов. + + > **Внимание**: При значении `false` горячее подключение блочных устройств через `.spec.blockDeviceRefs` не поддерживается. Изменения блочных устройств требуют перезагрузки ВМ. Горячее подключение через `VirtualMachineBlockDeviceAttachment` по-прежнему поддерживается. + liveMigrationPolicy: + description: | + Политика для процесса живой миграции: + + * `AlwaysSafe` — использовать безопасный вариант для автоматических миграций и для ручного запуска. Не включать замедление CPU. + * `PreferSafe` — использовать безопасный вариант для автоматических миграций. Замедление CPU можно включить вручную с помощью поля `force=true` в VMOP. + * `AlwaysForced` — включать замедление CPU для автоматических миграций и для ручного запуска. Нельзя отключить замедление CPU. + * `PreferForced` — включать замедление CPU для автоматических миграций. Замедление CPU можно выключить вручную с помощью поля `force=false` в VMOP. + memory: + description: | + Блок настроек оперативной памяти для виртуальной машины. + nodeSelector: + description: | + [По аналогии](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes//) c параметром подов `spec.nodeSelector` в Kubernetes. + osType: + description: | + Параметр позволяет выбрать тип используемой ОС, для которой будет создана ВМ с оптимальным набором необходимых виртуальных устройств и параметров. + + * `Windows` — для ОС семейства Microsoft Windows; + * `Generic` — для других типов ОС. + priorityClassName: + description: | + [По аналогии](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) с параметром подов `spec.priorityClassName` в Kubernetes. + provisioning: + description: | + Блок описания сценария начальной инициализации ВМ. + properties: + sysprepRef: + description: | + Ссылка на существующий ресурс со сценарием автоматизации Windows. + + Структура ресурса для типа `SysprepRef`: + + * `.data.autounattend.xml`; + * `.data.unattend.xml`. + properties: + kind: + description: | + Тип ресурса. + Используйте секрет с типом `provisioning.virtualization.deckhouse.io/sysprep`. + type: + description: | + Поддерживаемые параметры для использования сценария инициализации: + + * `UserData` — использовать сценарий `cloud-init` в секции `.spec.provisioning.UserData`; + * `UserDataRef` — использовать сценарий `cloud-init`, который находится в другом ресурсе; + * `SysprepRef` — использовать сценарий автоматизации установки Windows, который находится в другом ресурсе. + userData: + description: | + Текст сценария `cloud-init`. + + [Дополнительная информация о `cloud-init` и примеры конфигурации](https://cloudinit.readthedocs.io/en/latest/reference/examples.html). + userDataRef: + description: | + Ссылка на существующий ресурс со сценарием `cloud-init`. + + Структура ресурса для типа `userDataRef`: + + * `.data.userData`. + properties: + kind: + description: | + Тип ресурса. + runPolicy: + description: | + Параметр определяет политику запуска ВМ: + + * `AlwaysOn` — после создания ВМ всегда находится в работающем состоянии, даже в случае отключения средствами ОС; + * `AlwaysOff` — после создания ВМ всегда находится в выключенном состоянии; + * `Manual` — после создания ВМ выключается. Включение и выключение ВМ контролируется через API-сервисы или средства ОС; + * `AlwaysOnUnlessStoppedManually` — после создания ВМ всегда находится в работающем состоянии. ВМ можно выключить средствами ОС или воспользоваться командой для утилиты d8: `d8 v stop `. + terminationGracePeriodSeconds: + description: | + Период ожидания после подачи сигнала о прекращении работы ВМ (`SIGTERM`), по истечении которого работа ВМ принудительно завершается. + tolerations: + description: | + [По аналогии](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) с параметром подов `spec.tolerations` в Kubernetes. + items: + description: + Под, к которому привязан этот toleration, допускает + любой taint, соответствующий тройке + с учётом оператора . + properties: + effect: + description: + "Effect задаёт эффект taint, которому нужно + соответствовать. Пустое значение означает соответствие + всем эффектам. Если задано, допустимые значения: NoSchedule, + PreferNoSchedule и NoExecute." + key: + description: + Key — ключ taint, к которому применяется + toleration. Пустое значение означает соответствие + всем ключам. Если ключ пуст, оператор должен быть + Exists; такая комбинация означает соответствие всем + значениям и всем ключам. + operator: + description: + "Operator задаёт отношение ключа к значению. + Допустимые операторы: Exists и Equal, по умолчанию + Equal. Exists эквивалентен подстановочному значению, + поэтому под может допускать все taint определённой + категории." + tolerationSeconds: + description: + TolerationSeconds задаёт время, в течение + которого toleration (должен иметь эффект NoExecute, + иначе поле игнорируется) допускает taint. По умолчанию + не задано — taint допускается бесконечно (без вытеснения). + Нулевые и отрицательные значения трактуются системой + как 0 (немедленное вытеснение). + value: + description: + Value — значение taint, которому соответствует + toleration. Для оператора Exists значение должно быть + пустым, иначе — обычная строка. + topologySpreadConstraints: + items: + description: | + Задаёт способ распределения подходящих подов по заданной топологии. + properties: + labelSelector: + description: + LabelSelector используется для поиска подходящих + подов. Поды, соответствующие этому селектору меток, + учитываются при подсчёте числа подов в соответствующем + домене топологии. + properties: + matchExpressions: + description: + matchExpressions — список требований + селектора меток. Требования объединяются по И. + items: + description: | + Требование селектора меток — это селектор, содержащий ключ, набор значений и оператор, связывающий ключ со значениями. + properties: + key: + description: + key — ключ метки, к которому + применяется селектор. + operator: + description: + "operator задаёт отношение ключа + к набору значений. Допустимые операторы: + In, NotIn, Exists и DoesNotExist." + values: + description: + values — массив строковых значений. + Для операторов In и NotIn массив должен + быть непустым; для Exists и DoesNotExist + — пустым. При strategic merge patch массив + заменяется целиком. + matchLabels: + description: + matchLabels — набор пар {ключ, значение}. + Одна пара {ключ, значение} эквивалентна элементу + matchExpressions с полем key = «ключ», оператором + In и массивом values, содержащим только «значение». + Требования объединяются по И. + matchLabelKeys: + description: | + Набор ключей лейблов пода для выбора подов, по которым вычисляется распределение. + Ключи используются для получения значений из лейблов размещаемого пода; пары «ключ–значение» объединяются по И с `labelSelector`, чтобы выбрать группу существующих подов, по которым вычисляется распределение для нового пода. + Один и тот же ключ не может присутствовать одновременно в `matchLabelKeys` и `labelSelector`. + `matchLabelKeys` нельзя задавать, если `labelSelector` не задан. + Ключи, отсутствующие в лейблах размещаемого пода, игнорируются. + Пустой или незаданный список означает выбор только по `labelSelector`. + + Поле находится в статусе beta и требует включения feature gate `MatchLabelKeysInPodTopologySpread` (включён по умолчанию). + maxSkew: + description: | + Задаёт допустимую степень неравномерности распределения подов. + + При `whenUnsatisfiable=DoNotSchedule` — максимально допустимая разница между числом подходящих подов в целевой топологии и глобальным минимумом. + Глобальный минимум — минимальное число подходящих подов в подходящем домене (или ноль, если число подходящих доменов меньше `MinDomains`). + + Например, в кластере из трёх зон при `MaxSkew=1` и распределении подов с одинаковым `labelSelector` как 2/2/1 (в зонах zone1, zone2 и zone3 соответственно) глобальный минимум равен 1: + + * при `MaxSkew=1` новый под можно разместить только в zone3 (станет 2/2/2); размещение в zone1 или zone2 нарушит `MaxSkew`, так как ActualSkew составит 3−1; + * при `MaxSkew=2` новый под можно разместить в любой зоне. + + При `whenUnsatisfiable=ScheduleAnyway` поле используется, чтобы отдавать приоритет топологиям, которые его удовлетворяют. + + Обязательное поле. Значение по умолчанию — 1; значение 0 не допускается. + minDomains: + description: | + Задаёт минимальное число подходящих доменов. + + Если число подходящих доменов с совпадающими ключами топологии меньше `minDomains`, Pod Topology Spread считает «глобальный минимум» равным 0, и затем вычисляется перекос (Skew). + Когда число подходящих доменов с совпадающими ключами топологии равно или больше `minDomains`, это значение не влияет на планирование. + В результате, когда число подходящих доменов меньше `minDomains`, планировщик не разместит в этих доменах больше `maxSkew` подов. + + Если значение не задано, ограничение ведёт себя так, как если бы `MinDomains` было равно 1. + Допустимые значения — целые числа больше 0. + Если значение задано, `whenUnsatisfiable` должно быть `DoNotSchedule`. + + Например, в кластере из трёх зон при `MaxSkew=2`, `MinDomains=5` и распределении подов с одинаковым `labelSelector` как 2/2/2 число доменов меньше 5 (`MinDomains`), поэтому «глобальный минимум» считается равным 0. + В этой ситуации новый под с тем же `labelSelector` не может быть запланирован: если разместить его в любой из трёх зон, вычисленный перекос составит 3 (3−0), что нарушит `MaxSkew`. + nodeAffinityPolicy: + description: | + Определяет, как учитывать `nodeAffinity`/`nodeSelector` пода при вычислении перекоса распределения по топологии. Варианты: + + - `Honor` — в расчётах участвуют только узлы, соответствующие `nodeAffinity`/`nodeSelector`; + - `Ignore` — `nodeAffinity`/`nodeSelector` игнорируются, в расчётах участвуют все узлы. + + Если значение не задано, поведение эквивалентно политике `Honor`. + nodeTaintsPolicy: + description: | + Определяет, как учитывать taints узлов при вычислении перекоса распределения по топологии. Варианты: + + - `Honor` — участвуют узлы без taints, а также узлы с taints, для которых у размещаемого пода есть toleration; + - `Ignore` — taints узлов игнорируются, в расчётах участвуют все узлы. + + Если значение не задано, поведение эквивалентно политике `Ignore`. + topologyKey: + description: + TopologyKey — ключ меток узлов. Узлы с + меткой с этим ключом и одинаковыми значениями считаются + принадлежащими одной топологии. Каждая пара <ключ, + значение> рассматривается как «корзина», и в каждую + корзину стараются поместить сбалансированное число + подов. Домен — конкретный экземпляр топологии; подходящий + домен — домен, узлы которого удовлетворяют nodeAffinityPolicy + и nodeTaintsPolicy. Например, при TopologyKey «kubernetes.io/hostname» + доменом топологии является каждый узел, а при «topology.kubernetes.io/zone» + — каждая зона. Обязательное поле. + whenUnsatisfiable: + description: | + Задаёт, что делать с подом, если он не удовлетворяет ограничению распределения. + + - `DoNotSchedule` (по умолчанию) — планировщик не размещает под. + - `ScheduleAnyway` — планировщик размещает под в любом месте, отдавая приоритет топологиям, помогающим уменьшить перекос. + + Ограничение считается «невыполнимым» для нового пода тогда и только тогда, когда любое возможное размещение пода нарушило бы `MaxSkew` в какой-либо топологии. + Например, в кластере из трёх зон при `MaxSkew=1` и распределении подов с одинаковым `labelSelector` как 3/1/1 (в зонах zone1, zone2 и zone3 соответственно) при `WhenUnsatisfiable=DoNotSchedule` новый под можно разместить только в zone2 или zone3 (станет 3/2/1 или 3/1/2), так как ActualSkew (2−1) в zone2 (zone3) удовлетворяет `MaxSkew` (1). + Иными словами, кластер может оставаться несбалансированным, но планировщик не усилит дисбаланс. + + Обязательное поле. + usbDevices: + description: | + Список USB-устройств для подключения к виртуальной машине. + Устройства указываются по имени ресурса `USBDevice` в том же пространстве имен. + items: + properties: + name: + description: | + Имя ресурса `USBDevice` в том же пространстве имен. + virtualMachineClassName: + description: | + Имя ресурса VirtualMachineClass, который описывает требования к виртуальному CPU и памяти, а также политику размещения ресурсов. + virtualMachineIPAddressName: + description: | + Имя для связанного ресурса virtualMachineIPAddress. + + Указывается при необходимости использования ранее созданного IP-адреса ВМ. + + Если не указано явно, по умолчанию для ВМ создаётся ресурс virtualMachineIPAddress с именем, аналогичным ресурсу ВМ (`.metadata.name`). + status: + description: | + Наблюдаемое состояние пула виртуальных машин. + properties: + conditions: + description: | + Условия, описывающие текущее состояние пула. + items: + description: | + Подробные сведения об одном аспекте текущего состояния данного API-ресурса. + properties: + lastTransitionTime: + description: Время перехода условия из одного состояния в другое. + message: + description: Удобочитаемое сообщение с подробной информацией о последнем переходе. + observedGeneration: + description: | + `.metadata.generation`, на основе которого было установлено условие. + Например, если `.metadata.generation` в настоящее время имеет значение `12`, а `.status.conditions[x].observedGeneration` имеет значение `9`, то условие устарело. + reason: + description: Краткая причина последнего перехода состояния. + status: + description: | + Статус условия. Возможные значения: `True`, `False`, `Unknown`. + type: + description: Тип условия. + desiredTemplateHash: + description: | + Хеш текущего `virtualMachineTemplate` — ревизия, к которой контроллер приводит реплики. + observedGeneration: + description: | + Поколение спецификации, обработанное контроллером. + readyReplicas: + description: | + Число членов, готовых обслуживать нагрузку (без учёта `Terminating`). + replicas: + description: | + Число существующих членов пула, включая находящиеся в состоянии `Terminating`: такая машина всё ещё занимает ресурсы, поэтому это реальная ёмкость. + restartPendingReplicas: + description: | + Число реплик, спецификация которых уже приведена к новому шаблону, но изменения, требующие перезапуска, ещё не применены. + selector: + description: | + Строковый селектор меток, публикуемый контроллером для субресурса `scale`; HPA/KEDA читают его самостоятельно. + updatedReplicas: + description: | + Число реплик, фактически находящихся на ревизии `desiredTemplateHash` (полностью синхронизированных). diff --git a/crds/embedded/datavolumes.yaml b/crds/embedded/datavolumes.yaml deleted file mode 100644 index 1d9eda8338..0000000000 --- a/crds/embedded/datavolumes.yaml +++ /dev/null @@ -1,748 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: internalvirtualizationdatavolumes.cdi.internal.virtualization.deckhouse.io - labels: - heritage: deckhouse - module: virtualization - app.kubernetes.io/component: cdi -spec: - conversion: - strategy: None - group: cdi.internal.virtualization.deckhouse.io - names: - categories: - - intvirt - kind: InternalVirtualizationDataVolume - listKind: InternalVirtualizationDataVolumeList - plural: internalvirtualizationdatavolumes - shortNames: - - intvirtdv - singular: internalvirtualizationdatavolume - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The phase the data volume is in - jsonPath: .status.phase - name: Phase - type: string - - description: Transfer progress in percentage if known, N/A otherwise - jsonPath: .status.progress - name: Progress - type: string - - description: The number of times the transfer has been restarted. - jsonPath: .status.restartCount - name: Restarts - type: integer - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: DataVolume is an abstraction on top of PersistentVolumeClaims - to allow easy population of those PersistentVolumeClaims with relation to - VirtualMachines - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: DataVolumeSpec defines the DataVolume type specification - properties: - checkpoints: - description: Checkpoints is a list of DataVolumeCheckpoints, representing - stages in a multistage import. - items: - description: DataVolumeCheckpoint defines a stage in a warm migration. - properties: - current: - description: Current is the identifier of the snapshot created - for this checkpoint. - type: string - previous: - description: Previous is the identifier of the snapshot from - the previous checkpoint. - type: string - required: - - current - - previous - type: object - type: array - contentType: - description: 'DataVolumeContentType options: "kubevirt", "archive"' - enum: - - kubevirt - - archive - type: string - finalCheckpoint: - description: FinalCheckpoint indicates whether the current DataVolumeCheckpoint - is the final checkpoint. - type: boolean - preallocation: - description: Preallocation controls whether storage for DataVolumes - should be allocated in advance. - type: boolean - priorityClassName: - description: PriorityClassName for Importer, Cloner and Uploader pod - type: string - pvc: - description: PVC is the PVC specification - properties: - accessModes: - description: 'accessModes contains the desired access modes the - volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If the provisioner - or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified - data source. When the AnyVolumeDataSource feature gate is enabled, - dataSource contents will be copied to dataSourceRef, and dataSourceRef - contents will be copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, then dataSourceRef - will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the resource being - referenced. If APIGroup is not specified, the specified - Kind must be in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: 'dataSourceRef specifies the object from which to - populate the volume with data, if a non-empty volume is desired. - This may be any object from a non-empty API group (non core - object) or a PersistentVolumeClaim object. When this field is - specified, volume binding will only succeed if the type of the - specified object matches some installed volume populator or - dynamic provisioner. This field will replace the functionality - of the dataSource field and as such if both fields are non-empty, - they must have the same value. For backwards compatibility, - when namespace isn''t specified in dataSourceRef, both fields - (dataSource and dataSourceRef) will be set to the same value - automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, dataSource isn''t - set to the same value and must be empty. There are three important - differences between dataSource and dataSourceRef: * While dataSource - only allows two specific types of objects, dataSourceRef allows - any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), - dataSourceRef preserves all values, and generates an error if - a disallowed value is specified. * While dataSource only allows - local objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature - gate to be enabled. (Alpha) Using the namespace field of dataSourceRef - requires the CrossNamespaceVolumeDataSource feature gate to - be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the resource being - referenced. If APIGroup is not specified, the specified - Kind must be in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: Namespace is the namespace of resource being - referenced Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum resources the volume - should have. If RecoverVolumeExpansionFailure feature is enabled - users are allowed to specify resource requirements that are - lower than previous value but must still be higher than capacity - recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. Requests cannot exceed - Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes to consider - for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If - the operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A - single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is "key", - the operator is "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: 'storageClassName is the name of the StorageClass - required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume is required - by the claim. Value of Filesystem is implied when not included - in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the PersistentVolume - backing this claim. - type: string - type: object - source: - description: Source is the src of the data for the requested DataVolume - properties: - blank: - description: DataVolumeBlankImage provides the parameters to create - a new raw blank image for the PVC - type: object - gcs: - description: DataVolumeSourceGCS provides the parameters to create - a Data Volume from an GCS source - properties: - secretRef: - description: SecretRef provides the secret reference needed - to access the GCS source - type: string - url: - description: URL is the url of the GCS source - type: string - required: - - url - type: object - http: - description: DataVolumeSourceHTTP can be either an http or https - endpoint, with an optional basic auth user name and password, - and an optional configmap containing additional CAs - properties: - certConfigMap: - description: CertConfigMap is a configmap reference, containing - a Certificate Authority(CA) public key, and a base64 encoded - pem certificate - type: string - extraHeaders: - description: ExtraHeaders is a list of strings containing - extra headers to include with HTTP transfer requests - items: - type: string - type: array - secretExtraHeaders: - description: SecretExtraHeaders is a list of Secret references, - each containing an extra HTTP header that may include sensitive - information - items: - type: string - type: array - secretRef: - description: SecretRef A Secret reference, the secret should - contain accessKeyId (user name) base64 encoded, and secretKey - (password) also base64 encoded - type: string - url: - description: URL is the URL of the http(s) endpoint - type: string - required: - - url - type: object - imageio: - description: DataVolumeSourceImageIO provides the parameters to - create a Data Volume from an imageio source - properties: - certConfigMap: - description: CertConfigMap provides a reference to the CA - cert - type: string - diskId: - description: DiskID provides id of a disk to be imported - type: string - secretRef: - description: SecretRef provides the secret reference needed - to access the ovirt-engine - type: string - url: - description: URL is the URL of the ovirt-engine - type: string - required: - - diskId - - url - type: object - pvc: - description: DataVolumeSourcePVC provides the parameters to create - a Data Volume from an existing PVC - properties: - name: - description: The name of the source PVC - type: string - namespace: - description: The namespace of the source PVC - type: string - required: - - name - - namespace - type: object - registry: - description: DataVolumeSourceRegistry provides the parameters - to create a Data Volume from an registry source - properties: - certConfigMap: - description: CertConfigMap provides a reference to the Registry - certs - type: string - imageStream: - description: ImageStream is the name of image stream for import - type: string - pullMethod: - description: PullMethod can be either "pod" (default import), - or "node" (node docker cache based import) - type: string - secretRef: - description: SecretRef provides the secret reference needed - to access the Registry source - type: string - url: - description: 'URL is the url of the registry source (starting - with the scheme: docker, oci-archive)' - type: string - type: object - s3: - description: DataVolumeSourceS3 provides the parameters to create - a Data Volume from an S3 source - properties: - certConfigMap: - description: CertConfigMap is a configmap reference, containing - a Certificate Authority(CA) public key, and a base64 encoded - pem certificate - type: string - secretRef: - description: SecretRef provides the secret reference needed - to access the S3 source - type: string - url: - description: URL is the url of the S3 source - type: string - required: - - url - type: object - snapshot: - description: DataVolumeSourceSnapshot provides the parameters - to create a Data Volume from an existing VolumeSnapshot - properties: - name: - description: The name of the source VolumeSnapshot - type: string - namespace: - description: The namespace of the source VolumeSnapshot - type: string - required: - - name - - namespace - type: object - upload: - description: DataVolumeSourceUpload provides the parameters to - create a Data Volume by uploading the source - type: object - vddk: - description: DataVolumeSourceVDDK provides the parameters to create - a Data Volume from a Vmware source - properties: - backingFile: - description: BackingFile is the path to the virtual hard disk - to migrate from vCenter/ESXi - type: string - initImageURL: - description: InitImageURL is an optional URL to an image containing - an extracted VDDK library, overrides v2v-vmware config map - type: string - secretRef: - description: SecretRef provides a reference to a secret containing - the username and password needed to access the vCenter or - ESXi host - type: string - thumbprint: - description: Thumbprint is the certificate thumbprint of the - vCenter or ESXi host - type: string - url: - description: URL is the URL of the vCenter or ESXi host with - the VM to migrate - type: string - uuid: - description: UUID is the UUID of the virtual machine that - the backing file is attached to in vCenter/ESXi - type: string - type: object - type: object - sourceRef: - description: SourceRef is an indirect reference to the source of data - for the requested DataVolume - properties: - kind: - description: The kind of the source reference, currently only - "DataSource" is supported - type: string - name: - description: The name of the source reference - type: string - namespace: - description: The namespace of the source reference, defaults to - the DataVolume namespace - type: string - required: - - kind - - name - type: object - storage: - description: Storage is the requested storage specification - properties: - accessModes: - description: 'AccessModes contains the desired access modes the - volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'This field can be used to specify either: * An existing - VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) * An existing custom - resource that implements data population (Alpha) In order to - use custom resource types that implement data population, the - AnyVolumeDataSource feature gate must be enabled. If the provisioner - or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified - data source. If the AnyVolumeDataSource feature gate is enabled, - this field will always have the same contents as the DataSourceRef - field.' - properties: - apiGroup: - description: APIGroup is the group for the resource being - referenced. If APIGroup is not specified, the specified - Kind must be in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: 'Specifies the object from which to populate the - volume with data, if a non-empty volume is desired. This may - be any local object from a non-empty API group (non core object) - or a PersistentVolumeClaim object. When this field is specified, - volume binding will only succeed if the type of the specified - object matches some installed volume populator or dynamic provisioner. - This field will replace the functionality of the DataSource - field and as such if both fields are non-empty, they must have - the same value. For backwards compatibility, both fields (DataSource - and DataSourceRef) will be set to the same value automatically - if one of them is empty and the other is non-empty. There are - two important differences between DataSource and DataSourceRef: - * While DataSource only allows two specific types of objects, - DataSourceRef allows any non-core object, as well as PersistentVolumeClaim - objects. * While DataSource ignores disallowed values (dropping - them), DataSourceRef preserves all values, and generates an - error if a disallowed value is specified. (Beta) Using this - field requires the AnyVolumeDataSource feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the resource being - referenced. If APIGroup is not specified, the specified - Kind must be in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: Namespace is the namespace of resource being - referenced Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'Resources represents the minimum resources the volume - should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, defined - in spec.resourceClaims, that are used by this container. - \n This is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be - set for containers." - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of one entry in - pod.spec.resourceClaims of the Pod where this field - is used. It makes that resource available inside a - container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. Requests cannot exceed - Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: A label query over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If - the operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A - single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is "key", - the operator is "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: 'Name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume is required - by the claim. Value of Filesystem is implied when not included - in claim spec. - type: string - volumeName: - description: VolumeName is the binding reference to the PersistentVolume - backing this claim. - type: string - type: object - type: object - status: - description: DataVolumeStatus contains the current status of the DataVolume - properties: - claimName: - description: ClaimName is the name of the underlying PVC used by the - DataVolume. - type: string - conditions: - items: - description: DataVolumeCondition represents the state of a data - volume condition. - properties: - lastHeartbeatTime: - format: date-time - type: string - lastTransitionTime: - format: date-time - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - description: DataVolumeConditionType is the string representation - of known condition types - type: string - required: - - status - - type - type: object - type: array - phase: - description: Phase is the current phase of the data volume - type: string - progress: - description: DataVolumeProgress is the current progress of the DataVolume - transfer operation. Value between 0 and 100 inclusive, N/A if not - available - type: string - restartCount: - description: RestartCount is the number of times the pod populating - the DataVolume has restarted - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/crds/embedded/storageprofiles.yaml b/crds/embedded/storageprofiles.yaml new file mode 100644 index 0000000000..6e4287dfcf --- /dev/null +++ b/crds/embedded/storageprofiles.yaml @@ -0,0 +1,151 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + heritage: deckhouse + module: virtualization + app.kubernetes.io/component: virtualization-controller + name: storageprofiles.storage.virtualization.deckhouse.io +spec: + group: storage.virtualization.deckhouse.io + names: + kind: StorageProfile + listKind: StorageProfileList + plural: storageprofiles + singular: storageprofile + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: StorageProfile provides storage capability recommendations + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageProfileSpec defines specification for StorageProfile + properties: + claimPropertySets: + description: ClaimPropertySets is a provided set of properties applicable + to PVC + items: + description: ClaimPropertySet is a set of properties applicable + to PVC + properties: + accessModes: + description: |- + AccessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + maxItems: 4 + type: array + x-kubernetes-validations: + - message: Illegal AccessMode + rule: self.all(am, am in ['ReadWriteOnce', 'ReadOnlyMany', + 'ReadWriteMany', 'ReadWriteOncePod']) + volumeMode: + description: |- + VolumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + enum: + - Block + - Filesystem + type: string + required: + - accessModes + - volumeMode + type: object + maxItems: 8 + type: array + cloneStrategy: + description: CloneStrategy defines the preferred method for cloning + a volume + type: string + dataImportCronSourceFormat: + description: DataImportCronSourceFormat defines the format of the + DataImportCron-created disk image sources + type: string + snapshotClass: + description: SnapshotClass is optional specific VolumeSnapshotClass + for CloneStrategySnapshot. If not set, a VolumeSnapshotClass is + chosen according to the provisioner. + type: string + type: object + status: + description: StorageProfileStatus provides the most recently observed + status of the StorageProfile + properties: + claimPropertySets: + description: ClaimPropertySets computed from the spec and detected + in the system + items: + description: ClaimPropertySet is a set of properties applicable + to PVC + properties: + accessModes: + description: |- + AccessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + maxItems: 4 + type: array + x-kubernetes-validations: + - message: Illegal AccessMode + rule: self.all(am, am in ['ReadWriteOnce', 'ReadOnlyMany', + 'ReadWriteMany', 'ReadWriteOncePod']) + volumeMode: + description: |- + VolumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + enum: + - Block + - Filesystem + type: string + required: + - accessModes + - volumeMode + type: object + maxItems: 8 + type: array + cloneStrategy: + description: CloneStrategy defines the preferred method for cloning + a volume + type: string + dataImportCronSourceFormat: + description: DataImportCronSourceFormat defines the format of the + DataImportCron-created disk image sources + type: string + provisioner: + description: The Storage class provisioner plugin name + type: string + snapshotClass: + description: SnapshotClass is optional specific VolumeSnapshotClass + for CloneStrategySnapshot. If not set, a VolumeSnapshotClass is + chosen according to the provisioner. + type: string + storageClass: + description: The StorageClass name for which capabilities are defined + type: string + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml index 52e6c381b8..45622d1a0c 100644 --- a/crds/embedded/virtualmachines.yaml +++ b/crds/embedded/virtualmachines.yaml @@ -4461,7 +4461,7 @@ spec: ready type: boolean restoreInProgress: - description: RestoreInProgress is the name of the VirtualMachineRestore + description: RestoreInProgress is the name of the restore operation currently executing type: string runStrategy: diff --git a/crds/virtualmachinepools.yaml b/crds/virtualmachinepools.yaml new file mode 100644 index 0000000000..87cb59c02e --- /dev/null +++ b/crds/virtualmachinepools.yaml @@ -0,0 +1,1793 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + labels: + heritage: deckhouse + module: virtualization + name: virtualmachinepools.virtualization.deckhouse.io +spec: + group: virtualization.deckhouse.io + names: + categories: + - virtualization + kind: VirtualMachinePool + listKind: VirtualMachinePoolList + plural: virtualmachinepools + shortNames: + - vmpool + - vmpools + singular: virtualmachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current number of pool members (including Terminating). + jsonPath: .status.replicas + name: Replicas + type: integer + - description: Number of members ready to serve. + jsonPath: .status.readyReplicas + name: Ready + type: integer + - description: Time of resource creation. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: |- + VirtualMachinePool declaratively manages a group of identical virtual machines: + it keeps the requested number of replicas, scales via the standard `scale` + subresource, and reuses "heavy" disks across replica generations. + + The resource is available only in paid editions (EE/SE+) and is gated behind + the `VirtualMachinePool` module feature gate. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + VirtualMachinePoolSpec is the desired state of a VirtualMachinePool. + + The disks a replica gets are declared in two places that must stay in sync: + virtualDiskTemplates describes each per-replica disk, and the template's + blockDeviceRefs references those disks (by name, kind VirtualDisk) to set the + boot order and interleave shared images. The three rules below enforce a + bijection between them so neither list can carry a dangling entry. + + Every virtualDiskTemplates entry must be referenced by a VirtualDisk in the template. + Every VirtualDisk reference in the template must name a virtualDiskTemplates entry. + The reference is one-to-one: no virtualDiskTemplates entry is referenced twice. + properties: + replicas: + description: |- + Replicas is the desired number of virtual machines in the pool. + + The field is written only by its owner — an autoscaler or a human via the + `scale` subresource, or by the addressed scale-down handler. The controller + never writes it. Bounds are held by the autoscaler; the hard ceiling is the + namespace ResourceQuota. + format: int32 + minimum: 0 + type: integer + scaleDownPolicy: + description: |- + ScaleDownPolicy chooses how a replica is picked when the pool is scaled down + anonymously through the `scale` subresource. It is required and has no + default, forcing a conscious choice between "any replica may be killed" and + "only addressed removal is allowed". + + - `NewestFirst` — anonymous scale-down is allowed; the youngest replicas + (least accumulated state) are removed first. + - `OldestFirst` — anonymous scale-down is allowed; the oldest replicas are + removed first (faster rotation). + - `Explicit` — anonymous scale-down through `scale` is rejected by a + webhook; replicas can be removed only by address. For "busy" workloads + such as CI runners and VDI. + enum: + - NewestFirst + - OldestFirst + - Explicit + type: string + virtualDiskTemplates: + description: |- + VirtualDiskTemplates describes each per-replica disk (reclaim policy, size, + data source). Names are unique within the pool (list-map key) and every + template must be referenced by a VirtualDisk entry in + virtualMachineTemplate.spec.blockDeviceRefs, which sets the boot order (see + the bijection rule on the spec). A disk with reclaim Delete belongs to its + VirtualMachine and is removed with it; a disk with reclaim Retain belongs to + the pool, outlives the replica and is reused on a later scale-up. + items: + description: VirtualDiskTemplateSpec describes a per-replica disk. + properties: + name: + description: |- + Name identifies the disk template within the pool. It is a DNS-1123 label + (no dots), because it is embedded into VirtualDisk names. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + reclaim: + description: + Reclaim controls what happens to the disk when + its replica is removed. + properties: + keep: + default: 0 + description: |- + Keep is the number of free (Retain) disks always kept warm for fast + scale-up; these are immune to the ttl. Only meaningful with Retain. + format: int32 + minimum: 0 + type: integer + onScaleDown: + default: Delete + description: OnScaleDown is Delete (default) or Retain. + enum: + - Delete + - Retain + type: string + ttl: + description: |- + TTL is how long a free disk lives beyond the warm buffer before it is + garbage-collected. Only meaningful with Retain. + type: string + type: object + x-kubernetes-validations: + - message: "keep and ttl are only valid with onScaleDown: Retain" + rule: self.onScaleDown == 'Retain' || (self.keep == 0 && !has(self.ttl)) + - message: + keep requires ttl; without ttl free disks are never + garbage-collected, so keep would have no effect + rule: self.keep == 0 || has(self.ttl) + spec: + description: + Spec is the desired state of the disk (an ordinary + VirtualDiskSpec). + properties: + dataSource: + properties: + containerImage: + description: + Use an image stored in an external container + registry. Only registries with enabled TLS are supported. + To provide a custom Certificate Authority (CA) chain, + use the `caBundle` field. + properties: + caBundle: + description: + CA chain in Base64 format to verify + the container registry. + example: YWFhCg== + format: byte + type: string + image: + description: + Path to the image in the container + registry. + example: registry.example.com/images/slackware:15 + pattern: ^(?P(?:(?P(?:(?:localhost|[\w-]+(?:\.[\w-]+)+)(?::\d+)?)|[\w]+:\d+)/)?(?P[a-z0-9_.-]+(?:/[a-z0-9_.-]+)*))(?::(?P[\w][\w.-]{0,127}))?(?:@(?P[A-Za-z][A-Za-z0-9]*(?:[+.-_][A-Za-z][A-Za-z0-9]*)*:[0-9a-fA-F]{32,}))?$ + type: string + imagePullSecret: + properties: + name: + description: + Name of the secret keeping container + registry credentials, which must be located + in the same namespace. + type: string + type: object + required: + - image + type: object + http: + description: |- + Fill the image with data from an external URL. The following schemas are supported: + + * HTTP + * HTTPS + + For HTTPS schema, there is an option to skip the TLS verification. + properties: + caBundle: + description: + CA chain in Base64 format to verify + the URL. + example: YWFhCg== + format: byte + type: string + checksum: + description: + Checksum to verify integrity and consistency + of the downloaded file. The file must match all + specified checksums. + properties: + md5: + example: f3b59bed9f91e32fac1210184fcff6f5 + maxLength: 32 + minLength: 32 + pattern: ^[0-9a-fA-F]{32}$ + type: string + sha256: + example: 78be890d71dde316c412da2ce8332ba47b9ce7a29d573801d2777e01aa20b9b5 + maxLength: 64 + minLength: 64 + pattern: ^[0-9a-fA-F]{64}$ + type: string + type: object + url: + description: |- + URL of the file for creating an image. The following file formats are supported: + * qcow2 + * vmdk + * vdi + * iso + * raw + The file can be compressed into an archive in one of the following formats: + * gz + * xz + example: https://mirror.example.com/images/slackware-15.qcow.gz + pattern: ^http[s]?:\/\/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$ + type: string + required: + - url + type: object + objectRef: + description: + Use an existing VirtualImage, ClusterVirtualImage, + or VirtualDiskSnapshot resource to create a disk. + properties: + kind: + description: + Kind of the existing VirtualImage, + ClusterVirtualImage, or VirtualDiskSnapshot resource. + enum: + - ClusterVirtualImage + - VirtualImage + - VirtualDiskSnapshot + type: string + name: + description: + Name of the existing VirtualImage, + ClusterVirtualImage, or VirtualDiskSnapshot resource. + minLength: 1 + type: string + required: + - kind + - name + type: object + type: + description: |- + The following image sources are available for creating an image: + + * `HTTP`: From a file published on an HTTP/HTTPS service at a given URL. + * `ContainerImage`: From another image stored in a container registry. + * `ObjectRef`: From an existing resource. + * `Upload`: From data uploaded by the user via a special interface. + enum: + - HTTP + - ContainerImage + - ObjectRef + - Upload + type: string + type: object + x-kubernetes-validations: + - message: + HTTP requires http and cannot have ContainerImage + or ObjectRef. + rule: + "self.type == 'HTTP' ? has(self.http) && !has(self.containerImage) + && !has(self.objectRef) : true" + - message: + ContainerImage requires containerImage and cannot + have HTTP or ObjectRef. + rule: + "self.type == 'ContainerImage' ? has(self.containerImage) + && !has(self.http) && !has(self.objectRef) : true" + - message: + ObjectRef requires objectRef and cannot have + HTTP or ContainerImage. + rule: + "self.type == 'ObjectRef' ? has(self.objectRef) + && !has(self.http) && !has(self.containerImage) : true" + persistentVolumeClaim: + description: Settings for creating PVCs to store the disk. + properties: + size: + anyOf: + - type: integer + - type: string + description: |- + Desired size for PVC to store the disk. If the disk is created from an image, the size must be at least as large as the original unpacked image. + + This parameter can be omitted if the `.spec.dataSource` section is filled out. In this case, the controller will determine the disk size automatically, based on the size of the extracted image from the source specified in `.spec.dataSource`. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClassName: + description: |- + StorageClass name required by the claim. For details on using StorageClass for PVC, refer to https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1. + + When creating disks, the user can specify the required StorageClass. If not specified, the default StorageClass will be used. + + The disk features and virtual machine behavior depend on the selected StorageClass. + + The `VolumeBindingMode` parameter in the StorageClass affects the disk creation process. The following values are allowed: + - `Immediate`: The disk will be created and becomes available for use immediately after creation. + - `WaitForFirstConsumer`: The disk will be created when first used on the node where the virtual machine will be started. + + StorageClass supports multiple storage settings: + - Creating a block device (`Block`) or file system (`FileSystem`). + - Multiple access (`ReadWriteMany`) or single access (`ReadWriteOnce`). The `ReadWriteMany` disks support multiple access, which enables a "live" migration of virtual machines. In contrast, the `ReadWriteOnce` disks, which can be accessed from only one node, don't have this feature. + + For known storage types, Deckhouse automatically determines the most efficient settings when creating disks (by priority, in descending order): + 1. `Block` + `ReadWriteMany` + 2. `FileSystem` + `ReadWriteMany` + 3. `Block` + `ReadWriteOnce` + 4. `FileSystem` + `ReadWriteOnce` + type: string + type: object + type: object + required: + - name + - spec + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + virtualMachineTemplate: + description: |- + VirtualMachineTemplate is the template every replica is stamped from. Its + `spec` is an ordinary VirtualMachineSpec, so a replica is no different from a + manually created virtual machine. + properties: + metadata: + description: |- + Metadata applied to every replica. Arbitrary user labels and annotations are + allowed; the controller adds its managed pool labels on top. A curated + struct (not the full ObjectMeta) so the CRD schema exposes labels and + annotations instead of an opaque object. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + description: Spec of the virtual machine that backs each replica. + properties: + affinity: + description: |- + VMAffinity [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) as in the pods `spec.affinity` parameter in Kubernetes; + + The affinity setting is completely similar to the above documentation, the only difference is in the names of some parameters. In fact, the following analogs are used: + * podAffinity -> virtualMachineAndPodAffinity + * podAffinityTerm -> virtualMachineAndPodAffinityTerm + properties: + nodeAffinity: + description: + Node affinity is a group of node affinity + scheduling rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: + A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: + A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: + The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: + A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: + The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: + Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: + Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: + A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: + The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: + A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: + The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + virtualMachineAndPodAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + virtualMachineAndPodAffinityTerm: + description: + Required. A vm affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array + namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding vmAndPodAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - virtualMachineAndPodAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array + namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + virtualMachineAndPodAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + virtualMachineAndPodAffinityTerm: + description: + Required. A vm affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array + namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding vmAndPodAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - virtualMachineAndPodAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + mismatchLabelKeys: + items: + type: string + type: array + namespaceSelector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: + matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + blockDeviceRefs: + description: |- + List of block devices that can be mounted by disks belonging to the virtual machine. + The order of booting is determined by the order in the list. + items: + properties: + bootOrder: + description: |- + Boot order of the block device. A smaller value means a higher priority. + If the parameter is not set for any device, the boot order follows the device position in the list (starting from 1). + If the parameter is set for at least one device, the boot order is determined by the specified values. + minimum: 1 + type: integer + kind: + description: |- + The BlockDeviceKind is a type of the block device. Options are: + + * `ClusterVirtualImage` — Use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. + * `VirtualImage` — Use `VirtualImage` as the disk. This type is always mounted in RO mode. If the image is an iso-image, it will be mounted as a CDROM device. + * `VirtualDisk` — Use `VirtualDisk` as the disk. This type is always mounted in RW mode. + enum: + - ClusterVirtualImage + - VirtualImage + - VirtualDisk + type: string + name: + description: The name of attached resource. + type: string + required: + - kind + - name + type: object + maxItems: 16 + minItems: 1 + type: array + bootloader: + default: BIOS + description: |- + The BootloaderType defines bootloader for VM. + * BIOS - use legacy BIOS. + * EFI - use Unified Extensible Firmware (EFI/UEFI). + * EFIWithSecureBoot - use UEFI/EFI with SecureBoot support. + enum: + - BIOS + - EFI + - EFIWithSecureBoot + type: string + cpu: + description: CPUSpec specifies the CPU settings for the VM. + properties: + coreFraction: + description: |- + Guaranteed share of CPU that will be allocated to the VM. Specified as a percentage. + The range of available values is defined in the VirtualMachineClass sizing policy. + If not specified, the default value from the VirtualMachineClass will be used. + pattern: ^(100|[1-9][0-9]?|[1-9])%$ + type: string + cores: + description: + Specifies the number of cores inside the + VM. The value must be greater or equal 1. + format: int32 + minimum: 1 + type: integer + required: + - cores + type: object + disruptions: + default: + restartApprovalMode: Manual + description: |- + Disruptions describes the policy for applying changes that require rebooting the VM + Changes to some VM configuration settings require a reboot of the VM to apply them. This policy allows you to specify the behavior of how the VM will respond to such changes. + properties: + restartApprovalMode: + description: + "RestartApprovalMode defines a restart approving + mode: Manual or Automatic." + enum: + - Manual + - Automatic + type: string + type: object + enableParavirtualization: + default: true + description: |- + Use the `virtio` bus to connect virtual devices of the VM. Set false to disable `virtio` for this VM. + Note: To use paravirtualization mode, some operating systems require the appropriate drivers to be installed. + type: boolean + liveMigrationPolicy: + description: Live migration policy type. + enum: + - Manual + - Never + - AlwaysSafe + - PreferSafe + - AlwaysForced + - PreferForced + type: string + memory: + description: + MemorySpec specifies the memory settings for + the VM. + properties: + size: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - size + type: object + networks: + items: + properties: + id: + type: integer + name: + type: string + type: + type: string + virtualMachineMACAddressName: + type: string + required: + - type + type: object + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector must match a node's labels for the VM to be scheduled on that node. + [The same](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes//) as in the pods `spec.nodeSelector` parameter in Kubernetes. + type: object + osType: + default: Generic + description: |- + The OsType parameter allows you to select the type of used OS, for which a VM with an optimal set of required virtual devices and parameters will be created. + + * Windows - for Microsoft Windows family operating systems. + * Generic - for other types of OS. + enum: + - Windows + - Generic + type: string + priorityClassName: + description: + PriorityClassName [The same](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) as + in the pods `spec.priorityClassName` parameter in Kubernetes. + type: string + provisioning: + description: + Provisioning is a block allows you to configure + the provisioning script for the VM. + properties: + sysprepRef: + description: |- + SysprepRef is reference to an existing Windows sysprep automation. + Resource structure for the SysprepRef type: + * `.data.autounattend.xml`. + * `.data.unattend.xml`. + properties: + kind: + default: Secret + description: |- + The kind of existing Windows sysprep automation resource. + The following options are supported: + - Secret + enum: + - Secret + type: string + name: + type: string + required: + - name + type: object + type: + description: |- + ProvisioningType parameter defines the type of provisioning script: + + Parameters supported for using the provisioning script: + * UserData - use the cloud-init in the .spec.provisioning.UserData section. + * UserDataRef - use a cloud-init script that resides in a different resource. + * SysprepRef - Use a Windows Automation script that resides in a different resource. + More information: https://cloudinit.readthedocs.io/en/latest/reference/examples.html + type: string + userData: + description: Inline cloud-init userdata script. + type: string + userDataRef: + description: |- + UserDataRef is reference to an existing resource with a cloud-init script. + Resource structure for userDataRef type: + * `.data.userData`. + properties: + kind: + default: Secret + description: |- + The kind of existing cloud-init automation resource. + The following options are supported: + - Secret + enum: + - Secret + type: string + name: + type: string + required: + - name + type: object + required: + - type + type: object + x-kubernetes-validations: + - message: UserData cannot have userDataRef or sysprepRef. + rule: + "self.type == 'UserData' ? has(self.userData) && + !has(self.userDataRef) && !has(self.sysprepRef) : true" + - message: UserDataRef cannot have userData or sysprepRef. + rule: + "self.type == 'UserDataRef' ? has(self.userDataRef) + && !has(self.userData) && !has(self.sysprepRef) : true" + - message: SysprepRef cannot have userData or userDataRef. + rule: + "self.type == 'SysprepRef' ? has(self.sysprepRef) + && !has(self.userData) && !has(self.userDataRef) : true" + runPolicy: + default: AlwaysOnUnlessStoppedManually + description: |- + RunPolicy parameter defines the VM startup policy + * `AlwaysOn` - after creation the VM is always in a running state, even in case of its shutdown by OS means. + * `AlwaysOff` - after creation the VM is always in the off state. + * `Manual` - after creation the VM is switched off, the VM state (switching on/off) is controlled via sub-resources or OS means. + * `AlwaysOnUnlessStoppedManually` - after creation the VM is always in a running state. The VM can be shutdown by means of the OS or use the d8 utility: `d8 v stop `. + enum: + - AlwaysOn + - AlwaysOff + - Manual + - AlwaysOnUnlessStoppedManually + type: string + terminationGracePeriodSeconds: + default: 60 + description: + Grace period observed after signalling a VM to + stop after which the VM is force terminated. + format: int64 + type: integer + tolerations: + description: |- + Tolerations define rules to tolerate node taints. + The same](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) as in the pods `spec.tolerations` parameter in Kubernetes. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + items: + description: + TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + usbDevices: + description: |- + List of USB devices to attach to the virtual machine. + Devices are referenced by name of USBDevice resource in the same namespace. + items: + description: + USBDeviceSpecRef references a USB device by + name. + properties: + name: + description: + The name of USBDevice resource in the same + namespace. + type: string + required: + - name + type: object + maxItems: 8 + type: array + virtualMachineClassName: + description: + Name of the `VirtualMachineClass` resource describing + the requirements for a virtual CPU, memory and the resource + allocation policy and node placement policies for virtual + machines. + type: string + virtualMachineIPAddressName: + description: |- + Name for the associated `virtualMachineIPAddress` resource. + Specified when it is necessary to use a previously created IP address of the VM. + If not explicitly specified, by default a `virtualMachineIPAddress` resource is created for the VM with a name similar to the VM resource (`.metadata.name`). + type: string + required: + - blockDeviceRefs + - cpu + - liveMigrationPolicy + - memory + - virtualMachineClassName + type: object + type: object + required: + - scaleDownPolicy + - virtualDiskTemplates + - virtualMachineTemplate + type: object + x-kubernetes-validations: + - message: + each virtualDiskTemplates entry must be referenced by a VirtualDisk + entry in virtualMachineTemplate.spec.blockDeviceRefs + rule: + has(self.virtualMachineTemplate.spec) && has(self.virtualMachineTemplate.spec.blockDeviceRefs) + && self.virtualDiskTemplates.all(t, self.virtualMachineTemplate.spec.blockDeviceRefs.exists(r, + r.kind == 'VirtualDisk' && r.name == t.name)) + - message: + each VirtualDisk reference in virtualMachineTemplate.spec.blockDeviceRefs + must name a virtualDiskTemplates entry + rule: + has(self.virtualMachineTemplate.spec) && has(self.virtualMachineTemplate.spec.blockDeviceRefs) + && self.virtualMachineTemplate.spec.blockDeviceRefs.filter(r, r.kind + == 'VirtualDisk').all(r, self.virtualDiskTemplates.exists(t, t.name + == r.name)) + - message: + each virtualDiskTemplates entry must be referenced exactly + once (no duplicate VirtualDisk references) + rule: + has(self.virtualMachineTemplate.spec) && has(self.virtualMachineTemplate.spec.blockDeviceRefs) + && self.virtualMachineTemplate.spec.blockDeviceRefs.filter(r, r.kind + == 'VirtualDisk').size() == self.virtualDiskTemplates.size() + status: + description: VirtualMachinePoolStatus is the observed state of a VirtualMachinePool. + properties: + conditions: + description: Conditions describe the current state of the pool. + items: + description: + Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + desiredTemplateHash: + description: |- + DesiredTemplateHash is the hash of the current virtualMachineTemplate — the + revision the controller is converging replicas to (cf. updateRevision on a + StatefulSet). + type: string + observedGeneration: + description: + ObservedGeneration is the generation of the spec the + controller has processed. + format: int64 + type: integer + readyReplicas: + description: + ReadyReplicas is the number of members ready to serve + (Terminating excluded). + format: int32 + type: integer + replicas: + description: |- + Replicas is the number of existing members, including those in Terminating: + such a machine still occupies resources, so it is real capacity, not a phantom. + format: int32 + type: integer + restartPendingReplicas: + description: |- + RestartPendingReplicas is the number of replicas patched to the new template + whose disruptive part still awaits a restart. + format: int32 + type: integer + selector: + description: |- + Selector is the label selector the controller publishes for the `scale` + subresource; HPA/KEDA read it themselves. + type: string + updatedReplicas: + description: |- + UpdatedReplicas is the number of replicas effectively on DesiredTemplateHash + (fully synced). + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 1a1ff088d5..af4e5a3d16 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1076,7 +1076,7 @@ spec: virtualMachineIPAddressName: type: string description: | - Name of the VirtualMachineIPAddressName resource with an IP address of the VM. + Name of the VirtualMachineIPAddress resource with an IP address of the VM. ipAddress: type: string description: | diff --git a/docs/ADMIN_GUIDE.md b/docs/ADMIN_GUIDE.md index 300a3df7cb..d49fb13faf 100644 --- a/docs/ADMIN_GUIDE.md +++ b/docs/ADMIN_GUIDE.md @@ -1240,6 +1240,29 @@ How to start VM migration in the web interface: - Select `Migrate` from the pop-up menu. - Confirm or cancel the migration in the pop-up window. +#### Dedicated migration network + +By default, live migration traffic flows over the node's default network and competes with workload traffic. You can also route it over a dedicated VLAN provisioned by the [`sdn`](/modules/sdn/) module. + +Prerequisites: + +- The `sdn` module is enabled. +- A [SystemNetwork](/modules/sdn/cr.html#systemnetwork) resource exists and is in the `Ready` state. + +To enable the feature, set `spec.settings.liveMigration.network` on the `virtualization` ModuleConfig: use `type: SystemNetwork` and specify the name of the prepared `SystemNetwork` under `systemNetwork.name`. Once configured, every VM migration in the cluster runs over the specified `SystemNetwork` VLAN. + +```yaml +spec: + settings: + liveMigration: + network: + type: SystemNetwork + systemNetwork: + name: migration-net +``` + +To route migration traffic back over the default node network, remove the `network` block (it is the implicit default when unset). + #### Maintenance mode When working on nodes with virtual machines running, there is a risk of disrupting their performance. To avoid this, you can put a node into the maintenance mode and migrate the virtual machines to other free nodes. diff --git a/docs/ADMIN_GUIDE.ru.md b/docs/ADMIN_GUIDE.ru.md index 8e479ca9d3..d7a5e321df 100644 --- a/docs/ADMIN_GUIDE.ru.md +++ b/docs/ADMIN_GUIDE.ru.md @@ -1250,6 +1250,29 @@ spec: - Во всплывающем меню выберите `Мигрировать`. - Во всплывающем окне подтвердите или отмените миграцию. +#### Выделенная сеть для миграции + +По умолчанию трафик живой миграции идёт по основной сети узла и конкурирует за полосу пропускания с пользовательскими нагрузками. Его можно направить через выделенный VLAN, предоставляемый модулем [`sdn`](/modules/sdn/). + +Необходимые условия: + +- Включён модуль `sdn`. +- Ресурс [SystemNetwork](/modules/sdn/cr.html#systemnetwork) создан и находится в состоянии `Ready`. + +Чтобы включить возможность, задайте `spec.settings.liveMigration.network` в ресурсе ModuleConfig `virtualization`: укажите `type: SystemNetwork` и имя подготовленного `SystemNetwork` в поле `systemNetwork.name`. После настройки каждая миграция виртуальных машин в кластере выполняется по VLAN указанного `SystemNetwork`. + +```yaml +spec: + settings: + liveMigration: + network: + type: SystemNetwork + systemNetwork: + name: migration-net +``` + +Чтобы вернуть трафик миграции на основную сеть узла, удалите блок `network` (по умолчанию, если он не задан, используется сеть узла). + #### Режим обслуживания При выполнении работ на узлах с запущенными виртуальными машинами существует риск нарушения их работоспособности. Чтобы этого избежать, узел можно перевести в режим обслуживания и мигрировать виртуальные машины на другие свободные узлы. diff --git a/docs/FAQ.md b/docs/FAQ.md index 90a1ba21e0..8bf6146ed5 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -808,7 +808,7 @@ The command includes only virtual machines with assigned IP addresses in the `Ru 1. Optionally set host variables via annotations (for example, the SSH user): ```bash - d8 k -n demo-app annotate vm frontend provisioning.virtualization.deckhouse.io/ansible_user="cloud" + d8 k -n demo-app annotate vm frontend vars.provisioning.virtualization.deckhouse.io/ansible_user="cloud" ``` 1. Run Ansible with a dynamically generated inventory: diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index fc9efdd69b..aa84fb823f 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -808,7 +808,7 @@ ansible -m shell -a "uptime" \ 1. При необходимости задайте переменные хоста через аннотации (например, пользователя для SSH): ```bash - d8 k -n demo-app annotate vm frontend provisioning.virtualization.deckhouse.io/ansible_user="cloud" + d8 k -n demo-app annotate vm frontend vars.provisioning.virtualization.deckhouse.io/ansible_user="cloud" ``` 1. Запустите Ansible с динамически сформированным инвентарём: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 782947bbf5..d67057eb5c 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -3,11 +3,27 @@ title: "Release Notes" weight: 70 --- +## v1.9.3 + + +Release date: July 7, 2026. + + +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + +### Fixes + +- [module] Fixed slow downloading of images from DVCR to the node when attaching them to a virtual machine. +- [vm] Fixed a volume mount leak that could leave a VM with hotplugged images stuck in the Terminating state during deletion. + ## v1.9.2 + Release date: July 1, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [vm] Fixed reduced throughput during live migration of running VMs compared to v1.8.3. @@ -15,10 +31,13 @@ Release date: July 1, 2026. - [observability] Fixed loss of audit events and false `D8LogShipperDestinationErrors` alerts during certificate rotation of the `virtualization-audit` pod. ## v1.9.1 + Release date: June 24, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [vd] Fixed cancellation of virtual disk storage class changes and cancellation of local disk migration. @@ -34,10 +53,13 @@ Release date: June 24, 2026. - CVE-2026-42507 ## v1.9.0 + Release date: June 10, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vm] A restart is no longer required to attach and detach virtual disks and images via the virtual machine's `.spec.blockDeviceRefs`. @@ -70,20 +92,26 @@ Release date: June 10, 2026. - [vm] System virtual machine resources (pods with `d8v-hp-` and `d8v-vm-` prefixes) now run as the `deckhouse` user, without root privileges. ## v1.8.3 + Release date: June 3, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [vm] Fixed an issue that blocked virtual machine migration for VMs with additional network interfaces. - [vm] Fixed duplicate service pods (`d8v-hp-*`) when hot-unplugging disks from VMs. ## v1.8.2 + Release date: May 20, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Security - [module] Fixed vulnerabilities: @@ -99,6 +127,7 @@ Release date: May 20, 2026. - CVE-2026-42499 ## v1.8.1 + Release date: April 28, 2026. @@ -112,6 +141,7 @@ Release date: April 28, 2026. - [observability] Fixed CPU usage calculation on the virtual machine dashboard in HA clusters, where duplicated controller metrics could affect the displayed value. ## v1.8.0 + Release date: April 22, 2026. @@ -150,10 +180,13 @@ Release date: April 22, 2026. - CVE-2026-33997 ## v1.7.2 + Release date: May 20, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Security - [module] Fixed vulnerabilities: @@ -169,10 +202,13 @@ Release date: May 20, 2026. - CVE-2026-42499 ## v1.7.1 + Release date: April 21, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [vm] To update the firmware on virtual machines with a connected USB device, one of the following actions is required. A corresponding message will appear in the virtual machine status: @@ -203,10 +239,13 @@ Release date: April 21, 2026. - CVE-2026-33186 ## v1.7.0 + Release date: March 31, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vm] The order of additional network interfaces is now deterministic and does not change after virtual machine restarts. For this to work for virtual machines created on earlier versions, they must be restarted. @@ -231,10 +270,13 @@ Release date: March 31, 2026. - [usb] Fixed USB device detection on the host: duplicate USB devices could previously appear. ## v1.6.3 + Release date: April 21, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Security - [module] Fixed vulnerabilities: @@ -255,15 +297,19 @@ Release date: April 21, 2026. - CVE-2026-33186 ## v1.6.2 + Release date: March 23, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [module] The `virtualization` module requires Deckhouse Kubernetes Platform version 1.74.2 or later. This version includes a fix for quota validation when creating disks. ## v1.6.1 + Release date: March 10, 2026. @@ -279,6 +325,7 @@ Release date: March 10, 2026. - [module] Fixed vulnerabilities CVE-2026-24051 and CVE-2025-15558. ## v1.6.0 + Release date: March 2, 2026. @@ -306,15 +353,19 @@ Release date: March 2, 2026. - [vm] Added the `--from-file` flag to the `vlctl` utility for viewing domain information from a local libvirt XML file. ## v1.5.2 + Release date: March 5, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [vd] Fixed a potential `OOMKill` during the virtual disk creation on NFS. ## v1.5.1 + Release date: February 16, 2026. @@ -324,6 +375,7 @@ Release date: February 16, 2026. - [vd] Fixed an issue with creating a virtual disk from a virtual image stored on a `PersistentVolumeClaim` (with `.spec.storage` set to `PersistentVolumeClaim`). ## v1.5.0 + Release date: February 9, 2026. @@ -349,19 +401,25 @@ Release date: February 9, 2026. - [vd] When viewing disks, the name of the virtual machine they are attached to is now displayed (`d8 k get vd`). ## v1.4.1 + Release date: February 16, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Security - [module] Fixed vulnerabilities CVE-2025-61726, CVE-2025-61728, CVE-2025-61730, and CVE-2025-68121. ## v1.4.0 + Release date: January 23, 2026. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vd] Added support for changing the StorageClass of disks attached via [VirtualMachineBlockDeviceAttachment](/modules/virtualization/cr.html#virtualmachineblockdeviceattachment) (hotplug). @@ -376,10 +434,13 @@ Release date: January 23, 2026. - [vm] Added support for cloning virtual machines in the `Running` phase via [VirtualMachineOperation](/modules/virtualization/cr.html#virtualmachineoperation) of type `Clone`. ## v1.3.0 + Release date: December 16, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vmclass] Added the `.spec.sizingPolicies.defaultCoreFraction` field to the [VirtualMachineClass](/modules/virtualization/cr.html#virtualmachineclass) resource, allowing you to set the default `coreFraction` for virtual machines that use this class. @@ -392,15 +453,19 @@ Release date: December 16, 2025. - [observability] Fixed the display of virtual machine charts in clusters running in HA mode. ## v1.2.2 + Release date: December 5, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [module] Fixed RBAC access permissions for the `d8:use:role:user` role that prevented it from managing the [VirtualMachineOperation](/modules/virtualization/cr.html#virtualmachineoperation) resource. ## v1.2.1 + Release date: December 4, 2025. @@ -410,10 +475,13 @@ Release date: December 4, 2025. - [module] The deprecated part of the configuration has been removed, which could have prevented the virtualization module from upgrading in clusters running Kubernetes version 1.34 and above. ## v1.2.0 + Release date: November 28, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vmrestore] The [VirtualMachineRestore](/modules/virtualization/cr.html#virtualmachinerestore) resource is deprecated. Use the following resources instead: @@ -448,10 +516,13 @@ Release date: November 28, 2025. - [module] Fixed vulnerability CVE-2025-64324. ## v1.1.3 + Release date: November 21, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Security - [module] Fixed CVE-2025-64324, CVE-2025-64435, CVE-2025-64436, CVE-2025-58183, CVE-2025-58186, CVE-2025-58187, CVE-2025-58188, CVE-2025-52565, CVE-2025-52881, CVE-2025-31133. @@ -461,10 +532,13 @@ Release date: November 21, 2025. - [observability] The virtual machine overview dashboards (`Namespace / Virtual Machine` and `Namespace / Virtual Machines`) have been improved: in addition to the cluster level, they are now also available at the project level. ## v1.1.2 + Release date: November 5, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [vd] Fixed live disk migration between StorageClasses that use different drivers. Restrictions: @@ -472,10 +546,13 @@ Release date: November 5, 2025. - [vm] In the `Migrating` state, detailed error information is now displayed when a live migration of a virtual machine fails. ## v1.1.1 + Release date: October 16, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Fixes - [core] Fixed an issue in the containerd v2 where storage providing a PVC with the `Filesystem` type was incorrectly attached via [VirtualMachineBlockDeviceAttachment](/modules/virtualization/cr.html#virtualmachineblockdeviceattachment). @@ -495,10 +572,13 @@ Release date: October 16, 2025. - [module] Fixed vulnerabilities CVE-2025-58058 and CVE-2025-54410. ## v1.1.0 + Release date: October 6, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vm] Added the ability to migrate VMs using disks on local storage. Restrictions: @@ -525,10 +605,13 @@ Release date: October 6, 2025. - [observability] Fixed the graph on the virtual machine dashboard that displays memory copy statistics during VM migration. ## v1.0.0 + Release date: September 11, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### New features - [vm] Added protection to prevent a cloud image ([VirtualImage](/modules/virtualization/cr.html#virtualimage) \ [ClusterVirtualImage](/modules/virtualization/cr.html#clustervirtualimage)) from being connected as the first disk. Previously, this caused the VM to fail to start with the "No bootable device" error. @@ -547,10 +630,13 @@ Release date: September 11, 2025. - Fixed CVE-2025-47907. ## v0.25.0 + Release date: August 29, 2025. +**Note:** During the upgrade to this version, running virtual machines will be automatically migrated to update their firmware version. + ### Important notes before update In version v0.25.0, support for the module's operation with CRI containerd v2 has been added. diff --git a/docs/RELEASE_NOTES.ru.md b/docs/RELEASE_NOTES.ru.md index b083b61006..31a391c3b3 100644 --- a/docs/RELEASE_NOTES.ru.md +++ b/docs/RELEASE_NOTES.ru.md @@ -3,11 +3,27 @@ title: "Релизы" weight: 70 --- +## v1.9.3 + + +Дата релиза: 7 июля 2026. + + +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + +### Исправления + +- [module] Исправлено медленное скачивание образов из DVCR на узел для подключения к виртуальной машине. +- [vm] Исправлена утечка монтирований томов, из-за которой ВМ при наличии подключённых на лету образов (hotplug) могла зависнуть при удалении в состоянии Terminating. + ## v1.9.2 + Дата релиза: 1 июля 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [vm] Исправлено снижение скорости живой миграции работающих виртуальных машин по сравнению с v1.8.3. @@ -15,10 +31,13 @@ weight: 70 - [observability] Исправлена потеря событий аудита и ложные срабатывания алерта `D8LogShipperDestinationErrors` при ротации сертификатов пода `virtualization-audit`. ## v1.9.1 + Дата релиза: 24 июня 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [vd] Исправлена отмена изменения класса хранения виртуальных дисков и отмена миграции локальных дисков. @@ -34,10 +53,13 @@ weight: 70 - CVE-2026-42507 ## v1.9.0 + Дата релиза: 10 июня 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vm] Для подключения и отключения виртуальных дисков и образов через спецификацию виртуальной машины `.spec.blockDeviceRefs` перезагрузка ВМ больше не требуется. @@ -70,20 +92,26 @@ weight: 70 - [vm] Системные ресурсы виртуальной машины (поды с префиксами `d8v-hp-` и `d8v-vm-`) теперь работают от пользователя `deckhouse`, без root-прав. ## v1.8.3 + Дата релиза: 3 июня 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [vm] Исправили проблему блокировки миграции ВМ с дополнительными интерфейсами. - [vm] Исправили проблему с дублированием служебных подов (`d8v-hp-*`) при отключении (hot unplug) дисков ВМ. ## v1.8.2 + Дата релиза: 20 мая 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Безопасность - [module] Исправлены уязвимости: @@ -99,6 +127,7 @@ weight: 70 - CVE-2026-42499 ## v1.8.1 + Дата релиза: 28 апреля 2026. @@ -112,6 +141,7 @@ weight: 70 - [observability] Исправлен расчет загрузки CPU на дашборде виртуальной машины в HA-кластерах: дублирующиеся метрики контроллера больше не влияют на отображаемое значение. ## v1.8.0 + Дата релиза: 22 апреля 2026. @@ -150,10 +180,13 @@ weight: 70 - CVE-2026-33997 ## v1.7.2 + Дата релиза: 20 мая 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Безопасность - [module] Исправлены уязвимости: @@ -169,10 +202,13 @@ weight: 70 - CVE-2026-42499 ## v1.7.1 + Дата релиза: 21 апреля 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [vm] Для обновления ПО виртуальных машин с подключенным USB-устройством требуется выполнить одно из следующих действий. Соответствующее сообщение появится в статусе виртуальной машины: @@ -203,10 +239,13 @@ weight: 70 - CVE-2026-33186 ## v1.7.0 + Дата релиза: 31 марта 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vm] Порядок дополнительных сетевых интерфейсов теперь детерминирован и не меняется при рестартах виртуальных машин. Чтобы это работало на виртуальных машинах, созданных на ранних версиях, их необходимо перезапустить. @@ -231,10 +270,13 @@ weight: 70 - [usb] Исправлено обнаружение USB-устройств на хосте: ранее могли появляться дубликаты USB-устройств. ## v1.6.3 + Дата релиза: 21 апреля 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Безопасность - [module] Исправлены уязвимости: @@ -255,15 +297,19 @@ weight: 70 - CVE-2026-33186 ## v1.6.2 + Дата релиза: 23 марта 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [module] Для работы модуля `virtualization` требуется Deckhouse Kubernetes Platform версии не ниже 1.74.2. В этой версии исправлена валидация квот при создании дисков. ## v1.6.1 + Дата релиза: 10 марта 2026. @@ -279,6 +325,7 @@ weight: 70 - [module] Исправлены уязвимости CVE-2026-24051, CVE-2025-15558. ## v1.6.0 + Дата релиза: 2 марта 2026. @@ -306,15 +353,19 @@ weight: 70 - [vm] Для утилиты `vlctl` добавлен флаг `--from-file` для просмотра информации о домене из локального libvirt XML-файла. ## v1.5.2 + Дата релиза: 5 марта 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [vd] Исправлен возможный `OOMKill` при создании виртуального диска на NFS. ## v1.5.1 + Дата релиза: 16 февраля 2026. @@ -324,6 +375,7 @@ weight: 70 - [vd] Исправлена проблема при создании виртуального диска из виртуального образа, хранящегося на `PersistentVolumeClaim` (при значении `.spec.storage=PersistentVolumeClaim`). ## v1.5.0 + Дата релиза: 9 февраля 2026. @@ -349,19 +401,25 @@ weight: 70 - [vd] При просмотре дисков теперь отображается имя виртуальной машины, к которой они подключены (`d8 k get vd`). ## v1.4.1 + Дата релиза: 16 февраля 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Безопасность - [module] Исправлены уязвимости CVE-2025-61726, CVE-2025-61728, CVE-2025-61730 и CVE-2025-68121. ## v1.4.0 + Дата релиза: 23 января 2026. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vd] Добавлена поддержка изменения StorageClass для дисков, подключённых через [VirtualMachineBlockDeviceAttachment](/modules/virtualization/cr.html#virtualmachineblockdeviceattachment) (hotplug). @@ -376,10 +434,13 @@ weight: 70 - [vm] Добавлена поддержка клонирования виртуальных машин в состоянии `Running` через [VirtualMachineOperation](/modules/virtualization/cr.html#virtualmachineoperation) с типом `Clone`. ## v1.3.0 + Дата релиза: 16 декабря 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vmclass] В ресурсе [VirtualMachineClass](/modules/virtualization/cr.html#virtualmachineclass) добавлено поле `.spec.sizingPolicies.defaultCoreFraction`, позволяющее задать значение `coreFraction` по умолчанию для виртуальных машин, использующих этот класс. @@ -392,15 +453,19 @@ weight: 70 - [observability] В кластерах, работающих в HA режиме, исправлено отображение графиков по виртуальным машинам. ## v1.2.2 + Дата релиза: 5 декабря 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [module] Для роли `d8:use:role:user` исправлены права доступа RBAC, которые не позволяли управлять ресурсом [VirtualMachineOperation](/modules/virtualization/cr.html#virtualmachineoperation). ## v1.2.1 + Дата релиза: 4 декабря 2025. @@ -410,10 +475,13 @@ weight: 70 - [module] Удалена устаревшая часть конфигурации, из‑за которой обновление модуля виртуализации могло не выполняться в кластерах с Kubernetes версии 1.34 и выше. ## v1.2.0 + Дата релиза: 28 ноября 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vmrestore] Ресурс [VirtualMachineRestore](/modules/virtualization/cr.html#virtualmachinerestore) помечен как устаревший (deprecated). Вместо него используйте следующие ресурсы: @@ -448,10 +516,13 @@ weight: 70 - [module] Исправлена уязвимость CVE-2025-64324. ## v1.1.3 + Дата релиза: 21 ноября 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Безопасность - [module] Исправлены уязвимости CVE-2025-64324, CVE-2025-64435, CVE-2025-64436, CVE-2025-58183, CVE-2025-58186, CVE-2025-58187, CVE-2025-58188, CVE-2025-52565, CVE-2025-52881, CVE-2025-31133. @@ -461,10 +532,13 @@ weight: 70 - [observability] Доработаны дашборды обзора виртуальных машин (`Namespace / Virtual Machine` и `Namespace / Virtual Machines`): помимо уровня кластера, они теперь доступны и на уровне проекта. ## v1.1.2 + Дата релиза: 5 ноября 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [vd] Исправлена живая миграция дисков между StorageClass, использующими разные драйверы. Ограничения: @@ -472,10 +546,13 @@ weight: 70 - [vm] В состоянии `Migrating` при неуспешной живой миграции виртуальной машины добавлено отображение подробной информации об ошибке. ## v1.1.1 + Дата релиза: 16 октября 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Исправления - [core] Исправлена проблема в containerd v2, из-за которой хранилище, предоставляющее PVC с типом `FileSystem`, некорректно подключалось через [VirtualMachineBlockDeviceAttachment](/modules/virtualization/cr.html#virtualmachineblockdeviceattachment). @@ -495,10 +572,13 @@ weight: 70 - [module] Исправлены уязвимости CVE-2025-58058 и CVE-2025-54410. ## v1.1.0 + Дата релиза: 6 октября 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vm] Добавлена возможность миграции ВМ, использующих диски на локальных хранилищах. Ограничения: @@ -525,10 +605,13 @@ weight: 70 - [observability] На дашборде виртуальной машины исправлен график, отображающий статистику копирования памяти во время миграции ВМ. ## v1.0.0 + Дата релиза: 11 сентября 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Новые возможности - [vm] Добавлена защита от подключения cloud-образа ([VirtualImage](/modules/virtualization/cr.html#virtualimage) \ [ClusterVirtualImage](/modules/virtualization/cr.html#clustervirtualimage)) в качестве первого диска. Ранее это приводило к невозможности запуска ВМ с ошибкой "No bootable device". @@ -547,10 +630,13 @@ weight: 70 - Устранено CVE-2025-47907. ## v0.25.0 + Дата релиза: 29 августа 2025. +**Обратите внимание:** в процессе обновления до этой версии работающие виртуальные машины будут автоматически мигрированы для обновления версии прошивки. + ### Важная информация перед обновлением В версии v0.25.0 добавлена поддержка работы модуля с CRI containerd v2. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index ae584d9d62..48eadac4d5 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -1282,6 +1282,17 @@ If the VM uses 2 cores, it falls in the range of 1-4 cores. Then memory can be s In addition to VM sizing, the policy also allows you to implement the desired maximum oversubscription for VMs. For example, by specifying `coreFraction: 20%` in the policy, you guarantee any VM at least 20% of the CPU compute resources, which would effectively define a maximum possible oversubscription of 5:1. +If you try to create or update a VM whose configuration violates the sizing policy, the request is rejected with a message that names the parameter to change and the values to use. Each message about a specific policy also ends with the hint `check the sizing policy of the VirtualMachineClass or contact the administrator for more information` (omitted below for brevity). Examples for a class `supercpu` with the policy above: + +- Cores outside all ranges (`cores: 10`): `does not match any sizing policy of VirtualMachineClass "supercpu": its 10 CPU core(s) fall outside the allowed ranges (1-4, 5-8); set the number of cores (spec.cpu.cores) accordingly` +- Core fraction not allowed (`cores: 2`, `coreFraction: 30%`): `the CPU core fraction "30%" is not allowed; set the core fraction (spec.cpu.coreFraction) to one of: 5%, 10%, 20%, 50%, 100%` +- Memory out of range (`cores: 2`, `size: 16Gi`): `the memory size (16Gi) is out of the range allowed by the sizing policy; set the memory size (spec.memory.size) between 1Gi and 8Gi` +- Cores not on the step grid (`cores.step`): `the number of CPU cores (7) does not match the sizing policy step; set the number of cores (spec.cpu.cores) to 6 or 8` +- Memory not on the step grid (`memory.step`): `the memory size (1536Mi) does not match the sizing policy step; set the memory size (spec.memory.size) to 1Gi or 2Gi` +- Per-core memory out of range (`memory.perCore`): `the memory size (18Gi) is not allowed for 6 CPU core(s); set the memory size (spec.memory.size) between 6Gi and 12Gi, or change the number of cores (spec.cpu.cores) (the sizing policy allows between 1Gi and 2Gi of memory per core)` +- Per-core memory not on the step grid: `the memory size (2560Mi) does not match the per-core sizing policy step for 2 CPU core(s); set the memory size (spec.memory.size) to 2Gi or 4Gi, or change the number of cores (spec.cpu.cores)` +- Several violations at once: all reasons are listed in a single message under `does not match the sizing policy of VirtualMachineClass "supercpu" for several reasons:`. + ### Automatic CPU topology configuration The CPU topology of a virtual machine (VM) determines how the CPU cores are allocated across sockets. This is important to ensure optimal performance and compatibility with applications that may depend on the CPU configuration. In the VM configuration, you specify only the total number of processor cores, and the topology (the number of sockets and cores in each socket) is automatically calculated based on this value. @@ -3003,6 +3014,235 @@ The command cannot output data directly to the terminal. You must redirect the o After executing the command, you will receive a `debug-info.tar.gz` archive that contains all collected data in YAML format (for resources) and text files (for logs). This archive can be sent to technical support for problem analysis. +## Virtual machine pools + +{{< alert level="warning" >}} +Available in the EE and SE+ editions. Requires the `VirtualMachinePool` feature gate. +{{< /alert >}} + +The [VirtualMachinePool](cr.html#virtualmachinepool) resource maintains a requested number of identical virtual machines and lets you scale them via the `scale` subresource, a HorizontalPodAutoscaler (HPA), or KEDA. Its `virtualMachineTemplate.spec` is an ordinary `VirtualMachineSpec`, so a replica is no different from a manually created virtual machine. + +This functionality is disabled by default. To enable it, add `VirtualMachinePool` to the `.spec.settings.featureGates` array in the ModuleConfig `virtualization`: + +```yaml +kind: ModuleConfig +metadata: + name: virtualization +spec: + settings: + featureGates: + - VirtualMachinePool +``` + +Create a pool with the desired number of replicas and a template. Each per-replica disk is described once in `virtualDiskTemplates` (reclaim policy, size, data source), and the template's `blockDeviceRefs` references those disks — by name, with `kind: VirtualDisk` — to set the device (boot) order, exactly as in a plain `VirtualMachine`. Every `virtualDiskTemplates` entry must be referenced exactly once (admission enforces this bijection; disk-template names are unique). Alongside the per-replica disks you may list shared read-only images (`VirtualImage`/`ClusterVirtualImage`) — for example a common ISO/CD-ROM attached to every replica — they are not per-replica and need no `virtualDiskTemplates` entry. + +```bash +d8 k apply -f - <-`. Disks follow the same scheme: a per-replica (`Delete`) disk is named `-