diff --git a/01_install_requirements.sh b/01_install_requirements.sh index e85c59d19..c020c76da 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -25,6 +25,22 @@ if [ -z "${METAL3_DEV_ENV:-}" ]; then # Patch metal3-dev-env to use Ansible 8.x on centos9/rhel9. sed -i '/ANSIBLE_VERSION/{ s/10\.7\.0/8.7.0/; }' lib/common.sh + # Go tarball defaults hardcode linux-amd64; use GOARCH passed as extra var. + # Upstream fix: https://github.com/metal3-io/metal3-dev-env/pull/1694 + GO_DEFAULTS="vm-setup/roles/packages_installation/defaults/main.yml" + if grep -q 'linux-amd64' "${GO_DEFAULTS}"; then + sed -i 's/go_tarball: "go{{ go_version }}.linux-amd64.tar.gz"/go_tarball: "go{{ go_version }}.linux-{{ GOARCH | default('\''amd64'\'') }}.tar.gz"/' \ + "${GO_DEFAULTS}" + fi + + # RHEL 10: genisoimage replaced by xorriso (provides mkisofs), + # python3-bcrypt not in base repos (soft dep of passlib, not required). + if [[ "${DISTRO}" =~ ^(rhel10|centos10) ]]; then + PKG_DEFAULTS="vm-setup/roles/packages_installation/defaults/main.yml" + sed -i '/^ - genisoimage$/d' "${PKG_DEFAULTS}" + sed -i '/^ - python3-bcrypt$/d' "${PKG_DEFAULTS}" + fi + popd fi @@ -108,8 +124,16 @@ case $DISTRO in sudo ln -s /usr/bin/python3 /usr/bin/python || true PYTHON_DEVEL="python3-devel" ;; + "rhel10"|"centos10") + sudo dnf -y install python3-pip + if sudo subscription-manager identity > /dev/null 2>&1; then + sudo subscription-manager repos --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || true + fi + sudo ln -s /usr/bin/python3 /usr/bin/python || true + PYTHON_DEVEL="python3-devel" + ;; *) - echo -n "CentOS 9 or RHEL 9 required (el8 is no longer supported due to glibc requirements)" + echo -n "CentOS 9+, RHEL 9+, or compatible required (el8 is no longer supported due to glibc requirements)" exit 1 ;; esac diff --git a/02_configure_host.sh b/02_configure_host.sh index f6bcb1a97..14c5551a2 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -86,6 +86,20 @@ if [[ "${NODES_PLATFORM}" == "baremetal" ]] && [ -n "$NODES_FILE" ] ; then set -x fi +if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + sudo systemctl enable --now firewalld + configure_chronyd + + sudo firewall-cmd --zone=public --add-port=80/tcp --add-port=53/tcp --add-port=53/udp \ + --add-port=8000/tcp --add-port="${LOCAL_REGISTRY_PORT}"/tcp \ + --add-port="${INSTALLER_PROXY_PORT}"/tcp --add-port="${AGENT_BOOT_SERVER_PORT}"/tcp \ + --add-port=3260/tcp + + switch_to_internal_dns + + exit 0 +fi + # TODO - move this to metal3-dev-env. # This is to address the following error: # "msg": "internal error: Check the host setup: enabling IPv6 forwarding with RA routes without accept_ra set to 2 is likely to cause routes loss. Interfaces to look at: eno2" diff --git a/agent/05_agent_configure.sh b/agent/05_agent_configure.sh index e3ea74628..2c839ab28 100755 --- a/agent/05_agent_configure.sh +++ b/agent/05_agent_configure.sh @@ -182,6 +182,51 @@ function get_static_ips_and_macs() { fi } +function get_baremetal_ips_and_macs() { + AGENT_NODES_IPS=() + AGENT_NODES_IPSV6=() + AGENT_NODES_MACS=() + AGENT_NODES_HOSTNAMES=() + AGENT_MASTER_HOSTNAMES=() + AGENT_EXTRA_WORKERS_IPS=() + AGENT_EXTRA_WORKERS_IPSV6=() + AGENT_EXTRA_WORKERS_MACS=() + AGENT_EXTRA_WORKERS_HOSTNAMES=() + + IFS=',' read -ra ips <<< "${BAREMETAL_IPS}" + local total_nodes + total_nodes=$(jq '.nodes | length' "$NODES_FILE") + + local -a data_macs=() + if [[ -n "${BAREMETAL_MACS:-}" ]]; then + IFS=',' read -ra data_macs <<< "${BAREMETAL_MACS}" + fi + + for (( i=0; i < total_nodes; i++ )); do + local mac hostname ip + if [[ ${#data_macs[@]} -gt 0 ]]; then + mac="${data_macs[$i]}" + else + mac=$(node_val "$i" "ports[0].address") + fi + hostname=$(node_val "$i" "name") + ip="${ips[$i]}" + + add_ip_host_entry "$ip" "$hostname" + + if (( i < NUM_MASTERS )); then + AGENT_NODES_IPS+=("$ip") + AGENT_NODES_MACS+=("$mac") + AGENT_NODES_HOSTNAMES+=("$hostname") + AGENT_MASTER_HOSTNAMES+=("$hostname") + else + AGENT_EXTRA_WORKERS_IPS+=("$ip") + AGENT_EXTRA_WORKERS_MACS+=("$mac") + AGENT_EXTRA_WORKERS_HOSTNAMES+=("$hostname") + fi + done +} + function generate_extra_cluster_manifests() { mkdir -p "${EXTRA_MANIFESTS_PATH}" @@ -653,7 +698,15 @@ write_pull_secret # This is temporary and will go away when https://github.com/nmstate/nmstate is used sudo yum install -y nmstate -get_static_ips_and_macs +if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + # When 02_configure_host.sh is skipped (no libvirt), the NM dnsmasq + # plugin hasn't been activated. Enable it so configure_dnsmasq entries + # actually resolve. + switch_to_internal_dns + get_baremetal_ips_and_macs +else + get_static_ips_and_macs +fi get_nodes_bmc_info @@ -700,8 +753,15 @@ else # For ISO_NO_REGISTRY mode with workers, add api-int to libvirt DNS # Workers need to resolve api-int during bootstrap before joining the cluster if [[ "${NUM_MASTERS}" -gt "1" ]]; then - add_dns_entry "${API_VIPS%"${VIPS_SEPARATOR}"*}" "api-int" + if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + add_dnsmasq_multi_entry "apiintvip" "${API_VIPS}" + sudo systemctl reload NetworkManager + else + add_dns_entry "${API_VIPS%"${VIPS_SEPARATOR}"*}" "api-int" + fi fi fi -enable_isolated_baremetal_network +if [[ "${NODES_PLATFORM}" != "baremetal" ]]; then + enable_isolated_baremetal_network +fi diff --git a/agent/06_agent_create_cluster.sh b/agent/06_agent_create_cluster.sh index 8566ec575..9422dd27a 100755 --- a/agent/06_agent_create_cluster.sh +++ b/agent/06_agent_create_cluster.sh @@ -163,6 +163,111 @@ function attach_agent_iso() { } +function redfish_curl() { + local user="$1" password="$2" url="$3" + shift 3 + curl -s -u "${user}":"${password}" -k -H 'Content-Type: application/json' "$url" "$@" +} + +function mount_agent_iso_baremetal() { + local iso_url="${BAREMETAL_ISO_SERVER}" + local total_nodes + total_nodes=$(jq '.nodes | length' "$NODES_FILE") + + for (( i=0; i < total_nodes; i++ )); do + local address user password scheme system systemurl bmc_base + address=$(jq -r ".nodes[$i].driver_info.address" "$NODES_FILE") + user=$(jq -r ".nodes[$i].driver_info.username" "$NODES_FILE") + password=$(jq -r ".nodes[$i].driver_info.password" "$NODES_FILE") + local node_name + node_name=$(jq -r ".nodes[$i].name" "$NODES_FILE") + + if [[ ! $address =~ ^(redfish.*://)(.*)$ ]]; then + echo "ERROR: Unsupported BMC protocol for node $i ($address) — Redfish required" + exit 1 + fi + scheme="https://" + system="${BASH_REMATCH[2]}" + if [[ ${BASH_REMATCH[1]} =~ http: ]]; then + scheme="http://" + fi + systemurl="${scheme}${system}" + bmc_base="${scheme}${system%%/*}" + + local manager_id vmedia_uri cd_slot + manager_id=$(redfish_curl "$user" "$password" "${bmc_base}/redfish/v1/Managers" | jq -r '.Members[0]."@odata.id"') + vmedia_uri="${bmc_base}${manager_id}/VirtualMedia" + cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[] | select(."@odata.id" | test("[Cc][Dd]|[Dd][Vv][Dd]|2")) | ."@odata.id"' | head -1) + + if [[ -z "$cd_slot" ]]; then + cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[1]."@odata.id"') + fi + + local current_image + current_image=$(redfish_curl "$user" "$password" "${bmc_base}${cd_slot}" | jq -r '.Inserted // false') + if [[ "$current_image" == "true" ]]; then + echo "Ejecting existing VirtualMedia from ${node_name}..." + redfish_curl "$user" "$password" "${bmc_base}${cd_slot}/Actions/VirtualMedia.EjectMedia" \ + -d '{}' + sleep 2 + fi + + echo "Mounting ISO on ${node_name} via ${cd_slot}..." + redfish_curl "$user" "$password" "${bmc_base}${cd_slot}/Actions/VirtualMedia.InsertMedia" \ + -d "{\"Image\": \"${iso_url}\", \"Inserted\": true, \"WriteProtected\": true}" + + local inserted + inserted=$(redfish_curl "$user" "$password" "${bmc_base}${cd_slot}" | jq -r '.Inserted') + if [[ "$inserted" != "true" ]]; then + echo "WARNING: VirtualMedia reports Inserted=${inserted} for ${node_name}" + fi + + redfish_curl "$user" "$password" "$systemurl" -X PATCH \ + -d '{"Boot": {"BootSourceOverrideTarget": "Cd", "BootSourceOverrideEnabled": "Once"}}' + + redfish_curl "$user" "$password" "$systemurl/Actions/ComputerSystem.Reset" \ + -d '{"ResetType": "ForceRestart"}' + + echo "Node ${node_name} booting from ISO" + done +} + +function eject_agent_iso_baremetal() { + local total_nodes + total_nodes=$(jq '.nodes | length' "$NODES_FILE") + + for (( i=0; i < total_nodes; i++ )); do + local address user password scheme system bmc_base + address=$(jq -r ".nodes[$i].driver_info.address" "$NODES_FILE") + user=$(jq -r ".nodes[$i].driver_info.username" "$NODES_FILE") + password=$(jq -r ".nodes[$i].driver_info.password" "$NODES_FILE") + local node_name + node_name=$(jq -r ".nodes[$i].name" "$NODES_FILE") + + if [[ ! $address =~ ^(redfish.*://)(.*)$ ]]; then + continue + fi + scheme="https://" + system="${BASH_REMATCH[2]}" + if [[ ${BASH_REMATCH[1]} =~ http: ]]; then + scheme="http://" + fi + bmc_base="${scheme}${system%%/*}" + + local manager_id vmedia_uri cd_slot + manager_id=$(redfish_curl "$user" "$password" "${bmc_base}/redfish/v1/Managers" | jq -r '.Members[0]."@odata.id"') + vmedia_uri="${bmc_base}${manager_id}/VirtualMedia" + cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[] | select(."@odata.id" | test("[Cc][Dd]|[Dd][Vv][Dd]|2")) | ."@odata.id"' | head -1) + + if [[ -z "$cd_slot" ]]; then + cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[1]."@odata.id"') + fi + + echo "Ejecting ISO from ${node_name}..." + redfish_curl "$user" "$password" "${bmc_base}${cd_slot}/Actions/VirtualMedia.EjectMedia" -d '{}' + done +} + function attach_appliance_diskimage() { set_file_acl @@ -305,6 +410,10 @@ function wait_for_hosts_installed() { } function get_node0_ip() { + if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + echo "${BAREMETAL_IPS%%,*}" + return + fi # shellcheck disable=SC2059 node0_name=$(printf "${MASTER_HOSTNAME_FORMAT}" 0) node0_ip=$(sudo virsh net-dumpxml ostestbm | xmllint --xpath "string(//dns[*]/host/hostname[. = '${node0_name}']/../@ip)" -) @@ -617,16 +726,29 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in fi fi - attach_agent_iso master "$NUM_MASTERS" - attach_agent_iso worker "$NUM_WORKERS" - attach_agent_iso arbiter "$NUM_ARBITERS" - - if [[ "${ARCH}" == "aarch64" ]]; then - wait_for_hosts_installed || echo "WARNING: proceeding with CDROM eject despite host status issues" - eject_agent_iso master "$NUM_MASTERS" - eject_agent_iso worker "$NUM_WORKERS" - eject_agent_iso arbiter "$NUM_ARBITERS" - echo "aarch64: CDROM media ejected from all VMs" + if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + # Stage the ISO where BAREMETAL_ISO_SERVER can serve it. The ISO is + # generated under OCP_DIR (relative to dev-scripts), but the HTTP + # server root may be elsewhere (e.g. WORKING_DIR under nginx). + iso_file="${OCP_DIR}/agent.$(uname -m).iso" + if [[ -f "$iso_file" ]]; then + serve_dir="${WORKING_DIR}/${CLUSTER_NAME}" + mkdir -p "$serve_dir" + ln -sf "$(realpath "$iso_file")" "${serve_dir}/agent.$(uname -m).iso" + fi + mount_agent_iso_baremetal + else + attach_agent_iso master "$NUM_MASTERS" + attach_agent_iso worker "$NUM_WORKERS" + attach_agent_iso arbiter "$NUM_ARBITERS" + + if [[ "${ARCH}" == "aarch64" ]]; then + wait_for_hosts_installed || echo "WARNING: proceeding with CDROM eject despite host status issues" + eject_agent_iso master "$NUM_MASTERS" + eject_agent_iso worker "$NUM_WORKERS" + eject_agent_iso arbiter "$NUM_ARBITERS" + echo "aarch64: CDROM media ejected from all VMs" + fi fi ;; @@ -759,6 +881,10 @@ fi wait_for_cluster_ready +if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + eject_agent_iso_baremetal +fi + if [ ! -z "${AGENT_DEPLOY_MCE}" ]; then mce_complete_deployment fi diff --git a/agent/common.sh b/agent/common.sh index a0421a933..739cdf6fa 100644 --- a/agent/common.sh +++ b/agent/common.sh @@ -100,6 +100,10 @@ if [ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ] ; then fi function getRendezvousIP() { + if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then + echo "${BAREMETAL_IPS%%,*}" + return + fi node_zero_mac_address=$(sudo virsh domiflist "${AGENT_RENDEZVOUS_NODE_HOSTNAME}" | awk '$3 == "ostestbm" {print $5}') # ip neigh can return multiple addresses (IPv4, link-local IPv6, global IPv6) # on separate lines. Filter to a single appropriate address. diff --git a/config_example.sh b/config_example.sh index f9dbd4b8f..60466b872 100755 --- a/config_example.sh +++ b/config_example.sh @@ -480,6 +480,37 @@ set -x # #export NODES_PLATFORM=baremetal +# --- Baremetal ABI (Agent-Based Installation) variables --- +# These are used when NODES_PLATFORM=baremetal with the agent installer (make agent). +# All BAREMETAL_* vars except BAREMETAL_ISO_SERVER are auto-generated by +# the adoption tool (make baremetal-adopt) into config_baremetal_fencing.sh. + +# BAREMETAL_IPS - +# Comma-separated node IPs, order matches NODES_FILE entries. +# Required when NODES_PLATFORM=baremetal. +# +#export BAREMETAL_IPS="10.1.155.10,10.1.155.11" + +# BAREMETAL_API_VIP - +# API virtual IP for dnsmasq DNS on the provisioning host. +# Required when NODES_PLATFORM=baremetal. +# +#export BAREMETAL_API_VIP="10.1.155.100" + +# BAREMETAL_INGRESS_VIP - +# Ingress virtual IP. Defaults to BAREMETAL_API_VIP if unset +# (common for 2-node TNF where API and ingress share a VIP). +# +#export BAREMETAL_INGRESS_VIP="10.1.155.101" + +# BAREMETAL_ISO_SERVER - +# Full HTTP URL where the agent ISO is staged for Redfish VirtualMedia boot. +# This is the only baremetal var NOT auto-generated by adoption — the user +# must provide an HTTP server reachable from the BMCs. +# Required when NODES_PLATFORM=baremetal with agent installer. +# +#export BAREMETAL_ISO_SERVER="http://10.1.235.49:8080/agent.x86_64.iso" + # ENABLE_WORKLOAD_PARTITIONING - # Set to any non zero length string value to enable workload partitioning in the install config. # diff --git a/network.sh b/network.sh index 5f8f94c6b..97204e389 100755 --- a/network.sh +++ b/network.sh @@ -337,6 +337,10 @@ function set_api_and_ingress_vip() { if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then get_vips configure_dnsmasq "${API_VIPS}" "${INGRESS_VIPS}" + elif [ "${NODES_PLATFORM}" == "baremetal" ] && [ -n "${BAREMETAL_API_VIP:-}" ]; then + API_VIPS="${BAREMETAL_API_VIP}" + INGRESS_VIPS="${BAREMETAL_INGRESS_VIP:-${BAREMETAL_API_VIP}}" + configure_dnsmasq "${API_VIPS}" "${INGRESS_VIPS}" else # Specific for users *NOT* using devscript with KVM (virsh) for deploy. (Reads: baremetal) if [[ -z "${EXTERNAL_SUBNET_V4}" ]]; then