horazont
commented
1 year ago I tested this in a modified yaook/k8s cluster. For future reference, here is a non-exhaustive list of things I did:
managed-k8s# git diff HEAD
```patch
diff --git a/k8s-base/roles/ch-k8s-lbaas-controller/templates/controller-config.toml b/k8s-base/roles/ch-k8s-lbaas-controller/templates/controller-config.toml
index ef111d82..3b33e252 100644
--- a/k8s-base/roles/ch-k8s-lbaas-controller/templates/controller-config.toml
+++ b/k8s-base/roles/ch-k8s-lbaas-controller/templates/controller-config.toml
@@ -3,7 +3,9 @@ backend-layer='Pod'
{% else %}
backend-layer='NodePort'
{% endif %}
+port-manager='{{ ch_k8s_lbaas_port_manager }}'
+{% if ch_k8s_lbaas_port_manager == "openstack" %}
[openstack.auth]
region={{ lookup('env', 'OS_REGION_NAME') | to_json }}
username={{ lookup('env','OS_USERNAME') | to_json }}
@@ -20,12 +22,27 @@ application-credential-secret={{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET
use-floating-ips={{ ch_k8s_lbaas_use_floating_ips | ternary('true', 'false') }}
floating-ip-network-id={{ ch_k8s_lbaas_floating_ip_network_id | to_json }}
subnet-id={{ ch_k8s_lbaas_subnet_id | to_json }}
+{% elif ch_k8s_lbaas_port_manager == "static" %}
+[static]
+ipv4-addresses = [
+{% for addr in ch_k8s_lbaas_static_ipv4_addresses %}
+ {{ addr | to_json }},
+{% endfor %}
+]
+{% endif %}
[agents]
shared-secret={{ ch_k8s_lbaas_shared_secret | to_json }}
token-lifetime=60
+{% if ch_k8s_lbaas_port_manager == "openstack" %}
{% for node in groups['frontend'] %}
[[agents.agent]]
url="http://{{ hostvars[node]['local_ipv4_address'] }}:{{ ch_k8s_lbaas_agent_port }}"
{% endfor %}
+{% else %}
+{% for url in ch_k8s_lbaas_agent_urls %}
+[[agents.agent]]
+url={{ url | to_json }}
+{% endfor %}
+{% endif %}
diff --git a/templates/config.template.toml b/templates/config.template.toml
index c414c82a..10b1d440 100644
--- a/templates/config.template.toml
+++ b/templates/config.template.toml
@@ -83,6 +83,24 @@ shared_secret = "..." # REPLACE ME
version = "0.5.0"
agent_port = 15203
+# Configure which IP address ("port") manager to use. Two options are available:
+# - openstack: Uses OpenStack and the yaook/k8s gateway nodes to provision
+# LBaaS IP addresses ports.
+# - static: Uses a fixed set of IP addresses to use for load balancing. When the
+# static port manager is used, the `agent_urls` and `static_ipv4_addresses`
+# options must also be configured.
+#port_manager = "openstack"
+
+# List of IPv4 addresses which are usable for the static port manager. It is
+# your responsibility to ensure that the node(s) which run the agent(s) receive
+# traffic for these IPv4 addresses.
+#static_ipv4_addresses = []
+
+# Customize URLs for the agents. This will typically be a list of HTTP URLs
+# like http://agent_ip:15203. This option is only used if the port manager is
+# set to `static`, and must be set if the port manager is `static`.
+#agent_urls = []
+
# ANCHOR_END: ch-k8s-lbaas_config
# ANCHOR: kubernetes_basic_cluster_configuration
diff --git a/terraform/10-networking.tf b/terraform/10-networking.tf
index 70aa5dde..1396ae33 100644
--- a/terraform/10-networking.tf
+++ b/terraform/10-networking.tf
@@ -32,9 +32,17 @@ resource "openstack_networking_router_v2" "cluster_router" {
external_network_id = data.openstack_networking_network_v2.public_network.id
}
+resource "openstack_networking_port_v2" "cluster_router_port" {
+ name = "router-port"
+ network_id = openstack_networking_network_v2.cluster_network.id
+ fixed_ip {
+ subnet_id = openstack_networking_subnet_v2.cluster_subnet.id
+ }
+}
+
resource "openstack_networking_router_interface_v2" "cluster_router_iface" {
router_id = openstack_networking_router_v2.cluster_router.id
- subnet_id = openstack_networking_subnet_v2.cluster_subnet.id
+ port_id = openstack_networking_port_v2.cluster_router_port.id
}
resource "openstack_networking_router_interface_v2" "cluster_router_iface_v6" {
diff --git a/terraform/20-gateway.tf b/terraform/20-fws.tf
similarity index 98%
rename from terraform/20-gateway.tf
rename to terraform/20-fws.tf
index 06bf283c..7bad338d 100644
--- a/terraform/20-gateway.tf
+++ b/terraform/20-fws.tf
@@ -16,6 +16,7 @@ resource "openstack_networking_port_v2" "gw_vip_port" {
fixed_ip {
subnet_id = openstack_networking_subnet_v2.cluster_subnet.id
+ ip_address = openstack_networking_subnet_v2.cluster_subnet.gateway_ip
}
dynamic "fixed_ip" {
```
- Ran terraform
- Added a VyOS VM to the project using our inhouse VyOS image, cloudified via https://github.com/vyos/vyos-vm-images
- Configured VyOS to take the .1 IP address via keepalived
- Configured VyOS to route traffic to the IP of the router (which is not the .1)
- I allocated a new port for the k8s-vip
- I modified the hosts and other inventory files to set that IP address as
networking_fixed_ip - Modified the inventory file to use the masters as frontend
- Set
on_openstackanddualstack_supportto False - Remove mention of
networking_floating_ip - Rolled out stage2+3 with those modifications
- Configured BGP on VyOS to peer with one of the master nodes
- Deployed the LBaaS agent by hand with a handwritten config on VyOS, with disabled SNAT, using the .1 address as LB IP
- Deployed a LoadBalancer type service in order to check the configuration
- Curled the floating IP associated with the .1 private IP to confirm that things work
This PR adds compatibility for a VyOS bare-metal setup by:
port_managercomponent exchangeableNotes:
ToDos:
CheckPortExistsin OpenStack PortManagerFor VyOS 1.3, this agent config file would work:
And this controller config could be used:
And this sudo rule in
/etc/sudoers.d/lbaas: