guide for gateway devices with vlans and elastic addresses

[displague]

Users would benefit from a guide that demonstrates devices (count = size of address space) being connected to the gateway/vlan/addresses, with userdata that configures the vlan and an address from the range.

This guide should show the difference between a public address and managed address deployment since the latter does not need to provision elastic addresses.

_Originally posted by @displague in https://github.com/equinix/terraform-provider-metal/pull/157#discussion_r671446915_

[displague]

resource "metal_project" "test" {
    name = "tfacc-gateway-test"
}

resource "metal_vlan" "test" {
    description = "test VLAN in SV"
    metro       = "sv"
    project_id  = metal_project.test.id
}

resource "metal_reserved_ip_block" "pub_ips" {
  project_id = metal_project.test.id
  type       = "public_ipv4"
  metro      = "sv"
  quantity   = 8
}

resource "metal_gateway" "public" {
    project_id               = metal_project.test.id
    vlan_id                  = metal_vlan.test.id
    ip_reservation_id = metal_reserved_ip_block.pub_ips.id
}

resource "metal_gateway" "private" {
    project_id               = metal_project.test.id
    vlan_id                  = metal_vlan.test.id
    private_ipv4_subnet_size = 8
}

data "metal_reserved_ip_block" "priv_ips" {
  project_id = metal_project.test.id
  id = metal_gateway.private.ip_reservation_id
}

# this will not work without changes
resource "metal_device" "nodes" {
  count = 6

  # This is a big example from https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v2.html#network-config-v2
  # another example at https://docs.mirantis.com/container-cloud/latest/deployment-guide/deploy-bm-mgmt/bm-mgmt-deploy.html
  # (we would cut out most of this out, no bridges, no dhcp, just the bonds ethernet devices, addresses, DNS, and default routes - I think)
  # we would use data.metal_reserved.ip_block.priv_ips to figure out
  #  cidrhost("10.12.127.0/20", count.index+1) for the private address
  # and use metal_reserved.ip_block.pub_ips to figure out in a similar way
  # maybe we need https://www.terraform.io/docs/language/functions/cidrsubnet.html too, but I think EM attributes give us this
  userdata = <<<EOS
#cloud-config
network:
  version: 2
  ethernets:
    # opaque ID for physical interfaces, only referred to by other stanzas
    id0:
      match:
        macaddress: '00:11:22:33:44:55'
      wakeonlan: true
      dhcp4: true
      addresses:
        - 192.168.14.2/24
        - 2001:1::1/64
      gateway4: 192.168.14.1
      gateway6: 2001:1::2
      nameservers:
        search: [foo.local, bar.local]
        addresses: [8.8.8.8]
      # static routes
      routes:
        - to: 192.0.2.0/24
          via: 11.0.0.1
          metric: 3
    lom:
      match:
        driver: ixgbe
      # you are responsible for setting tight enough match rules
      # that only match one device if you use set-name
      set-name: lom1
      dhcp6: true
    switchports:
      # all cards on second PCI bus; unconfigured by themselves, will be added
      # to br0 below
      match:
        name: enp2*
      mtu: 1280
  bonds:
    bond0:
      interfaces: [id0, lom]
  bridges:
    # the key name is the name for virtual (created) interfaces; no match: and
    # set-name: allowed
    br0:
      # IDs of the components; switchports expands into multiple interfaces
      interfaces: [wlp1s0, switchports]
      dhcp4: true
  vlans:
    en-intra:
      id: 1
      link: id0
      dhcp4: yes
EOS 
}

resource "metal_device_network_type" "nodes" {
  device_id = metal_device.nodes.id
  type      = "hybrid" # technically, this still works.
  # maybe we should use layer2-bonded or individual for the example, I think
}

resource "metal_port_vlan_attachment" "test" {
  device_id = metal_device_network_type.test.id
  port_name = "bond0"
  vlan_vnid = metal_vlan.test.vxlan
}