search

28mm / blast-radius

Interactive visualizations of Terraform dependency graphs using d3.js
https://28mm.github.io/blast-radius-docs/
MIT License
2.04k stars 257 forks source link

graph.json crashing with coreos tectonic vmware terraform #23

Closed mathewusher closed 6 years ago

mathewusher commented 6 years ago

COREOS TECTONIC VERSION curl -O https://releases.tectonic.com/releases/tectonic_1.8.4-tectonic.3.zip

ERROR tf = Terraform(os.getcwd()) File "/usr/lib/python3.6/site-packages/blastradius/handlers/terraform.py", line 49, in init self.modules[name] = Terraform(directory=self.directory+'/'+source, settings=mod) File "/usr/lib/python3.6/site-packages/blastradius/handlers/terraform.py", line 26, in init self.config = hcl.load(config_io) File "/usr/lib/python3.6/site-packages/hcl/api.py", line 51, in load return loads(fp.read()) File "/usr/lib/python3.6/site-packages/hcl/api.py", line 62, in loads return HclParser().parse(s) File "/usr/lib/python3.6/site-packages/hcl/parser.py", line 307, in parse return self.yacc.parse(s, lexer=Lexer()) File "/usr/lib/python3.6/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File "/usr/lib/python3.6/site-packages/ply/yacc.py", line 1061, in parseopt_notrack lookahead = get_token() # Get the next token File "/usr/lib/python3.6/site-packages/hcl/lexer.py", line 275, in token return self.lex.token() File "/usr/lib/python3.6/site-packages/ply/lex.py", line 406, in token newtok = self.lexeoff(tok) File "/usr/lib/python3.6/site-packages/hcl/lexer.py", line 222, in t_heredoc_eof _raise_error(t, 'EOF before closing heredoc') File "/usr/lib/python3.6/site-packages/hcl/lexer.py", line 17, in _raise_error raise ValueError("Line %d, column %d, index %d: %s" % (lineno, column, lexpos, message)) ValueError: Line 438, column 1, index 12356: EOF before closing heredoc

Narrowed it down to these source objects and resources modules/ignition 'ignition_masters': {'source': '../../modules/ignition', 'base_domain': '${var.tectonic_base_domain}', 'bootstrap_upgrade_cl': '${var.tectonic_bootstrap_upgrade_cl}', 'cluster_name': '${var.tectonic_cluster_name}', 'container_images': '${var.tectonic_container_images}', 'etcd_advertise_name_list': '${data.template_file.etcd_hostname_list..rendered}', 'etcd_count': '${length(data.template_file.etcd_hostname_list..rendered)}', 'etcd_initial_cluster_list': '${data.template_file.etcd_hostname_list.*.rendered}', 'etcd_tls_enabled': '${var.tectonic_etcd_tls_enabled}', 'image_re': '${var.tectonic_image_re}', 'kube_dns_service_ip': '${module.bootkube.kube_dns_service_ip}', 'kubelet_cni_bin_dir': '${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }', 'kubelet_debug_config': '${var.tectonic_kubelet_debug_config}', 'kubelet_node_label': 'node-role.kubernetes.io/master', 'kubelet_node_taints': 'node-role.kubernetes.io/master=:NoSchedule', 'use_metadata': False, 'tectonic_vanilla_k8s': '${var.tectonic_vanilla_k8s}'},

modules/vmware/node 'masters': {'source': '../../modules/vmware/node', 'base_domain': '${var.tectonic_base_domain}', 'container_images': '${var.tectonic_container_images}', 'core_public_keys': ['${var.tectonic_vmware_ssh_authorized_key}'], 'dns_server': '${var.tectonic_vmware_node_dns}', 'gateways': '${var.tectonic_vmware_master_gateways}', 'hostname': '${var.tectonic_vmware_master_hostnames}', 'ign_bootkube_path_unit_id': '${module.bootkube.systemd_path_unit_id}', 'ign_bootkube_service_id': '${module.bootkube.systemd_service_id}', 'ign_docker_dropin_id': '${module.ignition_masters.docker_dropin_id}', 'ign_installer_kubelet_env_id': '${module.ignition_masters.installer_kubelet_env_id}', 'ign_installer_runtime_mappings_id': '${module.ignition_masters.installer_runtime_mappings_id}', 'ign_k8s_node_bootstrap_service_id': '${module.ignition_masters.k8s_node_bootstrap_service_id}', 'ign_kubelet_service_id': '${module.ignition_masters.kubelet_service_id}', 'ign_locksmithd_service_id': '${module.ignition_masters.locksmithd_service_id}', 'ign_max_user_watches_id': '${module.ignition_masters.max_user_watches_id}', 'ign_tectonic_path_unit_id': '${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}', 'ign_tectonic_service_id': '${module.tectonic.systemd_service_id}', 'image_re': '${var.tectonic_image_re}', 'instance_count': '${var.tectonic_master_count}', 'ip_address': '${var.tectonic_vmware_master_ip}', 'kubeconfig': '${module.bootkube.kubeconfig}', 'private_key': '${var.tectonic_vmware_ssh_private_key_path}', 'vm_disk_datastore': '${var.tectonic_vmware_master_datastore}', 'vm_disk_template': '${var.tectonic_vmware_vm_template}', 'vm_disk_template_folder': '${var.tectonic_vmware_vm_template_folder}', 'vm_memory': '${var.tectonic_vmware_master_memory}', 'vm_network_labels': '${var.tectonic_vmware_master_networks}', 'vm_vcpu': '${var.tectonic_vmware_master_vcpu}', 'vmware_clusters': '${var.tectonic_vmware_master_clusters}', 'vmware_datacenters': '${var.tectonic_vmware_master_datacenters}', 'vmware_folder': '${vsphere_folder.tectonic_vsphere_folder.path}', 'vmware_resource_pool': '${var.tectonic_vmware_master_resource_pool}'},

28mm commented 6 years ago

Hi @mathewusher

Thanks for the thorough report—I believe I may have encountered similar issue previously, arising from a difference in the way pyhcl and terraform handle heredoc termination...

I should have a chance to dig into this tomorrow—it would be nice to have a fix in place!

28mm commented 6 years ago

@mathewusher -- quick update. Here's what I believe is happening. PyHCL chokes on .../modules/ignition/etcd.tf 

[...]$ cd modules/ignition
[...]$ ipython
In [1]: import hcl
In [2]: with open('etcd.tf', 'r') as f:
   ...:     config = hcl.load(f)
   ...:     
[...snipped...]
ValueError: Line 72, column 0, index 2700: EOF before closing heredoc

This happens to be the only instance I can find in the codebase of a heredoc that isn't terminated at the beginning of a line. The offending passage is this:

  cert_options = <<EOF
--cert-file=/etc/ssl/etcd/server.crt \
  --key-file=/etc/ssl/etcd/server.key \
  --peer-cert-file=/etc/ssl/etcd/peer.crt \
  --peer-key-file=/etc/ssl/etcd/peer.key  \
  --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \
  --peer-client-cert-auth=trueEOF

As far as I can tell this valid HCL, but rejected by PyHCL. I'm going to cut an issue upstream and see what it will take to correct the inconsistency.

A workaround is possible, in the meantime, if you just want to see the graph.

cert_options = "--cert-file=/etc/ssl/etcd/server.crt --key-file=/etc/ssl/etcd/server.key --peer-cert-file=/etc/ssl/etcd/peer.crt --peer-key-file=/etc/ssl/etcd/peer.key --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt --peer-client-cert-auth=true"

Cheers, Patrick

28mm commented 6 years ago

@mathewusher

This should be fixed in pyHCL, now.

[...]$ pip3 uninstall pyhcl
[...]$ pip3 install pyhcl
In [1]: import hcl

In [2]: with open('etcd.tf', 'r') as f:
   ...:     obj = hcl.load(f)
   ...:     

In [3]: obj
Out[3]: 
{'data': {'ignition_systemd_unit': {'etcd': {'count': '${var.etcd_count}',
    'dropin': [{'content': '${data.template_file.etcd.*.rendered[count.index]}',
      'name': '40-etcd-cluster.conf'}],
    'enable': True,
    'name': 'etcd-member.service'}},
  'template_file': {'advertise_client_urls': {'count': '${var.etcd_count}',
    'template': '${local.scheme}://${var.etcd_advertise_name_list[count.index]}:2379'},
   'etcd': {'count': '${var.etcd_count}',
    'template': '${file("${path.module}/resources/dropins/40-etcd-cluster.conf")}',
    'vars': {'advertise_client_urls': '${data.template_file.advertise_client_urls.*.rendered[count.index]}',
     'cert_options': '${var.etcd_tls_enabled ? local.cert_options : ""}',
     'container_image': '${var.container_images["etcd"]}',
     'initial_advertise_peer_urls': '${data.template_file.initial_advertise_peer_urls.*.rendered[count.index]}',
     'initial_cluster': '${length(var.etcd_initial_cluster_list) > 0 ? format("--initial-cluster=%s", join(",", data.template_file.initial_cluster.*.rendered)) : ""}',
     'metadata_deps': '${var.use_metadata ? local.metadata_deps : ""}',
     'metadata_env': '${var.use_metadata ? local.metadata_env : ""}',
     'name': '${data.template_file.etcd_names.*.rendered[count.index]}',
     'scheme': '${local.scheme}'}},
   'etcd_names': {'count': '${var.etcd_count}',
    'template': '${var.cluster_name}-etcd-${count.index}${var.base_domain == "" ? "" : ".${var.base_domain}"}'},
   'initial_advertise_peer_urls': {'count': '${var.etcd_count}',
    'template': '${local.scheme}://${var.etcd_advertise_name_list[count.index]}:2380'},
   'initial_cluster': {'count': '${length(var.etcd_initial_cluster_list) > 0 ? var.etcd_count : 0}',
    'template': '${data.template_file.etcd_names.*.rendered[count.index]}=${local.scheme}://${local.etcd_initial_cluster_list[count.index]}:2380'}}},
 'locals': {'cert_options': '--cert-file=/etc/ssl/etcd/server.crt \\\n  --key-file=/etc/ssl/etcd/server.key \\\n  --peer-cert-file=/etc/ssl/etcd/peer.crt \\\n  --peer-key-file=/etc/ssl/etcd/peer.key \\\n  --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \\\n  --peer-client-cert-auth=true',
  'etcd_initial_cluster_list': '${concat(var.etcd_initial_cluster_list, list("dummy"))}',
  'metadata_deps': 'Requires=coreos-metadata.service\nAfter=coreos-metadata.service',
  'metadata_env': 'EnvironmentFile=/run/metadata/coreos',
  'scheme': '${var.etcd_tls_enabled ? "https" : "http"}'}}