search

Juniper / ansible-junos-stdlib

Junos modules for Ansible
Apache License 2.0
303 stars 157 forks source link

junos_get_facts displaying wrong serialnumber of the secondary node of a SRX cluster #235

Open epydd opened 7 years ago

epydd commented 7 years ago

The get_facts on a secondary node of a SRX Cluster displays the serialnumber of the primary node.

the playbook:

- name: Deploy Cluster
  hosts: PROV
  gather_facts: no
  connection: local
  roles:
    - Juniper.junos
  vars_files:
    - "vars/ZTP.yml"

  tasks:
    - wait_for:
        host: "{{ inventory_hostname }}"
        port: 830
        timeout: 5

    - junos_get_facts: 
        host: "{{ inventory_hostname }}"
      register: facts

    - debug:
        msg: "{{ facts }}"

the output:

PLAY [Deploy Cluster] **********************************************************

TASK [wait_for] ****************************************************
ok: [E0]
ok: [E1]

TASK [junos_get_facts] ***********************************************************
ok: [E0]
ok: [E1]

TASK [debug] *******************************************************************
ok: [E0] => {
    "msg": {
        "changed": false,
        "facts": {
            "HOME": "/root",
            "RE0": {
                "last_reboot_reason": "0x200:normal shutdown",
                "mastership_state": "master",
                "model": "RE-SRX320",
                "status": "OK",
                "up_time": "14 hours, 2 minutes, 47 seconds"
            },
            "RE1": null,
            "RE_hw_mi": true,
            "current_re": [
                "node0",
                "master",
                "fpc0",
                "node",
                "fwdd",
                "member",
                "pfem",
                "re0",
                "fpc0.pic0"
            ],
            "domain": null,
            "fqdn": "CL_E0",
            "has_2RE": true,
            "hostname": "CL_E0",
            "hostname_info": {
                "node0": "CL_E0",
                "node1": "CL_E1"
            },
            "ifd_style": "CLASSIC",
            "junos_info": {
                "node0": {
                    "object": {
                        "build": 4,
                        "major": [
                            15,
                            1
                        ],
                        "minor": [
                            49,
                            "D",
                            80
                        ],
                        "type": "X"
                    },
                    "text": "15.1X49-D80.4"
                },
                "node1": {
                    "object": {
                        "build": 4,
                        "major": [
                            15,
                            1
                        ],
                        "minor": [
                            49,
                            "D",
                            80
                        ],
                        "type": "X"
                    },
                    "text": "15.1X49-D80.4"
                }
            },
            "master": [
                "RE0",
                "RE0"
            ],
            "model": "SRX320",
            "model_info": {
                "node0": "SRX320",
                "node1": "SRX320"
            },
            "personality": "SRX_BRANCH",
            "re_info": {
                "default": {
                    "0": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    },
                    "default": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    }
                },
                "node0": {
                    "0": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    },
                    "default": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    }
                },
                "node1": {
                    "0": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    }
                }
            },
            "re_master": {
                "default": "0",
                "node0": "0",
                "node1": "0"
            },
            "serialnumber": "CW3116AF0099",
            "srx_cluster": true,
            "srx_cluster_id": "5",
            "switch_style": "VLAN_L2NG",
            "vc_capable": false,
            "vc_fabric": null,
            "vc_master": null,
            "vc_mode": null,
            "version": "15.1X49-D80.4",
            "version_RE0": "15.1X49-D80.4",
            "version_RE1": "15.1X49-D80.4",
            "version_info": {
                "build": 4,
                "major": [
                    15,
                    1
                ],
                "minor": [
                    49,
                    "D",
                    80
                ],
                "type": "X"
            },
            "virtual": false
        }
    }
}
ok: [E1] => {
    "msg": {
        "changed": false,
        "facts": {
            "HOME": "/root",
            "RE0": {
                "last_reboot_reason": "0x200:normal shutdown",
                "mastership_state": "master",
                "model": "RE-SRX320",
                "status": "OK",
                "up_time": "14 hours, 2 minutes, 47 seconds"
            },
            "RE1": null,
            "RE_hw_mi": true,
            "current_re": [
                "node1",
                "master",
                "fpc0",
                "node",
                "fwdd",
                "member",
                "pfem",
                "re0",
                "fpc0.pic0"
            ],
            "domain": null,
            "fqdn": "CL_E1",
            "has_2RE": true,
            "hostname": "CL_E1",
            "hostname_info": {
                "node0": "CL_E0",
                "node1": "CL_E1"
            },
            "ifd_style": "CLASSIC",
            "junos_info": {
                "node0": {
                    "object": {
                        "build": 4,
                        "major": [
                            15,
                            1
                        ],
                        "minor": [
                            49,
                            "D",
                            80
                        ],
                        "type": "X"
                    },
                    "text": "15.1X49-D80.4"
                },
                "node1": {
                    "object": {
                        "build": 4,
                        "major": [
                            15,
                            1
                        ],
                        "minor": [
                            49,
                            "D",
                            80
                        ],
                        "type": "X"
                    },
                    "text": "15.1X49-D80.4"
                }
            },
            "master": [
                "RE0",
                "RE0"
            ],
            "model": "SRX320",
            "model_info": {
                "node0": "SRX320",
                "node1": "SRX320"
            },
            "personality": "SRX_BRANCH",
            "re_info": {
                "default": {
                    "0": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    },
                    "default": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    }
                },
                "node0": {
                    "0": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    },
                    "default": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    }
                },
                "node1": {
                    "0": {
                        "last_reboot_reason": "0x200:normal shutdown",
                        "mastership_state": "master",
                        "model": "RE-SRX320",
                        "status": "OK"
                    }
                }
            },
            "re_master": {
                "default": "0",
                "node0": "0",
                "node1": "0"
            },
            "serialnumber": "CW3116AF0099",
            "srx_cluster": true,
            "srx_cluster_id": "5",
            "switch_style": "NONE",
            "vc_capable": false,
            "vc_fabric": null,
            "vc_master": null,
            "vc_mode": null,
            "version": "15.1X49-D80.4",
            "version_RE0": "15.1X49-D80.4",
            "version_RE1": "15.1X49-D80.4",
            "version_info": {
                "build": 4,
                "major": [
                    15,
                    1
                ],
                "minor": [
                    49,
                    "D",
                    80
                ],
                "type": "X"
            },
            "virtual": false
        }
    }
}
stacywsmith commented 7 years ago

@epydd Please provide the output of show chassis hardware | display xml from both the primary and secondary nodes of the SRX cluster.

epydd commented 7 years ago

@stacywsmith Heres the output:

root@CL_E0> show chassis hardware |display xml
<rpc-reply xmlns:junos="http://xml.juniper.net/junos/15.1X49/junos">
    <multi-routing-engine-results>

        <multi-routing-engine-item>

            <re-name>node0</re-name>

            <chassis-inventory xmlns="http://xml.juniper.net/junos/15.1X49/junos-chassis">
                <chassis junos:style="inventory">
                    <name>Chassis</name>
                    <serial-number>CW3116AF0099</serial-number>
                    <description>SRX320</description>
                    <chassis-module>
                        <name>Routing Engine</name>
                        <version>REV 0x07</version>
                        <part-number>650-065040</part-number>
                        <serial-number>CW3116AF0099</serial-number>
                        <description>RE-SRX320</description>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 0</name>
                        <description>FPC</description>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>6xGE,2xGE SFP Base PIC</description>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 2</name>
                        <version>REV 06</version>
                        <part-number>750-064613</part-number>
                        <serial-number>ACNW5129</serial-number>
                        <description>FPC</description>
                        <model-number></model-number>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>1x Serial mPIM (RoHS)</description>
                            <model-number></model-number>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>Power Supply 0</name>
                    </chassis-module>
                </chassis>
            </chassis-inventory>
        </multi-routing-engine-item>

        <multi-routing-engine-item>

            <re-name>node1</re-name>

            <chassis-inventory xmlns="http://xml.juniper.net/junos/15.1X49/junos-chassis">
                <chassis junos:style="inventory">
                    <name>Chassis</name>
                    <serial-number>CW3116AF0172</serial-number>
                    <description>SRX320</description>
                    <chassis-module>
                        <name>Routing Engine</name>
                        <version>REV 0x07</version>
                        <part-number>650-065040</part-number>
                        <serial-number>CW3116AF0172</serial-number>
                        <description>RE-SRX320</description>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 0</name>
                        <description>FPC</description>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>6xGE,2xGE SFP Base PIC</description>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 2</name>
                        <version>REV 06</version>
                        <part-number>750-064613</part-number>
                        <serial-number>ACNV8730</serial-number>
                        <description>FPC</description>
                        <model-number></model-number>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>1x Serial mPIM (RoHS)</description>
                            <model-number></model-number>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>Power Supply 0</name>
                    </chassis-module>
                </chassis>
            </chassis-inventory>
        </multi-routing-engine-item>

    </multi-routing-engine-results>
    <cli>
        <banner>{primary:node0}</banner>
    </cli>
</rpc-reply>
epydd commented 7 years ago

Is it possible to find out from the get_facts, wich of the nodes is the primary? the only way that i find out is to check the 'current-re' list and check if theres a node0 or node1 value...

Also, noted that you asked the output from both nodes... here they are:

Output node0:

root@CL_E0> show chassis hardware |display xml
<rpc-reply xmlns:junos="http://xml.juniper.net/junos/15.1X49/junos">
    <multi-routing-engine-results>

        <multi-routing-engine-item>

            <re-name>node0</re-name>

            <chassis-inventory xmlns="http://xml.juniper.net/junos/15.1X49/junos-chassis">
                <chassis junos:style="inventory">
                    <name>Chassis</name>
                    <serial-number>CW3116AF0099</serial-number>
                    <description>SRX320</description>
                    <chassis-module>
                        <name>Routing Engine</name>
                        <version>REV 0x07</version>
                        <part-number>650-065040</part-number>
                        <serial-number>CW3116AF0099</serial-number>
                        <description>RE-SRX320</description>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 0</name>
                        <description>FPC</description>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>6xGE,2xGE SFP Base PIC</description>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 2</name>
                        <version>REV 06</version>
                        <part-number>750-064613</part-number>
                        <serial-number>ACNW5129</serial-number>
                        <description>FPC</description>
                        <model-number></model-number>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>1x Serial mPIM (RoHS)</description>
                            <model-number></model-number>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>Power Supply 0</name>
                    </chassis-module>
                </chassis>
            </chassis-inventory>
        </multi-routing-engine-item>

        <multi-routing-engine-item>

            <re-name>node1</re-name>

            <chassis-inventory xmlns="http://xml.juniper.net/junos/15.1X49/junos-chassis">
                <chassis junos:style="inventory">
                    <name>Chassis</name>
                    <serial-number>CW3116AF0172</serial-number>
                    <description>SRX320</description>
                    <chassis-module>
                        <name>Routing Engine</name>
                        <version>REV 0x07</version>
                        <part-number>650-065040</part-number>
                        <serial-number>CW3116AF0172</serial-number>
                        <description>RE-SRX320</description>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 0</name>
                        <description>FPC</description>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>6xGE,2xGE SFP Base PIC</description>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 2</name>
                        <version>REV 06</version>
                        <part-number>750-064613</part-number>
                        <serial-number>ACNV8730</serial-number>
                        <description>FPC</description>
                        <model-number></model-number>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>1x Serial mPIM (RoHS)</description>
                            <model-number></model-number>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>Power Supply 0</name>
                    </chassis-module>
                </chassis>
            </chassis-inventory>
        </multi-routing-engine-item>

    </multi-routing-engine-results>
    <cli>
        <banner>{secondary:node0}</banner>
    </cli>
</rpc-reply>

ouput node1:

root@CL_E1> show chassis hardware |display xml
<rpc-reply xmlns:junos="http://xml.juniper.net/junos/15.1X49/junos">
    <multi-routing-engine-results>

        <multi-routing-engine-item>

            <re-name>node0</re-name>

            <chassis-inventory xmlns="http://xml.juniper.net/junos/15.1X49/junos-chassis">
                <chassis junos:style="inventory">
                    <name>Chassis</name>
                    <serial-number>CW3116AF0099</serial-number>
                    <description>SRX320</description>
                    <chassis-module>
                        <name>Routing Engine</name>
                        <version>REV 0x07</version>
                        <part-number>650-065040</part-number>
                        <serial-number>CW3116AF0099</serial-number>
                        <description>RE-SRX320</description>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 0</name>
                        <description>FPC</description>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>6xGE,2xGE SFP Base PIC</description>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 2</name>
                        <version>REV 06</version>
                        <part-number>750-064613</part-number>
                        <serial-number>ACNW5129</serial-number>
                        <description>FPC</description>
                        <model-number></model-number>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>1x Serial mPIM (RoHS)</description>
                            <model-number></model-number>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>Power Supply 0</name>
                    </chassis-module>
                </chassis>
            </chassis-inventory>
        </multi-routing-engine-item>

        <multi-routing-engine-item>

            <re-name>node1</re-name>

            <chassis-inventory xmlns="http://xml.juniper.net/junos/15.1X49/junos-chassis">
                <chassis junos:style="inventory">
                    <name>Chassis</name>
                    <serial-number>CW3116AF0172</serial-number>
                    <description>SRX320</description>
                    <chassis-module>
                        <name>Routing Engine</name>
                        <version>REV 0x07</version>
                        <part-number>650-065040</part-number>
                        <serial-number>CW3116AF0172</serial-number>
                        <description>RE-SRX320</description>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 0</name>
                        <description>FPC</description>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>6xGE,2xGE SFP Base PIC</description>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>FPC 2</name>
                        <version>REV 06</version>
                        <part-number>750-064613</part-number>
                        <serial-number>ACNV8730</serial-number>
                        <description>FPC</description>
                        <model-number></model-number>
                        <chassis-sub-module>
                            <name>PIC 0</name>
                            <description>1x Serial mPIM (RoHS)</description>
                            <model-number></model-number>
                        </chassis-sub-module>
                    </chassis-module>
                    <chassis-module>
                        <name>Power Supply 0</name>
                    </chassis-module>
                </chassis>
            </chassis-inventory>
        </multi-routing-engine-item>

    </multi-routing-engine-results>
    <cli>
        <banner>{primary:node1}</banner>
    </cli>
</rpc-reply>
stacywsmith commented 7 years ago

@epydd Thanks for including the output from node1 as well.

I need to think about the serialnumber fact a little more. Prior to PyEZ 2.1, the exact meaning of each fact (especially in the context of a multi-RE and/or multi-chassis system) had not been formally defined. I tried to define the meanings here: http://junos-pyez.readthedocs.io/en/2.1.1/jnpr.junos.facts.html

I tend to agree with you that the serialnumber should be reported for the "current" chassis. That's the definition I adopted for things like hostname and version. This probably also means adding a new serialnumber_info fact which is a dictionary keyed on each chassis (similar to what I did with the new version_info and hostname_info facts. Does this sound like a reasonable approach?

At present, there's no explicit way to find out if node0 or node1 is primary directly from junos_get_facts. There is a new master property of the PyEZ device object (https://github.com/Juniper/py-junos-eznc/blob/master/lib/jnpr/junos/device.py#L215), but it's not yet exposed in the junos_get_facts output.

However, the current_re fact will always contain a primary element in the list if the node is the primary node of an SRX cluster. In fact, that's the logic the PyEZ master property uses: https://github.com/Juniper/py-junos-eznc/blob/master/lib/jnpr/junos/device.py#L252-L258

I would certainly be open to a new issue, and pull request :-), to expose the PyEZ device's master and re_name properties in the junos_get_facts response.

epydd commented 7 years ago

@stacywsmith Thanks for the response! the serialnumber_info looks like a reasonable approach.

I managed to identify the primary (and secondary) node with the following:

    - name: "FACTS - Collecting cluster information"
      junos_get_facts: 
        host: "{{ inventory_hostname }}"
      register: facts

    - name:
      set_fact:
        secondary: "node1"
      when: "'node0' in facts.facts.current_re"

    - name:
      set_fact:
        secondary: "node0"
      when: "'node1' in facts.facts.current_re"

SSH to {{ inventory_hostname }} on a reth interface will land on the primary node, and then the output of get_fact would tell me if Im on node0 or node1. But this is a bad approach...

Regarding the primary element in the list of current_re, as you might notice, it did not happen on the first output I posted (master should not be a boolean? why is it a list of 'RE0'?). Also, if changing the primary node on the cluster (failover), the output of master still points fo RE0...

epydd commented 7 years ago

Im not familiar with git/github, but ill learn more to be able to create a pull request...

stacywsmith commented 7 years ago

Regarding the primary element in the list of current_re, as you might notice, it did not happen on the first output I posted (master should not be a boolean? why is it a list of 'RE0'?). Also, if changing the primary node on the cluster (failover), the output of master still points fo RE0...

For any facts which were previously implemented in PyEZ <= 2.0.1 I tried to ensure that PyEZ >= 2.1.0 would return the same value. I simply tried to document what the fact really is telling you, because sometimes it wasn't obvious. That's the case with the existing master fact.

Here's the definition of the existing master fact:

On a single chassis/node system, a string value of ‘RE0’ or ‘RE1’ indicating which RE is master. On a multi-chassis or multi-node system, the value is a list of these strings indicating whether RE0 or RE1 is master. There is one entry in the list for each chassis/node in the system.

So, it's a list for each chassis/node and each element in the list is telling you if RE0 or RE1 on that particular chassis is the master. In the case of a two-node cluster of SRX 320s, each node only has a single RE of RE0 and RE0 is the "master" on each node. In this case, the term "master" is referring to GRES/NSR, and not to an SRX cluster's primary/secondary roles. The bottom line is the existing master fact isn't really useful in your case. It will always be a list of [ 'RE0', 'RE0'] on a cluster of SRX 320s.

It's very interesting that the primary value isn't showing up in the current_re fact for your setup. Would it be possible to get the output of the show interfaces terse routing-instance __juniper_private1__ cli command from both the primary and secondary nodes in your setup?

epydd commented 7 years ago

Heres the output:

{primary:node0}
root@CL_E0> show interfaces terse routing-instance __juniper_private1__
Interface               Admin Link Proto    Local                 Remote
fab0.0                  up    down inet     30.81.0.200/24
fab1.0                  up    down inet     30.82.0.200/24
fxp1.0                  up    up   inet     129.80.0.1/2
                                   tnp      0x1500001
lo0.16385               up    up   inet     10.0.0.1            --> 0/0
                                            10.0.0.16           --> 0/0
                                            128.0.0.1           --> 0/0
                                            128.0.0.4           --> 0/0
                                            128.0.1.16          --> 0/0

{primary:node0}
root@CL_E0>
{secondary:node1}
root@CL_E1> show interfaces terse routing-instance __juniper_private1__
Interface               Admin Link Proto    Local                 Remote
fab0.0                  up    down inet     30.81.0.200/24
fab1.0                  up    down inet     30.82.0.200/24
fxp1.0                  up    up   inet     130.80.0.1/2
                                   tnp      0x2500001
lo0.16385               up    up   inet     10.0.0.1            --> 0/0
                                            10.0.0.16           --> 0/0
                                            128.0.0.1           --> 0/0
                                            128.0.0.4           --> 0/0
                                            128.0.1.16          --> 0/0

{secondary:node1}
root@CL_E1>
stacywsmith commented 6 years ago

This requires an enhancement to PyEZ to add a new serialnumber_info fact. Once this is done in PyEZ, the information will automatically be available in the facts provided by juniper_junos_facts.