Closed adambaumeister closed 7 months ago
cdot65
commented
7 months ago @adambaumeister , is there value in retaining the xmltodict dependency and xml_to_dict() function if we utilize the flatten_xml_to_dict() function?
I'm all for removing external dependencies where applicable, curious to know if the flatten_xml_to_dict() could be used as the sole flipper of xml to python dict objects
cdot65
commented
7 months ago Interesting to consider some of small differences: xml_to_dict() will present the object as a type OrderedDict whereas the flatten_xml_to_dict() will return as a type of dictionary. If we needed to maintain order (i.e. Security Policy), then xml_to_dict() would provide significant value. But we aren't using the function to parse XML configuration, so the value is mitigated.
What may prove more valuable is the fact that the xmltodict library will return empty values of an object as type None, whereas flatten_xml_to_dict() returns an empty dictionary. Unsure if there are situations where we'd need one over the other, just felt it appropriate to consider
Comparison between the two outputs:
xml_to_dict_output = OrderedDict(
[
(
"response",
OrderedDict(
[
("@status", "success"),
(
"result",
OrderedDict(
[
("enabled", "yes"),
(
"group",
OrderedDict(
[
("mode", "Active-Passive"),
(
"local-info",
OrderedDict(
[
("version", "1"),
("state", "active"),
("state-duration", "154350"),
(
"mgmt-ip",
"192.168.255.41/24",
),
("mgmt-ipv6", None),
("preemptive", "no"),
("promotion-hold", "2000"),
("hello-interval", "8000"),
("heartbeat-interval", "2000"),
("preempt-hold", "1"),
("monitor-fail-holdup", "0"),
("addon-master-holdup", "500"),
("ha1-encrypt-imported", "no"),
("mode", "Active-Passive"),
("platform-model", "PA-VM"),
(
"serial-num",
"007054000123456",
),
("vm-license", "vm100"),
("priority", "10"),
("max-flaps", "3"),
("preempt-flap-cnt", "0"),
("nonfunc-flap-cnt", "0"),
("mgmt-hb", "configured"),
("state-sync", "Complete"),
("state-sync-type", "ethernet"),
(
"active-passive",
OrderedDict(
[
(
"passive-link-state",
"shutdown",
),
(
"monitor-fail-holddown",
"1",
),
]
),
),
(
"ha1-ipaddr",
"169.250.0.1/24",
),
(
"ha1-macaddr",
"a2:d0:dd:11:a1:5d",
),
("ha1-port", "ethernet1/1"),
("ha1-encrypt-enable", "no"),
("ha1-link-mon-intv", "3000"),
(
"ha2-macaddr",
"06:95:06:87:f8:d2",
),
("ha2-port", "ethernet1/2"),
("ha2-keep-alive", "log-only"),
("ha2-ka-thresh", "10000"),
("build-rel", "11.0.0"),
(
"url-version",
"20240122.20208",
),
("app-version", "8799-8509"),
("iot-version", "111-466"),
("av-version", "0"),
("threat-version", "8799-8509"),
(
"vpnclient-version",
"Not Installed",
),
(
"gpclient-version",
"Not Installed",
),
("vm-license-type", "vm100"),
("DLP", "Match"),
("VMS", "Match"),
("build-compat", "Match"),
("url-compat", "Mismatch"),
("app-compat", "Match"),
("iot-compat", "Match"),
("av-compat", "Match"),
("threat-compat", "Match"),
("vpnclient-compat", "Match"),
("gpclient-compat", "Match"),
("vm-license-compat", "Match"),
]
),
),
(
"peer-info",
OrderedDict(
[
(
"conn-ha1",
OrderedDict(
[
(
"conn-status",
"up",
),
(
"conn-primary",
"yes",
),
(
"conn-desc",
"heartbeat status",
),
]
),
),
(
"conn-mgmt",
OrderedDict(
[
(
"conn-status",
"up",
),
(
"conn-desc",
"heartbeat status",
),
]
),
),
(
"conn-ha2",
OrderedDict(
[
(
"conn-primary",
"yes",
),
(
"conn-ka-enbled",
"yes",
),
(
"conn-desc",
"keep-alive status",
),
(
"conn-type",
"log-only",
),
("conn-hold", "0"),
(
"conn-status",
"up",
),
]
),
),
("conn-status", "up"),
("version", "1"),
("state", "passive"),
("state-duration", "153783"),
(
"last-error-reason",
"User requested",
),
(
"last-error-state",
"suspended",
),
("preemptive", "no"),
("mode", "Active-Passive"),
("platform-model", "PA-VM"),
(
"serial-num",
"007054000123456",
),
("vm-license", "vm100"),
("priority", "20"),
(
"mgmt-ip",
"192.168.255.42/24",
),
("mgmt-ipv6", None),
("ha1-ipaddr", "169.250.0.2"),
(
"ha1-macaddr",
"52:67:bc:61:da:e3",
),
(
"ha2-macaddr",
"a2:06:f5:33:08:e3",
),
("build-rel", "11.0.0"),
(
"url-version",
"0000.00.00.000",
),
("app-version", "8799-8509"),
("iot-version", "111-466"),
("av-version", "0"),
("threat-version", "8799-8509"),
(
"vpnclient-version",
"Not Installed",
),
(
"gpclient-version",
"Not Installed",
),
("vm-license-type", "vm100"),
("DLP", "4.0.0"),
("VMS", "4.0.2"),
]
),
),
(
"link-monitoring",
OrderedDict(
[
("enabled", "yes"),
("failure-condition", "any"),
("groups", None),
]
),
),
(
"path-monitoring",
OrderedDict(
[
("enabled", "yes"),
("failure-condition", "any"),
("virtual-wire", None),
("vlan", None),
("virtual-router", None),
]
),
),
("running-sync", "synchronized"),
("running-sync-enabled", "yes"),
]
),
),
]
),
),
]
),
)
]
)
flatten_xml_to_dict_output = {
"result": {
"enabled": "yes",
"group": {
"mode": "Active-Passive",
"local-info": {
"version": "1",
"state": "active",
"state-duration": "154350",
"mgmt-ip": "192.168.255.41/24",
"mgmt-ipv6": {},
"preemptive": "no",
"promotion-hold": "2000",
"hello-interval": "8000",
"heartbeat-interval": "2000",
"preempt-hold": "1",
"monitor-fail-holdup": "0",
"addon-master-holdup": "500",
"ha1-encrypt-imported": "no",
"mode": "Active-Passive",
"platform-model": "PA-VM",
"serial-num": "007054000123456",
"vm-license": "vm100",
"priority": "10",
"max-flaps": "3",
"preempt-flap-cnt": "0",
"nonfunc-flap-cnt": "0",
"mgmt-hb": "configured",
"state-sync": "Complete",
"state-sync-type": "ethernet",
"active-passive": {
"passive-link-state": "shutdown",
"monitor-fail-holddown": "1",
},
"ha1-ipaddr": "169.250.0.1/24",
"ha1-macaddr": "a2:d0:dd:11:a1:5d",
"ha1-port": "ethernet1/1",
"ha1-encrypt-enable": "no",
"ha1-link-mon-intv": "3000",
"ha2-macaddr": "06:95:06:87:f8:d2",
"ha2-port": "ethernet1/2",
"ha2-keep-alive": "log-only",
"ha2-ka-thresh": "10000",
"build-rel": "11.0.0",
"url-version": "20240122.20208",
"app-version": "8799-8509",
"iot-version": "111-466",
"av-version": "0",
"threat-version": "8799-8509",
"vpnclient-version": "Not Installed",
"gpclient-version": "Not Installed",
"vm-license-type": "vm100",
"DLP": "Match",
"VMS": "Match",
"build-compat": "Match",
"url-compat": "Mismatch",
"app-compat": "Match",
"iot-compat": "Match",
"av-compat": "Match",
"threat-compat": "Match",
"vpnclient-compat": "Match",
"gpclient-compat": "Match",
"vm-license-compat": "Match",
},
"peer-info": {
"conn-ha1": {
"conn-status": "up",
"conn-primary": "yes",
"conn-desc": "heartbeat status",
},
"conn-mgmt": {"conn-status": "up", "conn-desc": "heartbeat status"},
"conn-ha2": {
"conn-primary": "yes",
"conn-ka-enbled": "yes",
"conn-desc": "keep-alive status",
"conn-type": "log-only",
"conn-hold": "0",
"conn-status": "up",
},
"conn-status": "up",
"version": "1",
"state": "passive",
"state-duration": "153783",
"last-error-reason": "User requested",
"last-error-state": "suspended",
"preemptive": "no",
"mode": "Active-Passive",
"platform-model": "PA-VM",
"serial-num": "007054000123456",
"vm-license": "vm100",
"priority": "20",
"mgmt-ip": "192.168.255.42/24",
"mgmt-ipv6": {},
"ha1-ipaddr": "169.250.0.2",
"ha1-macaddr": "52:67:bc:61:da:e3",
"ha2-macaddr": "a2:06:f5:33:08:e3",
"build-rel": "11.0.0",
"url-version": "0000.00.00.000",
"app-version": "8799-8509",
"iot-version": "111-466",
"av-version": "0",
"threat-version": "8799-8509",
"vpnclient-version": "Not Installed",
"gpclient-version": "Not Installed",
"vm-license-type": "vm100",
"DLP": "4.0.0",
"VMS": "4.0.2",
},
"link-monitoring": {
"enabled": "yes",
"failure-condition": "any",
"groups": {},
},
"path-monitoring": {
"enabled": "yes",
"failure-condition": "any",
"virtual-wire": {},
"vlan": {},
"virtual-router": {},
},
"running-sync": "synchronized",
"running-sync-enabled": "yes",
},
}
}
adambaumeister
commented
7 months ago I agree on the xmltodict thing @cdot65, in my experience that library is only useful if you want to transform arbitary XML into a python data structure, but the use case here is a known XML format into pydantic models. There is another library that does literally that but it's probably too heavyweight when a simple recursive function and a mixin does the job.
There are also situations where you might want to flatten or change the data structure to make the pydantic models more logical than their XML representations, given that you will never need to reserialize them to XML. For example, instead of:
class HighAvailabilityPeer(BaseModel):
serial: str
class HighAvailability(BaseModel):
state: str
peer: HighAvailabilityPeer
class ManagedDevice(BaseModel):
ha: HighAvailabilityyou could instead do
class ManagedDevice(BaseModel):
ha_state: str
ha_peer_serial: strLG2M 🚀
Motivation
This pull request introduces the ability to connect to a Panorama instance instead of a single firewall, and pass a filter into the script. This filter allows the user to specify multiple devices in the one script run.
This PR does not introduce a time savings (besides the small amount of effort reduction in running the script against multiple ip addresses) but is the basis for future parallelization as the script now has the panorama view of all the managed devices.