gschanuel
commented
2 years ago Hello there.. after one year maybe you have already found a solution anyway, this is my logstash config to use with panos >= 9.1.3
I'm using metallb, so you may need to change the services annotations and type
Hope it helps someone
---
replicas: 2
# Allows you to add any config files in /usr/share/logstash/config/
# such as logstash.yml and log4j2.properties
#
# Note that when overriding logstash.yml, `http.host: 0.0.0.0` should always be included
# to make default probes work.
logstashConfig:
logstash.yml: |
http.host: "0.0.0.0"
pipeline.ecs_compatibility: disabled
queue.type: persisted
queue.max_bytes: 100gb
pipeline.workers: 10
pipeline.batch.size: 350
# Allows you to add any pipeline files in /usr/share/logstash/pipeline/
### ***warn*** there is a hardcoded logstash.conf in the image, override it first
logstashPipeline:
logstash.conf: |
input {
syslog {
timezone => "America/Sao_Paulo"
port => "5244"
type => "syslog"
tags => [ "PAN-OS_SysLog" ]
}
}
filter {
# Log types are "TRAFFIC", "THREAT", "CONFIG" and "SYSTEM". URL & Wildfire logs are inside Threat logs
# Log fields: https://www.paloaltonetworks.com/documentation/80/pan-os/pan-os/monitoring/syslog-field-descriptions
#clone {
# clones => ["plain"]
#}
#if "plain" not in [type]{
if ([message] =~ /TRAFFIC/) {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
"GeneratedTime", "SourceAddress", "DestinationAddress", "NATSourceIP", "NATDestinationIP", "RuleName",
"SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
"InboundInterface", "OutboundInterface", "LogAction", "FUTURE_USE", "SessionID",
"RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
"Protocol", "Action", "Bytes", "BytesSent", "BytesReceived", "Packets", "StartTime", "ElapsedTime",
"Category", "FUTURE_USE", "SequenceNumber", "ActionFlags", "SourceLocation",
"DestinationLocation", "FUTURE_USE", "PacketsSent", "PacketsReceived", "SessionEndReason",
"DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
"DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "ActionSource", "SourceVMUUID",
"DestinationVMUUID", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID", "ParentStartTime",
"TunnelType", "SCTPAssociationID", "SCTPChunks", "SCTPChunksSent", "SCTPChunksReceived", "RuleUUID",
"HTTP-2Connection", "AppFlapCount", "PolicyID", "LinkSwitches", "SD-WANCluster", "SD-WANDeviceType",
"SD-WANClusterType", "SD-WANSite", "DynamicUserGroupName"
]
}
mutate { rename => [ "[host]", "[host][name]"] }
mutate {
convert => [ "Bytes", "integer" ]
convert => [ "BytesReceived", "integer" ]
convert => [ "BytesSent", "integer" ]
convert => [ "ElapsedTime", "integer" ]
convert => [ "GeoIP.dma_code", "integer" ]
convert => [ "GeoIP.latitude", "float" ]
convert => [ "GeoIP.longitude", "float" ]
convert => [ "NATDestinationPort", "integer" ]
convert => [ "NATSourcePort", "integer" ]
convert => [ "Packets", "integer" ]
convert => [ "PacketsReceived", "integer" ]
convert => [ "PacketsSent", "integer" ]
convert => [ "SequenceNumber", "integer" ]
add_tag => [ "PAN-OS_Traffic"]
}
}
else if ([message] =~ /HIPMATCH/) {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
"GeneratedTime", "SourceUser", "VirtualSystem", "MachineName", "OperatingSystem",
"SourceAddress", "HIP", "RepeatCount", "HIPType", "FUTURE_USE", "FUTURE_USE", "SequenceNumber",
"ActionFlags", "DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2",
"DeviceGroupHierarchyLevel3", "DeviceGroupHierarchyLevel4", "VirtualSystemName",
"DeviceName", "VirtualSystemID", "IPv6SourceAddress", "HostID", "UserDeviceSerialNumber"
]
}
mutate { rename => [ "[host]", "[host][name]"] }
mutate {
convert => [ "GeoIP.dma_code", "integer" ]
convert => [ "GeoIP.latitude", "float" ]
convert => [ "GeoIP.longitude", "float" ]
add_tag => [ "PAN-OS_HIPMATCH"]
}
}
else if ([message] =~ /THREAT/) {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
"GeneratedTime", "SourceAddress", "DestinationAddress", "NATSourceIP", "NATDestinationIP", "RuleName",
"SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
"InboundInterface", "OutboundInterface", "LogAction", "FUTURE_USE", "SessionID",
"RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
"Protocol", "Action", "URL_Filename", "ThreatID", "Category", "Severity", "Direction",
"SequenceNumber", "ActionFlags", "SourceLocation", "DestinationLocation", "FUTURE_USE",
"ContentType", "PCAP_ID", "FileDigest", "Cloud", "URLIndex", "UserAgent", "FileType",
"X-Forwarded-For", "Referer", "Sender", "Subject", "Recipient", "ReportID",
"DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
"DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "FUTURE_USE", "SourceVMUUID",
"DestinationVMUUID", "HTTPMethod", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID",
"ParentStartTime", "TunnelType", "ThreatCategory", "ContentVersion", "FUTURE_USE",
"SCTPAssociationID", "PayloadProtocolID", "HTTPHeaders", "URLCategoryList", "RuleUUID",
"HTTP-2Connection", "DynamicUserGroupName"
]
}
mutate { rename => [ "[host]", "[host][name]"] }
mutate {
convert => [ "GeoIP.dma_code", "integer" ]
convert => [ "GeoIP.latitude", "float" ]
convert => [ "GeoIP.longitude", "float" ]
convert => [ "NATDestinationPort", "integer" ]
convert => [ "NATSourcePort", "integer" ]
convert => [ "SequenceNumber", "integer" ]
add_tag => ["PAN-OS_Threat"]
}
}
else if ([message] =~ /CONFIG/) {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Subtype", "FUTURE_USE", "GeneratedTime", "Host",
"VirtualSystem", "Command", "Admin", "Client", "Result", "ConfigurationPath", "BeforeChangeDetail",
"AfterChangeDetail", "SequenceNumber", "ActionFlags", "DeviceGroupHierarchyLevel1",
"DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3", "DeviceGroupHierarchyLevel4",
"VirtualSystemName", "DeviceName", "DeviceGroup", "AuditComment"
]
}
mutate { add_tag => [ "PAN-OS_Config"] }
mutate { rename => [ "[host]", "[host][name]"] }
}
else if ([message] =~ /SYSTEM/) {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Content_ThreatType", "FUTURE_USE", "GeneratedTime",
"VirtualSystem", "EventID", "Object", "FUTURE_USE", "FUTURE_USE", "Module", "Severity", "Description",
"SequenceNumber", "ActionFlags", "DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2",
"DeviceGroupHierarchyLevel3", "DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName"
]
}
mutate { add_tag => [ "PAN-OS_System"] }
mutate { rename => [ "[host]", "[host][name]"] }
}
else if ([message] =~ /GLOBALPROTECT/) {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE", "GeneratedTime",
"VirtualSystem", "EventID", "Stage", "AuthenticationMethod", "TunnelType", "SourceUser", "SourceRegion",
"MachineName", "PublicIP", "PublicIPv6", "PrivateIP", "PrivateIPv6", "HostID", "SerialNumber", "ClientVersion",
"ClientOS", "ClientOSVersion", "RepeatCount", "Reason", "Error", "Description", "Status", "Location",
"LoginDuration", "ConnectMethod", "ErrorCode", "Portal", "SequenceNumber", "ActionFlags"
]
}
mutate {
copy => { "Description" => "tmp_field1" }
}
#grok {
# match => { "tmp_field1" => "GlobalProtect (?<GlobalProtect.Action>[^\.]*)\. %{GREEDYDATA:tmp_field2}" }
#}
kv {
source => "tmp_field1"
target => "Desc"
trim_key => " "
trim_value => " ."
field_split => ","
value_split => ":"
}
mutate {
remove_field => ["tmp_field1"]
}
if [PublicIP] and [PublicIP] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
geoip {
source => "PublicIP"
target => "GeoIP"
}
}
mutate { add_tag => [ "PAN-OS_GLOBALPROTECT"] }
mutate { rename => [ "[host]", "[host][name]"] }
}
mutate {
# Original message has been fully parsed, so remove it.
remove_field => [ "message" ]
}
# Geolocate logs that have SourceIP if that SourceIP is a non-RFC1918 address
if [SourceAddress] and [SourceAddress] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
geoip {
source => "SourceAddress"
target => "SourceAddressGeo"
}
# Delete 0,0 in SourceAddressGeo.location if equal to 0,0
if ([SourceAddressGeo.location] and [SourceAddressGeo.location] =~ "0,0") {
mutate {
replace => [ "SourceAddressGeo.location", "" ]
}
}
}
# Geolocate logs that have DestinationAddress and if that DestinationAddress is a non-RFC1918 address
if [DestinationAddress] and [DestinationAddress] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
geoip {
source => "DestinationAddress"
target => "DestinationAddressGeo"
}
# Delete 0,0 in DestinationAddressGeo.location if equal to 0,0
if ([DestinationAddressGeo.location] and [DestinationAddressGeo.location] =~ "0,0") {
mutate {
replace => [ "DestinationAddressGeo.location", "" ]
}
}
}
# Takes the 5-tuple of source address, source port, destination address, destination port, and protocol and does a SHA1 hash to fingerprint the flow.This is a useful
# way to be able to do top N terms queries on flows, not just on one field.
if [SourceAddress] and [DestinationAddress] {
fingerprint {
concatenate_sources => true
method => "SHA1"
key => "logstash"
source => [ "SourceAddress", "SourcePort", "DestinationAddress", "DestinationPort", "Protocol" ]
}
}
#}
}
output {
if "PAN-OS_Traffic" in [tags] {
elasticsearch {
hosts => "logsys-ingest.elasticsearch:9200"
ssl => true
cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
ssl_certificate_verification => false
data_stream => "true"
data_stream_sync_fields => true
data_stream_dataset => "panos"
data_stream_namespace => "traffic"
}
}
else if "PAN-OS_Threat" in [tags] {
elasticsearch {
hosts => "logsys-ingest.elasticsearch:9200"
ssl => true
cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
ssl_certificate_verification => false
data_stream => "true"
data_stream_sync_fields => true
data_stream_dataset => "panos"
data_stream_namespace => "threat"
}
}
else if "PAN-OS_Config" in [tags] {
elasticsearch {
hosts => "logsys-ingest.elasticsearch:9200"
ssl => true
cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
ssl_certificate_verification => false
data_stream => "true"
data_stream_sync_fields => true
data_stream_dataset => "panos"
data_stream_namespace => "config"
}
}
else if "PAN-OS_System" in [tags] {
elasticsearch {
hosts => "logsys-ingest.elasticsearch:9200"
ssl => true
cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
ssl_certificate_verification => false
data_stream => "true"
data_stream_sync_fields => true
data_stream_dataset => "panos"
data_stream_namespace => "system"
}
}
else if "PAN-OS_HIPMATCH" in [tags] {
elasticsearch {
hosts => "logsys-ingest.elasticsearch:9200"
ssl => true
cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
ssl_certificate_verification => false
data_stream => "true"
data_stream_sync_fields => true
data_stream_dataset => "panos"
data_stream_namespace => "hip"
}
}
else if "PAN-OS_GLOBALPROTECT" in [tags] {
elasticsearch {
hosts => "logsys-ingest.elasticsearch:9200"
ssl => true
cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
ssl_certificate_verification => false
data_stream => "true"
data_stream_sync_fields => true
data_stream_dataset => "panos"
data_stream_namespace => "globalprotect"
}
}
#if "plain" in [type] {
# stdout {
# codec => rubydebug
# }
#}
}
# Extra environment variables to append to this nodeGroup
# This will be appended to the current 'env:' key. You can use any of the kubernetes env
# syntax here
extraEnvs:
- name: 'ELASTICSEARCH_USERNAME'
valueFrom:
secretKeyRef:
name: elastic-credentials
key: username
- name: 'ELASTICSEARCH_PASSWORD'
valueFrom:
secretKeyRef:
name: elastic-credentials
key: password
# A list of secrets and their paths to mount inside the pod
secretMounts:
- name: elastic-certificates
secretName: elastic-certificate-crt
path: /usr/share/elasticsearch/config/certs
resources:
requests:
cpu: "3"
memory: "3Gi"
limits:
cpu: "4"
memory: "5Gi"
volumeClaimTemplate:
accessModes: [ "ReadWriteOnce" ]
storageClassName: nfs-logsys
resources:
requests:
storage: 110Gi
persistence:
enabled: true
annotations: {}
service:
annotations:
external-dns.alpha.kubernetes.io/hostname: logsys-input-panos.foo.bar
type: LoadBalancer
ports:
- name: paloalto-input
port: 5244
protocol: TCP
Hi, First of all, thank you very much for the article. How can I do Globalprotect log forwarding for panos-10 thank you.