obriensystems
commented
9 months ago Prepare for Splunk Cloud log shipping via org level log sink to pubsub target via dataflow cluster in PSC
follow
- https://cloud.google.com/architecture/stream-logs-from-google-cloud-to-splunk
- https://cloud.google.com/architecture/stream-logs-from-google-cloud-to-splunk/deployment
Script in https://github.com/GoogleCloudPlatform/pubsec-declarative-toolkit/tree/gh651-splunk
use as base https://github.com/GoogleCloudPlatform/pubsec-declarative-toolkit/blob/gh446-hub/solutions/setup.sh
export BOOT_PROJECT_ID=bigquery-ol
export PROJECT_ID=bigquery-ol
gcloud config set project $PROJECT_ID
export SUPER_ADMIN_EMAIL=$(gcloud config list --format json|jq .core.account | sed 's/"//g')
export ORGANIZATION_ID=$(gcloud projects get-ancestors $BOOT_PROJECT_ID --format='get(id)' | tail -1)
export PROJECT_NUMBER=$(gcloud projects list --filter="${PROJECT_ID}" '--format=value(PROJECT_NUMBER)')
export NETWORK_NAME=hub
export SUBNET_NAME=hub-sn
export REGION=northamerica-northeast1
export INPUT_TOPIC_NAME=bqtopic
export INPUT_SUBSCRIPTION_NAME=bqtopic-sub
export DEAD_LETTER_TOPIC_NAME=bq-dead-topic
export DEAD_LETTER_SUBSCRIPTION_NAME=bq-dead-topic-sub
export LOG_SINK_SERVICE_ACCOUNT=service-${PROJECT_NUMBER}@gcp-sa-logging.iam.gserviceaccount.com
export ORGANIZATION_SINK_NAME=org-sink-splunk
echo "BOOT_PROJECT: $BOOT_PROJECT_ID"
echo "PROJECT_ID: $PROJECT_ID"
echo "SUPER_ADMIN_EMAIL: $SUPER_ADMIN_EMAIL"
echo "ORGANIZATION_ID: $ORGANIZATION_ID"
echo "PROJECT_NUMBER: $PROJECT_NUMBER"
echo "NETWORK_NAME: $NETWORK_NAME"
echo "SUBNET_NAME: $SUBNET_NAME"
echo "REGION: $REGION"
echo "INPUT_TOPIC_NAME: $INPUT_TOPIC_NAME"
echo "INPUT_SUBSCRIPTION_NAME: $INPUT_SUBSCRIPTION_NAME"
echo "DEAD_LETTER_TOPIC_NAME: $DEAD_LETTER_TOPIC_NAME"
echo "DEAD_LETTER_SUBSCRIPTION_NAME: $DEAD_LETTER_SUBSCRIPTION_NAME"
echo "LOG_SINK_SERVICE_ACCOUNT: $LOG_SINK_SERVICE_ACCOUNT"
echo "ORGANIZATION_SINK_NAME: $ORGANIZATION_SINK_NAME"
# enable services
gcloud services enable monitoring.googleapis.com
gcloud services enable secretmanager.googleapis.com
gcloud services enable compute.googleapis.com
gcloud services enable pubsub.googleapis.com
gcloud services enable dataflow.googleapis.com
# grant roles
gcloud organizations add-iam-policy-binding $ORGANIZATION_ID --member=user:$SUPER_ADMIN_EMAIL --role=roles/logging.configWriter --quiet > /dev/null 1>&1
gcloud organizations add-iam-policy-binding $ORGANIZATION_ID --member=user:$SUPER_ADMIN_EMAIL --role=roles/compute.networkAdmin --quiet > /dev/null 1>&1
gcloud organizations add-iam-policy-binding $ORGANIZATION_ID --member=user:$SUPER_ADMIN_EMAIL --role=roles/compute.securityAdmin --quiet > /dev/null 1>&1
gcloud organizations add-iam-policy-binding $ORGANIZATION_ID --member=user:$SUPER_ADMIN_EMAIL --role=roles/secretmanager.admin --quiet > /dev/null 1>&1
gcloud compute networks create $NETWORK_NAME --subnet-mode=custom
gcloud compute networks subnets create $SUBNET_NAME \
--network=$NETWORK_NAME \
--region=$REGION \
--range=192.168.1.0/24
gcloud compute firewall-rules create allow-internal-dataflow \
--network=$NETWORK_NAME \
--action=allow \
--direction=ingress \
--target-tags=dataflow \
--source-tags=dataflow \
--priority=0 \
--rules=tcp:12345-12346
gcloud compute routers create nat-router \
--network=$NETWORK_NAME \
--region=$REGION
gcloud compute routers nats create nat-config \
--router=nat-router \
--nat-custom-subnet-ip-ranges=$SUBNET_NAME \
--auto-allocate-nat-external-ips \
--region=$REGION
gcloud compute networks subnets update $SUBNET_NAME \
--enable-private-ip-google-access \
--region=$REGION
gcloud pubsub topics create $INPUT_TOPIC_NAME
gcloud pubsub subscriptions create \
--topic $INPUT_TOPIC_NAME $INPUT_SUBSCRIPTION_NAME
echo "PROJECT_ID: $PROJECT_ID"
echo "INPUT_TOPIC_NAME: $INPUT_TOPIC_NAME"
gcloud logging sinks create $ORGANIZATION_SINK_NAME \
pubsub.googleapis.com/projects/${PROJECT_ID}/topics/${INPUT_TOPIC_NAME} \
--organization=$ORGANIZATION_ID \
--include-children \
--log-filter='NOT logName:projects/bigquery-ol/logs/dataflow.googleapis.com'
## fix project and topic rendering above
#gcloud pubsub topics add-iam-policy-binding $INPUT_TOPIC_NAME \
#--member=serviceAccount:service-951469276805@gcp-sa-logging.iam.gserviceaccount.com \
#--role=roles/pubsub.publisher
gcloud pubsub topics add-iam-policy-binding $INPUT_TOPIC_NAME \
--member=serviceAccount:${LOG_SINK_SERVICE_ACCOUNT} \
--role=roles/pubsub.publisher
gcloud pubsub topics create $DEAD_LETTER_TOPIC_NAME
gcloud pubsub subscriptions create --topic $DEAD_LETTER_TOPIC_NAME $DEAD_LETTER_SUBSCRIPTION_NAME
https://cloud.google.com/architecture/stream-logs-from-google-cloud-to-splunk/deployment
running
michael@cloudshell:~/bigquery-ol/pubsec-declarative-toolkit/solutions (bigquery-ol)$ ./setup-splunk-sink.sh
Updated property [core/project].
BOOT_PROJECT: bigquery-ol
PROJECT_ID: bigquery-ol
SUPER_ADMIN_EMAIL: mi..ev
ORGANIZATION_ID: 583..68
PROJECT_NUMBER: 951469276805
NETWORK_NAME: hub
SUBNET_NAME: hub-sn
REGION: northamerica-northeast1
INPUT_TOPIC_NAME: bqtopic
INPUT_SUBSCRIPTION_NAME: bqtopic-sub
DEAD_LETTER_TOPIC_NAME: bq-dead-topic
DEAD_LETTER_SUBSCRIPTION_NAME: bq-dead-topic-sub
LOG_SINK_SERVICE_ACCOUNT: service-951469276805@gcp-sa-logging.iam.gserviceaccount.com
ORGANIZATION_SINK_NAME: org-sink-splunk
Operation "operations/acat.p2-951469276805-e274bf61-e2a0-461f-86e7-03a476cd3b27" finished successfully.
Operation "operations/acf.p2-951469276805-f4e3d475-3c23-494e-8a2b-84b1a67a5107" finished successfully.
Operation "operations/acf.p2-951469276805-d1965720-dd9c-4c5e-981b-eb96689b123f" finished successfully.
Updated IAM policy for organization [583675367868].
Updated IAM policy for organization [583675367868].
Updated IAM policy for organization [583675367868].
Updated IAM policy for organization [583675367868].
Created [https://www.googleapis.com/compute/v1/projects/bigquery-ol/global/networks/hub].
NAME: hub
SUBNET_MODE: CUSTOM
BGP_ROUTING_MODE: REGIONAL
IPV4_RANGE:
GATEWAY_IPV4:
Instances on this network will not be reachable until firewall rules
are created. As an example, you can allow all internal traffic between
instances as well as SSH, RDP, and ICMP by running:
$ gcloud compute firewall-rules create <FIREWALL_NAME> --network hub --allow tcp,udp,icmp --source-ranges <IP_RANGE>
$ gcloud compute firewall-rules create <FIREWALL_NAME> --network hub --allow tcp:22,tcp:3389,icmp
Created [https://www.googleapis.com/compute/v1/projects/bigquery-ol/regions/northamerica-northeast1/subnetworks/hub-sn].
NAME: hub-sn
REGION: northamerica-northeast1
NETWORK: hub
RANGE: 192.168.1.0/24
STACK_TYPE: IPV4_ONLY
IPV6_ACCESS_TYPE:
INTERNAL_IPV6_PREFIX:
EXTERNAL_IPV6_PREFIX:
Creating firewall...working..Created [https://www.googleapis.com/compute/v1/projects/bigquery-ol/global/firewalls/allow-internal-dataflow].
Creating firewall...done.
NAME: allow-internal-dataflow
NETWORK: hub
DIRECTION: INGRESS
PRIORITY: 0
ALLOW: tcp:12345-12346
DENY:
DISABLED: False
Creating router [nat-router]...done.
NAME: nat-router
REGION: northamerica-northeast1
NETWORK: hub
Creating NAT [nat-config] in router [nat-router]...done.
Updated [https://www.googleapis.com/compute/v1/projects/bigquery-ol/regions/northamerica-northeast1/subnetworks/hub-sn].
Created topic [projects/bigquery-ol/topics/bqtopic].
Created subscription [projects/bigquery-ol/subscriptions/bqtopic-sub].
Created [https://logging.googleapis.com/v2/organizations/583675367868/sinks/org-sink-splunk].
Please remember to grant `serviceAccount:service-org-583675367868@gcp-sa-logging.iam.gserviceaccount.com` the Pub/Sub Publisher role on the topic.
More information about sinks can be found at https://cloud.google.com/logging/docs/export/configure_export
writer identity
serviceAccount:service-org-583675367868@gcp-sa-logging.iam.gserviceaccount.com
inclusion filter
NOT logName:projects/bigquery-ol/logs/dataflow.googleapis.com
fix project id
hardcoded project but there is an issue selecting the target - logs are ok, just need to fix queue topic selection
gcloud pubsub topics add-iam-policy-binding $INPUT_TOPIC_NAME \
--member=serviceAccount:${LOG_SINK_SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com \
--role=roles/pubsub.publisher
ERROR: Policy modification failed. For a binding with condition, run "gcloud alpha iam policies lint-condition" to identify issues in condition.
ERROR: (gcloud.pubsub.topics.add-iam-policy-binding) INVALID_ARGUMENT: Invalid service account (service-951469276805@gcp-sa-logging.iam.gserviceaccount.com@bigquery-ol.iam.gserviceaccount.com).
fixed
gcloud pubsub topics add-iam-policy-binding $INPUT_TOPIC_NAME \
--member=serviceAccount:service-951469276805@gcp-sa-logging.iam.gserviceaccount.com \
--role=roles/pubsub.publisher
Updated IAM policy for topic [bqtopic].
bindings:
- members:
- serviceAccount:service-951469276805@gcp-sa-logging.iam.gserviceaccount.com
role: roles/pubsub.publisher
etag: BwYJvRbrKsU=
version: 1Triage issue with sink service account permission on the PubSub topic
existing sink
Writer Identity serviceAccount:service-org-583...68@gcp-sa-logging.iam.gserviceaccount.com
PubSub topic
Testing
echo "Granting roles/pubsub.publisher to SA: ${LOG_SINK_SERVICE_ACCOUNT} on PubSub topic: ${INPUT_TOPIC_NAME}"
gcloud pubsub topics add-iam-policy-binding $INPUT_TOPIC_NAME \
--member=serviceAccount:${LOG_SINK_SERVICE_ACCOUNT} \
--role=roles/pubsub.publisher
Granting roles/pubsub.publisher to SA: service-951469276805@gcp-sa-logging.iam.gserviceaccount.com on PubSub topic: bqtopic
Updated IAM policy for topic [bqtopic].
bindings:
- members:
- serviceAccount:service-951469276805@gcp-sa-logging.iam.gserviceaccount.com
role: roles/pubsub.publisher
etag: BwYKNQy6cWo=
version: 1
michael@cloudshell:~/bigquery-ol/pubsec-declarative-toolkit/solutions (bigquery-ol)$
Service account is mismatched
service-org-583...68@gcp-sa-logging.iam.gserviceaccount.com instead of service-951...805@gcp-sa-logging.iam.gserviceaccount.com
export PUBSUB_WRITER_IDENTITY_SERVICE_ACCOUNT=service-org-${ORGANIZATION_ID}@gcp-sa-logging.iam.gserviceaccount.com
echo "Granting roles/pubsub.publisher to SA: $PUBSUB_WRITER_IDENTITY_SERVICE_ACCOUNT} on PubSub topic: ${INPUT_TOPIC_NAME}"
gcloud pubsub topics add-iam-policy-binding $INPUT_TOPIC_NAME \
--member=serviceAccount:${PUBSUB_WRITER_IDENTITY_SERVICE_ACCOUNT} \
--role=roles/pubsub.publisher
PUBSUB_WRITER_IDENTITY_SERVICE_ACCOUNT: service-org-58...868@gcp-sa-logging.iam.gserviceaccount.com
ORGANIZATION_SINK_NAME: org-sink-splunk
Granting roles/pubsub.publisher to SA: service-org-58...68@gcp-sa-logging.iam.gserviceaccount.com} on PubSub topic: bqtopic
Updated IAM policy for topic [bqtopic].
bindings:
- members:
- serviceAccount:service-951...805@gcp-sa-logging.iam.gserviceaccount.com
- serviceAccount:service-org-58...68@gcp-sa-logging.iam.gserviceaccount.com
role: roles/pubsub.publisher
etag: BwYKNSG_KO0=
version: 1
Messages now
Forcing some logs via
SELECT * FROM `bigquery-ol.rollerblade.rollerblade` LIMIT 10002.7 messages / sec
Pulling messages
fmichaelobrien
Note Log sink publishing delays - I see an hour+ on certain sinks - when using pubsub we are near realtime
thank you team for - https://support.google.com/a/answer/7061566
Results
Log Sink to Dataflow (apache beam) pipeline to Http Endpoint Collector
Notes
Architecture/Requirements
Support 3rd party log aggregation support for Splunk for example as a logging sync target
drive/shadow to https://github.com/GoogleCloudPlatform/pbmm-on-gcp-onboarding/issues/318
review splunk specific integrations
https://splunkbase.splunk.com/app/3088?_ga=2.266051020.2131231099.1699452385-302992155.1699452385&_gl=1*1oxfrkk*_ga*MzAyOTkyMTU1LjE2OTk0NTIzODU.*_ga_GS7YF8S63Y*MTY5OTQ1MjM4Ni4xLjAuMTY5OTQ1MjM4Ni42MC4wLjA.*_ga_5EPM2P39FV*MTY5OTQ1MjM4Ni4xLjEuMTY5OTQ1MjM4OS41Ny4wLjA.
review ELK stack export simulation
review IBM qradar integration https://www.ibm.com/docs/en/qradar-common?topic=collector-setting-maximum-eps-rate