-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathmain.tf
139 lines (107 loc) · 4.71 KB
/
main.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# Pub/Sub topics
resource "google_pubsub_topic" "log-topic" {
name = "${var.dataflow_base_name}-log-topic"
project = var.gcp_log_project
}
resource "google_pubsub_topic" "log-dl-topic" {
name = "${var.dataflow_base_name}-log-dl-topic"
project = var.gcp_log_project
}
# Pub/Sub subscriptions
resource "google_pubsub_subscription" "log-sub" {
name = "${var.dataflow_base_name}-log-sub"
project = var.gcp_log_project
topic = google_pubsub_topic.log-topic.name
message_retention_duration = var.pubsub_msg_retention
}
resource "google_pubsub_subscription" "log-dl-sub" {
name = "${var.dataflow_base_name}-log-dl-sub"
project = var.gcp_log_project
topic = google_pubsub_topic.log-dl-topic.name
message_retention_duration = var.pubsub_msg_retention
}
# Log sinks
resource "google_logging_organization_sink" "log-org-sink" {
name = "${var.dataflow_base_name}-log-org-sink"
count = var.log_sink_org_enable == 1 ? 1 : 0
org_id = var.log_sink_org_id
include_children = true
filter = templatefile("${path.module}/log-org-sink-filter-catchall.tpl", {
org_id = var.log_sink_org_id,
log_project = var.gcp_log_project,
dataflow_name = var.dataflow_job_enable == 1 ? google_dataflow_job.splunk-job[0].name : ""
})
destination = "pubsub.googleapis.com/${google_pubsub_topic.log-topic.id}"
}
resource "google_logging_project_sink" "log-proj-sink" {
name = "${var.dataflow_base_name}-log-proj-sink"
count = var.log_sink_proj_enable == 1 ? 1 : 0
project = var.log_sink_proj_name
filter = templatefile("${path.module}/log-proj-sink-filter-catchall.tpl", {
log_project = var.gcp_log_project,
dataflow_name = var.dataflow_job_enable == 1 ? google_dataflow_job.splunk-job[0].name : ""
})
unique_writer_identity = true
destination = "pubsub.googleapis.com/${google_pubsub_topic.log-topic.id}"
}
# Pub/Sub topic IAM policy
resource "google_pubsub_topic_iam_member" "log-org-topic-sink-member" {
count = var.log_sink_org_enable == 1 ? 1 : 0
project = google_pubsub_topic.log-topic.project
topic = google_pubsub_topic.log-topic.name
role = "roles/pubsub.publisher"
member = google_logging_organization_sink.log-org-sink.0.writer_identity
}
resource "google_pubsub_topic_iam_member" "log-proj-topic-sink-member" {
count = var.log_sink_proj_enable == 1 ? 1 : 0
project = google_pubsub_topic.log-topic.project
topic = google_pubsub_topic.log-topic.name
role = "roles/pubsub.publisher"
member = google_logging_project_sink.log-proj-sink.0.writer_identity
}
# Bucket for Dataflow temp storage
resource "google_storage_bucket" "log-bucket" {
name = "${var.dataflow_base_name}-log-bucket"
project = var.gcp_log_project
location = var.gcp_region
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_object" "temp" {
name = "${var.dataflow_base_name}/temp/.ignore"
content = "IGNORE"
bucket = google_storage_bucket.log-bucket.name
depends_on = [google_storage_bucket.log-bucket]
}
resource "google_storage_bucket_object" "splunk-udf" {
name = "${var.dataflow_base_name}/js/splunk-udf.js"
content = templatefile("${path.module}/dataflow-udf-js.tpl", {
input_sub = google_pubsub_subscription.log-sub.name
})
bucket = google_storage_bucket.log-bucket.name
depends_on = [google_storage_bucket.log-bucket]
}
# Dataflow job
resource "google_dataflow_job" "splunk-job" {
name = "${var.dataflow_base_name}-job"
count = var.dataflow_job_enable == 1 ? 1 : 0
project = var.gcp_log_project
region = var.gcp_region
zone = var.gcp_zone
machine_type = var.dataflow_worker_machine_type
max_workers = var.dataflow_max_workers
template_gcs_path = "gs://dataflow-templates-${var.gcp_region}/latest/Cloud_PubSub_to_Splunk"
temp_gcs_location = "${google_storage_bucket.log-bucket.url}/${dirname(google_storage_bucket_object.temp.name)}"
parameters = {
inputSubscription = google_pubsub_subscription.log-sub.id
outputDeadletterTopic = google_pubsub_topic.log-dl-topic.id
url = var.splunk_hec_url
token = var.splunk_hec_token
javascriptTextTransformGcsPath = "${google_storage_bucket.log-bucket.url}/${google_storage_bucket_object.splunk-udf.name}"
javascriptTextTransformFunctionName = "transform"
disableCertificateValidation = var.splunk_hec_disable_cert_validation
batchCount = var.dataflow_batchCount,
parallelism = var.dataflow_parallelism
}
on_delete = "cancel"
}