Skip to content

Commit 5b2f436

Browse files
jchancellor-msjchancellor-ms
authored
syslog filtering sample for use in truncating AVS syslogs (Azure#273)
* syslog filtering sample for use in truncating AVS syslogs * Added grok rule to break out syslog messages * Updated URL to point to the main branch on ESLZ repo
1 parent 004ae0d commit 5b2f436

File tree

24 files changed

+1340
-0
lines changed

24 files changed

+1340
-0
lines changed

‎terraform/modules/avs_event_hub_for_logs/main.tf

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
########################################################################################
2+
#Deploy the Event hub items
3+
########################################################################################
4+
#Deploy the event hub namespace
5+
resource "azurerm_eventhub_namespace" "avs_log_processing" {
6+
name = var.eventhub_namespace_name
7+
location = var.rg_location
8+
resource_group_name = var.rg_name
9+
sku = "Standard"
10+
capacity = var.eventhub_capacity
11+
12+
tags = var.tags
13+
}
14+
15+
#deploy the event hub
16+
resource "azurerm_eventhub" "avs_log_processing" {
17+
name = var.eventhub_name
18+
namespace_name = azurerm_eventhub_namespace.avs_log_processing.name
19+
resource_group_name = var.rg_name
20+
partition_count = var.eventhub_partition_count
21+
message_retention = var.eventhub_message_retention_days
22+
}
23+
24+
#deploy the authorization rule for the diagnostic setting
25+
resource "azurerm_eventhub_namespace_authorization_rule" "avs_log_processing" {
26+
name = var.diagnostic_eventhub_authorization_rule_name
27+
namespace_name = azurerm_eventhub_namespace.avs_log_processing.name
28+
resource_group_name = var.rg_name
29+
30+
listen = true
31+
send = true
32+
manage = true
33+
}
34+
35+
#deploy the authorization rule for the plugin
36+
resource "azurerm_eventhub_authorization_rule" "avs_log_processing" {
37+
name = var.logstash_eventhub_authorization_rule_name
38+
namespace_name = azurerm_eventhub_namespace.avs_log_processing.name
39+
eventhub_name = azurerm_eventhub.avs_log_processing.name
40+
resource_group_name = var.rg_name
41+
42+
listen = true
43+
send = true
44+
manage = true
45+
}
46+
47+
#deploy an eventhub consumer group for use by the logstash plugin
48+
resource "azurerm_eventhub_consumer_group" "avs_log_processing" {
49+
name = var.consumer_group_name
50+
namespace_name = azurerm_eventhub_namespace.avs_log_processing.name
51+
eventhub_name = azurerm_eventhub.avs_log_processing.name
52+
resource_group_name = var.rg_name
53+
}
54+
55+
#deploy a storage account for use by the eventhub plugin to maintain state
56+
resource "azurerm_storage_account" "avs_log_processing" {
57+
name = var.plugin_storage_account_name
58+
resource_group_name = var.rg_name
59+
location = var.rg_location
60+
account_tier = "Standard"
61+
account_replication_type = "LRS"
62+
63+
tags = var.tags
64+
}
65+
66+
#############################################################################################
67+
# Telemetry Section - Toggled on and off with the telemetry variable
68+
# This allows us to get deployment frequency statistics for deployments
69+
# Re-using parts of the Core Enterprise Landing Zone methodology
70+
#############################################################################################
71+
locals {
72+
#create an empty ARM template to use for generating the deployment value
73+
telem_arm_subscription_template_content = <<TEMPLATE
74+
{
75+
"$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
76+
"contentVersion": "1.0.0.0",
77+
"parameters": {},
78+
"variables": {},
79+
"resources": [],
80+
"outputs": {
81+
"telemetry": {
82+
"type": "String",
83+
"value": "For more information, see https://aka.ms/alz/tf/telemetry"
84+
}
85+
}
86+
}
87+
TEMPLATE
88+
module_identifier = lower("avs_event_hub_for_logs")
89+
telem_arm_deployment_name = "${lower(var.guid_telemetry)}.${substr(local.module_identifier, 0, 20)}.${random_string.telemetry.result}"
90+
}
91+
92+
#create a random string for uniqueness
93+
resource "random_string" "telemetry" {
94+
length = 4
95+
special = false
96+
upper = false
97+
lower = true
98+
}
99+
100+
resource "azurerm_subscription_template_deployment" "telemetry_core" {
101+
count = var.module_telemetry_enabled ? 1 : 0
102+
103+
name = local.telem_arm_deployment_name
104+
location = var.rg_location
105+
template_content = local.telem_arm_subscription_template_content
106+
}

‎terraform/modules/avs_event_hub_for_logs/outputs.tf

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
output "event_hub_connection_string" {
2+
value = azurerm_eventhub_authorization_rule.avs_log_processing.primary_connection_string
3+
}
4+
5+
output "event_hub_consumer_group_name" {
6+
value = azurerm_eventhub_consumer_group.avs_log_processing.name
7+
}
8+
9+
output "event_hub_storage_account_name" {
10+
value = azurerm_storage_account.avs_log_processing.primary_connection_string
11+
}
12+
13+
output "event_hub_name" {
14+
value = azurerm_eventhub.avs_log_processing.name
15+
}
16+
17+
output "event_hub_authorization_rule_id" {
18+
value = azurerm_eventhub_namespace_authorization_rule.avs_log_processing.id
19+
}

‎terraform/modules/avs_event_hub_for_logs/readme.md

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
### General
2+
3+
* Description: This module creates an event hub namespace, eventhub, and consumer group as well as the two authorization rules used for processing. It also creates a storage account that is used by the event hub plugin as a witness to enable multiple consumers.
4+
5+
* The module leverages variables for naming and common values to be modified as part of the deployment.
6+
7+
8+

‎terraform/modules/avs_event_hub_for_logs/variables.tf

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
variable "rg_name" {
2+
type = string
3+
description = "The azure resource name for the resource group"
4+
}
5+
6+
variable "rg_location" {
7+
description = "Resource Group region location"
8+
default = "westus2"
9+
}
10+
11+
variable "eventhub_namespace_name" {
12+
type = string
13+
description = "The name for the eventhub namespace"
14+
}
15+
16+
variable "eventhub_capacity" {
17+
type = number
18+
description = "The number of eventhub capacity units on the namespace"
19+
default = 8
20+
}
21+
22+
variable "eventhub_name" {
23+
type = string
24+
description = "The name of the eventhub where the logs are being sent"
25+
}
26+
27+
variable "eventhub_partition_count" {
28+
type = number
29+
description = "The number of partitions for the eventhub"
30+
default = 2
31+
}
32+
33+
variable "eventhub_message_retention_days" {
34+
type = number
35+
description = "The number of days for message retention"
36+
default = 3
37+
}
38+
39+
variable "diagnostic_eventhub_authorization_rule_name" {
40+
type = string
41+
description = "Name for the authorization rule used by the eventhub diagnostic setting"
42+
}
43+
44+
variable "logstash_eventhub_authorization_rule_name" {
45+
type = string
46+
description = "Name for the authorization rule used by the logstash event hub plugin"
47+
}
48+
49+
variable "consumer_group_name" {
50+
type = string
51+
description = "The consumer group name for the event hub plugin to process the logs."
52+
}
53+
54+
variable "plugin_storage_account_name" {
55+
type = string
56+
description = "The storage account name for the storage account used by the logstash event hub plugin as a witness."
57+
}
58+
59+
variable "tags" {
60+
type = map(string)
61+
description = "List of the tags that will be assigned to each resource"
62+
}
63+
64+
#################################################################
65+
# telemetry variables
66+
#################################################################
67+
variable "module_telemetry_enabled" {
68+
type = bool
69+
description = "toggle the telemetry on/off for this module"
70+
default = true
71+
}
72+
73+
variable "guid_telemetry" {
74+
type = string
75+
description = "guid used for telemetry identification. Defaults to module guid, but overrides with root if needed."
76+
default = "0f9a8adc-9d37-40b3-aaed-ab34b95cf6dd"
77+
}
78+

0 commit comments

Comments
 (0)