From ae40c80195641fcd1e4c8fae056adc1c3a17d0ea Mon Sep 17 00:00:00 2001 From: Stanislav Kopp Date: Tue, 26 Aug 2025 18:33:19 +0200 Subject: [PATCH 1/3] STACKITCIN-311 Adjust Grafana TF module to allow Postgres datasources --- grafana/datasource/datasource.tf | 107 ++++++++++++++++++++----------- grafana/datasource/variables.tf | 44 ++++++------- 2 files changed, 91 insertions(+), 60 deletions(-) diff --git a/grafana/datasource/datasource.tf b/grafana/datasource/datasource.tf index a2eec52..2fc1eab 100644 --- a/grafana/datasource/datasource.tf +++ b/grafana/datasource/datasource.tf @@ -1,52 +1,81 @@ -# main.tf - -# Step 1: Create all Grafana data sources with their basic configuration. -# This resource establishes the fundamental properties of each data source. +# Step 1: Create the basic "shell" of each datasource. resource "grafana_data_source" "this" { for_each = var.datasources - name = each.key - type = each.value.type - url = var.datasource_urls[each.value.url_key] + name = each.key + type = each.value.type + url = var.datasource_urls[each.value.url_key] is_default = coalesce(each.value.is_default, false) - basic_auth_enabled = true - basic_auth_username = var.datasource_users[each.value.user_key] + # For HTTP Basic Auth (Loki, Prometheus, etc.) + # FIX: Changed 'user_key' to 'basic_auth_user_key' to match your variables.tf + basic_auth_enabled = each.value.basic_auth_user_key != null + basic_auth_username = each.value.basic_auth_user_key != null ? var.datasource_users[each.value.basic_auth_user_key] : null - secure_json_data_encoded = jsonencode({ - basicAuthPassword = var.datasource_passwords[each.value.pass_key] - }) + # For database usernames (like Postgres) + # This sets the username initially. + username = each.value.db_user_key != null ? var.datasource_users[each.value.db_user_key] : null - # Encodes initial, non-dependent JSON data. - # Configurations that depend on other datasource UIDs will be handled separately. - json_data_encoded = each.value.json_data != null ? jsonencode(each.value.json_data) : null - - # ignore changes made by the _config resource + # CRITICAL FIX: This resource must ignore attributes that are + # managed by the other 'config' resources below. lifecycle { ignore_changes = [ json_data_encoded, + secure_json_data_encoded, + # Also ignore username, as it can be managed/reported back differently by the API. + username, ] } } -# Step 2: Apply Loki-specific 'derivedFields' configuration. -# This resource targets Loki data sources that need to link to another data source (like Tempo). -# It runs after the initial data sources are created to resolve the UIDs. -resource "grafana_data_source_config" "loki_derived_fields" { - # Filter for datasources that are of type 'loki' and have 'derived_fields' defined. +# Step 2: Apply the main json_data for datasources like PostgreSQL. +resource "grafana_data_source_config" "json_data_main" { for_each = { for k, v in var.datasources : k => v - if v.type == "loki" && v.derived_fields != null + if v.json_data != null && v.derived_fields == null && v.traces_to_logs == null } - # The UID of the Loki data source to configure. uid = grafana_data_source.this[each.key].uid - # Construct the json_data with the derivedFields. + json_data_encoded = jsonencode(each.value.json_data) + + # This config must ignore the password, which is managed by the 'passwords' resource. + lifecycle { + ignore_changes = [secure_json_data_encoded] + } +} + +# Step 3: Apply passwords to all datasources that require one. +resource "grafana_data_source_config" "passwords" { + for_each = { + for k, v in var.datasources : k => v if v.pass_key != null + } + + uid = grafana_data_source.this[each.key].uid + + secure_json_data_encoded = jsonencode( + each.value.type == "grafana-postgresql-datasource" ? { + password = var.datasource_passwords[each.value.pass_key] + } : { + basicAuthPassword = var.datasource_passwords[each.value.pass_key] + } + ) + + # This config must ignore the main json_data, which is managed elsewhere. + lifecycle { + ignore_changes = [json_data_encoded] + } +} + +# Step 4: Apply Loki-specific 'derivedFields' configuration. +resource "grafana_data_source_config" "loki_derived_fields" { + for_each = { + for k, v in var.datasources : k => v if v.type == "loki" && v.derived_fields != null + } + uid = grafana_data_source.this[each.key].uid json_data_encoded = jsonencode({ derivedFields = [ for field in each.value.derived_fields : { - # The UID of the target data source (e.g., Tempo). datasourceUid = grafana_data_source.this[field.target_datasource_name].uid matcherRegex = field.matcher_regex name = field.name @@ -54,24 +83,21 @@ resource "grafana_data_source_config" "loki_derived_fields" { } ] }) + + # This config must ignore the password, which is managed elsewhere. + lifecycle { + ignore_changes = [secure_json_data_encoded] + } } -# Step 3: Apply Tempo-specific 'tracesToLogsV2' configuration. -# This resource targets Tempo data sources that need to link back to a logging data source (like Loki). +# Step 5: Apply Tempo-specific 'tracesToLogsV2' configuration. resource "grafana_data_source_config" "tempo_traces_to_logs" { - # Filter for datasources that are of type 'tempo' and have 'traces_to_logs' defined. for_each = { - for k, v in var.datasources : k => v - if v.type == "tempo" && v.traces_to_logs != null + for k, v in var.datasources : k => v if v.type == "tempo" && v.traces_to_logs != null } - - # The UID of the Tempo data source to configure. - uid = grafana_data_source.this[each.key].uid - - # Construct the json_data with the tracesToLogsV2 settings. + uid = grafana_data_source.this[each.key].uid json_data_encoded = jsonencode({ tracesToLogsV2 = { - # The UID of the target data source (e.g., Loki). datasourceUid = grafana_data_source.this[each.value.traces_to_logs.target_datasource_name].uid query = each.value.traces_to_logs.query customQuery = coalesce(each.value.traces_to_logs.custom_query, true) @@ -81,4 +107,9 @@ resource "grafana_data_source_config" "tempo_traces_to_logs" { spanEndTimeShift = each.value.traces_to_logs.span_end_time_shift } }) -} + + # This config must ignore the password, which is managed elsewhere. + lifecycle { + ignore_changes = [secure_json_data_encoded] + } +} \ No newline at end of file diff --git a/grafana/datasource/variables.tf b/grafana/datasource/variables.tf index cbf19db..94ebfa8 100644 --- a/grafana/datasource/variables.tf +++ b/grafana/datasource/variables.tf @@ -1,32 +1,33 @@ -# variables.tf - -# Define datasources (non-sensitive metadata only) variable "datasources" { description = < Date: Tue, 26 Aug 2025 18:38:36 +0200 Subject: [PATCH 2/3] Fixes --- grafana/README.md | 18 +++++++++--------- grafana/datasource/datasource.tf | 13 ++++++------- grafana/datasource/variables.tf | 2 +- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/grafana/README.md b/grafana/README.md index 856b2d9..d0482a4 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -83,17 +83,17 @@ module "datasource" { datasources = { Thanos-Common-Infra-PRD = { - type = "prometheus" - url_key = "thanos_coin_prd" - user_key = "thanos_coin_prd" - pass_key = "thanos_coin_prd" - is_default = true + type = "prometheus" + url_key = "thanos_coin_prd" + basic_auth_user_key = "thanos_coin_prd" + pass_key = "thanos_coin_prd" + is_default = true } Loki-Common-Infra-PRD = { - type = "loki" - url_key = "loki_coin_prd" - user_key = "loki_coin_prd" - pass_key = "loki_coin_prd" + type = "loki" + url_key = "loki_coin_prd" + basic_auth_user_key = "loki_coin_prd" + pass_key = "loki_coin_prd" } } diff --git a/grafana/datasource/datasource.tf b/grafana/datasource/datasource.tf index 2fc1eab..34a22fe 100644 --- a/grafana/datasource/datasource.tf +++ b/grafana/datasource/datasource.tf @@ -1,4 +1,4 @@ -# Step 1: Create the basic "shell" of each datasource. +# Create the basic "shell" of each datasource. resource "grafana_data_source" "this" { for_each = var.datasources @@ -8,7 +8,6 @@ resource "grafana_data_source" "this" { is_default = coalesce(each.value.is_default, false) # For HTTP Basic Auth (Loki, Prometheus, etc.) - # FIX: Changed 'user_key' to 'basic_auth_user_key' to match your variables.tf basic_auth_enabled = each.value.basic_auth_user_key != null basic_auth_username = each.value.basic_auth_user_key != null ? var.datasource_users[each.value.basic_auth_user_key] : null @@ -16,7 +15,7 @@ resource "grafana_data_source" "this" { # This sets the username initially. username = each.value.db_user_key != null ? var.datasource_users[each.value.db_user_key] : null - # CRITICAL FIX: This resource must ignore attributes that are + # This resource must ignore attributes that are # managed by the other 'config' resources below. lifecycle { ignore_changes = [ @@ -28,7 +27,7 @@ resource "grafana_data_source" "this" { } } -# Step 2: Apply the main json_data for datasources like PostgreSQL. +# Apply the main json_data for datasources like PostgreSQL. resource "grafana_data_source_config" "json_data_main" { for_each = { for k, v in var.datasources : k => v @@ -45,7 +44,7 @@ resource "grafana_data_source_config" "json_data_main" { } } -# Step 3: Apply passwords to all datasources that require one. +# Apply passwords to all datasources that require one. resource "grafana_data_source_config" "passwords" { for_each = { for k, v in var.datasources : k => v if v.pass_key != null @@ -67,7 +66,7 @@ resource "grafana_data_source_config" "passwords" { } } -# Step 4: Apply Loki-specific 'derivedFields' configuration. +# Apply Loki-specific 'derivedFields' configuration. resource "grafana_data_source_config" "loki_derived_fields" { for_each = { for k, v in var.datasources : k => v if v.type == "loki" && v.derived_fields != null @@ -90,7 +89,7 @@ resource "grafana_data_source_config" "loki_derived_fields" { } } -# Step 5: Apply Tempo-specific 'tracesToLogsV2' configuration. +# Apply Tempo-specific 'tracesToLogsV2' configuration. resource "grafana_data_source_config" "tempo_traces_to_logs" { for_each = { for k, v in var.datasources : k => v if v.type == "tempo" && v.traces_to_logs != null diff --git a/grafana/datasource/variables.tf b/grafana/datasource/variables.tf index 94ebfa8..b505e8d 100644 --- a/grafana/datasource/variables.tf +++ b/grafana/datasource/variables.tf @@ -15,7 +15,7 @@ EOT # Key to look up a basic auth username basic_auth_user_key = optional(string) - # ADDED BACK: Non-sensitive JSON data for Postgres, etc. + # Non-sensitive JSON data for Postgres, etc. json_data = optional(map(any)) # Linking Attributes (for Loki/Tempo) From d5a0c28c11b1379833acdb7520cc7c4dc9bb4d45 Mon Sep 17 00:00:00 2001 From: Stanislav Kopp Date: Wed, 27 Aug 2025 09:44:37 +0200 Subject: [PATCH 3/3] format --- grafana/datasource/datasource.tf | 4 ++-- grafana/datasource/variables.tf | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/grafana/datasource/datasource.tf b/grafana/datasource/datasource.tf index 34a22fe..dd31ba8 100644 --- a/grafana/datasource/datasource.tf +++ b/grafana/datasource/datasource.tf @@ -71,7 +71,7 @@ resource "grafana_data_source_config" "loki_derived_fields" { for_each = { for k, v in var.datasources : k => v if v.type == "loki" && v.derived_fields != null } - uid = grafana_data_source.this[each.key].uid + uid = grafana_data_source.this[each.key].uid json_data_encoded = jsonencode({ derivedFields = [ for field in each.value.derived_fields : { @@ -94,7 +94,7 @@ resource "grafana_data_source_config" "tempo_traces_to_logs" { for_each = { for k, v in var.datasources : k => v if v.type == "tempo" && v.traces_to_logs != null } - uid = grafana_data_source.this[each.key].uid + uid = grafana_data_source.this[each.key].uid json_data_encoded = jsonencode({ tracesToLogsV2 = { datasourceUid = grafana_data_source.this[each.value.traces_to_logs.target_datasource_name].uid diff --git a/grafana/datasource/variables.tf b/grafana/datasource/variables.tf index b505e8d..a349c98 100644 --- a/grafana/datasource/variables.tf +++ b/grafana/datasource/variables.tf @@ -5,18 +5,18 @@ Each datasource specifies type, keys to lookup URL/user/password, and optional configurations for linking data sources. EOT type = map(object({ - type = string - url_key = string - pass_key = optional(string) - is_default = optional(bool) + type = string + url_key = string + pass_key = optional(string) + is_default = optional(bool) # Key to look up a database username - db_user_key = optional(string) + db_user_key = optional(string) # Key to look up a basic auth username - basic_auth_user_key = optional(string) + basic_auth_user_key = optional(string) # Non-sensitive JSON data for Postgres, etc. - json_data = optional(map(any)) + json_data = optional(map(any)) # Linking Attributes (for Loki/Tempo) derived_fields = optional(list(object({