STACKITCIN-311 Adjust Grafana TF module to allow Postgres datasources

grafana/datasource/datasource.tf

@@ -1,52 +1,81 @@
-# main.tf
+# Step 1: Create the basic "shell" of each datasource.
-
-# Step 1: Create all Grafana data sources with their basic configuration.
-# This resource establishes the fundamental properties of each data source.
 resource "grafana_data_source" "this" {
   for_each = var.datasources
 
-  name     = each.key
+  name       = each.key
-  type     = each.value.type
+  type       = each.value.type
-  url      = var.datasource_urls[each.value.url_key]
+  url        = var.datasource_urls[each.value.url_key]
   is_default = coalesce(each.value.is_default, false)
 
-  basic_auth_enabled  = true
+  # For HTTP Basic Auth (Loki, Prometheus, etc.)
-  basic_auth_username = var.datasource_users[each.value.user_key]
+  # FIX: Changed 'user_key' to 'basic_auth_user_key' to match your variables.tf
+  basic_auth_enabled  = each.value.basic_auth_user_key != null
+  basic_auth_username = each.value.basic_auth_user_key != null ? var.datasource_users[each.value.basic_auth_user_key] : null
 
-  secure_json_data_encoded = jsonencode({
+  # For database usernames (like Postgres)
-    basicAuthPassword = var.datasource_passwords[each.value.pass_key]
+  # This sets the username initially.
-  })
+  username = each.value.db_user_key != null ? var.datasource_users[each.value.db_user_key] : null
 
-  # Encodes initial, non-dependent JSON data.
+  # CRITICAL FIX: This resource must ignore attributes that are
-  # Configurations that depend on other datasource UIDs will be handled separately.
+  # managed by the other 'config' resources below.
-  json_data_encoded = each.value.json_data != null ? jsonencode(each.value.json_data) : null
-
-  # ignore changes made by the _config resource
   lifecycle {
     ignore_changes = [
       json_data_encoded,
+      secure_json_data_encoded,
+      # Also ignore username, as it can be managed/reported back differently by the API.
+      username,
     ]
   }
 }
 
-# Step 2: Apply Loki-specific 'derivedFields' configuration.
+# Step 2: Apply the main json_data for datasources like PostgreSQL.
-# This resource targets Loki data sources that need to link to another data source (like Tempo).
+resource "grafana_data_source_config" "json_data_main" {
-# It runs after the initial data sources are created to resolve the UIDs.
-resource "grafana_data_source_config" "loki_derived_fields" {
-  # Filter for datasources that are of type 'loki' and have 'derived_fields' defined.
   for_each = {
     for k, v in var.datasources : k => v
-    if v.type == "loki" && v.derived_fields != null
+    if v.json_data != null && v.derived_fields == null && v.traces_to_logs == null
   }
 
-  # The UID of the Loki data source to configure.
   uid = grafana_data_source.this[each.key].uid
 
-  # Construct the json_data with the derivedFields.
+  json_data_encoded = jsonencode(each.value.json_data)
+
+  # This config must ignore the password, which is managed by the 'passwords' resource.
+  lifecycle {
+    ignore_changes = [secure_json_data_encoded]
+  }
+}
+
+# Step 3: Apply passwords to all datasources that require one.
+resource "grafana_data_source_config" "passwords" {
+  for_each = {
+    for k, v in var.datasources : k => v if v.pass_key != null
+  }
+
+  uid = grafana_data_source.this[each.key].uid
+
+  secure_json_data_encoded = jsonencode(
+    each.value.type == "grafana-postgresql-datasource" ? {
+      password = var.datasource_passwords[each.value.pass_key]
+      } : {
+      basicAuthPassword = var.datasource_passwords[each.value.pass_key]
+    }
+  )
+
+  # This config must ignore the main json_data, which is managed elsewhere.
+  lifecycle {
+    ignore_changes = [json_data_encoded]
+  }
+}
+
+# Step 4: Apply Loki-specific 'derivedFields' configuration.
+resource "grafana_data_source_config" "loki_derived_fields" {
+  for_each = {
+    for k, v in var.datasources : k => v if v.type == "loki" && v.derived_fields != null
+  }
+  uid               = grafana_data_source.this[each.key].uid
   json_data_encoded = jsonencode({
     derivedFields = [
       for field in each.value.derived_fields : {
-        # The UID of the target data source (e.g., Tempo).
         datasourceUid = grafana_data_source.this[field.target_datasource_name].uid
         matcherRegex  = field.matcher_regex
         name          = field.name
@@ -54,24 +83,21 @@ resource "grafana_data_source_config" "loki_derived_fields" {
       }
     ]
   })
+
+  # This config must ignore the password, which is managed elsewhere.
+  lifecycle {
+    ignore_changes = [secure_json_data_encoded]
+  }
 }
 
-# Step 3: Apply Tempo-specific 'tracesToLogsV2' configuration.
+# Step 5: Apply Tempo-specific 'tracesToLogsV2' configuration.
-# This resource targets Tempo data sources that need to link back to a logging data source (like Loki).
 resource "grafana_data_source_config" "tempo_traces_to_logs" {
-  # Filter for datasources that are of type 'tempo' and have 'traces_to_logs' defined.
   for_each = {
-    for k, v in var.datasources : k => v
+    for k, v in var.datasources : k => v if v.type == "tempo" && v.traces_to_logs != null
-    if v.type == "tempo" && v.traces_to_logs != null
   }
-
+  uid               = grafana_data_source.this[each.key].uid
-  # The UID of the Tempo data source to configure.
-  uid = grafana_data_source.this[each.key].uid
-
-  # Construct the json_data with the tracesToLogsV2 settings.
   json_data_encoded = jsonencode({
     tracesToLogsV2 = {
-      # The UID of the target data source (e.g., Loki).
       datasourceUid      = grafana_data_source.this[each.value.traces_to_logs.target_datasource_name].uid
       query              = each.value.traces_to_logs.query
       customQuery        = coalesce(each.value.traces_to_logs.custom_query, true)
@@ -81,4 +107,9 @@ resource "grafana_data_source_config" "tempo_traces_to_logs" {
       spanEndTimeShift   = each.value.traces_to_logs.span_end_time_shift
     }
   })
-}
+
+  # This config must ignore the password, which is managed elsewhere.
+  lifecycle {
+    ignore_changes = [secure_json_data_encoded]
+  }
+}

grafana/datasource/variables.tf

@@ -1,32 +1,33 @@
-# variables.tf
-
-# Define datasources (non-sensitive metadata only)
 variable "datasources" {
   description = <<EOT
 Map of datasources to create. Keys are datasource names.
-Each datasource specifies type (prometheus/loki/tempo), keys to lookup URL/user/password,
+Each datasource specifies type, keys to lookup URL/user/password,
 and optional configurations for linking data sources.
 EOT
   type = map(object({
-    type       = string           # e.g., prometheus, loki, tempo
+    type                  = string
-    url_key    = string           # key for URL lookup in datasource_urls map
+    url_key               = string
-    user_key   = string           # key for username lookup in datasource_users map
+    pass_key              = optional(string)
-    pass_key   = string           # key for password lookup in datasource_passwords map
+    is_default            = optional(bool)
-    is_default = optional(bool)   # true if this datasource should be Grafana default
-    json_data  = optional(map(any)) # Initial, non-dependent JSON data
 
-    # Optional: For Loki, to link to a tracing datasource like Tempo.
+    # Key to look up a database username
+    db_user_key           = optional(string)
+    # Key to look up a basic auth username
+    basic_auth_user_key   = optional(string)
+
+    # ADDED BACK: Non-sensitive JSON data for Postgres, etc.
+    json_data             = optional(map(any))
+
+    # Linking Attributes (for Loki/Tempo)
     derived_fields = optional(list(object({
-      target_datasource_name = string # The name of the target datasource (e.g., "tempo-main")
+      target_datasource_name = string
-      matcher_regex          = string # Regex to find the trace ID in logs.
+      matcher_regex          = string
-      name                   = string # Name for the derived field (e.g., "traceID")
+      name                   = string
-      url                    = string # URL template, e.g., "$${__value.raw}"
+      url                    = string
     })))
-
-    # Optional: For Tempo, to link to a logging datasource like Loki.
     traces_to_logs = optional(object({
-      target_datasource_name = string # The name of the target datasource (e.g., "loki-main")
+      target_datasource_name = string
-      query                  = string # The query to run in the target datasource.
+      query                  = string
       custom_query           = optional(bool)
       filter_by_span_id      = optional(bool)
       filter_by_trace_id     = optional(bool)
@@ -36,7 +37,6 @@ EOT
   }))
 }
 
-# Sensitive maps for URLs, usernames, passwords
 variable "datasource_urls" {
   description = "Map of datasource URLs, keyed by url_key"
   type        = map(string)
@@ -44,7 +44,7 @@ variable "datasource_urls" {
 }
 
 variable "datasource_users" {
-  description = "Map of datasource usernames, keyed by user_key"
+  description = "Map of datasource usernames, keyed by db_user_key or basic_auth_user_key"
   type        = map(string)
   sensitive   = true
 }
@@ -53,4 +53,4 @@ variable "datasource_passwords" {
   description = "Map of datasource passwords, keyed by pass_key"
   type        = map(string)
   sensitive   = true
-}
+}