Terraform and csv for data source

terraform

So I am trying to use Terraform to provision multiple VM's in vSphere. I have a CSV that contains the information for provisioning the guests. When I hard code the data sources the VM's are provisioned, but when I try to use the csv to populate those same sources Terraform plan tells me that the datacenter is not defined. I have tried both CSVDECODE and the interpolation methods, neither has worked.

example csvdecode:

locals {
  virtualmachines = csvdecode(file("${path.module}/virtualmachines.csv"))
}

data "vsphere_datacenter" "dc" {
  count = length(local.virtualmachines)
  name =  local.virtualmachines[count.index].dcname
}

example interpolation:


locals {
  virtualmachines = csvdecode(file("${path.module}/server_list.csv"))
}

    # Creating terraform resources from a CSV file using interpolation

    data "null_data_source" "csv_file" {
      inputs = {
        file_data = "${chomp(file("${path.module}/virtualmachines.csv"))}"
               }
    }

resource "null_resource" "csv_interpolation_method" {
  count = "${length(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))))}"

  triggers = {
    01 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 0)}"
    02 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 1)}"
    03 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 2)}"
    04 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 3)}"
    05 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 4)}"
    06 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 5)}"
    07 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 6)}"
    08 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 7)}"
    09 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 8)}"
    10 = "${element(split(",", element(slice(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")), 1, length(split("\n", lookup(data.null_data_source.csv_file.outputs, "file_data")))), count.index)), 9)}"
  }
}


locals {
  dcname    = null_resource.csv_interpolation_method.*.triggers.01
  storage   = null_resource.csv_interpolation_method.*.triggers.02
  pool      = null_resource.csv_interpolation_method.*.triggers.03
  vtemplate = null_resource.csv_interpolation_method.*.triggers.04
  vname     = null_resource.csv_interpolation_method.*.triggers.05
  cpu       = null_resource.csv_interpolation_method.*.triggers.06
  memory    = null_resource.csv_interpolation_method.*.triggers.07
  disk      = null_resource.csv_interpolation_method.*.triggers.08
  network   = null_resource.csv_interpolation_method.*.triggers.09
  guest_id  = null_resource.csv_interpolation_method.*.triggers.10
}

output "server_output" {
  value = "${local.dcname}"
}

data "vsphere_datacenter" "dc" {
  name = "${output.server_output}"
}

output "storage" {
  value = "${local.storage}"
}

data "vsphere_datastore" "datastore" {
  name          = "${local.storage}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

output "pool" {
  value = "${local.pool}"
}

data "vsphere_resource_pool" "pool" {
  name          = "${local.pool}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

output "network" {
  value = "${local.network}"
}

data "vsphere_network" "network" {
  name          = "${local.network}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

output "template" {
  value = "${local.vtemplate}"
}

data "vsphere_virtual_machine" "template" {
  name          = "${local.vtemplate}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}


resource "vsphere_virtual_machine" "vm" {
  count = length(local.virtualmachines)

  name             = local.virtualmachines[count.index].vname
  resource_pool_id = "${data.vsphere_resource_pool.pool.id}"
  datastore_id     = "${data.vsphere_datastore.datastore.id}"

  num_cpus                   = local.virtualmachines[count.index].cpu
  memory                     = local.virtualmachines[count.index].memory
  guest_id                   = local.virtualmachines[count.index].guest_id
  wait_for_guest_net_timeout = 0

  network_interface {
    network_id = "${data.vsphere_network.network.id}"
  }

  disk {
    label = "disk0"
    size  = local.virtualmachines[count.index].disk
  }

  clone {
    template_uuid = "${data.vsphere_virtual_machine.template.id}"

sample csv content:

dcname,storage,pool,vtemplate,vname,cpu,memory,disk,network,guest_id
txpla-w01-dc,txpla-w01-vsan,txpla-w01-shared01/Resources,AHLC75-template,terraform-test1,2,1024,80,Guest VLAN 1017,centos64Guest
txpla-w02-dc,txpla-w02-vsan,txpla-w02-shared01/Resources,AHLC75-template,terraform-test2,3,2048,80,Guest VLAN 1017,centos64Guest

Error message from interpolation method (it never makes it down to the resource)

A managed resource "output" "server_output" has not been declared in the root module

Any help would be appreciated..

Thank you

Best Answer

In Terraform, output blocks declare values to be passed to the calling module (or, if you're already in the root module, to be shown in the console output after applying).

You can't refer to them from within the module that defines them, and so Terraform is interpreting output.server_output as a resource reference, like you had a block in your configuration like resource "output" "server_output", from a hypothetical provider named "output" (which doesn't actually exist).

If you want to refer to a particular value from multiple places in the same module without duplicating its expression, you can use Local Values instead. You already have local.dcname, which is what output "server_output" refers to, so if you replace the output.server_output reference with local.dcname instead then it should produce the result value you were looking for.

Related Solutions

Terraform dynamic variable

I think an output map is the right approach. I'm not sure what went wrong when you tried it, but try something like:

output "subnets_map" {
  dev = "${aws_subnet.subnet_aza_dev.id}"
  test = "${aws_subnet.subnet_aza_test.id}"
}

Then

subnet_id = "${data.terraform_remote_state.env_shared.subnets_map[var.environment]}"

BTW I think it's better to use underscores rather than hyphens in HCL

Terraform – Choosing Credentials for a Remote State File

Your first question

if my provider setting is explicitly declaring the credentials to use inside the Terraform source code, why does my OS-level AWS configuration matter at all?

The error message "Failed to load backend: Error configuring the backend "s3"" is referring to your Backend S3 configuration.

Look in the file ./.terraform/terraform.tfstate and you will see the S3 Backend configuration.

The Terraform S3 Backend is different than the Terraform AWS Provider. The error message "No valid credential sources found for AWS Provider." is misleading. It implies that the AWS Provider configuration is used, which is false. S3 Backend credentials are configured separately and stored in the terraform.tfstate file.

Your OS-level AWS configuration matters because if no S3 Backend credentials are specified, as documented here https://www.terraform.io/docs/backends/types/s3.html, then Terraform defaults to using the following, in order:

Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
AWS Shared credentials file, default value is "~/.aws/credentials".

You didn't specify any credentials in your S3 Backend config so terraform is defaulting to the AWS Shared Credentials File.

Your S3 Backend configuration contains no credentials.

terraform {
  backend "s3" {
    bucket = "example_tf_states"
    key    = "global/vpc/us_east_1/example_state.tfstate"
    encrypt = true
    region     = "us-east-1"
  }
}

Your second question,

How can I make Terraform only use the creds defined in my Terraform configuration and ignore what's in my OS user profile?

First, Backends cannot contain interpolation, see https://www.terraform.io/docs/backends/config.html. So you cannot use any variables in the Backend config. e.g. this config is invalid

terraform {
  backend "s3" {
    bucket = "example_tf_states"
    key    = "global/vpc/us_east_1/example_state.tfstate"
    encrypt = true
    region     = "us-east-1"
    access_key = ${var.access_key}
    secret_key = ${var.secret_key}
  }
}

If you want to specify AWS credentials when running terraform init you specify backend configuration as options.

terraform init --backend-config="access_key=your_access_key" --backend-config="secret_key=your_secret_key"

This produces a S3 Backend config that looks like this, stored in the ./.terraform/terraform.tfstate file:

{
    "version": 3,
    "serial": 1,
    "lineage": "bd737d2d-1181-ed64-db57-467d14d2155a",
    "backend": {
        "type": "s3",
        "config": {
            "access_key": "your_access_key",
            "secret_key": "your_secret_key"
        },
        "hash": 9345827190033900985
    },

Again, the S3 Backend credentials are configured separately from your AWS Provider credentials.

Re-run terraform init and specify the credentials on the command line as --backend-config options to fix your error.

Best Answer

Related Solutions

Terraform dynamic variable

Terraform – Choosing Credentials for a Remote State File

Related Topic