I am trying to create a terraform module for AWS EMR cluster. I need to run multiple bootstrap scripts in EMR, where I am having errors.
For example:
main.tf
...
variable bootstrap_actions { type = "list"}
...
resource "aws_emr_cluster" "emr-cluster" {
name = "${var.emr_name}"
release_label = "${var.release_label}"
applications = "${var.applications}"
termination_protection = true
ec2_attributes {
subnet_id = "${data.aws_subnet.subnet.id}"
emr_managed_master_security_group = "${data.aws_security_group.emr_managed_master_security_group.id}"
emr_managed_slave_security_group = "${data.aws_security_group.emr_managed_slave_security_group.id}"
service_access_security_group = "${data.aws_security_group.service_access_security_group.id}"
additional_slave_security_groups = "${var.additional_slave_security_groups_id}"
instance_profile = "${var.instance_profile}"
key_name = "${var.key_name}"
}
master_instance_type = "${var.master_instance_type}"
core_instance_type = "${var.core_instance_type}"
core_instance_count = "${var.core_instance_count}"
tags {
BUSINESS_UNIT = "${var.BUSINESS_UNIT}"
BUSINESS_REGION = "${var.BUSINESS_REGION}"
CLIENT = "${var.CLIENT}"
ENVIRONMENT = "${var.env}"
PLATFORM = "${var.PLATFORM}"
Name = "${var.emr_name}"
}
bootstrap_action = "${var.bootstrap_actions}"
configurations = "test-fixtures/emr_configurations.json"
service_role = "${var.service_role}"
autoscaling_role = "${var.autoscaling_role}"
}
I passed all the variables including bootstrap as :
bootstrap_actions = [ "path=s3://bucket/bootstrap/hive/metastore/JSON41.sh,name=SERDE","path=s3://bucket/bootstrap/hive/hive-nofile-nproc-increase.sh,name=ulimit" ]
When I am applying the plan, I am getting error:
* aws_emr_cluster.emr-cluster: bootstrap_action.0: expected object, got invalid
* aws_emr_cluster.emr-cluster: bootstrap_action.1: expected object, got invalid
Does anyone have any idea on it? How can I pass multiple bootstrap actions here.
Please advise.
Thanks.
It works :
bootstrap_action = [
{
name = "custombootrstrap_test1"
path = "s3://${aws_s3_bucket.bucketlogs.bucket}/bootstrap-actions/master/configure-test1.sh"
},
{
name = "custombootrstrap_test2"
path = "s3://${aws_s3_bucket.bucketlogs.bucket}/bootstrap-actions/master/configure-test2.sh"
},
]
Doc: bootstrap_action - (Optional) list of bootstrap actions that will be run before Hadoop is started on the cluster nodes.
Your variable is list of strings, not list of objects. This variable is probably ok (not tested):
bootstrap_actions = [
{
path = "s3://bucket/bootstrap/hive/metastore/JSON41.sh"
name = "SERDE"
},
{
path = "s3://bucket/bootstrap/hive/hive-nofile-nproc-increase.sh"
name = "ulimit"
},
]
It should probably look more like this, https://github.com/hashicorp/terraform-provider-aws/issues/22416
bootstrap_action {
path = "s3://${var.emrS3LogUri}/folder1/folder2/emr1.sh"
name = "emr1"
args = ["s3://${var.emrS3LogUri}/folder1/certs/", var.s3BucketRegion]
}
bootstrap_action {
path = "s3://${var.emrS3LogUri}/folder1/folder2/emr2.sh"
name = "emr2"
}
Related
I am preparing to deploy Aurora MySQL Cluster, I have defined the following in the “variable.tf” under root:
variable "db_subnet_id" {
type = list
description = `DB Subnet IDs`
default = [“subnet-02ddc8565555aaa9b”,“subnet-1d30f3a41ce19635d”] #db-sub-eu-central-1a , db-sub-eu-central-1b
}
variable `db-sg` {
type = list
description = “List of standard security groups”
default = [`“sg-00cfed28101ea95ab”,“sg-08017f86bc12348e8”,“sg-0e8c67c7a3cd79a79”`]
}
variable `db_az` {
type = list
description = “Frankfurt Availability Zones”
default = [“eu-central-1a”,“eu-central-1b”]
}
how can I use those variables with the following parameters:
availability_zones
db_subnet_group_name
vpc_security_group_ids
I have tried to use this vailability_zones = var.db_az[count.index] but the following error shows up:
The “count” object can only be used in “module”, “resource”, and “data” blocks, and only when the “count” argument is set.
please advise, how can I use those variables with those parameters ?
Thanks
I am interested to see if anyone knows of any better alternative to using conditional count statements in Terraform. By "conditional count statement", I mean a statement where depending a condition like a variable input, count will evaluate to create either 0 or 1 of a resource.
A simple example:
resource "xxxxxxxx" "example" {
count = var.example != null ? 1 : 0
}
Here, the resource will only be created if var.example has a value (is not null), otherwise it will not get created.
Conditional counts usually work ok in practice, but sometimes in more complex uses than the one above it introduces a risk of getting an error during Terraform Plan where it cannot evaluate the result of the count pre-apply.
The "count" value depends on resource attributes that cannot be determined
until apply, so Terraform cannot predict how many instances will be created.
To work around this, use the -target argument to first apply only the
resources that the count depends on.
Is there any better way to achieve the same effect of creating resources on a conditional basis in Terraform?
Keeping in mind that your terraform config not shown I've used a general example of for_each instead of count.
This is an example of how 2 CNAME records will be created using & in this example terraform modules are used but the same can be done directly on terraform resources.
locals {
cname_records = {
"email" = ["email.domain.net."]
"imap" = ["imap.domain.net."]
}
}
module "aws_route53_record_CNAME" {
source = "app.terraform.io/terraform-cloud-org/route53-record/aws"
version = "1.0.0"
for_each = local.cname_records
records = each.value
name = each.key
zone_id = "YOUR-HOSTED-ZONE-ID"
type = "CNAME"
ttl = "300"
}
It varies by case, but there are some cases where you need to get around this. There is a clever trick, but it seems obtuse in the single item (binary exists or not) case. But in a case where there are actually multiple items, this should help.
This is a completely contrived example but I actually use the method quite a bit. In short, a list of things not known until apply, can be replaced by a list of objects with a key you don't care about. The purpose is that for_each doesn't mind if the value is unknown at plan-time, only if the key is.
Consider the following root module with these four modules.
main.tf
resource "aws_s3_bucket" "this" {
bucket = "h4s-test-bucket"
}
# module "with_count" { # cannot be determined until apply
# source = "./with-count"
# x = aws_s3_bucket.this.id
# }
module "with_for_each_over_item" { # handy workaround
source = "./with-for-each-over-item"
x = aws_s3_bucket.this.id
}
output "with_for_each_over_item" {
value = module.with_for_each_over_item
}
# module "with_for_each_list" { # cannot be determined until apply
# source = "./with-for-each-list"
# x = [aws_s3_bucket.this.id]
# }
module "with_for_each_list_better" { # handy workaround
source = "./with-for-each-list-better"
x = [{ y = aws_s3_bucket.this.id }]
}
output "with_for_each_list_better" {
value = module.with_for_each_list_better
}
module "with_for_each_list_best" { # handier workaround
source = "./with-for-each-list-best"
x = [aws_s3_bucket.this.id]
}
output "with_for_each_list_best" {
value = module.with_for_each_list_best
}
with-count/main.tf (problematic)
variable "x" {
type = string
default = null
}
resource "null_resource" "this" {
count = var.x != null ? 1 : 0
}
output "this" {
value = null_resource.this
}
with-for-each-over-item/main.tf (handy workaround)
variable "x" {
type = string
default = null
}
resource "null_resource" "this" {
for_each = { for i, v in [var.x] : i => v }
}
output "this" {
value = null_resource.this
}
with-for-each-list/main.tf (problematic)
variable "x" {
type = list(string)
default = []
}
resource "null_resource" "this" {
for_each = toset(var.x)
}
output "this" {
value = null_resource.this
}
with-for-each-list-better/main.tf (handy workaround)
variable "x" {
type = list(object({ y = string }))
default = []
}
resource "null_resource" "this" {
for_each = { for i, v in var.x : i => v }
}
output "this" {
value = null_resource.this
}
with-for-each-list-best/main.tf (handiest workaround)
variable "x" {
type = list(string)
default = []
}
resource "null_resource" "this" {
for_each = { for i, v in var.x : i => v }
}
output "this" {
value = null_resource.this
}
Summary
In cases where the variable has a value not known at plan-time, consider using an object where the key is known.
I am trying to find a way how to pass/map the list variable to kubernetes_deployment container environment variables. I run an asp.net core application in the container and I trying to modify the app with setting env variables to override the app settings.json. It is not a problem to override single values from settings.json, the problem is if I need to define/override the whole array there.
I have variable list like this in terrafrom:
variable "allowed_cars" {
type = list(
object({
manufacturer = string
model = string
})
)
}
Then, I have resource definition of kubernetes_deployment with containers.
I believe it could work if I set the env variables for the container like this:
env {
name = "App__AllowedCars__0__Manufacturer"
value = "xxx"
}
env {
name = "App__AllowedCars__0__Model"
value = "xxx"
}
env {
name = "App__AllowedCars__1__Manufacturer"
value = "xxx"
}
env {
name = "App__AllowedCars__1__Model"
value = "xxx"
}
...
Is there a way how to pass these env variables to the container in a dynamic way based on allowed_cars terraform variable? I don't know how many items will be defined for each environments, etc...
Thank you very much.
Something of this definition (used example of azurerm_container_group in azure)
environment_variables = "${merge(var.env_vars,var.secure_env_vars,local.master_env)}"
and then variables can be passed in
variable "env_vars" {
type = "map"
description = "envvaars"
default = {
WEB_USER = "locust"
HATCH_RATE = 25
LOCUST_COUNT = 50
LOCUST_FILE = "/locust/locustfile.py"
ATTACKED_HOST = "https://api-perf.yrdy.com"
}
}
variable "secure_env_vars" {
type = "map"
description = "secure env vars"
default = {
WEB_PASSWORD = "dummy"
API_KEY = "test"
}
}
As #ydaetskcoR mentioned, the dynamic block was solution for me.
https://www.terraform.io/docs/language/expressions/dynamic-blocks.html
dynamic "env" {
for_each = var.allowed_cars
content {
name = "App__AllowedCars__0__${env.key}__Manufacturer"
value = env.value["manufacturer"]
}
}
I have a list of tags that I use for different resources:
common_tags = {
Application = "${var.application}"
Environment = "${var.environment}"
}
I added an aws_autoscaling_group where tags can be also provided, but the structure required is different and has to be defined in this way:
tags = [{
key = "explicit1"
value = "value1"
propagate_at_launch = true
}]
I need to convert the map of tags to a list of maps with the new structure.
Right now I do this:
variable "application" {
default = "MyApplication"
}
variable "environment" {
default = "production"
}
locals {
common_tags = {
Application = "${var.application}"
Environment = "${var.environment}"
}
my_service_tags = "${list(
map("key", "Name", "value", "my_service_tags-${var.environment}", "propagate_at_launch", true),
map("key", "Application", "value", var.application, "propagate_at_launch", true),
map("key", "Environment", "value", var.environment, "propagate_at_launch", true))}"
}
So as you can see the tags are defined twice.
Is there any way of define the new structure (my_service_tags) using the previous one (common_tags) ?
Thanks.
Note: This question can also be found on the WiX mailing list.
I need to be able to check for the existence of an IIS7 website based on the website's description. If the website does not exist I need to cancel the installation. If the website exists I want to continue the installation. I also need to be able to save the site id of the website so that I may use it during an uninstall.
For debugging purposes I have hard coded the website's description. I do not see any indication that a check for the website is being made within the MSI log file. This is the code I am using:
<iis:WebSite Id="IISWEBSITE" Description="Default Web Site" SiteId="*">
<iis:WebAddress Id="IisWebAddress" Port="1"/>
</iis:WebSite>
<Condition Message="Website [IISWEBSITE] not found.">
<![CDATA[IISWEBSITE]]>
</Condition>
Using ORCA I can see that IIsWebAddress and IIsWebSite tables are added to the MSI. The values are:
IIsWebsite
WEB: IISWEBSITE
Description: Default Web Site
KeyAddress: IisWebAddress
Id: -1
IIsWebAddress
Address: IisWebAddress
Web_: IISWEBSITE
Port: 1
Secure: 0
With the above code, the installation is halted with the error message "Website not found". It appears that IISWEBSITE is never getting set. Though, I know that "Default Web Site" exists. I know that I must be missing something, but what?
How can I perform a simple check for the existence of a website in IIS 7?
I too had same problem.
I wrote a custom action to check the version of IIS from registry.
On the basis of registry value create virtual directory
I wrote a custom action in Javascript to do this. If you are assuming IIS7, then you can use the appcmd.exe tool, and just invoke it from within Javascript to get the list of sites. In theory, it's pretty simple to do. But in practice, there's a bunch of hoops you need to jump through.
Here's what I came up with:
function RunAppCmd(command, deleteOutput) {
var shell = new ActiveXObject("WScript.Shell"),
fso = new ActiveXObject("Scripting.FileSystemObject"),
tmpdir = fso.GetSpecialFolder(SpecialFolders.TemporaryFolder),
tmpFileName = fso.BuildPath(tmpdir, fso.GetTempName()),
windir = fso.GetSpecialFolder(SpecialFolders.WindowsFolder),
appcmd = fso.BuildPath(windir,"system32\\inetsrv\\appcmd.exe") + " " + command,
rc;
deleteOutput = deleteOutput || false;
LogMessage("shell.Run("+appcmd+")");
// use cmd.exe to redirect the output
rc = shell.Run("%comspec% /c " + appcmd + "> " + tmpFileName, WindowStyle.Hidden, true);
LogMessage("shell.Run rc = " + rc);
if (deleteOutput) {
fso.DeleteFile(tmpFileName);
}
return {
rc : rc,
outputfile : (deleteOutput) ? null : tmpFileName
};
}
// GetWebSites_Appcmd()
//
// Gets website info using Appcmd.exe, only on IIS7+ .
//
// The return value is an array of JS objects, one per site.
//
function GetWebSites_Appcmd() {
var r, fso, textStream, sites, oneLine, record,
ParseOneLine = function(oneLine) {
// split the string: capture quoted strings, or a string surrounded
// by parens, or lastly, tokens separated by spaces,
var tokens = oneLine.match(/"[^"]+"|\(.+\)|[^ ]+/g),
// split the 3rd string: it is a set of properties separated by colons
props = tokens[2].slice(1,-1),
t2 = props.match(/\w+:.+?(?=,\w+:|$)/g),
bindingsString = t2[1],
ix1 = bindingsString.indexOf(':'),
t3 = bindingsString.substring(ix1+1).split(','),
L1 = t3.length,
bindings = {}, i, split, obj, p2;
for (i=0; i<L1; i++) {
split = t3[i].split('/');
obj = {};
if (split[0] == "net.tcp") {
p2 = split[1].split(':');
obj.port = p2[0];
}
else if (split[0] == "net.pipe") {
p2 = split[1].split(':');
obj.other = p2[0];
}
else if (split[0] == "http") {
p2 = split[1].split(':');
obj.ip = p2[0];
if (p2[1]) {
obj.port = p2[1];
}
obj.hostname = "";
}
else {
p2 = split[1].split(':');
obj.hostname = p2[0];
if (p2[1]) {
obj.port = p2[1];
}
}
bindings[split[0]] = obj;
}
// return the object describing the website
return {
id : t2[0].split(':')[1],
name : "W3SVC/" + t2[0].split(':')[1],
description : tokens[1].slice(1,-1),
bindings : bindings,
state : t2[2].split(':')[1] // started or not
};
};
LogMessage("GetWebSites_Appcmd() ENTER");
r = RunAppCmd("list sites");
if (r.rc !== 0) {
// 0x80004005 == E_FAIL
throw new Exception("ApplicationException", "exec appcmd.exe returned nonzero rc ("+r.rc+")", 0x80004005);
}
fso = new ActiveXObject("Scripting.FileSystemObject");
textStream = fso.OpenTextFile(r.outputfile, OpenMode.ForReading);
sites = [];
// Read from the file and parse the results.
while (!textStream.AtEndOfStream) {
oneLine = textStream.ReadLine();
record = ParseOneLine(oneLine);
LogMessage(" site: " + record.name);
sites.push(record);
}
textStream.Close();
fso.DeleteFile(r.outputfile);
LogMessage("GetWebSites_Appcmd() EXIT");
return sites;
}