From c57bc5ab2737803730c83a40197ca5792c5b6ef1 Mon Sep 17 00:00:00 2001 From: Satish Malireddi Date: Fri, 25 Oct 2019 15:39:45 -0700 Subject: [PATCH 1/5] node10 support (#573) --- .../provisioners/cookbooks/jenkins/attributes/default.rb | 6 +++--- installer/terraform/scripts/modifyCodebase.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/installer/terraform/provisioners/cookbooks/jenkins/attributes/default.rb b/installer/terraform/provisioners/cookbooks/jenkins/attributes/default.rb index 42fbc4ff..b243f8b2 100644 --- a/installer/terraform/provisioners/cookbooks/jenkins/attributes/default.rb +++ b/installer/terraform/provisioners/cookbooks/jenkins/attributes/default.rb @@ -19,12 +19,12 @@ default['maven']['setup_bin'] = true #Node cookbook property -default['nodejs']['version'] = '8' +default['nodejs']['version'] = '10' default['nodejs']['install_method'] = 'package' #This monkeypatch is necessary because the node cookbook brokenly defaults to a 6.x sources.list otherwise case node['platform_family'] when 'debian' - override['nodejs']['repo'] = 'https://deb.nodesource.com/node_8.x' + override['nodejs']['repo'] = 'https://deb.nodesource.com/node_10.x' when 'rhel', 'amazon' - default['nodejs']['repo'] = "https://rpm.nodesource.com/pub_8.x/el/$releasever/$basearch" + default['nodejs']['repo'] = "https://rpm.nodesource.com/pub_10.x/el/$releasever/$basearch" end diff --git a/installer/terraform/scripts/modifyCodebase.sh b/installer/terraform/scripts/modifyCodebase.sh index ab442e68..000820c3 100755 --- a/installer/terraform/scripts/modifyCodebase.sh +++ b/installer/terraform/scripts/modifyCodebase.sh @@ -121,7 +121,7 @@ do # shellcheck disable=SC2086 #Updating to service catalog -provider_runtime="nodejs8.10" +provider_runtime="nodejs10.x" pushToCatalog "$tablename" "$element" "$service_type" "${deployment_targets[@]}" "$provider_runtime" done From c2976d99677cc7650509e015d21a329254143ac3 Mon Sep 17 00:00:00 2001 From: rajeevr2715 Date: Wed, 27 Nov 2019 11:52:03 +0530 Subject: [PATCH 2/5] Node10 upgrade issue in existing jenkins --- .../cookbooks/jenkins/recipes/setupjenkins.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/installer/terraform/provisioners/cookbooks/jenkins/recipes/setupjenkins.rb b/installer/terraform/provisioners/cookbooks/jenkins/recipes/setupjenkins.rb index b97e7910..c0aa82f6 100644 --- a/installer/terraform/provisioners/cookbooks/jenkins/recipes/setupjenkins.rb +++ b/installer/terraform/provisioners/cookbooks/jenkins/recipes/setupjenkins.rb @@ -2,6 +2,21 @@ apt_update 'update' #force-update apt cache on Debian-derivatives to avoid pkg fetch errors package 'git' include_recipe 'maven::default' + +# Node won't upgrade to latest version in centos if already a version is present. +if platform_family?('rhel') + bash 'uninstall-nodejs' do + code <<-EOH + if [ -f /etc/os-release ]; then + rm -fr /var/cache/yum/* + yum remove -y nodejs + rm /etc/yum.repos.d/nodesource* + yum clean all + fi + EOH + end +end + include_recipe 'nodejs' include_recipe 'cloudcli' From 1dd3e617a4a988e785611352c518b4d6cc0f7d8d Mon Sep 17 00:00:00 2001 From: rajeevr2715 Date: Wed, 20 Nov 2019 18:36:17 +0530 Subject: [PATCH 3/5] Merged Automated script to delete lambda vpc enis in available state --- installer/terraform/commands.tf | 4 ++ .../terraform/scripts/clean_lambda_eni.py | 49 +++++++++++++++++++ installer/terraform/vpc_subnet.tf | 4 ++ 3 files changed, 57 insertions(+) create mode 100644 installer/terraform/scripts/clean_lambda_eni.py diff --git a/installer/terraform/commands.tf b/installer/terraform/commands.tf index 9139e7ef..b4d8a558 100755 --- a/installer/terraform/commands.tf +++ b/installer/terraform/commands.tf @@ -62,3 +62,7 @@ variable "config_cmd" { type = "string" default = "./scripts/config.py" } +variable "cleanEni_cmd" { + type = "string" + default = "./scripts/clean_lambda_eni.py" +} diff --git a/installer/terraform/scripts/clean_lambda_eni.py b/installer/terraform/scripts/clean_lambda_eni.py new file mode 100644 index 00000000..df83e04f --- /dev/null +++ b/installer/terraform/scripts/clean_lambda_eni.py @@ -0,0 +1,49 @@ +import boto3 +import sys +import time + + +def cleanup(client, vpc_id, sg_id): + in_use_found = False + try: + response = client.describe_network_interfaces( + Filters=[ + { + 'Name': 'vpc-id', + 'Values': [ + str(vpc_id), + ] + }, + { + 'Name': 'group-id', + 'Values': [ + str(sg_id), + ] + }, + ], + ) + for item in response['NetworkInterfaces']: + if "AWS Lambda VPC ENI" in item['Description']: + if item['Status'] == 'in-use': + in_use_found = True + if item['Status'] == 'available': + # delete the ENI + print("Item to delete: " + str(item['NetworkInterfaceId'])) + response = client.delete_network_interface( + NetworkInterfaceId=str(item['NetworkInterfaceId']) + ) + if in_use_found: + print("Still In use ENIs found") + time.sleep(30) + cleanup(client, vpc_id, sg_id) + else: + return True + except Exception as message: + print(message) + time.sleep(30) + cleanup(client, vpc_id, sg_id) + + +if __name__ == u"__main__": + client = boto3.client('ec2') + cleanup(client, sys.argv[1], sys.argv[2]) diff --git a/installer/terraform/vpc_subnet.tf b/installer/terraform/vpc_subnet.tf index 655c813b..dd4e28bf 100644 --- a/installer/terraform/vpc_subnet.tf +++ b/installer/terraform/vpc_subnet.tf @@ -211,6 +211,10 @@ resource "aws_subnet" "subnet_for_ecs_private" { availability_zone = "${element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)}" cidr_block = "${cidrsubnet(data.aws_vpc.vpc_data.cidr_block, ceil(log(4 * 2, 2)), 2 + count.index)}" tags = "${merge(var.additional_tags, local.common_tags)}" + provisioner "local-exec" { + when = "destroy" + command = "python ${var.cleanEni_cmd} ${data.aws_vpc.vpc_data.id} ${aws_security_group.vpc_sg.id}" + } } resource "aws_route_table" "privateroute" { From af1ddd6794d2c7440e062a4c9e642a8fea0afa03 Mon Sep 17 00:00:00 2001 From: rajeevr2715 Date: Tue, 3 Dec 2019 18:35:18 +0530 Subject: [PATCH 4/5] Handle ENI delete scenario1 fix --- installer/terraform/scripts/clean_lambda_eni.py | 2 -- installer/terraform/vpc_subnet.tf | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/installer/terraform/scripts/clean_lambda_eni.py b/installer/terraform/scripts/clean_lambda_eni.py index df83e04f..e7e9f254 100644 --- a/installer/terraform/scripts/clean_lambda_eni.py +++ b/installer/terraform/scripts/clean_lambda_eni.py @@ -40,8 +40,6 @@ def cleanup(client, vpc_id, sg_id): return True except Exception as message: print(message) - time.sleep(30) - cleanup(client, vpc_id, sg_id) if __name__ == u"__main__": diff --git a/installer/terraform/vpc_subnet.tf b/installer/terraform/vpc_subnet.tf index dd4e28bf..e1bad42e 100644 --- a/installer/terraform/vpc_subnet.tf +++ b/installer/terraform/vpc_subnet.tf @@ -213,7 +213,8 @@ resource "aws_subnet" "subnet_for_ecs_private" { tags = "${merge(var.additional_tags, local.common_tags)}" provisioner "local-exec" { when = "destroy" - command = "python ${var.cleanEni_cmd} ${data.aws_vpc.vpc_data.id} ${aws_security_group.vpc_sg.id}" + command = "python ${var.cleanEni_cmd} ${data.aws_vpc.vpc_data.id} ${var.dockerizedJenkins == 1 ? join(" ", aws_security_group.vpc_sg.*.id) : aws_security_group.vpc_sg_es_kibana.id }" + on_failure = "continue" } } From 8d43b98501a7b1fe7b68081d9f88509f83793375 Mon Sep 17 00:00:00 2001 From: rajeevr2715 Date: Tue, 3 Dec 2019 18:35:52 +0530 Subject: [PATCH 5/5] Apigee changed related to node10 lambda rename --- features/extensions/apigee/install.py | 2 +- features/extensions/apigee/terraformBugWorkaround.py | 7 +++---- features/extensions/apigee/uninstall.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/features/extensions/apigee/install.py b/features/extensions/apigee/install.py index b09c5473..adc0a898 100644 --- a/features/extensions/apigee/install.py +++ b/features/extensions/apigee/install.py @@ -108,7 +108,7 @@ def install(region, stackprefix, jazz_apiendpoint, jazz_userpass, jenkins_url, runTerraform(region, stackprefix, True) # TODO remove this entire module when the terraform bug is fixed click.secho('\nLinking new role to existing gateway function', fg='blue') - linkNewRoleToExistingFunctionWithCLI(stackprefix + "-" + gatewayFunctionName) + linkNewRoleToExistingFunctionWithCLI(stackprefix + "_" + gatewayFunctionName) jazz_userpass_list = ''.join(list(jazz_userpass)).split() jazz_username, jazz_password = jazz_userpass_list[0], jazz_userpass_list[1] jenkins_userpass_list = ''.join(list(jenkins_userpass)).split() diff --git a/features/extensions/apigee/terraformBugWorkaround.py b/features/extensions/apigee/terraformBugWorkaround.py index 45b4a9fa..6cea6319 100644 --- a/features/extensions/apigee/terraformBugWorkaround.py +++ b/features/extensions/apigee/terraformBugWorkaround.py @@ -10,8 +10,7 @@ terraformPreviousRoleOutput = "previous-role-arn" featureName = "Apigee" # The lambda function created has the format of ENVPREFIX-jazz-apigee-proxy-aws-STAGE -gatewayFunctionName = "jazz-apigee-proxy-aws-prod" -terraformGatewayResource = "aws_lambda_function.jazz-apigee-proxy" +gatewayFunctionName = "jazz_apigee-proxy-aws_prod" def runTerraform(region, envPrefix, install): @@ -30,9 +29,9 @@ def runTerraform(region, envPrefix, install): '-var', 'region={0}'.format(region), '-var', 'jazz_aws_accountid={0}'.format(getAWSAccountID()), '-var', 'env_prefix={0}'.format(envPrefix), - '-var', 'gateway_func_arn={0}'.format(getFunctionArn(envPrefix + "-" + + '-var', 'gateway_func_arn={0}'.format(getFunctionArn(envPrefix + "_" + gatewayFunctionName)), - '-var', 'previous_role_arn={0}'.format(getFunctionRole(envPrefix + "-" + + '-var', 'previous_role_arn={0}'.format(getFunctionRole(envPrefix + "_" + gatewayFunctionName)) ], cwd='extensions/apigee/terraform') diff --git a/features/extensions/apigee/uninstall.py b/features/extensions/apigee/uninstall.py index 98534652..59a0cb2a 100644 --- a/features/extensions/apigee/uninstall.py +++ b/features/extensions/apigee/uninstall.py @@ -56,7 +56,7 @@ def uninstall(region, stackprefix, jazz_userpass, jazz_apiendpoint, jenkins_url, click.secho('\nRestoring old role to gateway function', fg='blue') # Restore old role first, before we destroy the Terraform resources - restoreOldRoleToExistingFunctionWithCLI(stackprefix + "-" + gatewayFunctionName) + restoreOldRoleToExistingFunctionWithCLI(stackprefix + "_" + gatewayFunctionName) jazz_userpass_list = ''.join(list(jazz_userpass)).split() jazz_username, jazz_password = jazz_userpass_list[0], jazz_userpass_list[1] jenkins_userpass_list = ''.join(list(jenkins_userpass)).split()