From 0448f7e125bf05a9354851c1f0f78bb52e4d09e1 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Fri, 8 Sep 2023 11:38:17 +0100 Subject: [PATCH] Remove karpenter experiment --- karpenter-experiment/.terraform.lock.hcl | 118 ----------------------- karpenter-experiment/eks.tf | 111 --------------------- karpenter-experiment/helm.tf | 42 -------- karpenter-experiment/karpenter.tf | 41 -------- karpenter-experiment/main.tf | 29 ------ 5 files changed, 341 deletions(-) delete mode 100644 karpenter-experiment/.terraform.lock.hcl delete mode 100644 karpenter-experiment/eks.tf delete mode 100644 karpenter-experiment/helm.tf delete mode 100644 karpenter-experiment/karpenter.tf delete mode 100644 karpenter-experiment/main.tf diff --git a/karpenter-experiment/.terraform.lock.hcl b/karpenter-experiment/.terraform.lock.hcl deleted file mode 100644 index ab96fc3..0000000 --- a/karpenter-experiment/.terraform.lock.hcl +++ /dev/null @@ -1,118 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/gavinbunney/kubectl" { - version = "1.14.0" - constraints = "~> 1.14" - hashes = [ - "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", - "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", - "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", - "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", - "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", - "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", - "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", - "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", - "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", - "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", - ] -} - -provider "registry.terraform.io/hashicorp/aws" { - version = "4.34.0" - constraints = "~> 4.0" - hashes = [ - "h1:TMVXbfjowAI4MjMDCU7AJwCUzfufoSC/v6/v85sAOlg=", - "zh:2bdc9b908008c1e874d8ba7e2cfabd856cafb63c52fef51a1fdeef2f5584bffd", - "zh:43c5364e3161be3856e56494cbb8b21d513fc05875f1b40e66e583602154dd0a", - "zh:44e763adae92489f223f65866c1f8b5342e7e85b95daa8d1f483a2afb47f7db3", - "zh:62bfabb3a1a31814cb3fadc356539d8253b95abacfffd90984affb54c9a53a86", - "zh:6495ce67897d2d5648d466c09e8588e837c2878322988738a95c06926044b05d", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:b1546b2ac61d7cc27a8eba160ae1b6ac581d2c4af824a400d6511e4998da398a", - "zh:c8c362c5527f0533bde581e41cdb2bdf42aea557762f326dbfa122fdf001fb10", - "zh:c8cc28fb41f73ca09f590bace2204ea325f6116be0bbce6abfebd393d028f12c", - "zh:db0601c9bd12ca028d60ac87e85320285ebc64857715f7908dd6a283e5f44d45", - "zh:e64d2193236d05348ba2e8b99650d9274e5af80be39b3ee28bbe442b0684d8a3", - "zh:ff6228f3751e1f0ee7dc086d09e32d39ca6556f0b5267f36aae67881d29ace94", - ] -} - -provider "registry.terraform.io/hashicorp/cloudinit" { - version = "2.2.0" - constraints = ">= 2.0.0" - hashes = [ - "h1:tQLNREqesrdCQ/bIJnl0+yUK+XfdWzAG0wo4lp10LvM=", - "zh:76825122171f9ea2287fd27e23e80a7eb482f6491a4f41a096d77b666896ee96", - "zh:795a36dee548e30ca9c9d474af9ad6d29290e0a9816154ad38d55381cd0ab12d", - "zh:9200f02cb917fb99e44b40a68936fd60d338e4d30a718b7e2e48024a795a61b9", - "zh:a33cf255dc670c20678063aa84218e2c1b7a67d557f480d8ec0f68bc428ed472", - "zh:ba3c1b2cd0879286c1f531862c027ec04783ece81de67c9a3b97076f1ce7f58f", - "zh:bd575456394428a1a02191d2e46af0c00e41fd4f28cfe117d57b6aeb5154a0fb", - "zh:c68dd1db83d8437c36c92dc3fc11d71ced9def3483dd28c45f8640cfcd59de9a", - "zh:cbfe34a90852ed03cc074601527bb580a648127255c08589bc3ef4bf4f2e7e0c", - "zh:d6ffd7398c6d1f359b96f5b757e77b99b339fbb91df1b96ac974fe71bc87695c", - "zh:d9c15285f847d7a52df59e044184fb3ba1b7679fd0386291ed183782683d9517", - "zh:f7dd02f6d36844da23c9a27bb084503812c29c1aec4aba97237fec16860fdc8c", - ] -} - -provider "registry.terraform.io/hashicorp/helm" { - version = "2.7.1" - constraints = "~> 2.5" - hashes = [ - "h1:11oWNeohjD8Fy9S7WQSKY3GmDZi7gVdMRp8/Wqxn410=", - "zh:13e2467092deeff01c4cfa2b54ba4510aa7a9b06c58f22c4215b0f4333858364", - "zh:4549843db4fdf5d8150e8c0734e67b54b5c3bcfc914e3221e6952f428fb984d2", - "zh:55b5f83ed52f93dd00a73c33c948326052efd700350c19e63bb1679b12bfcda6", - "zh:749397e41393289eb0ef6efd0a75911d29b8aa7f48e5d6813b4b350dad91acbd", - "zh:7a4a2c95b055f6c8e70d1fc7a4cc4fd6e4f04845be36e40d42d31dfc13db37b8", - "zh:8143e5b8218857052505c805b570889b862c618ce6cbfbddb98938ff7a5901d3", - "zh:856d94b3b34d6204d66c6de4feab4737c74dba037ad64e4c613e8eec61d17f1a", - "zh:b9b037f1edda209022df1c7fc906786970524873e27b061f3355cb9bbed2cf08", - "zh:c433b27f52a0600490af07f8b217ab0b1048ba347d68e6fe478aba18634e78d9", - "zh:da133748368c6e27b433cd7faeb7b800536c8651e7af0415452901dfc7577dbf", - "zh:eecc63c2dec8aafa2ffd7426800c3e1a5e31e848be01ea9511ad0184dce15945", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.14.0" - constraints = ">= 2.10.0" - hashes = [ - "h1:FFeFf2j2ipbMlrbhmIv8M7bzX3Zq8SQHeFkkQGALh1k=", - "zh:1363fcd6eb3c63113eaa6947a4e7a9f78a6974ea344e89b662d97a78e2ccb70c", - "zh:166352455666b7d584705ceeb00f24fb9b884ab84e3a1a6019dc45d6539c9174", - "zh:4615249ce5311f6fbea9738b25b6e6159e7dcf4693b0a24bc6a5720d1bfd38d0", - "zh:5205343f8e6cfa89d2f9a312edddcf263755bc294a5216555c390244df826f17", - "zh:60b7d9b5da2d1a13bc9cdfe5be75da2e3d1034617dff51ef3f0beb72fe801879", - "zh:61b73d78ef03f0b38ff567b78f2984089eb17724fd8d0f92943b7e522cf31e39", - "zh:69dfe1278eecc6049736d74c3fa2d1f384035621ec5d72f8b180e3b25b45b592", - "zh:7746656be1b437e43f7324898cd4548d7e8cad5308042ba38cb45c4fecbf38fe", - "zh:7e573462091aaf2e6a37edeee33ee4d8f4c37f9a35c331e0f3a60caf078c88c1", - "zh:a05e1f02b2385679087a7059944cac7fb1d71dd042601ee4d0d26e9808d14dd5", - "zh:d8d5d52af1aa55160fec601a1006552d9b6fe21e97313850a1e79bc026e99cfe", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.3" - constraints = ">= 3.0.0" - hashes = [ - "h1:TtxH+PuCVPoWUpkMhjdz4QB1i7A4/Ag9kIouPabKDrI=", - "zh:0b34a21c535db27a71d1b76a635352ad5dc31d81cdee3d34926629c8919990a1", - "zh:160008f7a89ec99f1a79d05bd9c2f11679edbcac5ecf43f618d0a260d003d5ce", - "zh:2b951a9c75be26ff320d979c121832866db75f08bcbe326cf5290c8497e345a4", - "zh:5032695a3fb47914fa6037f21e7f06d0b684ffddde0c758477a63008476e97d0", - "zh:581f518e5104e5bdbdb9f5af35628dcb04a6fee11c6279e4ac1480d7a2bde39f", - "zh:66f36f9da00e7a39952e9e4cf158b552a98d8afabbb74804bf27b872b4fdefc6", - "zh:a159589b44d353376f86d05f1819952445f9a1c00a3bcc2745217d6f513f5147", - "zh:de077f6b852c8d637dbd9b9542ffd590db3d96f9f718acee6645ac6e22365b3c", - "zh:e0ea922619aea6b97f416fd8c129af944df9f37b81417ae7b970c6895498843e", - "zh:e7705f4f35332f598e6d4078c7e7e734e4728c5ec8642a50672e29d51d8f3be3", - "zh:eb64e7a5ebd37b8f59acc91e033897a5d454998f70e57047c94dbfb649503320", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} diff --git a/karpenter-experiment/eks.tf b/karpenter-experiment/eks.tf deleted file mode 100644 index 82fcfb8..0000000 --- a/karpenter-experiment/eks.tf +++ /dev/null @@ -1,111 +0,0 @@ -module "vpc" { - # https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest - source = "terraform-aws-modules/vpc/aws" - version = "3.14.2" - - name = local.cluster_name - cidr = "10.0.0.0/16" - - azs = ["eu-west-2a", "eu-west-2b", "eu-west-2c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] - - enable_nat_gateway = true - single_nat_gateway = true - one_nat_gateway_per_az = false - - public_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 - } -} - -module "eks" { - # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest - source = "terraform-aws-modules/eks/aws" - version = "18.29.0" - - cluster_name = local.cluster_name - cluster_version = "1.22" - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - # Required for Karpenter role below - enable_irsa = true - - node_security_group_additional_rules = { - ingress_nodes_karpenter_port = { - description = "Cluster API to Node group for Karpenter webhook" - protocol = "tcp" - from_port = 8443 - to_port = 8443 - type = "ingress" - source_cluster_security_group = true - } - } - - node_security_group_tags = { - # NOTE - if creating multiple security groups with this module, only tag the - # security group that Karpenter should utilize with the following tag - # (i.e. - at most, only one security group should have this tag in your account) - "karpenter.sh/discovery/${local.cluster_name}" = local.cluster_name - } - - # Only need one node to get Karpenter up and running. - # This ensures core services such as VPC CNI, CoreDNS, etc. are up and running - # so that Karpenter can be deployed and start managing compute capacity as required - eks_managed_node_groups = { - initial = { - instance_types = ["m5.large"] - # Not required nor used - avoid tagging two security groups with same tag as well - create_security_group = false - - # Ensure enough capacity to run 2 Karpenter pods - min_size = 2 - max_size = 3 - desired_size = 2 - - iam_role_additional_policies = [ - # Required by Karpenter - "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" - ] - - tags = { - # This will tag the launch template created for use by Karpenter - "karpenter.sh/discovery/${local.cluster_name}" = local.cluster_name - } - } - } -} - -resource "aws_iam_instance_profile" "karpenter" { - name = "KarpenterNodeInstanceProfile-${local.cluster_name}" - role = module.eks.eks_managed_node_groups["initial"].iam_role_name -} - -module "karpenter_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.3.1" - - role_name = "karpenter-controller-${local.cluster_name}" - attach_karpenter_controller_policy = true - - karpenter_tag_key = "karpenter.sh/discovery/${local.cluster_name}" - karpenter_controller_cluster_id = module.eks.cluster_id - karpenter_controller_node_iam_role_arns = [ - module.eks.eks_managed_node_groups["initial"].iam_role_arn - ] - - oidc_providers = { - ex = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] - } - } -} \ No newline at end of file diff --git a/karpenter-experiment/helm.tf b/karpenter-experiment/helm.tf deleted file mode 100644 index 2413ddd..0000000 --- a/karpenter-experiment/helm.tf +++ /dev/null @@ -1,42 +0,0 @@ -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", local.cluster_name] - } - } -} - -resource "helm_release" "karpenter" { - namespace = "karpenter" - create_namespace = true - - name = "karpenter" - repository = "oci://public.ecr.aws/karpenter" - chart = "karpenter" - version = "v0.18.0" - - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter_irsa.iam_role_arn - } - - set { - name = "clusterName" - value = module.eks.cluster_id - } - - set { - name = "clusterEndpoint" - value = module.eks.cluster_endpoint - } - - set { - name = "aws.defaultInstanceProfile" - value = aws_iam_instance_profile.karpenter.name - } -} \ No newline at end of file diff --git a/karpenter-experiment/karpenter.tf b/karpenter-experiment/karpenter.tf deleted file mode 100644 index 1f82a44..0000000 --- a/karpenter-experiment/karpenter.tf +++ /dev/null @@ -1,41 +0,0 @@ -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] - } -} - -resource "kubectl_manifest" "karpenter_provisioner" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner - metadata: - name: default - spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - provider: - subnetSelector: - Name: "*private*" - securityGroupSelector: - karpenter.sh/discovery/${module.eks.cluster_id}: ${module.eks.cluster_id} - tags: - karpenter.sh/discovery/${module.eks.cluster_id}: ${module.eks.cluster_id} - ttlSecondsAfterEmpty: 30 - YAML - - depends_on = [ - helm_release.karpenter - ] -} \ No newline at end of file diff --git a/karpenter-experiment/main.tf b/karpenter-experiment/main.tf deleted file mode 100644 index 6337a0f..0000000 --- a/karpenter-experiment/main.tf +++ /dev/null @@ -1,29 +0,0 @@ -terraform { - required_version = "~> 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 4.0" - } - helm = { - source = "hashicorp/helm" - version = "~> 2.5" - } - kubectl = { - source = "gavinbunney/kubectl" - version = "~> 1.14" - } - } -} - -provider "aws" { - region = "eu-west-2" -} - -locals { - cluster_name = "karpenter-demo" - partition = data.aws_partition.current.partition -} - -data "aws_partition" "current" {} \ No newline at end of file