From 3f5745e669a85274c58c3ce138dfa5fa1e69d30c Mon Sep 17 00:00:00 2001 From: d-sch Date: Thu, 20 Feb 2025 20:49:25 +0000 Subject: [PATCH 1/6] Terraform Structuring Lessons --- ...025-02-02-terraform-structuring-lessons.md | 54 +++++-------------- 1 file changed, 12 insertions(+), 42 deletions(-) diff --git a/content/posts/2025-02-02-terraform-structuring-lessons.md b/content/posts/2025-02-02-terraform-structuring-lessons.md index cc193c5..baa746f 100644 --- a/content/posts/2025-02-02-terraform-structuring-lessons.md +++ b/content/posts/2025-02-02-terraform-structuring-lessons.md @@ -1,5 +1,5 @@ --- -title: Terraform Structuring Lessons +title: Terraform module structure description: "" date: 2025-02-02T18:57:01.827Z preview: "" @@ -12,33 +12,25 @@ Working with Terraform takes some time to get comfortable with. One of the great I want to share some lessons learned after taking over larger applications infrastructure project without being able to reach the initial author. -# Obstacles For Reading Into Existing Terraform Code -Terraform code for an cloud infrastructure doesn't make it easy to understand the full picture easily. -There is no real beginning or end. Having files containing multiple resources hide all the complexity. -Having only one large file to start with is the worst case. Following the dependencies and build a mental -overview overloads the mental resource on multiple levels. Simply remembering multiple resources and there -dependencies at once is slowing your working memory and increase the error probability. +# Obstacles Reading Into Existing Terraform Code +Terraform code for an cloud infrastructure doesn't make it easy to understand the full picture easily. There is no real beginning or end. Having files containing multiple resources hide all the complexity. Having only one large file to start with is the worst case. +Following the dependencies and build a mental overview overloads the mental resource on multiple levels. Simply remembering multiple resources and there dependencies at once is slowing your working memory and increase the error probability. # Remove Obstacles By Refactor First -This learning lead me to conclude that it is takes too much time to try to understand unstructured Terraform code. -And it is painful too. Working with a lot of files containing multiple resource definitions I found doing refactoring -first instead really worth the effort. +This learning lead me to conclude that it is painful and takes too much time to try to understand unstructured Terraform code. Working with a lot of files containing multiple resource definitions I found doing refactoring first worth the effort instead. -This makes it much easier do get into a infrastructure. Breaking these into atomic files and applying a -good file naming convention already helps. You get an understanding of the different parts of the infrastructure -just by looking at the files. +This makes it much easier do get into a infrastructure. Breaking these into atomic files and applying a good naming convention already helps to get an understanding of the different infrastructure parts. # Conventions For Readable Readable Code ## Don't ### Declare Multiple Resources Into One File Multiple resource within one file make it complicate to find and understand changes -- First you must identify the scope of the change, e.g. which resource is affected. +- First you must identify the scope of the change. Which resource is affected. - Understand the change -- Finally make sure this affects the correct resource or property +- Finally make sure this affects the correct resource -This requires highest concentration and drains a lot of power. This leads to errors and at the same time -makes them harder to detect. +This requires highest concentration and drains a lot of power. This leads to errors and at the same time makes them harder to detect. ## Do ### Have Atomic Resource Files @@ -52,29 +44,7 @@ Using a good file naming convention you can identify important resource informat A good naming convention - orders related resources -- enables to get an high-level overview without reading into the files -- provides most important information like resource type +- enables to get an overview without reading into the files +- provides most important information like resource type -### Use Modules -The usage of modules can help to adhere to one or multiple of these principles -- Structure complex infrastructure - -Defining an infrastructure component that requires a large number of dependent or interdependent resources -is hard. Especially if all resources are in flat file or even a flat structure files. Especially naming gets -cumbersome. - -- don't repeat yourself - DRY - -If you create multiple infrastructure components using similar resource definitions it is a good idea to place -these into modules. - -- re-use - -While implementing multiple infrastructure components finally pattern emerge. These can be used to define modules -used by multiple infrastructure components. First within the project. Matured modules can be stored within a separate -git repository and referenced by the infrastructure components these. - -# Closing -This provides an overview about what to take care of to form a high quality IaC project. -In further posts I will highlight more details of the different aspects mentioned above. -And I plan to create posts sharing details what to take care of while building AWS infrastructure components. +# \ No newline at end of file From 49f2f17f905a09821ff90df6e27ee4a72fe2599e Mon Sep 17 00:00:00 2001 From: d-sch Date: Thu, 20 Feb 2025 20:50:31 +0000 Subject: [PATCH 2/6] Terraform Structuring Lessons --- ...025-02-02-terraform-structuring-lessons.md | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/content/posts/2025-02-02-terraform-structuring-lessons.md b/content/posts/2025-02-02-terraform-structuring-lessons.md index baa746f..cc193c5 100644 --- a/content/posts/2025-02-02-terraform-structuring-lessons.md +++ b/content/posts/2025-02-02-terraform-structuring-lessons.md @@ -1,5 +1,5 @@ --- -title: Terraform module structure +title: Terraform Structuring Lessons description: "" date: 2025-02-02T18:57:01.827Z preview: "" @@ -12,25 +12,33 @@ Working with Terraform takes some time to get comfortable with. One of the great I want to share some lessons learned after taking over larger applications infrastructure project without being able to reach the initial author. -# Obstacles Reading Into Existing Terraform Code -Terraform code for an cloud infrastructure doesn't make it easy to understand the full picture easily. There is no real beginning or end. Having files containing multiple resources hide all the complexity. Having only one large file to start with is the worst case. -Following the dependencies and build a mental overview overloads the mental resource on multiple levels. Simply remembering multiple resources and there dependencies at once is slowing your working memory and increase the error probability. +# Obstacles For Reading Into Existing Terraform Code +Terraform code for an cloud infrastructure doesn't make it easy to understand the full picture easily. +There is no real beginning or end. Having files containing multiple resources hide all the complexity. +Having only one large file to start with is the worst case. Following the dependencies and build a mental +overview overloads the mental resource on multiple levels. Simply remembering multiple resources and there +dependencies at once is slowing your working memory and increase the error probability. # Remove Obstacles By Refactor First -This learning lead me to conclude that it is painful and takes too much time to try to understand unstructured Terraform code. Working with a lot of files containing multiple resource definitions I found doing refactoring first worth the effort instead. +This learning lead me to conclude that it is takes too much time to try to understand unstructured Terraform code. +And it is painful too. Working with a lot of files containing multiple resource definitions I found doing refactoring +first instead really worth the effort. -This makes it much easier do get into a infrastructure. Breaking these into atomic files and applying a good naming convention already helps to get an understanding of the different infrastructure parts. +This makes it much easier do get into a infrastructure. Breaking these into atomic files and applying a +good file naming convention already helps. You get an understanding of the different parts of the infrastructure +just by looking at the files. # Conventions For Readable Readable Code ## Don't ### Declare Multiple Resources Into One File Multiple resource within one file make it complicate to find and understand changes -- First you must identify the scope of the change. Which resource is affected. +- First you must identify the scope of the change, e.g. which resource is affected. - Understand the change -- Finally make sure this affects the correct resource +- Finally make sure this affects the correct resource or property -This requires highest concentration and drains a lot of power. This leads to errors and at the same time makes them harder to detect. +This requires highest concentration and drains a lot of power. This leads to errors and at the same time +makes them harder to detect. ## Do ### Have Atomic Resource Files @@ -44,7 +52,29 @@ Using a good file naming convention you can identify important resource informat A good naming convention - orders related resources -- enables to get an overview without reading into the files -- provides most important information like resource type +- enables to get an high-level overview without reading into the files +- provides most important information like resource type -# \ No newline at end of file +### Use Modules +The usage of modules can help to adhere to one or multiple of these principles +- Structure complex infrastructure + +Defining an infrastructure component that requires a large number of dependent or interdependent resources +is hard. Especially if all resources are in flat file or even a flat structure files. Especially naming gets +cumbersome. + +- don't repeat yourself - DRY + +If you create multiple infrastructure components using similar resource definitions it is a good idea to place +these into modules. + +- re-use + +While implementing multiple infrastructure components finally pattern emerge. These can be used to define modules +used by multiple infrastructure components. First within the project. Matured modules can be stored within a separate +git repository and referenced by the infrastructure components these. + +# Closing +This provides an overview about what to take care of to form a high quality IaC project. +In further posts I will highlight more details of the different aspects mentioned above. +And I plan to create posts sharing details what to take care of while building AWS infrastructure components. From a0830fe4075b99750f7e126d6201eade50547174 Mon Sep 17 00:00:00 2001 From: d-sch Date: Sat, 22 Feb 2025 21:01:11 +0000 Subject: [PATCH 3/6] IaC Tales --- content/posts/2025-01-31-IaC Tales.md | 91 +++++++++++++++++++ ...025-02-02-terraform-structuring-lessons.md | 3 +- 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 content/posts/2025-01-31-IaC Tales.md diff --git a/content/posts/2025-01-31-IaC Tales.md b/content/posts/2025-01-31-IaC Tales.md new file mode 100644 index 0000000..292f07d --- /dev/null +++ b/content/posts/2025-01-31-IaC Tales.md @@ -0,0 +1,91 @@ +--- +title: IaC Tales +description: "" +date: 2025-01-31T21:39:47.928Z +preview: "" +draft: true +tags: [terraform,opentofu,terragrunt,aws] +categories: [IaC] +--- +Modern tales of software engineers in the IaC land. + +# Prelude +Back into infrastructure land after first contact some time ago. +Once again delving into the lower levels. Stumbling over... ok, no more physical wires. + +But finding your way around scattered, virtual floor plans. Reaching from infrastructure provider documentation over internal documentation, policies, pre-defined infrastructure, security and finally compliance checks. + +I never imagined what it could do to you if you scripted a solid part of infrastructure just to find out that it is rejected somewhere in the build pipelines. + +# The Quest Begins +First assess the party. +- AWS Cloud +- Terraform/OpenTofu +- Terragrunt +- Atlantis + +## AWS Cloud +A world full of documentations. User and developer guides are waiting. Even bigger the reference of +spells, receipts and ingredients. +To be fair have once a sneak peek introduction into the service provider. +Having heard about the myths, receipts, scripts and ingredients provided is really helpful. +But given the story is still not concluded, there are always new books and chapters added. + +## Terraform/OpenTofu +Versatile magic using spells trained to generate wished resources in foreign provider systems. +But to do so replaces all receipts and ingredients documented of the provider with foreign language spells. +These use the same terms but with different syntax it understands. + +Keep this in mind. You don't say what resources you want but which you wish should be. +After that reference these not yet there resources by name to build on top what will become. +First it let you see what you wish to be and creates it in the provider system you referenced. +If your wish clashes with the provider system you are in charge to find what prevents your dream come true. + +Take good care of the provider systems after your resources created. If resources change their names or get lost you are in trouble. +Because the spells taking notes of what was done and expect the world to look as written. +Be sure the notes are stored safe and available to everyone who adds new resources. +You have to recite each spell manually one resource after another to make it re-examine what reality in the provider system looks like. + +## Terragrunt +Handles the Terraform wand and translate your instructions without repeating yourself. +Using the same syntax but provide more grammar to express correct objectives and parameters. +Train it once and re-use in multiple situations. +Store your spell scripts in the wisely. Check for sure and check twice the place. +Prevent unfortunate things from happen. And be sure to plan you script storage structure well. +If not your project slowly descends into a pit of copying and pasting. + +## Atlantis +Makes sure that only secure and compliant spells are directed to the provider service. +Directly useable within the GitHub Octocat pull requests. +Prevents changes from become undone without trace. Wish forward only that resource were once there no longer be. +Don't fall into the trap. Your wishes are granted and fulfilled. +But not fixed forever and for others to see. Make sure you finally merge your casted spells. + +# How it begins +Once there was the proclamation that all folks of the code tinkers now are entitled to build their own AWS cloud underpinning. +The members of the operational guild will give guidance and do the final inspection and approval. +Everything in the AWS cloud will be written in a custom spells and hand over to Terragrunt. +All the spells are automatically checked for security and compliance. +It was told that this will make things easyier for everyone. + +# The journey starts +Slowly one code tinker after another went to find his way into the meta world of the internet. +Moving from one website to the next. First guided by the web search. Later by the mighty AI prompt. +Both behaving like the oracles from the past. Able to give clues but never authoritative. +Reading documentations in different styles. Foreign naming here and there. Sometimes similar but different. +Sometime different but the same. Spells provide in Terraform based on recipes in AWS. + +# Obstacles and responsibilities +Spells provide features but not the details of the used recipe. The tinkers have to fill in both into there memory. +Do the mapping is their work. Spells sometimes behave in unexpected ways. Finding ways to work around added to the code tinkers tasks. +But not all spells are approved by the Operators. This is tested after implementation. +After development the goal still not reached. They find out about the latest requirements of security and policies. +Your infrastructure is not using the official gate they are told. You need to settle inside the city boundaries. +You need to register a official address. Giving the final spell collection to Atlantis is their next step. +Now their spells are checked and if not approved once more the infrastructure code has to be changed. Don't use forbidden spells. +Documented only in the depths of the repository providing the Atlantis configuration. + +# (Happy) Ending +Finally the infrastructure is deployed. The work is done. The code tinkers wonder if they are still software engineers or now cloud solution engineers or DevOps or IT. Or all at once? +Later one tries to get into fixing an application and has a hard time to remember the details of the underlying framework used. +Even the code looks foreign after the time in infrastructure as code land. \ No newline at end of file diff --git a/content/posts/2025-02-02-terraform-structuring-lessons.md b/content/posts/2025-02-02-terraform-structuring-lessons.md index cc193c5..cb5ad76 100644 --- a/content/posts/2025-02-02-terraform-structuring-lessons.md +++ b/content/posts/2025-02-02-terraform-structuring-lessons.md @@ -4,10 +4,9 @@ description: "" date: 2025-02-02T18:57:01.827Z preview: "" draft: false -tags: [terraform] +tags: [terraform,opentofu] categories: [IaC] --- -# Overview Working with Terraform takes some time to get comfortable with. One of the greatest myths in all the examples is the `main.tf`. Starting a new infrastructure it can make sense to begin with one file an put all resources in there. But after reaching several pages in editor things can easily go down hill from there. I want to share some lessons learned after taking over larger applications infrastructure project without being able to reach the initial author. From 4ab1b7e08e0e67bf2d9a4b45ae89c00317f13e07 Mon Sep 17 00:00:00 2001 From: d-sch Date: Mon, 24 Feb 2025 21:58:37 +0000 Subject: [PATCH 4/6] IaC Tales --- content/posts/2025-01-31-IaC Tales.md | 33 ++++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/content/posts/2025-01-31-IaC Tales.md b/content/posts/2025-01-31-IaC Tales.md index 292f07d..c5ed4c6 100644 --- a/content/posts/2025-01-31-IaC Tales.md +++ b/content/posts/2025-01-31-IaC Tales.md @@ -56,36 +56,47 @@ If not your project slowly descends into a pit of copying and pasting. ## Atlantis Makes sure that only secure and compliant spells are directed to the provider service. -Directly useable within the GitHub Octocat pull requests. -Prevents changes from become undone without trace. Wish forward only that resource were once there no longer be. +Directly reachable within the GitHub Octocat pull requests comment. +Prevents changes from becoming undone without trace. Wish forward only. So only resources that were once will no longer be. Don't fall into the trap. Your wishes are granted and fulfilled. -But not fixed forever and for others to see. Make sure you finally merge your casted spells. +But not fixed forever and for others to see. Make sure you finally merge your casted spells of your pull request. # How it begins Once there was the proclamation that all folks of the code tinkers now are entitled to build their own AWS cloud underpinning. The members of the operational guild will give guidance and do the final inspection and approval. Everything in the AWS cloud will be written in a custom spells and hand over to Terragrunt. All the spells are automatically checked for security and compliance. -It was told that this will make things easyier for everyone. +It was told that this will make things easier for everyone. # The journey starts Slowly one code tinker after another went to find his way into the meta world of the internet. Moving from one website to the next. First guided by the web search. Later by the mighty AI prompt. -Both behaving like the oracles from the past. Able to give clues but never authoritative. +Both behaving like the oracles from the past. Able to give clues but never authoritative answers. Reading documentations in different styles. Foreign naming here and there. Sometimes similar but different. Sometime different but the same. Spells provide in Terraform based on recipes in AWS. # Obstacles and responsibilities -Spells provide features but not the details of the used recipe. The tinkers have to fill in both into there memory. +Spells provide features but not the details of the used recipe. The tinkers have to fill in both into their memory. Do the mapping is their work. Spells sometimes behave in unexpected ways. Finding ways to work around added to the code tinkers tasks. -But not all spells are approved by the Operators. This is tested after implementation. -After development the goal still not reached. They find out about the latest requirements of security and policies. -Your infrastructure is not using the official gate they are told. You need to settle inside the city boundaries. -You need to register a official address. Giving the final spell collection to Atlantis is their next step. +Slowly wishes formulated and bigger and bigger collections of spells depending on each other are formulated and written down. + +# Work to do reach approval +Beware this is reviewed and testified after implementation and the spell collection is done. But not all spells are approved by the Operators. +The final goal still not reached after all the time in development. Now the tinkers need to find out about the latest requirements of security and policies. +Your infrastructure is not using the official gates they are told. You need to settle inside the city boundaries and the documented zones. +You need to register a official address. + +Rephrase wishes and cast new spells, fixing dependencies and fighting strange and undocumented behaviors that is the tinkers duty now. + +# The final trial +Finally it is time to let the wishes get real. Giving the final spell collection to Atlantis is the final step. Now their spells are checked and if not approved once more the infrastructure code has to be changed. Don't use forbidden spells. Documented only in the depths of the repository providing the Atlantis configuration. +If your code is not worthy you have to go back refine it until it eventual pass. # (Happy) Ending Finally the infrastructure is deployed. The work is done. The code tinkers wonder if they are still software engineers or now cloud solution engineers or DevOps or IT. Or all at once? Later one tries to get into fixing an application and has a hard time to remember the details of the underlying framework used. -Even the code looks foreign after the time in infrastructure as code land. \ No newline at end of file +Even the code looks foreign after the time in infrastructure as code land. + +TBC \ No newline at end of file From cade6d9057c2cae7081755bd8c382ccc36d120d1 Mon Sep 17 00:00:00 2001 From: d-sch Date: Mon, 24 Feb 2025 22:00:01 +0000 Subject: [PATCH 5/6] IaC Tales --- content/posts/2025-01-31-IaC Tales.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/posts/2025-01-31-IaC Tales.md b/content/posts/2025-01-31-IaC Tales.md index c5ed4c6..19c89ef 100644 --- a/content/posts/2025-01-31-IaC Tales.md +++ b/content/posts/2025-01-31-IaC Tales.md @@ -3,7 +3,7 @@ title: IaC Tales description: "" date: 2025-01-31T21:39:47.928Z preview: "" -draft: true +draft: false tags: [terraform,opentofu,terragrunt,aws] categories: [IaC] --- From 8fc92f55fe5cd36a49ef77b239b031ca35c23155 Mon Sep 17 00:00:00 2001 From: d-sch Date: Tue, 25 Feb 2025 21:16:59 +0000 Subject: [PATCH 6/6] Fix hugo local build --- .../dart-sass/devcontainer-feature.json | 22 +++++++++++++++++++ .devcontainer/dart-sass/install.sh | 5 +++++ .devcontainer/devcontainer.json | 6 ++++- .gitignore | 1 + 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/dart-sass/devcontainer-feature.json create mode 100644 .devcontainer/dart-sass/install.sh diff --git a/.devcontainer/dart-sass/devcontainer-feature.json b/.devcontainer/dart-sass/devcontainer-feature.json new file mode 100644 index 0000000..7cc8824 --- /dev/null +++ b/.devcontainer/dart-sass/devcontainer-feature.json @@ -0,0 +1,22 @@ +{ + "name": "dart-sass", + "id": "dart-sass", + "version": "1.0.0", + "description": "Install dart-sass", + "options": { + "version": { + "type": "string", + "proposals": [ + "1.85.1" + ], + "default": "1.85.1", + "description": "Select a pre-made greeting, or enter your own" + } + }, + "containerEnv": { + "PATH": "/opt/dart-sass:${PATH}" + }, + "installsAfter": [ + "ghcr.io/devcontainers/features/common-utils" + ] +} diff --git a/.devcontainer/dart-sass/install.sh b/.devcontainer/dart-sass/install.sh new file mode 100644 index 0000000..9d3695e --- /dev/null +++ b/.devcontainer/dart-sass/install.sh @@ -0,0 +1,5 @@ +VERSION=${VERSION:-undefined} + +(cd /tmp && curl https://github.com/sass/dart-sass/releases/download/${VERSION}/dart-sass-${VERSION}-linux-x64.tar.gz -fsSLO --compressed) +tar -C /opt/ -xzvf /tmp/dart-sass-${VERSION}-linux-x64.tar.gz +rm /tmp/dart-sass-${VERSION}-linux-x64.tar.gz \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0d07718..ac22c75 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,11 @@ "name": "Dev Container", "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { - "ghcr.io/devcontainers/features/hugo:1": {} + "ghcr.io/devcontainers/features/hugo:1": { + "extended": true + }, + "ghcr.io/devcontainers/features/node:1": {}, + "./dart-sass": {} }, "customizations": { "vscode": { diff --git a/.gitignore b/.gitignore index d13aecb..9950427 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ resources/ go.mod go.sum resources/_gen/ +docs/ \ No newline at end of file