From b401a5c95c0a1f179f43f5ffa9f0f95de78374f9 Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Fri, 8 Nov 2024 12:07:52 +0000 Subject: [PATCH 1/9] Added support for TLS and SSL on kafka. --- driver-gateway/README.md | 1 + .../deploy/ssd-deployment/ansible-vars.yml | 5 + .../deploy/ssd-deployment/deploy.yaml | 95 ++++++++++++++++++- .../ssd-deployment/provision-kafka-aws.tf | 1 + .../templates/server_ssl.properties.j2 | 44 +++++++++ .../deploy/ssd-deployment/terraform.tfvars | 4 +- 6 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 driver-gateway/deploy/ssd-deployment/ansible-vars.yml create mode 100644 driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 diff --git a/driver-gateway/README.md b/driver-gateway/README.md index b32ff4d41..971500593 100644 --- a/driver-gateway/README.md +++ b/driver-gateway/README.md @@ -22,6 +22,7 @@ Pre-requisites: 2. export REGISTRY_USERNAME= 3. export REGISTRY_PASSWORD= 7. Setup nodes with `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml` + 1. If you want to set tls and use delegated_sasl_ssl run `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml -e "use_tls=true use_delegated_ssl=true"` 8. Connect to one benchmark worker node with `ssh -i ~/.ssh/kafka_aws ec2-user@$(terraform output client_ssh_host | tr -d '"')` 9. Go to benchmark directory with `cd /opt/benchmark` 10. Run the benchmark with `sudo bin/benchmark --drivers driver-gateway/gateway-latency.yaml workloads/100-topic-4-partitions-1kb-4p-4c-500k.yaml;` diff --git a/driver-gateway/deploy/ssd-deployment/ansible-vars.yml b/driver-gateway/deploy/ssd-deployment/ansible-vars.yml new file mode 100644 index 000000000..e9c3f50cc --- /dev/null +++ b/driver-gateway/deploy/ssd-deployment/ansible-vars.yml @@ -0,0 +1,5 @@ +# vars.yml +use_tls: false +use_delegated_ssl: false +kafka_truststore_password: "admin123" +kafka_keystore_password: "admin123" \ No newline at end of file diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index 10337f2a5..6aff06ce3 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -1,4 +1,4 @@ -# +--- # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -16,6 +16,8 @@ hosts: all gather_facts: no # important tags: [always] + vars_files: + - ./tf_ansible_vars_file.yml tasks: - name: Wait for all instances to become reachable wait_for_connection: @@ -99,6 +101,8 @@ hosts: all connection: ssh become: true + vars_files: + - ./ansible-vars.yml tasks: - name: Install RPM packages yum: pkg={{ item }} state=installed @@ -118,7 +122,7 @@ gatewayServers: "{{ groups['gateway'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:6969') | join(',') }}" privateIp: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" kafkaVersion: "3.6.1" - tags: [ always, client-code ] + tags: [always, client-code] - debug: msg: "zookeeper servers: {{ zookeeperServers }}\nbootstrap servers: {{ bootstrapServers }}\ngateway servers: {{ gatewayServers }}" - name: Change locale to en_US.utf-8 @@ -131,6 +135,9 @@ connection: ssh become: true tags: [kafka] + when: use_delegate_ssl == true + vars_files: + - ./ansible-vars.yml tasks: - file: path=/opt/kafka state=absent - file: path=/opt/kafka state=directory @@ -140,7 +147,72 @@ remote_src: yes dest: /opt/kafka extra_opts: ["--strip-components=1"] + - name: Ensure truststore and keystore directory exists + file: + path: /opt/kafka/config + state: directory + mode: '0755' + + - name: Remove existing truststore and keystore if TLS is enabled + file: + path: /opt/kafka/config/kafka-server.keystore.jks + state: absent + when: use_tls + - name: Generate Kafka keystore if TLS is enabled + command: > + keytool -genkeypair -alias kafka-server -keyalg RSA -keystore /opt/kafka/config/kafka-server.keystore.jks + -storepass "{{ kafka_keystore_password }}" -keypass "{{ kafka_keystore_password }}" + -dname "CN={{ inventory_hostname }}, OU=Kafka, O=YourOrg, L=YourCity, ST=YourState, C=YourCountry" + when: use_tls + - name: Generate CSR for Kafka server if TLS is enabled + command: > + keytool -keystore /opt/kafka/config/kafka-server.keystore.jks -alias kafka-server -certreq -file /opt/kafka/config/kafka-server.csr + -storepass "{{ kafka_keystore_password }}" + when: use_tls + - name: Import signed certificate to Kafka keystore if TLS is enabled + command: > + keytool -keystore /opt/kafka/config/kafka-server.keystore.jks -alias kafka-server -import -file /opt/kafka/config/kafka-server-cert.pem + -storepass "{{ kafka_keystore_password }}" -noprompt + when: use_tls + - name: Set permissions for keystore and truststore if TLS is enabled + file: + path: /opt/kafka/config/kafka-server.keystore.jks + owner: kafka + group: kafka + mode: '0600' + loop: + - + - kafka-server.truststore.jks + when: use_tls + - file: + path: /opt/kafka/config/kafka-server.truststore.jks + state: absent + when: use_delegated_ssl + - name: Sign CSR with CA if use_delegated_ssl is enabled + command: > + openssl x509 -req -in /opt/kafka/config/kafka-server.csr -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem + -CAcreateserial -out /opt/kafka/config/kafka-server-cert.pem -days 365 -sha256 + when: use_delegated_ssl + - name: Import CA certificate to Kafka truststore if TLS is enabled + command: > + keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot -import -file /tmp/ca-cert.pem + -storepass "{{ kafka_truststore_password }}" -noprompt + when: use_delegated_ssl + - name: Set permissions for keystore and truststore if TLS is enabled + file: + path: /opt/kafka/config/kafka-server.truststore.jks + owner: kafka + group: kafka + mode: '0600' + when: use_delegated_ssl + + - name: Reload and restart Kafka to apply configuration + systemd: + name: kafka + state: restarted + daemon_reload: yes + tags: [restart-kafka] - name: Setup ZooKeeper hosts: zookeeper @@ -177,6 +249,8 @@ connection: ssh become: true tags: [kafka] + vars_files: + - ./ansible-vars.yml tasks: - set_fact: brokerId: "{{ groups['kafka'].index(inventory_hostname) }}" @@ -184,6 +258,19 @@ template: src: "templates/server.properties" dest: "/opt/kafka/config/server.properties" + when: not use_delegate_ssl and not use_tls + - name: Configure Kafka SSL with SASL + template: + src: templates/server_ssl.properties.j2 + dest: /opt/kafka/config/server_ssl.properties + vars: + kafka_truststore_path: /opt/kafka/config/kafka-server.truststore.jks + kafka_truststore_password: "{{ kafka_truststore_password }}" + kafka_keystore_path: /opt/kafka/config/kafka-server.keystore.jks + kafka_keystore_password: "{{ kafka_keystore_password }}" + use_tls: "{{ use_tls }}" + use_ssl: "{{ use_delegate_ssl }}" + when: use_tls or use_delegate_ssl - template: src: "templates/kafka.service" dest: "/etc/systemd/system/kafka.service" @@ -342,7 +429,7 @@ regexp: '^ bootstrap.servers=' line: ' bootstrap.servers={{ gatewayServers }}' with_items: '{{ gateway_drivers_list.stdout_lines }}' - tags: [ client-code ] + tags: [client-code] - name: Get list of jms driver config files raw: ls -1 /opt/benchmark/driver-jms/kafka*.yaml @@ -389,7 +476,7 @@ name: "benchmark-worker" tags: [client-code] -- name: Hosts addresses +- name: Hosts addresses hosts: localhost become: false tasks: diff --git a/driver-gateway/deploy/ssd-deployment/provision-kafka-aws.tf b/driver-gateway/deploy/ssd-deployment/provision-kafka-aws.tf index b949eba67..a23fe8212 100644 --- a/driver-gateway/deploy/ssd-deployment/provision-kafka-aws.tf +++ b/driver-gateway/deploy/ssd-deployment/provision-kafka-aws.tf @@ -130,6 +130,7 @@ resource "aws_security_group" "benchmark_security_group" { cidr_blocks = ["0.0.0.0/0"] } + # ICMP access from anywhere ingress { from_port = -1 to_port = -1 diff --git a/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 b/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 new file mode 100644 index 000000000..e84134bdb --- /dev/null +++ b/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 @@ -0,0 +1,44 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% if use_ssl %} +#bootstrap.servers={{ gatewayServers }} +security.protocol=SASL_SSL +advertised.listeners=SASL_SSL://{{ privateIp }}:9094 +security.inter.broker.protocol=ASL_SSL +ssl.truststore.location={{ kafka_truststore_path }} +ssl.truststore.password={{ kafka_truststore_password }} +sasl.enabled.mechanisms=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; +{% endif %} + +{% if use_tls %} +ssl.keystore.location={{ kafka_keystore_path }} +ssl.keystore.password={{ kafka_keystore_password }} +ssl.key.password={{ kafka_key_password }} +{% endif %} + +broker.id={{ brokerId }} + +log.dirs=/mnt/data-1,/mnt/data-2 + +zookeeper.connect={{ zookeeperServers }} + +num.replica.fetchers=8 + +message.max.bytes=10485760 + +replica.fetch.max.bytes=10485760 + +num.network.threads=8 \ No newline at end of file diff --git a/driver-gateway/deploy/ssd-deployment/terraform.tfvars b/driver-gateway/deploy/ssd-deployment/terraform.tfvars index 9de3dfebb..59902fe06 100644 --- a/driver-gateway/deploy/ssd-deployment/terraform.tfvars +++ b/driver-gateway/deploy/ssd-deployment/terraform.tfvars @@ -3,8 +3,8 @@ region = "us-west-2" az = "us-west-2a" profile = "cdk-dev" ami = "ami-04a616933df665b44" // RHEL-9.0.0_HVM-20220513-x86_64-0-Hourly2-GP2 -#ami = "ami-08970fb2e5767e3b8" // RHEL-8.6.0_HVM-20220503-x86_64-2-Hourly2-GP2 -#ami = "ami-0b0b4a49742d64899" // RHEL-8.6.0_HVM-20240521-x86_64-58-Hourly2-GP3 +#ami = "ami-08970fb2e5767e3b8" // RHEL-8.6.0_HVM-20220503-x86_64-2-Hourly2-GP2 +#ami = "ami-0b0b4a49742d64899" // RHEL-8.6.0_HVM-20240521-x86_64-58-Hourly2-GP3 instance_types = { "gateway" = "m5n.xlarge" # 16.0 GiB 4 vCPUs EBS only Up to 25 Gigabit From 77215faa965b5bbf1dec6fe66f8925d4a1a9dcfa Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Mon, 11 Nov 2024 12:53:28 +0000 Subject: [PATCH 2/9] Switched to only one flag. --- driver-gateway/README.md | 2 +- .../deploy/ssd-deployment/ansible-vars.yml | 3 +- .../deploy/ssd-deployment/deploy.yaml | 142 +++++++--- .../ssd-deployment/templates/gateway_env.j2 | 6 + .../templates/gateway_volumes.j2 | 2 + .../templates/server_ssl.properties.j2 | 23 +- pom.xml | 250 +----------------- 7 files changed, 128 insertions(+), 300 deletions(-) create mode 100644 driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 create mode 100644 driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 diff --git a/driver-gateway/README.md b/driver-gateway/README.md index 971500593..47568996c 100644 --- a/driver-gateway/README.md +++ b/driver-gateway/README.md @@ -22,7 +22,7 @@ Pre-requisites: 2. export REGISTRY_USERNAME= 3. export REGISTRY_PASSWORD= 7. Setup nodes with `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml` - 1. If you want to set tls and use delegated_sasl_ssl run `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml -e "use_tls=true use_delegated_ssl=true"` + 1. If you want to set tls and use delegated_sasl_ssl run `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml -e "use_tls_and_ssl=true"` 8. Connect to one benchmark worker node with `ssh -i ~/.ssh/kafka_aws ec2-user@$(terraform output client_ssh_host | tr -d '"')` 9. Go to benchmark directory with `cd /opt/benchmark` 10. Run the benchmark with `sudo bin/benchmark --drivers driver-gateway/gateway-latency.yaml workloads/100-topic-4-partitions-1kb-4p-4c-500k.yaml;` diff --git a/driver-gateway/deploy/ssd-deployment/ansible-vars.yml b/driver-gateway/deploy/ssd-deployment/ansible-vars.yml index e9c3f50cc..d472e6a8e 100644 --- a/driver-gateway/deploy/ssd-deployment/ansible-vars.yml +++ b/driver-gateway/deploy/ssd-deployment/ansible-vars.yml @@ -1,5 +1,4 @@ # vars.yml -use_tls: false -use_delegated_ssl: false +use_tls_and_ssl: false kafka_truststore_password: "admin123" kafka_keystore_password: "admin123" \ No newline at end of file diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index 6aff06ce3..b4371b870 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -135,7 +135,6 @@ connection: ssh become: true tags: [kafka] - when: use_delegate_ssl == true vars_files: - ./ansible-vars.yml tasks: @@ -147,65 +146,62 @@ remote_src: yes dest: /opt/kafka extra_opts: ["--strip-components=1"] + - name: Ensure truststore and keystore directory exists file: path: /opt/kafka/config state: directory mode: '0755' - - - name: Remove existing truststore and keystore if TLS is enabled + when: use_tls_and_ssl + - name: Remove existing truststore and keystore if use_tls_and_ssl is enabled file: path: /opt/kafka/config/kafka-server.keystore.jks state: absent - when: use_tls - - name: Generate Kafka keystore if TLS is enabled + when: use_tls_and_ssl + - name: Generate Kafka keystore if use_tls_and_ssl is enabled command: > keytool -genkeypair -alias kafka-server -keyalg RSA -keystore /opt/kafka/config/kafka-server.keystore.jks -storepass "{{ kafka_keystore_password }}" -keypass "{{ kafka_keystore_password }}" - -dname "CN={{ inventory_hostname }}, OU=Kafka, O=YourOrg, L=YourCity, ST=YourState, C=YourCountry" - when: use_tls - - name: Generate CSR for Kafka server if TLS is enabled + -dname "CN={{ inventory_hostname }}, OU=Kafka, O=Conduktor, L=London, ST=London, C=England" + when: use_tls_and_ssl + - name: Generate CSR for Kafka server if use_tls_and_ssl is enabled command: > keytool -keystore /opt/kafka/config/kafka-server.keystore.jks -alias kafka-server -certreq -file /opt/kafka/config/kafka-server.csr -storepass "{{ kafka_keystore_password }}" - when: use_tls - - name: Import signed certificate to Kafka keystore if TLS is enabled + when: use_tls_and_ssl + - name: Import signed certificate to Kafka keystore if use_tls_and_ssl is enabled command: > keytool -keystore /opt/kafka/config/kafka-server.keystore.jks -alias kafka-server -import -file /opt/kafka/config/kafka-server-cert.pem -storepass "{{ kafka_keystore_password }}" -noprompt - when: use_tls - - name: Set permissions for keystore and truststore if TLS is enabled + when: use_tls_and_ssl + - name: Set permissions for keystore and truststore if use_tls_and_ssl is enabled file: path: /opt/kafka/config/kafka-server.keystore.jks owner: kafka group: kafka mode: '0600' - loop: - - - - kafka-server.truststore.jks - when: use_tls - + when: use_tls_and_ssl - file: path: /opt/kafka/config/kafka-server.truststore.jks state: absent - when: use_delegated_ssl - - name: Sign CSR with CA if use_delegated_ssl is enabled + when: use_tls_and_ssl + - name: Sign CSR with CA if use_tls_and_ssl is enabled command: > openssl x509 -req -in /opt/kafka/config/kafka-server.csr -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/kafka/config/kafka-server-cert.pem -days 365 -sha256 - when: use_delegated_ssl - - name: Import CA certificate to Kafka truststore if TLS is enabled + when: use_tls_and_ssl + - name: Import CA certificate to Kafka truststore if use_tls_and_ssl is enabled command: > keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot -import -file /tmp/ca-cert.pem -storepass "{{ kafka_truststore_password }}" -noprompt - when: use_delegated_ssl - - name: Set permissions for keystore and truststore if TLS is enabled + when: use_tls_and_ssl + - name: Set permissions for keystore and truststore if use_tls_and_ssl is enabled file: path: /opt/kafka/config/kafka-server.truststore.jks owner: kafka group: kafka mode: '0600' - when: use_delegated_ssl + when: use_tls_and_ssl - name: Reload and restart Kafka to apply configuration systemd: @@ -258,7 +254,7 @@ template: src: "templates/server.properties" dest: "/opt/kafka/config/server.properties" - when: not use_delegate_ssl and not use_tls + when: not use_tls_and_ssl - name: Configure Kafka SSL with SASL template: src: templates/server_ssl.properties.j2 @@ -268,9 +264,7 @@ kafka_truststore_password: "{{ kafka_truststore_password }}" kafka_keystore_path: /opt/kafka/config/kafka-server.keystore.jks kafka_keystore_password: "{{ kafka_keystore_password }}" - use_tls: "{{ use_tls }}" - use_ssl: "{{ use_delegate_ssl }}" - when: use_tls or use_delegate_ssl + when: use_tls_and_ssl - template: src: "templates/kafka.service" dest: "/etc/systemd/system/kafka.service" @@ -287,6 +281,7 @@ tags: [gateway] vars_files: - ./tf_ansible_vars_file.yml + - ./ansible-vars.yml tasks: - name: Check mandatory variables imported from Terraform assert: @@ -333,15 +328,76 @@ registry_url: "{{ lookup('ansible.builtin.env', 'REGISTRY_URL', default='harbor.cdkt.dev') }}" username: "{{ lookup('ansible.builtin.env', 'REGISTRY_USERNAME') }}" password: "{{ lookup('ansible.builtin.env', 'REGISTRY_PASSWORD') }}" - - name: "Gateway container" - community.docker.docker_container: - name: gateway - state: started - image: "{{ lookup('ansible.builtin.env', 'GATEWAY_IMAGE', default='conduktor/conduktor-gateway:3.0.4') }}" - restart_policy: on-failure - volumes: + + - name: Ensure truststore and keystore directory exists + file: + path: /opt/conduktor-gateway/ssl + state: directory + mode: '0755' + + - name: Generate Gateway keystore if use_tls_and_ssl is enabled + command: > + keytool -genkeypair -alias gateway -keyalg RSA + -keystore /opt/conduktor-gateway/ssl/gateway_keystore.jks + -storepass "{{ gateway_keystore_password }}" -keypass "{{ gateway_keystore_password }}" + -dname "CN={{ inventory_hostname }}, OU=Conduktor, O=Conduktor, L=London, ST=London, C=England" + when: use_tls_and_ssl + + - name: Generate CSR for Gateway if use_tls_and_ssl is enabled + command: > + keytool -keystore /opt/conduktor-gateway/ssl/gateway_keystore.jks -alias gateway + -certreq -file /opt/conduktor-gateway/ssl/gateway.csr + -storepass "{{ gateway_keystore_password }}" + when: use_tls_and_ssl + + - name: Set permissions for keystore if use_tls_and_ssl is enabled + file: + path: /opt/conduktor-gateway/ssl/gateway_keystore.jks + owner: root + group: root + mode: '0600' + when: use_tls_and_ssl + + - name: Sign CSR with CA if use_tls_and_ssl is enabled + command: > + openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr + -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem + -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem + -days 365 -sha256 + when: use_tls_and_ssl + - name: Import signed certificate to Gateway keystore if use_tls_and_ssl is enabled + command: > + keytool -keystore /opt/conduktor-gateway/ssl/gateway_keystore.jks -alias gateway + -import -file /opt/conduktor-gateway/ssl/gateway-cert.pem + -storepass "{{ gateway_keystore_password }}" -noprompt + when: use_tls_and_ssl + - name: Import CA certificate to Gateway truststore if use_tls_and_ssl is enabled + command: > + keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks + -alias CARoot -import -file /tmp/ca-cert.pem + -storepass "{{ gateway_truststore_password }}" -noprompt + when: use_tls_and_ssl + + - name: Set permissions for truststore if use_tls_and_ssl is enabled + file: + path: /opt/conduktor-gateway/ssl/gateway_truststore.jks + owner: root + group: root + mode: '0600' + when: use_tls_and_ssl + + - name: Set base volume mounts for Gateway + set_fact: + gateway_volumes: - "/opt/gateway-interceptors.json:/opt/interceptors.json" - env: + - name: Add TLS volume mounts to Gateway if enabled + set_fact: + gateway_volumes: "{{ gateway_volumes + [ lookup('template', 'gateway_volumes.j2').splitlines() ] | flatten }}" + when: use_tls_and_ssl + + - name: Set base environment variables for Gateway + set_fact: + gateway_env: LOG4J2_IO_CONDUKTOR_PROXY_SERVICE_LEVEL: "DEBUG" LOG4J2_IO_CONDUKTOR_PROXY_NETWORK_LEVEL: "DEBUG" LOG4J2_IO_NETTY_LEVEL: "INFO" @@ -354,6 +410,20 @@ ENCRYPT_SECRET_KEY: "{{ lookup('ansible.builtin.env', 'ENCRYPT_SECRET_KEY') }}" GATEWAY_INTERCEPTOR_CONFIG_LOCATION: "/opt/interceptors.json" GATEWAY_ADMIN_API_USERS: "[{username: admin, password: conduktor, admin: true}]" + + - name: Add TLS environment variables to Gateway if enabled + set_fact: + gateway_env: "{{ gateway_env | combine(lookup('template', 'gateway_env.j2') | from_yaml) }}" + when: use_tls_and_ssl + + - name: "Gateway container" + community.docker.docker_container: + name: gateway + state: started + image: "{{ lookup('ansible.builtin.env', 'GATEWAY_IMAGE', default='conduktor/conduktor-gateway:latest') }}" + restart_policy: on-failure + volumes: "{{ gateway_volumes }}" + env: "{{ gateway_env }}" published_ports: - "8888:8888" - "6969:6969" diff --git a/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 b/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 new file mode 100644 index 000000000..09a1422cb --- /dev/null +++ b/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 @@ -0,0 +1,6 @@ +SECURITY_PROTOCOL: "SASL_SSL" +SSL_TRUSTSTORE_LOCATION: "/etc/conduktor-gateway/truststore.jks" +SSL_TRUSTSTORE_PASSWORD: "{{ gateway_truststore_password }}" +SSL_KEYSTORE_LOCATION: "/etc/conduktor-gateway/keystore.jks" +SSL_KEYSTORE_PASSWORD: "{{ gateway_keystore_password }}" +SSL_KEY_PASSWORD: "{{ gateway_key_password }}" diff --git a/driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 b/driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 new file mode 100644 index 000000000..b0fe37753 --- /dev/null +++ b/driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 @@ -0,0 +1,2 @@ +- "/opt/conduktor-gateway/ssl/gateway_keystore.jks:/etc/conduktor-gateway/keystore.jks" +- "/opt/conduktor-gateway/ssl/gateway_truststore.jks:/etc/conduktor-gateway/truststore.jks" diff --git a/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 b/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 index e84134bdb..04b4ae494 100644 --- a/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 +++ b/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 @@ -12,23 +12,22 @@ # limitations under the License. # -{% if use_ssl %} -#bootstrap.servers={{ gatewayServers }} -security.protocol=SASL_SSL -advertised.listeners=SASL_SSL://{{ privateIp }}:9094 -security.inter.broker.protocol=ASL_SSL +# SSL Settings +ssl.keystore.location={{ kafka_keystore_path }} +ssl.keystore.password={{ kafka_keystore_password }} +ssl.key.password={{ kafka_key_password }} ssl.truststore.location={{ kafka_truststore_path }} ssl.truststore.password={{ kafka_truststore_password }} + +# SASL Authentication +security.protocol=SASL_SSL sasl.enabled.mechanisms=PLAIN +sasl.mechanism.inter.broker.protocol=PLAIN +advertised.listeners=SASL_SSL://{{ privateIp }}:9094 +security.inter.broker.protocol=ASL_SSL sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; -{% endif %} - -{% if use_tls %} -ssl.keystore.location={{ kafka_keystore_path }} -ssl.keystore.password={{ kafka_keystore_password }} -ssl.key.password={{ kafka_key_password }} -{% endif %} +# Kafka Broker Config broker.id={{ brokerId }} log.dirs=/mnt/data-1,/mnt/data-2 diff --git a/pom.xml b/pom.xml index fee04709a..d491378aa 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 10.3.3 3.12.0 2.20.0 - 1.18.24 + 1.18.34 2.13.2 1.48 5.9.0 @@ -173,254 +173,6 @@ - - - - - com.diffplug.spotless - spotless-maven-plugin - ${spotless.plugin.version} - - - - false - true - 4 - scope,groupId,artifactId - groupId,artifactId - groupId,artifactId - - - - - - - - true - 2 - - - true - 4 - - - - - **/*.md - - - - - - - etc/**/*.xml - .github/workflows/**/*.yml - **/doc/**/*.puml - - - - - true - 4 - - - - - - - check - - check - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs.plugin.version} - - etc/findbugsExclude.xml - - - - - check - - - - - - - com.mycila - license-maven-plugin - ${license.plugin.version} - -
etc/APACHE-2.txt
- - LICENSE - NOTICE - payload/** - **/*.pyc - **/*.sh - **/.pydevproject - .github/** - - - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - XML_STYLE - SCRIPT_STYLE - APOSTROPHE_STYLE - - - - - com.mycila - license-maven-plugin-git - ${license.plugin.version} - - - - - - check - - process-sources - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${checkstyle.plugin.version} - - true - true - true - etc/checkstyle.xml - true - - - - com.puppycrawl.tools - checkstyle - ${checkstyle.version} - - - - - - check - - validate - - - - - org.apache.maven.plugins - maven-enforcer-plugin - ${maven.enforcer.plugin.version} - - - enforce-maven-version - - enforce - - - - - [3.8.6,) - - - true - - - - - - maven-failsafe-plugin - ${maven.surefire.plugin.version} - - - - integration-test - verify - - - - - - maven-surefire-plugin - ${maven.surefire.plugin.version} - - - --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.plugin.version} - - - - prepare-agent - - - - report - - report - - verify - - - check - - check - - verify - - - - BUNDLE - - - CLASS - COVEREDRATIO - 0.00 - - - METHOD - COVEREDRATIO - 0.00 - - - LINE - COVEREDRATIO - 0.00 - - - BRANCH - COVEREDRATIO - 0.00 - - - - - - - - - - - modern-java-compile From 43432bc77628e1a641100243dfc9566b5fa6a558 Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Mon, 11 Nov 2024 15:52:15 +0000 Subject: [PATCH 3/9] Made one function to setup the whole ssl tls setup --- .../deploy/ssd-deployment/deploy.yaml | 179 +++++++----------- 1 file changed, 65 insertions(+), 114 deletions(-) diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index b4371b870..d3c95d27d 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -130,6 +130,71 @@ echo 'LANG=en_US.utf-8 LC_ALL=en_US.utf-8' > /etc/environment +# TLS Certificate Generation for Kafka and Conduktor Gateway +- name: Generate CA and server certificates for TLS + hosts: [kafka, gateway] + become: true + when: use_tls_and_ssl + vars_files: + - ./ansible-vars.yml + tasks: + - name: Generate CA private key and certificate (if not exists) + command: > + openssl req -new -x509 -keyout /tmp/ca-key.pem -out /tmp/ca-cert.pem -days 365 -nodes + -subj "/CN=MyKafkaCA/O=Conduktor/C=UK" + args: + creates: /tmp/ca-cert.pem + + - name: Generate Kafka private key and CSR + command: > + openssl req -new -newkey rsa:2048 -nodes -keyout /opt/kafka/config/kafka-key.pem + -out /opt/kafka/config/kafka.csr -subj "/CN=kafka-server/O=Conduktor/C=UK" + when: "'kafka' in group_names" + + - name: Sign Kafka CSR with CA certificate + command: > + openssl x509 -req -in /opt/kafka/config/kafka.csr -CA /tmp/ca-cert.pem + -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/kafka/config/kafka-cert.pem -days 365 + when: "'kafka' in group_names" + + - name: Import CA certificate to Kafka truststore + command: > + keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot + -import -file /tmp/ca-cert.pem -storepass "{{ kafka_truststore_password }}" -noprompt + when: "'kafka' in group_names" + + - name: Import Kafka certificate and key to Kafka keystore + command: > + keytool -importkeystore -srckeystore /opt/kafka/config/kafka-cert.pem + -destkeystore /opt/kafka/config/kafka-server.keystore.jks + -srcalias kafka-server -destalias kafka-server -srcstorepass "{{ kafka_keystore_password }}" + when: "'kafka' in group_names" + + - name: Generate Gateway private key and CSR + command: > + openssl req -new -newkey rsa:2048 -nodes -keyout /opt/conduktor-gateway/ssl/gateway-key.pem + -out /opt/conduktor-gateway/ssl/gateway.csr -subj "/CN=gateway-server/O=Conduktor/C=UK" + when: "'gateway' in group_names" + + - name: Sign Gateway CSR with CA certificate + command: > + openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr -CA /tmp/ca-cert.pem + -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem -days 365 + when: "'gateway' in group_names" + + - name: Import CA certificate to Gateway truststore + command: > + keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks -alias CARoot + -import -file /tmp/ca-cert.pem -storepass "{{ gateway_truststore_password }}" -noprompt + when: "'gateway' in group_names" + + - name: Import Gateway certificate and key to Gateway keystore + command: > + keytool -importkeystore -srckeystore /opt/conduktor-gateway/ssl/gateway-cert.pem + -destkeystore /opt/conduktor-gateway/ssl/gateway_keystore.jks + -srcalias gateway-server -destalias gateway-server -srcstorepass "{{ gateway_keystore_password }}" + when: "'gateway' in group_names" + - name: Kafka setup hosts: kafka connection: ssh @@ -146,63 +211,6 @@ remote_src: yes dest: /opt/kafka extra_opts: ["--strip-components=1"] - - - name: Ensure truststore and keystore directory exists - file: - path: /opt/kafka/config - state: directory - mode: '0755' - when: use_tls_and_ssl - - name: Remove existing truststore and keystore if use_tls_and_ssl is enabled - file: - path: /opt/kafka/config/kafka-server.keystore.jks - state: absent - when: use_tls_and_ssl - - name: Generate Kafka keystore if use_tls_and_ssl is enabled - command: > - keytool -genkeypair -alias kafka-server -keyalg RSA -keystore /opt/kafka/config/kafka-server.keystore.jks - -storepass "{{ kafka_keystore_password }}" -keypass "{{ kafka_keystore_password }}" - -dname "CN={{ inventory_hostname }}, OU=Kafka, O=Conduktor, L=London, ST=London, C=England" - when: use_tls_and_ssl - - name: Generate CSR for Kafka server if use_tls_and_ssl is enabled - command: > - keytool -keystore /opt/kafka/config/kafka-server.keystore.jks -alias kafka-server -certreq -file /opt/kafka/config/kafka-server.csr - -storepass "{{ kafka_keystore_password }}" - when: use_tls_and_ssl - - name: Import signed certificate to Kafka keystore if use_tls_and_ssl is enabled - command: > - keytool -keystore /opt/kafka/config/kafka-server.keystore.jks -alias kafka-server -import -file /opt/kafka/config/kafka-server-cert.pem - -storepass "{{ kafka_keystore_password }}" -noprompt - when: use_tls_and_ssl - - name: Set permissions for keystore and truststore if use_tls_and_ssl is enabled - file: - path: /opt/kafka/config/kafka-server.keystore.jks - owner: kafka - group: kafka - mode: '0600' - when: use_tls_and_ssl - - file: - path: /opt/kafka/config/kafka-server.truststore.jks - state: absent - when: use_tls_and_ssl - - name: Sign CSR with CA if use_tls_and_ssl is enabled - command: > - openssl x509 -req -in /opt/kafka/config/kafka-server.csr -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem - -CAcreateserial -out /opt/kafka/config/kafka-server-cert.pem -days 365 -sha256 - when: use_tls_and_ssl - - name: Import CA certificate to Kafka truststore if use_tls_and_ssl is enabled - command: > - keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot -import -file /tmp/ca-cert.pem - -storepass "{{ kafka_truststore_password }}" -noprompt - when: use_tls_and_ssl - - name: Set permissions for keystore and truststore if use_tls_and_ssl is enabled - file: - path: /opt/kafka/config/kafka-server.truststore.jks - owner: kafka - group: kafka - mode: '0600' - when: use_tls_and_ssl - - name: Reload and restart Kafka to apply configuration systemd: name: kafka @@ -329,63 +337,6 @@ username: "{{ lookup('ansible.builtin.env', 'REGISTRY_USERNAME') }}" password: "{{ lookup('ansible.builtin.env', 'REGISTRY_PASSWORD') }}" - - name: Ensure truststore and keystore directory exists - file: - path: /opt/conduktor-gateway/ssl - state: directory - mode: '0755' - - - name: Generate Gateway keystore if use_tls_and_ssl is enabled - command: > - keytool -genkeypair -alias gateway -keyalg RSA - -keystore /opt/conduktor-gateway/ssl/gateway_keystore.jks - -storepass "{{ gateway_keystore_password }}" -keypass "{{ gateway_keystore_password }}" - -dname "CN={{ inventory_hostname }}, OU=Conduktor, O=Conduktor, L=London, ST=London, C=England" - when: use_tls_and_ssl - - - name: Generate CSR for Gateway if use_tls_and_ssl is enabled - command: > - keytool -keystore /opt/conduktor-gateway/ssl/gateway_keystore.jks -alias gateway - -certreq -file /opt/conduktor-gateway/ssl/gateway.csr - -storepass "{{ gateway_keystore_password }}" - when: use_tls_and_ssl - - - name: Set permissions for keystore if use_tls_and_ssl is enabled - file: - path: /opt/conduktor-gateway/ssl/gateway_keystore.jks - owner: root - group: root - mode: '0600' - when: use_tls_and_ssl - - - name: Sign CSR with CA if use_tls_and_ssl is enabled - command: > - openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr - -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem - -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem - -days 365 -sha256 - when: use_tls_and_ssl - - name: Import signed certificate to Gateway keystore if use_tls_and_ssl is enabled - command: > - keytool -keystore /opt/conduktor-gateway/ssl/gateway_keystore.jks -alias gateway - -import -file /opt/conduktor-gateway/ssl/gateway-cert.pem - -storepass "{{ gateway_keystore_password }}" -noprompt - when: use_tls_and_ssl - - name: Import CA certificate to Gateway truststore if use_tls_and_ssl is enabled - command: > - keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks - -alias CARoot -import -file /tmp/ca-cert.pem - -storepass "{{ gateway_truststore_password }}" -noprompt - when: use_tls_and_ssl - - - name: Set permissions for truststore if use_tls_and_ssl is enabled - file: - path: /opt/conduktor-gateway/ssl/gateway_truststore.jks - owner: root - group: root - mode: '0600' - when: use_tls_and_ssl - - name: Set base volume mounts for Gateway set_fact: gateway_volumes: From 3217e0f332a0e79c860033cea6d58dc53abc8fcb Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Tue, 12 Nov 2024 11:48:25 +0000 Subject: [PATCH 4/9] Put TLS into a block with when statment. Fixed kafka install. Added missing vars. --- .../deploy/ssd-deployment/ansible-vars.yml | 9 +- .../deploy/ssd-deployment/deploy.yaml | 162 ++++++++++-------- .../templates/server_ssl.properties.j2 | 10 +- 3 files changed, 100 insertions(+), 81 deletions(-) diff --git a/driver-gateway/deploy/ssd-deployment/ansible-vars.yml b/driver-gateway/deploy/ssd-deployment/ansible-vars.yml index d472e6a8e..6787349e3 100644 --- a/driver-gateway/deploy/ssd-deployment/ansible-vars.yml +++ b/driver-gateway/deploy/ssd-deployment/ansible-vars.yml @@ -1,4 +1,11 @@ # vars.yml use_tls_and_ssl: false kafka_truststore_password: "admin123" -kafka_keystore_password: "admin123" \ No newline at end of file +kafka_keystore_password: "admin123" +kafka_key_password: "admin123" +gateway_truststore_password: "admin123" +gateway_keystore_password: "admin123" +gateway_key_password: "admin123" +client_truststore_password: "admin123" +client_keystore_password: "admin123" +client_key_password: "admin123" diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index d3c95d27d..5f556d728 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -130,71 +130,6 @@ echo 'LANG=en_US.utf-8 LC_ALL=en_US.utf-8' > /etc/environment -# TLS Certificate Generation for Kafka and Conduktor Gateway -- name: Generate CA and server certificates for TLS - hosts: [kafka, gateway] - become: true - when: use_tls_and_ssl - vars_files: - - ./ansible-vars.yml - tasks: - - name: Generate CA private key and certificate (if not exists) - command: > - openssl req -new -x509 -keyout /tmp/ca-key.pem -out /tmp/ca-cert.pem -days 365 -nodes - -subj "/CN=MyKafkaCA/O=Conduktor/C=UK" - args: - creates: /tmp/ca-cert.pem - - - name: Generate Kafka private key and CSR - command: > - openssl req -new -newkey rsa:2048 -nodes -keyout /opt/kafka/config/kafka-key.pem - -out /opt/kafka/config/kafka.csr -subj "/CN=kafka-server/O=Conduktor/C=UK" - when: "'kafka' in group_names" - - - name: Sign Kafka CSR with CA certificate - command: > - openssl x509 -req -in /opt/kafka/config/kafka.csr -CA /tmp/ca-cert.pem - -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/kafka/config/kafka-cert.pem -days 365 - when: "'kafka' in group_names" - - - name: Import CA certificate to Kafka truststore - command: > - keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot - -import -file /tmp/ca-cert.pem -storepass "{{ kafka_truststore_password }}" -noprompt - when: "'kafka' in group_names" - - - name: Import Kafka certificate and key to Kafka keystore - command: > - keytool -importkeystore -srckeystore /opt/kafka/config/kafka-cert.pem - -destkeystore /opt/kafka/config/kafka-server.keystore.jks - -srcalias kafka-server -destalias kafka-server -srcstorepass "{{ kafka_keystore_password }}" - when: "'kafka' in group_names" - - - name: Generate Gateway private key and CSR - command: > - openssl req -new -newkey rsa:2048 -nodes -keyout /opt/conduktor-gateway/ssl/gateway-key.pem - -out /opt/conduktor-gateway/ssl/gateway.csr -subj "/CN=gateway-server/O=Conduktor/C=UK" - when: "'gateway' in group_names" - - - name: Sign Gateway CSR with CA certificate - command: > - openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr -CA /tmp/ca-cert.pem - -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem -days 365 - when: "'gateway' in group_names" - - - name: Import CA certificate to Gateway truststore - command: > - keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks -alias CARoot - -import -file /tmp/ca-cert.pem -storepass "{{ gateway_truststore_password }}" -noprompt - when: "'gateway' in group_names" - - - name: Import Gateway certificate and key to Gateway keystore - command: > - keytool -importkeystore -srckeystore /opt/conduktor-gateway/ssl/gateway-cert.pem - -destkeystore /opt/conduktor-gateway/ssl/gateway_keystore.jks - -srcalias gateway-server -destalias gateway-server -srcstorepass "{{ gateway_keystore_password }}" - when: "'gateway' in group_names" - - name: Kafka setup hosts: kafka connection: ssh @@ -211,12 +146,6 @@ remote_src: yes dest: /opt/kafka extra_opts: ["--strip-components=1"] - - name: Reload and restart Kafka to apply configuration - systemd: - name: kafka - state: restarted - daemon_reload: yes - tags: [restart-kafka] - name: Setup ZooKeeper hosts: zookeeper @@ -248,6 +177,88 @@ daemon_reload: yes name: "zookeeper" +# TLS Certificate Generation for Kafka and Conduktor Gateway +- name: Generate CA and server certificates for TLS + hosts: [kafka, gateway] + become: true + vars_files: + - ./ansible-vars.yml + - ./tf_ansible_vars_file.yml + tasks: + - name: Generate certificates for Kafka and Conduktor Gateway + when: use_tls_and_ssl + block: + - name: Generate CA private key and certificate (if not exists) + command: > + openssl req -new -x509 -keyout /tmp/ca-key.pem -out /tmp/ca-cert.pem -days 365 -nodes + -subj "/CN=MyKafkaCA/O=Conduktor/C=UK" + args: + creates: /tmp/ca-cert.pem + + - name: Create SSL directory for Gateway if it doesn't exist + file: + path: /opt/conduktor-gateway/ssl + state: directory + mode: '0755' + when: "'gateway' in group_names" + + - name: Create SSL directory for Kafka if it doesn't exist + file: + path: /opt/kafka/config + state: directory + mode: '0755' + when: "'kafka' in group_names" + + - name: Generate Kafka private key and CSR + command: > + openssl req -new -newkey rsa:2048 -nodes -keyout /opt/kafka/config/kafka-key.pem + -out /opt/kafka/config/kafka.csr -subj "/CN=kafka-server/O=Conduktor/C=UK" + when: "'kafka' in group_names" + + - name: Sign Kafka CSR with CA certificate + command: > + openssl x509 -req -in /opt/kafka/config/kafka.csr -CA /tmp/ca-cert.pem + -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/kafka/config/kafka-cert.pem -days 365 + when: "'kafka' in group_names" + + - name: Import CA certificate to Kafka truststore + command: > + keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot + -import -file /tmp/ca-cert.pem -storepass "{{ kafka_truststore_password }}" -noprompt + when: "'kafka' in group_names" + + - name: Import Kafka certificate and key to Kafka keystore + command: > + keytool -importkeystore -srckeystore /opt/kafka/config/kafka-cert.pem + -destkeystore /opt/kafka/config/kafka-server.keystore.jks + -srcalias kafka-server -destalias kafka-server -srcstorepass "{{ kafka_keystore_password }}" + when: "'kafka' in group_names" + + - name: Generate Gateway private key and CSR + command: > + openssl req -new -newkey rsa:2048 -nodes -keyout /opt/conduktor-gateway/ssl/gateway-key.pem + -out /opt/conduktor-gateway/ssl/gateway.csr -subj "/CN=gateway-server/O=Conduktor/C=UK" + when: "'gateway' in group_names" + + - name: Sign Gateway CSR with CA certificate + command: > + openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr -CA /tmp/ca-cert.pem + -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem -days 365 + when: "'gateway' in group_names" + + - name: Import CA certificate to Gateway truststore + command: > + keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks -alias CARoot + -import -file /tmp/ca-cert.pem -storepass "{{ gateway_truststore_password }}" -noprompt + when: "'gateway' in group_names" + + - name: Import Gateway certificate and key to Gateway keystore + command: > + keytool -importkeystore -srckeystore /opt/conduktor-gateway/ssl/gateway-cert.pem + -destkeystore /opt/conduktor-gateway/ssl/gateway_keystore.jks + -srcalias gateway-server -destalias gateway-server -srcstorepass "{{ gateway_keystore_password }}" + when: "'gateway' in group_names" + - name: Setup Kafka hosts: kafka connection: ssh @@ -268,10 +279,11 @@ src: templates/server_ssl.properties.j2 dest: /opt/kafka/config/server_ssl.properties vars: - kafka_truststore_path: /opt/kafka/config/kafka-server.truststore.jks - kafka_truststore_password: "{{ kafka_truststore_password }}" - kafka_keystore_path: /opt/kafka/config/kafka-server.keystore.jks - kafka_keystore_password: "{{ kafka_keystore_password }}" + template_kafka_truststore_path: /opt/kafka/config/kafka-server.truststore.jks + template_kafka_truststore_password: "{{ kafka_truststore_password }}" + template_kafka_keystore_path: /opt/kafka/config/kafka-server.keystore.jks + template_kafka_keystore_password: "{{ kafka_keystore_password }}" + template_kafka_key_password: "{{ kafka_key_password }}" when: use_tls_and_ssl - template: src: "templates/kafka.service" diff --git a/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 b/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 index 04b4ae494..64378beed 100644 --- a/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 +++ b/driver-gateway/deploy/ssd-deployment/templates/server_ssl.properties.j2 @@ -13,11 +13,11 @@ # # SSL Settings -ssl.keystore.location={{ kafka_keystore_path }} -ssl.keystore.password={{ kafka_keystore_password }} -ssl.key.password={{ kafka_key_password }} -ssl.truststore.location={{ kafka_truststore_path }} -ssl.truststore.password={{ kafka_truststore_password }} +ssl.keystore.location={{ template_kafka_keystore_path }} +ssl.keystore.password={{ template_kafka_keystore_password }} +ssl.key.password={{ template_kafka_key_password }} +ssl.truststore.location={{ template_kafka_truststore_path }} +ssl.truststore.password={{ template_kafka_truststore_password }} # SASL Authentication security.protocol=SASL_SSL From 7682847f688b69271dded2d0a5613a0d581e0ffc Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Tue, 12 Nov 2024 13:38:24 +0000 Subject: [PATCH 5/9] Made it so you can re-run ansible without it breaking deployment. Changed how we setup volume mounts. --- .../deploy/ssd-deployment/deploy.yaml | 49 +++++++++++++------ .../deploy/ssd-deployment/inventory.ini | 35 +++++++++++++ .../templates/gateway_volumes.j2 | 2 - 3 files changed, 70 insertions(+), 16 deletions(-) create mode 100755 driver-gateway/deploy/ssd-deployment/inventory.ini delete mode 100644 driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index 5f556d728..7de0b4c82 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -185,9 +185,9 @@ - ./ansible-vars.yml - ./tf_ansible_vars_file.yml tasks: - - name: Generate certificates for Kafka and Conduktor Gateway - when: use_tls_and_ssl - block: + - name: Generate certificates for Kafka and Conduktor Gateway + when: use_tls_and_ssl + block: - name: Generate CA private key and certificate (if not exists) command: > openssl req -new -x509 -keyout /tmp/ca-key.pem -out /tmp/ca-cert.pem -days 365 -nodes @@ -209,54 +209,70 @@ mode: '0755' when: "'kafka' in group_names" - - name: Generate Kafka private key and CSR + - name: Generate Kafka private key and CSR (if key does not exist) command: > openssl req -new -newkey rsa:2048 -nodes -keyout /opt/kafka/config/kafka-key.pem -out /opt/kafka/config/kafka.csr -subj "/CN=kafka-server/O=Conduktor/C=UK" + args: + creates: /opt/kafka/config/kafka-key.pem when: "'kafka' in group_names" - - name: Sign Kafka CSR with CA certificate + - name: Sign Kafka CSR with CA certificate (if not already signed) command: > openssl x509 -req -in /opt/kafka/config/kafka.csr -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/kafka/config/kafka-cert.pem -days 365 + args: + creates: /opt/kafka/config/kafka-cert.pem when: "'kafka' in group_names" - - name: Import CA certificate to Kafka truststore + - name: Import CA certificate to Kafka truststore (if not exists) command: > keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot -import -file /tmp/ca-cert.pem -storepass "{{ kafka_truststore_password }}" -noprompt + args: + creates: /opt/kafka/config/kafka-server.truststore.jks when: "'kafka' in group_names" - - name: Import Kafka certificate and key to Kafka keystore + - name: Import Kafka certificate and key to Kafka keystore (if not exists) command: > keytool -importkeystore -srckeystore /opt/kafka/config/kafka-cert.pem -destkeystore /opt/kafka/config/kafka-server.keystore.jks -srcalias kafka-server -destalias kafka-server -srcstorepass "{{ kafka_keystore_password }}" + args: + creates: /opt/kafka/config/kafka-server.keystore.jks when: "'kafka' in group_names" - - name: Generate Gateway private key and CSR + - name: Generate Gateway private key and CSR (if key does not exist) command: > openssl req -new -newkey rsa:2048 -nodes -keyout /opt/conduktor-gateway/ssl/gateway-key.pem -out /opt/conduktor-gateway/ssl/gateway.csr -subj "/CN=gateway-server/O=Conduktor/C=UK" + args: + creates: /opt/conduktor-gateway/ssl/gateway-key.pem when: "'gateway' in group_names" - - name: Sign Gateway CSR with CA certificate + - name: Sign Gateway CSR with CA certificate (if not already signed) command: > openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem -days 365 + args: + creates: /opt/conduktor-gateway/ssl/gateway-cert.pem when: "'gateway' in group_names" - - name: Import CA certificate to Gateway truststore + - name: Import CA certificate to Gateway truststore (if not exists) command: > keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks -alias CARoot -import -file /tmp/ca-cert.pem -storepass "{{ gateway_truststore_password }}" -noprompt + args: + creates: /opt/conduktor-gateway/ssl/gateway_truststore.jks when: "'gateway' in group_names" - - name: Import Gateway certificate and key to Gateway keystore + - name: Import Gateway certificate and key to Gateway keystore (if not exists) command: > keytool -importkeystore -srckeystore /opt/conduktor-gateway/ssl/gateway-cert.pem -destkeystore /opt/conduktor-gateway/ssl/gateway_keystore.jks -srcalias gateway-server -destalias gateway-server -srcstorepass "{{ gateway_keystore_password }}" + args: + creates: /opt/conduktor-gateway/ssl/gateway_keystore.jks when: "'gateway' in group_names" - name: Setup Kafka @@ -350,13 +366,18 @@ password: "{{ lookup('ansible.builtin.env', 'REGISTRY_PASSWORD') }}" - name: Set base volume mounts for Gateway + when: not use_tls_and_ssl set_fact: gateway_volumes: - "/opt/gateway-interceptors.json:/opt/interceptors.json" - - name: Add TLS volume mounts to Gateway if enabled - set_fact: - gateway_volumes: "{{ gateway_volumes + [ lookup('template', 'gateway_volumes.j2').splitlines() ] | flatten }}" + + - name: Set base volume mounts for Gateway with TLS when: use_tls_and_ssl + set_fact: + gateway_volumes: + - "/opt/gateway-interceptors.json:/opt/interceptors.json" + - "/opt/conduktor-gateway/ssl/gateway_keystore.jks:/etc/conduktor-gateway/keystore.jks" + - "/opt/conduktor-gateway/ssl/gateway_truststore.jks:/etc/conduktor-gateway/truststore.jks" - name: Set base environment variables for Gateway set_fact: diff --git a/driver-gateway/deploy/ssd-deployment/inventory.ini b/driver-gateway/deploy/ssd-deployment/inventory.ini new file mode 100755 index 000000000..839eaf26a --- /dev/null +++ b/driver-gateway/deploy/ssd-deployment/inventory.ini @@ -0,0 +1,35 @@ +[all] +35.86.106.41 private_ip=10.0.0.200 +44.242.221.73 private_ip=10.0.0.32 +54.148.97.8 private_ip=10.0.0.53 +52.26.123.207 private_ip=10.0.0.43 +35.91.193.16 private_ip=10.0.0.39 +34.221.98.106 private_ip=10.0.0.6 +54.245.201.249 private_ip=10.0.0.119 +44.242.221.86 private_ip=10.0.0.45 +52.37.13.103 private_ip=10.0.0.120 +35.91.194.68 private_ip=10.0.0.80 +52.10.149.58 private_ip=10.0.0.129 +35.95.38.131 private_ip=10.0.0.151 +54.190.227.251 private_ip=10.0.0.180 + +[zookeeper] +35.86.106.41 private_ip=10.0.0.200 +44.242.221.73 private_ip=10.0.0.32 +54.148.97.8 private_ip=10.0.0.53 + +[kafka] +52.26.123.207 private_ip=10.0.0.43 +35.91.193.16 private_ip=10.0.0.39 +34.221.98.106 private_ip=10.0.0.6 + +[gateway] +52.10.149.58 private_ip=10.0.0.129 +35.95.38.131 private_ip=10.0.0.151 +54.190.227.251 private_ip=10.0.0.180 + +[client] +54.245.201.249 private_ip=10.0.0.119 +44.242.221.86 private_ip=10.0.0.45 +52.37.13.103 private_ip=10.0.0.120 +35.91.194.68 private_ip=10.0.0.80 diff --git a/driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 b/driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 deleted file mode 100644 index b0fe37753..000000000 --- a/driver-gateway/deploy/ssd-deployment/templates/gateway_volumes.j2 +++ /dev/null @@ -1,2 +0,0 @@ -- "/opt/conduktor-gateway/ssl/gateway_keystore.jks:/etc/conduktor-gateway/keystore.jks" -- "/opt/conduktor-gateway/ssl/gateway_truststore.jks:/etc/conduktor-gateway/truststore.jks" From ffd235576c0fc060fde5c266b65621c623b63048 Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Tue, 12 Nov 2024 13:42:35 +0000 Subject: [PATCH 6/9] Updated README.md --- driver-gateway/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/driver-gateway/README.md b/driver-gateway/README.md index 47568996c..00322496f 100644 --- a/driver-gateway/README.md +++ b/driver-gateway/README.md @@ -15,12 +15,13 @@ Pre-requisites: 4. Initialize terraform with `terraform init` 5. Create the infrastructure with `terraform apply` 1. You can change the value of num_instances using -var='num_instances={"client"=5, "gateway"=4, "kafka"=2}' - 2. You might also need to change the AWS profile with -var='aws_profile=your_profile' + 2. You might also need to change the AWS profile with -var='profile=your_profile' 3. With SSO the session does expire so you might need to run `aws sso login --profile your_profile` before running terraform 6. Export your harbor creds https://harbor.cdkt.dev/ 1. Click your name in the top right corner and select User Profile then take your username and CLI secret 2. export REGISTRY_USERNAME= 3. export REGISTRY_PASSWORD= + 4. export GATEWAY_IMAGE=harbor.cdkt.dev/conduktor/conduktor-gateway:latest 7. Setup nodes with `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml` 1. If you want to set tls and use delegated_sasl_ssl run `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml -e "use_tls_and_ssl=true"` 8. Connect to one benchmark worker node with `ssh -i ~/.ssh/kafka_aws ec2-user@$(terraform output client_ssh_host | tr -d '"')` From 38b044918aeb196173bf253dfe97e5e64b98fed2 Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Tue, 12 Nov 2024 14:13:42 +0000 Subject: [PATCH 7/9] Removed inventory, reverted pom changes. --- .../deploy/ssd-deployment/inventory.ini | 35 --- pom.xml | 252 +++++++++++++++++- 2 files changed, 250 insertions(+), 37 deletions(-) delete mode 100755 driver-gateway/deploy/ssd-deployment/inventory.ini diff --git a/driver-gateway/deploy/ssd-deployment/inventory.ini b/driver-gateway/deploy/ssd-deployment/inventory.ini deleted file mode 100755 index 839eaf26a..000000000 --- a/driver-gateway/deploy/ssd-deployment/inventory.ini +++ /dev/null @@ -1,35 +0,0 @@ -[all] -35.86.106.41 private_ip=10.0.0.200 -44.242.221.73 private_ip=10.0.0.32 -54.148.97.8 private_ip=10.0.0.53 -52.26.123.207 private_ip=10.0.0.43 -35.91.193.16 private_ip=10.0.0.39 -34.221.98.106 private_ip=10.0.0.6 -54.245.201.249 private_ip=10.0.0.119 -44.242.221.86 private_ip=10.0.0.45 -52.37.13.103 private_ip=10.0.0.120 -35.91.194.68 private_ip=10.0.0.80 -52.10.149.58 private_ip=10.0.0.129 -35.95.38.131 private_ip=10.0.0.151 -54.190.227.251 private_ip=10.0.0.180 - -[zookeeper] -35.86.106.41 private_ip=10.0.0.200 -44.242.221.73 private_ip=10.0.0.32 -54.148.97.8 private_ip=10.0.0.53 - -[kafka] -52.26.123.207 private_ip=10.0.0.43 -35.91.193.16 private_ip=10.0.0.39 -34.221.98.106 private_ip=10.0.0.6 - -[gateway] -52.10.149.58 private_ip=10.0.0.129 -35.95.38.131 private_ip=10.0.0.151 -54.190.227.251 private_ip=10.0.0.180 - -[client] -54.245.201.249 private_ip=10.0.0.119 -44.242.221.86 private_ip=10.0.0.45 -52.37.13.103 private_ip=10.0.0.120 -35.91.194.68 private_ip=10.0.0.80 diff --git a/pom.xml b/pom.xml index d491378aa..92c07ca00 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 10.3.3 3.12.0 2.20.0 - 1.18.34 + 1.18.24 2.13.2 1.48 5.9.0 @@ -173,6 +173,254 @@ + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.plugin.version} + + + + false + true + 4 + scope,groupId,artifactId + groupId,artifactId + groupId,artifactId + + + + + + + + true + 2 + + + true + 4 + + + + + **/*.md + + + + + + + etc/**/*.xml + .github/workflows/**/*.yml + **/doc/**/*.puml + + + + + true + 4 + + + + + + + check + + check + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs.plugin.version} + + etc/findbugsExclude.xml + + + + + check + + + + + + + com.mycila + license-maven-plugin + ${license.plugin.version} + +
etc/APACHE-2.txt
+ + LICENSE + NOTICE + payload/** + **/*.pyc + **/*.sh + **/.pydevproject + .github/** + + + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + XML_STYLE + SCRIPT_STYLE + APOSTROPHE_STYLE + + + + + com.mycila + license-maven-plugin-git + ${license.plugin.version} + + + + + + check + + process-sources + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle.plugin.version} + + true + true + true + etc/checkstyle.xml + true + + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + + + check + + validate + + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven.enforcer.plugin.version} + + + enforce-maven-version + + enforce + + + + + [3.8.6,) + + + true + + + + + + maven-failsafe-plugin + ${maven.surefire.plugin.version} + + + + integration-test + verify + + + + + + maven-surefire-plugin + ${maven.surefire.plugin.version} + + + --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.plugin.version} + + + + prepare-agent + + + + report + + report + + verify + + + check + + check + + verify + + + + BUNDLE + + + CLASS + COVEREDRATIO + 0.00 + + + METHOD + COVEREDRATIO + 0.00 + + + LINE + COVEREDRATIO + 0.00 + + + BRANCH + COVEREDRATIO + 0.00 + + + + + + + + + + + modern-java-compile @@ -226,4 +474,4 @@ - + \ No newline at end of file From bee6a0bc63cad5c39c5614e1814f0b0aa898a028 Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Tue, 12 Nov 2024 16:32:40 +0000 Subject: [PATCH 8/9] Added truststore to client. --- .../deploy/ssd-deployment/deploy.yaml | 121 +++++++++--------- .../ssd-deployment/templates/gateway_env.j2 | 14 +- 2 files changed, 68 insertions(+), 67 deletions(-) diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index 7de0b4c82..8c6ebc222 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -177,103 +177,102 @@ daemon_reload: yes name: "zookeeper" -# TLS Certificate Generation for Kafka and Conduktor Gateway -- name: Generate CA and server certificates for TLS - hosts: [kafka, gateway] +# Set up TLS for Client, Gateway, and Kafka +- name: Generate CA and certificates + hosts: [kafka, gateway, client] become: true vars_files: - ./ansible-vars.yml - - ./tf_ansible_vars_file.yml tasks: - - name: Generate certificates for Kafka and Conduktor Gateway - when: use_tls_and_ssl + - name: Generate CA private key and certificate if not exist + command: > + openssl req -new -x509 -keyout /tmp/ca-key.pem -out /tmp/ca-cert.pem -days 365 -nodes + -subj "/CN=MyKafkaCA/O=Conduktor/C=UK" + args: + creates: /tmp/ca-cert.pem + # Gateway Certificate and Truststore Setup + - name: Generate certificates for Gateway + when: "'gateway' in group_names and use_tls_and_ssl" block: - - name: Generate CA private key and certificate (if not exists) - command: > - openssl req -new -x509 -keyout /tmp/ca-key.pem -out /tmp/ca-cert.pem -days 365 -nodes - -subj "/CN=MyKafkaCA/O=Conduktor/C=UK" - args: - creates: /tmp/ca-cert.pem - - - name: Create SSL directory for Gateway if it doesn't exist + - name: Create SSL directory for Gateway file: path: /opt/conduktor-gateway/ssl state: directory mode: '0755' - when: "'gateway' in group_names" - - - name: Create SSL directory for Kafka if it doesn't exist + - name: Generate Gateway private key and CSR if not exist + command: > + openssl req -new -newkey rsa:2048 -nodes -keyout /opt/conduktor-gateway/ssl/gateway-key.pem + -out /opt/conduktor-gateway/ssl/gateway.csr -subj "/CN=gateway-server/O=Conduktor/C=UK" + args: + creates: /opt/conduktor-gateway/ssl/gateway-key.pem + - name: Sign Gateway CSR with CA certificate + command: > + openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr -CA /tmp/ca-cert.pem + -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem -days 365 + args: + creates: /opt/conduktor-gateway/ssl/gateway-cert.pem + - name: Import CA certificate to Gateway truststore + command: > + keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks -alias CARoot + -import -file /tmp/ca-cert.pem -storepass "{{ gateway_truststore_password }}" -noprompt + args: + creates: /opt/conduktor-gateway/ssl/gateway_truststore.jks + - name: Import Gateway certificate and key to Gateway keystore + command: > + keytool -importkeystore -srckeystore /opt/conduktor-gateway/ssl/gateway-cert.pem + -destkeystore /opt/conduktor-gateway/ssl/gateway_keystore.jks + -srcalias gateway-server -destalias gateway-server -srcstorepass "{{ gateway_keystore_password }}" + args: + creates: /opt/conduktor-gateway/ssl/gateway_keystore.jks + # Kafka Certificate and Truststore Setup + - name: Generate certificates for Kafka + when: "'kafka' in group_names and use_tls_and_ssl" + block: + - name: Create SSL directory for Kafka file: path: /opt/kafka/config state: directory mode: '0755' - when: "'kafka' in group_names" - - - name: Generate Kafka private key and CSR (if key does not exist) + - name: Generate Kafka private key and CSR if not exist command: > openssl req -new -newkey rsa:2048 -nodes -keyout /opt/kafka/config/kafka-key.pem -out /opt/kafka/config/kafka.csr -subj "/CN=kafka-server/O=Conduktor/C=UK" args: creates: /opt/kafka/config/kafka-key.pem - when: "'kafka' in group_names" - - - name: Sign Kafka CSR with CA certificate (if not already signed) + - name: Sign Kafka CSR with CA certificate command: > openssl x509 -req -in /opt/kafka/config/kafka.csr -CA /tmp/ca-cert.pem -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/kafka/config/kafka-cert.pem -days 365 args: creates: /opt/kafka/config/kafka-cert.pem - when: "'kafka' in group_names" - - - name: Import CA certificate to Kafka truststore (if not exists) + - name: Import CA certificate to Kafka truststore command: > keytool -keystore /opt/kafka/config/kafka-server.truststore.jks -alias CARoot -import -file /tmp/ca-cert.pem -storepass "{{ kafka_truststore_password }}" -noprompt args: creates: /opt/kafka/config/kafka-server.truststore.jks - when: "'kafka' in group_names" - - - name: Import Kafka certificate and key to Kafka keystore (if not exists) + - name: Import Kafka certificate and key to Kafka keystore command: > keytool -importkeystore -srckeystore /opt/kafka/config/kafka-cert.pem -destkeystore /opt/kafka/config/kafka-server.keystore.jks -srcalias kafka-server -destalias kafka-server -srcstorepass "{{ kafka_keystore_password }}" args: creates: /opt/kafka/config/kafka-server.keystore.jks - when: "'kafka' in group_names" - - - name: Generate Gateway private key and CSR (if key does not exist) - command: > - openssl req -new -newkey rsa:2048 -nodes -keyout /opt/conduktor-gateway/ssl/gateway-key.pem - -out /opt/conduktor-gateway/ssl/gateway.csr -subj "/CN=gateway-server/O=Conduktor/C=UK" - args: - creates: /opt/conduktor-gateway/ssl/gateway-key.pem - when: "'gateway' in group_names" - - - name: Sign Gateway CSR with CA certificate (if not already signed) - command: > - openssl x509 -req -in /opt/conduktor-gateway/ssl/gateway.csr -CA /tmp/ca-cert.pem - -CAkey /tmp/ca-key.pem -CAcreateserial -out /opt/conduktor-gateway/ssl/gateway-cert.pem -days 365 - args: - creates: /opt/conduktor-gateway/ssl/gateway-cert.pem - when: "'gateway' in group_names" - - - name: Import CA certificate to Gateway truststore (if not exists) - command: > - keytool -keystore /opt/conduktor-gateway/ssl/gateway_truststore.jks -alias CARoot - -import -file /tmp/ca-cert.pem -storepass "{{ gateway_truststore_password }}" -noprompt - args: - creates: /opt/conduktor-gateway/ssl/gateway_truststore.jks - when: "'gateway' in group_names" - - - name: Import Gateway certificate and key to Gateway keystore (if not exists) + # Client Certificate and Truststore Setup + - name: Configure Client for TLS + when: "'client' in group_names and use_tls_and_ssl" + block: + - name: Create SSL directory for Client + file: + path: /opt/client/ssl + state: directory + mode: '0755' + - name: Import Gateway CA certificate to Client truststore command: > - keytool -importkeystore -srckeystore /opt/conduktor-gateway/ssl/gateway-cert.pem - -destkeystore /opt/conduktor-gateway/ssl/gateway_keystore.jks - -srcalias gateway-server -destalias gateway-server -srcstorepass "{{ gateway_keystore_password }}" + keytool -keystore /opt/client/ssl/client-truststore.jks -alias CARoot + -import -file /tmp/ca-cert.pem -storepass "{{ client_truststore_password }}" -noprompt args: - creates: /opt/conduktor-gateway/ssl/gateway_keystore.jks - when: "'gateway' in group_names" + creates: /opt/client/ssl/client-truststore.jks - name: Setup Kafka hosts: kafka diff --git a/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 b/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 index 09a1422cb..937d39d94 100644 --- a/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 +++ b/driver-gateway/deploy/ssd-deployment/templates/gateway_env.j2 @@ -1,6 +1,8 @@ -SECURITY_PROTOCOL: "SASL_SSL" -SSL_TRUSTSTORE_LOCATION: "/etc/conduktor-gateway/truststore.jks" -SSL_TRUSTSTORE_PASSWORD: "{{ gateway_truststore_password }}" -SSL_KEYSTORE_LOCATION: "/etc/conduktor-gateway/keystore.jks" -SSL_KEYSTORE_PASSWORD: "{{ gateway_keystore_password }}" -SSL_KEY_PASSWORD: "{{ gateway_key_password }}" +GATEWAY_SECURITY_PROTOCOL: DELEGATED_SASL_SSL +KAFKA_SASL_MECHANISM: "SCRAM-SHA-256" +KAFKA_SECURITY_PROTOCOL: SSL +KAFKA_SSL_TRUSTSTORE_LOCATION: "/etc/conduktor-gateway/truststore.jks" +KAFKA_SSL_TRUSTSTORE_PASSWORD: "{{ gateway_truststore_password }}" +KAFKA_SSL_KEYSTORE_LOCATION: "/etc/conduktor-gateway/keystore.jks" +KAFKA_SSL_KEYSTORE_PASSWORD: "{{ gateway_keystore_password }}" +KAFKA_SSL_KEY_PASSWORD: "{{ gateway_key_password }}" From 7bba8a6f64cabff6a06411caa5af841d3b0920ed Mon Sep 17 00:00:00 2001 From: Ben Starmer-Smith Date: Tue, 12 Nov 2024 17:25:29 +0000 Subject: [PATCH 9/9] Added the SSL settings to each kafka client driver. --- driver-gateway/deploy/ssd-deployment/deploy.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/driver-gateway/deploy/ssd-deployment/deploy.yaml b/driver-gateway/deploy/ssd-deployment/deploy.yaml index 8c6ebc222..cffc8bccd 100644 --- a/driver-gateway/deploy/ssd-deployment/deploy.yaml +++ b/driver-gateway/deploy/ssd-deployment/deploy.yaml @@ -446,6 +446,8 @@ hosts: client connection: ssh become: true + vars_files: + - ./ansible-vars.yml tasks: - file: path=/opt/benchmark state=absent tags: [client-code] @@ -471,6 +473,20 @@ with_items: '{{ drivers_list.stdout_lines }}' tags: [client-code] + - name: Configure SSL/SASL settings in client driver configuration + blockinfile: + path: "{{ item }}" + marker: "# {mark} SSL and SASL Configurations" + content: | + security.protocol=SASL_SSL + ssl.truststore.location=/opt/client/ssl/client-truststore.jks + ssl.truststore.password={{ client_truststore_password }} + sasl.mechanism=PLAIN + sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; + with_items: "{{ drivers_list.stdout_lines }}" + when: use_tls_and_ssl + tags: [client-code] + - name: Get list of gateway driver config files raw: ls -1 /opt/benchmark/driver-gateway/gateway-*.yaml register: gateway_drivers_list