From 0f4e463e02d1fefe12cceb66e81192f29b3c947a Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Mon, 11 Sep 2017 15:50:25 -0500 Subject: [PATCH 01/14] WIP for Kafka Manager --- config/deploy/test.rb | 2 + config/settings/test.yml | 42 ++++++++++++- lib/bash/debian/kafka_manager.sh | 18 +++--- lib/boot.rb | 1 + lib/kafka_manager/README.md | 19 ++++++ .../kafka_manager_configure.rake | 62 +++++++++++++++++++ lib/kafka_manager/kafka_manager_helpers.rb | 24 +++++++ lib/kafka_manager/kafka_manager_install.rake | 23 +++++++ lib/kafka_manager/kafka_manager_nodes.rake | 43 +++++++++++++ lib/kafka_manager/kafka_manager_service.rake | 42 +++++++++++++ lib/ubuntu/ubuntu_helper.rb | 4 ++ 11 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 lib/kafka_manager/README.md create mode 100644 lib/kafka_manager/kafka_manager_configure.rake create mode 100644 lib/kafka_manager/kafka_manager_helpers.rb create mode 100644 lib/kafka_manager/kafka_manager_install.rake create mode 100644 lib/kafka_manager/kafka_manager_nodes.rake create mode 100644 lib/kafka_manager/kafka_manager_service.rake diff --git a/config/deploy/test.rb b/config/deploy/test.rb index 1630746..f7d9624 100644 --- a/config/deploy/test.rb +++ b/config/deploy/test.rb @@ -42,6 +42,8 @@ server 'test_kafka2', user: 'ubuntu', roles: %w[ubuntu kafka] server 'test_kafka3', user: 'ubuntu', roles: %w[ubuntu kafka] +server 'test_kafka_manager', user: 'ubuntu', roles: %w[ubuntu kafka_manager] + # role-based syntax # ================== diff --git a/config/settings/test.yml b/config/settings/test.yml index 91fe34b..0661f43 100644 --- a/config/settings/test.yml +++ b/config/settings/test.yml @@ -108,6 +108,7 @@ aws: # - advertised.listeners test_kafka_configuration: resource: configuration + tag_service: kafka kafka_home: "/opt/kafka" # symlinks to /opt/kafka_{SCALA_VERSION}-{KAFKA_VERSION} kafka_version: "0.11.0.0" scala_version: "2.11" @@ -148,6 +149,38 @@ aws: tag_name: test_kafka3 availability_zone: us-west-2c # need one node in each zone + # --- + # Kafka Manager + + test_kafka_manager_configuration: + resource: configuration + tag_service: kafka_manager + basicAuthentication: + enabled: false + username: "admin" + password: "password" + realm: "Kafka-Manager" + excluded: + - "/api/health" + # KMClusterManagerFeature - allows adding, updating, deleting cluster from Kafka Manager + # KMTopicManagerFeature - allows adding, updating, deleting topic from a Kafka cluster + # KMPreferredReplicaElectionFeature - allows running of preferred replica election for a Kafka cluster + # KMReassignPartitionsFeature - allows generating partition assignments and reassigning partitions + features: + - KMClusterManagerFeature + - KMTopicManagerFeature + - KMPreferredReplicaElectionFeature + - KMReassignPartitionsFeature + + test_kafka_manager: + <<: *TestDefault + tag_service: kafka_manager + tag_group: test_kafka + tag_name: test_kafka_manager + security_groups: + - test_ssh_security_group + - test_kafka_security_group + # --- # Mesos Nodes # - note definition and use of defaults: MesosMasterDefaults, MesosAgentDefaults @@ -267,9 +300,16 @@ aws: group_id: ip_permissions: - ip_protocol: tcp - from_port: 9092 + from_port: 9092 # kafka clients to_port: 9092 ip_ranges: - cidr_ip: '0.0.0.0/0' ipv_6_ranges: - cidr_ipv_6: '::/0' + - ip_protocol: tcp + from_port: 9000 # kafka manager + to_port: 9000 + ip_ranges: + - cidr_ip: '0.0.0.0/0' + ipv_6_ranges: + - cidr_ipv_6: '::/0' diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index 690966b..6c12a19 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -1,6 +1,13 @@ +#!/usr/bin/env bash + +if [ -d /usr/share/kafka-manager ]; then + echo "Kafka manager is installed" + exit +fi # --- # Download +cd /tmp git clone https://github.com/yahoo/kafka-manager @@ -10,16 +17,9 @@ cd kafka-manager sbt debian:packageBin -exit - - # --- # Install -cd ../ -DIST=$(basename ${SRC}/core/build/distributions/kafka_*-${VER}.tgz .tgz) -tar zxf ${SRC}/core/build/distributions/${DIST}.tgz -C /usr/local/ -rm -f /usr/local/kafka -ln -s /usr/local/$DIST /usr/local/kafka - +sudo dpkg -i -R target/ +# dpkg -L kafka-manager diff --git a/lib/boot.rb b/lib/boot.rb index 96124739..1afffe3 100644 --- a/lib/boot.rb +++ b/lib/boot.rb @@ -24,5 +24,6 @@ require_relative 'redhat/redhat_helper' require_relative 'kafka/kafka_helpers' +require_relative 'kafka_manager/kafka_manager_helpers' require_relative 'zookeeper/zookeeper_helpers' diff --git a/lib/kafka_manager/README.md b/lib/kafka_manager/README.md new file mode 100644 index 0000000..7277195 --- /dev/null +++ b/lib/kafka_manager/README.md @@ -0,0 +1,19 @@ +# Kafka Manager + +- https://github.com/yahoo/kafka-manager + +## Configuration + +- https://github.com/yahoo/kafka-manager#configuration + +See `lib/kafka/kafka_manager_configure.rake` + +Look for "kafka_manager" in: + - `config/settings/{stage}.yml` + - `config/deploy/{stage}.rb` + +## Capistrano tasks + +```bash +$ bundle exec cap -T | grep kafka_manager +``` diff --git a/lib/kafka_manager/kafka_manager_configure.rake b/lib/kafka_manager/kafka_manager_configure.rake new file mode 100644 index 0000000..3ff3cd9 --- /dev/null +++ b/lib/kafka_manager/kafka_manager_configure.rake @@ -0,0 +1,62 @@ +require_relative 'kafka_manager_helpers' + +# Kafka Manager Configuration +# https://github.com/yahoo/kafka-manager#configuration +# +namespace :kafka_manager do + namespace :service do + def kafka_manager_conf + @kafka_manager_conf ||= capture('ls /usr/share/kafka-manager/conf/application.conf') + end + + # basicAuthentication + # basicAuthentication.enabled=false + # basicAuthentication.username="admin" + # basicAuthentication.password="password" + # basicAuthentication.realm="Kafka-Manager" + # basicAuthentication.excluded=["/api/health"] # ping the health of your instance without authentification + def kafka_manager_authentication + return unless configuration.basicAuthentication.enabled + enabled = "basicAuthentication.enabled=#{configuration.basicAuthentication.enabled}" + sudo("sed -i -e 's#basicAuthentication.enabled=.*##{enabled}#' #{kafka_manager_conf}") + # basicAuthentication.username="admin" + username = "basicAuthentication.username=\"#{configuration.basicAuthentication.username}\"" + sudo("sed -i -e 's#basicAuthentication.username=.*##{username}#' #{kafka_manager_conf}") + # basicAuthentication.password="password" + password = "basicAuthentication.password=\"#{configuration.basicAuthentication.password}\"" + sudo("sed -i -e 's#basicAuthentication.password=.*##{password}#' #{kafka_manager_conf}") + end + + # application.features=["KMClusterManagerFeature","KMTopicManagerFeature", + # "KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"] + # + # KMClusterManagerFeature - allows adding, updating, deleting cluster from Kafka Manager + # KMTopicManagerFeature - allows adding, updating, deleting topic from a Kafka cluster + # KMPreferredReplicaElectionFeature - allows running of preferred replica election for a Kafka cluster + # KMReassignPartitionsFeature - allows generating partition assignments and reassigning partitions + def kafka_manager_features + features = "application.features=[#{configuration.features.join(',')}]" + sudo("sed -i -e 's#application.features=.*##{features}#' #{kafka_manager_conf}") + end + + # Set kafka-manager.zkhosts (note the /kafka chroot path) + # - for multiple ZooKeeper instances, the kafka-manager.zkhosts should be a + # comma-separated string listing the IP addresses and port numbers + # of all the ZooKeeper instances. + def kafka_manager_zookeeper_connect + # Note the use of a '#' in sed delimiter, because connections may contain `/` chars + zk = ZookeeperHelpers.connections(false).join(',') + zoo_connect = "kafka-manager.zkhosts=#{zk}/kafka" + sudo("sed -i -e 's#kafka-manager.zkhosts=.*##{zoo_connect}#' #{kafka_manager_conf}") + end + + desc 'Configure Kafka Manager' + task :configure do + on roles(:kafka_manager), in: :parallel do |host| + kafka_manager_features + kafka_manager_zookeeper_connect + end + end + end +end + diff --git a/lib/kafka_manager/kafka_manager_helpers.rb b/lib/kafka_manager/kafka_manager_helpers.rb new file mode 100644 index 0000000..fdbfddf --- /dev/null +++ b/lib/kafka_manager/kafka_manager_helpers.rb @@ -0,0 +1,24 @@ + +# Utilities for working with Kafka +module KafkaManagerHelpers + + module_function + + SERVICE = 'kafka_manager'.freeze + + # KAFKA_HOME_DEFAULT = '/opt/kafka'.freeze + + def settings + @settings ||= ServiceSettings.new SERVICE + end + + def manager + @manager ||= ServiceManager.new SERVICE + end + + def configuration + settings.configuration + end + +end + diff --git a/lib/kafka_manager/kafka_manager_install.rake b/lib/kafka_manager/kafka_manager_install.rake new file mode 100644 index 0000000..d6da7d0 --- /dev/null +++ b/lib/kafka_manager/kafka_manager_install.rake @@ -0,0 +1,23 @@ +require_relative 'kafka_manager_helpers' + +# Kafka Manager Installation +# https://github.com/yahoo/kafka-manager +# +namespace :kafka_manager do + namespace :service do + def install_kafka_manager + install_java8 + sudo(ubuntu_helper.sbt) + sudo(ubuntu_helper.kafka_manager) + end + + desc 'Install Kafka Manager service' + task :install do + on roles(:kafka_manager), in: :parallel do |host| + install_kafka_manager + KafkaManagerHelpers.manager.reboot_node(host_settings) + end + end + end +end + diff --git a/lib/kafka_manager/kafka_manager_nodes.rake b/lib/kafka_manager/kafka_manager_nodes.rake new file mode 100644 index 0000000..bdc7fae --- /dev/null +++ b/lib/kafka_manager/kafka_manager_nodes.rake @@ -0,0 +1,43 @@ +require_relative 'kafka_manager_helpers' + +namespace :kafka_manager do + namespace :nodes do + desc 'List settings in this project' + task :check_settings do + KafkaManagerHelpers.settings.nodes.each do |params| + puts JSON.pretty_generate(JSON.parse(params.to_json)) + end + end + + desc 'Create nodes' + task :create do + KafkaManagerHelpers.manager.create_nodes + end + + desc 'Find and describe all nodes' + task :find do + KafkaManagerHelpers.manager.describe_nodes + end + + desc 'Reboot Kafka systems - WARNING, can reset IPs' + task :reboot do + KafkaManagerHelpers.manager.reboot_nodes + end + + desc 'Terminate nodes' + task :terminate do + KafkaManagerHelpers.manager.terminate_nodes + end + + desc 'Compose public entries for ~/.ssh/config for nodes' + task :ssh_config_public do + puts KafkaManagerHelpers.manager.ssh_config + end + + desc 'Compose entries for /etc/hosts using public IPs' + task :etc_hosts_public do + puts KafkaManagerHelpers.manager.etc_hosts.join("\n") + end + end +end + diff --git a/lib/kafka_manager/kafka_manager_service.rake b/lib/kafka_manager/kafka_manager_service.rake new file mode 100644 index 0000000..946a055 --- /dev/null +++ b/lib/kafka_manager/kafka_manager_service.rake @@ -0,0 +1,42 @@ +require_relative 'kafka_manager_helpers' + +namespace :kafka_manager do + namespace :service do + def kafka_manager_running? + pid = capture('ls /usr/share/kafka-manager/RUNNING_PID') + ! pid.nil? + end + + desc 'Start Kafka Manager' + task :start do + on roles(:kafka_manager) do |host| + # TODO: Create 'kafka' user/group to run the service + if kafka_manager_running? + puts "#{host.hostname} is already running Kafka Manager" + else + sudo('kafka-manager') + end + end + end + + desc 'Status of Kafka Manager' + task :status do + on roles(:kafka_manager) do |host| + if kafka_running? + puts "#{host.hostname} is running Kafka Manager" + else + puts "#{host.hostname} is not running Kafka Manager" + end + end + end + + desc 'Stop Kafka Manager' + task :stop do + on roles(:kafka_manager) do + # Ignore the exit(1) status when it's not running already + sudo('${KAFKA_BIN}/kafka-server-stop.sh || true') + end + end + end +end + diff --git a/lib/ubuntu/ubuntu_helper.rb b/lib/ubuntu/ubuntu_helper.rb index 6968329..b60bbf2 100644 --- a/lib/ubuntu/ubuntu_helper.rb +++ b/lib/ubuntu/ubuntu_helper.rb @@ -79,6 +79,10 @@ def kafka_bin(kafka_ver = '') "#{script_path}/kafka_bin.sh #{kafka_ver} > #{log_path}/kafka_bin.log" end + def kafka_manager + "#{script_path}/kafka_manager.sh #{kafka_ver} > #{log_path}/kafka_manager.log" + end + def log_path_files "find #{log_path} -type f" end From 047f1b73adc131ac1ef1d78a4ca8218cc8755d46 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Fri, 22 Sep 2017 13:35:38 -0700 Subject: [PATCH 02/14] puts data from ServiceManager#describe_nodes --- lib/kafka/kafka_nodes.rake | 2 +- lib/kafka_manager/kafka_manager_nodes.rake | 2 +- lib/zookeeper/zookeeper_nodes.rake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kafka/kafka_nodes.rake b/lib/kafka/kafka_nodes.rake index 164027a..9bf9af2 100644 --- a/lib/kafka/kafka_nodes.rake +++ b/lib/kafka/kafka_nodes.rake @@ -16,7 +16,7 @@ namespace :kafka do desc 'Find and describe all nodes' task :find do - KafkaHelpers.manager.describe_nodes + puts KafkaHelpers.manager.describe_nodes.join("\n") end desc 'Reboot Kafka systems - WARNING, can reset IPs' diff --git a/lib/kafka_manager/kafka_manager_nodes.rake b/lib/kafka_manager/kafka_manager_nodes.rake index bdc7fae..1a60ef6 100644 --- a/lib/kafka_manager/kafka_manager_nodes.rake +++ b/lib/kafka_manager/kafka_manager_nodes.rake @@ -16,7 +16,7 @@ namespace :kafka_manager do desc 'Find and describe all nodes' task :find do - KafkaManagerHelpers.manager.describe_nodes + puts KafkaManagerHelpers.manager.describe_nodes.join("\n") end desc 'Reboot Kafka systems - WARNING, can reset IPs' diff --git a/lib/zookeeper/zookeeper_nodes.rake b/lib/zookeeper/zookeeper_nodes.rake index 07b42f5..73e584b 100644 --- a/lib/zookeeper/zookeeper_nodes.rake +++ b/lib/zookeeper/zookeeper_nodes.rake @@ -21,7 +21,7 @@ namespace :zookeeper do desc 'Find and describe all nodes' task :find do - ZookeeperHelpers.manager.describe_nodes + puts ZookeeperHelpers.manager.describe_nodes.join("\n") end desc 'Compose public entries for ~/.ssh/config for nodes' From c3e575144bf4e5686d2fd0d50449601bd7dcd593 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Mon, 25 Sep 2017 08:27:07 -0700 Subject: [PATCH 03/14] WIP for Kafka Manager --- lib/kafka_manager/kafka_manager_service.rake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/kafka_manager/kafka_manager_service.rake b/lib/kafka_manager/kafka_manager_service.rake index 946a055..e9c6537 100644 --- a/lib/kafka_manager/kafka_manager_service.rake +++ b/lib/kafka_manager/kafka_manager_service.rake @@ -14,6 +14,7 @@ namespace :kafka_manager do if kafka_manager_running? puts "#{host.hostname} is already running Kafka Manager" else + # TODO: does it need to be run by sudo? sudo('kafka-manager') end end @@ -34,7 +35,8 @@ namespace :kafka_manager do task :stop do on roles(:kafka_manager) do # Ignore the exit(1) status when it's not running already - sudo('${KAFKA_BIN}/kafka-server-stop.sh || true') + # sudo('${KAFKA_BIN}/kafka-server-stop.sh || true') + puts 'TODO' end end end From a79db4b518f858fc1b302cc61a05085fee36b076 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Mon, 25 Sep 2017 09:45:35 -0700 Subject: [PATCH 04/14] UbuntuHelper#kafka_manager requires no kafka_ver --- lib/ubuntu/ubuntu_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ubuntu/ubuntu_helper.rb b/lib/ubuntu/ubuntu_helper.rb index b60bbf2..1d16859 100644 --- a/lib/ubuntu/ubuntu_helper.rb +++ b/lib/ubuntu/ubuntu_helper.rb @@ -80,7 +80,7 @@ def kafka_bin(kafka_ver = '') end def kafka_manager - "#{script_path}/kafka_manager.sh #{kafka_ver} > #{log_path}/kafka_manager.log" + "#{script_path}/kafka_manager.sh > #{log_path}/kafka_manager.log" end def log_path_files From 8e4fa1f197954186b7c5af78fa39065d1b191dc1 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Mon, 25 Sep 2017 09:46:00 -0700 Subject: [PATCH 05/14] README - refer to service docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c4e6486..e4bc9ff 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Utils to provision services on AWS Incomplete - work in progress. +See also `lib/{service}/README.md` documentation. + # Goals - use ruby AWS API client to provision AWS resources - use capistrano for service installation/configuration From 8a1515575ed5df52004d06978d9a25df5955bacf Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Mon, 25 Sep 2017 10:08:54 -0700 Subject: [PATCH 06/14] Kafka Manager install using binary package --- lib/bash/debian/kafka_manager.sh | 13 +++++++++++++ lib/kafka_manager/kafka_manager_install.rake | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index 6c12a19..993f0bb 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -5,6 +5,14 @@ if [ -d /usr/share/kafka-manager ]; then exit fi + +curl -s https://packagecloud.io/install/repositories/spuder/kafka-manager/script.deb.sh | sudo bash + +sudo apt install kafka-manager + +exit + + # --- # Download cd /tmp @@ -23,3 +31,8 @@ sbt debian:packageBin sudo dpkg -i -R target/ # dpkg -L kafka-manager + +# --- +# Cleanup + +sbt clean clean-files diff --git a/lib/kafka_manager/kafka_manager_install.rake b/lib/kafka_manager/kafka_manager_install.rake index d6da7d0..7083512 100644 --- a/lib/kafka_manager/kafka_manager_install.rake +++ b/lib/kafka_manager/kafka_manager_install.rake @@ -7,7 +7,7 @@ namespace :kafka_manager do namespace :service do def install_kafka_manager install_java8 - sudo(ubuntu_helper.sbt) + # sudo(ubuntu_helper.sbt) sudo(ubuntu_helper.kafka_manager) end From 675d8524a9772a5ad179bba866e74c4a214b16ac Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Mon, 25 Sep 2017 12:02:05 -0700 Subject: [PATCH 07/14] WIP on kafka-manager --- lib/bash/debian/kafka_manager.sh | 1 - lib/kafka_manager/kafka_manager_install.rake | 3 ++- lib/kafka_manager/kafka_manager_service.rake | 4 ++-- lib/ubuntu/ubuntu_helper.rb | 4 ++++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index 993f0bb..c14e499 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -5,7 +5,6 @@ if [ -d /usr/share/kafka-manager ]; then exit fi - curl -s https://packagecloud.io/install/repositories/spuder/kafka-manager/script.deb.sh | sudo bash sudo apt install kafka-manager diff --git a/lib/kafka_manager/kafka_manager_install.rake b/lib/kafka_manager/kafka_manager_install.rake index 7083512..1449dea 100644 --- a/lib/kafka_manager/kafka_manager_install.rake +++ b/lib/kafka_manager/kafka_manager_install.rake @@ -7,7 +7,7 @@ namespace :kafka_manager do namespace :service do def install_kafka_manager install_java8 - # sudo(ubuntu_helper.sbt) + sudo(ubuntu_helper.upstart) sudo(ubuntu_helper.kafka_manager) end @@ -15,6 +15,7 @@ namespace :kafka_manager do task :install do on roles(:kafka_manager), in: :parallel do |host| install_kafka_manager + # reboot to enable the upstart service KafkaManagerHelpers.manager.reboot_node(host_settings) end end diff --git a/lib/kafka_manager/kafka_manager_service.rake b/lib/kafka_manager/kafka_manager_service.rake index e9c6537..e1f8c05 100644 --- a/lib/kafka_manager/kafka_manager_service.rake +++ b/lib/kafka_manager/kafka_manager_service.rake @@ -14,7 +14,7 @@ namespace :kafka_manager do if kafka_manager_running? puts "#{host.hostname} is already running Kafka Manager" else - # TODO: does it need to be run by sudo? + # It requires sudo to create PID and log files sudo('kafka-manager') end end @@ -23,7 +23,7 @@ namespace :kafka_manager do desc 'Status of Kafka Manager' task :status do on roles(:kafka_manager) do |host| - if kafka_running? + if kafka_manager_running? puts "#{host.hostname} is running Kafka Manager" else puts "#{host.hostname} is not running Kafka Manager" diff --git a/lib/ubuntu/ubuntu_helper.rb b/lib/ubuntu/ubuntu_helper.rb index 1d16859..be966ee 100644 --- a/lib/ubuntu/ubuntu_helper.rb +++ b/lib/ubuntu/ubuntu_helper.rb @@ -99,6 +99,10 @@ def sbt "#{script_path}/sbt.sh > #{log_path}/sbt.log" end + def upstart + "apt-get install upstart > #{log_path}/upstart.log" + end + def zookeeper "#{script_path}/zookeeper.sh > #{log_path}/zookeeper.log" end From 390050c0ac61e45b6d46785da5d14d19d48c3ba9 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Tue, 26 Sep 2017 07:59:55 -0700 Subject: [PATCH 08/14] KafkaManager - installation can enable systemd --- lib/bash/debian/kafka_manager.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index c14e499..bbc12f6 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -7,7 +7,29 @@ fi curl -s https://packagecloud.io/install/repositories/spuder/kafka-manager/script.deb.sh | sudo bash -sudo apt install kafka-manager +sudo apt-get install kafka-manager + + +#--- +# Enable systemd +# https://github.com/yahoo/kafka-manager/issues/373 + +cat > /tmp/kafka-manager.service < Date: Tue, 26 Sep 2017 08:03:13 -0700 Subject: [PATCH 09/14] KafkaManager - remove upstart dependency --- lib/kafka_manager/kafka_manager_install.rake | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/kafka_manager/kafka_manager_install.rake b/lib/kafka_manager/kafka_manager_install.rake index 1449dea..12f683a 100644 --- a/lib/kafka_manager/kafka_manager_install.rake +++ b/lib/kafka_manager/kafka_manager_install.rake @@ -7,7 +7,6 @@ namespace :kafka_manager do namespace :service do def install_kafka_manager install_java8 - sudo(ubuntu_helper.upstart) sudo(ubuntu_helper.kafka_manager) end @@ -15,8 +14,6 @@ namespace :kafka_manager do task :install do on roles(:kafka_manager), in: :parallel do |host| install_kafka_manager - # reboot to enable the upstart service - KafkaManagerHelpers.manager.reboot_node(host_settings) end end end From abc9a2bb69f5de7bb46488261d6f027eb4b7f2dd Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Tue, 26 Sep 2017 08:32:32 -0700 Subject: [PATCH 10/14] KafkaManager - installation revised --- lib/bash/debian/kafka_manager.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index bbc12f6..9eee54c 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if [ -d /usr/share/kafka-manager ]; then +if which kafka-manager; then echo "Kafka manager is installed" exit fi @@ -31,6 +31,8 @@ EOF sudo mv /tmp/kafka-manager.service /lib/systemd/system/ sudo systemctl enable kafka-manager.service +# TODO: Create 'kafka' user/group to run the service + exit From 8b580c3c0f1ac5187ddbedcbed1b348bf8cc075c Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Tue, 26 Sep 2017 08:33:18 -0700 Subject: [PATCH 11/14] KafkaManager - service can use systemd --- lib/kafka_manager/kafka_manager_service.rake | 27 ++++---------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/lib/kafka_manager/kafka_manager_service.rake b/lib/kafka_manager/kafka_manager_service.rake index e1f8c05..2f7261b 100644 --- a/lib/kafka_manager/kafka_manager_service.rake +++ b/lib/kafka_manager/kafka_manager_service.rake @@ -2,41 +2,24 @@ require_relative 'kafka_manager_helpers' namespace :kafka_manager do namespace :service do - def kafka_manager_running? - pid = capture('ls /usr/share/kafka-manager/RUNNING_PID') - ! pid.nil? - end - desc 'Start Kafka Manager' task :start do - on roles(:kafka_manager) do |host| - # TODO: Create 'kafka' user/group to run the service - if kafka_manager_running? - puts "#{host.hostname} is already running Kafka Manager" - else - # It requires sudo to create PID and log files - sudo('kafka-manager') - end + on roles(:kafka_manager) do + sudo('service kafka-manager start') end end desc 'Status of Kafka Manager' task :status do - on roles(:kafka_manager) do |host| - if kafka_manager_running? - puts "#{host.hostname} is running Kafka Manager" - else - puts "#{host.hostname} is not running Kafka Manager" - end + on roles(:kafka_manager) do + execute('service kafka-manager status || true') end end desc 'Stop Kafka Manager' task :stop do on roles(:kafka_manager) do - # Ignore the exit(1) status when it's not running already - # sudo('${KAFKA_BIN}/kafka-server-stop.sh || true') - puts 'TODO' + sudo('service kafka-manager stop') end end end From 8de7b057406bea938418fc28359d6c3e3225e9a6 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Tue, 26 Sep 2017 08:36:37 -0700 Subject: [PATCH 12/14] KafkaManager install revised --- lib/bash/debian/kafka_manager.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index 9eee54c..bed817e 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -7,7 +7,7 @@ fi curl -s https://packagecloud.io/install/repositories/spuder/kafka-manager/script.deb.sh | sudo bash -sudo apt-get install kafka-manager +sudo apt-get install -y -q kafka-manager #--- From 18552201bed819dba3919e460d1291c71b975fc5 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Tue, 26 Sep 2017 09:47:58 -0700 Subject: [PATCH 13/14] KafkaManager - installation revised --- lib/bash/debian/kafka_manager.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index bed817e..f5d1c35 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -23,12 +23,14 @@ After=network.target WantedBy=multi-user.target [Service] -ExecStart=/usr/bin/kafka-manager +ExecStart=/usr/bin/kafka-manager -Dpidfile.path=/var/run/kafka-manager.pid -Dconfig.file=/etc/kafka-manager/application.conf -Dlogger.file=/etc/kafka-manager/logger.xml Type=simple Restart=always EOF sudo mv /tmp/kafka-manager.service /lib/systemd/system/ +sudo chown root:root /lib/systemd/system/kafka-manager.service +sudo chmod 644 /lib/systemd/system/kafka-manager.service sudo systemctl enable kafka-manager.service # TODO: Create 'kafka' user/group to run the service From 106846a5bcb3fb4213cb402c6b7f3c0b0afdc042 Mon Sep 17 00:00:00 2001 From: "Darren L. Weber, Ph.D" Date: Tue, 17 Oct 2017 13:43:24 -0700 Subject: [PATCH 14/14] WIP --- .../kafka_manager_configure.rake | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/lib/kafka_manager/kafka_manager_configure.rake b/lib/kafka_manager/kafka_manager_configure.rake index 3ff3cd9..f69ed3a 100644 --- a/lib/kafka_manager/kafka_manager_configure.rake +++ b/lib/kafka_manager/kafka_manager_configure.rake @@ -5,8 +5,30 @@ require_relative 'kafka_manager_helpers' # namespace :kafka_manager do namespace :service do - def kafka_manager_conf - @kafka_manager_conf ||= capture('ls /usr/share/kafka-manager/conf/application.conf') + + + # /etc/default/kafka-manager defines: + # + #JAVA_OPTS="-Dpidfile.path=/var/run/kafka-manager.pid -Dconfig.file=/etc/kafka-manager/application.conf -Dlogger.file=/etc/kafka-manager/logger.xml $JAVA_OPTS" + #PIDFILE="/var/run/kafka-manager.pid" + + ########################################################################## + # BEGIN from https://github.com/yahoo/kafka-manager/issues/373 + + # In /opt/kafka-manager-1.3.3.6/conf/application.ini + # + # -Dapplication.home=/opt/kafka-manager-1.3.3.6 + # -Dpidfile.path=/opt/kafka-manager-1.3.3.6/kafka-manager.pid + # -Dhttp.port=8080 + + # END from https://github.com/yahoo/kafka-manager/issues/373 + ########################################################################## + + + + def kafka_manager_application_conf + # @kafka_manager_application_conf ||= capture('ls /usr/share/kafka-manager/conf/application.conf') + @kafka_manager_application_conf ||= capture('ls /etc/kafka-manager/application.conf') end # basicAuthentication @@ -16,15 +38,16 @@ namespace :kafka_manager do # basicAuthentication.realm="Kafka-Manager" # basicAuthentication.excluded=["/api/health"] # ping the health of your instance without authentification def kafka_manager_authentication - return unless configuration.basicAuthentication.enabled - enabled = "basicAuthentication.enabled=#{configuration.basicAuthentication.enabled}" - sudo("sed -i -e 's#basicAuthentication.enabled=.*##{enabled}#' #{kafka_manager_conf}") + return unless KafkaManagerHelpers.configuration.basicAuthentication.enabled + auth = KafkaManagerHelpers.configuration.basicAuthentication + enabled = 'basicAuthentication.enabled=true' + sudo("sed -i -e 's#basicAuthentication.enabled=.*##{enabled}#' #{kafka_manager_application_conf}") # basicAuthentication.username="admin" - username = "basicAuthentication.username=\"#{configuration.basicAuthentication.username}\"" - sudo("sed -i -e 's#basicAuthentication.username=.*##{username}#' #{kafka_manager_conf}") + username = "basicAuthentication.username=\"#{auth.username}\"" + sudo("sed -i -e 's#basicAuthentication.username=.*##{username}#' #{kafka_manager_application_conf}") # basicAuthentication.password="password" - password = "basicAuthentication.password=\"#{configuration.basicAuthentication.password}\"" - sudo("sed -i -e 's#basicAuthentication.password=.*##{password}#' #{kafka_manager_conf}") + password = "basicAuthentication.password=\"#{auth.password}\"" + sudo("sed -i -e 's#basicAuthentication.password=.*##{password}#' #{kafka_manager_application_conf}") end # application.features=["KMClusterManagerFeature","KMTopicManagerFeature", @@ -35,8 +58,8 @@ namespace :kafka_manager do # KMPreferredReplicaElectionFeature - allows running of preferred replica election for a Kafka cluster # KMReassignPartitionsFeature - allows generating partition assignments and reassigning partitions def kafka_manager_features - features = "application.features=[#{configuration.features.join(',')}]" - sudo("sed -i -e 's#application.features=.*##{features}#' #{kafka_manager_conf}") + features = "application.features=[#{KafkaManagerHelpers.configuration.features.join(',')}]" + sudo("sed -i -e 's#application.features=.*##{features}#' #{kafka_manager_application_conf}") end # Set kafka-manager.zkhosts (note the /kafka chroot path) @@ -44,10 +67,13 @@ namespace :kafka_manager do # comma-separated string listing the IP addresses and port numbers # of all the ZooKeeper instances. def kafka_manager_zookeeper_connect + # kafka-manager.zkhosts="my-zookeeper-connection-string" + # kafka-manager.base-zk-path="/a-chroot" - don't see this in the installed file + # # Note the use of a '#' in sed delimiter, because connections may contain `/` chars zk = ZookeeperHelpers.connections(false).join(',') zoo_connect = "kafka-manager.zkhosts=#{zk}/kafka" - sudo("sed -i -e 's#kafka-manager.zkhosts=.*##{zoo_connect}#' #{kafka_manager_conf}") + sudo("sed -i -e 's#kafka-manager.zkhosts=.*##{zoo_connect}#' #{kafka_manager_application_conf}") end desc 'Configure Kafka Manager'