diff --git a/README.md b/README.md index c4e6486..e4bc9ff 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Utils to provision services on AWS Incomplete - work in progress. +See also `lib/{service}/README.md` documentation. + # Goals - use ruby AWS API client to provision AWS resources - use capistrano for service installation/configuration diff --git a/config/deploy/test.rb b/config/deploy/test.rb index 1630746..f7d9624 100644 --- a/config/deploy/test.rb +++ b/config/deploy/test.rb @@ -42,6 +42,8 @@ server 'test_kafka2', user: 'ubuntu', roles: %w[ubuntu kafka] server 'test_kafka3', user: 'ubuntu', roles: %w[ubuntu kafka] +server 'test_kafka_manager', user: 'ubuntu', roles: %w[ubuntu kafka_manager] + # role-based syntax # ================== diff --git a/config/settings/test.yml b/config/settings/test.yml index 91fe34b..0661f43 100644 --- a/config/settings/test.yml +++ b/config/settings/test.yml @@ -108,6 +108,7 @@ aws: # - advertised.listeners test_kafka_configuration: resource: configuration + tag_service: kafka kafka_home: "/opt/kafka" # symlinks to /opt/kafka_{SCALA_VERSION}-{KAFKA_VERSION} kafka_version: "0.11.0.0" scala_version: "2.11" @@ -148,6 +149,38 @@ aws: tag_name: test_kafka3 availability_zone: us-west-2c # need one node in each zone + # --- + # Kafka Manager + + test_kafka_manager_configuration: + resource: configuration + tag_service: kafka_manager + basicAuthentication: + enabled: false + username: "admin" + password: "password" + realm: "Kafka-Manager" + excluded: + - "/api/health" + # KMClusterManagerFeature - allows adding, updating, deleting cluster from Kafka Manager + # KMTopicManagerFeature - allows adding, updating, deleting topic from a Kafka cluster + # KMPreferredReplicaElectionFeature - allows running of preferred replica election for a Kafka cluster + # KMReassignPartitionsFeature - allows generating partition assignments and reassigning partitions + features: + - KMClusterManagerFeature + - KMTopicManagerFeature + - KMPreferredReplicaElectionFeature + - KMReassignPartitionsFeature + + test_kafka_manager: + <<: *TestDefault + tag_service: kafka_manager + tag_group: test_kafka + tag_name: test_kafka_manager + security_groups: + - test_ssh_security_group + - test_kafka_security_group + # --- # Mesos Nodes # - note definition and use of defaults: MesosMasterDefaults, MesosAgentDefaults @@ -267,9 +300,16 @@ aws: group_id: ip_permissions: - ip_protocol: tcp - from_port: 9092 + from_port: 9092 # kafka clients to_port: 9092 ip_ranges: - cidr_ip: '0.0.0.0/0' ipv_6_ranges: - cidr_ipv_6: '::/0' + - ip_protocol: tcp + from_port: 9000 # kafka manager + to_port: 9000 + ip_ranges: + - cidr_ip: '0.0.0.0/0' + ipv_6_ranges: + - cidr_ipv_6: '::/0' diff --git a/lib/bash/debian/kafka_manager.sh b/lib/bash/debian/kafka_manager.sh index 690966b..f5d1c35 100755 --- a/lib/bash/debian/kafka_manager.sh +++ b/lib/bash/debian/kafka_manager.sh @@ -1,6 +1,46 @@ +#!/usr/bin/env bash + +if which kafka-manager; then + echo "Kafka manager is installed" + exit +fi + +curl -s https://packagecloud.io/install/repositories/spuder/kafka-manager/script.deb.sh | sudo bash + +sudo apt-get install -y -q kafka-manager + + +#--- +# Enable systemd +# https://github.com/yahoo/kafka-manager/issues/373 + +cat > /tmp/kafka-manager.service < #{log_path}/kafka_bin.log" end + def kafka_manager + "#{script_path}/kafka_manager.sh > #{log_path}/kafka_manager.log" + end + def log_path_files "find #{log_path} -type f" end @@ -95,6 +99,10 @@ def sbt "#{script_path}/sbt.sh > #{log_path}/sbt.log" end + def upstart + "apt-get install upstart > #{log_path}/upstart.log" + end + def zookeeper "#{script_path}/zookeeper.sh > #{log_path}/zookeeper.log" end diff --git a/lib/zookeeper/zookeeper_nodes.rake b/lib/zookeeper/zookeeper_nodes.rake index 07b42f5..73e584b 100644 --- a/lib/zookeeper/zookeeper_nodes.rake +++ b/lib/zookeeper/zookeeper_nodes.rake @@ -21,7 +21,7 @@ namespace :zookeeper do desc 'Find and describe all nodes' task :find do - ZookeeperHelpers.manager.describe_nodes + puts ZookeeperHelpers.manager.describe_nodes.join("\n") end desc 'Compose public entries for ~/.ssh/config for nodes'