Added support for Accumulo.

laurentedel · Oct 28, 2018 · 3790deb · 3790deb
1 parent 6baf5dd
commit 3790deb
Show file tree

Hide file tree

Showing 11 changed files with 175 additions and 1 deletion.
diff --git a/INSTALL_AWS.md b/INSTALL_AWS.md
@@ -297,6 +297,11 @@ Modify the file at `~/ansible-hortonworks/playbooks/group_vars/all` to set the c
 | `.admin_password`              | The password for the Superset admin user.                                               |
 | smartsense_security_options`.admin_password` | The password for the Activity Explorer's Zeppelin admin user.             |
 | logsearch_security_options`.admin_password`  | The password for the Log Search admin user.                               |
+| accumulo_security_options      | Security related options for Accumulo (such as passwords).                              |
+| `.root_password`               | Password for the Accumulo root user. This password will be used to initialize Accumulo and to create the trace user.       |
+| `.instance_secret`             | A secret unique to a given instance that all Accumulo server processes must know in order to communicate with one another. |
+| `.trace_user`                  | User that the tracer process uses to write tracing data to Accumulo.                    |
+| `.trace_password`              | Password for the trace user.                                                            |
 
 ### ambari configuration
 

diff --git a/INSTALL_Azure.md b/INSTALL_Azure.md
@@ -301,6 +301,11 @@ Modify the file at `~/ansible-hortonworks/playbooks/group_vars/all` to set the c
 | `.admin_password`              | The password for the Superset admin user.                                               |
 | smartsense_security_options`.admin_password` | The password for the Activity Explorer's Zeppelin admin user.             |
 | logsearch_security_options`.admin_password`  | The password for the Log Search admin user.                               |
+| accumulo_security_options      | Security related options for Accumulo (such as passwords).                              |
+| `.root_password`               | Password for the Accumulo root user. This password will be used to initialize Accumulo and to create the trace user.       |
+| `.instance_secret`             | A secret unique to a given instance that all Accumulo server processes must know in order to communicate with one another. |
+| `.trace_user`                  | User that the tracer process uses to write tracing data to Accumulo.                    |
+| `.trace_password`              | Password for the trace user.                                                            |
 
 ### ambari configuration
 

diff --git a/INSTALL_GCE.md b/INSTALL_GCE.md
@@ -362,6 +362,11 @@ Modify the file at `~/ansible-hortonworks/playbooks/group_vars/all` to set the c
 | `.admin_password`              | The password for the Superset admin user.                                               |
 | smartsense_security_options`.admin_password` | The password for the Activity Explorer's Zeppelin admin user.             |
 | logsearch_security_options`.admin_password`  | The password for the Log Search admin user.                               |
+| accumulo_security_options      | Security related options for Accumulo (such as passwords).                              |
+| `.root_password`               | Password for the Accumulo root user. This password will be used to initialize Accumulo and to create the trace user.       |
+| `.instance_secret`             | A secret unique to a given instance that all Accumulo server processes must know in order to communicate with one another. |
+| `.trace_user`                  | User that the tracer process uses to write tracing data to Accumulo.                    |
+| `.trace_password`              | Password for the trace user.                                                            |
 
 ### ambari configuration
 

diff --git a/INSTALL_OpenStack.md b/INSTALL_OpenStack.md
@@ -327,6 +327,11 @@ Modify the file at `~/ansible-hortonworks/playbooks/group_vars/all` to set the c
 | `.admin_password`              | The password for the Superset admin user.                                               |
 | smartsense_security_options`.admin_password` | The password for the Activity Explorer's Zeppelin admin user.             |
 | logsearch_security_options`.admin_password`  | The password for the Log Search admin user.                               |
+| accumulo_security_options      | Security related options for Accumulo (such as passwords).                              |
+| `.root_password`               | Password for the Accumulo root user. This password will be used to initialize Accumulo and to create the trace user.       |
+| `.instance_secret`             | A secret unique to a given instance that all Accumulo server processes must know in order to communicate with one another. |
+| `.trace_user`                  | User that the tracer process uses to write tracing data to Accumulo.                    |
+| `.trace_password`              | Password for the trace user.                                                            |
 
 ### ambari configuration
 

diff --git a/INSTALL_static.md b/INSTALL_static.md
@@ -293,6 +293,11 @@ Modify the file at `~/ansible-hortonworks/playbooks/group_vars/all` to set the c
 | `.admin_password`              | The password for the Superset admin user.                                               |
 | smartsense_security_options`.admin_password` | The password for the Activity Explorer's Zeppelin admin user.             |
 | logsearch_security_options`.admin_password`  | The password for the Log Search admin user.                               |
+| accumulo_security_options      | Security related options for Accumulo (such as passwords).                              |
+| `.root_password`               | Password for the Accumulo root user. This password will be used to initialize Accumulo and to create the trace user.       |
+| `.instance_secret`             | A secret unique to a given instance that all Accumulo server processes must know in order to communicate with one another. |
+| `.trace_user`                  | User that the tracer process uses to write tracing data to Accumulo.                    |
+| `.trace_password`              | Password for the trace user.                                                            |
 
 ### ambari configuration
 

diff --git a/README.md b/README.md
@@ -173,7 +173,7 @@ Currently, these playbooks are divided into the following parts:
 ### Dynamic blueprint supported features
 > The components that will be installed are only those defined in the `blueprint_dynamic` [variable](playbooks/group_vars/all#L161).
 > - Supported in this case means all prerequites (databases, passwords, required configs) are taken care of and the component is deployed successfully on the chosen `host_group`.
-- [x] HDP Services: `HDFS`, `YARN + MapReduce2`, `Hive`, `HBase`, `Oozie`, `ZooKeeper`, `Storm`, `Atlas`, `Kafka`, `Knox`, `Log Search`, `Ranger`, `Ranger KMS`, `SmartSense`, `Spark2`, `Zeppelin`, `Druid`, `Superset`
+- [x] HDP Services: `HDFS`, `YARN + MapReduce2`, `Hive`, `HBase`, `Accumulo`, `Oozie`, `ZooKeeper`, `Storm`, `Atlas`, `Kafka`, `Knox`, `Log Search`, `Ranger`, `Ranger KMS`, `SmartSense`, `Spark2`, `Zeppelin`, `Druid`, `Superset`
 - [x] HDF Services: `NiFi`, `NiFi Registry`, `Schema Registry`, `Streaming Analytics Manager`, `ZooKeeper`, `Storm`, `Kafka`, `Knox`, `Ranger`, `Log Search`
 - [x] HA Configuration: NameNode, ResourceManager, Hive, HBase, Ranger KMS, Druid
 - [x] Secure clusters with MIT KDC (Ambari managed)

diff --git a/playbooks/check_dynamic_blueprint.yml b/playbooks/check_dynamic_blueprint.yml
@@ -108,3 +108,10 @@
         msg: "One or more of the following components are missing from the blueprint: DRUID_BROKER, DRUID_COORDINATOR, DRUID_ROUTER, DRUID_MIDDLEMANAGER, DRUID_HISTORICAL, DRUID_OVERLORD."
       when: druid_hosts|length > 0 and
            ['DRUID_BROKER', 'DRUID_COORDINATOR', 'DRUID_ROUTER', 'DRUID_MIDDLEMANAGER', 'DRUID_HISTORICAL', 'DRUID_OVERLORD'] | difference(blueprint_all_services) | length > 0
+
+    - name: Fail if the required Accumulo components are not part of the blueprint
+      fail:
+        msg: "One or more of the following components are missing from the blueprint: ACCUMULO_CLIENT, ACCUMULO_MASTER, ACCUMULO_MONITOR, ACCUMULO_TRACER, ACCUMULO_GC, ACCUMULO_TSERVER."
+      when: (['ACCUMULO_CLIENT', 'ACCUMULO_MASTER', 'ACCUMULO_MONITOR', 'ACCUMULO_TRACER', 'ACCUMULO_GC', 'ACCUMULO_TSERVER'] | difference(blueprint_all_clients | union(blueprint_all_services)) | length > 0)
+            and
+            (['ACCUMULO_CLIENT', 'ACCUMULO_MASTER', 'ACCUMULO_MONITOR', 'ACCUMULO_TRACER', 'ACCUMULO_GC', 'ACCUMULO_TSERVER'] | difference(blueprint_all_clients | union(blueprint_all_services)) | length < 6)
diff --git a/playbooks/group_vars/all b/playbooks/group_vars/all
@@ -132,6 +132,11 @@ smartsense_security_options:
 logsearch_security_options:
   admin_password: "{{ default_password }}"                # the password for the Logsearch admin user
 
+accumulo_security_options:
+  root_password: "{{ default_password }}"                 # the password for the Accumulo root user
+  instance_secret: "{{ default_password }}"               # a secret unique to a given instance
+  trace_user: "trace"                                     # user that the tracer process uses to write tracing data to Accumulo
+  trace_password: "{{ default_password }}"                # password for the trace user
 
 ##########################
 ## ambari configuration ##

diff --git a/playbooks/group_vars/example-hdp-singlenode-with-accumulo b/playbooks/group_vars/example-hdp-singlenode-with-accumulo
@@ -0,0 +1,42 @@
+###########
+#
+# These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON.
+# This blueprint will build a single node HDP instance with Ambari and some of the more common services plus Accumulo.
+#
+###########
+
+blueprint_name: '{{ cluster_name }}_blueprint'                  # the name of the blueprint as it will be stored in Ambari
+blueprint_file: 'blueprint_dynamic.j2'                          # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON
+blueprint_dynamic:                                              # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON
+  - host_group: "hdp-singlenode"
+    clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT']
+    services:
+      - ZOOKEEPER_SERVER
+      - NAMENODE
+      - SECONDARY_NAMENODE
+      - DATANODE
+      - RESOURCEMANAGER
+      - NODEMANAGER
+      - APP_TIMELINE_SERVER
+      - HISTORYSERVER
+      - HIVE_SERVER
+      - HIVE_METASTORE
+      - WEBHCAT_SERVER
+      - OOZIE_SERVER
+      - ACCUMULO_MASTER
+      - ACCUMULO_MONITOR
+      - ACCUMULO_TRACER
+      - ACCUMULO_GC
+      - ACCUMULO_TSERVER
+      - SPARK2_JOBHISTORYSERVER
+      - ZEPPELIN_MASTER
+      - KNOX_GATEWAY
+      - HST_SERVER
+      - ACTIVITY_ANALYZER
+      - ACTIVITY_EXPLORER
+      - HST_AGENT
+      - AMBARI_SERVER
+      - INFRA_SOLR
+      - METRICS_COLLECTOR
+      - METRICS_GRAFANA
+      - METRICS_MONITOR
diff --git a/playbooks/group_vars/example-hdp3-ha-3-masters-with-accumulo b/playbooks/group_vars/example-hdp3-ha-3-masters-with-accumulo
@@ -0,0 +1,78 @@
+###########
+#
+# These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON.
+# This blueprint will build an HDP cluster with Accumulo, made of 3 master services nodes and 1 or more worker nodes.
+# One of the master services nodes is running Ambari and other utility services - here called the management node.
+# The other two master services are running the main HA Hadoop services, including the Accumulo masters.
+#
+###########
+
+blueprint_name: '{{ cluster_name }}_blueprint'                  # the name of the blueprint as it will be stored in Ambari
+blueprint_file: 'blueprint_dynamic.j2'                          # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON
+blueprint_dynamic:                                              # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON
+  - host_group: "hdp-masternode-01"
+    clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT']
+    services:
+      - ZOOKEEPER_SERVER
+      - NAMENODE
+      - ZKFC
+      - JOURNALNODE
+      - RESOURCEMANAGER
+      - HIVE_SERVER
+      - HIVE_METASTORE
+      - ACCUMULO_MASTER
+      - ACCUMULO_MONITOR
+      - ACCUMULO_TRACER
+      - ACCUMULO_GC
+      - KAFKA_BROKER
+      - ACTIVITY_ANALYZER
+      - HST_AGENT
+      - METRICS_MONITOR
+  - host_group: "hdp-masternode-02"
+    clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT']
+    services:
+      - ZOOKEEPER_SERVER
+      - NAMENODE
+      - ZKFC
+      - JOURNALNODE
+      - RESOURCEMANAGER
+      - APP_TIMELINE_SERVER
+      - YARN_REGISTRY_DNS
+      - TIMELINE_READER
+      - HIVE_SERVER
+      - HIVE_METASTORE
+      - ACCUMULO_MASTER
+      - ACCUMULO_MONITOR
+      - ACCUMULO_TRACER
+      - ACCUMULO_GC
+      - KAFKA_BROKER
+      - HISTORYSERVER
+      - SPARK2_JOBHISTORYSERVER
+      - ACTIVITY_ANALYZER
+      - HST_AGENT
+      - METRICS_MONITOR
+  - host_group: "hdp-management"
+    clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT']
+    services:
+      - ZOOKEEPER_SERVER
+      - JOURNALNODE
+      - OOZIE_SERVER
+      - KNOX_GATEWAY
+      - ZEPPELIN_MASTER
+      - AMBARI_SERVER
+      - INFRA_SOLR
+      - HST_SERVER
+      - ACTIVITY_ANALYZER
+      - ACTIVITY_EXPLORER
+      - HST_AGENT
+      - METRICS_COLLECTOR
+      - METRICS_GRAFANA
+      - METRICS_MONITOR
+  - host_group: "hdp-worker"
+    clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT']
+    services:
+      - DATANODE
+      - NODEMANAGER
+      - ACCUMULO_TSERVER
+      - HST_AGENT
+      - METRICS_MONITOR
diff --git a/playbooks/roles/ambari-blueprint/templates/blueprint_dynamic.j2 b/playbooks/roles/ambari-blueprint/templates/blueprint_dynamic.j2
@@ -696,6 +696,23 @@
         "oozie_user" : "oozie"
       }
     },
+{% if 'ACCUMULO_MASTER' in blueprint_all_services %}
+    {
+      "accumulo-site" : {
+        "instance.volumes": "hdfs://{% if namenode_groups|length > 1 %}{{ hdfs_ha_name }}{% else %}%HOSTGROUP::{{ namenode_groups[0] }}%:8020{% endif %}/apps/accumulo/data",
+        "trace.user": "{{ accumulo_security_options.trace_user | default("trace") }}",
+        "instance.zookeeper.timeout": "30s"
+      }
+    },
+    {
+      "accumulo-env" : {
+        "accumulo_root_password": "{{ accumulo_security_options.root_password }}",
+        "instance_secret": "{{ accumulo_security_options.instance_secret }}",
+        "trace_password": "{{ accumulo_security_options.trace_password }}",
+        "accumulo_instance_name": "hdp-accumulo-instance"
+      }
+    },
+{% endif %}
     {
       "hbase-site" : {
         {% if namenode_groups|length > 1 -%}