Merge branch 'master' into master

laurentedel · Jan 23, 2019 · 98228aa · 98228aa
2 parents e57e0d3 + 1fbad84
commit 98228aa
Show file tree

Hide file tree

Showing 9 changed files with 70 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@ ansible-hortonworks
 
 These Ansible playbooks will build a Hortonworks cluster (Hortonworks Data Platform and / or Hortonworks DataFlow) using Ambari Blueprints. For a full list of supported features check [below](#features).
 
-- Tested with: HDP 3.0, HDP 2.4 -> 2.6.5, HDP Search 3.0.0.0, HDF 2.0 -> 3.2, Ambari 2.4 -> 2.7.
+- Tested with: HDP 3.0 -> 3.1, HDP 2.4 -> 2.6.5, HDP Search 3.0 -> 4.0, HDF 2.0 -> 3.3, Ambari 2.4 -> 2.7 (the versions must be matched as per the [support matrix](https://supportmatrix.hortonworks.com)).
 
 - This includes building the Cloud infrastructure (optional) and taking care of the prerequisites.
 
@@ -55,7 +55,7 @@ Therefore, these Ansible playbooks try to take advantage of Blueprint's `host_gr
 </p>
 
 - If the blueprint is dynamic, these `host_groups` are defined in the [variable file](playbooks/group_vars/all#L162) and they need to match the Ansible inventory groups that will run those components.
-- If the blueprint is static, these `host_groups` are defined in the [blueprint itself](playbooks/roles/ambari-blueprint/files/blueprint_hdfs_only.j2#L29) and they need to match the Ansible inventory groups that will run those components.
+- If the blueprint is static, these `host_groups` are defined in the [blueprint itself](playbooks/roles/ambari-blueprint/files/blueprint_hdfs_only.json#L27) and they need to match the Ansible inventory groups that will run those components.
 
 
 ### Cloud inventory

diff --git a/playbooks/group_vars/all b/playbooks/group_vars/all
@@ -5,16 +5,16 @@
 
 cluster_name: 'mytestcluster'
 
-ambari_version: '2.7.1.0'                                 # must be the 4-part full version number
+ambari_version: '2.7.3.0'                                 # must be the 4-part full version number
 
-hdp_version: '3.0.1.0'                                    # must be the 4-part full version number
+hdp_version: '3.1.0.0'                                    # must be the 4-part full version number
 hdp_build_number: 'auto'                                  # the HDP build number from docs.hortonworks.com (if set to 'auto', Ansible will try to get it from the repository)
 
-hdf_version: '3.2.0.0'                                    # must be the 4-part full version number
+hdf_version: '3.3.0.0'                                    # must be the 4-part full version number
 hdf_build_number: 'auto'                                  # the HDF build number from docs.hortonworks.com (if set to 'auto', Ansible will try to get it from the repository)
 
-hdpsearch_version: '3.0.0'                                # must be the full version number
-hdpsearch_build_number: '100'                             # the HDP Search build number from docs.hortonworks.com (hardcoded to 100 for the moment)
+hdpsearch_version: '4.0.0'                                # must be the full version number
+hdpsearch_build_number: '400'                             # the HDP Search build number from docs.hortonworks.com
 
 repo_base_url: 'http://public-repo-1.hortonworks.com'     # change this if using a Local Repository
 
@@ -25,6 +25,7 @@ repo_base_url: 'http://public-repo-1.hortonworks.com'     # change this if using
 
 external_dns: yes                                         # set to yes to use the existing DNS (when no, it will update the /etc/hosts file - must be set to 'no' when using Azure)
 disable_firewall: no                                      # set to yes to disable the existing local firewall service (iptables, firewalld, ufw)
+timezone: UTC
 
 
 ########################
@@ -138,6 +139,7 @@ accumulo_security_options:
   trace_user: "trace"                                     # user that the tracer process uses to write tracing data to Accumulo
   trace_password: "{{ default_password }}"                # password for the trace user
 
+
 ##########################
 ## ambari configuration ##
 ##########################
@@ -216,5 +218,4 @@ hdfs_ha_name: "{{ cluster_name | regex_replace('_','-') }}"
 ### advanced configuration ##
 #############################
 
-is_vm_docker_containers: 'no' # docker containers have some specifics like mount points for example
-timezone: UTC
+is_vm_docker_containers: 'no'                             # docker containers have some specifics like mount points for example
diff --git a/playbooks/roles/ambari-blueprint/templates/blueprint_dynamic.j2 b/playbooks/roles/ambari-blueprint/templates/blueprint_dynamic.j2
@@ -832,7 +832,7 @@
       "name" : "{{ blueprint_item.host_group }}",
       "configurations" : [ ],
       "components" : [
-        {% for client in blueprint_item.clients -%}
+        {% for client in blueprint_item.clients|default([]) -%}
         { "name" : "{{ client }}" },
         {% endfor %}
 {% if security|lower == "mit-kdc" or security|lower == "active-directory" -%}

diff --git a/playbooks/roles/ambari-blueprint/templates/cluster_template.j2 b/playbooks/roles/ambari-blueprint/templates/cluster_template.j2
@@ -29,7 +29,7 @@
   When using a static blueprint, this might sometimes fail, as it simply does the regex check in the blueprint's content.
 #}
   "host_groups" :[
-    {% for group in groups|sort if group not in ['all', 'ambari-server', 'hadoop-cluster', 'ungrouped'] and groups[group]|length > 0 and (blueprint_file == 'blueprint_dynamic.j2' and group in blueprint_dynamic|map(attribute='host_group')|list) or (blueprint_file != 'blueprint_dynamic.j2' and blueprint_static['host_groups'] | string is search(group)) %}
+    {% for group in groups|sort if group not in ['all', 'ungrouped'] and groups[group]|length > 0 and (blueprint_file == 'blueprint_dynamic.j2' and group in blueprint_dynamic|map(attribute='host_group')|list) or (blueprint_file != 'blueprint_dynamic.j2' and blueprint_static['host_groups'] | string is search(group)) %}
 
     {
       "name" : "{{ group }}",

diff --git a/playbooks/roles/ambari-config/templates/vdf-HDF-3.3-latest.xml.j2 b/playbooks/roles/ambari-config/templates/vdf-HDF-3.3-latest.xml.j2
@@ -0,0 +1 @@
+vdf-HDF-3.3.0.0.xml.j2
diff --git a/playbooks/roles/ambari-config/templates/vdf-HDF-3.3.0.0.xml.j2 b/playbooks/roles/ambari-config/templates/vdf-HDF-3.3.0.0.xml.j2
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="version_definition.xsd">
+  <release>
+{% include 'vdf-HDF-part-release.j2' %}
+  </release>
+  <manifest>
+    <service id="KNOX-100" name="KNOX" version="1.0.0"/>
+    <service id="KAFKA-200" name="KAFKA" version="2.0.0"/>
+    <service id="NIFI_REGISTRY-030" name="NIFI_REGISTRY" version="0.3.0"/>
+    <service id="REGISTRY-054" name="REGISTRY" version="0.5.4"/>
+    <service id="RANGER-120" name="RANGER" version="1.2.0"/>
+    <service id="RANGER_KMS-120" name="RANGER_KMS" version="1.2.0"/>
+    <service id="ZOOKEEPER-346" name="ZOOKEEPER" version="3.4.6"/>
+    <service id="MINIFI-060" name="MINIFI" version="0.6.0"/>
+    <service id="NIFI-180" name="NIFI" version="1.8.0"/>
+    <service id="STORM-121" name="STORM" version="1.2.1"/>
+    <service id="STREAMLINE-060" name="STREAMLINE" version="0.6.0"/>
+  </manifest>
+  <available-services/>
+  <repository-info>
+{% include 'vdf-HDF-part-repository-info.j2' %}
+  </repository-info>
+</repository-version>
diff --git a/playbooks/roles/ambari-config/templates/vdf-HDP-3.1-latest.xml.j2 b/playbooks/roles/ambari-config/templates/vdf-HDP-3.1-latest.xml.j2
@@ -0,0 +1 @@
+vdf-HDP-3.1.0.0.xml.j2
diff --git a/playbooks/roles/ambari-config/templates/vdf-HDP-3.1.0.0.xml.j2 b/playbooks/roles/ambari-config/templates/vdf-HDP-3.1.0.0.xml.j2
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="version_definition.xsd">
+  <release>
+{% include 'vdf-HDP-part-release.j2' %}
+  </release>
+  <manifest>
+    <service id="HDFS-311" name="HDFS" version="3.1.1"/>
+    <service id="YARN-311" name="YARN" version="3.1.1"/>
+    <service id="MAPREDUCE2-311" name="MAPREDUCE2" version="3.1.1"/>
+    <service id="STORM-121" name="STORM" version="1.2.1"/>
+    <service id="PIG-0160" name="PIG" version="0.16.0"/>
+    <service id="TEZ-091" name="TEZ" version="0.9.1"/>
+    <service id="HIVE-310" name="HIVE" version="3.1.0"/>
+    <service id="OOZIE-431" name="OOZIE" version="4.3.1"/>
+    <service id="KAFKA-200" name="KAFKA" version="2.0.0"/>
+    <service id="HBASE-202" name="HBASE" version="2.0.2"/>
+    <service id="KNOX-100" name="KNOX" version="1.0.0"/>
+    <service id="RANGER-120" name="RANGER" version="1.2.0"/>
+    <service id="RANGER_KMS-120" name="RANGER_KMS" version="1.2.0"/>
+    <service id="ZOOKEEPER-346" name="ZOOKEEPER" version="3.4.6"/>
+    <service id="ATLAS-110" name="ATLAS" version="1.1.0"/>
+    <service id="SQOOP-147" name="SQOOP" version="1.4.7"/>
+    <service id="ZEPPELIN-080" name="ZEPPELIN" version="0.8.0"/>
+    <service id="SPARK2-232" name="SPARK2" version="2.3.2"/>
+    <service id="DRUID-0121" name="DRUID" version="0.12.1"/>
+    <service id="ACCUMULO-170" name="ACCUMULO" version="1.7.0"/>
+  </manifest>
+  <available-services/>
+  <repository-info>
+{% include 'vdf-HDP-part-repository-info.j2' %}
+  </repository-info>
+</repository-version>
diff --git a/playbooks/set_variables.yml b/playbooks/set_variables.yml
@@ -245,7 +245,7 @@
         - name: Populate the all services and clients lists
           set_fact:
             blueprint_all_services: "{{ blueprint_all_services | union(item.services) }}"
-            blueprint_all_clients: "{{ blueprint_all_clients | union(item.clients) }}"
+            blueprint_all_clients: "{{ blueprint_all_clients | union(item.clients|default([])) }}"
           when: groups[item.host_group] is defined and groups[item.host_group]|length > 0
           with_items: "{{ blueprint_dynamic }}"
           no_log: True