[WIP] 4.0 complete Chef stack (#37)

4.0 complete Chef stack

.travis.yml

@@ -1,11 +1,24 @@
-addons:
-  apt:
-    packages:
-    - awscli
-install: true
+language: python
+python:
+   - '3.6'
+install:
+  - pip install awscli cfn-lint
 script:
-- aws s3 cp backendless_chef.yaml s3://aws-native-chef-server/backendless_chef_test.yaml
-- aws cloudformation validate-template --template-url https://s3.amazonaws.com/aws-native-chef-server/backendless_chef_test.yaml
+- cfn-lint -f json *.yaml
+- aws cloudformation validate-template --template-body file://main.yaml
+- aws cloudformation validate-template --template-body file://chef_elasticsearch.yaml
+- aws cloudformation validate-template --template-body file://chef_rds.yaml
+- aws cloudformation validate-template --template-body file://automate.yaml
+- aws cloudformation validate-template --template-body file://chef_server_ha.yaml
+- aws cloudformation validate-template --template-body file://chef_server_ops_dashboard.yaml
+- aws cloudformation validate-template --template-body file://supermarket.yaml
+env:
+  global:
+  - AWS_DEFAULT_REGION=us-east-1
+  # AWS_ACCESS_KEY_ID
+  - secure: "McXO5ZJKxDbfQYoAPiRGsm/UppTovtOcvrntEEIt53vulQS5beeo3c82tcJ8qxkUoKudreWv9E05j8wAIyOA5kKenlmapQYkgNMb1ROZXlklJOh5urhqvYn6mi0nv4MiOidPBvOPgXOm/mkr4tvo40YWa3950sjjrmcDqNugsSl0rsX5tF6/RzJ27xH4AuX8Pp8VqNKNiv9Gkjfy3+b+HPIgmw1esNxTetpjFtstZdz2OHQU6IPcg6tUhu9BbUWaLm7S/iO+E32vgKRheOOh6QWQKF1KFQNvjVQhJHXuAA60HFNrk506rm4y2do4CR8COqMcbjcOILF0CWYG15oMA5+zttUzcicPiXs2de0v0i0y9msAkXwxJuMRFu59p33KnXfEgRAUEwKawefYA7NnLHCWuC4XDmzR163Q7qSZ9aNsMMEvs2dmHe4jtepVXmBQOm6J0eCwh3WZTGZ1oEs5R6Kcoy3RhihUvuxIqp9ZBm6QHJMfoffqCm4ua31coee/H3BNMzQn+eZ6cH7eKy4Kp8LoIFHLzZu+O5f8fx/y5vtMZuc5g45mt7IARe82wGGZrap/gIFLkHO7o+WhoYs9YQtpDBn/S4MlpDbB+ySdCVQc/EtqpxhA7PcpjVSUWHfB0Ik/SfevBlCpl/9DvQI9VxcUPHeTHT4g/heZELxf0Mw="
+  # AWS_SECRET_ACCESS_KEY
+  - secure: "NRTvDe1Eii9iZ0A9SVzN02C5Ud9Gd5F7YOALRoO4+PHTq8+yjuaaQbk9BlsC2yMWqlLNbnClJ0HHR4RjuM84mKIS3Wy8koi9ZJa43D7ZEqjqGf/GbBxF3oJ7ThEXqWuL8UcxmXarHIE+joUxMtWjNh2n+9//9YIgCELL2QnQ5duEkkNpeXlzPsgkLpeUVTnTLD5n1xPsGhr2ffAss1ktHoYQakyu+/8OWV1SmAjN/i1i3BCYfRZBWE/8HsIu5CMQPb95rUTANtlJVmcrEuKBs3I0TQIWFlWrREpREX1Ggh0zqX3za+3khJoSWMXS5ZcSsNQoz2YieMTANU1bEFGaFm+d7oZbj8LX9Xn/VeVOjCoBE85VSKkalFfrQ25lJCYVHxmpkQWUom73fMJqgBdI/QuS7EtV5w9imy5fnAvRAYvv4R+UVonwFJ2GVzMIWaZcFag4V035j6fhV36E2iFZdWGtXWS0eMR+77+vXxqy79rS34uuyEYLJp0CMJj/S4uqLlqbl7T7+Law4dlSYXJW5+vchN/wbuTx6jXUGnjQ91crxTMR4akMSXttt01MLdjJcS9qgelV51oEKiElo6T/sAF4LennB09E5csl5wvphwKxEq1QZgxrVPoR3HZ3obOJylYwENTREFxYVtBtuK1nxYBc73pcgBnA38N9vxnxFdQ="
 deploy:
   provider: s3
   access_key_id:
@@ -16,9 +29,3 @@ deploy:
   acl: public_read
   on:
     branch: master
-env:
-  global:
-  - secure: vHqHE/Du2MHf8xGHtB21wZOf2sNpFxelx+eCIdffAq7EpwhmY6de60Q5Ywk6ODx4yke21tcrWfLJNH8EWEeemvd8Neq6GRvj+5ezNoE16gucFJDcQ9blIHZGiPOVuMPZILrbGfFG3Hnkxu4jBK/8gATr5Sq0eBuFMmPFexZo6QKynayAzMiYdsnkaMOfhH0wPXXZLQOpN+GO4UmG7gjPs8cTCZUHj0OzWkU6/0AtEe7hLm5KFfeVVh4QWoipOEXRX+9Wmijtr29H1tomEuFin5VZpToZnE+3iHW1VhMtbsSmOdswX4JnjUa0upG8uOzvYmioNrq8Ae4wYdX3/nNBkpKNfPAOFqdS5b4tVEeeDCGKT/bLV+1g0FxK1svFst2H/IT8GGnsoeGKoGOUmSYEVC6nRe0+74wZs5vi2VmqcwffgQdB0PJFlgnFKxe7r52mT6biLRWbtb4nP4Sr+mBIwIZB4sKuIa1vtdgj5APGEpSAa0DW1lp6oBxhyWPXg7/Fmy4H5mZXS7gZXQoHX9/cPeHd3cdd9bNtmcK+HOMIwrmP6mxGIbUPUXlOrgCE3ugiPsVC8uRlCobjYJbrbOHuQToh/TdZN9oAbJqwjiR5mCJuaHc34/ZleNDLPpdJTyYs9ARBft1c/wWqx9BnexoLuzeTAuVs1hb8yYzjHtHpEsE=
-  - secure: Mxngxb+f+edoK42od1M1ojTbOr/Ha3fyEkjYzvX1gF7P0P2imm4utiBciJdJq1AnxXLKPYwoSMYMNvkXCpWdcOrz9JF4Hux+aJ1f7wzssaMgVAdfi8ZM438HXTz7LgkdkZ/d/FX+CgWszx37aHd9AfzH/x9/wQDEuAf+PvSPjGVW86V8jYZ74ZYgyJVnXdkOXwdKOT4TqWp/C7XQvzOJdm8soEFllEh5tuyCDj5SqGfjCj6XzEXQIVAoBujSiIVs6N1cSY3qayy63ReinBk29BcdhThSpBr1OefBcJ33GTk02R0mlqUeVncZ1PWRdNmizf5qPB9+dDbAhEoZzrk6fcE+lNLWvIKV/jhEnumQGmLWwEpZGm20K2AVqDGcsyKdvOzP4J43xpkHHvMMBOP5wqjbJ3UhPA4xlu6BVpQgSYF12YBkjMTKecqhzcPkmbFatuN9kNpGxKyEGL12KzkbrXVVJ2WCank1C7XYvPPFi14GgVb7dyB3GOGP5mvfCQw6tnYfkGdvQKoXzkBK7i/5LuLl3R9wmBUsCc+pflHEdLEPJ6BeWE3WmjEPsZTYayaEW8YXjU0AIHGfvXVB7U4GuUVaqIOltSdt2j5C730ak4OwriwHKsbI1TTkHETcwvPkyk3FYmUW4HoxOndLcXtcDIqp1PwECe7doeTbTcBelho=
-  - secure: tU3ZGSVTVRBarT1ZOH3y/fu/fGKaphN/48ICQbfT6LVCZ0g+6vNp0QUaukiJaDeoyQvyJOX1iXyTYtxMZy3++PTJJ/JU2nvrFEaE0Y9l3LdnMC+v2cZtDWOAROUqBgWIQL9bDH5YIR6Iqb6DtLdp70qHvfUQhHjte/n8ej1n6Kr8avaUU/jfGU6GesjXIRitz2dKh62IJrMNG0GThhAMjH50BNg5uXVgSsUXBrDUZcc7KHlSHI1hvf3pt35/irMgflVM4yDhJwKpKXY/szxcGkn1na4MhaJGLZPupmkYpoHjdRjBvBI9he5wbFucm/coSyRo6GLqE0APi4AICNQS/gCIjDeEG0vgIh3YrERVdIfJFp3La85pbPUmwA9c4zIbyVBRtUUKDpjdhJYoS9Mj4+mLSBvAI0TxG/CNU8Hbg7VMHLIW05SvPFAk7GMF8MsM9jQHjl3KztRupyw3v3x6qPb9qezJYzBVJxZ1GeHSWG3roAmqKnoN39EUMtcQglh/2fS1HeDfU6O+gN8ZxK+g0jILTxyAgj++3x0pZ45B+OSTh7fuReT6uAZYZHjF1Vg1PFK5Ek7gjmksOO3QH+G62u9j+7iPlFWkYAfj0/gHUsN+roe5o7FvjJmVhhIb5x35aX0IONNg6eWr7shAA9aLCcR/26aY+ouL0juTwYb564U=
-  - secure: Tad2u9nKOcE5LGe/ruLbJP9FuGWZiEIPKEYp7di+AV/EnYUrPN3i2BP2uv/atC0Uwwj5HbEWnNqdOZqJCMafPacetVvNGtYXxati9PeWbg9eGtm6GOGnJQ3GmBgC0vxAt8MmjAOu7o4tK8X3FCiaDiiJtsIf+tohUADKfMV+28NHyd1RXls2tW+BybtNOodUiydZDLIAxTiqePUlHcQdgW65HRseHOpSUnKbzyrsOrBqnWtIm6/kXNVJwxVxTHAsNNMRSVldiI+JZtlsQPByxI7e/ielzxOU+He3AvNmzbvmX9cWIfbixSEgvFO+hUCQdz1lFEjuSPMZI+hGxAu76ks7hVu9WBaThE+rgWZcT4SPIWTFI4OWrodEZI56q/GI4aOA4S+3AZRyd2R1rS9IGAgNPCLu95SyCpn+Pl4yOVnGuKTMs8O2CodjVIzG0iQC3IJA+9hqho4yrYMnn17O4itvUnTx2YXdybs3AKRdp7ivKeAXK3iQL9HOsI+itTgy616scW7+Tefj3a53cHpNkUX2jLWm4Z1C/fbAhFOPGu5FNRjagHFDO04WoMxMWQWRpi7j4j1tBVEuUYzliKFEZwYZrR01fH9Bq6ybuOpKN8wBfrEORf45BaZYI4+8WXdXdtEV8ors4EDfO3wm+w4YRxSKr/+Nst54P6ooJmgGT+E=

README.md

@@ -1,9 +1,12 @@
-# AWS Native Chef Server Cluster
-A Chef Server cluster utilizing Amazon services for high availability, auto-scaling and DBaaS
+# AWS Native Chef Stack
+A complete Chef Stack including:
+- Chef Automate 2 server, using EC2 Auto-Recovery
+- Chef Server cluster utilizing Amazon services for high availability, auto-scaling and DBaaS
+- Chef Supermarket server, using EC2 Auto-Recovery
 
 ![Chef Server Architecture Diagram](/images/arch-diagram.png?raw=true "Architecture Diagram")
 
-# What does this template provision?
+# What does the chef_server_ha template provision?
 - A "bootstrap" frontend in an Auto Scaling Group of 1.
 - A second frontend in an Auto Scaling Group that will automatically scale up to a configured maximum (default 3)
 - A Multi-AZ Elastic Load Balancer
@@ -14,13 +17,13 @@ A Chef Server cluster utilizing Amazon services for high availability, auto-scal
 
 ![Dashboard Example](/images/opsdashboard.png?raw=true "Architecture Diagram")
 
-
 # Using it
 
 ## Requirements
 * A working knowledge and comfort level with CloudFormation so that you can read and understand this template for your self
 * Permissions to create all of the types of resources specified in this template (IAM roles, Database subnets, etc)
 * A valid SSL certificate ARN (from the AWS Certificate Manager service)
+* A Route53 hosted zone (optional but strongly recommended)
 
 ## Prerequisites
 
@@ -31,15 +34,20 @@ Before you fire it up, there are a few things you should make sure you have prep
 ## Fire up the Chef Server stack
 
 You can launch this stack with the push of a button:
-<p><a href="https://console.aws.amazon.com/cloudformation/home#/stacks/new?templateURL=https:%2F%2Fs3.amazonaws.com%2Faws-native-chef-server%2Fbackendless_chef.yaml&amp;stackName=my-chef-cluster" target="_blank"><img src="https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png" alt="Launch Stack" /></a></p>
+<p><a href="https://console.aws.amazon.com/cloudformation/home#/stacks/new?templateURL=https:%2F%2Fs3.amazonaws.com%2Faws-native-chef-server%2Fmain.yaml&amp;stackName=my-chef-stack" target="_blank"><img src="https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png" alt="Launch Stack" /></a></p>
 
 However, the most repeatable and least error-prone way to launch this stack is to use the `aws` command-line. First copy file `stack_parameters.json.example` to `stack_parameters.json`, make the necessary changes, then run this command:
 
 ```bash
 MYBUCKET=aws-native-chef-server
+MYID=mycompany
+# Configure the automate_stack_parameters.json and then launch the cloudformation stack:
+# If you're using your own bucket, uncomment the next line:
+# aws s3 sync . s3://$MYBUCKET/ --exclude "*" --include "*.yaml" --include "files/*" && \
+aws cloudformation validate-template --template-url https://s3.amazonaws.com/$MYBUCKET/main.yaml && \
 aws cloudformation create-stack \
-  --stack-name irving-backendless-chef \
-  --template-url https://s3.amazonaws.com/$MYBUCKET/backendless_chef.yaml \
+  --stack-name ${MYID}-chef-stack \
+  --template-url https://s3.amazonaws.com/$MYBUCKET/main.yaml \
   --capabilities CAPABILITY_IAM \
   --on-failure DO_NOTHING \
   --parameters file://stack_parameters.json
@@ -51,11 +59,13 @@ If you've made changes to the template content or parameters and you wish to upd
 
 ```bash
 MYBUCKET=aws-native-chef-server
-aws s3 cp backendless_chef.yaml s3://$MYBUCKET/
-aws cloudformation validate-template --template-url https://s3.amazonaws.com/$MYBUCKET/backendless_chef.yaml
+MYID=mycompany
+# If you're using your own bucket, uncomment the next line:
+# aws s3 sync . s3://$MYBUCKET/ --exclude "*" --include "*.yaml" --include "files/*" && \
+aws cloudformation validate-template --template-url https://s3.amazonaws.com/$MYBUCKET/main.yaml && \
 aws cloudformation update-stack \
-  --stack-name irving-backendless-chef \
-  --template-url https://s3.amazonaws.com/$MYBUCKET/backendless_chef.yaml \
+  --stack-name ${MYID}-chef-stack \
+  --template-url https://s3.amazonaws.com/$MYBUCKET/main.yaml \
   --capabilities CAPABILITY_IAM \
   --parameters file://stack_parameters.json
 ```
@@ -67,12 +77,13 @@ Note: For production instances it is recommended to use the CloudFormation conso
 If you're using a bastion host and need to SSH from the outside:
 
 ```bash
-ssh -o ProxyCommand="ssh -W %h:%p -q ec2-user@bastion" -l ec2-user <chef server private ip>
+ssh -o ProxyCommand="ssh -W %h:%p -q user@bastion" -l user <chef server private ip>
 ```
+Where "user" is `ec2-user` on the RHEL AMI, and `centos` on the CentOS AMI
 
-otherwise just login as `ec2-user` to the private IPs of the chef servers
+otherwise just SSH directly to the public IPs of the chef servers
 
-## Upgrading
+## Upgrading the Chef Server
 
 If a new Chef Server or Manage package comes out, the process for upgrading is simple and requires no downtime:
 
@@ -111,21 +122,6 @@ Yes, it is significantly more robust and easier to operate.
 
 Contributions are welcomed!
 
-# Developer notes
-
-## RegionMap
-To update the region map execute the following lines in your terminal and then paste the results into the `AWSRegion2AMI` mappings section of the template:
-
-```bash
-AMAZON_RELEASE='amzn-ami-hvm-2018.03.0.20180622-x86_64-gp2'
-regions=$(aws ec2 describe-regions --query "Regions[].RegionName" --output text)
-for region in $regions; do
-  ami=$(aws --region $region ec2 describe-images \
-  --filters "Name=name,Values=${AMAZON_RELEASE}" \
-  --query "Images[0].ImageId" --output "text")
-  printf "    $region:\n      AMI: $ami\n"; done
-```
-
 # Credits
 
 This project was inspired by the work of [Levi Smith](https://github.com/TheFynx) of the Hearst Automation Team and published at [HearstAT/cfn_backendless_chef](https://github.com/HearstAT/cfn_backendless_chef).  Thanks Levi!
@@ -134,4 +130,4 @@ Contributors:
 - Irving Popovetsky
 - Joshua Hudson
 - Levi Smith
-- Daniel Bright
+- Daniel Bright