diff --git a/chef-templates/jenkins/CHANGELOG.md b/chef-templates/jenkins/CHANGELOG.md new file mode 100644 index 0000000..ccf1464 --- /dev/null +++ b/chef-templates/jenkins/CHANGELOG.md @@ -0,0 +1,5 @@ +### 2016-05-10 + +* Initial Jenkins Master / Slave import. +* Initial import of RL10 volume scripts. +* Initial import of generic CAT which will launch a master / slave cluster. diff --git a/chef-templates/jenkins/Jenkins-Master.yml b/chef-templates/jenkins/Jenkins-Master.yml new file mode 100644 index 0000000..08379c6 --- /dev/null +++ b/chef-templates/jenkins/Jenkins-Master.yml @@ -0,0 +1,138 @@ +Name: Jenkins Master - Chef Solo (RightLink 10.6.0) +Description: "Chef Client ServerTemplate for RightLink10. RightLink10 is a new agent + to connect servers to the RightScale platform that is very lightweight and simple + to install.\n_It is not a plug-in upgrade from the older RightLink v6 series._\nSee + [http://docs.rightscale.com/rl/about.html](http://docs.rightscale.com/rl/about.html) + for details.\n\nThis base ST runs a script to update the packaging system and enable + automatic security updates, and a collectd install script to enable RightScale monitoring. + \n\nThe base ST can run on most any Linux distro that supports cloud-init. It is + recommended to use the standard distro images in various clouds.\n\n__Requirements__\n\n* + Chef Server or Hosted Chef\n\n__Tested Linux distros:__\n\n* Ubuntu 14.04 x86_64\n\n__Documenation__\n\n* + [Overview](http://docs.rightscale.com/st/rl10/chef-client/overview.html)\n\n\n__Tested + compute clouds:__\n\n* AWS\n" +Inputs: + COLLECTD_SERVER: env:RS_TSS + RS_INSTANCE_UUID: env:RS_INSTANCE_UUID +RightScripts: + Boot: + - Name: RL10 Linux Wait For EIP + Revision: 5 + Publisher: RightScale + - Name: RL10 Linux Setup Hostname + Revision: 8 + Publisher: RightScale + - Name: RL10 Linux Enable Managed Login + Revision: 12 + Publisher: RightScale + - Name: RL10 Linux Setup Alerts + Revision: 2 + Publisher: RightScale + - Name: RL10 Linux Setup Automatic Upgrade + Revision: 4 + Publisher: RightScale + - RL5_6_10_Setup_Custom_Logrotate_Configs.sh + - Storage_Toolbox_Stripe-chef.sh + - RL10_Jenkins_Install_Master.sh + Decommission: + - RL10_Linux_Shutdown_Reason.sh + - Storage_Toolbox_Decommission-chef.sh + Operational: + - RL10_Linux_Setup_Automatic_Upgrade.sh + - RL10_Linux_Upgrade.sh + - Storage_Toolbox_Schedule-chef.sh + - Storage_Toolbox_Backup-chef.sh +MultiCloudImages: +- Name: Ubuntu_14.04_x64 + Revision: 70 + Publisher: RightScale +- Name: Ubuntu_12.04_x64 + Revision: 66 + Publisher: RightScale +- Name: Ubuntu_12.04_x64_KVM + Revision: 31 + Publisher: RightScale +- Name: Ubuntu_14.04_x64_KVM + Revision: 31 + Publisher: RightScale +- Name: Ubuntu_16.04_x64 + Revision: 5 + Publisher: RightScale +- Name: Ubuntu_16.04_x64_KVM + Revision: 3 + Publisher: RightScale +- Name: CentOS_6.x_x64 + Revision: 25 + Publisher: RightScale +- Name: CentOS_6.x_x64_KVM + Revision: 30 + Publisher: RightScale +- Name: CentOS_7.x_x64 + Revision: 33 + Publisher: RightScale +- Name: CentOS_7.x_x64_KVM + Revision: 30 + Publisher: RightScale +- Name: RHEL_6.x_x64_KVM + Revision: 9 + Publisher: RightScale +- Name: RHEL_7.x_x64_KVM + Revision: 9 + Publisher: RightScale +- Name: RHEL_6.x_x64 + Revision: 13 + Publisher: RightScale +- Name: RHEL_7.x_x64 + Revision: 11 + Publisher: RightScale +Alerts: +- Name: rs instance terminated + Description: Raise an alert if the instance has been terminated abnormally, i.e. + not through the RightScale interface or by an elasticity daemon resizing server + arrays. + Clause: If RS/server.state == terminated for 1 minutes Then escalate critical +- Name: rs instance stranded + Description: Raise an alert if the instance enters the stranded state. + Clause: If RS/server-failure.state == stranded for 1 minutes Then escalate warning +- Name: rs instance not responding + Description: Raise an alert if the instance fails to send monitoring information + for 5 minutes. + Clause: If cpu-0/cpu-idle.value == NaN for 5 minutes Then escalate critical +- Name: rs cpu busy + Description: Raise an alert if the idle time is too low. + Clause: If cpu-0/cpu-idle.value < 15 for 3 minutes Then escalate warning +- Name: rs cpu overloaded + Description: Raise an alert when the cpu idle time is too low. + Clause: If cpu-0/cpu-idle.value < 3 for 5 minutes Then escalate critical +- Name: rs cpu I/O wait + Description: Raise an alert if disk io is too high. + Clause: If cpu-0/cpu-wait.value > 40 for 15 minutes Then escalate warning +- Name: rs low space in root partition + Description: Raise an alert if the available space in the root partition is too + low. This alert may be modified on an instance to match the metric name df/df-root.free + instead if the instance is running collectd 4. See the RL10 Linux Setup Alerts + RightScript (rll/setup-alerts.sh) for more details. + Clause: If df-root/df_complex-free.value < 1073741824 for 5 minutes Then escalate + critical +- Name: rs high network tx activity + Description: Raise an alert if the amount of network data transmitted is too high. + This alert may be modified or cloned on an instance to match the actual network + interfaces that are present. See the RL10 Linux Setup Alerts RightScript (rll/setup-alerts.sh) + for more details. + Clause: If interface-eth0/if_octets.tx > 10000000 for 10 minutes Then escalate critical +- Name: rs high network rx activity + Description: Raise an alert if the amount of network data received is too high. + This alert may be modified or cloned on an instance to match the actual network + interfaces that are present. See the RL10 Linux Setup Alerts RightScript (rll/setup-alerts.sh) + for more details. + Clause: If interface-eth0/if_octets.rx > 50000000 for 30 minutes Then escalate critical +- Name: rs low swap space + Description: Raise alert if the free swap space is too low. This alert may be removed + from an instance if swap is not enabled. See the RL10 Linux Setup Alerts RightScript + (rll/setup-alerts.sh) for more details. + Clause: If swap/swap-free.value < 104857600 for 5 minutes Then escalate critical +- Name: rs memory low + Description: Raise an alert if free memory is too low. + Clause: If memory/memory-free.value < 1000000 for 1 minutes Then escalate critical +- Name: rs out of memory + Description: Raise an alert when the server is out of free memory. + Clause: If memory/memory-free.value == 0 for 1 minutes Then escalate critical diff --git a/chef-templates/jenkins/Jenkins-Slave.yml b/chef-templates/jenkins/Jenkins-Slave.yml new file mode 100644 index 0000000..2851584 --- /dev/null +++ b/chef-templates/jenkins/Jenkins-Slave.yml @@ -0,0 +1,138 @@ +Name: Jenkins Slave - Chef Solo (RightLink 10.6.0) +Description: "Chef Client ServerTemplate for RightLink10. RightLink10 is a new agent + to connect servers to the RightScale platform that is very lightweight and simple + to install.\n_It is not a plug-in upgrade from the older RightLink v6 series._\nSee + [http://docs.rightscale.com/rl/about.html](http://docs.rightscale.com/rl/about.html) + for details.\n\nThis base ST runs a script to update the packaging system and enable + automatic security updates, and a collectd install script to enable RightScale monitoring. + \n\nThe base ST can run on most any Linux distro that supports cloud-init. It is + recommended to use the standard distro images in various clouds.\n\n__Requirements__\n\n* + Chef Server or Hosted Chef\n\n__Tested Linux distros:__\n\n* Ubuntu 14.04 x86_64\n\n__Documenation__\n\n* + [Overview](http://docs.rightscale.com/st/rl10/chef-client/overview.html)\n\n\n__Tested + compute clouds:__\n\n* AWS\n" +Inputs: + COLLECTD_SERVER: env:RS_TSS + RS_INSTANCE_UUID: env:RS_INSTANCE_UUID +RightScripts: + Boot: + - Name: RL10 Linux Wait For EIP + Revision: 5 + Publisher: RightScale + - Name: RL10 Linux Setup Hostname + Revision: 8 + Publisher: RightScale + - Name: RL10 Linux Enable Managed Login + Revision: 12 + Publisher: RightScale + - Name: RL10 Linux Setup Alerts + Revision: 2 + Publisher: RightScale + - Name: RL10 Linux Setup Automatic Upgrade + Revision: 4 + Publisher: RightScale + - RL5_6_10_Setup_Custom_Logrotate_Configs.sh + - Storage_Toolbox_Stripe-chef.sh + - RL10_Jenkins_Install_Slave.sh + Decommission: + - RL10_Linux_Shutdown_Reason.sh + - Storage_Toolbox_Decommission-chef.sh + Operational: + - RL10_Linux_Setup_Automatic_Upgrade.sh + - RL10_Linux_Upgrade.sh + - Storage_Toolbox_Schedule-chef.sh + - Storage_Toolbox_Backup-chef.sh +MultiCloudImages: +- Name: Ubuntu_14.04_x64 + Revision: 70 + Publisher: RightScale +- Name: Ubuntu_12.04_x64 + Revision: 66 + Publisher: RightScale +- Name: Ubuntu_12.04_x64_KVM + Revision: 31 + Publisher: RightScale +- Name: Ubuntu_14.04_x64_KVM + Revision: 31 + Publisher: RightScale +- Name: Ubuntu_16.04_x64 + Revision: 5 + Publisher: RightScale +- Name: Ubuntu_16.04_x64_KVM + Revision: 3 + Publisher: RightScale +- Name: CentOS_6.x_x64 + Revision: 25 + Publisher: RightScale +- Name: CentOS_6.x_x64_KVM + Revision: 30 + Publisher: RightScale +- Name: CentOS_7.x_x64 + Revision: 33 + Publisher: RightScale +- Name: CentOS_7.x_x64_KVM + Revision: 30 + Publisher: RightScale +- Name: RHEL_6.x_x64_KVM + Revision: 9 + Publisher: RightScale +- Name: RHEL_7.x_x64_KVM + Revision: 9 + Publisher: RightScale +- Name: RHEL_6.x_x64 + Revision: 13 + Publisher: RightScale +- Name: RHEL_7.x_x64 + Revision: 11 + Publisher: RightScale +Alerts: +- Name: rs instance terminated + Description: Raise an alert if the instance has been terminated abnormally, i.e. + not through the RightScale interface or by an elasticity daemon resizing server + arrays. + Clause: If RS/server.state == terminated for 1 minutes Then escalate critical +- Name: rs instance stranded + Description: Raise an alert if the instance enters the stranded state. + Clause: If RS/server-failure.state == stranded for 1 minutes Then escalate warning +- Name: rs instance not responding + Description: Raise an alert if the instance fails to send monitoring information + for 5 minutes. + Clause: If cpu-0/cpu-idle.value == NaN for 5 minutes Then escalate critical +- Name: rs cpu busy + Description: Raise an alert if the idle time is too low. + Clause: If cpu-0/cpu-idle.value < 15 for 3 minutes Then escalate warning +- Name: rs cpu overloaded + Description: Raise an alert when the cpu idle time is too low. + Clause: If cpu-0/cpu-idle.value < 3 for 5 minutes Then escalate critical +- Name: rs cpu I/O wait + Description: Raise an alert if disk io is too high. + Clause: If cpu-0/cpu-wait.value > 40 for 15 minutes Then escalate warning +- Name: rs low space in root partition + Description: Raise an alert if the available space in the root partition is too + low. This alert may be modified on an instance to match the metric name df/df-root.free + instead if the instance is running collectd 4. See the RL10 Linux Setup Alerts + RightScript (rll/setup-alerts.sh) for more details. + Clause: If df-root/df_complex-free.value < 1073741824 for 5 minutes Then escalate + critical +- Name: rs high network tx activity + Description: Raise an alert if the amount of network data transmitted is too high. + This alert may be modified or cloned on an instance to match the actual network + interfaces that are present. See the RL10 Linux Setup Alerts RightScript (rll/setup-alerts.sh) + for more details. + Clause: If interface-eth0/if_octets.tx > 10000000 for 10 minutes Then escalate critical +- Name: rs high network rx activity + Description: Raise an alert if the amount of network data received is too high. + This alert may be modified or cloned on an instance to match the actual network + interfaces that are present. See the RL10 Linux Setup Alerts RightScript (rll/setup-alerts.sh) + for more details. + Clause: If interface-eth0/if_octets.rx > 50000000 for 30 minutes Then escalate critical +- Name: rs low swap space + Description: Raise alert if the free swap space is too low. This alert may be removed + from an instance if swap is not enabled. See the RL10 Linux Setup Alerts RightScript + (rll/setup-alerts.sh) for more details. + Clause: If swap/swap-free.value < 104857600 for 5 minutes Then escalate critical +- Name: rs memory low + Description: Raise an alert if free memory is too low. + Clause: If memory/memory-free.value < 1000000 for 1 minutes Then escalate critical +- Name: rs out of memory + Description: Raise an alert when the server is out of free memory. + Clause: If memory/memory-free.value == 0 for 1 minutes Then escalate critical diff --git a/chef-templates/jenkins/README.md b/chef-templates/jenkins/README.md new file mode 100644 index 0000000..95c0623 --- /dev/null +++ b/chef-templates/jenkins/README.md @@ -0,0 +1,43 @@ +### README: + +* Upload YAML files to your account with [right_st](https://github.com/rightscale/right_st) + +### IMPORTANT INPUTS: + +#### Jenkins Related: + +* `MASTER_IP`: Reachable address for the Jenkins slaves to contact and add themselves to the cluster. Usually the private IP address of the Jenkins Master host. + +* `SWARM_PLUGIN_VERSION`: Version of the Jenkins Swarm plugin to install which allows for slave / master discovery. Currently defaults to `3.4`. + +* `DESCRIPTION`: Text name of jenkins slave instances. + +* `AUTO_DISCOVERY_ADDRESS`: Enable this if using UDP based discovery. Not required. Current defaults allow Jenkins slaves to discover master via Swarm plugin and connecting to `MASTER_IP`. + +* `MASTER_PORT`: Default port Jenkins listens on. Defaults to `8080`. + +#### Storage Related: + +* `BACKUP_KEEP_DAILIES`: Number of daily backups to keep. Defaults to `14`. + +* `BACKUP_KEEP_LAST`: Number of snapshots to keep. Defaults to `60`. + +* `BACKUP_KEEP_MONTHLIES`: Number of monthly backups to keep. Defaults to `12`. + +* `BACKUP_KEEP_WEEKLIES`: Number of weekly backups to keep. Defaults to `6`. + +* `BACKUP_KEEP_YEARLIES`: Number of yearly backups to keep. Defaults to `2`. + +* `STOR_BACKUP_LINEAGE`: Name of backup lineage to use for snapshots. + +* `DEVICE_MOUNT_POINT`: Mount point of data volume. Defaults to `/var/lib/jenkins`, which is the Jenkins home / work directory. + +* `DEVICE_NICKNAME`: Name of the LVM device to be created. Defaults to `data_storage`. + +* `DEVICE_COUNT`: Number of devices to create which will make up the underlying LVM volume. + +* `DEVICE_DESTROY_ON_DECOMMISSION`: If set to true, the devices will be destroyed on decommission. Defaults to `false`. + +* `STOR_RESTORE_LINEAGE`: If set, restore from the supplied backup name. + +* `STOR_RESTORE_TIMESTAMP`: The filesystem to be used on the data volume. Defaults to `ext4`. \ No newline at end of file diff --git a/chef-templates/jenkins/RL10_Jenkins_Install_Master.sh b/chef-templates/jenkins/RL10_Jenkins_Install_Master.sh new file mode 100755 index 0000000..ac9e08a --- /dev/null +++ b/chef-templates/jenkins/RL10_Jenkins_Install_Master.sh @@ -0,0 +1,122 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: RL10 Jenkins Install Master +# Description: Install and configure Jenkins master server +# Inputs: +# LOG_LEVEL: +# Category: CHEF +# Description: The log level for the chef install +# Input Type: single +# Required: true +# Advanced: true +# Possible Values: +# - text:info +# - text:warn +# - text:fatal +# - text:debug +# Default: text:info +# COOKBOOK_VERSION: +# Category: JENKINS +# Description: 'The jenkins cookbook version to install from. This allows for multiple versions +# of the same cookbook in the attachments. (e.g. If attachments is jenkins-201704111.tar the version is 201704111.)' +# Input Type: single +# Required: true +# Advanced: true +# Default: text:201704183 +# SWARM_PLUGIN_VERSION: +# Category: JENKINS +# Description: 'The swarm plugin version to use.' +# Input Type: single +# Required: true +# Advanced: true +# Default: text:3.4 +# Attachments: +# - rsc_jenkins-201704183.tar +# ... + +set -x +set -e + +# https://github.com/berkshelf/berkshelf-api/issues/112 +export LC_CTYPE=en_US.UTF-8 + +if [ ! -e /usr/bin/chef-client ]; then + curl -L https://omnitruck.chef.io/install.sh | sudo bash -s -- "12.19.36" +fi + +chef_dir="/home/rightscale/.chef" + +# if [ -e $chef_dir/cookbooks ]; then +# echo "Jenkins already installed. Exiting." +# exit 0 +# fi + +rm -rf $chef_dir +mkdir -p $chef_dir/chef-install +chmod -R 0777 $chef_dir/chef-install + +mkdir -p $chef_dir/cookbooks +chown -R 0777 $chef_dir/cookbooks + +#install packages when on ubuntu +if which apt-get >/dev/null 2>&1; then + apt-get -y update + apt-get install -y build-essential git #ruby2.0 ruby2.0-dev +fi + +#install packages for centos +if which yum >/dev/null 2>&1; then + yum groupinstall -y 'Development Tools' + yum install -y libxml2 libxml2-devel libxslt libxslt-devel git +fi + +#install berkshelf +/opt/chef/embedded/bin/gem install berkshelf -v '4.3.5' --no-ri --no-rdoc + +#checkout the chef server cookbook and install dependent cookbooks using berkshelf +cd $chef_dir + +# Download cookbooks from RS Attachments + +if [ -f $RS_ATTACH_DIR/rsc_jenkins-$COOKBOOK_VERSION.tar ]; then + tar -xvf $RS_ATTACH_DIR/rsc_jenkins-$COOKBOOK_VERSION.tar +fi + +/opt/chef/embedded/bin/berks vendor $chef_dir/cookbooks + +cd $HOME + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi + +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + + "tags": [ + ] + }, + "rightscale":{ + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id" + }, + "rsc_jenkins":{ + "swarm" : { + "version" : "$SWARM_PLUGIN_VERSION" + } + }, + "run_list": ["recipe[rsc_jenkins::master]","recipe[rsc_jenkins::swarm-plugin]"] +} +EOF + +cat < $chef_dir/solo.rb +cookbook_path "$chef_dir/cookbooks" +data_bag_path "$chef_dir/data_bags" +EOF + +#cp -f /tmp/environment /etc/environment +/sbin/mkhomedir_helper rightlink + +chef-solo -l $LOG_LEVEL -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/RL10_Jenkins_Install_Slave.sh b/chef-templates/jenkins/RL10_Jenkins_Install_Slave.sh new file mode 100755 index 0000000..eae630d --- /dev/null +++ b/chef-templates/jenkins/RL10_Jenkins_Install_Slave.sh @@ -0,0 +1,290 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: RL10 Jenkins Install Slave +# Description: Install and configure Jenkins master server +# Inputs: +# LOG_LEVEL: +# Category: CHEF +# Description: The log level for the chef install +# Input Type: single +# Required: true +# Advanced: true +# Possible Values: +# - text:info +# - text:warn +# - text:fatal +# - text:debug +# Default: text:info +# COOKBOOK_VERSION: +# Category: JENKINS +# Description: 'The jenkins cookbook version to install from. This allows for multiple versions +# of the same cookbook in the attachments. (e.g. If attachments is jenkins-201704111.tar the version is 201704111.)' +# Input Type: single +# Required: true +# Advanced: true +# Default: text:201704183 +# NAME: +# Category: JENKINS +# Description: 'The name of the jenkins slave server.' +# Input Type: single +# Required: true +# Advanced: false +# MASTER_IP: +# Category: JENKINS +# Description: 'The fqdn or IP address of the master.' +# Input Type: single +# Required: true +# Advanced: false +# SWARM_PLUGIN_VERSION: +# Category: JENKINS +# Description: 'The version of the swam plugin to install. https://wiki.jenkins-ci.org/display/JENKINS/Swarm+Plugin' +# Input Type: single +# Required: true +# Advanced: true +# Default: text:3.4 +# DESCRIPTION: +# Category: JENKINS +# Description: 'Description of slave instances.' +# Input Type: single +# Required: false +# Advanced: false +# AUTO_DISCOVERY_ADDRESS: +# Category: JENKINS +# Description: 'Use this address for udp-based auto-discovery' +# Input Type: single +# Required: false +# Advanced: true +# CANDIDATE_TAG: +# Category: JENKINS +# Description: 'Show swarm candidate with tag only' +# Input Type: single +# Required: false +# Advanced: true +# Default: text:false +# Possible Values: ["text:true", "text:false"] +# DELETE_EXISTING: +# Category: JENKINS +# Description: 'Deletes any existing slave with the same name.' +# Input Type: single +# Required: false +# Advanced: true +# Default: text:true +# Possible Values: ["text:true", "text:false"] +# DISABLE_UNIQUE_ID: +# Category: JENKINS +# Description: 'Disables clients unique ID.' +# Input Type: single +# Required: false +# Advanced: true +# Default: text:false +# Possible Values: ["text:true", "text:false"] +# DISABLE_SSL_VERIFICATION: +# Category: JENKINS +# Description: 'Disables SSL verification. Must be set to true if HTTP is being used.' +# Input Type: single +# Required: false +# Advanced: true +# Default: text:true +# Possible Values: ["text:true", "text:false"] +# EXECUTORS: +# Category: JENKINS +# Description: 'Number of executors.' +# Input Type: single +# Required: false +# Advanced: true +# Default: text:2 +# LABELS: +# Category: JENKINS +# Description: 'Whitespace-separated list of labels to be assigned for this slave.' +# Input Type: single +# Required: false +# Advanced: true +# MASTER_PORT: +# Category: JENKINS +# Description: 'The port the Jenkins master is listening on.' +# Input Type: single +# Required: true +# Advanced: true +# Default: text:8080 +# MASTER_PROTOCOL: +# Category: JENKINS +# Description: 'The http(s) protocol the Jenkins master is listening on.' +# Input Type: single +# Required: true +# Advanced: true +# Default: text:http +# Possible Values: ["text:http", "text:https"] +# MODE: +# Category: JENKINS +# Description: 'The mode controlling how Jenkins allocates jobs to slaves. Can be either "normal" (utilize this slave as much as possible) or "exclusive" (leave this machine for tied jobs only) Default: normal.' +# Input Type: single +# Required: false +# Advanced: true +# Possible Values: ["text:normal", "text:exclusive"] +# NO_RETRY_AFTER_CONNECTED: +# Category: JENKINS +# Description: 'Do not retry if a successful connection gets closed.' +# Input Type: single +# Required: false +# Advanced: true +# Default: text:false +# Possible Values: ["text:true", "text:false"] +# JENKINS_PASSWORD: +# Category: JENKINS +# Description: 'The Jenkins user password.' +# Input Type: single +# Required: false +# Advanced: true +# RETRY: +# Category: JENKINS +# Description: 'Number of retrys before giving up.' +# Input Type: single +# Required: false +# Advanced: true +# JENKINS_USERNAME: +# Category: JENKINS +# Description: 'The Jenkins user name.' +# Input Type: single +# Required: false +# Advanced: true +# RETRY_BACK_OFF_STRATEGY: +# Category: JENKINS +# Description: 'The mode controlling retry wait time.' +# Input Type: single +# Required: false +# Advanced: true +# Possible Values: ["text:linear", "text:exponential", "text:none"] +# RETRY_INTERVAL: +# Category: JENKINS +# Description: 'Time to wait before retry in seconds.' +# Input Type: single +# Required: false +# Advanced: true +# SSL_FINGER_PRINTS: +# Category: JENKINS +# Description: 'Whitespace-separated list of accepted certificate fingerprints (SHA-256/Hex), otherwise system truststore will be used.' +# Input Type: single +# Required: false +# Advanced: true +# MAX_RETRY_INTERVAL: +# Category: JENKINS +# Description: 'Max time to wait before retry in seconds.' +# Input Type: single +# Required: false +# Advanced: true +# Attachments: +# - rsc_jenkins-201704183.tar +# ... + +# Make Name URL Safe +# This should probably be done in the Jenkins cookbook +NAME=`echo "$NAME" | sed "s/ /_/g" | sed "s/[&$+,\/:;=?@#{}<>|%^]//g" | sed "s/[\[\]]//g"` + +set -x +set -e + +# https://github.com/berkshelf/berkshelf-api/issues/112 +export LC_CTYPE=en_US.UTF-8 + +if [ ! -e /usr/bin/chef-client ]; then + curl -L https://omnitruck.chef.io/install.sh | sudo bash -s -- "12.19.36" +fi + +chef_dir="/home/rightscale/.chef" + +# if [ -e $chef_dir/cookbooks ]; then +# echo "Jenkins already installed. Exiting." +# exit 0 +# fi + +rm -rf $chef_dir +mkdir -p $chef_dir/chef-install +chmod -R 0777 $chef_dir/chef-install + +mkdir -p $chef_dir/cookbooks +chown -R 0777 $chef_dir/cookbooks + +#install packages when on ubuntu +if which apt-get >/dev/null 2>&1; then + apt-get -y update + apt-get install -y build-essential git #ruby2.0 ruby2.0-dev +fi + +#install packages for centos +if which yum >/dev/null 2>&1; then + yum groupinstall -y 'Development Tools' + yum install -y libxml2 libxml2-devel libxslt libxslt-devel git +fi + +#install berkshelf +/opt/chef/embedded/bin/gem install berkshelf -v '4.3.5' --no-ri --no-rdoc + +#checkout the chef server cookbook and install dependent cookbooks using berkshelf +cd $chef_dir + +# # Download cookbooks from RS Attachments + +if [ -f $RS_ATTACH_DIR/rsc_jenkins-$COOKBOOK_VERSION.tar ]; then + tar -xvf $RS_ATTACH_DIR/rsc_jenkins-$COOKBOOK_VERSION.tar +fi + +/opt/chef/embedded/bin/berks vendor $chef_dir/cookbooks + +cd $HOME + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi + +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + + "tags": [ + ] + }, + "rightscale":{ + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id" + }, + "rsc_jenkins":{ + "swarm" : { + "version": "$SWARM_PLUGIN_VERSION" + }, + "slave": { + "name": "$NAME", + "description": "$DESCRIPTION", + "master": "$MASTER_IP", + "auto-discovery-address": "$AUTO_DISCOVERY_ADDRESS", + "candidate-tag": $CANDIDATE_TAG, + "delete-existing-clients": $DELETE_EXISTING, + "disable-clients-unique-id": $DISABLE_UNIQUE_ID, + "disable-ssl-verification": $DISABLE_SSL_VERIFICATION, + "executors": "$EXECUTORS", + "labels": "$LABELS", + "master_port": "$MASTER_PORT", + "master_protocol": "$MASTER_PROTOCOL", + "mode": "$MODE", + "no-retry-after-connected": $NO_RETRY_AFTER_CONNECTED, + "password": "$JENKINS_PASSWORD", + "retry": "$RETRY", + "retry-back-off-strategy": "$RETRY_BACK_OFF_STRATEGY", + "retry-interval": "$RETRY_INTERVAL", + "ssl-finger-prints": "$SSL_FINGER_PRINTS", + "username": "$JENKINS_USERNAME", + "maxRetryInterval": "$MAX_RETRY_INTERVAL" + } + }, + "run_list": ["recipe[rsc_jenkins::slave]"] +} +EOF + +cat < $chef_dir/solo.rb +cookbook_path "$chef_dir/cookbooks" +EOF + +#cp -f /tmp/environment /etc/environment +/sbin/mkhomedir_helper rightlink + +chef-solo -l $LOG_LEVEL -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/RL5_6_10_Setup_Custom_Logrotate_Configs.sh b/chef-templates/jenkins/RL5_6_10_Setup_Custom_Logrotate_Configs.sh new file mode 100755 index 0000000..c091364 --- /dev/null +++ b/chef-templates/jenkins/RL5_6_10_Setup_Custom_Logrotate_Configs.sh @@ -0,0 +1,75 @@ +#!/bin/bash -ex +# --- +# RightScript Name: RL5/6/10 Setup Custom Logrotate Configs +# Inputs: +# CONFIGS: +# Category: Logging +# Description: Space separated list of configurations to download from attachments. +# Input Type: single +# Required: true +# Advanced: false +# Default: text:chef +# Attachments: +# - chef +# ... +# RL5/6/10 Setup Custom Logrotate Configs +# This script will copy the custom logrotate configs [attached to this RightScript] and change logrotate to run hourly versus daily. It's currently designed for Ubuntu 12, but shouldn't require much modification for newer versions of Ubuntu or even CentOS as logrotate hasn't changed much recently. +# +# Written by: Bryan Karaffa [bryan.karaffa@rightscale.com] + + +# Operational Code +# This is where you put the steps for install for each version of RightLink. + +# RL5 and RL6 share the same limitation. Use the same code for both versions. +do_RL56 () { + echo "Executed do_RL56()" + # List Contents of attachments + ls -al $RS_ATTACH_DIR/ + # Copy attachments containing custom logrotate configs + for f in $CONFIGS; do + cp -rf $RS_ATTACH_DIR/$f /etc/logrotate.d/ + done + # Set Logrotate to run hourly versus daily + if [ ! -f /etc/cron.hourly/logrotate ]; then cp -rf /etc/cron.daily/logrotate /etc/cron.hourly/logrotate; chmod +x /etc/cron.hourly/logrotate; fi + # List Current Logrotate Config Files + ls -al /etc/logrotate.d/ +} + +do_RL10 () { + echo "Executed do_RL10()" + # List Contents of attachments + ls -al $RS_ATTACH_DIR/ + # Copy attachments containing custom logrotate configs + for f in $CONFIGS; do + sudo cp -rf $RS_ATTACH_DIR/$f /etc/logrotate.d/ + done +# Cleanup the RightScript that may accidently get copied + if [ -f /etc/logrotate.d/__script-0 ]; then sudo rm -f /etc/logrotate.d/__script-0; fi + # Set Logrotate to run hourly versus daily + if [ ! -f /etc/cron.hourly/logrotate ]; then sudo cp -rf /etc/cron.daily/logrotate /etc/cron.hourly/logrotate; sudo chmod +x /etc/cron.hourly/logrotate; fi + # List Current Logrotate Config Files + ls -al /etc/logrotate.d/ +} + + + +# Get the RightLink Version +if [ -f /etc/rightscale.d/rightscale-release ]; then + + if grep -q '6.*.*' /etc/rightscale.d/rightscale-release; then rightlink_version=6; + elif grep -q '5.*.*' /etc/rightscale.d/rightscale-release; then rightlink_version=5; + fi +elif [ -f /var/lib/rightscale-identity ]; then rightlink_version=10; +else + echo "RightLink version could not be identified or is older than version 5." + exit 1 +fi +echo "This instance is running RightLink Version $rightlink_version" + +# Run the function associated with the version of RightLink that is installed. +case $rightlink_version in + 5) do_RL56;; + 6) do_RL56;; + 10) do_RL10;; +esac diff --git a/chef-templates/jenkins/Storage_Toolbox_Backup-chef.sh b/chef-templates/jenkins/Storage_Toolbox_Backup-chef.sh new file mode 100755 index 0000000..6b55d19 --- /dev/null +++ b/chef-templates/jenkins/Storage_Toolbox_Backup-chef.sh @@ -0,0 +1,162 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: Storage Toolbox Backup - chef +# Description: Create a backup of all volumes attached to the server +# Inputs: +# BACKUP_KEEP_DAILIES: +# Category: Storage +# Description: 'Number of daily backups to keep. Example: 14' +# Input Type: single +# Required: false +# Advanced: false +# Default: text:14 +# BACKUP_KEEP_LAST: +# Category: Storage +# Description: "Number of snapshots to keep. Example: 60\r\n" +# Input Type: single +# Required: false +# Advanced: false +# Default: text:60 +# BACKUP_KEEP_MONTHLIES: +# Category: Storage +# Description: 'Number of monthly backups to keep. Example: 12' +# Input Type: single +# Required: false +# Advanced: false +# Default: text:12 +# BACKUP_KEEP_WEEKLIES: +# Category: Storage +# Description: 'Number of weekly backups to keep. Example: 6' +# Input Type: single +# Required: false +# Advanced: false +# Default: text:14 +# BACKUP_KEEP_YEARLIES: +# Category: Storage +# Description: "Number of yearly backups to keep. Example: 2\r\n" +# Input Type: single +# Required: false +# Advanced: false +# Default: text:2 +# STOR_BACKUP_LINEAGE: +# Category: Storage +# Input Type: single +# Required: true +# Advanced: false +# DEVICE_MOUNT_POINT: +# Category: Storage +# Description: 'The mount point to mount the device on. Example: /var/lib/jenkins' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:/var/lib/jenkins +# DEVICE_NICKNAME: +# Category: Storage +# Description: 'Nickname for the device. rs-storage::volume uses this for the filesystem +# label, which is restricted to 12 characters. If longer than 12 characters, the +# filesystem label will be set to the first 12 characters. Example: data_storage' +# Input Type: single +# Required: true +# Advanced: false +# Attachments: +# - rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +# ... + +set -x +set -e + +# https://github.com/berkshelf/berkshelf-api/issues/112 +export LC_CTYPE=en_US.UTF-8 + +if [ ! -e /usr/bin/chef-client ]; then + curl -L https://www.opscode.com/chef/install.sh | sudo bash -s -- -v 12.19.36 +fi + +HOME=/home/rightscale +export PATH=${PATH}:/usr/local/sbin:/usr/local/bin + +/sbin/mkhomedir_helper rightlink + +export chef_dir=$HOME/.chef + +rm -rf $chef_dir +mkdir -p $chef_dir/chef-install +chmod -R 0777 $chef_dir/chef-install + +mkdir -p $chef_dir/cookbooks +chown -R 0777 $chef_dir/cookbooks + +#install packages when on ubuntu +if which apt-get >/dev/null 2>&1; then + apt-get -y update + apt-get install -y build-essential git #ruby2.0 ruby2.0-dev +fi + +#install packages for centos +if which yum >/dev/null 2>&1; then + yum groupinstall -y 'Development Tools' + yum install -y libxml2 libxml2-devel libxslt libxslt-devel git +fi + +#install berkshelf +/opt/chef/embedded/bin/gem install berkshelf -v '4.3.5' --no-ri --no-rdoc + +#checkout the chef server cookbook and install dependent cookbooks using berkshelf +cd $chef_dir + +# Download cookbooks from RS Attachments + +if [ -f $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar ]; then + tar -xvf $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +fi + +/opt/chef/embedded/bin/berks vendor $chef_dir/cookbooks + +#get instance data to pass to chef server +instance_data=$(/usr/local/bin/rsc --rl10 cm15 index_instance_session /api/sessions/instance) +instance_uuid=$(echo $instance_data | /usr/local/bin/rsc --x1 '.monitoring_id' json) +instance_id=$(echo $instance_data | /usr/local/bin/rsc --x1 '.resource_uid' json) + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi +# add the rightscale env variables to the chef runtime attributes +# http://docs.rightscale.com/cm/ref/environment_inputs.html +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + "tags": [] + }, + + "rightscale": { + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id" + }, + "rs-storage": { + "device": { + "mount_point":"$DEVICE_MOUNT_POINT", + "nickname":"$DEVICE_NICKNAME" + }, + "backup":{ + "lineage":"$STOR_BACKUP_LINEAGE", + "keep":{ + "dailies":"$BACKUP_KEEP_DAILIES", + "keep_last":"$BACKUP_KEEP_LAST", + "monthlies":"$BACKUP_KEEP_MONTHLIES", + "weeklies":"$BACKUP_KEEP_WEEKLIES", + "yearlies":"$BACKUP_KEEP_YEARLIES" + } + } + }, + + "run_list": ["recipe[rs-storage::backup]"] +} +EOF + +cat < $chef_dir/solo.rb +cookbook_path "$chef_dir/cookbooks" +data_bag_path "$chef_dir/data_bags" +EOF + +chef-solo -l info -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/Storage_Toolbox_Decommission-chef.sh b/chef-templates/jenkins/Storage_Toolbox_Decommission-chef.sh new file mode 100755 index 0000000..24411b8 --- /dev/null +++ b/chef-templates/jenkins/Storage_Toolbox_Decommission-chef.sh @@ -0,0 +1,92 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: Storage Toolbox Decommission - chef +# Inputs: +# DEVICE_COUNT: +# Category: Storage +# Description: "The number of devices to create and use in the Logical Volume. If +# this value is set to more than 1, it will create the specified number of devices +# and create an LVM on the devices.\r\n" +# Input Type: single +# Required: true +# Advanced: false +# DEVICE_DESTROY_ON_DECOMMISSION: +# Category: Storage +# Description: If set to true, the devices will be destroyed on decommission. +# Input Type: single +# Required: true +# Advanced: false +# Default: text:false +# Possible Values: +# - text:true +# - text:false +# DEVICE_MOUNT_POINT: +# Category: Storage +# Description: 'The mount point to mount the device on. Example: /var/lib/jenkins' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:/var/lib/jenkins +# DEVICE_NICKNAME: +# Category: Storage +# Description: 'Nickname for the device. rs-storage::volume uses this for the filesystem +# label, which is restricted to 12 characters. If longer than 12 characters, the +# filesystem label will be set to the first 12 characters. Example: data_storage' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:data_storage +# Attachments: +# - rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +# ... + +set -e + +HOME=/home/rightscale + +sudo /sbin/mkhomedir_helper rightlink + +export chef_dir=$HOME/.chef +mkdir -p $chef_dir + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi + +#get instance data to pass to chef server +instance_data=$(/usr/local/bin/rsc --rl10 cm15 index_instance_session /api/sessions/instance) +instance_uuid=$(echo $instance_data | /usr/local/bin/rsc --x1 '.monitoring_id' json) +instance_id=$(echo $instance_data | /usr/local/bin/rsc --x1 '.resource_uid' json) + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi +# add the rightscale env variables to the chef runtime attributes +# http://docs.rightscale.com/cm/ref/environment_inputs.html +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + "tags": [] + }, + + "rightscale": { + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id", + "decom_reason":"${DECOM_REASON}" + }, + + "rs-storage": { + "device":{ + "count":"$DEVICE_COUNT", + "destroy_on_decommission":"$DEVICE_DESTROY_ON_DECOMMISSION", + "mount_point":"$DEVICE_MOUNT_POINT", + "nickname":"$DEVICE_NICKNAME" + } + }, + + "run_list": ["recipe[rs-storage::decommission]"] +} +EOF + +chef-solo -l info -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/Storage_Toolbox_Schedule-chef.sh b/chef-templates/jenkins/Storage_Toolbox_Schedule-chef.sh new file mode 100755 index 0000000..6a7746a --- /dev/null +++ b/chef-templates/jenkins/Storage_Toolbox_Schedule-chef.sh @@ -0,0 +1,125 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: Storage Toolbox Schedule - chef +# Description: 'Enable/disable periodic backups ' +# Inputs: +# SCHEDULE_ENABLE: +# Category: Storage +# Description: Enable or disable periodic backup schedule +# Input Type: single +# Required: false +# Advanced: false +# Possible Values: +# - text:true +# - text:false +# SCHEDULE_HOUR: +# Category: Storage +# Description: 'The hour to schedule the backup on. This value should abide by crontab +# syntax. Use ''*'' for taking'' + '' backups every hour. Example: 23' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:23 +# SCHEDULE_MINUTE: +# Category: Storage +# Description: 'The minute to schedule the backup on. This value should abide by +# crontab syntax. Example: 30' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:15 +# Attachments: +# - rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +# ... + +set -x +set -e + +# https://github.com/berkshelf/berkshelf-api/issues/112 +export LC_CTYPE=en_US.UTF-8 + +if [ ! -e /usr/bin/chef-client ]; then + curl -L https://www.opscode.com/chef/install.sh | sudo bash -s -- -v 12.19.36 +fi + +HOME=/home/rightscale +export PATH=${PATH}:/usr/local/sbin:/usr/local/bin + +/sbin/mkhomedir_helper rightlink + +export chef_dir=$HOME/.chef + +rm -rf $chef_dir +mkdir -p $chef_dir/chef-install +chmod -R 0777 $chef_dir/chef-install + +mkdir -p $chef_dir/cookbooks +chown -R 0777 $chef_dir/cookbooks + +#install packages when on ubuntu +if which apt-get >/dev/null 2>&1; then + apt-get -y update + apt-get install -y build-essential git #ruby2.0 ruby2.0-dev +fi + +#install packages for centos +if which yum >/dev/null 2>&1; then + yum groupinstall -y 'Development Tools' + yum install -y libxml2 libxml2-devel libxslt libxslt-devel git +fi + +#install berkshelf +/opt/chef/embedded/bin/gem install berkshelf -v '4.3.5' --no-ri --no-rdoc + +#checkout the chef server cookbook and install dependent cookbooks using berkshelf +cd $chef_dir + +# Download cookbooks from RS Attachments + +if [ -f $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar ]; then + tar -xvf $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +fi + +/opt/chef/embedded/bin/berks vendor $chef_dir/cookbooks + +#get instance data to pass to chef server +instance_data=$(/usr/local/bin/rsc --rl10 cm15 index_instance_session /api/sessions/instance) +instance_uuid=$(echo $instance_data | /usr/local/bin/rsc --x1 '.monitoring_id' json) +instance_id=$(echo $instance_data | /usr/local/bin/rsc --x1 '.resource_uid' json) + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi + +# add the rightscale env variables to the chef runtime attributes +# http://docs.rightscale.com/cm/ref/environment_inputs.html +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + "tags": [] + }, + + "rightscale": { + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id" + }, + + "rs-storage": { + "schedule":{ + "enable":"$SCHEDULE_ENABLE", + "hour":$SCHEDULE_HOUR, + "minute":$SCHEDULE_MINUTE + } + }, + + "run_list": ["recipe[rs-storage::schedule]"] +} +EOF + +cat < $chef_dir/solo.rb +cookbook_path "$chef_dir/cookbooks" +data_bag_path "$chef_dir/data_bags" +EOF + +chef-solo -l info -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/Storage_Toolbox_Stripe-chef.sh b/chef-templates/jenkins/Storage_Toolbox_Stripe-chef.sh new file mode 100755 index 0000000..50ea0e4 --- /dev/null +++ b/chef-templates/jenkins/Storage_Toolbox_Stripe-chef.sh @@ -0,0 +1,205 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: Storage Toolbox Stripe - chef +# Description: Creates volumes, attaches them to the server, and sets up a striped LVM +# Inputs: +# DEVICE_IOPS: +# Category: Storage +# Description: 'IO Operations Per Second to use for the device. Currently this value +# is only used on AWS clouds. Example: 100' +# Input Type: single +# Required: false +# Advanced: false +# DEVICE_MOUNT_POINT: +# Category: Storage +# Description: 'The mount point to mount the device on. Example: /var/lib/jenkins' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:/var/lib/jenkins +# DEVICE_VOLUME_SIZE: +# Category: Storage +# Description: 'Size of the volume or logical volume to create (in GB). Example: +# 10' +# Input Type: single +# Required: true +# Advanced: false +# DEVICE_VOLUME_TYPE: +# Category: Storage +# Description: 'Volume Type to use for creating volumes. Example: gp2' +# Input Type: single +# Required: false +# Advanced: false +# STOR_RESTORE_LINEAGE: +# Category: Storage +# Description: 'The lineage name to restore backups. Example: staging' +# Input Type: single +# Required: false +# Advanced: false +# STOR_RESTORE_TIMESTAMP: +# Category: Storage +# Description: 'The timestamp (in seconds since UNIX epoch) to select a backup to +# restore from. The backup selected will have been created on or before this timestamp. +# Example: 1391473172' +# Input Type: single +# Required: false +# Advanced: false +# DEVICE_FILESYSTEM: +# Category: Storage +# Description: 'The filesystem to be used on the device. Defaults are based on OS +# and determined in attributes/defaults.rb. Example: ext4' +# Input Type: single +# Required: false +# Advanced: false +# Default: text:ext4 +# DEVICE_COUNT: +# Category: Storage +# Description: "The number of devices to create and use in the Logical Volume. If +# this value is set to more than 1, it will create the specified number of devices +# and create an LVM on the devices.\r\n" +# Input Type: single +# Required: true +# Advanced: false +# Default: text:2 +# DEVICE_NICKNAME: +# Category: Storage +# Description: 'Nickname for the device. rs-storage::volume uses this for the filesystem +# label, which is restricted to 12 characters. If longer than 12 characters, the +# filesystem label will be set to the first 12 characters. Example: data_storage' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:data_storage +# Attachments: +# - rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +# ... + +set -x +set -e + +HOME=/home/rightscale +export PATH=${PATH}:/usr/local/sbin:/usr/local/bin + +# https://github.com/berkshelf/berkshelf-api/issues/112 +export LC_CTYPE=en_US.UTF-8 + +if [ ! -e /usr/bin/chef-client ]; then + curl -L https://www.opscode.com/chef/install.sh | sudo bash -s -- -v 12.19.36 +fi + +sudo /sbin/mkhomedir_helper rightlink + +export chef_dir=$HOME/.chef +mkdir -p $chef_dir + +rm -rf $chef_dir +mkdir -p $chef_dir/chef-install +chmod -R 0777 $chef_dir/chef-install + +mkdir -p $chef_dir/cookbooks +chown -R 0777 $chef_dir/cookbooks + +#install packages when on ubuntu +if which apt-get >/dev/null 2>&1; then + apt-get -y update + apt-get install -y build-essential git #ruby2.0 ruby2.0-dev +fi + +#install packages for centos +if which yum >/dev/null 2>&1; then + yum groupinstall -y 'Development Tools' + yum install -y libxml2 libxml2-devel libxslt libxslt-devel git +fi + + +#install berkshelf +/opt/chef/embedded/bin/gem install berkshelf -v '4.3.5' --no-ri --no-rdoc + +#checkout the chef server cookbook and install dependent cookbooks using berkshelf +cd $chef_dir + +# Download cookbooks from RS Attachments +if [ -f $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar ]; then + tar -xvf $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +fi + +/opt/chef/embedded/bin/berks vendor $chef_dir/cookbooks + +#get instance data to pass to chef server +instance_data=$(/usr/local/bin/rsc --rl10 cm15 index_instance_session /api/sessions/instance) +instance_uuid=$(echo $instance_data | /usr/local/bin/rsc --x1 '.monitoring_id' json) +instance_id=$(echo $instance_data | /usr/local/bin/rsc --x1 '.resource_uid' json) + +device_iops='' +if [ -n "$DEVICE_IOPS" ];then + device_iops="\"iops\":\"$DEVICE_IOPS\"," +fi + +device_filesystem='' +if [ -n "$DEVICE_FILESYSTEM" ];then + device_filesystem="\"filesystem\":\"$DEVICE_FILESYSTEM\"," +fi +device_volume_type='' +if [ -n "$DEVICE_VOLUME_TYPE" ];then + device_volume_type="\"volume_type\":\"$DEVICE_VOLUME_TYPE\"" +fi + +restore_lineage='' +if [ -n "$STOR_RESTORE_LINEAGE" ];then + restore_lineage="\"lineage\":\"$STOR_RESTORE_LINEAGE\"$comma" +fi + +restore_timestamp='' +comma="" +if [ -n "$STOR_RESTORE_TIMESTAMP" ];then + comma="," + restore_timestamp="\"timestamp\":\"$STOR_RESTORE_TIMESTAMP\"" +fi + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi +# add the rightscale env variables to the chef runtime attributes +# http://docs.rightscale.com/cm/ref/environment_inputs.html +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + "tags": [] + }, + + "rightscale": { + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id" + }, + "apt":{"compile_time_update":true}, + "build-essential":{"compile_time":true}, + "rs-storage": { + "device":{ + "count":"$DEVICE_COUNT", + $device_filesystem + $device_iops + $device_volume_type + "mount_point":"$DEVICE_MOUNT_POINT", + "nickname":"$DEVICE_NICKNAME", + "volume_size":"$DEVICE_VOLUME_SIZE" + + }, + "restore":{ + $restore_lineage$comma + $restore_timestamp + } + + }, + + "run_list": ["recipe[apt]","recipe[build-essential]", + "recipe[rs-storage::default]","recipe[rs-storage::stripe]"] +} +EOF + +cat < $chef_dir/solo.rb +cookbook_path "$chef_dir/cookbooks" +data_bag_path "$chef_dir/data_bags" +EOF + +chef-solo -l info -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/Storage_Toolbox_Volume-chef.sh b/chef-templates/jenkins/Storage_Toolbox_Volume-chef.sh new file mode 100755 index 0000000..02634ca --- /dev/null +++ b/chef-templates/jenkins/Storage_Toolbox_Volume-chef.sh @@ -0,0 +1,197 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: Storage Toolbox Volume - chef +# Description: 'Creates a volume and attaches it to the server ' +# Inputs: +# DEVICE_IOPS: +# Category: Storage +# Description: 'IO Operations Per Second to use for the device. Currently this value +# is only used on AWS clouds. Example: 100' +# Input Type: single +# Required: false +# Advanced: false +# DEVICE_MOUNT_POINT: +# Category: Storage +# Description: 'The mount point to mount the device on. Example: /var/lib/jenkins' +# Input Type: single +# Required: true +# Advanced: false +# Default: text:/var/lib/jenkins +# DEVICE_NICKNAME: +# Category: Storage +# Description: 'Nickname for the device. rs-storage::volume uses this for the filesystem +# label, which is restricted to 12 characters. If longer than 12 characters, the +# filesystem label will be set to the first 12 characters. Example: data_storage' +# Input Type: single +# Required: true +# Advanced: false +# DEVICE_VOLUME_SIZE: +# Category: Storage +# Description: "Size of the volume or logical volume to create (in GB). Example: +# 10\r\n" +# Input Type: single +# Required: true +# Advanced: false +# DEVICE_VOLUME_TYPE: +# Category: Storage +# Description: "Volume Type to use for creating volumes. Example: gp2\r\n" +# Input Type: single +# Required: false +# Advanced: false +# STOR_RESTORE_LINEAGE: +# Category: Storage +# Description: 'The lineage name to restore backups. Example: staging' +# Input Type: single +# Required: false +# Advanced: false +# STOR_RESTORE_TIMESTAMP: +# Category: Storage +# Description: 'The timestamp (in seconds since UNIX epoch) to select a backup to +# restore from. The backup selected will have been created on or before this timestamp. +# Example: 1391473172' +# Input Type: single +# Required: false +# Advanced: false +# DEVICE_FILESYSTEM: +# Category: Storage +# Description: 'The filesystem to be used on the device. Defaults are based on OS +# and determined in attributes/defaults.rb. Example: ext4' +# Input Type: single +# Required: false +# Advanced: false +# Default: text:ext4 +# Attachments: +# - rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +# ... + +set -x +set -e + +# https://github.com/berkshelf/berkshelf-api/issues/112 +export LC_CTYPE=en_US.UTF-8 + +if [ ! -e /usr/bin/chef-client ]; then + curl -L https://www.opscode.com/chef/install.sh | sudo bash -s -- -v 12.19.36 +fi + +HOME=/home/rightscale +export PATH=${PATH}:/usr/local/sbin:/usr/local/bin + +/sbin/mkhomedir_helper rightlink + +export chef_dir=$HOME/.chef + +rm -rf $chef_dir +mkdir -p $chef_dir/chef-install +chmod -R 0777 $chef_dir/chef-install + +mkdir -p $chef_dir/cookbooks +chown -R 0777 $chef_dir/cookbooks + +#install packages when on ubuntu +if which apt-get >/dev/null 2>&1; then + apt-get -y update + apt-get install -y build-essential git #ruby2.0 ruby2.0-dev +fi + +#install packages for centos +if which yum >/dev/null 2>&1; then + yum groupinstall -y 'Development Tools' + yum install -y libxml2 libxml2-devel libxslt libxslt-devel git +fi + +#install berkshelf +/opt/chef/embedded/bin/gem install berkshelf -v '4.3.5' --no-ri --no-rdoc + +#checkout the chef server cookbook and install dependent cookbooks using berkshelf +cd $chef_dir + +# Download cookbooks from RS Attachments + +if [ -f $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar ]; then + tar -xvf $RS_ATTACH_DIR/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar +fi + +/opt/chef/embedded/bin/berks vendor $chef_dir/cookbooks + +# get instance data to pass to chef server +instance_data=$(/usr/local/bin/rsc --rl10 cm15 index_instance_session /api/sessions/instance) +instance_uuid=$(echo $instance_data | /usr/local/bin/rsc --x1 '.monitoring_id' json) +instance_id=$(echo $instance_data | /usr/local/bin/rsc --x1 '.resource_uid' json) + +device_iops='' +if [ -n "$DEVICE_IOPS" ];then + device_iops="\"iops\":\"$DEVICE_IOPS\"," +fi + +device_volume_type='' +if [ -n "$DEVICE_VOLUME_TYPE" ];then + device_volume_type="\"volume_type\":\"$DEVICE_VOLUME_TYPE\"," +fi + +device_filesystem='' +if [ -n "$DEVICE_FILESYSTEM" ];then + device_filesystem="\"filesystem\":\"$DEVICE_FILESYSTEM\"," +fi + +restore_lineage='' +if [ -n "$STOR_RESTORE_LINEAGE" ];then + restore_lineage="\"lineage\":\"$STOR_RESTORE_LINEAGE\"$comma" +fi + +restore_timestamp='' +comma="" +if [ -n "$STOR_RESTORE_TIMESTAMP" ];then + comma="," + restore_timestamp="\"timestamp\":\"$STOR_RESTORE_TIMESTAMP\"" +fi + +if [ -e $chef_dir/chef.json ]; then + rm -f $chef_dir/chef.json +fi +# add the rightscale env variables to the chef runtime attributes +# http://docs.rightscale.com/cm/ref/environment_inputs.html +cat < $chef_dir/chef.json +{ + "name": "${HOSTNAME}", + "normal": { + "tags": [] + }, + + "apt":{"compile_time_update":true}, + "build-essential":{"compile_time":true}, + + "rightscale": { + "instance_uuid":"$instance_uuid", + "instance_id":"$instance_id" + }, + + "rs-storage": { + "device":{ + $device_filesystem + $device_iops + "mount_point":"$DEVICE_MOUNT_POINT", + "nickname":"$DEVICE_NICKNAME", + $device_volume_type + "volume_size":"$DEVICE_VOLUME_SIZE" + }, + "restore":{ + $restore_lineage$comma + $restore_timestamp + } + + }, + + "run_list": ["recipe[apt]","recipe[build-essential]", + "recipe[rs-storage::default]","recipe[rs-storage::volume]"] +} +EOF + +cat < $chef_dir/solo.rb +cookbook_path "$chef_dir/cookbooks" +data_bag_path "$chef_dir/data_bags" +EOF + +/sbin/mkhomedir_helper rightlink + +chef-solo -l info -L /var/log/chef.log -j $chef_dir/chef.json -c $chef_dir/solo.rb diff --git a/chef-templates/jenkins/Update_R53_A_Record.sh b/chef-templates/jenkins/Update_R53_A_Record.sh new file mode 100755 index 0000000..f72def8 --- /dev/null +++ b/chef-templates/jenkins/Update_R53_A_Record.sh @@ -0,0 +1,65 @@ +#! /usr/bin/sudo /bin/bash +# --- +# RightScript Name: Update R53 A Record +# Description: Updates an R53 A record +# Inputs: +# R53_HOSTED_ZONE_ID: +# Category: R53 +# Description: Hosted Zone ID +# Input Type: single +# Required: true +# Advanced: false +# R53_TTL: +# Category: R53 +# Description: A record TTL +# Input Type: single +# Required: true +# Advanced: true +# Default: text:60 +# FQDN: +# Category: R53 +# Description: FQDN of the instance. (e.g. myserver.example.tld) +# Input Type: single +# Required: true +# Advanced: false +# IPADDRESS: +# Category: R53 +# Description: IP Address of the instance. +# Input Type: single +# Required: true +# Advanced: true +# Default: env:PRIVATE_IP +# Attachments: [] +# ... +echo "Installing awscli" +apt-get install -y awscli + +cat << EOF > /tmp/r53.json +{ + "Comment": "optional comment about the changes in this change batch request", + "Changes": [ + { + "Action": "UPSERT", + "ResourceRecordSet": { + "Name": "$FQDN", + "Type": "A", + "TTL": $R53_TTL, + "ResourceRecords": [ + { + "Value": "$IPADDRESS" + } + ] + } + } + ] +} +EOF + +aws route53 --region us-west-2 change-resource-record-sets --hosted-zone-id $R53_HOSTED_ZONE_ID --change-batch file:///tmp/r53.json +if [ $? -eq 0 ]; then + echo "$FQDN A record set to $IPADDRESS" +else + echo "Error setting $FQDN" + exit 1 +fi +rm -rf /tmp/r53.json diff --git a/chef-templates/jenkins/attachments/chef b/chef-templates/jenkins/attachments/chef new file mode 100644 index 0000000..4ffa342 --- /dev/null +++ b/chef-templates/jenkins/attachments/chef @@ -0,0 +1,8 @@ +/var/log/chef.log { + rotate 14 + daily + size 256M + compress + missingok + notifempty +} diff --git a/chef-templates/jenkins/attachments/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar b/chef-templates/jenkins/attachments/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar new file mode 100644 index 0000000..1f5e1e8 Binary files /dev/null and b/chef-templates/jenkins/attachments/rs-storage-d669f08f87743e072eba619a3fba6a9c9dd6bc89.tar differ diff --git a/chef-templates/jenkins/attachments/rsc_jenkins-201704183.tar b/chef-templates/jenkins/attachments/rsc_jenkins-201704183.tar new file mode 100644 index 0000000..4820ac8 Binary files /dev/null and b/chef-templates/jenkins/attachments/rsc_jenkins-201704183.tar differ diff --git a/chef-templates/jenkins/jenkins.cat.rb b/chef-templates/jenkins/jenkins.cat.rb new file mode 100644 index 0000000..053ee56 --- /dev/null +++ b/chef-templates/jenkins/jenkins.cat.rb @@ -0,0 +1,129 @@ +name "Jenkins Master and Slaves" +rs_ca_ver 20161221 +short_description "Jenkins Master and Slave cluster" + +parameter "param_slave_count" do + label "Jenkins Slave Count" + type "string" + operations [ "operation_set_slave_count", "launch" ] +end + +resource "server_1", type: "server" do + name "jenkins-master" + cloud "EC2 us-east-1" + instance_type "t2.large" + ssh_key_href "/api/clouds/1/ssh_keys/13FIKG64LL5SG" + subnet_hrefs "/api/clouds/1/subnets/C2II06OI99TMO" + security_group_hrefs [ "/api/clouds/1/security_groups/ATG27T4SM9AOL" ] + server_template find("stefhen-jenkins-master") +end + +resource "server_array_1", type: "server_array" do + name "jenkins-slaves" + cloud "EC2 us-east-1" + instance_type "t2.large" + ssh_key_href "/api/clouds/1/ssh_keys/13FIKG64LL5SG" + subnet_hrefs "/api/clouds/1/subnets/C2II06OI99TMO" + security_group_hrefs [ "/api/clouds/1/security_groups/ATG27T4SM9AOL" ] + server_template find("stefhen-jenkins-slave") + state "enabled" + array_type "alert" + elasticity_params do { + "bounds" => { + "min_count" => $param_slave_count, + "max_count" => 20 + }, + "pacing" => { + "resize_calm_time" => 5, + "resize_down_by" => 1, + "resize_up_by" => 1 + }, + "alert_specific_params" => { + "decision_threshold" => 51, + "voters_tag_predicate" => "jenkins-slave" + } + } end +end + +operation "launch" do + description "Launch the application" + definition "generated_launch" + output_mappings do { + $output_jenkins_master_ip => join(["http://", @server_1.public_ip_address, ":8080/"]) + } end +end + +operation "stop" do + description "Bring the array size to 0" + definition "disable_and_shrink_array" +end + +operation "start" do + description "Enable array" + definition "enable_array" +end + +operation "operation_set_slave_count" do + description "Sets the number of slaves to the provided parameter" + definition "set_slave_count" +end + +operation "operation_launch_slave" do + description "Manually adds one Jenkins slave" + definition "launch_slave" +end + +output "output_jenkins_master_ip" do + label "Jenkins" + description "Jenkins Master IP Address" + +end + +define wait_for_array_to_reach_size(@array, $size) do + sub task_name: "wait for array to reach size", timeout: 2h do + sleep_until(size(@array.current_instances()) == $size) + end +end + +parameter "param_vol_size" do + label "Jenkins Data Volume Size" + type "string" + operations "launch" +end + +define generated_launch(@server_1, @server_array_1, $param_slave_count, $param_vol_size) return @server_1 do + $inp = { + "DESCRIPTION": "text:Jenkins Slaves", + "MASTER_IP": "env:jenkins-master:PRIVATE_IP", + "NAME": "text:jenkins-slaves", + "STOR_BACKUP_LINEAGE": "text:jenkins_backup", + "DEVICE_VOLUME_SIZE": "text:" + $param_vol_size + } + + @@deployment.multi_update_inputs(inputs: $inp) + + concurrent do + provision(@server_1) + provision(@server_array_1) + end +end + +define disable_and_shrink_array(@server_array_1) return @server_array_1 do + @server_array_1.update(server_array: { state: "disabled" }) + @server_array_1.multi_terminate() + call wait_for_array_to_reach_size(@server_array_1, 0) +end + +define enable_array(@server_array_1) return @server_array_1 do + $desired_size = @server_array_1.elasticity_params["bounds"]["min_count"] + @server_array_1.update(server_array: { state: "enabled" }) + call wait_for_array_to_reach_size(@server_array_1, $desired_size) +end + +define set_slave_count(@server_array_1, $param_slave_count) do + @server_array_1.update(server_array: { elasticity_params: { bounds: { min_count: $param_slave_count } } } ) +end + +define launch_slave(@server_array_1) do + @server_array_1.launch() +end