From 157db04bc4d906ddb5330b34cb9a6e384845057b Mon Sep 17 00:00:00 2001 From: Mike Greiling Date: Sun, 8 Feb 2015 04:54:43 -0600 Subject: [PATCH] initial commit --- .gitignore | 7 + Rakefile | 71 ++++++++ build/.gitignore | 1 + opsworks/Vagrantfile.template | 11 ++ opsworks/client.yml | 16 ++ opsworks/opsworks | 51 ++++++ opsworks/opsworks.rb | 294 ++++++++++++++++++++++++++++++++++ opsworks/pre_config.yml | 61 +++++++ preseed/preseed.cfg | 31 ++++ provision/cleanup.sh | 41 +++++ provision/minimize.sh | 43 +++++ provision/network.sh | 17 ++ provision/opsworks.sh | 73 +++++++++ provision/update.sh | 7 + provision/vagrant.sh | 28 ++++ provision/vmtools.sh | 37 +++++ template/ubuntu1204.json | 106 ++++++++++++ template/ubuntu1404.json | 106 ++++++++++++ 18 files changed, 1001 insertions(+) create mode 100644 .gitignore create mode 100644 Rakefile create mode 100644 build/.gitignore create mode 100644 opsworks/Vagrantfile.template create mode 100755 opsworks/client.yml create mode 100644 opsworks/opsworks create mode 100644 opsworks/opsworks.rb create mode 100755 opsworks/pre_config.yml create mode 100644 preseed/preseed.cfg create mode 100755 provision/cleanup.sh create mode 100644 provision/minimize.sh create mode 100644 provision/network.sh create mode 100755 provision/opsworks.sh create mode 100644 provision/update.sh create mode 100755 provision/vagrant.sh create mode 100755 provision/vmtools.sh create mode 100644 template/ubuntu1204.json create mode 100644 template/ubuntu1404.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f3f117 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +packer_cache/ +.vagrant/ +*.box +*.iso +*.tar.gz +output-* \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..5d42521 --- /dev/null +++ b/Rakefile @@ -0,0 +1,71 @@ + +# first is default +BOXES = %w( ubuntu1404 ubuntu1204 ).freeze + +# namespace for each provider +provider_builder = lambda do |provider| + namespace provider do + desc "Build a box for #{provider} using packer (default: #{BOXES.first})" + task :build, [:box] do | t, args| build_box box_arg(args), provider; end + + desc "Install a #{provider} box with vagrant (default: #{BOXES.first})" + task :install, [:box] do | t, args| install_box box_arg(args), provider; end + + desc "Remove an installed vagrant box for #{provider} (default: #{BOXES.first})" + task :remove, [:box] do | t, args| remove_box box_arg(args), provider; end + + desc "Build all boxes for #{provider} using packer" + task 'build-all' do BOXES.each { |box| build_box box, provider }; end + + desc "Install all #{provider} boxes with vagrant" + task 'install-all' do BOXES.each { |box| install_box box, provider }; end + + desc "Remove all #{provider} boxes from vagrant" + task 'remove-all' do BOXES.each { |box| remove_box box, provider }; end + end +end + +provider_builder.call(:virtualbox) +provider_builder.call(:vmware) + +desc "Remove compiled assets and cached files" +task :clean do + sh 'rm -f build/*.box && rm -rf packer_cache' +end + +# shortcuts to virtualbox tasks with no namespace +task :build, [:box] => 'virtualbox:build' +task :install, [:box] => 'virtualbox:install' +task :remove, [:box] => 'virtualbox:remove' + +# build a box for the specified provider +def build_box(box, provider) + log "Building #{box} for #{provider}" + sh "rm -f build/#{box}-opsworks-#{provider}.box && packer build -only=#{provider}-iso template/#{box}.json" +end + +# build a box with vagrant +def install_box(box, provider) + file = "build/#{box}-opsworks-#{provider}.box" + build_box(box, provider) unless File.exists?(file) + + log "Installing #{box}-opsworks for #{provider} with vagrant" + sh "vagrant box add #{file} --force --name=#{box}-opsworks" +end + +# remove a box from vagrant +def remove_box(box, provider) + log "Removing #{box} for #{provider}" + provider = 'vmware_desktop' if provider.to_s == 'vmware' + sh "vagrant box remove #{box} --provider=#{provider}" +end + +# validate input for tasks with optional parameters +def box_arg(args) + abort "Invalid box provided" if args[:box] && !BOXES.include?(args[:box]) + args[:box] || BOXES.first +end + +def log(msg) + puts "==> Rake: #{msg}" +end diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..621099e --- /dev/null +++ b/build/.gitignore @@ -0,0 +1 @@ +*.box \ No newline at end of file diff --git a/opsworks/Vagrantfile.template b/opsworks/Vagrantfile.template new file mode 100644 index 0000000..9c8ed37 --- /dev/null +++ b/opsworks/Vagrantfile.template @@ -0,0 +1,11 @@ + +# require vagrant version 1.7.0 to allow for the named provisioners syntax +Vagrant.require_version(">= 1.7.0") +box_root = File.dirname(__FILE__) + +Vagrant.configure("2") do |config| + config.vm.synced_folder box_root, '/tmp/vagrant-opsworks' + config.vm.provision 'opsworks', type: 'shell' do |shell| + shell.inline = '/bin/bash /tmp/vagrant-opsworks/opsworks "$@"' + end +end diff --git a/opsworks/client.yml b/opsworks/client.yml new file mode 100755 index 0000000..4010476 --- /dev/null +++ b/opsworks/client.yml @@ -0,0 +1,16 @@ +# opsworks chef client config file +# /var/lib/aws/opsworks/client.yml +--- + +:local_mode: true +:log_level: :info +:ohai_plugin_path: /opt/aws/opsworks/current/plugins +:cookbook_path: ["/opt/aws/opsworks/current/merged-cookbooks"] +:default_cookbooks_path: /opt/aws/opsworks/current/cookbooks +:site_cookbooks_path: /opt/aws/opsworks/current/site-cookbooks +:merged_cookbooks_path: /opt/aws/opsworks/current/merged-cookbooks +:berkshelf_cookbooks_path: /opt/aws/opsworks/current/berkshelf-cookbooks +:berkshelf_cache_path: /var/lib/aws/opsworks/berkshelf_cache +:file_cache_path: /var/lib/aws/opsworks/cache +:search_nodes_path: /var/lib/aws/opsworks/data/nodes +:data_bag_path: /var/lib/aws/opsworks/data/data_bags diff --git a/opsworks/opsworks b/opsworks/opsworks new file mode 100644 index 0000000..c278bf8 --- /dev/null +++ b/opsworks/opsworks @@ -0,0 +1,51 @@ +#!/bin/bash + +# exit if no arguments provided +if [[ $# -eq 0 ]]; then + echo "Nothing to do..." + exit +fi + +# install opsworks agent if not already available +if [[ ! -f /opt/aws/opsworks/current/VERSION ]]; then + + # decide which agent version to install + if [[ -z $OPSWORKS_AGENT_VERSION ]]; then + OPSWORKS_AGENT_VERSION='33200020141203204624' + fi + + # install dependencies + echo "Installing OpsWorks agent dependencies" + apt-get -yq update + apt-get -yq install curl libxml2-dev libxslt-dev libyaml-dev + + # create expected directories and populate them with expected config files + mkdir -p /{etc,opt,var/{log,lib}}/aws/opsworks/ /var/lib/cloud/ + cp $(dirname "${BASH_SOURCE[0]}")/{client.yml,pre_config.yml} /var/lib/aws/opsworks/ + + # create a temporary directory to work in + TMP_DIR=$(mktemp -d "opsworks-installer.XXXXXXXX") + trap 'rm -rf "$TMP_DIR"' EXIT + pushd $TMP_DIR + + # download and install the requested opsworks agent + echo "Installing OpsWorks agent version $OPSWORKS_AGENT_VERSION" + wget -nv -O opsworks-agent-installer.tgz https://opsworks-instance-agent.s3.amazonaws.com/$OPSWORKS_AGENT_VERSION/opsworks-agent-installer.tgz + tar -xzpof opsworks-agent-installer.tgz + cd opsworks-agent-installer/opsworks-agent/bin/ + ./installer_wrapper.sh -R opsworks-instance-assets.s3.amazonaws.com + + # return to our previous location and destroy our temporary directory + popd > /dev/null + rm -rf "$TMP_DIR" + trap - EXIT + + # ensure we can access the opsworks chef gem + echo "export PATH=\$PATH:/opt/aws/opsworks/current/bin" > /etc/profile.d/opsworks_path.sh +fi + +# set the vagrant shared directory as cwd if available +[ -d /vagrant ] && cd /vagrant + +# run our provisioner +env /opt/aws/opsworks/local/bin/ruby $(dirname "${BASH_SOURCE[0]}")/opsworks.rb "$@" \ No newline at end of file diff --git a/opsworks/opsworks.rb b/opsworks/opsworks.rb new file mode 100644 index 0000000..f5626bf --- /dev/null +++ b/opsworks/opsworks.rb @@ -0,0 +1,294 @@ +require 'date' +require 'json' +require 'tmpdir' +require 'fileutils' + +module OpsWorks + + DNA_BASE = { + "ssh_users" => { + "1000" => { + "name" => "vagrant", + "public_key" => nil, + "sudoer" => true + } + }, + "dependencies" => { + "gem_binary" => "/usr/local/bin/gem", + "gems" => {}, + "debs" => {} + }, + "ebs" => { + "devices" => {}, + "raids" => {} + }, + "opsworks" => { + "activity" => "setup_and_deploy", + "valid_client_activities" => ["setup_and_deploy"], + "agent_version" => 0, + "ruby_version" => "2.0.0", + "ruby_stack" => "ruby", + "rails_stack" => { + "name" => nil + }, + "stack" => { + "name" => "Vagrant Stack", + "elb-load-balancers" => [], + "rds_instances" => [] + }, + "layers" => {}, + "instance" => { + "ip" => "127.0.0.1", + "private_ip" => "127.0.0.1", + "layers" => [] + } + }, + "deploy" => {}, + "opsworks_rubygems" => { + "version" => "2.2.2" + }, + "opsworks_bundler" => { + "version" => "1.5.3", + "manage_package" => nil + }, + "opsworks_custom_cookbooks" => { + "enabled" => false, + "scm" => { + "type" => "git", + "repository" => nil, + "user" => nil, + "password" => nil, + "revision" => nil, + "ssh_key" => nil + }, + "manage_berkshelf" => nil, + "recipes" => [] + }, + "chef_environment" => "_default", + "recipes" => [ + "opsworks_custom_cookbooks::load", + "opsworks_custom_cookbooks::execute" + ] + } + + DNA_DEPLOY_BASE = { + "deploy_to" => nil, + "application" => nil, + "deploying_user" => nil, + "domains" => [], + "application_type" => nil, + "mounted_at" => nil, + "rails_env" => nil, + "ssl_support" => false, + "ssl_certificate" => nil, + "ssl_certificate_key" => nil, + "ssl_certificate_ca" => nil, + "document_root" => nil, + "restart_command" => "echo 'restarting app'", + "sleep_before_restart" => 0, + "symlink_before_migrate" => {}, + "symlinks" => {}, + "database" => {}, + "migrate" => false, + "auto_bundle_on_deploy" => true, + "scm" => { + "scm_type" => "git", + "repository" => nil, + "revision" => nil, + "ssh_key" => nil, + "user" => nil, + "password" => nil + } + } + + def self.provision(*args) + if agent_revision < Date.today.prev_month(4) + warn "Warning: OpsWorks agent version #{agent_version} is over four months old, consider updating..." + end + + log "Checking dependencies..." + check_dependencies + + log "Reading input..." + dna = compile_json expand_paths(args) + + log "Parsing deployments..." + dna['deploy'].each do |name, app| + + # if repo points to a local path, trick opsworks into receiving it as a git repo + if app['scm']['repository'] && app['scm']['repository'] !~ /^(?:[a-z]+:)?\/\//i + if !Dir.exist?(app['scm']['repository']) + raise "Local app '#{name}' could not be found at '#{app['scm']['repository']}'" + end + app['scm']['repository'] = prepare_deployment(app['scm']['repository']) + end + end + + log "Parsing custom cookbooks..." + if dna['opsworks_custom_cookbooks']['enabled'] + cookbooks = dna['opsworks_custom_cookbooks'] + + # if repo points to a local path, trick opsworks into receiving it as a git repo + if cookbooks['scm']['repository'] && cookbooks['scm']['repository'] !~ /^(?:[a-z]+:)?\/\//i + if !Dir.exist?(cookbooks['scm']['repository']) + raise "Local custom cookbooks could not be found at '#{cookbooks['scm']['repository']}'" + end + cookbooks['scm']['repository'] = prepare_deployment(cookbooks['scm']['repository']) + + # autodetect berkshelf support + if cookbooks['manage_berkshelf'].nil? + berksfile = cookbooks['scm']['repository'].sub(/[\/\\]+$/,'') + '/Berksfile' + cookbooks['manage_berkshelf'] = File.exist?(berksfile) + end + end + + # remove the local cache to force opsworks to update custom cookbooks + log "Purging local cookbooks cache from '/opt/aws/opsworks/current/site-cookbooks'..." + FileUtils.rm_rf('/opt/aws/opsworks/current/site-cookbooks/') + end + + if dna['opsworks']['instance']['hostname'] + log "Setting instance hostname..." + set_hostname dna['opsworks']['instance']['hostname'] + end + + # run some base recipes if none explicitly provided + if dna['opsworks_custom_cookbooks']['recipes'].empty? + dna['opsworks_custom_cookbooks']['recipes']= %w( + recipe[opsworks_initial_setup] + recipe[ssh_host_keys] + recipe[ssh_users] + recipe[dependencies] + recipe[ebs] + recipe[agent_version] + recipe[opsworks_stack_state_sync] + recipe[opsworks_cleanup] + ) + end + + # ensure we don't set the agent version to anything lower than the current version + dna['opsworks']['agent_version'] = [agent_version, dna['opsworks']['agent_version']].max + + log "Generating dna.json..." + dna_file = save_json_tempfile dna, 'dna.json' + + log "Running opsworks agent..." + + # AWS currently does not set UTF-8 as default encoding + system({"LANG" => "POSIX"}, "opsworks-agent-cli run_command -f #{dna_file}") + + rescue StandardError => e + warn "Error: #{e}" + exit false + end + + def self.save_json_tempfile(data, name) + tmp_dir = Dir.mktmpdir('vagrant-opsworks') + File.chmod(0755, tmp_dir) + + tmp_file = "#{tmp_dir}/#{name}" + File.open(tmp_file, 'w') { |f| f.write JSON.pretty_generate(data) } + File.chmod(0755, tmp_file) + + tmp_file + end + + def self.log(msg) + puts msg + end + + def self.check_dependencies + `apt-get -yq install git 2>&1` if `which git`.empty? + end + + def self.set_hostname(hostname) + if !File.readlines('/etc/hosts').grep(/(?=[^\.\w-]|$)#{hostname}(?=[^\.\w-]|$)/).any? + File.open('/etc/hosts', 'a') do |f| + f.puts "\n127.0.0.1\t#{hostname}.localdomain #{hostname}\n" + end + end + File.write('/etc/hostname', hostname) + system('hostname', hostname) + end + + def self.expand_paths(args) + files = [] + args.each do |file| + if File.exist?(file) + files << file + elsif file.include? '*' + files += Dir.glob(file) + else + raise "The file '#{file}' does not appear to exist." + end + end + files + end + + def self.compile_json(files) + # combine all json files into one hash, starting with our base hash to + # provide some sensible defaults + dna = files.reduce(DNA_BASE) do |dna, file| + log "Processing '#{file}'..." + json = File.read(file).strip || '{}' + json = JSON.parse(json) + deep_merge(dna, json) + end + + # ensure each layer has some required fields including instances with both + # private and public ip addresses + dna['opsworks']['layers'].each do |name, layer| + next unless Hash === layer + layer['name'] ||= name + layer['elb-load-balancers'] ||= [] + layer['instances'] ||= {} + + next unless Hash === layer['instances'] + layer['instances'].each do |name, instance| + next unless Hash === instance + instance['private_ip'] ||= instance['ip'] + instance['ip'] ||= instance['private_ip'] + end + end + + # merge some default values into each app definition + dna['deploy'].each do |name, app| + app.replace deep_merge(DNA_DEPLOY_BASE, app) + app['application'] ||= name + end + + dna + rescue JSON::ParserError => e + raise "The file '#{file}' does not appear to be valid JSON. (error: #{e})" + end + + def self.prepare_deployment(path) + tmp_dir = Dir.mktmpdir('vagrant-opsworks') + File.chmod(0755, tmp_dir) + FileUtils.cp_r("#{path}/.", tmp_dir) + Dir.chdir(tmp_dir) do + `find . -name '.git*' -exec rm -rf {} \\; 2>&1; git init; git add .; git -c user.name='Vagrant' -c user.email=none commit -m 'Create temporary repository for deployment.'` + end + tmp_dir + end + + def self.deep_merge(a, b) + a.merge(b) { |_, a, b| Hash === a && Hash === b ? deep_merge(a, b) : b } + end + + def self.agent_version + File.read('/opt/aws/opsworks/current/REVISION')[/\d\d\d\d\-\d\d-\d\d-\d\d:\d\d:\d\d (\d+)/, 1].to_i + end + + def self.agent_revision + date_string = File.read('/opt/aws/opsworks/current/REVISION')[/(\d\d\d\d\-\d\d-\d\d-\d\d:\d\d:\d\d)/, 1] + raise 'Unable to parse agent revision' unless date_string + DateTime.strptime date_string, '%Y-%m-%d-%H:%M:%S' + end +end + +# automatically run provisioner +if __FILE__ == $0 + STDOUT.sync = true + OpsWorks.provision *ARGV +end diff --git a/opsworks/pre_config.yml b/opsworks/pre_config.yml new file mode 100755 index 0000000..a5f3e18 --- /dev/null +++ b/opsworks/pre_config.yml @@ -0,0 +1,61 @@ +# opsworks-agent pre-config file +# /var/lib/aws/opsworks/pre_config.yml +--- + +:hostname: opsworks-vagrant +:identity: ffffffff-ffff-ffff-ffff-ffffffffffff +:agent_installer_base_url: opsworks-instance-agent.s3.amazonaws.com +:agent_installer_tgz: opsworks-agent-installer.tgz +:verbose: false +:instance_service_region: us-east-1 +:instance_service_endpoint: localhost +:instance_service_port: '65535' +:instance_public_key: | + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA30W1qY/kuT3H0QJAWaOs + 0tLI8E64OKKCsK6908NL8ReroxsYHZHjzl5+ZNw7YE4QENBZ4eWR4gbC2z/YT/5b + U6LrZZUCuIdSL+9cMyBUUOTEV1W1Z5EtZkB8PgoDm2+SSDnL9F6Tww+oiD27kA4n + xa68i6Y0+tGARAkC11tX7z7Yg0cfkLYXcY3weueXagbIdF+esxuVFXj+Jz/oyBwP + qU1lVWQ1b3ZYv2XPnmMuJPfdENI/0YHkXm1JMhdseEUPS3TuZalqpDIbG/9m1ZMB + GiH5UUshMm2GldDjjEnVV024K+WFi7ww76+pJcdZ8NcyLXVzLEH/CiwhnQigFzw3 + xQIDAQAB + -----END PUBLIC KEY----- +:instance_private_key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEA30W1qY/kuT3H0QJAWaOs0tLI8E64OKKCsK6908NL8ReroxsY + HZHjzl5+ZNw7YE4QENBZ4eWR4gbC2z/YT/5bU6LrZZUCuIdSL+9cMyBUUOTEV1W1 + Z5EtZkB8PgoDm2+SSDnL9F6Tww+oiD27kA4nxa68i6Y0+tGARAkC11tX7z7Yg0cf + kLYXcY3weueXagbIdF+esxuVFXj+Jz/oyBwPqU1lVWQ1b3ZYv2XPnmMuJPfdENI/ + 0YHkXm1JMhdseEUPS3TuZalqpDIbG/9m1ZMBGiH5UUshMm2GldDjjEnVV024K+WF + i7ww76+pJcdZ8NcyLXVzLEH/CiwhnQigFzw3xQIDAQABAoIBAQDF6Ji6kJ4BxU2V + axV3X6oVxlnvCRgqu4J08q+5QefS8VRm4+FgdK0lhIUtCjpnh0qeXNEPd9r0K2IV + zmYDokd5v3RBOvCKeQjVDKsBdqrGecHAWGzQPNOtS4PVyjKgWSmlc/XhyuPXh82v + 1minrKR8igL/Fnjny0STChnGo2Uy3y3X089VIWlkKskSTLe73jTN7EeI47+LHJul + ULiV8Oh7YbEHf4w6Fz+1LNxcCJg8rNYeJKrGbCBq/ImygiJp14+oTU2nCDQSBJqF + bdUPhtpQDlDsAsfgBavYb3fHc0ZBVbCwM84VSdktKz9QbINI2Ugh+lI201woORlZ + MCqClgwRAoGBAPs1vapbsNbb+zRIxKRKfOLcay3O9efPyUqOPIMmn3pbpxk0GEBg + KQXO5PZmDP25oLA3bLvowVMMgShY1WRAr7ReloIRHaARVo+If8c7G83nePycYZ3f + C/ztfzsKpBO2SVzUkQIzh6rogK/rKUa/FCwZ/Gnf93v/j5RyUmLWXoZrAoGBAOOH + l/yCaxdY5khc1uEbCQDzT9lqWKcgFjoVuWO7o4vallNIDnCDd7olGbR2oIWbJAtB + mKpXLIWVjcRPXGp9Kr07mvkL+/mO2m6tG/vFJDh5Jge/qoOK4d1KEb2y+PwK+XRK + /SkAEcT6qpLMdz5W/Oy/wBc8b/BeBJkSoF1/weaPAoGAZ3eGDBHB720htTI3k/d+ + Iq5okrCIhhcGMGgPMnGJBAuV8oKLbpBstRC3K2ly9lorfgkGBwth/QPMesLD+YvP + ErpWwXGtQw2BGpM9FeEZnaA2K815Q13oASAM5FOIqvnMk6iVpVN2EIW84zg3gwUW + mOeHGFCADZmAGMNRfZYPzssCgYAnCWB+JjIRc2MvDx6eyHCnBReyCZjkM5EcrhV8 + kvjgScR4zWgMzcGA4lSiraekxJVOiRaUQxiUYrBL+gG1E3x9svhHulKk4ml/i5u9 + enlYZxCrS6sJno5Z1RduIIKvW4Ko/SSqICTsUsVpIkNjIrGKPOvMEMZzyu2nBZcV + 85Fk6QKBgQDLfvbS010HhYuEoW9NtSgeSxaJaCTU82EURBoL3F1umPcc1+EmtvnE + WbjdSwiVT7tb8F/yM6wIGjHEaNdIPdIakHEPU4MfJFDe/+aPwk2M1MCCpbERgyJR + m4SlK1pWQsPZ6Em0C+bw+dnkThEXXb3kfWSxnuh0d7JjfX+Fd/8F6Q== + -----END RSA PRIVATE KEY----- +:charlie_public_key: | + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAni7eKdm34oaCvGiw96Fk + lyLX+aPfInYzilkk+AY3pXF6nijpQ2cm3ZeM2EoqZFTv3a/meosNBAs3Q3Sy1e4G + 7Ibn/xwMof+iSBvimx3PGKFzNP0BhY9yS6AMEMxtmqksHb0glwmFeJcomdhxZV1F + ziWTtL6ZEyvCg0I7rxGm1ceQmD25eK90VcZVh4LJtNfnwcZRM4eC+KK9Qllxw5hW + vB4Z52JMMZbEG9MYCLydWSY9rnVkAyQ0ngJUaJ3q7JsbkBV/J5BcrGgcbioR1k+h + INRoHwBQU9WnT/x8W+N6vwJb4o6v2hBR1H2GSDLwyZ7wC8EVH+XafWYpU1g/nSEe + aQIDAQAB + -----END PUBLIC KEY----- +:wait_between_runs: '60' diff --git a/preseed/preseed.cfg b/preseed/preseed.cfg new file mode 100644 index 0000000..18eb826 --- /dev/null +++ b/preseed/preseed.cfg @@ -0,0 +1,31 @@ +choose-mirror-bin mirror/http/proxy string +d-i base-installer/kernel/override-image string linux-server +d-i clock-setup/utc boolean true +d-i clock-setup/utc-auto boolean true +d-i finish-install/reboot_in_progress note +d-i grub-installer/only_debian boolean true +d-i grub-installer/with_other_os boolean true +d-i partman-auto-lvm/guided_size string max +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/method string lvm +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true +d-i partman-lvm/device_remove_lvm boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true +d-i partman/confirm_write_new_label boolean true +d-i passwd/user-fullname string vagrant +d-i passwd/user-uid string 900 +d-i passwd/user-password password vagrant +d-i passwd/user-password-again password vagrant +d-i passwd/username string vagrant +d-i pkgsel/include string openssh-server ntp curl nfs-common linux-headers-$(uname -r) build-essential perl dkms +d-i pkgsel/install-language-support boolean false +d-i pkgsel/update-policy select unattended-upgrades +d-i pkgsel/upgrade select full-upgrade +d-i time/zone string UTC +d-i user-setup/allow-password-weak boolean true +d-i user-setup/encrypt-home boolean false +tasksel tasksel/first multiselect standard, ubuntu-server \ No newline at end of file diff --git a/provision/cleanup.sh b/provision/cleanup.sh new file mode 100755 index 0000000..f7bfe0c --- /dev/null +++ b/provision/cleanup.sh @@ -0,0 +1,41 @@ +#!/bin/bash -eux + +# Force grub to boot without user interation +echo "GRUB_RECORDFAIL_TIMEOUT=10" >> /etc/default/grub +update-grub + +echo "==> Cleaning up tmp" +rm -rf /tmp/* + +# Remove Bash history +unset HISTFILE +rm -f /root/.bash_history +rm -f /home/vagrant/.bash_history + +# Clean up log files +find /var/log -type f | while read f; do echo -ne '' > $f; done; + +echo "==> Clearing last login information" +>/var/log/lastlog +>/var/log/wtmp +>/var/log/btmp + +# Whiteout root +count=$(df --sync -kP / | tail -n1 | awk -F ' ' '{print $4}') +let count-- +dd if=/dev/zero of=/tmp/whitespace bs=1024 count=$count +rm /tmp/whitespace + +# Whiteout /boot +count=$(df --sync -kP /boot | tail -n1 | awk -F ' ' '{print $4}') +let count-- +dd if=/dev/zero of=/boot/whitespace bs=1024 count=$count +rm /boot/whitespace + +# Zero out the free space to save space in the final image +dd if=/dev/zero of=/EMPTY bs=1M +rm -f /EMPTY + +# Make sure we wait until all the data is written to disk, otherwise +# Packer might quite too early before the large files are deleted +sync \ No newline at end of file diff --git a/provision/minimize.sh b/provision/minimize.sh new file mode 100644 index 0000000..f2657f7 --- /dev/null +++ b/provision/minimize.sh @@ -0,0 +1,43 @@ +#!/bin/bash -eux + +echo "==> Installed packages before cleanup" +dpkg --get-selections | grep -v deinstall + +# Remove some packages to get a minimal install +echo "==> Removing all linux kernels except the currrent one" +dpkg --list | awk '{ print $2 }' | grep 'linux-image-3.*-generic' | grep -v $(uname -r) | xargs apt-get -y purge +echo "==> Removing linux source" +dpkg --list | awk '{ print $2 }' | grep linux-source | xargs apt-get -y purge +# echo "==> Removing development packages" +# dpkg --list | awk '{ print $2 }' | grep -- '-dev$' | xargs apt-get -y purge +echo "==> Removing documentation" +dpkg --list | awk '{ print $2 }' | grep -- '-doc$' | xargs apt-get -y purge +echo "==> Removing X11 libraries" +apt-get -y purge libx11-data xauth libxmuu1 libxcb1 libx11-6 libxext6 +echo "==> Removing obsolete networking components" +apt-get -y purge ppp pppconfig pppoeconf +echo "==> Removing other oddities" +apt-get -y purge popularity-contest installation-report landscape-common wireless-tools wpasupplicant ubuntu-serverguide + +# Clean up the apt cache +apt-get -y autoremove --purge +apt-get -y autoclean +apt-get -y clean + +# Clean up orphaned packages with deborphan +apt-get -y install deborphan +while [ -n "$(deborphan --guess-all --libdevel)" ]; do + deborphan --guess-all --libdevel | xargs apt-get -y purge +done +apt-get -y purge deborphan dialog + +# echo "==> Removing man pages" +# rm -rf /usr/share/man/* +# echo "==> Removing APT files" +# find /var/lib/apt -type f | xargs rm -f +echo "==> Removing anything in /usr/src" +rm -rf /usr/src/* +echo "==> Removing any docs" +rm -rf /usr/share/doc/* +echo "==> Removing caches" +find /var/cache -type f -exec rm -rf {} \; \ No newline at end of file diff --git a/provision/network.sh b/provision/network.sh new file mode 100644 index 0000000..5d2cf94 --- /dev/null +++ b/provision/network.sh @@ -0,0 +1,17 @@ +#!/bin/bash -eux + +# Make sure udev does not block our network - http://6.ptmc.org/?p=164 +echo "==> Cleaning up udev rules" +rm -rf /dev/.udev/ +rm /lib/udev/rules.d/75-persistent-net-generator.rules + +if [ -d "/var/lib/dhcp" ]; then + echo "==> Cleaning up leftover dhcp leases" + rm /var/lib/dhcp/* +fi + +# Add delay to prevent "vagrant reload" from failing +echo "pre-up sleep 2" >> /etc/network/interfaces + +# Disable DNS reverse lookup +echo "UseDNS no" >> /etc/ssh/sshd_config \ No newline at end of file diff --git a/provision/opsworks.sh b/provision/opsworks.sh new file mode 100755 index 0000000..4487806 --- /dev/null +++ b/provision/opsworks.sh @@ -0,0 +1,73 @@ +#!/bin/bash -eux + +# set the agent version to be installed +AGENT_VERSION="33300020141216163306" + +echo "==> Generating chef json for first OpsWorks run" +TMPDIR=$(mktemp -d) && trap 'rm -rf "$TMPDIR"' EXIT +mkdir -p $TMPDIR/cookbooks + +# Create a base json file to execute some default recipes +cat < $TMPDIR/dna.json +{ + "opsworks_custom_cookbooks": { + "enabled": true, + "scm": { + "repository": "$TMPDIR/cookbooks" + }, + "manage_berkshelf": true, + "recipes": [ + "recipe[opsworks_initial_setup]", + "recipe[ssh_host_keys]", + "recipe[ssh_users]", + "recipe[mysql::client]", + "recipe[dependencies]", + "recipe[ebs]", + "recipe[opsworks_ganglia::client]", + "recipe[apt]", + "recipe[deploy::default]", + "recipe[opsworks_ganglia::configure-client]", + "recipe[agent_version]", + "recipe[opsworks_stack_state_sync]", + "recipe[opsworks_cleanup]" + ] + } +} +EOT + +# Use Berkshelf to pre-load some commonly-used community cookbooks +cat <> $TMPDIR/cookbooks/Berksfile +source "https://supermarket.getchef.com" + +# pre-load some opscode community cookbooks +cookbook "apt", ">= 2.0" +cookbook "apache2" +cookbook "aws" +cookbook "bluepill" +cookbook "build-essential" +cookbook "couchdb" +cookbook "cron" +cookbook "git" +cookbook "haproxy" +cookbook "memcached" +cookbook "mongodb" +cookbook "mysql" +cookbook "newrelic" +cookbook "nginx" +cookbook "nodejs" +cookbook "ohai" +cookbook "postgresql" +cookbook "php" +cookbook "php-fpm" +cookbook "python" +cookbook "redisio" +cookbook "rsyslog" +cookbook "runit" +cookbook "sysctl" +cookbook "yum" +cookbook "yum-epel" +EOT + +echo "==> Installing and running OpsWorks agent" +chmod +x /tmp/opsworks/opsworks +env OPSWORKS_AGENT_VERSION="$AGENT_VERSION" /tmp/opsworks/opsworks $TMPDIR/dna.json \ No newline at end of file diff --git a/provision/update.sh b/provision/update.sh new file mode 100644 index 0000000..6dcf25f --- /dev/null +++ b/provision/update.sh @@ -0,0 +1,7 @@ +#!/bin/bash -eux + +echo "==> Updating packages" + +sudo find /var/lib/apt/lists -type f -exec rm -v {} \; +apt-get -y update +apt-get -y upgrade \ No newline at end of file diff --git a/provision/vagrant.sh b/provision/vagrant.sh new file mode 100755 index 0000000..554c849 --- /dev/null +++ b/provision/vagrant.sh @@ -0,0 +1,28 @@ +#!/bin/bash -eux + +# https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub +VAGRANT_INSECURE_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" + +# Store build time +date > /etc/vagrant_box_build_time + +# Create Vagrant user (if not already present) +if ! id -u vagrant >/dev/null 2>&1; then + echo "==> Creating vagrant user" + /usr/sbin/groupadd vagrant + /usr/sbin/useradd vagrant -g vagrant -G sudo -d /home/vagrant --create-home + echo "vagrant:vagrant" | chpasswd +fi + +echo "==> Giving vagrant sudo powers" +echo "vagrant ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +echo "==> Installing vagrant key" +mkdir -p /home/vagrant/.ssh +chmod 700 /home/vagrant/.ssh +echo "${VAGRANT_INSECURE_KEY}" > /home/vagrant/.ssh/authorized_keys +chmod 600 /home/vagrant/.ssh/authorized_keys +chown -R vagrant:vagrant /home/vagrant/.ssh + +# Remove those annoying "stdin: is not a tty" messages when running vagrant +sed -i "s/mesg n/tty -s \&\& mesg n/g" /root/.profile \ No newline at end of file diff --git a/provision/vmtools.sh b/provision/vmtools.sh new file mode 100755 index 0000000..c682e0a --- /dev/null +++ b/provision/vmtools.sh @@ -0,0 +1,37 @@ +#!/bin/bash -eux + +if [ -f /home/vagrant/.vbox_version ]; then + echo "==> Installing VirtualBox guest additions" + # Assuming the following packages are installed + # apt-get install -y linux-headers-$(uname -r) build-essential perl + # apt-get install -y dkms + + VBOX_VERSION=$(cat /home/vagrant/.vbox_version) + mount -o loop /home/vagrant/VBoxGuestAdditions_$VBOX_VERSION.iso /mnt + sh /mnt/VBoxLinuxAdditions.run + umount /mnt + rm /home/vagrant/VBoxGuestAdditions_$VBOX_VERSION.iso + rm /home/vagrant/.vbox_version + + if [[ $VBOX_VERSION = "4.3.10" ]]; then + ln -s /opt/VBoxGuestAdditions-4.3.10/lib/VBoxGuestAdditions /usr/lib/VBoxGuestAdditions + fi +fi + +if [ -f /home/vagrant/vmware_tools.iso ]; then + echo "==> Installing VMware Tools" + # Assuming the following packages are installed + # apt-get install -y linux-headers-$(uname -r) build-essential perl + + cd /tmp + mkdir -p /mnt/cdrom + mount -o loop /home/vagrant/vmware_tools.iso /mnt/cdrom + tar zxf /mnt/cdrom/VMwareTools-*.tar.gz -C /tmp/ + + /tmp/vmware-tools-distrib/vmware-install.pl -d + + rm /home/vagrant/vmware_tools.iso + umount /mnt/cdrom + rmdir /mnt/cdrom + rm -rf /tmp/VMwareTools-* +fi \ No newline at end of file diff --git a/template/ubuntu1204.json b/template/ubuntu1204.json new file mode 100644 index 0000000..a34d00a --- /dev/null +++ b/template/ubuntu1204.json @@ -0,0 +1,106 @@ +{ + "builders": [ + { + "vm_name": "ubuntu1204-opsworks", + "type": "virtualbox-iso", + "guest_os_type": "Ubuntu_64", + "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso", + "virtualbox_version_file": ".vbox_version", + "iso_urls": [ + "http://releases.ubuntu.com/12.04/ubuntu-12.04.5-server-amd64.iso", + "http://nl.releases.ubuntu.com/12.04/ubuntu-12.04.5-server-amd64.iso" + ], + "iso_checksum": "7540ace2d6cdee264432f5ed987236d32edef798", + "iso_checksum_type": "sha1", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "http_directory": "preseed", + "headless": true, + "boot_wait": "5s", + "boot_command": [ + "", + "/install/vmlinuz ", + "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/preseed.cfg ", + "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", + "hostname={{.Name}} ", + "fb=false debconf/frontend=noninteractive ", + "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", + "keyboard-configuration/variant=USA console-setup/ask_detect=false ", + "initrd=/install/initrd.gz -- " + ], + "shutdown_command": "echo 'vagrant' | sudo -S -E shutdown -P now", + "hard_drive_interface": "sata" + }, + { + "vm_name": "ubuntu1204-opsworks", + "type": "vmware-iso", + "guest_os_type": "ubuntu-64", + "iso_urls": [ + "http://releases.ubuntu.com/12.04/ubuntu-12.04.5-server-amd64.iso", + "http://nl.releases.ubuntu.com/12.04/ubuntu-12.04.5-server-amd64.iso" + ], + "iso_checksum": "7540ace2d6cdee264432f5ed987236d32edef798", + "iso_checksum_type": "sha1", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "http_directory": "preseed", + "headless": true, + "boot_wait": "5s", + "boot_command": [ + "", + "/install/vmlinuz ", + "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/preseed.cfg ", + "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", + "hostname={{.Name}} ", + "fb=false debconf/frontend=noninteractive ", + "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", + "keyboard-configuration/variant=USA console-setup/ask_detect=false ", + "initrd=/install/initrd.gz -- " + ], + "shutdown_command": "echo 'vagrant' | sudo -S -E shutdown -P now", + "tools_upload_flavor": "linux", + "tools_upload_path": "vmware_tools.iso", + "vmdk_name": "disk", + "disk_type_id": "0", + "vmx_data": { + "MemTrimRate": "0", + "sched.mem.pshare.enable": "FALSE", + "mainMem.useNamedFile": "FALSE", + "prefvmx.minVmMemPct": "100" + } + } + ], + "provisioners": [ + { + "type": "file", + "source": "opsworks", + "destination": "/tmp" + }, + { + "type": "shell", + "execute_command": "echo 'vagrant' | sudo -S -E bash '{{.Path}}'", + "scripts": [ + "provision/update.sh", + "provision/network.sh", + "provision/vagrant.sh", + "provision/vmtools.sh", + "provision/minimize.sh", + "provision/opsworks.sh", + "provision/cleanup.sh" + ] + } + ], + "post-processors": [ + { + "type": "vagrant", + "output": "build/ubuntu1204-opsworks-{{.Provider}}.box", + "include": [ + "opsworks/opsworks", + "opsworks/opsworks.rb", + "opsworks/pre_config.yml", + "opsworks/client.yml" + ], + "vagrantfile_template": "opsworks/Vagrantfile.template" + } + ] +} \ No newline at end of file diff --git a/template/ubuntu1404.json b/template/ubuntu1404.json new file mode 100644 index 0000000..19e15cb --- /dev/null +++ b/template/ubuntu1404.json @@ -0,0 +1,106 @@ +{ + "builders": [ + { + "vm_name": "ubuntu1404-opsworks", + "type": "virtualbox-iso", + "guest_os_type": "Ubuntu_64", + "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso", + "virtualbox_version_file": ".vbox_version", + "iso_urls": [ + "http://releases.ubuntu.com/14.04/ubuntu-14.04.1-server-amd64.iso", + "http://nl.releases.ubuntu.com/14.04/ubuntu-14.04.1-server-amd64.iso" + ], + "iso_checksum": "401c5f6666fe2879ac3a9a3247b487723410cf88", + "iso_checksum_type": "sha1", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "http_directory": "preseed", + "headless": true, + "boot_wait": "5s", + "boot_command": [ + "", + "/install/vmlinuz ", + "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/preseed.cfg ", + "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", + "hostname={{.Name}} ", + "fb=false debconf/frontend=noninteractive ", + "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", + "keyboard-configuration/variant=USA console-setup/ask_detect=false ", + "initrd=/install/initrd.gz -- " + ], + "shutdown_command": "echo 'vagrant' | sudo -S -E shutdown -P now", + "hard_drive_interface": "sata" + }, + { + "vm_name": "ubuntu1404-opsworks", + "type": "vmware-iso", + "guest_os_type": "ubuntu-64", + "iso_urls": [ + "http://releases.ubuntu.com/14.04/ubuntu-14.04.1-server-amd64.iso", + "http://nl.releases.ubuntu.com/14.04/ubuntu-14.04.1-server-amd64.iso" + ], + "iso_checksum": "401c5f6666fe2879ac3a9a3247b487723410cf88", + "iso_checksum_type": "sha1", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "http_directory": "preseed", + "headless": true, + "boot_wait": "5s", + "boot_command": [ + "", + "/install/vmlinuz ", + "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/preseed.cfg ", + "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", + "hostname={{.Name}} ", + "fb=false debconf/frontend=noninteractive ", + "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", + "keyboard-configuration/variant=USA console-setup/ask_detect=false ", + "initrd=/install/initrd.gz -- " + ], + "shutdown_command": "echo 'vagrant' | sudo -S -E shutdown -P now", + "tools_upload_flavor": "linux", + "tools_upload_path": "vmware_tools.iso", + "vmdk_name": "disk", + "disk_type_id": "0", + "vmx_data": { + "MemTrimRate": "0", + "sched.mem.pshare.enable": "FALSE", + "mainMem.useNamedFile": "FALSE", + "prefvmx.minVmMemPct": "100" + } + } + ], + "provisioners": [ + { + "type": "file", + "source": "opsworks", + "destination": "/tmp" + }, + { + "type": "shell", + "execute_command": "echo 'vagrant' | sudo -S -E bash '{{.Path}}'", + "scripts": [ + "provision/update.sh", + "provision/network.sh", + "provision/vagrant.sh", + "provision/vmtools.sh", + "provision/minimize.sh", + "provision/opsworks.sh", + "provision/cleanup.sh" + ] + } + ], + "post-processors": [ + { + "type": "vagrant", + "output": "build/ubuntu1404-opsworks-{{.Provider}}.box", + "include": [ + "opsworks/opsworks", + "opsworks/opsworks.rb", + "opsworks/pre_config.yml", + "opsworks/client.yml" + ], + "vagrantfile_template": "opsworks/Vagrantfile.template" + } + ] +} \ No newline at end of file