From 5167ede3177fb9aad8ebeafe556b41d75c3f1a76 Mon Sep 17 00:00:00 2001 From: "Eric (iCloud) Brown" Date: Fri, 5 Feb 2021 10:22:18 -0800 Subject: [PATCH] Ongoing mac-setup.sh cleanup Summary: * Can pass parameter whether to install apps or not * Install rust * Install mkcert * Minor bug fixes * Separate a few more things Issue: https://khanacademy.atlassian.net/browse/INFRA-5864 Test Plan: Startup clean VM and run mac-setup.sh. Ensure that it works with no args, -a and -n. Then finish all other setup.sh and startup KA. (There are still a few known issues in this area.) Reviewers: #devops, boris, davidbraley Reviewed By: #devops, boris, davidbraley Subscribers: davidbraley, boris, csilvers Differential Revision: https://phabricator.khanacademy.org/D69082 --- bin/edit-system-config.sh | 47 +++++++++++++++++++++++++++++++ bin/install-mac-apps.sh | 47 ++++++++++++++++++++++++++++--- bin/install-mac-mkcert.py | 22 +++++++++++++++ bin/install-mac-rust.py | 12 ++++++++ linux-setup.sh | 2 ++ mac-setup-elevated.sh | 58 +++++++++++++++++++++++++++++++++++++-- mac-setup-normal.sh | 1 + mac-setup.sh | 42 +++++++++++++++++++++++++--- profile.default | 3 ++ setup.sh | 54 ++++++++---------------------------- zprofile.default | 3 ++ 11 files changed, 238 insertions(+), 53 deletions(-) create mode 100755 bin/edit-system-config.sh create mode 100755 bin/install-mac-mkcert.py create mode 100755 bin/install-mac-rust.py diff --git a/bin/edit-system-config.sh b/bin/edit-system-config.sh new file mode 100755 index 0000000..5da9d5c --- /dev/null +++ b/bin/edit-system-config.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# History: +# * Functionality moved from setup.sh to edit-system-config.sh +# (called from linux-setup.sh and mac-setup-elevated.sh). + +# Bail on any errors +set -e + +# Install in $HOME by default, but can set an alternate destination via $1. +ROOT=${1-$HOME} +mkdir -p "$ROOT" + +echo "Modifying system configs" + +# This command avoids the spew when you deploy the Khan Academy +# appengine app: +# Cannot guess mime-type for XXX. Using application/octet-stream +line="application/octet-stream less eot ttf woff otf as fla sjs flash tmpl" +if [ -s /usr/local/etc/mime.types ]; then + # Replace any existing line with 'less' and 'eot' with the new line. + grep -v 'less eot' /usr/local/etc/mime.types | \ + sudo sh -c "cat; echo '$line' > /usr/local/etc/mime.types" +else + sudo sh -c 'echo "$line" > /usr/local/etc/mime.types' +fi +sudo chmod a+r /usr/local/etc/mime.types + +# If there is no ssh key, make one. +mkdir -p "$ROOT/.ssh" +if [ ! -e "$ROOT/.ssh/id_rsa" -a ! -e "$ROOT/.ssh/id_dsa" ]; then + ssh-keygen -q -N "" -t rsa -f "$ROOT/.ssh/id_rsa" +fi + +# if the user does not have a global gitignore file configured, reference +# ours (or whatever is in the default location +if ! git config --global core.excludesfile > /dev/null; then + git config --global core.excludesfile ~/.gitignore +fi +# cleanup from previous versions: remove ~/.gitignore.khan symlink if exists +rm -f ~/.gitignore.khan + +# Apple is very picky on permsions of files zsh loads +ZSHSHARE="/usr/local/share/zsh" +if [[ -d "${ZSHSHARE}" ]]; then + chmod -R 755 "${ZSHSHARE}" +fi diff --git a/bin/install-mac-apps.sh b/bin/install-mac-apps.sh index 77c06a1..ab0bc8c 100755 --- a/bin/install-mac-apps.sh +++ b/bin/install-mac-apps.sh @@ -1,10 +1,46 @@ #!/bin/bash -# User may be prompted for password so brew can run sudo +# This script is normally called by mac-setup.sh (with relevant arguments) + +# User may be prompted for password so brew can run sudo when installing +# - google-drive-file-stream (currently broken in homebrew 2/4/2021) +# - zoom # Bail on any errors set -e +SCRIPT=$(basename $0) + +usage() { + cat </dev/null 2>&1; then echo "$app is not installed, installing $app" - brew install --cask $app || warn "Failed to install $app, perhaps it is already installed." + brew install --cask $app || echo "Failed to install $app, perhaps it is already installed." else echo "$app already installed" fi diff --git a/bin/install-mac-mkcert.py b/bin/install-mac-mkcert.py new file mode 100755 index 0000000..e2a85c4 --- /dev/null +++ b/bin/install-mac-mkcert.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Install mkcert and setup a CA. + +This very simple script exists because we want it to be called along with all +the other scripts that require elevated permissions (sudo) and because it +requires a reboot after completion. +""" + +import subprocess + +result = subprocess.run(['which', 'mkcert'], capture_output=True) +if result.returncode != 0: + subprocess.run(['brew', 'install', 'mkcert'], check=True) + # The following will ask for your password + subprocess.run(['mkcert', '-install'], check=True) + + print(""" +You have installed mkcert (used to make khanacademy.dev work) +A CA has been added to your system and browser certificate trust stores. + +You must REBOOT your machine for browsers to recognize new CA. +""") diff --git a/bin/install-mac-rust.py b/bin/install-mac-rust.py new file mode 100755 index 0000000..6d72f56 --- /dev/null +++ b/bin/install-mac-rust.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +"""Install rust & cargo on a mac.""" + +# TODO(ebrown): Make something that works on mac & linux +# TODO(ebrown): Install a specific version of rust/cargo +# TODO(ebrown): Tweak khan startup scripts instead of rustup-init doing it + +import subprocess + +subprocess.run(['brew', 'install', 'rustup-init'], check=True) +subprocess.run(['rustup-init', '-y', '-t', 'wasm32-wasi', '--no-modify-path'], + check=True) diff --git a/linux-setup.sh b/linux-setup.sh index 18fd61e..19a80c6 100755 --- a/linux-setup.sh +++ b/linux-setup.sh @@ -323,4 +323,6 @@ install_postgresql # TODO (boris): Setup pyenv (see mac_setup:install_python_tools) # https://opencafe.readthedocs.io/en/latest/getting_started/pyenv/ +"$DEVTOOLS_DIR"/khan-dotfiles/bin/edit-system-config.sh + trap - EXIT diff --git a/mac-setup-elevated.sh b/mac-setup-elevated.sh index 60deeda..560726a 100755 --- a/mac-setup-elevated.sh +++ b/mac-setup-elevated.sh @@ -6,8 +6,50 @@ # Bail on any errors set -e +SCRIPT=$(basename $0) + +usage() { + cat << EOF +usage: $SCRIPT [options] + --root Use specified directory as root (instead of HOME). + --all Install all user apps. + --none Install no user apps. +EOF +} + +# Install in $HOME by default, but can set an alternate destination via $1. +ROOT="${ROOT:-$HOME}" + +APPS= + +# Process command line arguments +while [[ "$1" != "" ]]; do + case $1 in + -r | --root) + shift + ROOT=$1 + ;; + -a | --all) + APPS="-a" + ;; + -n | --none) + APPS="-n" + ;; + -h | --help) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac + shift +done + +echo foobar + # The directory to which all repositories will be cloned. -ROOT=${1-$HOME} REPOS_DIR="$ROOT/khan" # Derived path location constants @@ -37,8 +79,20 @@ install_protoc() { echo "This setup script needs your password to install things as root." sudo sh -c 'echo Thanks' +# Add github to known_hosts (one less prompt when QAing script) +mkdir -p ~/.ssh +grep -q github.com ~/.ssh/known_hosts 2>/dev/null || \ + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" \ + >> ~/.ssh/known_hosts + "$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-homebrew.py +# Other brew related installers that require sudo + +"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-mkcert.py + +"$DEVTOOLS_DIR"/khan-dotfiles/bin/edit-system-config.sh + # It used to be we needed to install xcode-tools, now homebrew does this for us #"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-gcc.sh @@ -48,4 +102,4 @@ install_protoc # (as well as in khan-linter for linting those jobs) install_mac_java -"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-apps.sh +"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-apps.sh "$APPS" diff --git a/mac-setup-normal.sh b/mac-setup-normal.sh index d3d926f..5608d33 100755 --- a/mac-setup-normal.sh +++ b/mac-setup-normal.sh @@ -372,6 +372,7 @@ update_git install_node install_go +"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-rust.py "$DEVTOOLS_DIR"/khan-dotfiles/bin/mac-setup-postgres.py install_nginx diff --git a/mac-setup.sh b/mac-setup.sh index f796dac..ec83920 100755 --- a/mac-setup.sh +++ b/mac-setup.sh @@ -3,11 +3,45 @@ # Bail on any errors set -e -# TODO(ericbrown): Support --quiet command line argument -# TODO(ericbrown): Use python?? +SCRIPT=$(basename $0) + +usage() { + cat << EOF +usage: $SCRIPT [options] + --root Use specified directory as root (instead of HOME). + --all Install all user apps. + --none Install no user apps. +EOF +} + +# Install in $HOME by default, but can set an alternate destination via $1. +ROOT="${ROOT:-$HOME}" + +# Process command line arguments +while [[ "$1" != "" ]]; do + case $1 in + -r | --root) + shift + ROOT=$1 + ;; + -a | --all) + APPS="-a" + ;; + -n | --none) + APPS="-n" + ;; + -h | --help) + usage + exit 0 + ;; + *) + usage + exit 1 + esac + shift +done # The directory to which all repositories will be cloned. -ROOT=${1-$HOME} REPOS_DIR="$ROOT/khan" # Derived path location constants @@ -43,7 +77,7 @@ read -p "Press enter to continue..." # Note that ensure parsing arguments (above) doesn't hide anything # Run setup that requires sudo access -"$DEVTOOLS_DIR"/khan-dotfiles/mac-setup-elevated.sh +"$DEVTOOLS_DIR"/khan-dotfiles/mac-setup-elevated.sh "$APPS" # Run setup that does NOT require sudo access "$DEVTOOLS_DIR"/khan-dotfiles/mac-setup-normal.sh diff --git a/profile.default b/profile.default index b551569..ac07950 100644 --- a/profile.default +++ b/profile.default @@ -5,3 +5,6 @@ if [ -s ~/.profile.khan ]; then . ~/.profile.khan fi +if [ -s ~/.cargo/env ]; then + . ~/.cargo/env +fi diff --git a/setup.sh b/setup.sh index 31364a7..6734602 100755 --- a/setup.sh +++ b/setup.sh @@ -35,6 +35,9 @@ DIR=$(dirname "$0") # should we install webapp? (disable for mobile devs or to make testing faster) WEBAPP="${WEBAPP:-true}" +# Will contain a string on a mac and be empty on linux +IS_MAC=$(which sw_vers || echo "") + trap exit_warning EXIT # from shared-functions.sh warnings="" @@ -138,43 +141,6 @@ install_dotfiles() { . ~/.profile } -edit_system_config() { - echo "Modifying system configs" - - # This command avoids the spew when you deploy the Khan Academy - # appengine app: - # Cannot guess mime-type for XXX. Using application/octet-stream - line="application/octet-stream less eot ttf woff otf as fla sjs flash tmpl" - if [ -s /usr/local/etc/mime.types ]; then - # Replace any existing line with 'less' and 'eot' with the new line. - grep -v 'less eot' /usr/local/etc/mime.types | \ - sudo sh -c "cat; echo '$line' > /usr/local/etc/mime.types" - else - sudo sh -c 'echo "$line" > /usr/local/etc/mime.types' - fi - sudo chmod a+r /usr/local/etc/mime.types - - # If there is no ssh key, make one. - mkdir -p "$ROOT/.ssh" - if [ ! -e "$ROOT/.ssh/id_rsa" -a ! -e "$ROOT/.ssh/id_dsa" ]; then - ssh-keygen -q -N "" -t rsa -f "$ROOT/.ssh/id_rsa" - fi - - # if the user does not have a global gitignore file configured, reference - # ours (or whatever is in the default location - if ! git config --global core.excludesfile > /dev/null; then - git config --global core.excludesfile ~/.gitignore - fi - # cleanup from previous versions: remove ~/.gitignore.khan symlink if exists - rm -f ~/.gitignore.khan - - # Apple is very picky on permsions of files zsh loads - ZSHSHARE="/usr/local/share/zsh" - if [[ -d "${ZSHSHARE}" ]]; then - chmod -R 755 "${ZSHSHARE}" - fi -} - # clone a repository without any special sauce. should only be used in order to # bootstrap ka-clone, or if you are certain you don't want a khanified repo. # $1: url of the repository to clone. $2: directory to put repo @@ -256,8 +222,15 @@ install_deps() { # Need to install yarn first before run `make install_deps` # in webapp. + echo "Installing yarn" if ! which yarn >/dev/null 2>&1; then - sudo npm install -g yarn + if [[ -n "${IS_MAC}" ]]; then + # Mac does not require root - npm is in /usr/local via brew + npm install -g yarn + else + # Linux requires sudo permissions + sudo npm install -g yarn + fi fi # Install all the requirements for khan @@ -347,14 +320,9 @@ setup_arc() { check_dependencies -# Run sudo once at the beginning to get the necessary permissions. -echo "This setup script needs your password to install things as root." -sudo sh -c 'echo Thanks' - # the order of these individually doesn't matter but they should come first update_userinfo install_dotfiles -edit_system_config # the order for these is (mostly!) important, beware clone_repos install_and_setup_gcloud diff --git a/zprofile.default b/zprofile.default index 5eefd89..3b9f711 100644 --- a/zprofile.default +++ b/zprofile.default @@ -5,3 +5,6 @@ if [ -s ~/.zprofile.khan ]; then . ~/.zprofile.khan fi +if [ -s ~/.cargo/env ]; then + . ~/.cargo/env +fi