diff --git a/README.markdown b/README.markdown index 9435647..9b3f6f0 100644 --- a/README.markdown +++ b/README.markdown @@ -57,6 +57,24 @@ A few notes to keep in mind: Ask in `#infrastructure-devops` if you have any questions, and thanks for the contributions! +## Testing `khan-dotfiles` on Linux + +It is easy to test the Khan-dotfiles on linux using a Docker container. +Use a command like `docker run -it ubuntu:22.04` to create a new +linux instance. Then run: +``` +# apt-get update +# apt-get install -y git sudo make +# useradd -U -d /home/testuser -m -s /bin/bash -G sudo testuser +# echo testuser:U6aMy0wojraho | chpasswd -e # the empty password +# su - testuser +$ mkdir -p khan/devtools +$ cd khan/devtools +$ git clone https://github.com/Khan/khan-dotfiles +$ cd khan-dotfiles +$ make +``` + ## Credits The `khan-dotfiles` are now maintained by the DevOps group within the diff --git a/containers/kabuild/README.md b/containers/kabuild/README.md index 1eb457c..637f102 100644 --- a/containers/kabuild/README.md +++ b/containers/kabuild/README.md @@ -24,7 +24,7 @@ The only skaffold command we're really using here is "skaffold build" ## Why NOT linux-setup.sh -linux-setup.sh & linux-functions.sh could possibly be used instead of +linux-setup.sh could possibly be used instead of doing things explicity in the Dockerfile. But that was tried and there was quite a bit to debug. And in the long term, DevOps anticipates that all services are built with relatively simple containers instead of a mega diff --git a/linux-functions.sh b/linux-functions.sh deleted file mode 100755 index c30088a..0000000 --- a/linux-functions.sh +++ /dev/null @@ -1,303 +0,0 @@ -#!/bin/sh -# This is largely a copy of linux-setup.sh -# TODO(ericbrown): Reduce code redundancy by refactoring to share code - -install_java() { - sudo apt-get install -y openjdk-11-jdk - # We ask you to select a java version (interactively) in case you have more - # than one installed. If there's only one, it'll just select that version - # by default. - sudo update-alternatives --config java - sudo update-alternatives --config javac -} - -install_go() { - if ! has_recent_go; then # has_recent_go is from shared-functions.sh - # This PPA is needed for ubuntus <20 but not >=20 - # (and it doesn't install for them anyway) - sudo add-apt-repository -y ppa:longsleep/golang-backports && sudo apt-get update -qq -y || sudo add-apt-repository -y -r ppa:longsleep/golang-backports - sudo apt-get install -y "golang-$DESIRED_GO_VERSION" - # The ppa installs go into /usr/lib/go-/bin/go - # Let's link that to somewhere likely to be on $PATH - sudo cp -sf /usr/lib/"go-$DESIRED_GO_VERSION"/bin/* /usr/local/bin/ - else - echo "golang already installed" - fi -} - -# NOTE: if you add a package here, check if you should also add it -# to webapp's Dockerfile. -install_packages() { - updated_apt_repo="" - - # This is needed to get the add-apt-repository command. - # apt-transport-https may not be strictly necessary, but can help - # for future updates. - sudo apt-get install -y software-properties-common apt-transport-https - - # To get the most recent nodejs, later. - if ls /etc/apt/sources.list.d/ 2>&1 | grep -q chris-lea-node_js; then - # We used to use the (obsolete) chris-lea repo, remove that if needed - sudo add-apt-repository -y -r ppa:chris-lea/node.js - sudo rm -f /etc/apt/sources.list.d/chris-lea-node_js* - updated_apt_repo=yes - fi - if ! ls /etc/apt/sources.list.d/ 2>&1 | grep -q nodesource || \ - ! grep -q node_12.x /etc/apt/sources.list.d/nodesource.list; then - # This is a simplified version of https://deb.nodesource.com/setup_12.x - wget -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add - - cat <&1 | grep -q git-core-ppa; then - sudo add-apt-repository -y ppa:git-core/ppa - updated_apt_repo=yes - fi - - # To get chrome, later. - if [ ! -s /etc/apt/sources.list.d/google-chrome.list ]; then - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" \ - | sudo tee /etc/apt/sources.list.d/google-chrome.list - wget -O- https://dl-ssl.google.com/linux/linux_signing_key.pub \ - | sudo apt-key add - - updated_apt_repo=yes - fi - - # Register all that stuff we just did. - if [ -n "$updated_apt_repo" ]; then - sudo apt-get update -qq -y || true - fi - - # This is needed for ubuntu >=20, but not prior ones. - sudo apt-get install -y python-is-python2 || true - - # Install pip manually. - curl https://bootstrap.pypa.io/get-pip.py --output get-pip.py - sudo python2 get-pip.py - # Delete get-pip.py after we're finish running it. - rm -f get-pip.py - # Match webapp's version version. - sudo pip install pip==20.1.1 - - # Install virtualenv and pychecker manually; ubuntu - # dropped support for them in ubuntu >=20 (since they're python2) - sudo pip install virtualenv==20.0.23 - - # We do not have pychecker on the mac (in /usr/local or virtualenv) - # Current install creates a broken /usr/local/bin/pychecker - #sudo pip install http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz/download - - # Needed to develop at Khan: git, python, node (js). - # lib{freetype6{,-dev},{png,jpeg}-dev} are needed for PIL - # imagemagick is needed for image resizing and other operations - # lib{xml2,xslt}-dev are needed for lxml - # libyaml-dev is needed for pyyaml - # libncurses-dev and libreadline-dev are needed for readline - # nodejs is used for various frontendy stuff in webapp, as well as our js - # services. We standardize on version 12 (the latest version suppported - # on appengine standard). - # redis is needed to run memorystore on dev - # docker is needed to run dev-webapp containers - # unzip is needed for other scripts - # tree sysdig iftop tcpflow are for diagnostics - # cargo is for rust / fastly - # python-dev supplies Python.h for things built by setup.sh (but why?) - # libncurses-dev is needed to build the readline we want - # python3-pip pipenv is required - # yarnpkg is the new node pkg mgr - # TODO(benkraft): Pull the version we want from webapp somehow. - # curl for various scripts (including setup.sh) - sudo apt-get install -y git \ - python-dev \ - python-mode python-setuptools \ - libfreetype6 libfreetype6-dev libpng-dev libjpeg-dev \ - imagemagick \ - libxslt1-dev \ - libyaml-dev \ - libncurses-dev libreadline-dev \ - nodejs=12* \ - redis-server \ - docker \ - curl \ - jq \ - unzip tree sysdig iftop tcpflow \ - cargo cargo-doc \ - python-dev \ - gcc \ - python3-pip pipenv - - # We need npm 6 or greater to support node12. That's the default - # for nodejs, but we may have overridden it before in a way that - # makes it impossible to upgrade, so we reinstall nodejs if our - # npm version is 5.x.x. - if expr "`npm --version`" : 5 >/dev/null 2>&1; then - sudo apt-get purge -y nodejs - sudo apt-get install -y "nodejs=12*" - fi - - # Ubuntu installs as /usr/bin/nodejs but the rest of the world expects - # it to be `node`. - if ! [ -f /usr/bin/node ] && [ -f /usr/bin/nodejs ]; then - sudo ln -s /usr/bin/nodejs /usr/bin/node - fi - - # Ubuntu's nodejs doesn't install npm, but if you get it from the PPA, - # it does (and conflicts with the separate npm package). So install it - # if and only if it hasn't been installed already. - if ! which npm >/dev/null 2>&1 ; then - sudo apt-get install -y npm - fi - # Make sure we have the preferred version of npm - # TODO(benkraft): Pull this version number from webapp somehow. - # We need npm 6 or greater to support node12. This is a particular npm6 - # version known to work. - sudo npm install -g npm@6.14.4 - sudo npm install -g yarn - - # Not technically needed to develop at Khan, but we assume you have it. - sudo apt-get install -y unrar ack-grep - - # Not needed for Khan, but useful things to have. - sudo apt-get install -y ntp abiword curl diffstat expect gimp \ - mplayer netcat netpbm screen w3m vim emacs google-chrome-stable - - # If you don't have the other ack installed, ack is shorter than ack-grep - # This might fail if you already have ack installed, so let it fail silently. - sudo dpkg-divert --local --divert /usr/bin/ack --rename --add \ - /usr/bin/ack-grep || echo "Using installed ack" - - # Needed to install printer drivers, and to use the printer scanner - sudo apt-get install -y apparmor-utils xsane - - # We use java for our google cloud dataflow jobs that live in webapp - # (as well as in khan-linter for linting those jobs) - install_java - - # We use go for our code, going forward - install_go -} - -install_slack() { - # Get the latest slack deb file and install it. - if ! which slack >/dev/null 2>&1 ; then - case `uname -m` in - *86) arch=i386;; - x86_64) arch=amd64;; - *) echo "WARNING: Cannot install slack: no client for `uname -m`";; - esac - if [ -n "$arch" ]; then - sudo apt-get install -y gconf2 gconf-service libgtk2.0-0 libappindicator1 - rm -rf /tmp/slack.deb - deb_url="$(wget -O- https://slack.com/downloads/instructions/ubuntu | grep -o 'https.*\.deb' | head -n1)" - if [ -n "$deb_url" ]; then - wget -O/tmp/slack.deb "$deb_url" || echo "WARNING: Cannot install slack: couldn't download $deb_url" - sudo dpkg -i /tmp/slack.deb - else - echo "WARNING: Cannot install slack: couldn't find .deb URL" - fi - fi - fi -} - -install_protoc() { - # The linux and mac installation process is the same aside from the - # platform-dependent zip archive. - install_protoc_common https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip -} - -install_watchman() { - if ! which watchman ; then - update "Installing watchman..." - builddir="$DEVTOOLS_DIR/watchman/" - if [ ! -d "$builddir" ]; then - mkdir -p "$builddir" - git clone https://github.com/facebook/watchman.git "$builddir" - fi - - ( - # Adapted from https://medium.com/@saurabh.friday/install-watchman-on-ubuntu-18-04-ba23c56eb23a - cd "$builddir" - sudo apt-get install -y autoconf automake build-essential python-dev libtool libssl-dev - git checkout tags/v4.9.0 - ./autogen.sh - # --enable-lenient is required for newer versions of GCC, which is - # stricter with certain constructs. - ./configure --enable-lenient - make - sudo make install - ) - fi -} - -install_postgresql() { - # postgresql-14 (not 14) is in normal repos for 20.04 - # TODO(ericbrown): We do WANT postgresql-14 - INFRA-5679 - - #sudo apt-get remove postgresql-11 postgresql-13 postgresql-client-11 postgresql-client-12 postgresql-client-13 - #sudo apt-get remove postgresql-client-common postgresql-common - - # Instructions taken from - # https://pgdash.io/blog/postgres-11-getting-started.html - # Postgres 14 is not available in 18.04, so we need to add the pg apt repository. - #curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - #sudo add-apt-repository -y "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -c -s`-pgdg main" - #sudo apt-get update - - POSTGRESQL_VERSION=14 - echo "Installing postgresql-${POSTGRESQL_VERSION}" - sudo apt-get install -y postgresql-${POSTGRESQL_VERSION} postgresql-contrib libpq-dev - - # Set up authentication to allow connections from the postgres user with no - # password. This matches the authentication setup that homebrew installs on - # a mac. Unlike a mac, we do not need to create a postgres user manually. - # We are doing this with a here-document because paths are different when - # this is called from packer scripts. - sudo cp "/etc/postgresql/${POSTGRESQL_VERSION}/main/pg_hba.conf" \ - "/etc/postgresql/${POSTGRESQL_VERSION}/main/pg_hba.conf.bak" - sudo tee /etc/postgresql/${POSTGRESQL_VERSION}/main/pg_hba.conf >/dev/null <&1 | grep -q chris-lea-node_js; then @@ -133,10 +134,10 @@ EOF # To get chrome, later. if [ ! -s /etc/apt/sources.list.d/google-chrome.list ]; then - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" \ - | sudo tee /etc/apt/sources.list.d/google-chrome.list wget -O- https://dl-ssl.google.com/linux/linux_signing_key.pub \ - | sudo apt-key add - + | sudo gpg --no-default-keyring --keyring /etc/apt/keyrings/google-chrome.gpg --import + echo 'deb [signed-by=/etc/apt/keyrings/google-chrome.gpg] http://dl.google.com/linux/chrome/deb/ stable main' \ + | sudo tee /etc/apt/sources.list.d/google-chrome.list updated_apt_repo=yes fi @@ -147,9 +148,7 @@ EOF fi # Python3 is needed to run the python services (e.g. ai-guide-core). - # We pin it at python3.8 at the moment, but will move it to python3.11 soon - # TODO(csilvers, GL-1195): remove python3.8 ai-guide-core is on python3.11 - sudo apt-get install -y python3.8 python3.8-venv + # We are on python3.11 now sudo apt-get install -y python3.11 python3.11-venv # Python2 is needed for development. First try the Ubuntu 22.04+ packages, then @@ -197,6 +196,9 @@ EOF # services. We standardize on version 16. # redis is needed to run memorystore on dev # libnss3-tools is a pre-req for mkcert, see install_mkcert for details. + # python3-venv is needed for the deploy virtualenv + # cargo is needed to run fastly-khancademy-dev + # docker is needed to run dev/server, lsof and uuid-runtime to run hotel # TODO(benkraft): Pull the version we want from webapp somehow. sudo apt-get install -y git \ libfreetype6 libfreetype6-dev libpng-dev libjpeg-dev \ @@ -209,7 +211,9 @@ EOF unzip \ jq \ libnss3-tools \ - python3-pip + python3-dev python3-setuptools python3-pip python3-venv \ + cargo cargo-doc \ + docker lsof uuid-runtime # We need npm 8 or greater to support node16. That's the default # for nodejs, but we may have overridden it before in a way that @@ -243,7 +247,8 @@ EOF # Not needed for Khan, but useful things to have. sudo apt-get install -y ntp abiword diffstat expect gimp \ - mplayer netcat netpbm screen w3m vim emacs google-chrome-stable + mplayer netcat iftop tcpflow netpbm screen w3m \ + vim emacs google-chrome-stable # If you don't have the other ack installed, ack is shorter than ack-grep # This might fail if you already have ack installed, so let it fail silently. @@ -325,7 +330,7 @@ install_postgresql() { } install_rust() { - builddir=$(mktemp -d -t rustup.XXXXX) + builddir=$(mktemp -d -t rustup.XXXXX) ( cd "$builddir" diff --git a/mac-setup-normal.sh b/mac-setup-normal.sh index ea27fea..45018fe 100755 --- a/mac-setup-normal.sh +++ b/mac-setup-normal.sh @@ -35,38 +35,6 @@ DEVTOOLS_DIR="$REPOS_DIR/devtools" # Load shared setup functions. . "$DEVTOOLS_DIR"/khan-dotfiles/shared-functions.sh -# for printing standard echoish messages -notice () { - printf " $1\n" -} - -# for printing logging messages that *may* be replaced by -# a success/warn/error message -info () { - printf " [ \033[00;34m..\033[0m ] $1" -} - -# for printing prompts that expect user input and will be -# replaced by a success/warn/error message -user () { - printf "\r [ \033[0;33m??\033[0m ] $1 " -} - -# for replacing previous input prompts with success messages -success () { - printf "\r\033[2K [ \033[00;32mOK\033[0m ] $1\n" -} - -# for replacing previous input prompts with warnings -warn () { - printf "\r\033[2K [\033[0;33mWARN\033[0m] $1\n" -} - -# for replacing previous prompts with errors -error () { - printf "\r\033[2K [\033[0;31mFAIL\033[0m] $1\n" -} - trap exit_warning EXIT # from shared-functions.sh @@ -79,31 +47,6 @@ update_path() { . .profile.khan } -maybe_generate_ssh_keys () { - # Create a public key if need be. - info "Checking for ssh keys" - mkdir -p ~/.ssh - if [ -s ~/.ssh/id_rsa ] || [ -s ~/.ssh/id_ecdsa ] - then - # TODO(ebrown): Verify these key(s) have passphrases on them - success "Found existing ssh keys" - else - echo - echo "Creating your ssh key pair for this machine" - echo "Please DO NOT use an empty passphrase" - APPLE_SSH_ADD_BEHAVIOR=macos ssh-keygen -t ecdsa -f ~/.ssh/id_ecdsa - # Old: ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa - success "Generated an rsa ssh key at ~/.ssh/id_ecdsa" - APPLE_SSH_ADD_BEHAVIOR=macos ssh-add -K - success "Added key to your keychain" - echo "Your ssh public key is:" - cat ~/.ssh/id_ecdsa.pub - echo "Please manually copy this public key to https://github.com/settings/keys." - read -p "Press enter when you've done this..." - fi - return 0 -} - copy_ssh_key () { if [ -e ~/.ssh/id_rsa ] then diff --git a/setup.sh b/setup.sh index 2e30f7c..cf786ce 100755 --- a/setup.sh +++ b/setup.sh @@ -73,7 +73,10 @@ check_dependencies() { # file as well. This is a noop if it's already in there. mkdir -p ~/.ssh grep -q github.com ~/.ssh/known_hosts 2>/dev/null || \ - ssh-keyscan github.com >> ~/.ssh/khown_hosts + ssh-keyscan github.com >> ~/.ssh/known_hosts + + # Make sure we have the ssh keys we need to pull from git. + maybe_generate_ssh_keys # in shared-functions.sh # You need to have run the setup to install binaries: node, npm/etc. if ! npm --version >/dev/null; then @@ -336,10 +339,10 @@ install_our_lovely_cli() { npm install } -install_dotfiles - check_dependencies +install_dotfiles + update_userinfo # the order for these is (mostly!) important, beware @@ -352,7 +355,7 @@ install_hooks # pre-req: clone_repos download_db_dump # pre-req: install_deps create_pg_databases # pre-req: install_deps create_default_keeper_config # pre-req: update_userinfo -install_keeper python3 +install_keeper echo echo "---------------------------------------------------------------------" diff --git a/shared-functions.sh b/shared-functions.sh index 26379a2..67d9749 100644 --- a/shared-functions.sh +++ b/shared-functions.sh @@ -1,5 +1,37 @@ bad_usage_get_yn_input=100 +# for printing standard echoish messages +notice() { + printf " $1\n" +} + +# for printing logging messages that *may* be replaced by +# a success/warn/error message +info() { + printf " [ \033[00;34m..\033[0m ] $1" +} + +# for printing prompts that expect user input and will be +# replaced by a success/warn/error message +user() { + printf "\r [ \033[0;33m??\033[0m ] $1 " +} + +# for replacing previous input prompts with success messages +success() { + printf "\r\033[2K [ \033[00;32mOK\033[0m ] $1\n" +} + +# for replacing previous input prompts with warnings +warn() { + printf "\r\033[2K [\033[0;33mWARN\033[0m] $1\n" +} + +# for replacing previous prompts with errors +error() { + printf "\r\033[2K [\033[0;31mFAIL\033[0m] $1\n" +} + # Replacement for clone_repo() function using ka-clone tool for local config # If run on an existing repository, will *update* and do --repair # Arguments: @@ -292,7 +324,39 @@ EOF fi } -# install keeper +maybe_generate_ssh_keys() { + # Create a public key if need be. + info "Checking for ssh keys" + mkdir -p ~/.ssh + if [ -s ~/.ssh/id_rsa ] || [ -s ~/.ssh/id_ecdsa ]; then + # TODO(ebrown): Verify these key(s) have passphrases on them + success "Found existing ssh keys" + else + echo + echo "Creating your ssh key pair for this machine" + echo "Please DO NOT use an empty passphrase" + APPLE_SSH_ADD_BEHAVIOR=macos ssh-keygen -t ecdsa -f ~/.ssh/id_ecdsa + # Old: ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa + success "Generated an rsa ssh key at ~/.ssh/id_ecdsa" + echo "Your ssh public key is:" + cat ~/.ssh/id_ecdsa.pub + echo "Please manually copy this public key to https://github.com/settings/keys." + read -p "Press enter when you've done this..." + fi + + # Add the keys to the keychain if needed + if [ -z "`ssh-add -l 2>/dev/null`" ]; then + ssh-add >/dev/null || { + # ssh-agent isn't running, let's fix that + eval $(ssh-agent -s) + ssh-add + } + success "Added key to your keychain" + fi + + return 0 +} + install_keeper() { # NOTE(miguel): we have had issues in our deploy system and with devs # in their local environment with keeper throttling requests since