From 6355e82e3ca89f56216eebad34165fc0cfcb19e7 Mon Sep 17 00:00:00 2001 From: Craig Silverstein Date: Wed, 6 Mar 2024 13:25:51 -0800 Subject: [PATCH] Remove all vestiges of python2 from khan-dotfiles. (#101) ## Summary: We no longer use python2 anywhere! (Well, we still use it in one place: `make topic-icons`, anyone who wants to run that will now need to install python2 manually.) Let's remove it from khan-dotfiles. That simplifies the dotfiles a fair bit, and also simplifies the users' dev experience since the virtualenv was causing problems. I did this by running `git grep -e 'python[^3]' -e 'virtualenv'` and deleting anything that looked no longer relevant. Issue: https://khanacademy.atlassian.net/browse/INFRA-9771 ## Test plan: TODO -- will need to test on new installs of both mac and linux, and I have no idea how to do that! Author: csilvers Reviewers: dbraley, csilvers, #infra-platform Required Reviewers: Approved By: dbraley Checks: Pull Request URL: https://github.com/Khan/khan-dotfiles/pull/101 --- .profile.khan | 32 ----------------------- Makefile | 4 --- bin/install-mac-homebrew.py | 2 +- bin/install-mac-python2.py | 47 --------------------------------- bin/mac-uninstall-brew.sh | 2 +- bin/pip2 | 3 --- bin/rebuild_virtualenv.sh | 47 --------------------------------- bin/system_report.sh | 3 +-- containers/kabuild/Dockerfile | 16 ------------ linux-setup.sh | 35 ++----------------------- mac-setup-elevated.sh | 2 +- mac-setup-normal.sh | 26 ------------------- setup-gcloud.sh | 3 --- setup.sh | 27 +------------------ shared-functions.sh | 49 +---------------------------------- 15 files changed, 8 insertions(+), 290 deletions(-) delete mode 100755 bin/install-mac-python2.py delete mode 100755 bin/pip2 delete mode 100755 bin/rebuild_virtualenv.sh diff --git a/.profile.khan b/.profile.khan index d8743a3..b129af1 100755 --- a/.profile.khan +++ b/.profile.khan @@ -57,16 +57,6 @@ if which brew >/dev/null 2>&1; then mkdir -p "$KA_CONFIG_CACHE_DIR" echo $BREW_PREFIX > $BREW_PREFIX_HINT_FILE fi - - # Add the unversioned binaries from our backported homebrew python@2 formula, - # if they have brew and have our python@2 formula installed - # See: https://github.com/Khan/homebrew-repo/blob/master/Formula/python%402.rb - - # This will exist if native python2 is installed (usually only on intel) - PY2_BIN=$BREW_PREFIX - # On a M1 machine, we run python2 via rosetta out of /usr/local - [ -d "$PY2_BIN" ] || PY2_BIN="/usr/local/opt/python@2" - export PATH="$PY2_BIN/libexec/bin:$PATH" fi # Make sure git (and other apps) prefer 'vim' to 'vi'. @@ -104,27 +94,5 @@ if [ `uname -s` = Darwin ]; then alias timeout=gtimeout fi -# If you do NOT want to have the python2 environment auto-activated, -# add the following to the top of your ~/.profile script: -# export DO_NOT_ACTIVATE_VIRTUALENV_KHAN27=1 -# -if [ -z "$DO_NOT_ACTIVATE_VIRTUALENV_KHAN27" ]; then - # Activate Python2.7 virtualenv if present - # This is desirable if you're touching the python monolith - if [ -s ~/.virtualenv/khan27/bin/activate ]; then - . ~/.virtualenv/khan27/bin/activate - else - echo "[WARN] Could not find '~/.virtualenv/khan27/bin/activate'"\ - "- All processes that depend on Python may be negatively impacted."\ - "Please run 'make' from the dotfiles directory to fix this." - fi -fi - -# Since we auto-activate the khan27 virtualenv, we don't want pipenv using any -# virtualenv it happens to find itself in, but instead to use its own. -# TODO(benkraft): If in the future we move to pipenv everywhere, we can -# dispense with the auto-activated virtualenvs, and get rid of this. -export PIPENV_IGNORE_VIRTUALENVS=1 - # Source the generated profile, if it exists. [ -f $HOME/.profile-generated.khan ] && source $HOME/.profile-generated.khan diff --git a/Makefile b/Makefile index b501208..73c4ed4 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,3 @@ os-install: common-install: ./setup.sh - -virtualenv: - ./bin/rebuild_virtualenv.sh - diff --git a/bin/install-mac-homebrew.py b/bin/install-mac-homebrew.py index 57afda9..22cfe10 100755 --- a/bin/install-mac-homebrew.py +++ b/bin/install-mac-homebrew.py @@ -117,7 +117,7 @@ def _validate_and_install_homebrew(self, force_x86=False): # unbeknownst!) # subprocess.run(['brew', 'tap', 'brew/cask-versions'], check=True) - # This is where we store our own formula, including a python@2 backport + # This is where we store our own formulas. subprocess.run(brew_runner + ["tap", "khan/repo"], check=True) def validate_and_install_homebrew(self): diff --git a/bin/install-mac-python2.py b/bin/install-mac-python2.py deleted file mode 100755 index 747e5a7..0000000 --- a/bin/install-mac-python2.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -"""Install Khan's python2.""" - -import argparse -import platform -import re -import subprocess - -parser = argparse.ArgumentParser() -parser.add_argument( - "--force", help="Force install of Khan's python2", action="store_true" -) -args = parser.parse_args() - -which = subprocess.run(["which", "python2"], capture_output=True, text=True) -is_installed = ( - which.returncode == 0 and which.stdout.strip() != "/usr/bin/python2" -) -if is_installed: - print("Already running a non-system python2.") - -if args.force or not is_installed: - action = "reinstall" if is_installed else "install" - print("Installing python2 from khan/repo. This may take a few minutes.") - if platform.uname().machine == "arm64": - brew_runner = ["arch", "-x86_64", "/usr/local/bin/brew"] - else: - brew_runner = ["brew"] - subprocess.run(brew_runner + [action, "khan/repo/python@2"], check=True) - -# Get version of pip2 -pip2_version = "" -pip2_version_str = subprocess.run( - ["pip2", "--version"], capture_output=True, text=True -) -if pip2_version_str: - match = re.match(r"\w+ (\d+)", pip2_version_str.stdout) - if match: - pip2_version = match.group(1) - -if pip2_version and pip2_version > "19": - print("Reverting pip2 from version: " + pip2_version_str.stdout.strip()) - subprocess.run(["pip2", "install", "pip<20", "-U"], check=True) - -# Simple diagnostics -subprocess.run(["pip2", "--version"]) -print("which python2: " + which.stdout.strip()) diff --git a/bin/mac-uninstall-brew.sh b/bin/mac-uninstall-brew.sh index 0a634c2..5add0b0 100755 --- a/bin/mac-uninstall-brew.sh +++ b/bin/mac-uninstall-brew.sh @@ -40,7 +40,7 @@ if [[ -e "/usr/local/bin/brew" ]]; then sudo rm -rf /usr/local/var/postgresql* fi -echo "Removing khan python2 virtualenv..." +echo "Removing the (obsolete) khan python2 virtualenv..." rm -rf ~/.virtualenv/khan27 echo "Removing other things that may upgrade if user reinstalls..." diff --git a/bin/pip2 b/bin/pip2 deleted file mode 100755 index b4ca6ba..0000000 --- a/bin/pip2 +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python2 -m pip $@ \ No newline at end of file diff --git a/bin/rebuild_virtualenv.sh b/bin/rebuild_virtualenv.sh deleted file mode 100755 index 36ae075..0000000 --- a/bin/rebuild_virtualenv.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Quick way to rebuild the python virtualenv. This is a common way to end up -# with a broken dev setup; and we don't need to re-run all of khan-dotfiles to -# fix it. This script is best-effort: if something goes wrong we'll just tell -# the user to fall back to khan-dotfiles. - -# Bail on any errors -set -e - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd -P )/.." -KHAN_DIR="$SCRIPT_DIR/../.." - -venv_exit_warning() { - echo "*** FATAL ERROR: unable to rebuild virtualenv! ***" - echo "*** Try a full run of khan-dotfiles instead, ***" - echo "*** or contact @dev-support. ***" -} - -trap venv_exit_warning EXIT - -# Load shared setup functions. -. "$SCRIPT_DIR"/shared-functions.sh - -# We're about to do an rm -rf; make sure it looks like the right directory. -# For most (all) users, $VIRTUAL_ENV should be $HOME/.virtualenv/khan27 but -# we'll be a little flexible. -if [[ "$VIRTUAL_ENV" != */.virtualenv/khan27 ]]; then - echo "VIRTUAL_ENV=$VIRTUAL_ENV doesn't look like a " \ - "Khan virtualenv, not removing." - exit 1 -fi - -echo "Ensure proper version of python2 virutalenv is installed" -install_python2_virtualenv - -echo "Cleaning out old virtualenv" -rm -rf "$VIRTUAL_ENV" -# We probably just removed `which python`, force the shell to find a new one. -hash -r -echo "Creating and activating new virtualenv" -create_and_activate_virtualenv "$VIRTUAL_ENV" # from shared-functions.sh -echo "Reinstalling webapp deps" -( cd "$KHAN_DIR/webapp" && make python_deps ) -echo "Success! You may need to restart any open terminals or running dev-servers." - -trap - EXIT diff --git a/bin/system_report.sh b/bin/system_report.sh index 9592e83..ecf0a32 100755 --- a/bin/system_report.sh +++ b/bin/system_report.sh @@ -133,11 +133,10 @@ tool_version java -version # Python tooling header "Python" tool_version python --version -tool_version python2 --version tool_version python3 --version tool_version pip --version kv "VIRTUAL_ENV" "$VIRTUAL_ENV" -kv "sys.path" "$(python2 -c 'import sys; print sys.path')" +kv "sys.path" "$(python3 -c 'import sys; print(sys.path)')" # TODO(dbraley): (l) check software-properties-common # TODO(dbraley): (l) check apt-trasport-https diff --git a/containers/kabuild/Dockerfile b/containers/kabuild/Dockerfile index a30841e..5bf2f9f 100644 --- a/containers/kabuild/Dockerfile +++ b/containers/kabuild/Dockerfile @@ -68,20 +68,6 @@ ENV DESIRED_GO_VERSION=1.21 RUN apt-get install -y golang-${DESIRED_GO_VERSION} RUN cp -sf /usr/lib/go-${DESIRED_GO_VERSION}/bin/* /usr/local/bin/ -# Python2 -ENV VENV=.virtualenv/khan27 -RUN apt-get install -y python2-dev python-setuptools -RUN ln -s /usr/bin/python2 /usr/bin/python -WORKDIR /tmp -RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py -RUN python2 get-pip.py pip==19.3.1 -RUN rm -f get-pip.py -WORKDIR /root -RUN pip install virtualenv==20.0.23 -RUN pip install http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz/download -RUN virtualenv -q --python="$(which python2)" --always-copy "/root/${VENV}" -RUN /root/${VENV}/bin/pip install -U "pip<20" setuptools - RUN apt-get install -y \ libfreetype6 libfreetype6-dev libpng-dev libjpeg-dev \ imagemagick \ @@ -128,8 +114,6 @@ WORKDIR /home/${USER_NAME} RUN mkdir -m 700 .ssh RUN mkdir khan RUN ssh-keyscan github.com >> ~/.ssh/known_hosts -RUN virtualenv -q --python="$(which python2)" --always-copy "/home/${USER_NAME}/${VENV}" -RUN /home/${USER_NAME}/${VENV}/bin/pip install -U "pip<20" setuptools COPY --chown=${USER_NAME}:${GROUP_NAME} .profile.khan .profile.khan COPY --chown=${USER_NAME}:${GROUP_NAME} .bash_profile.khan .bash_profile.khan COPY --chown=${USER_NAME}:${GROUP_NAME} .bashrc.khan .bashrc.khan diff --git a/linux-setup.sh b/linux-setup.sh index c5591ea..6a3210b 100755 --- a/linux-setup.sh +++ b/linux-setup.sh @@ -126,7 +126,7 @@ EOF updated_apt_repo=yes fi - # To get python3.8, later. + # To get the most recent python, later. if ! ls /etc/apt/sources.list.d/ 2>&1 | grep -q deadsnakes; then sudo add-apt-repository -y ppa:deadsnakes/ppa updated_apt_repo=yes @@ -151,41 +151,9 @@ EOF # We are on python3.11 now sudo apt-get install -y python3.11 python3.11-venv - # Python2 is needed for development. First try the Ubuntu 22.04+ packages, then - # the Ubuntu <22.04 packages if that fails. - sudo apt-get install -y python2-dev python-setuptools || sudo apt-get install -y python-dev python-mode python-setuptools - - # This is needed for Ubuntu >=20, but not prior ones. It no longer exists - # as of Ubuntu 22.04. - sudo apt-get install -y python-is-python2 || true - - # If we're on Ubuntu 22.04+, installing python-is-python2 didn't do anything, so - # we create the symlink ourselves. - if ! [ -f /usr/bin/python ]; then - sudo ln -s /usr/bin/python2 /usr/bin/python - fi - # Install curl for setup script usage sudo apt-get install -y curl - # Install pip manually. - curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py - # Match webapp's version. - sudo python2 get-pip.py pip==19.3.1 - # Delete get-pip.py after we're finish running it. - rm -f get-pip.py - - # Install virtualenv and pychecker manually; ubuntu - # dropped support for them in ubuntu >=20 (since they're python2) - sudo pip install virtualenv==20.0.23 - sudo pip install http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz/download - - # get-pip.py will remove the system pip3 binary if it previously existed, - # but it won't remove the package, so installing the package again won't - # restore it. Here we remove the package if it exists, so that the next - # apt-get command will install it properly. - sudo apt-get remove -y python3-pip || true - # Needed to develop at Khan: git, node (js). # lib{freetype6{,-dev},{png,jpeg}-dev} are needed for PIL # imagemagick is needed for image resizing and other operations @@ -212,6 +180,7 @@ EOF jq \ libnss3-tools \ python3-dev python3-setuptools python3-pip python3-venv \ + python-is-python3 \ cargo cargo-doc \ docker lsof uuid-runtime diff --git a/mac-setup-elevated.sh b/mac-setup-elevated.sh index 9fc7afa..1f4e3be 100755 --- a/mac-setup-elevated.sh +++ b/mac-setup-elevated.sh @@ -68,7 +68,7 @@ echo "This setup script needs your password to install things as root." sudo sh -c 'echo Thanks' if [[ $(uname -m) = "arm64" ]]; then - # install rosetta on M1 (required for openjdk, python2 and other things) + # install rosetta on M1 (required for openjdk and other things) # This will work here, but it requires input and I'd rather just have it in docs #sudo softwareupdate --install-rosetta diff --git a/mac-setup-normal.sh b/mac-setup-normal.sh index 45018fe..7908a90 100755 --- a/mac-setup-normal.sh +++ b/mac-setup-normal.sh @@ -153,17 +153,6 @@ update_git() { fi } -install_python2() { - # We only do this if python2 == /usr/bin/python2, which means it's system python - if [ "$(which python2)" != "/usr/bin/python2" ]; then - success "Already running a non-system python2." - return - fi - - info "Installing python2 from khan/repo. This may take a few minutes." - brew86 install khan/repo/python@2 -} - install_node() { if ! which node >/dev/null 2>&1; then # Install node 16: It's LTS and the latest version supported on @@ -302,12 +291,6 @@ install_jq() { install_python_tools() { # Python3 is needed to run the python services (e.g. ai-guide-core). - # We pin it at python3.8 at the moment, but will move it to python3.11 soon - # TODO(csilvers, GL-1195): remove python3.8 ai-guide-core is on python3.11 - if ! which python3.8 >/dev/null 2>&1; then - info "Installing python 3.8\n" - brew install python@3.8 - fi if ! which python3.11 >/dev/null 2>&1; then info "Installing python 3.11\n" brew install python@3.11 @@ -318,13 +301,6 @@ install_python_tools() { if ! brew ls pyenv >/dev/null 2>&1; then info "Installing pyenv\n" brew install pyenv - # At the moment, we depend on MacOS coming with python 2.7. If that - # stops, or we want to align the python versions with the linux - # dotfiles more effectively, we could do it with pyenv: - # `pyenv install 2.7.16 ; pyenv global 2.7.16` - # Because the linux dotfiles do not yet install pyenv, holding off on - # using pyenv to enforce python version until either that happens, or - # MacOs stops including python 2.7 by default. else success "pyenv already installed" fi @@ -366,8 +342,6 @@ install_openssl install_jq update_git -"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-python2.py - install_node install_go diff --git a/setup-gcloud.sh b/setup-gcloud.sh index 0ee144e..ba0c8c9 100755 --- a/setup-gcloud.sh +++ b/setup-gcloud.sh @@ -2,9 +2,6 @@ # This is used to be in setup.sh: install_and_setup_gcloud() # It was broken and is easier to debug separately -# TODO(ericbrown): Shell scripts hard to debug - rewrite in python -# shared-functions.sh are an issue - # Bail on any errors set -e diff --git a/setup.sh b/setup.sh index cf786ce..c55aadd 100755 --- a/setup.sh +++ b/setup.sh @@ -132,21 +132,6 @@ install_dotfiles() { fi done - # If users are using a shell other than bash, the updates we make won't - # get picked up. They'll have to activate the virtualenv in their shell - # config; if they haven't, the rest of the script will fail. - # TODO(benkraft): Add more specific instructions for other common shells, - # or just write dotfiles for them. - shell="`basename "$SHELL"`" - if [ -z "$VIRTUAL_ENV" ] && [ "$shell" != bash ] && [ "$shell" != zsh ]; then - add_fatal_error "Your default shell is $shell, not bash or zsh, so you'll" \ - "need to update its config manually to activate our" \ - "virtualenv. You can follow the instructions at" \ - "khanacademy.org/r/virtualenvs to create a new" \ - "virtualenv and then export its path in the" \ - "VIRTUAL_ENV environment variable before trying again." - fi - # *.template files are also copied so the user can change them. Unlike the # "default" files above, these do not include KA code, they are merely # useful defaults we want to install if the user doesnt have anything @@ -238,21 +223,11 @@ clone_repos() { kaclone_repair_self } -# Sets up virtualenv and pipenv +# Sets up pipenv setup_python() { - echo "Installing python2 virtualenv" - install_python2_virtualenv - - create_and_activate_virtualenv "$ROOT/.virtualenv/khan27" - # Used by various infra projects for managing python3 environment echo "Installing pipenv for python3" - - # WARNING: This overrwrites the python2 virtualenv pip3_install -q pipenv - - echo "Reinstall python2 virtualenv (will overwrite python3 version)" - install_python2_virtualenv } # Must have cloned the repos first. diff --git a/shared-functions.sh b/shared-functions.sh index 67d9749..f0314b3 100644 --- a/shared-functions.sh +++ b/shared-functions.sh @@ -166,7 +166,7 @@ install_protoc_common() { zip_url=$1 # We use protocol buffers in webapp's event log stream infrastructure. This - # installs the protocol buffer compiler (which generates python & java code + # installs the protocol buffer compiler (which generates go & java code # from the protocol buffer definitions), as well as a go-based compiler # plugin that allows us to generate bigquery schemas as well. @@ -214,53 +214,6 @@ has_recent_go() { -o "$go_minor_version" -ge "$DESIRED_GO_MINOR_VERISON" ] } -# Install the version of python2 virtualenv that we want -install_python2_virtualenv() { - # Install virtualenv. - # https://docs.google.com/document/d/1zrmm6byPImfbt7wDyS8PpULwnEckSxna2jhSl38cWt8 - # Must do --force-reinstall or it will NOT automatically overwrite - # python3 version of virtualenv if it accidentally gets installed. - python2 -m pip install virtualenv==20.0.23 --force-reinstall -} - -# Creates a webapp virtualenv in $1, if none exists, then activates it. -# -# Assumes pip and virtualenv are already installed. -# -# Arguments: -# $1: driectory in which to put the virtualenv, typically ~/.virtualenv/khan27. -create_and_activate_virtualenv() { - # On a arm64 mac, we MUST use the python2 version of virtualenv - VIRTUALENV=$(which virtualenv) - if [[ -n ${IS_MAC_ARM} ]]; then - VIRTUALENV=/usr/local/bin/virtualenv - if [[ -z "${VIRTUALENV}" ]]; then - /usr/local/bin/python2 -m pip install virtualenv - fi - fi - - if [ ! -d "$1" ]; then - ${VIRTUALENV} -q --python="$(which python2)" --always-copy "$1" - fi - - # Activate the virtualenv. - . "$1/bin/activate" - - # pip may get broken by virtualenv for some reason. We're better off - # calling `python -m pip` so we'll just swap in a script that does - # that for us. - if ! pip --version 2>/dev/null ; then - cp bin/pip `which pip` - cp bin/pip2 `which pip2` - fi - - # pip20+ stopped supporting python2.7, so we need to make sure - # we are using an older pip. - if ! pip --version | grep -q "pip 1[0-9]"; then - pip install -U "pip<20" setuptools - fi -} - # pip-install something globally. Recent pythons don't let you do # that (https://peps.python.org/pep-0668/), but we do it anyway. # $@: the arguments to `pip install`.