Merge pull request #116 from ucbds-infra/fix-gs

.gitignore

@@ -15,4 +15,5 @@ docs/_build
 .Rhistory
 .ionide
 gs-testing-stuff
-.mypy.cache
+.mypy.cache
+.OTTER_LOG

Dockerfile

@@ -52,4 +52,4 @@ RUN apt-get clean && \
 ADD requirements.txt /tmp/requirements.txt
 RUN pip install -r /tmp/requirements.txt
 
-RUN pip install otter-grader==1.0.1
+RUN pip install otter-grader==1.1.0

docs/changelog.md

@@ -1,5 +1,14 @@
 # Changelog
 
+**v1.0.2:**
+
+* Moved Gradescope grading to inside conda env within container due to change in Gradescope's grading image
+* Added ability to specify additional tests to be run in cell metadata without need of explicit `Notebook.check` cells
+
+**v1.0.1:**
+
+* Fixed bug with specification of overwriting requirements in Otter Generate
+
 **v1.0.0:**
 
 * Changed structure of CLI into six main commands: `otter assign`, `otter check`, `otter export`, `otter generate`, `otter grade`, and `otter service`

docs/execution.md

@@ -20,6 +20,7 @@ If students are submitting IPython notebooks (`.ipynb` files), they are executed
             <li>The code lines are run through an <code>IPython.core.inputsplitter.IPythonInputSplitter</code></li>
             <li>The string is run in the dummy environment with <code>otter.Notebook.export</code> and <code>otter.Notebook._log_event</code> patched so that they don't run</li>
             <li>If the run was successful, the cell contents are added to a collection string. If it fails, the error is ignored unless otherwise indicated.</li>
+            <li>If the cell has metadata indicating that a check should be run after that cell, the code for the check is added.</li>
         </ol>
     </li>
     <li>The collection string is turned into an abstract syntax tree that is then transformed so that all calls of any form to <code>otter.Notebook.check</code> have their return values appended to a prenamed list which is in the dummy environemnt.</li>
@@ -30,6 +31,22 @@ If students are submitting IPython notebooks (`.ipynb` files), they are executed
 
 The grades for each test are then collected from the list to which they were appended in the dummy environment and any additional tests are run against this resulting environment. Running in this manner has one main advantage: it is robust to variable name collisions. If two tests rely on a variable of the same name, they will not be stymied by the variable being changed between the tests, because the results for one test are collected from when that check is called rather than being run at the end of execution.
 
+It is also possible to specify tests to run in the cell metadata without the need of explicit calls to `Notebook.check`. To do so, add an `otter` field to the cell metadata with the following structure:
+
+```json
+{
+    "otter": {
+        "tests": [
+            "q1",
+            "q2",
+            "etc."
+        ]
+    }
+}
+```
+
+The strings within `tests` should correspond to filenames within the tests directory (without the `.py` extension) as would be passed to `Notebook.check`. Inserting this into the cell metadata will cause Otter to insert a call to a covertly imported `Notebook` instance with the correct filename and collect the results _at that point of execution_, allowing for robustness to variable name collisions without explicit calls to `Notebook.check`. _Note that these tests are not currently seen by any checks in the notebook, so a call to `Notebook.check_all` will not include the results of these tests._
+
 ## Scripts
 
 If students are submitting Python scripts, they are executed as follows:

otter/execute/execute_notebook.py

@@ -13,9 +13,11 @@
 from IPython.core.inputsplitter import IPythonInputSplitter
 
 from .check_wrapper import CheckCallWrapper
+
 from ..utils import hide_outputs
 
 IGNORE_CELL_TAG = "otter_ignore"
+CELL_METADATA_KEY = "otter"
 
 def filter_ignored_cells(nb):
     """
@@ -34,7 +36,7 @@ def filter_ignored_cells(nb):
         metadata = cell.get("metadata", {})
         tags = metadata.get("tags", [])
 
-        if IGNORE_CELL_TAG in tags:
+        if IGNORE_CELL_TAG in tags or metadata.get(CELL_METADATA_KEY, {}).get("ignore", False):
             del nb['cells'][i]
 
     return nb
@@ -69,10 +71,14 @@ def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False,
         # add display from IPython
         global_env["display"] = display
 
+        # add dummy Notebook class so that we can collect results w/out altering how the CheckCallWrapper
+        # needs to function
+        from ..check.notebook import Notebook
+        notebook_class_name = f"Notebook_{secret}"
+        global_env[notebook_class_name] = Notebook
+
         source = ""
-        # if gradescope:
-        #     source = "import sys\nsys.path.append(\"/autograder/submission\")\n"
-        # el
+
         if cwd:
             source = f"import sys\nsys.path.append(r\"{cwd}\")\n"
             exec(source, global_env)
@@ -83,6 +89,9 @@ def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False,
             global_env["np"] = np
             global_env["random"] = random
 
+        if test_dir is None:
+            test_dir = "/home/tests"
+
         # Before rewriting AST, find cells of code that generate errors.
         # One round of execution is done beforehand to mimic the Jupyter notebook style of running
         # (e.g. code runs up to the point of execution).
@@ -115,10 +124,7 @@ def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False,
                                 # if gradescope:
                                 #     line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/autograder/submission/tests\")", line)
                                 # el
-                                if test_dir:
-                                    line = re.sub(r"otter\.Notebook\(.*?\)", f"otter.Notebook(\"{test_dir}\")", line)
-                                else:
-                                    line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/home/tests\")", line)
+                                line = re.sub(r"otter\.Notebook\(.*?\)", f"otter.Notebook(\"{test_dir}\")", line)
                                 code_lines.append(line)
                                 if source_is_str_bool:
                                     code_lines.append('\n')
@@ -137,6 +143,15 @@ def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False,
                     if not ignore_errors:
                         raise
 
+            # add checks from metadata
+            otter_config = cell.get("metadata", {}).get(CELL_METADATA_KEY, {})
+            check_results_list_name = f"check_results_{secret}"
+            if otter_config.get("tests", []):
+                tests = otter_config.get("tests", [])
+                for test in tests:
+                    source += f"\n{check_results_list_name}.append({notebook_class_name}('{test_dir}').check('{test}'))\n"
+
+
         tree = ast.parse(source)
         # # CODE BELOW COMMENTED OUT BECAUSE the only check function is within the Notebook class
         # if find_check_assignment(tree) or find_check_definition(tree):

otter/generate/autograder.py

@@ -15,23 +15,16 @@
 from .token import APIClient
 
 TEMPLATES_DIR = pkg_resources.resource_filename(__name__, "templates")
-SETUP_SH_PATH = os.path.join(TEMPLATES_DIR, "setup.sh")
-PYTHON_REQUIREMENTS_PATH = os.path.join(TEMPLATES_DIR, "requirements.txt")
-R_REQUIREMENTS_PATH = os.path.join(TEMPLATES_DIR, "requirements.r")
-RUN_AUTOGRADER_PATH = os.path.join(TEMPLATES_DIR, "run_autograder")
-MINICONDA_INSTALL_URL = "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"
-
-with open(SETUP_SH_PATH) as f:
-    SETUP_SH = Template(f.read())
-
-with open(PYTHON_REQUIREMENTS_PATH) as f:
-    PYTHON_REQUIREMENTS = Template(f.read())
-
-with open(R_REQUIREMENTS_PATH) as f:
-    R_REQUIREMENTS = Template(f.read())
-
-with open(RUN_AUTOGRADER_PATH) as f:
-    RUN_AUTOGRADER = Template(f.read())
+MINICONDA_INSTALL_URL = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64.sh"
+OTTER_ENV_NAME = "otter-gradescope-env"
+TEMPLATE_FILE_PATHS = {
+    "setup.sh": os.path.join(TEMPLATES_DIR, "setup.sh"),
+    "requirements.txt": os.path.join(TEMPLATES_DIR, "requirements.txt"),
+    "requirements.r": os.path.join(TEMPLATES_DIR, "requirements.r"),
+    "run_autograder": os.path.join(TEMPLATES_DIR, "run_autograder"),
+    "run_otter.py": os.path.join(TEMPLATES_DIR, "run_otter.py"),
+    "environment.yml":  os.path.join(TEMPLATES_DIR, "environment.yml"),
+}
 
 def main(args):
     """
@@ -53,8 +46,13 @@ def main(args):
     args.lang = args.lang.lower()
     assert args.lang.lower() in ["python", "r"], f"{args.lang} is not a supported language"
 
+    templates = {}
+    for fn, fp in TEMPLATE_FILE_PATHS.items():
+        with open(fp) as f:
+            templates[fn] = Template(f.read())
+
     # format run_autograder
-    run_autograder = RUN_AUTOGRADER.render(
+    run_otter_py = templates["run_otter.py"].render(
         threshold = str(args.threshold),
         points = str(args.points),
         show_stdout = str(args.show_stdout),
@@ -72,11 +70,20 @@ def main(args):
         autograder_dir = str(args.autograder_dir),
     )
 
+    run_autograder = templates["run_autograder"].render(
+        autograder_dir = str(args.autograder_dir),
+    )
+
     # format setup.sh
-    setup_sh = SETUP_SH.render(
+    setup_sh = templates["setup.sh"].render(
         autograder_dir = str(args.autograder_dir),
         miniconda_install_url = MINICONDA_INSTALL_URL,
         ottr_branch = "stable",
+        otter_env_name = OTTER_ENV_NAME,
+    )
+
+    environment_yml = templates["environment.yml"].render(
+        otter_env_name = OTTER_ENV_NAME,
     )
 
     # create tmp directory to zip inside
@@ -97,15 +104,15 @@ def main(args):
             f = open(os.devnull)
 
         # render the templates
-        python_requirements = PYTHON_REQUIREMENTS.render(
+        python_requirements = templates["requirements.txt"].render(
             other_requirements = f.read() if args.lang.lower() == "python" else "",
             overwrite_requirements = args.lang.lower() == "python" and args.overwrite_requirements
         )
 
         # reset the stream
         f.seek(0)
 
-        r_requirements = R_REQUIREMENTS.render(
+        r_requirements = templates["requirements.r"].render(
             other_requirements = f.read() if args.lang.lower() == "r" else "",
             overwrite_requirements = args.lang.lower() == "python" and args.overwrite_requirements
 
@@ -132,6 +139,12 @@ def main(args):
         with open(os.path.join(os.getcwd(), "tmp", "run_autograder"), "w+") as f:
             f.write(run_autograder)
 
+        with open(os.path.join(os.getcwd(), "tmp", "run_otter.py"), "w+") as f:
+            f.write(run_otter_py)
+
+        with open(os.path.join(os.getcwd(), "tmp", "environment.yml"), "w+") as f:
+            f.write(environment_yml)
+
         # copy files into tmp
         if len(args.files) > 0:
             os.mkdir(os.path.join("tmp", "files"))
@@ -158,8 +171,8 @@ def main(args):
         if os.path.exists(zip_path):
             os.remove(zip_path)
 
-        zip_cmd = ["zip", "-r", zip_path, "run_autograder", "requirements.r",
-                "setup.sh", "requirements.txt", "tests"]
+        zip_cmd = ["zip", "-r", zip_path, "run_autograder", "run_otter.py", "requirements.r",
+                "setup.sh", "requirements.txt", "environment.yml", "tests"]
 
         if r_requirements:
             zip_cmd += ["requirements.r"]

otter/generate/templates/environment.yml

@@ -0,0 +1,12 @@
+name: {{ otter_env_name }}
+channels:
+    - defaults
+    - conda-forge
+dependencies:
+    - python=3.7
+    - pip
+    - r-base 
+    - r-essentials 
+    - r-devtools
+    - pip:
+        - -r requirements.txt

otter/generate/templates/requirements.txt

@@ -15,6 +15,6 @@ rpy2
 jupytext
 numpy==1.16.0
 tornado==5.1.1
-otter-grader==1.0.1
+otter-grader==1.1.0
 {% endif %}{% if other_requirements %}
 {{ other_requirements }}{% endif %}

otter/generate/templates/run_autograder

@@ -1,28 +1,6 @@
-#!/usr/bin/env python3
+#!/usr/bin/env bash
 
-import os
-import subprocess
-
-from otter.generate.run_autograder import main as run_autograder
-
-config = {
-    "score_threshold": {{ threshold }},
-    "points_possible": {{ points }},
-    "show_stdout_on_release": {{ show_stdout }},
-    "show_hidden_tests_on_release": {{ show_hidden }},
-    "seed": {{ seed }},
-    "grade_from_log": {{ grade_from_log }},
-    "serialized_variables": {{ serialized_variables }},
-    "public_multiplier": {{ public_multiplier }},
-    "token": {% if token %}'{{ token }}'{% else %}None{% endif %},
-    "course_id": '{{ course_id }}',
-    "assignment_id": '{{ assignment_id }}',
-    "filtering": {{ filtering }},
-    "pagebreaks": {{ pagebreaks }},
-    "debug": False,
-    "autograder_dir": '{{ autograder_dir }}',
-    "lang": '{{ lang }}',
-}
-
-if __name__ == "__main__":
-    run_autograder(config)
+export PATH="/root/miniconda3/bin:$PATH"
+source /root/miniconda3/etc/profile.d/conda.sh
+conda activate otter-gradescope-env
+python {{ autograder_dir }}/source/run_otter.py

otter/generate/templates/run_otter.py

@@ -0,0 +1,30 @@
+"""
+Runs Otter on Gradescope with the configurations specified below
+"""
+
+import os
+import subprocess
+
+from otter.generate.run_autograder import main as run_autograder
+
+config = {
+    "score_threshold": {{ threshold }},
+    "points_possible": {{ points }},
+    "show_stdout_on_release": {{ show_stdout }},
+    "show_hidden_tests_on_release": {{ show_hidden }},
+    "seed": {{ seed }},
+    "grade_from_log": {{ grade_from_log }},
+    "serialized_variables": {{ serialized_variables }},
+    "public_multiplier": {{ public_multiplier }},
+    "token": {% if token %}'{{ token }}'{% else %}None{% endif %},
+    "course_id": '{{ course_id }}',
+    "assignment_id": '{{ assignment_id }}',
+    "filtering": {{ filtering }},
+    "pagebreaks": {{ pagebreaks }},
+    "debug": False,
+    "autograder_dir": '{{ autograder_dir }}',
+    "lang": '{{ lang }}',
+}
+
+if __name__ == "__main__":
+    run_autograder(config)

otter/generate/templates/setup.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-apt-get clean
-apt-get update
-apt-get install -y python3.7 python3-pip python3.7-dev
+# apt-get clean
+# apt-get update
+# apt-get install -y python3.7 python3-pip python3.7-dev
 
 apt-get clean
 apt-get update
@@ -13,11 +13,11 @@ apt-get install -y texlive-xetex texlive-fonts-recommended texlive-generic-recom
 wget --quiet -O /tmp/wkhtmltopdf.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.bionic_amd64.deb
 apt-get install -y /tmp/wkhtmltopdf.deb
 
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+# update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
 
 apt-get clean
 apt-get update
-apt-get install -y install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libcurl4-openssl-dev
+apt-get install -y build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libcurl4-openssl-dev
 
 # install conda
 wget -nv -O {{ autograder_dir }}/source/miniconda_install.sh "{{ miniconda_install_url }}"
@@ -28,16 +28,24 @@ echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
 export PATH=/root/miniconda3/bin:$PATH
 export TAR="/bin/tar"
 
-# install R dependencies
-conda install --yes r-base r-essentials 
-conda install --yes r-devtools -c conda-forge
+# # install R dependencies
+# conda install --yes r-base r-essentials 
+# conda install --yes r-devtools -c conda-forge
+
+# # install requirements
+# pip3 install -r {{ autograder_dir }}/source/requirements.txt
+# pip install -r {{ autograder_dir }}/source/requirements.txt
+# Rscript {{ autograder_dir }}/source/requirements.r
+
+# install dependencies with conda
+conda env create -f {{ autograder_dir }}/source/environment.yml
+conda run -n {{ otter_env_name }} Rscript {{ autograder_dir }}/source/requirements.r
 
-# install requirements
-pip3 install -r {{ autograder_dir }}/source/requirements.txt
-pip install -r {{ autograder_dir }}/source/requirements.txt
-Rscript {{ autograder_dir }}/source/requirements.r
+# set conda shell
+conda init --all
 
 # install ottr; not sure why it needs to happen twice but whatever
 git clone --single-branch -b {{ ottr_branch }} https://github.com/ucbds-infra/ottr.git {{ autograder_dir }}/source/ottr
 cd {{ autograder_dir }}/source/ottr 
-Rscript -e "devtools::install()" || Rscript -e "devtools::install()"
+conda run -n {{ otter_env_name }} Rscript -e "devtools::install\\(\\)"
+conda run -n {{ otter_env_name }} Rscript -e "devtools::install\\(\\)"