Add course source directory for local testing without git

If the git origin is empty in the course settings, then the course
source files are copied from the new course source directory
(if the local source directory is enabled in the settings).
Course directories may be mounted to the source directory in local
testing so that changes in the course may be built without any
git setup.

It used to be (and still is) possible to leave the git origin empty
and mount course source directories directly to the build directory,
but that setup is prone to file permission issues. The build fails
if the build process in the container does not have write access
to the build directory.

In the run-gitmanager container image, remove COURSES_PATH
`/srv/courseshuey` since it is not useful any longer.
Unset git origin and leave it empty in the database setup
for the default course.

access/config.py

@@ -263,6 +263,10 @@ def file_paths(key: str, source: ConfigSource = ConfigSource.PUBLISH) -> Tuple[s
             CourseConfig.version_id_path(key, source=source),
         )
 
+    @staticmethod
+    def local_source_path_to(key: str, *paths: str) -> str:
+        return CourseConfig._path_to(settings.LOCAL_COURSE_SOURCE_PATH, key, *paths)
+
     def static_path_to(self, *paths: str) -> Optional[str]:
         if self.data.static_dir is Undefined:
             return None

builder/builder.py

@@ -402,8 +402,6 @@ def push_event(
 
     update = updates[-1]
 
-    path = CourseConfig.path_to(course_key)
-
     log_stream = StringIO()
     log_handler = logging.StreamHandler(log_stream)
     build_logger.addHandler(log_handler)
@@ -419,6 +417,18 @@ def push_event(
             pull_status = pull(str(build_path), course.git_origin, course.git_branch, logger=build_logger)
             if not pull_status:
                 return
+        elif settings.LOCAL_COURSE_SOURCE_PATH:
+            path = CourseConfig.local_source_path_to(course_key)
+            build_logger.debug(f"Course origin not set: copying the course sources from {path} to the build directory.")
+
+            def ignore_func(directory, contents):
+                # Do not copy the .git directory in shutil.copytree().
+                if '.git' in contents:
+                    return ('.git',)
+                return ()
+
+            shutil.rmtree(build_path, ignore_errors=True)
+            shutil.copytree(path, build_path, symlinks=True, ignore=ignore_func)
         else:
             build_logger.warning(f"Course origin not set: skipping git update\n")
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM apluslms/service-base:django-1.13
+FROM apluslms/service-base:django-1.14
 
 # Set container related configuration via environment variables.
 ENV CONTAINER_TYPE="gitmanager" \

docker/Dockerfile.huey

@@ -14,9 +14,6 @@ RUN touch /etc/services.d/postgresql/down \
   # Combine Django settings for the task queue worker (Huey).
  && cp /srv/gitmanager-cont-settings.py /srv/gitmanagerhuey-cont-settings.py \
  && cat /srv/gitmanagerhuey-custom-cont-settings.py >> /srv/gitmanagerhuey-cont-settings.py \
-  # Course directory for the worker.
-  # The course directory may be mounted here so that the container does not need to clone any git repos.
- && mkdir -p /srv/courseshuey/ \
  && :
 
 

docker/docker-compose-full.yml

@@ -51,8 +51,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - /tmp/aplus:/tmp/aplus
       - coursebuild:/srv/courses
-      - .:/srv/courseshuey/default:ro
-      - .:/srv/courses/git_repo/default:ro
+      - .:/srv/courses/source/default:ro
       #- $HOME/gitmanager/:/srv/gitmanager/:ro
     depends_on:
       - gitmanager

docker/docker-compose-immediate.yml

@@ -7,9 +7,7 @@ version: '3'
 # The gitmanager container runs the build tasks immediately
 # with an in-memory storage (no Redis needed).
 # The course git repo is mounted to the gitmanager container and
-# gitmanager git-clones the course for the build.
-# The disadvantage is that git-cloning includes only committed changes, that is,
-# uncommitted changes are not part of the build in this configuration.
+# gitmanager copies the course for the build.
 
 volumes:
   data:
@@ -38,7 +36,7 @@ services:
       - data:/data
       - /var/run/docker.sock:/var/run/docker.sock
       - /tmp/aplus:/tmp/aplus
-      - .:/srv/courses/git_repo/default:ro
+      - .:/srv/courses/source/default:ro
       #- $HOME/gitmanager/:/srv/gitmanager/:ro
     ports:
       - "8070:8070"

docker/rootfs/srv/db-gitmanager-bootstrap.py

@@ -10,7 +10,7 @@ def create_default_courses():
     course = Course.objects.create(
         key='default',
         remote_id=1,
-        git_origin='/srv/courses/git_repo/default',
+        git_origin='',
         git_branch='master',
         webhook_secret='',
     )

docker/rootfs/srv/gitmanager-cont-settings.py

@@ -26,6 +26,7 @@
 STORE_PATH = '/srv/courses/course_store'
 COURSES_PATH = '/srv/courses/publish'
 BUILD_PATH = '/tmp/aplus/gitmanager/build'
+LOCAL_COURSE_SOURCE_PATH = '/srv/courses/source'
 # See the BUILD_MODULE script for details
 BUILD_MODULE_SETTINGS = {
     'HOST_BUILD_PATH': BUILD_PATH,

docker/rootfs/srv/gitmanagerhuey-custom-cont-settings.py

@@ -2,11 +2,6 @@
 # with some modifications for the Huey worker (task queue consumer).
 # Dockerfile.huey adds gitmanager-cont-settings.py to the start of this file.
 
-# In the worker, COURSES_PATH does not need to be in the same device as STORE_PATH
-# because the worker never writes to COURSES_PATH.
-# The worker only copies the build output to STORE_PATH.
-COURSES_PATH = '/srv/courseshuey'
-
 # Connect to the database in the gitmanager web app.
 DATABASES['default'].update({
     'HOST': 'gitmanager',

gitmanager/settings.py

@@ -55,6 +55,8 @@
 BUILD_MODULE_SETTINGS = {
   "HOST_BUILD_PATH": BUILD_PATH,
 }
+# local course source directory for testing the build without cloning anything from git
+LOCAL_COURSE_SOURCE_PATH = None
 
 # Whether to use the X-SendFile
 USE_X_SENDFILE = False