Merge pull request #665 from chrispyles/mamba

ucbds-infra · Jul 16, 2023 · 08a1052 · 08a1052
2 parents 62ceab1 + 753b879
commit 08a1052
Show file tree

Hide file tree

Showing 28 changed files with 220 additions and 204 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,7 @@
 * Removed `linux/amd64` platform specification for Docker images in Otter Grade
 * Updated Otter Assign to normalize notebooks before writing them with `nbformat.validator.normalize` per [#658](https://github.com/ucbds-infra/otter-grader/issues/658)
 * Added summary of assignment questions to Otter Assign logging per [#564](https://github.com/ucbds-infra/otter-grader/issues/564)
+* Converted to Mamba from Conda for package management in grading containers per [#660](https://github.com/ucbds-infra/otter-grader/issues/660)
 
 **v4.4.0:**
 

diff --git a/Dockerfile b/Dockerfile
@@ -10,20 +10,24 @@ COPY --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 # Install apt packages
 RUN apt-get update && apt-get upgrade -y && apt-get install -y wget make build-essential
 
-# Install conda
-RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-    -O /tmp/miniconda_install.sh && \
-    chmod +x /tmp/miniconda_install.sh && \
-    /tmp/miniconda_install.sh -b
-ENV PATH=/root/miniconda3/bin:$PATH
+# Install mamba
+RUN if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
+        then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+            -O /tmp/mamba_install.sh ; \
+        else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+            -O /tmp/mamba_install.sh ; \
+    fi && \
+    chmod +x /tmp/mamba_install.sh && \
+    /tmp/mamba_install.sh -b
+ENV PATH=/root/mambaforge/bin:$PATH
 
-# Create conda environment
+# Create mamba environment
 COPY environment.yml requirements.txt requirements-export.txt requirements-test.txt /tmp/
 RUN mkdir -p /tmp/docs && touch /tmp/docs/requirements.txt
-RUN conda env create -f /tmp/environment.yml
+RUN mamba env create -f /tmp/environment.yml
+RUN mamba run -n otter-grader Rscript -e 'install.packages("ottr", dependencies=TRUE, repos="https://cran.r-project.org")'
 ADD . /root/otter-grader
-RUN conda run -n otter-grader Rscript -e 'install.packages("ottr", dependencies=TRUE, repos="https://cran.r-project.org")'
-RUN conda run -n otter-grader pip install /root/otter-grader
+RUN mamba run -n otter-grader pip install /root/otter-grader
 
 # Make script executable
 RUN chmod +x /root/otter-grader/bin/run_tests
diff --git a/bin/run_tests b/bin/run_tests
@@ -3,8 +3,9 @@
 # Runs Otter's tests and copies the resulting coverage.xml file into the /tmp directory. This script
 # should only be used inside a Docker container created by /Dockerfile.
 
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
 cd /root/otter-grader
-conda activate otter-grader
+mamba activate otter-grader
 make testcov PYTESTOPTS="-vv" && coverage xml -i && cp ./coverage.xml /tmp/coverage.xml
diff --git a/otter/generate/__init__.py b/otter/generate/__init__.py
@@ -23,7 +23,6 @@
 
 
 DEFAULT_PYTHON_VERSION = "3.7"
-MINICONDA_INSTALL_URL = "https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh"
 OTTER_ENV_NAME = "otter-env"
 OTTR_VERSION = "1.2.0"
 TEMPLATE_DIR = pkg_resources.resource_filename(__name__, "templates")
@@ -222,7 +221,6 @@ def main(*, tests_dir="./tests", output_path="autograder.zip", config=None, no_c
     template_context = {
         "autograder_dir": ag_config.autograder_dir,
         "otter_env_name": OTTER_ENV_NAME,
-        "miniconda_install_url": MINICONDA_INSTALL_URL,
         "ottr_version": OTTR_VERSION,
         "channel_priority_strict": channel_priority_strict,
         "has_r_requirements": False,

diff --git a/otter/generate/templates/common/run_autograder b/otter/generate/templates/common/run_autograder
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
-conda activate {{ otter_env_name }}
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
+mamba activate {{ otter_env_name }}
 python {{ autograder_dir }}/source/run_otter.py
diff --git a/otter/generate/templates/python/setup.sh b/otter/generate/templates/python/setup.sh
@@ -6,22 +6,22 @@ apt-get update
 apt-get install -y wget pandoc texlive-xetex texlive-fonts-recommended texlive-plain-generic \
     build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libgit2-dev texlive-lang-chinese
 
-# install conda
+# install mamba
 if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
-    then wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
-        -O {{ autograder_dir }}/source/miniconda_install.sh ; \
-    else wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        -O {{ autograder_dir }}/source/miniconda_install.sh ; \
+    then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+        -O {{ autograder_dir }}/source/mamba_install.sh ; \
+    else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+        -O {{ autograder_dir }}/source/mamba_install.sh ; \
 fi
-chmod +x {{ autograder_dir }}/source/miniconda_install.sh
-{{ autograder_dir }}/source/miniconda_install.sh -b
-echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
+chmod +x {{ autograder_dir }}/source/mamba_install.sh
+{{ autograder_dir }}/source/mamba_install.sh -b
+echo "export PATH=/root/mambaforge/bin:\$PATH" >> /root/.bashrc
 
-export PATH=/root/miniconda3/bin:$PATH
+export PATH=/root/mambaforge/bin:$PATH
 export TAR="/bin/tar"
 
-# install dependencies with conda
-conda env create -f {{ autograder_dir }}/source/environment.yml
+# install dependencies with mamba
+mamba env create -f {{ autograder_dir }}/source/environment.yml
 
-# set conda shell
-conda init --all
+# set mamba shell
+mamba init --all
diff --git a/otter/generate/templates/r/setup.sh b/otter/generate/templates/r/setup.sh
@@ -11,27 +11,27 @@ apt-get install -y libnlopt-dev cmake libfreetype6-dev libpng-dev libtiff5-dev l
     build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libgit2-dev texlive-lang-chinese \
     libxft-dev
 
-# install conda
+# install mamba
 if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
-    then wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
-        -O {{ autograder_dir }}/source/miniconda_install.sh ; \
-    else wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        -O {{ autograder_dir }}/source/miniconda_install.sh ; \
+    then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+        -O {{ autograder_dir }}/source/mamba_install.sh ; \
+    else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+        -O {{ autograder_dir }}/source/mamba_install.sh ; \
 fi
-chmod +x {{ autograder_dir }}/source/miniconda_install.sh
-{{ autograder_dir }}/source/miniconda_install.sh -b
-echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
+chmod +x {{ autograder_dir }}/source/mamba_install.sh
+{{ autograder_dir }}/source/mamba_install.sh -b
+echo "export PATH=/root/mambaforge/bin:\$PATH" >> /root/.bashrc
 
-export PATH=/root/miniconda3/bin:$PATH
+export PATH=/root/mambaforge/bin:$PATH
 export TAR="/bin/tar"
 
-# install dependencies with conda{% if channel_priority_strict %}
-conda config --set channel_priority strict{% endif %}
-conda env create -f {{ autograder_dir }}/source/environment.yml{% if has_r_requirements %}
-conda run -n {{ otter_env_name }} Rscript {{ autograder_dir }}/source/requirements.r{% endif %}
+# install dependencies with mamba{% if channel_priority_strict %}
+mamba config --set channel_priority strict{% endif %}
+mamba env create -f {{ autograder_dir }}/source/environment.yml{% if has_r_requirements %}
+mamba run -n {{ otter_env_name }} Rscript {{ autograder_dir }}/source/requirements.r{% endif %}
 
-# set conda shell
-conda init --all
+# set mamba shell
+mamba init --all
 
-# install ottr; not sure why it needs to happen twice but whatever
-conda run -n otter-env Rscript -e 'install.packages("https://cran.r-project.org/package=ottr&version={{ ottr_version }}", dependencies=TRUE, repos=NULL)'
+# install ottr
+mamba run -n otter-env Rscript -e 'install.packages("https://cran.r-project.org/package=ottr&version={{ ottr_version }}", dependencies=TRUE, repos=NULL)'
diff --git a/otter/run/run_autograder/autograder_config.py b/otter/run/run_autograder/autograder_config.py
@@ -104,8 +104,8 @@ class AutograderConfig(fica.Config):
     )
 
     miniconda_path = fica.Key(
-        description="the path to the miniconda install directory",
-        default="/root/miniconda3",
+        description="the path to the mamba install directory",
+        default="/root/mambaforge",
     )
 
     plugins = fica.Key(

diff --git a/test/files/Dockerfile b/test/files/Dockerfile
@@ -1,6 +1,6 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
-ENV PATH=/root/miniconda3/bin:$PATH
+ENV PATH=/root/mambaforge/bin:$PATH
 ADD . /tmp/otter-grader
-RUN conda run -n otter-env pip install /tmp/otter-grader
+RUN mamba run -n otter-env pip install /tmp/otter-grader
 RUN rm -rf /tmp/*
diff --git a/test/test_assign/files/example-autograder-correct/run_autograder b/test/test_assign/files/example-autograder-correct/run_autograder
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
-conda activate otter-env
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
+mamba activate otter-env
 python /autograder/source/run_otter.py
diff --git a/test/test_assign/files/example-autograder-correct/setup.sh b/test/test_assign/files/example-autograder-correct/setup.sh
@@ -6,22 +6,22 @@ apt-get update
 apt-get install -y wget pandoc texlive-xetex texlive-fonts-recommended texlive-plain-generic \
     build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libgit2-dev texlive-lang-chinese
 
-# install conda
+# install mamba
 if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
-    then wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
-    else wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
+    then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+        -O /autograder/source/mamba_install.sh ; \
+    else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+        -O /autograder/source/mamba_install.sh ; \
 fi
-chmod +x /autograder/source/miniconda_install.sh
-/autograder/source/miniconda_install.sh -b
-echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
+chmod +x /autograder/source/mamba_install.sh
+/autograder/source/mamba_install.sh -b
+echo "export PATH=/root/mambaforge/bin:\$PATH" >> /root/.bashrc
 
-export PATH=/root/miniconda3/bin:$PATH
+export PATH=/root/mambaforge/bin:$PATH
 export TAR="/bin/tar"
 
-# install dependencies with conda
-conda env create -f /autograder/source/environment.yml
+# install dependencies with mamba
+mamba env create -f /autograder/source/environment.yml
 
-# set conda shell
-conda init --all
+# set mamba shell
+mamba init --all
diff --git a/test/test_assign/files/gs-autograder-correct/run_autograder b/test/test_assign/files/gs-autograder-correct/run_autograder
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
-conda activate otter-env
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
+mamba activate otter-env
 python /autograder/source/run_otter.py
diff --git a/test/test_assign/files/gs-autograder-correct/setup.sh b/test/test_assign/files/gs-autograder-correct/setup.sh
@@ -6,22 +6,22 @@ apt-get update
 apt-get install -y wget pandoc texlive-xetex texlive-fonts-recommended texlive-plain-generic \
     build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libgit2-dev texlive-lang-chinese
 
-# install conda
+# install mamba
 if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
-    then wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
-    else wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
+    then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+        -O /autograder/source/mamba_install.sh ; \
+    else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+        -O /autograder/source/mamba_install.sh ; \
 fi
-chmod +x /autograder/source/miniconda_install.sh
-/autograder/source/miniconda_install.sh -b
-echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
+chmod +x /autograder/source/mamba_install.sh
+/autograder/source/mamba_install.sh -b
+echo "export PATH=/root/mambaforge/bin:\$PATH" >> /root/.bashrc
 
-export PATH=/root/miniconda3/bin:$PATH
+export PATH=/root/mambaforge/bin:$PATH
 export TAR="/bin/tar"
 
-# install dependencies with conda
-conda env create -f /autograder/source/environment.yml
+# install dependencies with mamba
+mamba env create -f /autograder/source/environment.yml
 
-# set conda shell
-conda init --all
+# set mamba shell
+mamba init --all
diff --git a/test/test_assign/files/rmd-autograder-correct/run_autograder b/test/test_assign/files/rmd-autograder-correct/run_autograder
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
-conda activate otter-env
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
+mamba activate otter-env
 python /autograder/source/run_otter.py
diff --git a/test/test_assign/files/rmd-autograder-correct/setup.sh b/test/test_assign/files/rmd-autograder-correct/setup.sh
@@ -11,26 +11,26 @@ apt-get install -y libnlopt-dev cmake libfreetype6-dev libpng-dev libtiff5-dev l
     build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libgit2-dev texlive-lang-chinese \
     libxft-dev
 
-# install conda
+# install mamba
 if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
-    then wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
-    else wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
+    then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+        -O /autograder/source/mamba_install.sh ; \
+    else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+        -O /autograder/source/mamba_install.sh ; \
 fi
-chmod +x /autograder/source/miniconda_install.sh
-/autograder/source/miniconda_install.sh -b
-echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
+chmod +x /autograder/source/mamba_install.sh
+/autograder/source/mamba_install.sh -b
+echo "export PATH=/root/mambaforge/bin:\$PATH" >> /root/.bashrc
 
-export PATH=/root/miniconda3/bin:$PATH
+export PATH=/root/mambaforge/bin:$PATH
 export TAR="/bin/tar"
 
-# install dependencies with conda
-conda config --set channel_priority strict
-conda env create -f /autograder/source/environment.yml
+# install dependencies with mamba
+mamba config --set channel_priority strict
+mamba env create -f /autograder/source/environment.yml
 
-# set conda shell
-conda init --all
+# set mamba shell
+mamba init --all
 
-# install ottr; not sure why it needs to happen twice but whatever
-conda run -n otter-env Rscript -e 'install.packages("https://cran.r-project.org/package=ottr&version=1.2.0", dependencies=TRUE, repos=NULL)'
+# install ottr
+mamba run -n otter-env Rscript -e 'install.packages("https://cran.r-project.org/package=ottr&version=1.2.0", dependencies=TRUE, repos=NULL)'
diff --git a/test/test_generate/files/autograder-correct/run_autograder b/test/test_generate/files/autograder-correct/run_autograder
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
-conda activate otter-env
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
+mamba activate otter-env
 python /autograder/source/run_otter.py
diff --git a/test/test_generate/files/autograder-correct/setup.sh b/test/test_generate/files/autograder-correct/setup.sh
@@ -6,22 +6,22 @@ apt-get update
 apt-get install -y wget pandoc texlive-xetex texlive-fonts-recommended texlive-plain-generic \
     build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev libgit2-dev texlive-lang-chinese
 
-# install conda
+# install mamba
 if [ $(uname -p) = "arm" ] || [ $(uname -p) = "aarch64" ] ; \
-    then wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
-    else wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        -O /autograder/source/miniconda_install.sh ; \
+    then wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh \
+        -O /autograder/source/mamba_install.sh ; \
+    else wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
+        -O /autograder/source/mamba_install.sh ; \
 fi
-chmod +x /autograder/source/miniconda_install.sh
-/autograder/source/miniconda_install.sh -b
-echo "export PATH=/root/miniconda3/bin:\$PATH" >> /root/.bashrc
+chmod +x /autograder/source/mamba_install.sh
+/autograder/source/mamba_install.sh -b
+echo "export PATH=/root/mambaforge/bin:\$PATH" >> /root/.bashrc
 
-export PATH=/root/miniconda3/bin:$PATH
+export PATH=/root/mambaforge/bin:$PATH
 export TAR="/bin/tar"
 
-# install dependencies with conda
-conda env create -f /autograder/source/environment.yml
+# install dependencies with mamba
+mamba env create -f /autograder/source/environment.yml
 
-# set conda shell
-conda init --all
+# set mamba shell
+mamba init --all
diff --git a/test/test_generate/files/autograder-custom-env/run_autograder b/test/test_generate/files/autograder-custom-env/run_autograder
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-export PATH="/root/miniconda3/bin:$PATH"
-source /root/miniconda3/etc/profile.d/conda.sh
-conda activate otter-env
+export PATH="/root/mambaforge/bin:$PATH"
+source /root/mambaforge/etc/profile.d/conda.sh
+source /root/mambaforge/etc/profile.d/mamba.sh
+mamba activate otter-env
 python /autograder/source/run_otter.py