Merge main back into this branch

.github/workflows/docker-publish-and-test.yml

@@ -238,14 +238,8 @@ jobs:
 
       - name: Test minimal image
         run: |
-          bash test/run_tests.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test pm_fast_tests
-          bash test/run_tests.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test sm_fast_tests
-          bash test/run_vcs.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test pm_fast_vcs
-          bash test/run_vcs.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test sm_fast_vcs
-          bash test/run_tests.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test sp_sm_test
-
-      # bash test/run_tests.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test equilibriumcrystalstructure
-
+          bash test/run_all.sh ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test all
+
       # Re-run the previous buildx action, except this time push the image if needed. Because of caching, this
       # does not rebuild the image.
       #

README.md

@@ -81,6 +81,8 @@ Alternatively, if you want the full image, do this:
 docker pull ghcr.io/openkim/developer-platform
 docker run -it --name kim_dev ghcr.io/openkim/developer-platform bash
 ```
+*NOTE:* If you wish to install and use JupyterLab to work in your container (see section regarding IDEs at the bottom of the page), you must expose a port by e.g. adding the argument `-p 8888:8888` to your `docker run` command.
+
 The container will automatically stop when you close the original bash session
 that was opened by doing `docker run` above; this can be verified by doing
 `docker ps`, which will reveal no running containers.  This will *not* cause
@@ -161,6 +163,38 @@ resource allocation on a container-by-container basis can be found at
 https://docs.docker.com/config/containers/resource_constraints/.  In linux, resource
 control must be done on a container-by-container basis.
 
+## IDEs (VSCode and Jupyter Lab)
+
+To work inside the container using VSCode, you can use the "Dev Containers"
+extension which should be installed by default. Select "Remote Explorer" in
+the left sidebar, then choose "Dev Containers" in the drop down menu. Your
+container should be under "Other Containers", and you can attach VSCode to it
+using one of the icons on the right. See image below with relevant UI elements:
+
+![VSCode screenshot](doc_img/vscode.png)
+
+Once you have attached VSCode to your container once, it will appear under 
+"Dev Containers" rather than "Other Containers".
+
+To use Jupyter Lab to access the container, you must have issued your `docker run` command
+with the option to expose a port (e.g. `-p 8888:8888`). Then, you can install Jupyter Lab using
+```
+sudo pip install jupyterlab
+```
+It is a large installation, so we do not include it as not all users will need it. Then you can use JupyterLab by running
+
+```
+jupyter lab --ip 0.0.0.0
+```
+inside the container, or from your host terminal by running 
+```
+docker exec kim_dev jupyter lab --ip 0.0.0.0
+```
+where `kim_dev` should be replaced if you named your container something different.
+You should now be able to access the Jupyter Lab IDE inside your container by
+pointing your browser at one of the displayed URLs (typically the one starting with `http://127.0.0.1:8888/lab?token=`).
+
+
 ## References
 
 [1] "The OpenKIM Processing Pipeline: A Cloud-Based Automatic Materials

docker/config/Dockerfile

@@ -136,5 +136,12 @@ RUN chmod 4755 /usr/bin/sudo
 RUN chown -R openkim:openkim /home/openkim/
 RUN chown -R openkim:openkim /pipeline/
 
+
+# TODO: Maybe a better way to do this? I want kim-property to be editable
+RUN pip3 uninstall -y kim-property
+
 USER openkim
+
+RUN pip3 --no-cache-dir install kim-property==2.6.3
+
 WORKDIR /home/openkim/

docker/config/excerpts/query_local/helper_functions.py

@@ -542,8 +542,8 @@ def item_is_latest(kimcode):
     version than the one given.
     """
     # Discard human-readable prefix, if any
-    name, leader, num, ver = parse_kim_code(kimcode)
-    short_id = ("_").join((leader, num, ver))
+    _, leader, num, ver = parse_kim_code(kimcode)
+    shortcode = ("_").join((leader, num))
     leader = leader.lower()
 
     if leader in ["te", "vc"]:
@@ -558,9 +558,9 @@ def item_is_latest(kimcode):
     # Don't need to turn on 'history' since we're interested in the latest
     query = {
         "database": "data",
-        "query": {"meta." + item_type + ".short-id": short_id},
-        "project": ["meta." + item_type + ".version"],
-        "sort": [["meta." + item_type + ".version", -1]],
+        "query": {"meta." + item_type + ".shortcode": shortcode},
+        "project": ["meta." + item_type + ".kimid-version"],
+        "sort": [["meta." + item_type + ".kimid-version", -1]],
         "limit": 1,
     }
 

docker/config/excerpts/template.py

@@ -129,6 +129,9 @@ def intercept_query(query, subject_name, local, infofile):
 
         # If we're querying from pipeline.stdin.tpl for a stale Model, add history and sort accordingly
         if add_history:
+            print ("WARNING: Querying for stale models will currently return ALL results for that model.\n"
+                   "If your pipeline.stdin.tpl query does not include a 'limit':1 option,\n"
+                   "you may get duplicate and/or 'double stale' (stale model and stale test) results.\n")
             query["history"] = True
             orig_sort = query.get("sort", None)
             if orig_sort:

docker/config/excerpts/util.py

@@ -184,7 +184,6 @@ def item_is_latest(kimcode):
         # Discard human-readable prefix, if any
         _, leader, num, ver = kimcodes.parse_kim_code(kimcode)
         short_id = ("_").join((leader, num, ver))
-        leader = leader.lower()
 
         # Query remote database
         query = {
@@ -200,12 +199,10 @@ def item_is_latest(kimcode):
         # onto a single key, a scalar is returned)
         if isinstance(query_result, list):
             if query_result:
-                latest_ver = query_result[0]
+                return query_result[0]
             else:
                 # Couldn't find any results, so we must have the latest version of the item
                 # as far as the local database is concerned
                 return True
         else:
-            latest_ver = query_result
-
-        return ver == latest_ver
+            return query_result

docker/git/Dockerfile

@@ -20,14 +20,14 @@ RUN git clone -q https://github.com/openkim/kim-python-utils -b master ${PACKAGE
   && git checkout e4e21b202264373a9f33dfc47b6e05c0af625950
 RUN git clone -q https://github.com/openkim/crystal-genome-util -b main ${PACKAGE_DIR}/crystal-genome-util \
   && cd ${PACKAGE_DIR}/crystal-genome-util \
-  && git checkout 92e0b4c54ed4ca4e7ef1630499df2c8651bc2333 
-RUN git clone -q https://github.com/openkim/kim-property.git -b spoof-version ${PACKAGE_DIR}/kim-property \
-  && cd ${PACKAGE_DIR}/kim-property \
-  && git checkout 61f230c616c9636ea649ad67c26fd6a0a2115e0c
+  && git checkout e18a2d62fc3e391acbf9c98a28efaebca914b007
 RUN git clone -q https://github.com/lammps/lammps -b stable_2Aug2023_update1 ${PACKAGE_DIR}/lammps
 RUN git clone -q https://gitlab.com/openkim/ase -b user-species ${PACKAGE_DIR}/ase \
   && cd ${PACKAGE_DIR}/ase \
-  && git checkout e00c8da0f84fb3a726c411f7ef2792241434c9a3
+  && git checkout abe2d6c6e265f7ba6e79cc9b437ef1940731eccc
 RUN git clone -q https://gitlab.com/micronano_public/MDpp -b release ${PACKAGE_DIR}/MD++ \
   && cd ${PACKAGE_DIR}/MD++ \
   && git checkout f7d64a7720a4bc1602371a128c8db7779fcf8dcb
+RUN git clone -q https://github.com/openkim/kim-tools -b main ${PACKAGE_DIR}/kim-tools \
+  && cd ${PACKAGE_DIR}/kim-tools \
+  && git checkout 003b84fdab9f955c607816fbbe3827c3ae3a4bc7

docker/install/Dockerfile

@@ -38,7 +38,8 @@ RUN ${PIP} install packaging==20.9
 RUN ${PIP} install markupsafe==2.0.1
 RUN ${PIP} install Jinja2==2.11.3
 RUN ${PIP} install edn_format==0.7.5
-RUN ${PIP} install kim-edn==1.3.1
+RUN ${PIP} install kim-edn==1.4.1
+RUN ${PIP} install kim-property==2.6.3
 RUN ${PIP} install kim-query==3.0.0
 RUN ${PIP} install simplejson==3.17.2
 RUN ${PIP} install numpy==1.19.5
@@ -88,18 +89,8 @@ RUN cd ${PACKAGE_DIR} \
 #########################################
 ## numdifftools
 #########################################
-# Note that installing from the github repo rather than PyPI requires that git
-# be installed because it uses pbr (https://docs.openstack.org/pbr/latest/). If
-# we really want, we can build an sdist when we clone the numdifftools repo via
-# `python setup.py sdist && pip install dist/*`, but I'm forgoing this for now
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update -qq \
- && apt-get install --no-install-recommends -qqy git \
- && cd ${PACKAGE_DIR}/numdifftools \
- && ${PIP} install . \
- && apt-get purge -y git \
- && apt-get clean \
- && rm -fr /var/lib/apt/lists/*
+RUN cd ${PACKAGE_DIR}/numdifftools \
+ && ${PIP} install . 
 
 #########################################
 ## kimpy
@@ -134,6 +125,7 @@ RUN cd ${PACKAGE_DIR}/lammps/ \
       -D PKG_SMTBQ=yes \
       -D PKG_EXTRA-PAIR=yes \
       -D PKG_CORESHELL=yes \
+      -D PKG_EXTRA-FIX=yes \
       ../cmake \
  && make -j2 \
  && make install \
@@ -159,17 +151,17 @@ RUN cd ${PACKAGE_DIR}/ase/ \
 #########################################
 RUN cd ${PACKAGE_DIR}/kim-python-utils \
  && ${PIP} install .
-
+ 
 #########################################
-## kim-property (temporary spoof-version branch)
+## crystal-genome-util
 #########################################
-RUN cd ${PACKAGE_DIR}/kim-property \
+RUN cd ${PACKAGE_DIR}/crystal-genome-util \
  && ${PIP} install .
 
 #########################################
-## crystal-genome-util
+## kim-tools
 #########################################
-RUN cd ${PACKAGE_DIR}/crystal-genome-util \
+RUN cd ${PACKAGE_DIR}/kim-tools \
  && ${PIP} install .
 
 #########################################

docker/sys/Dockerfile

@@ -39,5 +39,6 @@ RUN apt-get update -qq \
  valgrind \
  tree \
  libfreetype6-dev \
+ git \
  && apt-get clean \
  && rm -fr /var/lib/apt/lists/*

test/run_vcs.sh → test/run_all.sh

@@ -1,2 +1,2 @@
-DOCKER_COMMAND="cd /home/openkim/test_scripts_and_data && bash set_up_and_run_$2.sh && bash compare_vcs.sh $2"
-docker run --rm --mount type=bind,src=$PWD/test/test_scripts_and_data,target=/home/openkim/test_scripts_and_data --env LD_LIBRARY_PATH=:/usr/local/lib $1 /bin/bash -c "$DOCKER_COMMAND"
+DOCKER_COMMAND="cd /home/openkim/test_scripts_and_data && bash set_up_and_run_$2.sh && python compare_dbs.py $2 && bash compare_vcs.sh $2"
+docker run --rm --mount type=bind,src=$PWD/test/test_scripts_and_data,target=/home/openkim/test_scripts_and_data --env LD_LIBRARY_PATH=:/usr/local/lib $1 /bin/bash -c "$DOCKER_COMMAND"

test/run_tests.sh

@@ -1,3 +1,2 @@
 DOCKER_COMMAND="cd /home/openkim/test_scripts_and_data && bash set_up_and_run_$2.sh && python compare_dbs.py $2"
-docker run --rm --mount type=bind,src=$PWD/test/test_scripts_and_data,target=/home/openkim/test_scripts_and_data --env LD_LIBRARY_PATH=:/usr/local/lib $1 /bin/bash -c "$DOCKER_COMMAND"
-
+docker run --rm --mount type=bind,src=$PWD/test/test_scripts_and_data,target=/home/openkim/test_scripts_and_data --env LD_LIBRARY_PATH=:/usr/local/lib $1 /bin/bash -c "$DOCKER_COMMAND"

test/test_scripts_and_data/all.grades

@@ -0,0 +1,13 @@
+P
+P
+P
+P
+P
+P
+P
+P
+N/A
+P
+P
+P
+P