Penquins update, bugfixes and basic CI

Penquins update, improvements, bugfixes and basic CI

.flake8

@@ -0,0 +1,16 @@
+# See:
+#
+# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes  (E, W)
+# https://flake8.pycqa.org/en/latest/user/error-codes.html (F)
+# https://github.com/PyCQA/flake8-bugbear
+#
+# for error codes.  And
+#
+# https://flake8.pycqa.org/en/latest/user/violations.html#selecting-violations-with-flake8
+#
+# for error classes selected below.
+
+[flake8]
+max-line-length = 80
+select = C,E,F,W,B,B950
+ignore = E501, W503

.github/workflows/docker_build.yaml

@@ -0,0 +1,75 @@
+name: Docker Build
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  docker_build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install "wheel>=0.36.0"
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+          key: ${{ runner.os }}-${{ hashFiles('**/requirements*.txt') }}
+      - name: create secrets.json
+        run: |
+          touch secrets.json
+          cat >> secrets.json << EOL
+          {
+            "server" : {
+              "admin_username": "ADMIN",
+              "admin_password": "PASSWORD"
+            },
+            "database": {
+              "admin": "mongoadmin",
+              "admin_pwd": "mongoadminsecret",
+              "user": "user",
+              "pwd": "pwd"
+            },
+            "kowalski": {
+              "instances": {
+                "gloria": {
+                  "protocol": "https",
+                  "host": "gloria.caltech.edu",
+                  "port": 443,
+                  "token": null
+                },
+                "melman": {
+                  "protocol": "https",
+                  "host": "melman.caltech.edu",
+                  "port": 443,
+                  "token": null
+                },
+                "kowalski": {
+                  "protocol": "https",
+                  "host": "kowalski.caltech.edu",
+                  "port": 443,
+                  "token": null
+                }
+              }
+            }
+          }
+          EOL
+      - name: Build image
+        run: |
+          docker volume create ztf_variable_marshal_mongodb
+          docker volume create ztf_variable_marshal_data
+          docker run -d --restart always --name ztf_variable_marshal_mongo_1 -p 27025:27017 \
+          -v ztf_variable_marshal_mongodb:/data/db \
+          -e MONGO_INITDB_ROOT_USERNAME=mongoadmin -e MONGO_INITDB_ROOT_PASSWORD=mongoadminsecret \
+          mongo:latest
+          docker build --rm -t ztf_variable_marshal:latest -f Dockerfile .

.github/workflows/lint.yaml

@@ -0,0 +1,26 @@
+name: lint
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  lint:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install "wheel>=0.36.0"
+          pip install pre-commit
+          pre-commit install
+      - name: Formatting and linting checks
+        run: pre-commit run --all-files

.github/workflows/test.yaml

@@ -0,0 +1,112 @@
+name: Test
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+      - name: Fetch and set up Kowalski
+        uses: actions/checkout@v2
+        with:
+          repository: skyportal/kowalski
+          path: kowalski
+      - name: Configure and spin up Kowalski, ingest test data
+        run: |
+          cd kowalski
+          python -m pip install --upgrade pip
+          sed -i'' -e 's/4000/6000/g' config.defaults.yaml
+          sed -i'' -e 's/4000/6000/g' docker-compose.defaults.yaml
+          cp docker-compose.defaults.yaml docker-compose.yaml
+          make docker_build && make docker_up
+          make docker_test
+
+      - name: Debug Check that Kowalski is up
+        run: |
+          curl -X GET http://localhost:6000
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install "wheel>=0.36.0"
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+          key: ${{ runner.os }}-${{ hashFiles('**/requirements*.txt') }}
+      - name: create secrets.json
+        run: |
+          touch secrets.json
+          cat >> secrets.json << EOL
+          {
+            "server" : {
+              "admin_username": "admin",
+              "admin_password": "admin"
+            },
+            "database": {
+              "admin": "mongoadmin",
+              "admin_pwd": "mongoadminsecret",
+              "user": "user",
+              "pwd": "pwd"
+            },
+            "kowalski": {
+              "instances": {
+                "kowalski": {
+                  "protocol": "http",
+                  "host": "172.17.0.1",
+                  "port": 6000,
+                  "username": "admin",
+                  "password": "admin"
+                }
+              }
+            }
+          }
+          EOL
+
+      - name: Less workers for testing
+        run: |
+          sed -i'' -e 's/-w 8/-w 2/g' Dockerfile
+
+      - name: Add volumes
+        run: |
+          docker volume create ztf_variable_marshal_mongodb
+          docker volume create ztf_variable_marshal_data
+
+      - name: Start mongo
+        run: |
+          docker run -d --restart always --name ztf_variable_marshal_mongo_1 -p 27025:27017 --expose 27025 \
+          -v ztf_variable_marshal_mongodb:/data/db \
+          -e MONGO_INITDB_ROOT_USERNAME=mongoadmin -e MONGO_INITDB_ROOT_PASSWORD=mongoadminsecret \
+          mongo:latest
+
+      - name: Build the marshal
+        run: |
+          docker build --rm -t ztf_variable_marshal:latest -f Dockerfile .
+
+      - name: Start the marshal
+        run: |
+          docker run --name ztf_variable_marshal -d --restart always -p 8000:4000 -v ztf_variable_marshal_data:/data --link ztf_variable_marshal_mongo_1:mongo ztf_variable_marshal:latest
+
+      - name: Wait for marshal to start
+        run: |
+          sleep 15
+
+      - name: Docker Logs
+        run: |
+          docker logs ztf_variable_marshal
+
+      - name: Run tests
+        run: |
+          sleep 10
+          pip install requests pymongo pytest
+          python -m pytest test.py

.gitignore

@@ -0,0 +1,2 @@
+secrets.json
+__pycache__/

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/python/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        pass_filenames: true
+        exclude: nb|.ipynb_checkpoints|data|dev|letsencrypt|logs|dask-worker-space
+  - repo: https://github.com/PyCQA/flake8
+    rev: 3.8.4
+    hooks:
+      - id: flake8
+        pass_filenames: true
+        exclude: nb|.ipynb_checkpoints|data|dev|letsencrypt|logs|dask-worker-space|__init__.py

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7
+FROM python:3.10.10
 
 # Install: vim, git, cron
 RUN apt-get update && apt-get -y install apt-file && apt-file update && apt-get -y install vim && \

dev/spec_dbsp.json

@@ -10763,4 +10763,4 @@
       "fluxerr": 0.5
     }
   ]
-}
+}

docker-compose.yml

@@ -47,4 +47,4 @@ services:
       - MONGO_INITDB_ROOT_PASSWORD=mongoadminsecret
     volumes:
     - mongodb:/data/db
-    restart: always
+    restart: always

readme.md

@@ -17,7 +17,7 @@ A dockerized Variable Marshal for [ZTF](https://ztf.caltech.edu) powered by `aio
 
 ## Python client library `zvm`
 
-Install the client library [zvm.py](https://github.com/dmitryduev/ztf-variable-marshal/blob/master/zvm.py), 
+Install the client library [zvm.py](https://github.com/dmitryduev/ztf-variable-marshal/blob/master/zvm.py),
 with `pip` into your environment:
 
 ```bash
@@ -28,12 +28,12 @@ pip install git+https://github.com/dmitryduev/ztf-variable-marshal.git
 
 A tutorial on how to programmatically interact with the ZVM:
 
-See [this jupyter notebook](https://github.com/dmitryduev/ztf-variable-marshal/blob/master/nb/api.ipynb), or 
+See [this jupyter notebook](https://github.com/dmitryduev/ztf-variable-marshal/blob/master/nb/api.ipynb), or
 [![Open In Google Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dmitryduev/ztf-variable-marshal/blob/master/nb/api.ipynb)
 
 ---
 
-## Production service  
+## Production service
 
 ### Set-up instructions
 
@@ -59,15 +59,33 @@ Create `secrets.json` with confidential/secret data:
     "pwd": "pwd"
   },
   "kowalski": {
-    "username": "USERNAME",
-    "password": "PASSWORD"
+    "instances": {
+      "gloria": {
+        "protocol": "https",
+        "host": "gloria.caltech.edu",
+        "port": 443,
+        "token": null // set to your token
+      },
+      "melman": {
+        "protocol": "https",
+        "host": "melman.caltech.edu",
+        "port": 443,
+        "token": null // set to your token
+      },
+      "kowalski": {
+        "protocol": "https",
+        "host": "kowalski.caltech.edu",
+        "port": 443,
+        "token": null // set to your token
+      }
+    },
   }
 }
 ```
 
 #### Using `docker-compose` (for production)
 
-Change `skipper.caltech.edu` in `docker-compose.yml` and in `traefik/traefik.toml` if necessary. 
+Change `skipper.caltech.edu` in `docker-compose.yml` and in `traefik/traefik.toml` if necessary.
 
 Run `docker-compose` to build and start the service (may have to sudo if acme complains):
 ```bash
@@ -91,10 +109,10 @@ docker volume create ztf_variable_marshal_mongodb
 docker volume create ztf_variable_marshal_data
 ```
 
-Pull, build, and launch the MongoDB container. Feel free to change u/p for the admin, 
+Pull, build, and launch the MongoDB container. Feel free to change u/p for the admin,
 but make sure to change `secrets.json` and `docker-compose.yml` correspondingly.
 ```bash
-docker run -d --restart always --name ztf_variable_marshal_mongo_1 -p 27025:27017 \
+docker run -d --restart always --name ztf_variable_marshal_mongo_1 -p 27025:27017 --expose 27025 \
        -v ztf_variable_marshal_mongodb:/data/db \
        -e MONGO_INITDB_ROOT_USERNAME=mongoadmin -e MONGO_INITDB_ROOT_PASSWORD=mongoadminsecret \
        mongo:latest
@@ -115,4 +133,4 @@ docker run -it --rm --name ztf_variable_marshal -p 8000:4000 -v ztf_variable_mar
 
 ```
 
-`ztf_variable_marshal` will be available on port 8000 of the `Docker` host machine. 
+`ztf_variable_marshal` will be available on port 8000 of the `Docker` host machine.

setup.py

@@ -1,9 +1,13 @@
 from setuptools import setup
 
-setup(name='zvm', version='0.0.1', py_modules=['zvm'],
-      install_requires=['pymongo>=3.4.0',
-                        'pytest>=3.3.0',
-                        # 'httpx>=0.7.5',
-                        'requests>=2.18.4']
-      )
-
+setup(
+    name="zvm",
+    version="0.0.1",
+    py_modules=["zvm"],
+    install_requires=[
+        "pymongo>=3.4.0",
+        "pytest>=3.3.0",
+        # 'httpx>=0.7.5',
+        "requests>=2.18.4",
+    ],
+)

test.py

@@ -0,0 +1,16 @@
+from zvm import zvm
+
+
+# for testing, we just run a simple query with the test credentials to see if the server is up
+def test_connection():
+    z = zvm(
+        protocol="http",
+        host="0.0.0.0",
+        port=8000,
+        username="admin",
+        password="admin",
+    )
+
+    # check connection
+    connection = z.check_connection()
+    assert connection is True