Correctly run tests in the CI

.coveragerc

@@ -0,0 +1,13 @@
+[report]
+include_namespace_packages = True
+skip_empty = True
+
+[run]
+omit =
+    **/tests/*
+    **/build/*
+    **/setup.py
+    **/dev/*
+    homeserver/app/*
+    **/cache/manage.py
+    **/migrations/*

Dockerfile.test

@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1
+FROM python:3.11.4
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHON_BIN=python
+
+WORKDIR /code
+
+# install system dependencies
+RUN apt update && apt install git -y
+
+# cleanup apt cache
+RUN rm -rf /var/lib/apt/lists/*
+
+COPY test-config /test-config
+COPY fractal /code/fractal
+COPY pyproject.toml README.md /code/
+COPY tests /code/tests
+
+COPY .coveragerc conftest.py pytest.ini /code/
+
+# install modules
+RUN pip3 install -e /code[dev]
+
+ENTRYPOINT [ "/test-config/entrypoint.sh" ]

Makefile

@@ -3,7 +3,8 @@
 TEST = ""
 
 test-ci:
-	docker compose up --exit-code-from test
+	docker compose up synapse --build --force-recreate -d --wait
+	docker compose up test --build --exit-code-from test
 
 setup:
 	python test-config/prepare-test.py

conftest.py

@@ -11,6 +11,11 @@ def mock_getpass():
         yield
 
 
+@pytest.fixture
+def test_homeserver_url() -> str:
+    return os.environ.get("TEST_HOMESERVER_URL", "http://localhost:8008")
+
+
 @pytest.fixture(autouse=True)
 def cleanup():
     yield

docker-compose.yaml

@@ -1,7 +1,34 @@
 services:
+  synapse:
+    image: fractal-cli-synapse:latest
+    build:
+      context: ./synapse/config
+      dockerfile: Dockerfile.synapse
+      args:
+        SYNAPSE_SERVER_NAME: "localhost"
+        SYNAPSE_REPORT_STATS: "no"
+    healthcheck:
+      test: curl localhost:8008/health
+      interval: 1s
+      timeout: 10s
+      retries: 10
+    labels:
+      - "org.homeserver=true"
+    # --timeout on up doesn't work with --exit-code-from. This ensures the synapse
+    # container is stopped immediately when the device exits
+    stop_signal: SIGKILL
   test:
-    image: python:3.11-slim
-    entrypoint: bash -c 'pip install poetry==1.7.1 && poetry install && poetry run pytest'
+    image: fractal-cli-test:test
+    build:
+      context: ./
+      dockerfile: Dockerfile.test
+    depends_on:
+      synapse:
+        condition: service_healthy
+    environment:
+      ENV: test
+      TEST_CONFIG_DIR: /test-config
+      TEST_HOMESERVER_URL: http://synapse:8008
     volumes:
-      - .:/code
+      - /var/run/docker.sock:/var/run/docker.sock
     working_dir: /code

pyproject.toml

@@ -14,7 +14,13 @@ toml = "^0.10.2"
 appdirs = "^1.4.4"
 clicz = ">=0.0.2"
 pytest = { version = "^7.4.3", optional = true }
+pytest-asyncio = { version = "^0.21.1", optional = true }
+pytest-cov = { version = "^4.1.0", optional = true }
+pytest-mock = { version = "^3.11.1", optional = true }
+docker = { version = "^6.1.3", optional = true }
+ipython = { version = "^8.17.2", optional = true }
 asgiref = ">=3.7.2"
+fractal-matrix-client = { git = "https://github.com/fractalnetworksco/fractal-matrix-client.git", branch = "main" }
 
 
 [tool.poetry.group.dev.dependencies]
@@ -28,7 +34,7 @@ build-backend = "poetry.core.masonry.api"
 fractal = "fractal.cli.__main__:main"
 
 [tool.poetry.extras]
-dev = ["pytest", "pytest-cov", "pytest-mock", "ipython"]
+dev = ["docker", "pytest", "pytest-cov", "pytest-mock", "pytest-asyncio", "ipython"]
 
 [tool.poetry.plugins."fractal.plugins"]
 "auth" = "fractal.cli.controllers.auth"

synapse/config/Dockerfile.synapse

@@ -0,0 +1,19 @@
+FROM matrixdotorg/synapse:v1.95.1
+
+ARG SYNAPSE_SERVER_NAME=localhost
+ARG SYNAPSE_REPORT_STATS=no
+
+RUN apt update; apt install sqlite3 -y
+RUN rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /data
+
+RUN bash -c 'python /start.py generate'
+
+COPY config_to_add.yaml /config_to_add.yaml
+
+# append config to homeserver.yaml only if it's not already there
+RUN cat /config_to_add.yaml >> /data/homeserver.yaml
+
+ENTRYPOINT bash -c ' \
+    python /start.py &> /dev/null'

synapse/config/config_to_add.yaml

@@ -0,0 +1,18 @@
+
+enable_registration: true
+registration_requires_token: true
+presence_enabled: false
+max_upload_size: 5000M
+
+# increasing default rate limiting burst counts to avoid tests hanging for unauthenticated
+# requtests to login
+rc_login:
+  address:
+    per_second: 0.15
+    burst_count: 656565
+  account:
+    per_second: 0.18
+    burst_count: 656565
+  failed_attempts:
+    per_second: 0.19
+    burst_count: 656565

synapse/docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3.9"
+services:
+  synapse:
+    image: homeserver-synapse:latest
+    build:
+      context: ./config
+      dockerfile: Dockerfile.synapse
+      args:
+        SYNAPSE_SERVER_NAME: "localhost"
+        SYNAPSE_REPORT_STATS: "no"
+    ports:
+      - 8008:8008
+    healthcheck:
+      test: curl localhost:8008/health
+      interval: 5s
+      timeout: 10s
+      retries: 5
+    labels:
+      - "org.homeserver=true"
+    restart: "unless-stopped"
+  element:
+    image: vectorim/element-web:latest
+    ports:
+      - "8009:80"
+    restart: "unless-stopped"

test-config/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# expected environment variables:
+# ENV - environment name (e.g. test, dev, prod)
+# TEST_CONFIG_DIR - path to the test-config directory
+
+set -e
+
+PREPARE_SCRIPT="$TEST_CONFIG_DIR/prepare-test.py"
+
+python3 "$PREPARE_SCRIPT"
+
+# environment file should be created by prepare-test.py
+source "$TEST_CONFIG_DIR/fractal_cli.$ENV.env"
+
+cd /code
+
+pytest -v --asyncio-mode=auto --cov=/code/fractal --cov-report=lcov --cov-report=term tests/

test-config/prepare-test.py

@@ -0,0 +1,90 @@
+"""
+This script is intended to be ran by the test container's entrypoint.
+
+Ensures that the provided test user exists and creates a room for testing.
+On Success, writes necessary environment variables to .env.testing.
+"""
+
+import asyncio
+import os
+from sys import exit
+
+import docker
+from aiofiles import open
+from docker.models.containers import Container
+from nio import AsyncClient, LoginError, RoomCreateError
+
+ENV = os.environ.get("ENV", "dev")
+TEST_CONFIG_DIR = os.environ.get("TEST_CONFIG_DIR", ".")
+TEST_ENV_FILE = os.environ.get("TEST_ENV_FILE", f"{TEST_CONFIG_DIR}/fractal_cli.{ENV}.env")
+TEST_HOMESERVER_URL = os.environ.get("TEST_HOMESERVER_URL", "http://localhost:8008")
+TEST_USER_USERNAME = os.environ.get("TEST_USER_USERNAME", "admin")
+TEST_USER_PASSWORD = os.environ.get("TEST_USER_PASSWORD", "admin")
+PYTHON_BIN = os.environ.get("PYTHON_BIN", "venv/bin/python")
+
+
+async def main():
+    # this is blocking but doesn't matter since this is an entrypoint
+    try:
+        # get homeserver container
+        docker_client = docker.from_env()
+        synapse_container = docker_client.containers.list(
+            filters={"label": "org.homeserver=true"}
+        )[0]
+        # asserting here so that the Container type hint works : )
+        assert isinstance(synapse_container, Container)
+    except Exception:
+        print("No homeserver container found")
+        print("Launch synapse container in /synapse")
+        exit(1)
+
+    # create admin user on synapse if it doesn't exist
+    result = synapse_container.exec_run(
+        f"register_new_matrix_user -c /data/homeserver.yaml -a -u {TEST_USER_USERNAME} -p {TEST_USER_PASSWORD} http://localhost:8008"
+    )
+
+    if "User ID already taken" not in result.output.decode("utf-8") and result.exit_code != 0:
+        print(result.output.decode("utf-8"))
+        exit(1)
+
+    # login
+    matrix_client = AsyncClient(TEST_HOMESERVER_URL, TEST_USER_USERNAME)
+    login_res = await matrix_client.login(TEST_USER_PASSWORD)
+    if isinstance(login_res, LoginError):
+        print(f"Error logging in: {login_res.message}")
+        exit(1)
+
+    # disable rate limiting for the created test user
+    print(f"Disabling rate limiting for user: {matrix_client.user_id}")
+    result = synapse_container.exec_run(
+        f"sqlite3 /data/homeserver.db -cmd \"INSERT INTO ratelimit_override values ('{matrix_client.user_id}', 0, 0);\" .exit"
+    )
+    if "UNIQUE constraint failed" not in result.output.decode("utf-8") and result.exit_code != 0:
+        print(result.output.decode("utf-8"))
+        exit(1)
+
+    # restart synapse container
+    print("Restarting synapse container")
+    synapse_container.restart()
+
+    print("Creating room")
+    # create room
+    # This always creates a new room. This is okay since we want a fresh start
+    room_create_res = await matrix_client.room_create(name="Test Room")
+    if isinstance(room_create_res, RoomCreateError):
+        print(f"Error creating room: {room_create_res.message}")
+        exit(1)
+
+    # write environment file
+    async with open(TEST_ENV_FILE, "w") as f:
+        await f.write(
+            f'export HS_USER_ID="{matrix_client.user_id}"\nexport MATRIX_ROOM_ID="{room_create_res.room_id}"\nexport MATRIX_ACCESS_TOKEN="{matrix_client.access_token}"\nexport MATRIX_HOMESERVER_URL="{TEST_HOMESERVER_URL}"\nexport PYTHON_BIN="{PYTHON_BIN}"\nexport HS_OWNER_ID="{matrix_client.user_id}"\nexport HS_DEVICE_ID="{matrix_client.user_id}"\n'
+        )
+
+    await matrix_client.close()
+
+    print("Successfully prepared")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_login.py

@@ -1,6 +1,6 @@
 from fractal.cli.controllers.auth import AuthController
 
 
-def test_login_with_password(mock_getpass):
+def test_login_with_password(mock_getpass, test_homeserver_url):
     auth_cntrl = AuthController()
-    auth_cntrl.login("@admin:localhost", homeserver_url="http://localhost:8008")
+    auth_cntrl.login("@admin:localhost", homeserver_url=test_homeserver_url)