Feat: Fixed db migration issues from v3.2.0 to 3.3.0

.github/workflows/integration_test.yaml

@@ -13,6 +13,6 @@ jobs:
       trivy-image-config: "trivy.yaml"
       juju-channel: 3.1/stable
       channel: 1.28-strict/stable
-      modules: '["test_charm", "test_saml", "test_users"]'
+      modules: '["test_charm", "test_saml", "test_users", "test_db_migration"]'
       self-hosted-runner: true
       self-hosted-runner-label: "edge"

charmcraft.yaml

@@ -12,5 +12,6 @@ bases:
       channel: "20.04"
 parts:
   charm:
+    build-packages: [cargo, rustc, pkg-config, libffi-dev, libssl-dev]
     charm-python-packages: [setuptools, pip]  # https://discourse.charmhub.io/t/install-or-update-python-packages-before-packing-a-charm/5158
     charm-binary-python-packages: [cosl]  # https://github.com/canonical/charmcraft/issues/1269

discourse_rock/patches/db_migrations.patch

@@ -0,0 +1,16 @@
+diff --git a/db/post_migrate/20240212034010_drop_deprecated_columns.rb b/db/post_migrate/20240212034010_drop_deprecated_columns.rb
+index 0899da20..015fc6d5 100644
+--- a/db/post_migrate/20240212034010_drop_deprecated_columns.rb
++++ b/db/post_migrate/20240212034010_drop_deprecated_columns.rb
+@@ -19,6 +19,11 @@ class DropDeprecatedColumns < ActiveRecord::Migration[7.0]
+   }
+
+   def up
++    execute <<~SQL
++      DROP TRIGGER IF EXISTS invites_user_id_readonly ON invites;
++      DROP TRIGGER IF EXISTS invites_redeemed_at_readonly ON invites;
++      DROP TRIGGER IF EXISTS user_api_keys_scopes_readonly ON user_api_keys;
++    SQL
+     DROPPED_COLUMNS.each { |table, columns| Migration::ColumnDropper.execute_drop(table, columns) }
+   end
+

discourse_rock/rockcraft.yaml

@@ -192,6 +192,7 @@ parts:
       - git
     after: [discourse, patches]
     override-stage: |
+      git -C srv/discourse/app apply patches/db_migrations.patch
       git -C srv/discourse/app apply patches/lp1903695.patch
       git -C srv/discourse/app apply patches/discourse-charm.patch
       git -C srv/discourse/app apply patches/sigterm.patch

testing_database/creating-the-testing-database.md

@@ -0,0 +1,34 @@
+# Creating the testing database
+At the writing this document, the testing database is created using Discourse v3.2.0.
+
+## Creating the database
+First of all we need to deploy the Discourse following the [tutorial](https://github.com/canonical/discourse-k8s-operator/blob/main/docs/tutorial.md).
+
+Then, we need to create a 2 new users for testing using the actions:
+
+`juju run discourse-k8s/0 create-user email=email@example.com admin=true`
+`juju run discourse-k8s/0 create-user email=email2@example.com`
+
+Please note that the first user is an admin and the second one is not. Also please not the passwords that are generated automatically by the command.
+
+Now open the Discourse URL in a browser, login with the first user (admin) and create a new topic. Reply to this topic as the admin user again. Then, login with the second user and reply to this topic. Then login with the first user and approve the second users reply.
+
+## Exporting the database
+
+First we need to get the database password:
+`juju run postgresql-k8s/0 get-password username=operator`
+
+Ssh into the database
+`juju ssh --container postgresql postgresql-k8s/0 bash`
+
+Create a folder to dump the db
+`mkdir -p /srv/dump/`
+
+Dump the db. Ip here is the unit ip
+`pg_dump -Fc -h 10.1.187.134 -U operator -d discourse > "/srv/dump/testing_database.sql"`
+
+Exit the container
+`exit`
+
+Copy the dump into local file system.
+`juju scp --container postgresql postgresql-k8s/0:/srv/dump/testing_database.sql./testing_database.sql`

tests/integration/test_db_migration.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Discourse integration tests."""
+
+import logging
+
+import pytest
+from botocore.config import Config
+from ops.model import WaitingStatus
+from pytest_operator.plugin import Model, OpsTest
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.asyncio
+@pytest.mark.abort_on_fail
+async def test_db_migration(model: Model, ops_test: OpsTest, pytestconfig: Config, run_action):
+    """
+    arrange: preload postgres with a mock db that is created in Discource v3.2.0
+    act: deploy and integrate with discourse v3.3.0
+    assert: discourse must be active idle, previously it was creating migration
+    errors related to not being able to delete some columns because of triggers
+    """
+    postgres_app = await model.deploy(
+        "postgresql-k8s",
+        channel="14/stable",
+        series="jammy",
+        revision=300,
+        trust=True,
+        config={"profile": "testing"},
+    )
+    async with ops_test.fast_forward():
+        await model.wait_for_idle(apps=[postgres_app.name], status="active")
+    await postgres_app.set_config(
+        {
+            "plugin_hstore_enable": "true",
+            "plugin_pg_trgm_enable": "true",
+        }
+    )
+    await model.wait_for_idle(apps=[postgres_app.name], status="active")
+    db_pass = await run_action(postgres_app.name, "get-password", username="operator")
+    db_pass = db_pass["password"]
+    return_code, _, scp_err = await ops_test.juju(
+        "scp",
+        "--container",
+        "postgresql",
+        "./testing_database/testing_database.sql",
+        f"{postgres_app.units[0].name}:.",
+    )
+
+    assert return_code == 0, scp_err
+
+    return_code, _, ssh_err = await ops_test.juju(
+        "ssh",
+        "--container",
+        "postgresql",
+        postgres_app.units[0].name,
+        "createdb -h localhost -U operator --password discourse",
+        stdin=str.encode(f"{db_pass}\n"),
+    )
+    assert return_code == 0, ssh_err
+
+    return_code, _, ssh_err = await ops_test.juju(
+        "ssh",
+        "--container",
+        "postgresql",
+        postgres_app.units[0].name,
+        "pg_restore -h localhost -U operator\
+              --password -d discourse\
+                  --no-owner --clean --if-exists ./testing_database.sql",
+        stdin=str.encode(f"{db_pass}\n"),
+    )
+    assert return_code == 0, ssh_err
+    redis_app = await model.deploy("redis-k8s", series="jammy", channel="latest/edge")
+    await model.wait_for_idle(apps=[redis_app.name], status="active")
+
+    resources = {"discourse-image": pytestconfig.getoption("--discourse-image")}
+    charm = await ops_test.build_charm(".")
+    await model.deploy("nginx-ingress-integrator", series="focal", trust=True)
+    app_name = "discourse-k8s"
+    discourse_application = await model.deploy(
+        charm,
+        resources=resources,
+        application_name=app_name,
+        series="focal",
+    )
+    await model.wait_for_idle(apps=[app_name], status="waiting")
+    unit = discourse_application.units[0]
+    assert unit.workload_status == WaitingStatus.name  # type: ignore
+    await model.add_relation(app_name, "postgresql-k8s:database")
+    await model.add_relation(app_name, "redis-k8s")
+    await model.add_relation(app_name, "nginx-ingress-integrator")
+    await model.wait_for_idle(apps=[app_name], status="active", raise_on_error=True)
+    await model.wait_for_idle(apps=[app_name], status="active", raise_on_error=True)