[PROD] TTAHUB-2886 Display FEI root cause on goal card, TTAHUB-3036 Filter out TR goals from similarity api

.circleci/config.yml

@@ -424,7 +424,7 @@ parameters:
     type: string
   dev_git_branch: # change to feature branch to test deployment
     description: "Name of github branch that will deploy to dev"
-    default: "mb/TTAHUB-3007/no-disallowed-urls"
+    default: "al-ttahub-2939-add-fei-root-cause-to-goal-card"
     type: string
   sandbox_git_branch:  # change to feature branch to test deployment
     default: "jp/3005/ipd-courses-widget"

.env.example

@@ -57,8 +57,6 @@ AWS_ELASTICSEARCH_ENDPOINT=http://localhost:9200
 AWS_ELASTICSEARCH_ACCESS_KEY=admin
 AWS_ELASTICSEARCH_SECRET_KEY=admin
 
-FONTAWESOME_NPM_AUTH_TOKEN=whatever_it_is_it_isnt_this
-
 # Email Address that notifications should come from
 FROM_EMAIL_ADDRESS=ttasmarthub@test.gov
 

README.md

@@ -36,12 +36,12 @@ those services are already running on your machine.
 
 1. Make sure Docker is installed. To check run `docker ps`.
 2. Make sure you have Node 18.20.3 installed.
-4. Copy `.env.example` to `.env`.
-6. Change the `FONTAWESOME_NPM_AUTH_TOKEN`, `AUTH_CLIENT_ID` and `AUTH_CLIENT_SECRET` variables to to values found in the team Keybase account. If you don't have access to Keybase, please ask in the acf-head-start-eng slack channel for access.
-7. Optionally, set `CURRENT_USER` to your current user's uid:gid. This will cause files created by docker compose to be owned by your user instead of root.
-3. Run `yarn docker:reset`. This builds the frontend and backend, installs dependencies, then runs database migrations and seeders. If this returns errors that the version of nodejs is incorrect, you may have older versions of the containers built. Delete those images and it should rebuild them.
-10. Run `yarn docker:start` to start the application. The [frontend][frontend] will be available on `localhost:3000`  and the [backend][backend] will run on `localhost:8080`, [API documentation][API documentation] will run on `localhost:5003`, and [minio][minio] will run on `localhost:9000`.
-11. Run `yarn docker:stop` to stop the servers and remove the docker containers.
+3. Copy `.env.example` to `.env`.
+4. Change the `AUTH_CLIENT_ID` and `AUTH_CLIENT_SECRET` variables to to values found in the team Keybase account. If you don't have access to Keybase, please ask in the acf-head-start-eng slack channel for access.
+5. Optionally, set `CURRENT_USER` to your current user's uid:gid. This will cause files created by docker compose to be owned by your user instead of root.
+6. Run `yarn docker:reset`. This builds the frontend and backend, installs dependencies, then runs database migrations and seeders. If this returns errors that the version of nodejs is incorrect, you may have older versions of the containers built. Delete those images and it should rebuild them. If you are using a newer Mac with the Apple Silicon chipset, puppeteer install fails with the message: ```"The chromium binary is not available for arm64"```. See the section immediately following this one, entitled "Apple Silicon & Chromium" for instructions on how to proceed.
+7. Run `yarn docker:start` to start the application. The [frontend][frontend] will be available on `localhost:3000`  and the [backend][backend] will run on `localhost:8080`, [API documentation][API documentation] will run on `localhost:5003`, and [minio][minio] will run on `localhost:9000`.
+8. Run `yarn docker:stop` to stop the servers and remove the docker containers.
 
 The frontend [proxies requests](https://create-react-app.dev/docs/proxying-api-requests-in-development/) to paths it doesn't recognize to the backend.
 
@@ -52,6 +52,21 @@ We use an AWS OpenSearch docker image (Elasticsearch fork) and require that the
 * `AWS_ELASTICSEARCH_ACCESS_KEY=admin`
 * `AWS_ELASTICSEARCH_SECRET_KEY=admin`
 
+#### Apple Silicon & Chromium
+On a Mac with Apple Silicon, puppeteer install fails with the message:
+```"The chromium binary is not available for arm64"```
+
+See [docker-compose.override.yml](docker-compose.override.yml) and uncomment the relevant lines to skip downloading chromium and use the host's binary instead.
+
+You will need to have chromium installed (you probably do not). The recommended installation method is to use brew: `brew install chromium --no-quarantine`
+
+To ~/.zshrc (or your particular shell config), you'll need to add:
+
+```sh
+export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+export PUPPETEER_EXECUTABLE_PATH=`which chromium`
+```
+
 #### Local build
 
 You can also run build commands directly on your host (without docker). Make sure you install dependencies when changing execution method. You could see some odd errors if you install dependencies for docker and then run yarn commands directly on the host, especially if you are developing on windows. If you want to use the host yarn commands be sure to run `yarn deps:local` before any other yarn commands. Likewise if you want to use docker make sure you run `yarn docker:deps`.

docker-compose.debug.yml

@@ -15,7 +15,6 @@ services:
       - POSTGRES_HOST=postgres_docker
       - REDIS_HOST=redis
       - SMTP_HOST=mailcatcher
-      - FONTAWESOME_NPM_AUTH_TOKEN
       # On an M1 mac, puppeteer install fails with the message:
       # "The chromium binary is not available for arm64"
       #

docker-compose.dss.yml

@@ -19,7 +19,6 @@ services:
       - POSTGRES_DB=ttasmarthub
       - DATABASE_URL=postgres://postgres:secretpass@postgres_docker:5432/ttasmarthub
       - SESSION_SECRET=notasecret
-      - FONTAWESOME_NPM_AUTH_TOKEN
       - BYPASS_SOCKETS="true"
     volumes:
       - ".:/app:rw"

docker-compose.override.yml

@@ -14,7 +14,6 @@ services:
       - POSTGRES_HOST=postgres_docker
       - REDIS_HOST=redis
       - SMTP_HOST=mailcatcher
-      - FONTAWESOME_NPM_AUTH_TOKEN
       # On an M1 mac, puppeteer install fails with the message:
       # "The chromium binary is not available for arm64"
       #
@@ -43,7 +42,6 @@ services:
       - "./packages:/packages:ro"
     environment:
       - BACKEND_PROXY=http://backend:8080
-      - FONTAWESOME_NPM_AUTH_TOKEN
       - REACT_APP_WEBSOCKET_URL
   worker:
     build:

docker-compose.test.yml

@@ -29,7 +29,6 @@ services:
       - "./scripts:/app/scripts"
     environment:
       - BACKEND_PROXY=http://test-backend:8080
-      - FONTAWESOME_NPM_AUTH_TOKEN
       - REACT_APP_WEBSOCKET_URL
     networks:
       - ttadp-test

frontend/src/components/GoalCards/GoalCard.js

@@ -216,6 +216,11 @@ export default function GoalCard({
   const internalLeftMargin = hideCheckbox ? '' : 'desktop:margin-left-5';
   const border = erroneouslySelected || deleteError ? 'smart-hub-border-base-error' : 'smart-hub-border-base-lighter';
 
+  const getResponses = () => {
+    const responses = goal.responses.length ? goal.responses[0].response : [];
+    return responses.map((r) => r).join(', ');
+  };
+
   return (
     <article
       className={`ttahub-goal-card usa-card padding-3 radius-lg border ${border} width-full maxw-full margin-bottom-2`}
@@ -277,6 +282,20 @@ export default function GoalCard({
               goalNumbers={goalNumbers}
             />
           </p>
+          {
+              goal.isFei
+                ? (
+                  <div className="grid-row">
+                    <p className="usa-prose text-bold margin-bottom-0 margin-top-1 margin-right-1">
+                      Root cause:
+                    </p>
+                    <p className="usa-prose margin-bottom-0 margin-top-1">
+                      { getResponses() }
+                    </p>
+                  </div>
+                )
+                : null
+          }
         </div>
         <div className="ttahub-goal-card__goal-column ttahub-goal-card__goal-column__goal-source padding-right-3">
           <p className="usa-prose text-bold margin-y-0">Goal source</p>

frontend/src/components/GoalCards/__tests__/GoalCard.js

@@ -28,6 +28,7 @@ describe('GoalCard', () => {
     source: 'The inferno',
     createdVia: 'rtr',
     onAR: true,
+    responses: [],
     sessionObjectives: [{
       title: 'Session objective 1',
       trainingReportId: 'TR-1',
@@ -132,6 +133,13 @@ describe('GoalCard', () => {
     expect(screen.getByText(/The inferno/i)).toBeInTheDocument();
   });
 
+  it('shows the fei root causes', () => {
+    renderGoalCard(DEFAULT_PROPS,
+      { ...goal, isFei: true, responses: [{ response: ['root cause 1', 'root cause 2', 'root cause 3'] }] });
+    expect(screen.getByText('Root cause:')).toBeInTheDocument();
+    expect(screen.getByText(/root cause 1, root cause 2, root cause 3/i)).toBeInTheDocument();
+  });
+
   it('hides the checkbox when hideCheckbox is true', () => {
     renderGoalCard({ ...DEFAULT_PROPS, hideCheckbox: true });
     expect(screen.queryByRole('checkbox')).not.toBeInTheDocument();

similarity_api/src/sim/compute.py

@@ -40,9 +40,10 @@ def compute_goal_similarities(recipient_id: int, alpha: float, cluster: bool):
       AND NULLIF(TRIM(g."name"), '') IS NOT NULL
     -- -------------------------------------------
     -- Only needed to prevent goals created from non-approved reports from being merged
-      AND ((ar."approvedAt" IS NOT NULL
-        AND g."createdVia"::text = 'activityReport')
-        OR (g."createdVia"::text != 'activityReport')
+    -- Prevent goals created via TR.
+      AND (
+        (ar."approvedAt" IS NOT NULL AND g."createdVia"::text = 'activityReport')
+        OR (g."createdVia"::text NOT IN ( 'activityReport', 'tr'))
       )
     -- -------------------------------------------
     ;
@@ -193,7 +194,8 @@ def find_similar_goals(recipient_id, goal_name, alpha, include_curated_templates
         JOIN "Recipients" r
           ON gr."recipientId" = r."id"
         WHERE r."id" = :recipient_id
-          AND NULLIF(TRIM(g."name"), '') IS NOT NULL;
+          AND NULLIF(TRIM(g."name"), '') IS NOT NULL
+          AND g."createdVia"::text != 'tr';
         """,
         {'recipient_id': recipient_id}
     )    

src/constants.js

@@ -259,7 +259,7 @@ const MAINTENANCE_CATEGORY = {
 
 const GOAL_CREATED_VIA = ['imported', 'activityReport', 'rtr', 'merge', 'admin', 'tr'];
 
-const CURRENT_GOAL_SIMILARITY_VERSION = 3;
+const CURRENT_GOAL_SIMILARITY_VERSION = 4;
 
 module.exports = {
   CURRENT_GOAL_SIMILARITY_VERSION,

src/migrations/20240625000000-correct-fei-root-cause.js

@@ -0,0 +1,124 @@
+const {
+  prepMigration,
+} = require('../lib/migration');
+
+module.exports = {
+  up: async (queryInterface) => queryInterface.sequelize.transaction(
+    async (transaction) => {
+      await prepMigration(queryInterface, transaction, __filename);
+      await queryInterface.sequelize.query(/* sql */`
+
+      -- Find all the ActivityReportGoalFieldResponses that have the
+      -- invalid 'Transportation' value
+      DROP TABLE IF EXISTS argfr_to_update;
+      CREATE TEMP TABLE argfr_to_update
+      AS
+      SELECT
+        argfr.id argfrid,
+        arg."goalId" gid
+      FROM "ActivityReportGoalFieldResponses" argfr
+      JOIN "ActivityReportGoals" arg
+        ON "activityReportGoalId" = arg.id
+      WHERE 'Transportation' = ANY(response);
+
+      -- Find all the GoalFieldResponses that have the
+      -- invalid 'Transportation' value
+      DROP TABLE IF EXISTS gfr_to_update;
+      CREATE TEMP TABLE gfr_to_update
+      AS
+      SELECT
+        id gfrid,
+        "goalId" gid
+      FROM "GoalFieldResponses"
+      WHERE 'Transportation' = ANY(response);
+
+      -- Make sure this is only the one recipient being updated.
+      -- If 'Transportation' somehow spread somewhere else then
+      -- we don't know how to correct it accurately.
+      DROP TABLE IF EXISTS recipient_list;
+      CREATE TEMP TABLE recipient_list
+      AS
+      SELECT DISTINCT gr."recipientId"
+      FROM argfr_to_update atu
+      JOIN "Goals" g
+        ON g.id = atu.gid
+      JOIN "Grants" gr
+        ON g."grantId" = gr.id
+      UNION
+      SELECT DISTINCT gr."recipientId"
+      FROM gfr_to_update atu
+      JOIN "Goals" g
+        ON g.id = atu.gid
+      JOIN "Grants" gr
+        ON g."grantId" = gr.id
+      ;
+
+      -- As a protective step, this will create a divide by zero error and
+      -- rollback the transaction if there is more than
+      -- one recipient found with 'Transportation' responses
+      SELECT 1/
+        (
+          LEAST(2, (SELECT COUNT(*) FROM recipient_list))
+          - 2
+        )
+      ;
+
+      -- Perform the actual updates to ActivityReportGoalFieldResponses
+      CREATE TEMP TABLE argfr_updates
+      AS
+      WITH updater AS (
+      UPDATE "ActivityReportGoalFieldResponses" argfr
+      SET response = ARRAY_REPLACE(response,'Transportation','Family Circumstances')
+      FROM argfr_to_update u
+      WHERE argfrid = argfr.id
+      RETURNING
+        argfr.id argfrid,
+        'ActivityReportGoalFieldResponses' tablename
+      ) SELECT * FROM updater
+      ;
+
+      -- Perform the actual updates to GoalFieldResponses
+      CREATE TEMP TABLE gfr_updates
+      AS
+      WITH updater AS (
+      UPDATE "GoalFieldResponses" gfr
+      SET response = ARRAY_REPLACE(response,'Transportation','Family Circumstances')
+      FROM gfr_to_update u
+      WHERE gfrid = gfr.id
+      RETURNING
+        gfr.id gfrid,
+        'GoalFieldResponses' tablename
+      ) SELECT * FROM updater
+      ;
+
+
+      -- A quick count of the results that is expected to be:
+      -- update_cnt |            tablename
+      -- -----------+----------------------------------
+      --          3 | GoalFieldResponses
+      --         10 | ActivityReportGoalFieldResponses
+      SELECT
+        COUNT(*) update_cnt,
+        tablename
+      FROM gfr_updates
+      GROUP BY 2
+      UNION
+      SELECT
+        COUNT(*),
+        tablename
+      FROM argfr_updates
+      GROUP BY 2
+      ;
+
+
+        `, { transaction });
+    },
+  ),
+
+  down: async (queryInterface) => queryInterface.sequelize.transaction(
+    async (transaction) => {
+      await prepMigration(queryInterface, transaction, __filename);
+      // Reversing this should be a separate migration
+    },
+  ),
+};

src/services/recipient.js

@@ -26,6 +26,7 @@ import {
   ActivityReportCollaborator,
   ActivityReportApprover,
   ActivityReportObjective,
+  GoalTemplateFieldPrompt,
 } from '../models';
 import orderRecipientsBy from '../lib/orderRecipientsBy';
 import {
@@ -41,6 +42,7 @@ import {
   findOrFailExistingGoal,
   responsesForComparison,
 } from '../goalServices/helpers';
+import getCachedResponse from '../lib/cache';
 
 export async function allArUserIdsByRecipientAndRegion(recipientId, regionId) {
   const reports = await ActivityReport.findAll({
@@ -511,6 +513,24 @@ export async function getGoalsByActivityRecipient(
     ...filters
   },
 ) {
+  // Get the GoalTemplateFieldPrompts where title is 'FEI root cause'.
+  const feiCacheKey = 'feiRootCauseFieldPrompt';
+  const feiResponse = await getCachedResponse(
+    feiCacheKey,
+    async () => {
+      const feiRootCauseFieldPrompt = await GoalTemplateFieldPrompt.findOne({
+        attributes: ['goalTemplateId'],
+        where: {
+          title: 'FEI root cause',
+        },
+      });
+      return JSON.stringify({
+        feiRootCauseFieldPrompt,
+      });
+    },
+    JSON.parse,
+  );
+
   // Scopes.
   const { goal: scopes } = await filtersToScopes(filters, { goal: { recipientId } });
 
@@ -589,6 +609,7 @@ export async function getGoalsByActivityRecipient(
           ELSE 7 END`),
       'status_sort'],
       [sequelize.literal(`CASE WHEN "Goal"."id" IN (${sanitizedIds}) THEN 1 ELSE 2 END`), 'merged_id'],
+      [sequelize.literal(`COALESCE("Goal"."goalTemplateId", 0) = ${feiResponse.feiRootCauseFieldPrompt.goalTemplateId}`), 'isFei'],
     ],
     where: goalWhere,
     include: [
@@ -868,6 +889,8 @@ export async function getGoalsByActivityRecipient(
       collaborators: [],
       onAR: current.onAR,
       sessionObjectives: [],
+      responses: current.responses,
+      isFei: current.dataValues.isFei,
     };
 
     goalToAdd.collaborators.push(...createCollaborators(current));