feat: Classify themes, endpoint management

html/modules/custom/reliefweb_openai/config/install/reliefweb_openai.settings.yml

@@ -1,2 +1,6 @@
 token: 'xyzzy'
 api_org: 'unocha'
+aws_access_key: ''
+aws_secret_key: ''
+aws_region: 'eu-central-1'
+aws_endpoint_theme_classifier: 'arn:aws:comprehend:eu-central-1:694216630861:document-classifier-endpoint/rw-themes'

html/modules/custom/reliefweb_openai/config/schema/reliefweb_openai.schema.yml

@@ -8,3 +8,15 @@ reliefweb_openai.settings:
     api_org:
       type: string
       label: 'OpenAI organization ID'
+    aws_access_key:
+      type: string
+      label: 'AWS Access key'
+    aws_secret_key:
+      type: string
+      label: 'AWS Secret key'
+    aws_region:
+      type: string
+      label: 'AWS Region'
+    aws_endpoint_theme_classifier:
+      type: string
+      label: 'AWS Theme classifier endpoint'

html/modules/custom/reliefweb_openai/drush.services.yml

@@ -4,3 +4,9 @@ services:
     arguments: ['@database', '@entity_type.manager', '@account_switcher', '@http_client', '@logger.factory', '@state']
     tags:
       - { name: drush.command }
+  reliefweb_openai.aws.commands:
+    class: \Drupal\reliefweb_openai\Command\ReliefwebOpenAIAwsCommand
+    arguments: ['@database', '@entity_type.manager', '@account_switcher', '@http_client', '@logger.factory', '@state']
+    tags:
+      - { name: drush.command }
+

html/modules/custom/reliefweb_openai/reliefweb_openai.module

@@ -12,6 +12,7 @@ use Drupal\Core\Ajax\HtmlCommand;
 use Drupal\Core\Entity\EntityInterface;
 use Drupal\Core\Form\FormStateInterface;
 use Drupal\node\Entity\Node;
+use Symfony\Component\Validator\Constraints\Length;
 
 /**
  * Implements hook_form_alter().
@@ -264,18 +265,20 @@ function reliefweb_openai_content_node_suggest_aws(array &$form, FormStateInterf
 
     $access_key = $config->get('aws_access_key');
     $secret_key = $config->get('aws_secret_key');
+    $region = $config->get('aws_region');
+    $endpoint = $config->get('aws_endpoint_theme_classifier');
 
     $client = new ComprehendClient([
-      'region' => 'eu-central-1',
+      'region' => $region,
       'version' => 'latest',
       'credentials' => [
-        'key' => $access_key,
-        'secret' => $secret_key,
-      ],
+      'key' => $access_key,
+      'secret' => $secret_key,
+      ]
     ]);
 
     $result = $client->classifyDocument([
-      'EndpointArn' => 'arn:aws:comprehend:eu-central-1:694216630861:document-classifier-endpoint/rw20230711',
+      'EndpointArn' => $endpoint,
       'Text' => $body,
     ]);
 
@@ -291,3 +294,78 @@ function reliefweb_openai_content_node_suggest_aws(array &$form, FormStateInterf
     return $response;
   }
 }
+
+function testit() {
+  $body = file_get_contents('/var/www/testpdf.txt');
+  $chunks = explode('|||', chunk_split($body, 4000, '|||'));
+
+  $results = [];
+
+  foreach ($chunks as $chunk) {
+    $results[] = reliefweb_openai_http_call([
+      'model' => 'text-davinci-003',
+      'prompt' => $chunk . "\n\nTl;dr",
+      'temperature' => 1,
+      'max_tokens' => 260,
+      'top_p' => 1.0,
+      'frequency_penalty' => 0.0,
+      'presence_penalty' => 1
+    ]);
+  }
+
+  print_r($results);
+}
+
+
+function testit2() {
+  $body = file_get_contents('/var/www/testpdf.txt');
+  $chunks = explode('|||', chunk_split($body, 9000, '|||'));
+
+  $results = [];
+
+  foreach ($chunks as $index => $chunk) {
+    print('Processing ' . $index . ' of ' . count($chunks)) . "\n";
+
+    if (strlen($chunk) < 100) {
+      continue;
+    }
+
+    $results[] = reliefweb_openai_http_call_chat(
+      [
+        'model' => 'gpt-3.5-turbo-16k',
+        'messages' => [
+          [
+            'role' => 'user',
+            'content' => "Summerize the following text:\n\n" . $chunk,
+          ],
+        ],
+        'temperature' => .8,
+        'max_tokens' => 300,
+      ],
+    );
+  }
+
+  $text = '';
+  foreach ($results as $row) {
+    $text .= $row['choices'][0]['message']['content'] ?? '';
+    $text .= "\n";
+  }
+
+  print_r($text);
+
+  $result = reliefweb_openai_http_call_chat(
+    [
+      'model' => 'gpt-3.5-turbo-16k',
+      'messages' => [
+        [
+          'role' => 'user',
+          'content' => "Summerize the following text:\n\n" . $text,
+        ],
+      ],
+      'temperature' => .8,
+      'max_tokens' => 300,
+    ],
+  );
+
+  print_r($result['choices'][0]['message']['content']);
+}

html/modules/custom/reliefweb_openai/src/Command/ReliefwebOpenAIAwsCommand.php

@@ -0,0 +1,192 @@
+<?php
+
+namespace Drupal\reliefweb_openai\Command;
+
+use Aws\Comprehend\ComprehendClient;
+use Consolidation\OutputFormatters\Options\FormatterOptions;
+use Consolidation\OutputFormatters\StructuredData\RowsOfFields;
+use Consolidation\SiteAlias\SiteAliasManagerAwareInterface;
+use Consolidation\SiteAlias\SiteAliasManagerAwareTrait;
+use Consolidation\SiteProcess\ProcessManagerAwareTrait;
+use Drupal\Core\Database\Connection;
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Drupal\Core\Logger\LoggerChannelFactoryInterface;
+use Drupal\Core\State\StateInterface;
+use Drush\Commands\DrushCommands;
+use GuzzleHttp\ClientInterface;
+
+/**
+ * ReliefWeb Open AI Drush commandfile.
+ */
+class ReliefwebOpenAIAwsCommand extends DrushCommands implements SiteAliasManagerAwareInterface {
+
+  // Drush traits.
+  use ProcessManagerAwareTrait;
+  use SiteAliasManagerAwareTrait;
+
+  /**
+   * The database connection.
+   *
+   * @var \Drupal\Core\Database\Connection
+   */
+  protected $database;
+
+  /**
+   * The entity type manager.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
+   */
+  protected $entityTypeManager;
+
+  /**
+   * The account switcher.
+   *
+   * @var \Drupal\Core\Session\AccountSwitcherInterface
+   */
+  protected $accountSwitcher;
+
+  /**
+   * An http client.
+   *
+   * @var \GuzzleHttp\ClientInterface
+   */
+  protected $httpClient;
+
+  /**
+   * The logger factory.
+   *
+   * @var \Drupal\Core\Logger\LoggerChannelFactoryInterface
+   */
+  protected $loggerFactory;
+
+  /**
+   * The state store.
+   *
+   * @var Drupal\Core\State\StateInterface
+   */
+  protected $state;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function __construct(
+    Connection $database,
+    EntityTypeManagerInterface $entity_type_manager,
+    $account_switcher,
+    ClientInterface $http_client,
+    LoggerChannelFactoryInterface $logger_factory,
+    StateInterface $state,
+  ) {
+    $this->database = $database;
+    $this->entityTypeManager = $entity_type_manager;
+    $this->accountSwitcher = $account_switcher;
+    $this->httpClient = $http_client;
+    $this->loggerFactory = $logger_factory;
+    $this->state = $state;
+  }
+
+  /**
+   * List endpoints.
+   *
+   * @command reliefweb_openai:aws_endpoints:list
+   * @usage reliefweb_openai:aws_endpoints:list
+   *   List AWS endpoints.
+   * @validate-module-enabled reliefweb_openai
+   */
+  public function listEndpoints(array $options = [
+    'format' => 'table',
+  ]) : RowsOfFields {
+    $config = \Drupal::config('reliefweb_openai.settings');
+
+    $access_key = $config->get('aws_access_key');
+    $secret_key = $config->get('aws_secret_key');
+    $region = $config->get('aws_region');
+
+    $client = new ComprehendClient([
+      'region' => $region,
+      'version' => 'latest',
+      'credentials' => [
+      'key' => $access_key,
+      'secret' => $secret_key,
+      ]
+    ]);
+
+    $result = $client->listEndpoints();
+    $data = [];
+
+    foreach ($result->get('EndpointPropertiesList') as $row) {
+      $data[$row['EndpointArn']] = [
+        'EndpointArn' => $row['EndpointArn'],
+        'Status' => $row['Status'],
+        'DesiredInferenceUnits' => $row['DesiredInferenceUnits'],
+      ];
+    }
+
+    return new RowsOfFields($data);
+  }
+
+  /**
+   * Create endpoint.
+   *
+   * @command reliefweb_openai:aws_endpoints:create
+   * @usage reliefweb_openai:aws_endpoints:create
+   *   Create AWS endpoints.
+   * @validate-module-enabled reliefweb_openai
+   */
+  public function createEndpoint() : string {
+    $config = \Drupal::config('reliefweb_openai.settings');
+
+    $access_key = $config->get('aws_access_key');
+    $secret_key = $config->get('aws_secret_key');
+    $region = $config->get('aws_region');
+
+    $client = new ComprehendClient([
+      'region' => $region,
+      'version' => 'latest',
+      'credentials' => [
+      'key' => $access_key,
+      'secret' => $secret_key,
+      ]
+    ]);
+
+    $result = $client->createEndpoint([
+      'EndpointName' => 'rw-themes',
+      'ModelArn' => 'arn:aws:comprehend:eu-central-1:694216630861:document-classifier/RW-Job-Tagging/version/v0-0-3',
+      'DesiredInferenceUnits' => 1,
+    ]);
+
+    return 'Endpoint will be created, takes 5-10 minutes to complete';
+  }
+
+  /**
+   * Delete endpoint.
+   *
+   * @command reliefweb_openai:aws_endpoints:delete
+   * @usage reliefweb_openai:aws_endpoints:delete
+   *   Delete AWS endpoints.
+   * @validate-module-enabled reliefweb_openai
+   */
+  public function deleteEndpoints(string $arn) : string {
+    $config = \Drupal::config('reliefweb_openai.settings');
+
+    $access_key = $config->get('aws_access_key');
+    $secret_key = $config->get('aws_secret_key');
+    $region = $config->get('aws_region');
+
+    $client = new ComprehendClient([
+      'region' => $region,
+      'version' => 'latest',
+      'credentials' => [
+      'key' => $access_key,
+      'secret' => $secret_key,
+      ]
+    ]);
+
+    $client->deleteEndpoint([
+      'EndpointArn' => $arn,
+    ]);
+
+    return 'Endpoint will be deleted';
+  }
+
+}

html/modules/custom/reliefweb_openai/src/Form/ApiSettingsForm.php

@@ -43,6 +43,34 @@ public function buildForm(array $form, FormStateInterface $form_state) {
       '#description' => $this->t('The organization name or ID on your OpenAI account. This is required for some OpenAI services to work correctly.'),
     ];
 
+    $form['aws_access_key'] = [
+      '#type' => 'textfield',
+      '#title' => $this->t('AWS Access key'),
+      '#default_value' => $this->config('reliefweb_openai.settings')->get('aws_access_key'),
+      '#description' => $this->t('AWS Access key'),
+    ];
+
+    $form['aws_secret_key'] = [
+      '#type' => 'textfield',
+      '#title' => $this->t('AWS Secret key'),
+      '#default_value' => $this->config('reliefweb_openai.settings')->get('aws_secret_key'),
+      '#description' => $this->t('AWS Secret key'),
+    ];
+
+    $form['aws_region'] = [
+      '#type' => 'textfield',
+      '#title' => $this->t('AWS Region'),
+      '#default_value' => $this->config('reliefweb_openai.settings')->get('aws_region'),
+      '#description' => $this->t('AWS Region'),
+    ];
+
+    $form['aws_endpoint_theme_classifier'] = [
+      '#type' => 'textfield',
+      '#title' => $this->t('AWS Theme classifier endpoint'),
+      '#default_value' => $this->config('reliefweb_openai.settings')->get('aws_endpoint_theme_classifier'),
+      '#description' => $this->t('AWS Theme classifier endpoint'),
+    ];
+
     return parent::buildForm($form, $form_state);
   }
 
@@ -53,6 +81,10 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
     $this->config('reliefweb_openai.settings')
       ->set('token', $form_state->getValue('token'))
       ->set('api_org', $form_state->getValue('api_org'))
+      ->set('aws_access_key', $form_state->getValue('aws_access_key'))
+      ->set('aws_secret_key', $form_state->getValue('aws_secret_key'))
+      ->set('aws_region', $form_state->getValue('aws_region'))
+      ->set('aws_endpoint_theme_classifier', $form_state->getValue('aws_endpoint_theme_classifier'))
       ->save();
     parent::submitForm($form, $form_state);
   }