diff --git a/README.md b/README.md index e06b32988..2aa9e5e9d 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,10 @@ * [Installation](#installation) * [Register ClassifAI account](#register-classifai-account) * [Set Up NLU Language Processing](#set-up-language-processing-via-ibm-watson) -* [Set Up ChatGPT Language Processing](#set-up-language-processing-via-openai) +* [Set Up OpenAI ChatGPT Language Processing](#set-up-language-processing-via-openai-chatgpt) +* [Set Up OpenAI Whisper Language Processing](#set-up-language-processing-via-openai-whisper) * [Set Up Computer Vision Image Processing](#set-up-image-processing-via-microsoft-azure) -* [Set Up DALL·E Image Processing](#set-up-image-processing-via-openai) +* [Set Up OpenAI DALL·E Image Processing](#set-up-image-processing-via-openai) * [Set Up Recommended Content](#set-up-recommended-content-via-microsoft-azure-personalizer) * [WP CLI Commands](#wp-cli-commands) * [FAQs](#frequently-asked-questions) @@ -27,6 +28,7 @@ * Automatically generate a summary of your content and store that as an excerpt using [OpenAI's ChatGPT](https://platform.openai.com/docs/guides/chat) * Generate new images on demand to use in-content or as a featured image using [OpenAI's DALL·E](https://platform.openai.com/docs/guides/images) +* Automatically generate transcripts of your audio files using [OpenAI's Whisper](https://platform.openai.com/docs/guides/speech-to-text) * Classify your content using [IBM Watson's Natural Language Understanding API](https://www.ibm.com/watson/services/natural-language-understanding/) and [Microsoft Azure's Computer Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/) * Supports Watson's [Categories](https://cloud.ibm.com/docs/natural-language-understanding?topic=natural-language-understanding-about#categories), [Keywords](https://cloud.ibm.com/docs/natural-language-understanding?topic=natural-language-understanding-about#keywords), [Concepts](https://cloud.ibm.com/docs/natural-language-understanding?topic=natural-language-understanding-about#concepts) & [Entities](https://cloud.ibm.com/docs/natural-language-understanding?topic=natural-language-understanding-about#entities) and Azure's [Describe Image](https://westus.dev.cognitive.microsoft.com/docs/services/5adf991815e1060e6355ad44/operations/56f91f2e778daf14a499e1fe) * Automatically classify content and images on save @@ -36,9 +38,9 @@ * BETA: Recommend content based on overall site traffic via [Azure Personalizer](https://azure.microsoft.com/en-us/services/cognitive-services/personalizer/) (note that we're gathering feedback on this feature and may significantly iterate depending on community input) * Bulk classify content with [WP-CLI](https://wp-cli.org/) -| Language Processing - Tagging | Recommended Content | Excerpt Generation | -| :-: | :-: | :-: | -| ![Screenshot of ClassifAI post tagging](assets/img/screenshot-1.png "Example of a Block Editor post with Watson Categories, Keywords, Concepts, and Entities.") | ![Screenshot of ClassifAI recommended content](assets/img/screenshot-2.png "Example of a Recommended Content Block with Azure Personalizer.") | ![Screenshot of ClassifAI excerpt generation](assets/img/screenshot-7.png "Example of automatic excerpt generation with OpenAI.") | +| Language Processing - Tagging | Recommended Content | Excerpt Generation | Audio Transcripts | +| :-: | :-: | :-: | :-: | +| ![Screenshot of ClassifAI post tagging](assets/img/screenshot-1.png "Example of a Block Editor post with Watson Categories, Keywords, Concepts, and Entities.") | ![Screenshot of ClassifAI recommended content](assets/img/screenshot-2.png "Example of a Recommended Content Block with Azure Personalizer.") | ![Screenshot of ClassifAI excerpt generation](assets/img/screenshot-7.png "Example of automatic excerpt generation with OpenAI.") | ![Screenshot of ClassifAI audio transcript generation](assets/img/screenshot-9.png "Example of automatic audio transcript generation with OpenAI.") | | Image Processing - Alt Text | Image Processing - Smart Cropping | Image Processing - Tagging | Image Processing - Generate Images | | :-: | :-: | :-: | :-: | @@ -49,7 +51,7 @@ * PHP 7.4+ * [WordPress](http://wordpress.org) 5.7+ * To utilize the NLU Language Processing functionality, you will need an active [IBM Watson](https://cloud.ibm.com/registration) account. -* To utilize the ChatGPT Language Processing functionality or DALL·E Image Processing functionality, you will need an active [OpenAI](https://platform.openai.com/signup) account. +* To utilize the ChatGPT or Whisper Language Processing functionality or DALL·E Image Processing functionality, you will need an active [OpenAI](https://platform.openai.com/signup) account. * To utilize the Computer Vision Image Processing functionality, you will need an active [Microsoft Azure](https://signup.azure.com/signup) account. ## Pricing @@ -58,7 +60,7 @@ Note that there is no cost to using ClassifAI itself. Both IBM Watson and Micros The service that powers ClassifAI's NLU Language Processing, IBM Watson's Natural Language Understanding ("NLU"), has a ["lite" pricing tier](https://www.ibm.com/cloud/watson-natural-language-understanding/pricing) that offers 30,000 free NLU items per month. -The service that powers ClassifAI's ChatGPT Language Processing and DALL·E Image Processing, OpenAI, has a limited free trial and then requires a [pay per usage](https://openai.com/pricing) plan. +The service that powers ClassifAI's ChatGPT and Whisper Language Processing and DALL·E Image Processing, OpenAI, has a limited free trial and then requires a [pay per usage](https://openai.com/pricing) plan. The service that powers ClassifAI's Computer Vision Image Processing, Microsoft Azure, has a ["free" pricing tier](https://azure.microsoft.com/en-us/pricing/details/cognitive-services/computer-vision/) that offers 20 transactions per minute and 5,000 transactions per month. @@ -134,7 +136,7 @@ ClassifAI is a sophisticated solution that we want organizations of all shapes a - Check for an email from `ClassifAI Team` which contains the registration key. - Note that the email will be sent from `opensource@10up.com`, so please whitelist this email address if needed. -### 2. Configure ClassifAI Registration Key under ClassifAI > ClassifAI +### 2. Configure ClassifAI Registration Key under Tools > ClassifAI - In the `Registered Email` field, enter the email you used for registration. - In the `Registration Key` field, enter the registration key from the email in step 1 above. @@ -150,7 +152,7 @@ ClassifAI is a sophisticated solution that we want organizations of all shapes a - Log into your account (accepting the privacy policy) and create a new [*Natural Language Understanding*](https://cloud.ibm.com/catalog/services/natural-language-understanding) Resource if you do not already have one. It may take a minute for your account to fully populate with the default resource group to use. - Click `Manage` in the left hand menu, then `Show credentials` on the Manage page to view the credentials for this resource. -### 2. Configure IBM Watson API Keys under ClassifAI > Language Processing > IBM Watson +### 2. Configure IBM Watson API Keys under Tools > ClassifAI > Language Processing > IBM Watson **The credentials screen will show either an API key or a username/password combination.** @@ -178,7 +180,7 @@ For more information, see https://cloud.ibm.com/docs/watson?topic=watson-endpoin ### 4. Save a Post/Page/CPT or run WP CLI command to batch classify your content -## Set Up Language Processing (via OpenAI) +## Set Up Language Processing (via OpenAI ChatGPT) ### 1. Sign up for OpenAI @@ -187,7 +189,7 @@ For more information, see https://cloud.ibm.com/docs/watson?topic=watson-endpoin * Log into your account and go to the [API key page](https://platform.openai.com/account/api-keys). * Click `Create new secret key` and copy the key that is shown. -### 2. Configure OpenAI API Keys under ClassifAI > Language Processing > OpenAI +### 2. Configure OpenAI API Keys under Tools > ClassifAI > Language Processing > OpenAI ChatGPT * Enter your API Key copied from the above step into the `API Key` field. @@ -203,6 +205,34 @@ For more information, see https://cloud.ibm.com/docs/watson?topic=watson-endpoin * Ensure this item has content saved. * Open the Excerpt panel in the sidebar and click on `Generate Excerpt` +## Set Up Language Processing (via OpenAI Whisper) + +Note that [OpenAI](https://platform.openai.com/docs/guides/speech-to-text) can create a transcript for audio files that meet the following requirements: +* The file must be presented in mp3, mp4, mpeg, mpga, m4a, wav, or webm format +* The file size must be less than 25 megabytes (MB) + +### 1. Sign up for OpenAI + +* [Sign up for an OpenAI account](https://platform.openai.com/signup) or sign into your existing one. +* If creating a new account, complete the verification process (requires confirming your email and phone number). +* Log into your account and go to the [API key page](https://platform.openai.com/account/api-keys). +* Click `Create new secret key` and copy the key that is shown. + +### 2. Configure OpenAI API Keys under Tools > ClassifAI > Language Processing > OpenAI Whisper + +* Enter your API Key copied from the above step into the `API Key` field. + +### 3. Enable specific features + +* Choose to enable the ability to automatically generate transcripts from supported audio files. +* Choose which user roles have access to this ability. +* Save changes and ensure a success message is shown. An error will show if API authentication fails. + +### 4. Upload a new audio file + +* Upload a new audio file. +* Check to make sure the transcript was stored in the Description field. + ## Set Up Image Processing (via Microsoft Azure) Note that [Computer Vision](https://docs.microsoft.com/en-us/azure/cognitive-services/computer-vision/home#image-requirements) can analyze and crop images that meet the following requirements: @@ -218,7 +248,7 @@ Note that [Computer Vision](https://docs.microsoft.com/en-us/azure/cognitive-ser - Click `Keys and Endpoint` in the left hand Resource Management menu to view the `Endpoint` URL for this resource. - Click the copy icon next to `KEY 1` to copy the API Key credential for this resource. -### 2. Configure Microsoft Azure API and Key under ClassifAI > Image Processing +### 2. Configure Microsoft Azure API and Key under Tools > ClassifAI > Image Processing - In the `Endpoint URL` field, enter your `API endpoint`. - In the `API Key` field, enter your `KEY 1`. @@ -239,7 +269,7 @@ Note that [Computer Vision](https://docs.microsoft.com/en-us/azure/cognitive-ser * Log into your account and go to the [API key page](https://platform.openai.com/account/api-keys). * Click `Create new secret key` and copy the key that is shown. -### 2. Configure OpenAI API Keys under ClassifAI > Image Processing > OpenAI +### 2. Configure OpenAI API Keys under Tools > ClassifAI > Image Processing > OpenAI * Enter your API Key copied from the above step into the `API Key` field. @@ -274,7 +304,7 @@ Note that [Personalizer](https://azure.microsoft.com/en-us/services/cognitive-se For more information, see https://docs.microsoft.com/en-us/azure/cognitive-services/personalizer/how-to-create-resource -### 2. Configure Microsoft Azure API and Key under ClassifAI > Recommended Content +### 2. Configure Microsoft Azure API and Key under Tools > ClassifAI > Recommended Content - In the `Endpoint URL` field, enter your `Endpoint` URL from Step 1 above. - In the `API Key` field, enter your `KEY 1` from Step 1 above. diff --git a/assets/img/screenshot-6.png b/assets/img/screenshot-6.png index fd461d3f1..9117400ae 100644 Binary files a/assets/img/screenshot-6.png and b/assets/img/screenshot-6.png differ diff --git a/assets/img/screenshot-9.png b/assets/img/screenshot-9.png new file mode 100644 index 000000000..f0a4f7486 Binary files /dev/null and b/assets/img/screenshot-9.png differ diff --git a/includes/Classifai/Admin/BulkActions.php b/includes/Classifai/Admin/BulkActions.php index 5d30bd10e..49eb2678e 100644 --- a/includes/Classifai/Admin/BulkActions.php +++ b/includes/Classifai/Admin/BulkActions.php @@ -2,12 +2,15 @@ namespace Classifai\Admin; use Classifai\Providers\Azure\ComputerVision; +use Classifai\Providers\OpenAI\Whisper; +use Classifai\Providers\OpenAI\Whisper\Transcribe; use function Classifai\get_supported_post_types; /** * Handle bulk actions. */ class BulkActions { + /** * Check to see if we can register this class. * @@ -27,34 +30,61 @@ public function can_register() { */ private $computer_vision; + /** + * @var \Classifai\Providers\OpenAI\Whisper + */ + private $whisper; + /** * Register the actions needed. */ public function register() { - $post_types = get_supported_post_types(); + $this->register_language_processing_hooks(); + $this->register_image_processing_hooks(); + + add_action( 'admin_notices', [ $this, 'bulk_action_admin_notice' ] ); + } + + /** + * Register bulk actions for language processing. + */ + public function register_language_processing_hooks() { + $nlu_post_types = get_supported_post_types(); + + // Set up the save post handler if we have any post types for NLU. + if ( ! empty( $nlu_post_types ) ) { + $this->save_post_handler = new SavePostHandler(); + } + + // Merge our post types together and make them unique. + $post_types = array_unique( array_merge( $nlu_post_types, [] ) ); if ( empty( $post_types ) ) { return; } - $this->save_post_handler = new SavePostHandler(); - $this->computer_vision = new ComputerVision( false ); - foreach ( $post_types as $post_type ) { add_filter( "bulk_actions-edit-$post_type", [ $this, 'register_bulk_actions' ] ); add_filter( "handle_bulk_actions-edit-$post_type", [ $this, 'bulk_action_handler' ], 10, 3 ); if ( is_post_type_hierarchical( $post_type ) ) { - add_action( 'page_row_actions', [ $this, 'register_row_action' ], 10, 2 ); + add_filter( 'page_row_actions', [ $this, 'register_row_action' ], 10, 2 ); } else { - add_action( 'post_row_actions', [ $this, 'register_row_action' ], 10, 2 ); + add_filter( 'post_row_actions', [ $this, 'register_row_action' ], 10, 2 ); } } + } + + /** + * Register bulk actions for the Computer Vision provider. + */ + public function register_image_processing_hooks() { + $this->computer_vision = new ComputerVision( false ); + $this->whisper = new Whisper( false ); add_filter( 'bulk_actions-upload', [ $this, 'register_media_bulk_actions' ] ); add_filter( 'handle_bulk_actions-upload', [ $this, 'media_bulk_action_handler' ], 10, 3 ); - - add_action( 'admin_notices', [ $this, 'bulk_action_admin_notice' ] ); + add_filter( 'media_row_actions', [ $this, 'register_media_row_action' ], 10, 2 ); } /** @@ -77,17 +107,22 @@ public function register_bulk_actions( $bulk_actions ) { * @return array */ public function register_media_bulk_actions( $bulk_actions ) { - $settings = $this->computer_vision->get_settings(); + $computer_vision_settings = $this->computer_vision->get_settings(); + $whisper_enabled = $this->whisper->is_feature_enabled(); if ( - 'no' !== $settings['enable_image_tagging'] || + 'no' !== $computer_vision_settings['enable_image_tagging'] || ! empty( $this->computer_vision->get_alt_text_settings() ) ) { - $bulk_actions['scan_image'] = __( 'Scan Image', 'classifai' ); + $bulk_actions['scan_image'] = __( 'Scan image', 'classifai' ); + } + + if ( isset( $computer_vision_settings['enable_smart_cropping'] ) && '1' === $computer_vision_settings['enable_smart_cropping'] ) { + $bulk_actions['smart_crop'] = __( 'Smart crop', 'classifai' ); } - if ( isset( $settings['enable_smart_cropping'] ) && '1' === $settings['enable_smart_cropping'] ) { - $bulk_actions['smart_crop'] = __( 'Smart Crop', 'classifai' ); + if ( ! is_wp_error( $whisper_enabled ) ) { + $bulk_actions['transcribe'] = __( 'Transcribe audio', 'classifai' ); } return $bulk_actions; @@ -108,11 +143,16 @@ public function bulk_action_handler( $redirect_to, $doaction, $post_ids ) { } foreach ( $post_ids as $post_id ) { - $this->save_post_handler->classify( $post_id ); + // Handle NLU classification. + if ( is_a( $this->save_post_handler, '\Classifai\Admin\SavePostHandler' ) ) { + $this->save_post_handler->classify( $post_id ); + } } - $redirect_to = remove_query_arg( [ 'bulk_classified', 'bulk_scanned', 'bulk_cropped' ], $redirect_to ); + + $redirect_to = remove_query_arg( [ 'bulk_classified', 'bulk_scanned', 'bulk_cropped', 'bulk_transcribed' ], $redirect_to ); $redirect_to = add_query_arg( 'bulk_classified', count( $post_ids ), $redirect_to ); - return $redirect_to; + + return esc_url_raw( $redirect_to ); } /** @@ -127,25 +167,34 @@ public function bulk_action_handler( $redirect_to, $doaction, $post_ids ) { public function media_bulk_action_handler( $redirect_to, $doaction, $attachment_ids ) { if ( empty( $attachment_ids ) || - ! in_array( $doaction, [ 'scan_image', 'smart_crop' ], true ) + ! in_array( $doaction, [ 'scan_image', 'smart_crop', 'transcribe' ], true ) ) { return $redirect_to; } + $action = ''; + foreach ( $attachment_ids as $attachment_id ) { + if ( 'transcribe' === $doaction ) { + $action = 'transcribed'; + $this->whisper->transcribe_audio( $attachment_id ); + continue; + } + $current_meta = wp_get_attachment_metadata( $attachment_id ); if ( 'smart_crop' === $doaction ) { + $action = 'cropped'; $this->computer_vision->smart_crop_image( $current_meta, $attachment_id ); - } else { + } elseif ( 'scan_image' === $doaction ) { + $action = 'scanned'; $this->computer_vision->generate_image_alt_tags( $current_meta, $attachment_id ); } } - $action = 'scan_image' === $doaction ? 'scanned' : 'cropped'; - - $redirect_to = remove_query_arg( [ 'bulk_classified', 'bulk_scanned', 'bulk_cropped' ], $redirect_to ); + $redirect_to = remove_query_arg( [ 'bulk_classified', 'bulk_scanned', 'bulk_cropped', 'bulk_transcribed' ], $redirect_to ); $redirect_to = add_query_arg( rawurlencode( "bulk_{$action}" ), count( $attachment_ids ), $redirect_to ); + return esc_url_raw( $redirect_to ); } @@ -154,11 +203,12 @@ public function media_bulk_action_handler( $redirect_to, $doaction, $attachment_ */ public function bulk_action_admin_notice() { - $classified = ! empty( $_GET['bulk_classified'] ) ? intval( wp_unslash( $_GET['bulk_classified'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended - $scanned = ! empty( $_GET['bulk_scanned'] ) ? intval( wp_unslash( $_GET['bulk_scanned'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended - $cropped = ! empty( $_GET['bulk_cropped'] ) ? intval( wp_unslash( $_GET['bulk_cropped'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended + $classified = ! empty( $_GET['bulk_classified'] ) ? intval( wp_unslash( $_GET['bulk_classified'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended + $scanned = ! empty( $_GET['bulk_scanned'] ) ? intval( wp_unslash( $_GET['bulk_scanned'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended + $cropped = ! empty( $_GET['bulk_cropped'] ) ? intval( wp_unslash( $_GET['bulk_cropped'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended + $transcribed = ! empty( $_GET['bulk_transcribed'] ) ? intval( wp_unslash( $_GET['bulk_transcribed'] ) ) : 0; // phpcs:ignore WordPress.Security.NonceVerification.Recommended - if ( ! $classified && ! $scanned && ! $cropped ) { + if ( ! $classified && ! $scanned && ! $cropped && ! $transcribed ) { return; } @@ -174,6 +224,10 @@ public function bulk_action_admin_notice() { $classified_posts_count = $cropped; $post_type = 'image'; $action = __( 'Cropped', 'classifai' ); + } elseif ( $transcribed ) { + $classified_posts_count = $transcribed; + $post_type = 'audio'; + $action = __( 'Transcribed', 'classifai' ); } $output = '

'; @@ -190,6 +244,7 @@ public function bulk_action_admin_notice() { $post_type ); $output .= '

'; + echo wp_kses( $output, [ @@ -211,7 +266,11 @@ public function bulk_action_admin_notice() { * @return array */ public function register_row_action( $actions, $post ) { - $post_types = get_supported_post_types(); + $post_types = []; + + if ( is_a( $this->save_post_handler, '\Classifai\Admin\SavePostHandler' ) ) { + $post_types = get_supported_post_types(); + } if ( ! in_array( $post->post_type, $post_types, true ) ) { return $actions; @@ -225,4 +284,33 @@ public function register_row_action( $actions, $post ) { return $actions; } + + /** + * Register media row actions. + * + * @param array $actions An array of action links for each attachment. + * @param \WP_Post $post WP_Post object for the current attachment. + * @return array + */ + public function register_media_row_action( $actions, $post ) { + $whisper_settings = $this->whisper->get_settings(); + $whisper_enabled = $this->whisper->is_feature_enabled( $post->ID ); + + if ( is_wp_error( $whisper_enabled ) ) { + return $actions; + } + + $transcribe = new Transcribe( $post->ID, $whisper_settings ); + + if ( $transcribe->should_process( $post->ID ) ) { + $actions['transcribe'] = sprintf( + '%s', + esc_url( wp_nonce_url( admin_url( sprintf( 'upload.php?action=transcribe&ids=%d&post_type=%s', $post->ID, $post->post_type ) ), 'bulk-media' ) ), + esc_html__( 'Transcribe', 'classifai' ) + ); + } + + return $actions; + } + } diff --git a/includes/Classifai/Providers/Azure/ComputerVision.php b/includes/Classifai/Providers/Azure/ComputerVision.php index e8c40c73f..e993a077b 100644 --- a/includes/Classifai/Providers/Azure/ComputerVision.php +++ b/includes/Classifai/Providers/Azure/ComputerVision.php @@ -485,7 +485,7 @@ public function maybe_rescan_image( $attachment_id ) { $image_url = get_largest_acceptable_image_url( get_attached_file( $attachment_id ), wp_get_attachment_url( $attachment_id ), - $metadata['sizes'], + $metadata['sizes'] ?? [], computer_vision_max_filesize() ); } diff --git a/includes/Classifai/Providers/OpenAI/APIRequest.php b/includes/Classifai/Providers/OpenAI/APIRequest.php index 6ed0e2517..333954e2c 100644 --- a/includes/Classifai/Providers/OpenAI/APIRequest.php +++ b/includes/Classifai/Providers/OpenAI/APIRequest.php @@ -30,7 +30,7 @@ class APIRequest { * * @param string $api_key OpenAI API key. */ - public function __construct( $api_key = '' ) { + public function __construct( string $api_key = '' ) { $this->api_key = $api_key; } @@ -41,7 +41,7 @@ public function __construct( $api_key = '' ) { * @param array $options Additional query params * @return array|WP_Error */ - public function get( $url, $options = [] ) { + public function get( string $url, array $options = [] ) { $this->add_headers( $options ); return $this->get_result( wp_remote_get( $url, $options ) ); // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.wp_remote_get_wp_remote_get } @@ -53,7 +53,7 @@ public function get( $url, $options = [] ) { * @param array $options Additional query params. * @return array|WP_Error */ - public function post( $url = '', $options = [] ) { + public function post( string $url = '', array $options = [] ) { $options = wp_parse_args( $options, [ @@ -64,6 +64,50 @@ public function post( $url = '', $options = [] ) { return $this->get_result( wp_remote_post( $url, $options ) ); // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.wp_remote_get_wp_remote_get } + /** + * Makes an authorized POST request with form data. + * + * @param string $url The OpenAI API URL. + * @param array $body The body of the request. + * @return array|WP_Error + */ + public function post_form( string $url = '', array $body = [] ) { + $boundary = wp_generate_password( 24, false ); + $payload = ''; + + // Take all our POST fields and transform them to work with form-data. + foreach ( $body as $name => $value ) { + $payload .= '--' . $boundary; + $payload .= "\r\n"; + + if ( 'file' === $name ) { + $payload .= 'Content-Disposition: form-data; name="file"; filename="' . basename( $value ) . '"' . "\r\n"; + $payload .= "\r\n"; + $payload .= file_get_contents( $value ); // phpcs:ignore + } else { + $payload .= 'Content-Disposition: form-data; name="' . esc_attr( $name ) . + '"' . "\r\n\r\n"; + $payload .= esc_attr( $value ); + } + + $payload .= "\r\n"; + } + + $payload .= '--' . $boundary . '--'; + + $options = [ + 'body' => $payload, + 'headers' => [ + 'Content-Type' => 'multipart/form-data; boundary=' . $boundary, + ], + 'timeout' => 60, // phpcs:ignore WordPressVIPMinimum.Performance.RemoteRequestTimeout.timeout_timeout + ]; + + $this->add_headers( $options ); + + return $this->get_result( wp_remote_post( $url, $options ) ); + } + /** * Get results from the response. * @@ -96,13 +140,18 @@ public function get_result( $response ) { * * @param array $options The header options, passed by reference. */ - public function add_headers( &$options = [] ) { + public function add_headers( array &$options = [] ) { if ( empty( $options['headers'] ) ) { $options['headers'] = []; } - $options['headers']['Authorization'] = $this->get_auth_header(); - $options['headers']['Content-Type'] = 'application/json'; + if ( ! isset( $options['headers']['Authorization'] ) ) { + $options['headers']['Authorization'] = $this->get_auth_header(); + } + + if ( ! isset( $options['headers']['Content-Type'] ) ) { + $options['headers']['Content-Type'] = 'application/json'; + } } /** diff --git a/includes/Classifai/Providers/OpenAI/ChatGPT.php b/includes/Classifai/Providers/OpenAI/ChatGPT.php index 982c94130..93614c3d7 100644 --- a/includes/Classifai/Providers/OpenAI/ChatGPT.php +++ b/includes/Classifai/Providers/OpenAI/ChatGPT.php @@ -44,7 +44,7 @@ class ChatGPT extends Provider { */ public function __construct( $service ) { parent::__construct( - 'OpenAI', + 'OpenAI ChatGPT', 'ChatGPT', 'openai_chatgpt', $service diff --git a/includes/Classifai/Providers/OpenAI/Whisper.php b/includes/Classifai/Providers/OpenAI/Whisper.php new file mode 100644 index 000000000..32b01a1fe --- /dev/null +++ b/includes/Classifai/Providers/OpenAI/Whisper.php @@ -0,0 +1,335 @@ +onboarding_options = array( + 'title' => __( 'OpenAI Whisper', 'classifai' ), + 'fields' => array( 'api-key' ), + 'features' => array( + 'enable_transcripts' => __( 'Generate transcripts from audio files', 'classifai' ), + ), + ); + } + + /** + * Register what we need for the plugin. + * + * This only fires if can_register returns true. + */ + public function register() { + add_action( 'add_attachment', [ $this, 'transcribe_audio' ] ); + add_filter( 'attachment_fields_to_edit', [ $this, 'add_buttons_to_media_modal' ], 10, 2 ); + add_action( 'add_meta_boxes_attachment', [ $this, 'setup_attachment_meta_box' ] ); + add_action( 'edit_attachment', [ $this, 'maybe_transcribe_audio' ] ); + } + + /** + * Check to see if the feature is enabled and a user has access. + * + * @param int $attachment_id Attachment ID to process. + * @return bool|WP_Error + */ + public function is_feature_enabled( int $attachment_id = 0 ) { + $settings = $this->get_settings(); + + // Check if valid authentication is in place. + if ( empty( $settings ) || ( isset( $settings['authenticated'] ) && false === $settings['authenticated'] ) ) { + return new WP_Error( 'auth', esc_html__( 'Please set up valid authentication with OpenAI.', 'classifai' ) ); + } + + // Check if the current user has permission. + $roles = $settings['roles'] ?? []; + $user_roles = wp_get_current_user()->roles ?? []; + + if ( empty( $roles ) || ! empty( array_diff( $user_roles, $roles ) ) ) { + return new WP_Error( 'no_permission', esc_html__( 'User role does not have permission.', 'classifai' ) ); + } + + if ( $attachment_id && ! current_user_can( 'edit_post', $attachment_id ) ) { + return new WP_Error( 'no_permission', esc_html__( 'User does not have permission to edit this attachment.', 'classifai' ) ); + } + + // Ensure feature is turned on. + if ( ! isset( $settings['enable_transcripts'] ) || 1 !== (int) $settings['enable_transcripts'] ) { + return new WP_Error( 'not_enabled', esc_html__( 'Transcripts are not enabled.', 'classifai' ) ); + } + + return true; + } + + /** + * Start the audio transcription process. + * + * @param int $attachment_id Attachment ID to process. + * @return WP_Error|bool + */ + public function transcribe_audio( $attachment_id = 0 ) { + $settings = $this->get_settings(); + $enabled = $this->is_feature_enabled( $attachment_id ); + + if ( is_wp_error( $enabled ) ) { + return $enabled; + } + + $transcribe = new Transcribe( intval( $attachment_id ), $settings ); + + return $transcribe->process(); + } + + /** + * Add new buttons to the media modal. + * + * @param array $form_fields Existing form fields. + * @param \WP_Post $attachment Attachment object. + * @return array + */ + public function add_buttons_to_media_modal( $form_fields, $attachment ) { + $enabled = $this->is_feature_enabled( $attachment->ID ); + + if ( is_wp_error( $enabled ) ) { + return $form_fields; + } + + $settings = $this->get_settings(); + $transcribe = new Transcribe( $attachment->ID, $settings ); + + if ( ! $transcribe->should_process( $attachment->ID ) ) { + return $form_fields; + } + + if ( is_array( $settings ) && isset( $settings['enable_transcripts'] ) && '1' === $settings['enable_transcripts'] ) { + $text = empty( get_the_content( null, false, $attachment ) ) ? __( 'Transcribe', 'classifai' ) : __( 'Re-transcribe', 'classifai' ); + + $form_fields['retranscribe'] = [ + 'label' => __( 'Transcribe audio', 'classifai' ), + 'input' => 'html', + 'html' => '', + 'show_in_edit' => false, + ]; + } + + return $form_fields; + } + + /** + * Add metabox on single attachment view to allow for transcription. + * + * @param \WP_Post $post Post object. + */ + public function setup_attachment_meta_box( $post ) { + $enabled = $this->is_feature_enabled( $post->ID ); + + if ( is_wp_error( $enabled ) ) { + return; + } + + $settings = $this->get_settings(); + $transcribe = new Transcribe( $post->ID, $settings ); + + if ( ! $transcribe->should_process( $post->ID ) ) { + return; + } + + if ( is_array( $settings ) && isset( $settings['enable_transcripts'] ) && '1' === $settings['enable_transcripts'] ) { + add_meta_box( + 'attachment_meta_box', + __( 'ClassifAI Audio Processing', 'classifai' ), + [ $this, 'attachment_meta_box' ], + 'attachment', + 'side', + 'high' + ); + } + } + + /** + * Display the attachment meta box. + * + * @param \WP_Post $post Post object. + */ + public function attachment_meta_box( $post ) { + $text = empty( get_the_content( null, false, $post ) ) ? __( 'Transcribe', 'classifai' ) : __( 'Re-transcribe', 'classifai' ); + + wp_nonce_field( 'classifai_openai_whisper_meta_action', 'classifai_openai_whisper_meta' ); + ?> + +
+
+ +
+
+ + is_feature_enabled( $attachment_id ); + + if ( is_wp_error( $enabled ) ) { + return; + } + + if ( ( defined( 'DOING_AUTOSAVE' ) && DOING_AUTOSAVE ) || ! current_user_can( 'edit_post', $attachment_id ) ) { + return; + } + + if ( empty( $_POST['classifai_openai_whisper_meta'] ) || ! wp_verify_nonce( sanitize_text_field( wp_unslash( $_POST['classifai_openai_whisper_meta'] ) ), 'classifai_openai_whisper_meta_action' ) ) { + return; + } + + if ( clean_input( 'retranscribe' ) ) { + // Remove to avoid infinite loop. + remove_action( 'edit_attachment', [ $this, 'maybe_transcribe_audio' ] ); + $this->transcribe_audio( $attachment_id ); + } + } + + /** + * Setup fields + */ + public function setup_fields_sections() { + $default_settings = $this->get_default_settings(); + + $this->setup_api_fields( $default_settings['api_key'] ); + + add_settings_field( + 'enable-transcripts', + esc_html__( 'Generate transcripts from audio files', 'classifai' ), + [ $this, 'render_input' ], + $this->get_option_name(), + $this->get_option_name(), + [ + 'label_for' => 'enable_transcripts', + 'input_type' => 'checkbox', + 'default_value' => $default_settings['enable_transcripts'], + 'description' => __( 'Automatically generate transcripts for supported audio files.', 'classifai' ), + ] + ); + + $roles = get_editable_roles() ?? []; + $roles = array_combine( array_keys( $roles ), array_column( $roles, 'name' ) ); + + add_settings_field( + 'roles', + esc_html__( 'Allowed roles', 'classifai' ), + [ $this, 'render_checkbox_group' ], + $this->get_option_name(), + $this->get_option_name(), + [ + 'label_for' => 'roles', + 'options' => $roles, + 'default_values' => $default_settings['roles'], + 'description' => __( 'Choose which roles are allowed to generate transcripts.', 'classifai' ), + ] + ); + } + + /** + * Sanitization for the options being saved. + * + * @param array $settings Array of settings about to be saved. + * + * @return array The sanitized settings to be saved. + */ + public function sanitize_settings( $settings ) { + $new_settings = $this->get_settings(); + $new_settings = array_merge( + $new_settings, + $this->sanitize_api_key_settings( $new_settings, $settings ) + ); + + if ( empty( $settings['enable_transcripts'] ) || 1 !== (int) $settings['enable_transcripts'] ) { + $new_settings['enable_transcripts'] = 'no'; + } else { + $new_settings['enable_transcripts'] = '1'; + } + + if ( isset( $settings['roles'] ) && is_array( $settings['roles'] ) ) { + $new_settings['roles'] = array_map( 'sanitize_text_field', $settings['roles'] ); + } else { + $new_settings['roles'] = array_keys( get_editable_roles() ?? [] ); + } + + return $new_settings; + } + + /** + * Resets settings for the provider. + */ + public function reset_settings() { + update_option( $this->get_option_name(), $this->get_default_settings() ); + } + + /** + * Default settings for Whisper. + * + * @return array + */ + private function get_default_settings() { + return [ + 'authenticated' => false, + 'api_key' => '', + 'enable_transcripts' => false, + 'roles' => array_keys( get_editable_roles() ?? [] ), + ]; + } + + /** + * Provides debug information related to the provider. + * + * @param array|null $settings Settings array. If empty, settings will be retrieved. + * @param boolean $configured Whether the provider is correctly configured. If null, the option will be retrieved. + * @return string|array + */ + public function get_provider_debug_information( $settings = null, $configured = null ) { + if ( is_null( $settings ) ) { + $settings = $this->sanitize_settings( $this->get_settings() ); + } + + $authenticated = 1 === intval( $settings['authenticated'] ?? 0 ); + $enable_transcript = 1 === intval( $settings['enable_transcripts'] ?? 0 ); + + return [ + __( 'Authenticated', 'classifai' ) => $authenticated ? __( 'yes', 'classifai' ) : __( 'no', 'classifai' ), + __( 'Generate transcripts', 'classifai' ) => $enable_transcript ? __( 'yes', 'classifai' ) : __( 'no', 'classifai' ), + __( 'Allowed roles', 'classifai' ) => implode( ', ', $settings['roles'] ?? [] ), + __( 'Latest response', 'classifai' ) => $this->get_formatted_latest_response( 'classifai_openai_whisper_latest_response' ), + ]; + } + +} diff --git a/includes/Classifai/Providers/OpenAI/Whisper/Transcribe.php b/includes/Classifai/Providers/OpenAI/Whisper/Transcribe.php new file mode 100644 index 000000000..0d0c25c10 --- /dev/null +++ b/includes/Classifai/Providers/OpenAI/Whisper/Transcribe.php @@ -0,0 +1,145 @@ +attachment_id = $attachment_id; + $this->settings = $settings; + } + + /** + * Transcribe the audio file. + * + * @return string|WP_Error + */ + public function process() { + if ( ! $this->should_process( $this->attachment_id ) ) { + return new WP_Error( 'process_error', esc_html__( 'Attachment does not meet processing requirements. Ensure the file type and size meet requirements.', 'classifai' ) ); + } + + $request = new APIRequest( $this->settings['api_key'] ?? '' ); + + /** + * Filter the request body before sending to Whisper. + * + * @since x.x.x + * @hook classifai_whisper_transcribe_request_body + * + * @param {array} $body Request body that will be sent to Whisper. + * @param {int} $attachment_id ID of attachment we are transcribing. + * + * @return {array} Request body. + */ + $body = apply_filters( + 'classifai_whisper_transcribe_request_body', + [ + 'file' => get_attached_file( $this->attachment_id ) ?? '', + 'model' => $this->whisper_model, + 'response_format' => 'json', + 'temperature' => 0, + ], + $this->attachment_id + ); + + // Make our API request. + $response = $request->post_form( + $this->get_api_url( $this->path ), + $body + ); + + set_transient( 'classifai_openai_whisper_latest_response', $response, DAY_IN_SECONDS * 30 ); + + // Extract out the text response, if it exists. + if ( ! is_wp_error( $response ) && isset( $response['text'] ) ) { + $response = $this->add_transcription( $response['text'] ); + } + + return $response; + } + + /** + * Add the transcribed text to the attachment. + * + * @param string $text Transcription result. + * @return string|WP_Error + */ + public function add_transcription( string $text = '' ) { + if ( empty( $text ) ) { + return new WP_Error( 'invalid_result', esc_html__( 'The transcription result is invalid.', 'classifai' ) ); + } + + /** + * Filter the text result returned from Whisper API. + * + * @since x.x.x + * @hook classifai_whisper_transcribe_result + * + * @param {string} $text Text extracted from the response. + * @param {int} $attachment_id The attachment ID. + * + * @return {string} + */ + $text = apply_filters( 'classifai_whisper_transcribe_result', $text, $this->attachment_id ); + + $update = wp_update_post( + [ + 'ID' => (int) $this->attachment_id, + 'post_content' => wp_kses_post( $text ), + ], + true + ); + + if ( is_wp_error( $update ) ) { + return $update; + } else { + return $text; + } + } + +} diff --git a/includes/Classifai/Providers/OpenAI/Whisper/Whisper.php b/includes/Classifai/Providers/OpenAI/Whisper/Whisper.php new file mode 100644 index 000000000..fa9c5158d --- /dev/null +++ b/includes/Classifai/Providers/OpenAI/Whisper/Whisper.php @@ -0,0 +1,86 @@ +whisper_url ), $path ); + } + + /** + * Should this attachment be processed. + * + * Ensure the file is a supported format and is under the maximum file size. + * + * @param int $attachment_id Attachment ID to process. + * @return boolean + */ + public function should_process( int $attachment_id ) { + $mime_type = get_post_mime_type( $attachment_id ); + $matched_extensions = explode( '|', array_search( $mime_type, wp_get_mime_types(), true ) ); + $process = false; + + foreach ( $matched_extensions as $ext ) { + if ( in_array( $ext, $this->file_formats, true ) ) { + $process = true; + } + } + + // If we have a proper file format, check the file size. + if ( $process ) { + $filesize = filesize( get_attached_file( $attachment_id ) ); + if ( ! $filesize || $filesize > $this->max_file_size ) { + $process = false; + } + } + + return $process; + } + +} diff --git a/includes/Classifai/Services/LanguageProcessing.php b/includes/Classifai/Services/LanguageProcessing.php index 78068d848..756f2d163 100644 --- a/includes/Classifai/Services/LanguageProcessing.php +++ b/includes/Classifai/Services/LanguageProcessing.php @@ -23,6 +23,7 @@ public function __construct() { [ 'Classifai\Providers\Watson\NLU', 'Classifai\Providers\OpenAI\ChatGPT', + 'Classifai\Providers\OpenAI\Whisper', ] ); } @@ -76,6 +77,24 @@ public function register_endpoints() { 'permission_callback' => [ $this, 'generate_post_excerpt_permissions_check' ], ] ); + + register_rest_route( + 'classifai/v1/openai', + 'generate-transcript/(?P\d+)', + [ + 'methods' => WP_REST_Server::READABLE, + 'callback' => [ $this, 'generate_audio_transcript' ], + 'args' => [ + 'id' => [ + 'required' => true, + 'type' => 'integer', + 'sanitize_callback' => 'absint', + 'description' => esc_html__( 'Attachment ID to generate transcript for.', 'classifai' ), + ], + ], + 'permission_callback' => [ $this, 'generate_audio_transcript_permissions_check' ], + ] + ); } /** @@ -238,4 +257,76 @@ public function generate_post_excerpt_permissions_check( WP_REST_Request $reques return true; } + /** + * Handle request to generate a transcript for given attachment ID. + * + * @param WP_REST_Request $request The full request object. + * @return \WP_REST_Response|WP_Error + */ + public function generate_audio_transcript( WP_REST_Request $request ) { + $attachment_id = $request->get_param( 'id' ); + $provider = ''; + + // Find the right provider class. + foreach ( $this->provider_classes as $provider_class ) { + if ( 'Whisper' === $provider_class->provider_service_name ) { + $provider = $provider_class; + } + } + + // Ensure we have a provider class. Should never happen but :shrug: + if ( ! $provider ) { + return new WP_Error( 'provider_class_required', esc_html__( 'Provider class not found.', 'classifai' ) ); + } + + return rest_ensure_response( $provider->transcribe_audio( $attachment_id ) ); + } + + /** + * Check if a given request has access to generate a transcript. + * + * This check ensures we have a valid user with proper capabilities + * making the request, that we are properly authenticated with OpenAI + * and that transcription is turned on. + * + * @param WP_REST_Request $request Full data about the request. + * @return WP_Error|bool + */ + public function generate_audio_transcript_permissions_check( WP_REST_Request $request ) { + $attachment_id = $request->get_param( 'id' ); + $post_type = get_post_type_object( 'attachment' ); + + // Ensure attachments are allowed in REST endpoints. + if ( empty( $post_type ) || empty( $post_type->show_in_rest ) ) { + return false; + } + + // Ensure we have a logged in user that can upload and change files. + if ( empty( $attachment_id ) || ! current_user_can( 'edit_post', $attachment_id ) || ! current_user_can( 'upload_files' ) ) { + return false; + } + + $settings = \Classifai\get_plugin_settings( 'language_processing', 'Whisper' ); + + // Check if valid authentication is in place. + if ( empty( $settings ) || ( isset( $settings['authenticated'] ) && false === $settings['authenticated'] ) ) { + return new WP_Error( 'auth', esc_html__( 'Please set up valid authentication with OpenAI.', 'classifai' ) ); + } + + // Check if transcription is turned on. + if ( empty( $settings ) || ( isset( $settings['enable_transcripts'] ) && 'no' === $settings['enable_transcripts'] ) ) { + return new WP_Error( 'not_enabled', esc_html__( 'Transcription is not currently enabled.', 'classifai' ) ); + } + + // Check if the current user's role is allowed. + $roles = $settings['roles'] ?? []; + $user_roles = wp_get_current_user()->roles ?? []; + + if ( empty( $roles ) || ! empty( array_diff( $user_roles, $roles ) ) ) { + return false; + } + + return true; + } + } diff --git a/readme.txt b/readme.txt index 7242e7e58..a74e7fbd3 100644 --- a/readme.txt +++ b/readme.txt @@ -23,13 +23,14 @@ Enhance your WordPress content with Artificial Intelligence and Machine Learning * Automatically scan images and PDF files for embedded text and save for use in WordPress * [Smartly crop images](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/computervision/generatethumbnail) around a region of interest identified by Computer Vision * Generate new images on demand to use in-content or as a featured image using [OpenAI's DALL·E](https://platform.openai.com/docs/guides/images) +* Automatically generate transcripts of your audio files using [OpenAI's Whisper](https://platform.openai.com/docs/guides/speech-to-text) * BETA: Recommend content based on overall site traffic via [Azure Personalizer](https://azure.microsoft.com/en-us/services/cognitive-services/personalizer/) (note that we're gathering feedback on this feature and may significantly iterate depending on community input) * Bulk classify content with [WP-CLI](https://wp-cli.org/) **Requirements** * To utilize the NLU Language Processing functionality, you will need an active [IBM Watson](https://cloud.ibm.com/registration) account. -* To utilize the ChatGPT Language Processing functionality or DALL·E Image Processing functionality, you will need an active [OpenAI](https://platform.openai.com/signup) account. +* To utilize the ChatGPT or Whisper Language Processing functionality or DALL·E Image Processing functionality, you will need an active [OpenAI](https://platform.openai.com/signup) account. * To utilize the Computer Vision Image Processing functionality, you will need an active [Microsoft Azure](https://signup.azure.com/signup) account. == Upgrade Notice == diff --git a/src/js/media.js b/src/js/media.js index 0ef8f2752..4833437ed 100644 --- a/src/js/media.js +++ b/src/js/media.js @@ -15,22 +15,25 @@ import { __ } from '@wordpress/i18n'; const imageTagsButton = document.getElementById( 'classifai-rescan-image-tags' ); - const ocrScanButton = document.getElementById( 'classifai-rescan-ocr' ); + const ocrScanButton = document.getElementById('classifai-rescan-ocr'); const smartCropButton = document.getElementById( 'classifai-rescan-smart-crop' ); - const readButton = document.getElementById( 'classifai-rescan-pdf' ); + const readButton = document.getElementById('classifai-rescan-pdf'); + const transcribeButton = document.getElementById( + 'classifai-retranscribe' + ); - if ( altTagsButton ) { - altTagsButton.addEventListener( 'click', ( e ) => - handleClick( { + if (altTagsButton) { + altTagsButton.addEventListener('click', (e) => + handleClick({ button: e.target, endpoint: '/classifai/v1/alt-tags/', - callback: ( resp ) => { + callback: (resp) => { const { enabledAltTextFields } = classifaiMediaVars; - if ( resp ) { - if ( enabledAltTextFields.includes( 'alt' ) ) { + if (resp) { + if (enabledAltTextFields.includes('alt')) { const textField = document.getElementById( 'attachment-details-two-column-alt-text' @@ -39,12 +42,12 @@ import { __ } from '@wordpress/i18n'; 'attachment-details-alt-text' ); - if ( textField ) { + if (textField) { textField.value = resp; } } - if ( enabledAltTextFields.includes( 'caption' ) ) { + if (enabledAltTextFields.includes('caption')) { const textField = document.getElementById( 'attachment-details-two-column-caption' @@ -53,14 +56,12 @@ import { __ } from '@wordpress/i18n'; 'attachment-details-caption' ); - if ( textField ) { + if (textField) { textField.value = resp; } } - if ( - enabledAltTextFields.includes( 'description' ) - ) { + if (enabledAltTextFields.includes('description')) { const textField = document.getElementById( 'attachment-details-two-column-description' @@ -69,32 +70,32 @@ import { __ } from '@wordpress/i18n'; 'attachment-details-description' ); - if ( textField ) { + if (textField) { textField.value = resp; } } } }, - } ) + }) ); } - if ( imageTagsButton ) { - imageTagsButton.addEventListener( 'click', ( e ) => - handleClick( { + if (imageTagsButton) { + imageTagsButton.addEventListener('click', (e) => + handleClick({ button: e.target, endpoint: '/classifai/v1/image-tags/', - } ) + }) ); } - if ( ocrScanButton ) { - ocrScanButton.addEventListener( 'click', ( e ) => - handleClick( { + if (ocrScanButton) { + ocrScanButton.addEventListener('click', (e) => + handleClick({ button: e.target, endpoint: '/classifai/v1/ocr/', - callback: ( resp ) => { - if ( resp ) { + callback: (resp) => { + if (resp) { const textField = document.getElementById( 'attachment-details-two-column-description' @@ -102,31 +103,55 @@ import { __ } from '@wordpress/i18n'; document.getElementById( 'attachment-details-description' ); - if ( textField ) { + if (textField) { textField.value = resp; } } }, - } ) + }) ); } - if ( smartCropButton ) { - smartCropButton.addEventListener( 'click', ( e ) => - handleClick( { + if (smartCropButton) { + smartCropButton.addEventListener('click', (e) => + handleClick({ button: e.target, endpoint: '/classifai/v1/smart-crop/', - } ) + }) ); } - if ( readButton ) { - readButton.addEventListener( 'click', ( e ) => { - const postID = e.target.getAttribute( 'data-id' ); - wp.apiRequest( { path: `/classifai/v1/read-pdf/${ postID }` } ); - e.target.setAttribute( 'disabled', 'disabled' ); - e.target.textContent = __( 'Read API requested!', 'classifai' ); - } ); + if (readButton) { + readButton.addEventListener('click', (e) => { + const postID = e.target.getAttribute('data-id'); + wp.apiRequest({ path: `/classifai/v1/read-pdf/${postID}` }); + e.target.setAttribute('disabled', 'disabled'); + e.target.textContent = __('Read API requested!', 'classifai'); + }); + } + + if (transcribeButton) { + transcribeButton.addEventListener('click', (e) => + handleClick({ + button: e.target, + endpoint: '/classifai/v1/openai/generate-transcript/', + callback: (resp) => { + if (resp) { + const textField = + document.getElementById( + 'attachment-details-two-column-description' + ) ?? + document.getElementById( + 'attachment-details-description' + ); + if (textField) { + textField.value = resp; + } + } + }, + buttonText: __('Re-transcribe', 'classifai'), + }) + ); } }; @@ -134,15 +159,15 @@ import { __ } from '@wordpress/i18n'; * Check the PDF Scanner status and disable button if in progress. */ const checkPdfReadStatus = () => { - const readButton = document.getElementById( 'classifai-rescan-pdf' ); + const readButton = document.getElementById('classifai-rescan-pdf'); - if ( ! readButton ) { + if (!readButton) { return; } - const postId = readButton.getAttribute( 'data-id' ); + const postId = readButton.getAttribute('data-id'); - $.ajax( { + $.ajax({ url: ajaxurl, type: 'POST', data: { @@ -150,38 +175,38 @@ import { __ } from '@wordpress/i18n'; attachment_id: postId, nonce: ClassifAI.ajax_nonce, }, - success: ( resp ) => { - if ( resp?.success ) { - if ( resp?.data?.running ) { - readButton.setAttribute( 'disabled', 'disabled' ); + success: (resp) => { + if (resp?.success) { + if (resp?.data?.running) { + readButton.setAttribute('disabled', 'disabled'); readButton.textContent = __( 'In progress!', 'classifai' ); - } else if ( resp?.data?.read ) { - readButton.textContent = __( 'Rescan', 'classifai' ); + } else if (resp?.data?.read) { + readButton.textContent = __('Rescan', 'classifai'); } } }, - } ); + }); }; - $( document ).ready( function() { - if ( wp.media ) { - wp.media.view.Modal.prototype.on( 'open', function() { - wp.media.frame.on( 'selection:toggle', handleButtonsClick ); - wp.media.frame.on( 'selection:toggle', checkPdfReadStatus ); - } ); + $(document).ready(function () { + if (wp.media) { + wp.media.view.Modal.prototype.on('open', function () { + wp.media.frame.on('selection:toggle', handleButtonsClick); + wp.media.frame.on('selection:toggle', checkPdfReadStatus); + }); } - if ( wp.media.frame ) { - wp.media.frame.on( 'edit:attachment', handleButtonsClick ); - wp.media.frame.on( 'edit:attachment', checkPdfReadStatus ); + if (wp.media.frame) { + wp.media.frame.on('edit:attachment', handleButtonsClick); + wp.media.frame.on('edit:attachment', checkPdfReadStatus); } // For new uploaded media. - if ( wp.Uploader && wp.Uploader.queue ) { - wp.Uploader.queue.on( 'reset', handleButtonsClick ); + if (wp.Uploader && wp.Uploader.queue) { + wp.Uploader.queue.on('reset', handleButtonsClick); } - } ); -} )( jQuery ); + }); +})(jQuery); diff --git a/tests/cypress/fixtures/audio.mp3 b/tests/cypress/fixtures/audio.mp3 new file mode 100644 index 000000000..6fc15969d Binary files /dev/null and b/tests/cypress/fixtures/audio.mp3 differ diff --git a/tests/cypress/integration/language-processing.test.js b/tests/cypress/integration/language-processing.test.js index 28d8df6e5..d4bbb8291 100644 --- a/tests/cypress/integration/language-processing.test.js +++ b/tests/cypress/integration/language-processing.test.js @@ -1,6 +1,6 @@ /* eslint jest/expect-expect: 0 */ -import { getChatGPTData } from '../plugins/functions'; +import { getChatGPTData, getWhisperData } from '../plugins/functions'; describe('Language processing Tests', () => { before(() => { @@ -152,7 +152,7 @@ describe('Language processing Tests', () => { cy.verifyPostTaxonomyTerms('tags', threshold / 100); }); - it( 'Can save OpenAI "Language Processing" settings', () => { + it( 'Can save OpenAI ChatGPT "Language Processing" settings', () => { cy.visit( '/wp-admin/tools.php?page=classifai&tab=language_processing&provider=openai_chatgpt' ); cy.get( '#api_key' ).clear().type( 'password' ); @@ -290,4 +290,97 @@ describe('Language processing Tests', () => { .should( 'not.exist' ); } ); } ); + + it('Can save OpenAI Whisper "Language Processing" settings', () => { + cy.visit( + '/wp-admin/tools.php?page=classifai&tab=language_processing&provider=openai_whisper' + ); + + cy.get('#api_key').clear().type('password'); + + cy.get('#enable_transcripts').check(); + cy.get('#openai_whisper_roles_administrator').check(); + cy.get('#submit').click(); + }); + + let audioEditLink = ''; + let mediaModalLink = ''; + + it('Can see OpenAI Whisper language processing actions on edit media page and verify generated data.', () => { + cy.visit('/wp-admin/media-new.php'); + cy.get('#plupload-upload-ui').should('exist'); + cy.get('#plupload-upload-ui input[type=file]').attachFile('audio.mp3'); + + cy.get('#media-items .media-item a.edit-attachment').should('exist'); + cy.get('#media-items .media-item a.edit-attachment') + .invoke('attr', 'href') + .then((editLink) => { + audioEditLink = editLink; + cy.visit(editLink); + }); + + // Verify metabox has processing actions. + cy.get('.postbox-header h2, #attachment_meta_box h2') + .first() + .contains('ClassifAI Audio Processing'); + cy.get('.misc-publishing-actions label[for=retranscribe]').contains( + 'Re-transcribe' + ); + + // Verify generated data. + cy.get('#attachment_content').should('have.value', getWhisperData()); + }); + + it('Can see OpenAI Whisper language processing actions on media model', () => { + const audioId = audioEditLink.split('post=')[1]?.split('&')[0]; + mediaModalLink = `wp-admin/upload.php?item=${audioId}`; + cy.visit(mediaModalLink); + cy.get('.media-modal').should('exist'); + + // Verify language processing actions. + cy.get('#classifai-retranscribe').contains('Re-transcribe'); + }); + + it('Can disable OpenAI Whisper language processing features', () => { + cy.visit( + '/wp-admin/tools.php?page=classifai&tab=language_processing&provider=openai_whisper' + ); + + // Disable features + cy.get('#enable_transcripts').uncheck(); + cy.get('#submit').click(); + + // Verify features are not present in attachment metabox. + cy.visit(audioEditLink); + cy.get('.misc-publishing-actions label[for=retranscribe]').should( + 'not.exist' + ); + + // Verify features are not present in media modal. + cy.visit(mediaModalLink); + cy.get('.media-modal').should('exist'); + cy.get('#classifai-retranscribe').should('not.exist'); + }); + + it('Can disable OpenAI Whisper language processing features by role', () => { + cy.visit( + '/wp-admin/tools.php?page=classifai&tab=language_processing&provider=openai_whisper' + ); + + // Disable admin role + cy.get('#enable_transcripts').check(); + cy.get('#openai_whisper_roles_administrator').uncheck(); + cy.get('#submit').click(); + + // Verify features are not present in attachment metabox. + cy.visit(audioEditLink); + cy.get('.misc-publishing-actions label[for=retranscribe]').should( + 'not.exist' + ); + + // Verify features are not present in media modal. + cy.visit(mediaModalLink); + cy.get('.media-modal').should('exist'); + cy.get('#classifai-retranscribe').should('not.exist'); + }); }); diff --git a/tests/cypress/plugins/functions.js b/tests/cypress/plugins/functions.js index af67bce58..919ba9800 100644 --- a/tests/cypress/plugins/functions.js +++ b/tests/cypress/plugins/functions.js @@ -2,6 +2,7 @@ import * as nluData from '../../test-plugin/nlu.json'; import * as chatgptData from '../../test-plugin/chatgpt.json'; import * as dalleData from '../../test-plugin/dalle.json'; import * as ocrData from '../../test-plugin/ocr.json'; +import * as whisperData from '../../test-plugin/whisper.json'; import * as imageData from '../../test-plugin/image_analyze.json'; import * as pdfData from '../../test-plugin/pdf.json'; @@ -52,6 +53,15 @@ export const getDalleData = () => { return dalleData.data; }; +/** + * Get data from test Whisper json file. + * + * @return {string[]} Whisper data. + */ +export const getWhisperData = () => { + return whisperData.text; +}; + /** * Get Image OCR data * diff --git a/tests/test-plugin/e2e-test-plugin.php b/tests/test-plugin/e2e-test-plugin.php index a9c917cb4..fb99ae1c7 100644 --- a/tests/test-plugin/e2e-test-plugin.php +++ b/tests/test-plugin/e2e-test-plugin.php @@ -23,6 +23,8 @@ function classifai_test_mock_http_requests( $preempt, $parsed_args, $url ) { $response = file_get_contents( __DIR__ . '/chatgpt.json' ); } elseif ( strpos( $url, 'https://api.openai.com/v1/chat/completions' ) !== false ) { $response = file_get_contents( __DIR__ . '/chatgpt.json' ); + } elseif ( strpos( $url, 'https://api.openai.com/v1/audio/transcriptions' ) !== false ) { + $response = file_get_contents( __DIR__ . '/whisper.json' ); } elseif ( strpos( $url, 'https://api.openai.com/v1/images/generations' ) !== false ) { $response = file_get_contents( __DIR__ . '/dalle.json' ); } elseif ( strpos( $url, 'http://e2e-test-image-processing.test/vision/v3.0/analyze' ) !== false ) { diff --git a/tests/test-plugin/whisper.json b/tests/test-plugin/whisper.json new file mode 100644 index 000000000..c1b94b7c3 --- /dev/null +++ b/tests/test-plugin/whisper.json @@ -0,0 +1,3 @@ +{ + "text": "Hello World!" +}