Merge branch 'master' of github.com:edenai/edenai-apis

.circleci/config.yml

@@ -0,0 +1,103 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
+jobs:
+  test:
+    docker:
+      - image: cimg/python:3.8
+    # Add steps to the job
+    # See: https://circleci.com/docs/2.0/configuration-reference/#steps
+    steps:
+      - checkout
+      - run:
+          name: Setup config files and folders
+          command: |
+            # Insall aws cli
+            pip install awscli --upgrade
+
+            # Copy settings file
+            aws s3 cp $S3PATH edenai_apis/api_keys --recursive
+
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install ffmpeg -y
+            pip install -r requirements.txt
+
+      - run:
+          name: Test
+          command: |
+            cd edenai_apis
+            mkdir test-results
+            export TEST_SCOPE="CICD" &&  pytest -vvv -n auto --maxprocesses=8 --dist loadgroup  --junitxml=test-results/junit.xml --cov
+      - store_test_results:
+          path: edenai_apis/test-results/junit.xml
+      - store_artifacts:
+          path: edenai_apis/htmlcov
+
+  check_providers:
+    docker:
+      - image: cimg/python:3.8
+    # Add steps to the job
+    # See: https://circleci.com/docs/2.0/configuration-reference/#steps
+    parameters:
+      interval:
+        type: string
+        default: nightly
+
+    steps:
+      - checkout
+      - run:
+          name: Setup config files and folders
+          command: |
+            sudo apt-get update
+            sudo apt-get install build-essential libssl-dev libasound2 wget
+            
+            wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+            sudo dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+
+            # Insall aws cli
+            pip install awscli --upgrade
+            # Copy settings file
+            aws s3 cp $S3PATH edenai_apis/api_keys --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install ffmpeg -y
+            pip install -e .
+            
+      - run:
+          name: check working providers
+          command: |
+            python edenai_apis/scripts/check_not_working_providers.py << parameters.interval >>
+
+
+# Invoke jobs via workflows
+# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
+workflows:
+  test-workflow:
+    jobs:
+      - test:
+          context:
+            - Edenai-back
+          filters:
+            branches:
+              only:
+                - master
+                - circleci
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 6 * * 1-5"
+          filters:
+            branches:
+              only:
+                - master
+                - release
+    jobs:
+     - check_providers

AVAILABLES_FEATURES_AND_PROVIDERS.md

@@ -223,6 +223,7 @@
 | **speech_to_text_async** | amazon |
 | | assembly |
 | | deepgram |
+| | faker |
 | | gladia |
 | | google |
 | | ibm |
@@ -484,6 +485,13 @@
 | **image** | face_compare |
 | | face_recognition |
 
+</details>
+<details><summary>faker</summary>
+
+| Features | Subfeatures |
+|----------|-------------|
+| **audio** | speech_to_text_async |
+
 </details>
 <details><summary>gladia</summary>
 

edenai_apis/apis/__init__.py

@@ -16,6 +16,7 @@
 from .elevenlabs import ElevenlabsApi
 from .emvista import EmvistaApi
 from .facepp import FaceppApi
+from .faker import FakerApi
 from .gladia import GladiaApi
 from .google import GoogleApi
 from .hireability import HireabilityApi

edenai_apis/apis/faker/__init__.py

@@ -0,0 +1 @@
+from .faker_api import FakerApi

edenai_apis/apis/faker/faker_api.py

@@ -0,0 +1,71 @@
+"""
+    Fake provider class used for tests
+"""
+from io import BufferedReader
+from random import randint
+from time import sleep
+from typing import Dict, List, Optional
+
+from edenai_apis.features import AudioInterface, ProviderInterface
+from edenai_apis.features.audio import SpeechToTextAsyncDataClass, SpeechDiarization
+from edenai_apis.utils.types import (
+    AsyncBaseResponseType,
+    AsyncLaunchJobResponseType,
+    AsyncPendingResponseType,
+    AsyncResponseType,
+)
+
+class FakerApi(ProviderInterface, AudioInterface):
+    provider_name = "faker"
+
+    def __init__(self, api_keys: Dict = {}) -> None:
+        super().__init__()
+
+    def audio__speech_to_text_async__launch_job(
+        self, 
+        file: str,
+        language: str, 
+        speakers: int, 
+        profanity_filter: bool, 
+        vocabulary: list,
+        audio_attributes: tuple,
+        file_url: str = "",
+        provider_params: dict = dict(),
+    ) -> AsyncLaunchJobResponseType:
+        sleep(randint(1, 3))
+        return AsyncLaunchJobResponseType(provider_job_id="SomeFakeID")
+
+    def audio__speech_to_text_async__get_job_result(
+        self, provider_job_id: str
+    ) -> AsyncBaseResponseType[SpeechToTextAsyncDataClass]:
+        sleep(randint(1, 3))
+
+        standardized_response = SpeechToTextAsyncDataClass(
+                text="empty",
+                diarization=SpeechDiarization(
+                    total_speakers = 1
+                )
+            )
+        provider_correct_response = AsyncResponseType[SpeechToTextAsyncDataClass](
+                original_response={},
+                standardized_response=standardized_response,
+                provider_job_id=provider_job_id,
+            )
+
+        chance_to_stop = randint(2, 1000)
+
+        if provider_job_id == 'FINISHED':
+            return provider_correct_response
+        if provider_job_id == 'ERROR':
+            raise Exception("error")
+        if provider_job_id == "pending":
+            return AsyncPendingResponseType[SpeechToTextAsyncDataClass](
+                provider_job_id=provider_job_id
+            )
+        if chance_to_stop < 250:
+            return provider_correct_response
+        if chance_to_stop > 994:
+            raise Exception("error")
+        return AsyncPendingResponseType[SpeechToTextAsyncDataClass](
+                provider_job_id=provider_job_id
+        )

edenai_apis/apis/faker/info.json

@@ -0,0 +1,53 @@
+{
+    "audio": {
+        "speech_to_text_async": {
+            "constraints": {
+                "languages": [
+                    "ar",
+                    "bg",
+                    "ca",
+                    "hr",
+                    "cs",
+                    "da",
+                    "nl",
+                    "en",
+                    "fa",
+                    "fi",
+                    "fr",
+                    "de",
+                    "el",
+                    "he",
+                    "hi",
+                    "hu",
+                    "id",
+                    "it",
+                    "ja",
+                    "ko",
+                    "lv",
+                    "lt",
+                    "ms",
+                    "cmn",
+                    "no",
+                    "pl",
+                    "pt",
+                    "ro",
+                    "ru",
+                    "sk",
+                    "sl",
+                    "es",
+                    "sv",
+                    "ta",
+                    "te",
+                    "tr"
+                ],
+                "file_extensions": [
+                    "flac",
+                    "wav",
+                    "mp3"
+                ],
+                "allow_null_language": true
+            },
+            "version": "v1"
+        }
+    }
+}

edenai_apis/apis/faker/outputs/audio/speech_to_text_async_output.json

@@ -0,0 +1,13 @@
+{
+  "status": "succeeded",
+  "provider_job_id": "SomeFakeID",
+  "original_response": {},
+  "standardized_response": {
+    "text": "empty",
+    "diarization": {
+      "total_speakers": 1,
+      "entries": [],
+      "error_message": null
+    }
+  }
+}

edenai_apis/scripts/check_not_working_providers.py

@@ -0,0 +1,61 @@
+import sys
+from pprint import pprint
+
+import requests
+from edenai_apis.interface import compute_output
+from edenai_apis.loaders.data_loader import FeatureDataEnum
+from edenai_apis.loaders.loaders import load_feature
+
+HOURLY = "hourly"
+
+if __name__ == "__main__":
+    interval = sys.argv[1]
+    not_working = []
+    query_is_working = "?is_working=False" if interval == HOURLY else ""
+    provider_subfeatures = requests.get(
+        url=f"https://api.edenai.run/v2/info/provider_subfeatures{query_is_working}"
+    ).json()
+    all_providers = [
+        (
+            provider["provider"]["name"],
+            provider["feature"]["name"],
+            provider["subfeature"]["name"],
+            provider.get("phase", ""),
+        )
+        for provider in provider_subfeatures
+    ]
+    for provider, feature, subfeature, phase in all_providers:
+        if phase == "create_project":
+            continue
+        try:
+            arguments = load_feature(
+                FeatureDataEnum.SAMPLES_ARGS,
+                feature=feature,
+                subfeature=subfeature,
+                phase=phase,
+            )
+        except NotImplementedError:
+            continue
+        try:
+            res = compute_output(
+                provider_name=provider,
+                feature=feature,
+                subfeature=subfeature,
+                args=arguments,
+                phase=phase,
+            )
+            if res["status"] == "fail":
+                raise Exception(res["error"])
+
+        except Exception as exc:
+            print(provider, feature, subfeature)
+            print(exc)
+            not_working.append((provider, feature, subfeature, exc))
+
+    print("=================================")
+    print("NOT WORKING PROVIDERS WITH ERRORS")
+    pprint(not_working)
+    print("=================================")
+
+    if not_working:
+        raise Exception