diff --git a/clowdapp.yaml b/clowdapp.yaml index 31452a2e..b199f0d6 100644 --- a/clowdapp.yaml +++ b/clowdapp.yaml @@ -2,85 +2,21 @@ apiVersion: v1 kind: Template metadata: - name: ros-backend + name: ros-backend-pcp-zeroconf objects: - apiVersion: cloud.redhat.com/v1alpha1 kind: ClowdApp metadata: - name: ros-backend + name: ros-backend-pcp-zeroconf spec: envName: ${ENV_NAME} - featureFlags: true - dependencies: - - host-inventory - - rbac - - engine deployments: - - name: api - replicas: ${{API_REPLICA_COUNT}} - webServices: - public: - enabled: true - apiPath: ros - podSpec: - image: ${IMAGE}:${IMAGE_TAG} - command: ["sh"] - args: ["-c", "export FLASK_APP=manage.py && poetry run flask db upgrade && poetry run flask seed && poetry run gunicorn ros.api.main:app"] - resources: - requests: - cpu: ${CPU_REQUEST_ROSRHEL} - memory: ${MEMORY_REQUEST_ROSRHEL} - limits: - cpu: ${CPU_LIMIT_ROSRHEL} - memory: ${MEMORY_LIMIT_ROSRHEL} - livenessProbe: - failureThreshold: 3 - httpGet: - path: /api/ros/v1/status - port: 8000 - scheme: HTTP - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 3 - readinessProbe: - httpGet: - path: /api/ros/v1/status - port: 8000 - scheme: HTTP - initialDelaySeconds: 3 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 - env: - - name: PATH_PREFIX - value: /api - - name: CLOWDER_ENABLED - value: ${CLOWDER_ENABLED} - - name: ENABLE_RBAC - value: "${ENABLE_RBAC}" - - name: DB_POOL_SIZE - value: ${DB_POOL_SIZE} - - name: DB_MAX_OVERFLOW - value: ${DB_MAX_OVERFLOW} - - name: UNLEASH_URL - value: ${UNLEASH_URL} - - name: UNLEASH_TOKEN - valueFrom: - secretKeyRef: - name: ${UNLEASH_SECRET_NAME} - key: CLIENT_ACCESS_TOKEN - optional: true - - name: BYPASS_UNLEASH - value: ${BYPASS_UNLEASH} - - - name: inventory-events-processor - replicas: ${{INVENTORY_PROCESSOR_REPLICA_COUNT}} + - name: pcp-generator + replicas: ${{PCP_GENERATOR_REPLICA_COUNT}} podSpec: image: ${IMAGE}:${IMAGE_TAG} command: ["bash"] - args: ["-c", "python -m ros.processor.inventory_events_consumer"] + args: ["-c", "python -m ros.processor.pcp_generator"] resources: requests: cpu: ${CPU_REQUEST_ROSRHEL} @@ -111,198 +47,22 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} - - name: DB_POOL_SIZE - value: ${DB_POOL_SIZE} - - name: DB_MAX_OVERFLOW - value: ${DB_MAX_OVERFLOW} - - name: UNLEASH_URL - value: ${UNLEASH_URL} - - name: UNLEASH_TOKEN - valueFrom: - secretKeyRef: - name: ${UNLEASH_SECRET_NAME} - key: CLIENT_ACCESS_TOKEN - optional: true - - name: BYPASS_UNLEASH - value: ${BYPASS_UNLEASH} - - name: engine-result-processor - replicas: ${{ENGINE_PROCESSOR_REPLICA_COUNT}} - podSpec: - image: ${IMAGE}:${IMAGE_TAG} - command: ["bash"] - args: ["-c", "python -m ros.processor.insights_engine_consumer"] - resources: - requests: - cpu: ${CPU_REQUEST_ROSRHEL} - memory: ${MEMORY_REQUEST_ROSRHEL} - limits: - cpu: ${CPU_LIMIT_ROSRHEL} - memory: ${MEMORY_LIMIT_ROSRHEL} - livenessProbe: - failureThreshold: 3 - httpGet: - path: / - port: 9000 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 3 - readinessProbe: - httpGet: - path: / - port: 9000 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 - env: - - name: CLOWDER_ENABLED - value: ${CLOWDER_ENABLED} - - name: DB_POOL_SIZE - value: ${DB_POOL_SIZE} - - name: DB_MAX_OVERFLOW - value: ${DB_MAX_OVERFLOW} - - name: UNLEASH_URL - value: ${UNLEASH_URL} - - name: UNLEASH_TOKEN - valueFrom: - secretKeyRef: - name: ${UNLEASH_SECRET_NAME} - key: CLIENT_ACCESS_TOKEN - optional: true - - name: BYPASS_UNLEASH - value: ${BYPASS_UNLEASH} - - - name: garbage-collector-processor - replicas: ${{GARBAGE_COLLECTOR_REPLICA_COUNT}} - podSpec: - image: ${IMAGE}:${IMAGE_TAG} - command: ["bash"] - args: ["-c", "python -m ros.processor.garbage_collector"] - resources: - requests: - cpu: ${CPU_REQUEST_ROSRHEL} - memory: ${MEMORY_REQUEST_ROSRHEL} - limits: - cpu: ${CPU_LIMIT_ROSRHEL} - memory: ${MEMORY_LIMIT_ROSRHEL} - livenessProbe: - failureThreshold: 3 - httpGet: - path: / - port: 9000 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 3 - readinessProbe: - httpGet: - path: / - port: 9000 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 - env: - - name: CLOWDER_ENABLED - value: ${CLOWDER_ENABLED} - - name: GARBAGE_COLLECTION_INTERVAL - value: ${GARBAGE_COLLECTION_INTERVAL} - - name: DAYS_UNTIL_STALE - value: ${DAYS_UNTIL_STALE} - - name: DB_POOL_SIZE - value: ${DB_POOL_SIZE} - - name: DB_MAX_OVERFLOW - value: ${DB_MAX_OVERFLOW} - - name: UNLEASH_URL - value: ${UNLEASH_URL} - - name: UNLEASH_TOKEN - valueFrom: - secretKeyRef: - name: ${UNLEASH_SECRET_NAME} - key: CLIENT_ACCESS_TOKEN - optional: true - - name: BYPASS_UNLEASH - value: ${BYPASS_UNLEASH} - - database: - name: ros - version: 13 - inMemoryDb: true kafkaTopics: - topicName: platform.inventory.events partitions: 1 - - topicName: platform.engine.results - partitions: 1 - - topicName: platform.notifications.ingress - partitions: 1 - testing: - iqePlugin: ros - - jobs: - - name: org-id-populator - podSpec: - image: quay.io/cloudservices/tenant-utils:latest - command: - - ./org-id-column-populator - - -C - - -a - - account - - -o - - org_id - - -t - - rh_accounts - - --ean-translator-addr - - http://${TENANT_TRANSLATOR_HOST}:${TENANT_TRANSLATOR_PORT} - - --prometheus-push-addr - - ${PROMETHEUS_PUSHGATEWAY} - env: - - name: TENANT_TRANSLATOR_HOST - value: ${TENANT_TRANSLATOR_HOST} - - name: TENANT_TRANSLATOR_PORT - value: ${TENANT_TRANSLATOR_PORT} - - name: LOG_FORMAT - value: ${POPULATOR_LOG_FORMAT} - - name: LOG_BATCH_FREQUENCY - value: '1s' - resources: - limits: - cpu: 300m - memory: 1Gi - requests: - cpu: 50m - memory: 512Mi parameters: -- description: Enable rbac needs to be "True" or "False" - name: ENABLE_RBAC - value: "True" - description : ClowdEnvironment name name: ENV_NAME required: true -- description: Replica count for api pod - name: API_REPLICA_COUNT - value: "1" -- description: Replica count for inventory processor pod - name: INVENTORY_PROCESSOR_REPLICA_COUNT - value: "1" -- description: Replica count for engine processor pod - name: ENGINE_PROCESSOR_REPLICA_COUNT - value: "1" -- description: Replica count for garbage collector pod - name: GARBAGE_COLLECTOR_REPLICA_COUNT +- description: Replica count for pcp generator + name: PCP_GENERATOR_REPLICA_COUNT value: "1" - description: Image NAME name: IMAGE required: true - value: quay.io/redhat-services-prod/insights-management-tenant/insights-resource-optimization/ros-backend + value: quay.io/repository/cloudservices/ros-backend - description: Image tag name: IMAGE_TAG required: true @@ -329,34 +89,3 @@ parameters: - description: Is clowder enabled name: CLOWDER_ENABLED value: "True" -- description: The interval at which garbage collection should run (in seconds) - name: GARBAGE_COLLECTION_INTERVAL - value: "86400" -- description: Number of days after which data is considered to be outdated - name: DAYS_UNTIL_STALE - value: "45" -- description: Host for the EAN to OrgId translator. - name: TENANT_TRANSLATOR_HOST - required: true - value: 'gateway.3scale-dev.svc.cluster.local' -- description: Port for the EAN to OrgId translator. - name: TENANT_TRANSLATOR_PORT - value: '8892' -- name: PROMETHEUS_PUSHGATEWAY - value: "localhost" -- name: POPULATOR_LOG_FORMAT - value: cloudwatch -- name: DB_POOL_SIZE - value: "20" -- name: DB_MAX_OVERFLOW - value: "20" - -# Feature flags -- description: Unleash secret name - name: UNLEASH_SECRET_NAME - value: bypass -- description: Unleash API url - name: UNLEASH_URL -- description: disable Unleash (feature flags), defaulting to fallback values - name: BYPASS_UNLEASH - value: 'false' diff --git a/ros/lib/config.py b/ros/lib/config.py index 93bfdc1e..f8dac2b3 100644 --- a/ros/lib/config.py +++ b/ros/lib/config.py @@ -141,6 +141,7 @@ def build_endpoint_url(ep): DAYS_UNTIL_STALE = int(os.getenv("DAYS_UNTIL_STALE", '45')) CW_LOGGING_FORMAT = '%(asctime)s - %(levelname)s - %(funcName)s - %(message)s' ROS_PROCESSOR_PORT = int(os.getenv("ROS_PROCESSOR_PORT", "8000")) +ROS_PCP_PROCESSOR_PORT = int(os.getenv("ROS_PCP_PROCESSOR_PORT", "8000")) ROS_API_PORT = int(os.getenv("ROS_API_PORT", "8000")) # Timeout in seconds to set against keys of deleted systems in a cache CACHE_TIMEOUT_FOR_DELETED_SYSTEM = int( diff --git a/ros/processor/__init__.py b/ros/processor/__init__.py index e69de29b..a4c387dd 100644 --- a/ros/processor/__init__.py +++ b/ros/processor/__init__.py @@ -0,0 +1,9 @@ +from flask import Flask + + +def create_app(): + app = Flask(__name__) + return app + + +app = create_app() diff --git a/ros/processor/main.py b/ros/processor/main.py deleted file mode 100644 index a95c68cf..00000000 --- a/ros/processor/main.py +++ /dev/null @@ -1,53 +0,0 @@ -from http.server import HTTPServer -from ros.lib.utils import PROCESSOR_INSTANCES, MonitoringHandler -from ros.lib.cw_logging import commence_cw_log_streaming -from ros.processor.inventory_events_consumer import InventoryEventsConsumer -from ros.processor.insights_engine_consumer import InsightsEngineConsumer -from ros.processor.garbage_collector import GarbageCollector -from prometheus_client import start_http_server -import threading -from ros.lib.config import METRICS_PORT, ROS_PROCESSOR_PORT - - -def process_engine_results(): - processor = InsightsEngineConsumer() - processor.processor_name = 'process-engine-results' - PROCESSOR_INSTANCES.append(processor) - processor.run() - - -def events_processor(): - processor = InventoryEventsConsumer() - processor.processor_name = 'events-processor' - PROCESSOR_INSTANCES.append(processor) - processor.run() - - -def garbage_collector(): - collector = GarbageCollector() - collector.processor_name = 'garbage-collector' - PROCESSOR_INSTANCES.append(collector) - collector.run() - - -def thread_monitor(): - server = HTTPServer(('', ROS_PROCESSOR_PORT), MonitoringHandler) - server.serve_forever() - - -if __name__ == "__main__": - commence_cw_log_streaming('ros-processor') - # Start processing in 2 different threads - engine_results = threading.Thread(name='process-engine-results', target=process_engine_results) - events = threading.Thread(name='events-processor', target=events_processor) - collector = threading.Thread(name='garbage-collector', target=garbage_collector) - threadmonitor = threading.Thread(name='thread-monitor', target=thread_monitor) - events.start() - engine_results.start() - collector.start() - threadmonitor.start() - start_http_server(int(METRICS_PORT)) - # Wait for threads to finish - events.join() - engine_results.join() - collector.join() diff --git a/ros/processor/pcp_generator.py b/ros/processor/pcp_generator.py new file mode 100644 index 00000000..d9eb6919 --- /dev/null +++ b/ros/processor/pcp_generator.py @@ -0,0 +1,16 @@ +from ros.processor import app +from ros.lib.config import get_logger +from ros.lib.config import ROS_PCP_PROCESSOR_PORT + + +logging = get_logger(__name__) + + +class PCPGenerator: + def __init__(self): + pass + + +if __name__ == "__main__": + logging.info("Flask server running on port %s", ROS_PCP_PROCESSOR_PORT) + app.run(host='0.0.0.0', port=ROS_PCP_PROCESSOR_PORT)