From fcf7b350bceb25d81226832bd6250252ca96f4d7 Mon Sep 17 00:00:00 2001 From: Carlos Camacho Date: Tue, 15 Dec 2020 16:43:49 +0100 Subject: [PATCH] feat: add simple agent skeleton This patch adds a simple skeleton code for the agent --- .github/workflows/release.yml | 14 ++++ agent/MANIFEST.in | 7 ++ agent/kubeinit/__init__.py | 21 +++++ agent/kubeinit/cli.py | 147 ++++++++++++++++++++++++++++++++++ agent/kubeinit/const.py | 26 ++++++ agent/kubeinit/get_banner.py | 42 ++++++++++ agent/kubeinit/lib.py | 41 ++++++++++ agent/kubeinit/logger.py | 64 +++++++++++++++ agent/requirements.txt | 14 ++++ agent/setup.py | 79 ++++++++++++++++++ agent/tests/test_sample.py | 31 +++++++ kubeinit/galaxy.yml | 2 +- tools/check_version.sh | 12 +++ tools/pypi_publish.sh | 82 +++++++++++++++++++ tox.ini | 5 ++ 15 files changed, 586 insertions(+), 1 deletion(-) create mode 100644 agent/MANIFEST.in create mode 100644 agent/kubeinit/__init__.py create mode 100644 agent/kubeinit/cli.py create mode 100644 agent/kubeinit/const.py create mode 100644 agent/kubeinit/get_banner.py create mode 100644 agent/kubeinit/lib.py create mode 100644 agent/kubeinit/logger.py create mode 100644 agent/requirements.txt create mode 100644 agent/setup.py create mode 100644 agent/tests/test_sample.py create mode 100755 tools/check_version.sh create mode 100755 tools/pypi_publish.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29c5c06c7..e42b3b1db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,6 +38,9 @@ jobs: sudo python3 -m pip install --upgrade virtualenv sudo python3 -m pip install --upgrade setuptools sudo python3 -m pip install -r ./test-requirements.txt + sudo python3 -m pip install tox shyaml + sudo python3 -m pip install twine + - name: Create collection build and publish run: | chmod +x ./tools/release.sh @@ -46,6 +49,17 @@ jobs: QUAY_KEY=${{ secrets.QUAY_KEY }} \ GALAXY_KEY=${{ secrets.GALAXY_KEY }} \ ./tools/release.sh + - name: Create pypi package and publish + run: | + ./tools/pypi_publish.sh -k ${{ secrets.PYPI_TOKEN }} + - name: Install latest Kubeinit agent version build locally if we published a new version + run: | + cd ./agent + if [ -d "./dist" ] + then + python3 -m pip install --force dist/* + kubeinit -v + fi release_docs: needs: release_sw diff --git a/agent/MANIFEST.in b/agent/MANIFEST.in new file mode 100644 index 000000000..75e3c6018 --- /dev/null +++ b/agent/MANIFEST.in @@ -0,0 +1,7 @@ +include *.py +include *.yaml +include requirements.txt +recursive-include kubeinit * +global-exclude *pyc +global-exclude .project +global-exclude .pydevproject diff --git a/agent/kubeinit/__init__.py b/agent/kubeinit/__init__.py new file mode 100644 index 000000000..471fbc309 --- /dev/null +++ b/agent/kubeinit/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + +from pkg_resources import get_distribution + +__version__ = get_distribution('kubeinit').version diff --git a/agent/kubeinit/cli.py b/agent/kubeinit/cli.py new file mode 100644 index 000000000..dc80cb750 --- /dev/null +++ b/agent/kubeinit/cli.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.com/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + + +import logging +import logging.handlers +import os +import threading +from argparse import ArgumentParser + +from kubeinit import __version__ +from kubeinit.const import KUBEINIT_LOG_FILE +from kubeinit.get_banner import get_banner +from kubeinit.lib import connect_to_aas, connect_to_aas2 + + +kubeinit_version = __version__ + +t1_stop = threading.Event() +t2_stop = threading.Event() + +handler = logging.handlers.WatchedFileHandler( + KUBEINIT_LOG_FILE) +formatter = logging.Formatter(logging.BASIC_FORMAT) +handler.setFormatter(formatter) +root = logging.getLogger() +root.setLevel(os.environ.get("LOGLEVEL", "INFO")) +root.addHandler(handler) + + +def main(): + """ + Application's entry point. + + Here, application's settings are read from the command line, + environment variables and CRD. Then, retrieving and processing + of Kubernetes events are initiated. + """ + parser = ArgumentParser( + description='Kubeinit - CLI', + prog='kubeinit' + ) + + parser.add_argument( + '-v', + '--version', + action='version', + version='%(prog)s ' + kubeinit_version + ) + + parser.add_argument( + '-b', + '--banner', + action='store_true', + help='Print Kubeinit.com banner' + ) + + subparsers = parser.add_subparsers(title="Kubeinit subcommands", + dest="command", + help=("These are the options " + "supported: \n" + "The listen option will " + "watch for CRD events. " + "The run option will " + "execute the Kubeinit " + "actions against the cluster.")) + + parser_connect = subparsers.add_parser('connect', help=("CLI options to run the " + "Kubeinit actions.")) + + parser_connect.add_argument( + '-k', + '--key', + default="", + type=str, + help=("The connection key:" + "--extra-vars thisisakey" + "Defaults to: empty")) + + subparsers.add_parser('status', help=("Show the connections status.")) + + parser_show = subparsers.add_parser('show', + help=("Get info from a connection.")) + + parser_show.add_argument( + "connection", + nargs='?', + default='', + type=str, + help=("Specify the connection to fetch details")) + + args = parser.parse_args() + + # print("Kubeinit called with the folowing parameters") + # print(parser.parse_args()) + + if args.banner: + print(get_banner()) + exit() + + try: + if (args.command == 'connect'): + print("We will watch for objects to process") + try: + print(args.key) + t1 = threading.Thread(target=connect_to_aas, + args=(t1_stop,)) + t1.start() + t2 = threading.Thread(target=connect_to_aas2, + args=(t2_stop,)) + t2.start() + except Exception as err: + print("Error: unable to start thread: " + err) + elif (args.command == 'status'): + print(u"\U0001F4DA" + " Listing the deployed actions" + " from the cluster.") + print(u"\U0001F4A1" + " For further information use:" + " kubeinit get [--debug]") + exit() + elif (args.command == 'show'): + print(u"\U0001F4E4" + " Getting the details from" + " an specific action.") + print(args.connection) + exit() + + except KeyboardInterrupt: + pass + except Exception as err: + raise RuntimeError('There is something wrong...' + err) + + while not t2_stop.is_set() or not t1_stop.is_set(): + pass diff --git a/agent/kubeinit/const.py b/agent/kubeinit/const.py new file mode 100644 index 000000000..6e7b2cfed --- /dev/null +++ b/agent/kubeinit/const.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + +import os +import tempfile + +# Main logging file +KUBEINIT_LOG_FOLDER = tempfile.gettempdir() +KUBEINIT_LOG_FILE = os.path.join(KUBEINIT_LOG_FOLDER, '/tmp/kubeinit.log') + +# Current branch +KUBEINIT_BRANCH = 'master' diff --git a/agent/kubeinit/get_banner.py b/agent/kubeinit/get_banner.py new file mode 100644 index 000000000..cea8d2a1c --- /dev/null +++ b/agent/kubeinit/get_banner.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + +from kubeinit import __version__ + +kubeinit_version = __version__ + + +def get_banner(): + """ + Get banner method. + + This method prints + the kubeinit.com banner. + """ + banner = """ +888 d8P 888 8888888 d8b 888 +888 d8P 888 888 Y8P 888 +888 d8P 888 888 888 +888d88K 888 888 88888b. .d88b. 888 88888b. 888 888888 +8888888b 888 888 888 "88b d8P Y8b 888 888 "88b 888 888 +888 Y88b 888 888 888 888 88888888 888 888 888 888 888 +888 Y88b Y88b 888 888 d88P Y8b. 888 888 888 888 Y88b. +888 Y88b "Y88888 88888P" "Y8888 8888888 888 888 888 "Y888 + (kubeinit.com) agent version {} +""".format(kubeinit_version) + return banner diff --git a/agent/kubeinit/lib.py b/agent/kubeinit/lib.py new file mode 100644 index 000000000..404c833b0 --- /dev/null +++ b/agent/kubeinit/lib.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + + +def connect_to_aas(stop, + api_client=None): + """ + Watch for action with timeouts. + + This method will listen for custom objects + that times out. + """ + while True: + return True + + +def connect_to_aas2(stop, + api_client=None): + """ + Watch for action with timeouts. + + This method will listen for custom objects + that times out. + """ + while True: + return True diff --git a/agent/kubeinit/logger.py b/agent/kubeinit/logger.py new file mode 100644 index 000000000..a1ac5051f --- /dev/null +++ b/agent/kubeinit/logger.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + +import logging +import sys +from logging.handlers import TimedRotatingFileHandler + +from kubeinit.const import KUBEINIT_LOG_FILE + +FORMATTER = logging.Formatter("%(asctime)s - %(name)s - " + "%(levelname)s - %(message)s") + + +def get_console_handler(): + """ + Get logger console handler. + + This is a main component of the input for the controller + """ + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(FORMATTER) + return console_handler + + +def get_file_handler(): + """ + Get logger file handler. + + This is a main component of the input for the controller + """ + file_handler = TimedRotatingFileHandler( + KUBEINIT_LOG_FILE, when='midnight') + file_handler.setFormatter(FORMATTER) + return file_handler + + +def get_logger(logger_name): + """ + Get logger. + + This is a main component of the input for the controller + """ + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) + # We do not want to print in the console + # logger.addHandler(get_console_handler()) + logger.addHandler(get_file_handler()) + logger.propagate = False + return logger diff --git a/agent/requirements.txt b/agent/requirements.txt new file mode 100644 index 000000000..abb118689 --- /dev/null +++ b/agent/requirements.txt @@ -0,0 +1,14 @@ +pyyaml +requests +ansible +openshift +j2cli +jinja2 +kubernetes +setuptools +PTable +rich +google-cloud +google-cloud-storage +google-cloud-firestore + diff --git a/agent/setup.py b/agent/setup.py new file mode 100644 index 000000000..8b2ee6c1f --- /dev/null +++ b/agent/setup.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + +import os +from sys import version_info + +from setuptools import find_packages, setup + +if version_info < (3, 5): + raise RuntimeError( + 'Python 3.5 or greater is required' + ) + +_NAME = 'kubeinit' +_DESCRIPTION = 'The Kubeinit CLI' +_REVISION = '0.6.5' + +kubeinit_revision = os.environ.get('KUBEINIT_REVISION', "") +if (kubeinit_revision != ""): + _REVISION = _REVISION + "." + kubeinit_revision + +if os.path.isfile('../README.md'): + with open('../README.md') as f: + long_description = f.read() +else: + long_description = _DESCRIPTION + +with open('requirements.txt') as f: + requirements = f.read().splitlines() + +setup( + name=_NAME, + version=_REVISION, + description=_DESCRIPTION, + long_description_content_type='text/markdown', + long_description=long_description, + url='https://www.kubeinit.com', + packages=find_packages(), + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Topic :: Scientific/Engineering', + 'Topic :: Software Development :: Testing', + ], + author='Carlos Camacho', + author_email='carloscamachoucv@gmail.com', + include_package_data=True, + install_requires=requirements, + entry_points={ + 'console_scripts': [ + '{0} = {0}.cli:main'.format(_NAME), + ] + } +) diff --git a/agent/tests/test_sample.py b/agent/tests/test_sample.py new file mode 100644 index 000000000..1f09d5d1c --- /dev/null +++ b/agent/tests/test_sample.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +""" +Copyright 2019 Kubeinit (kubeinit.com). + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + + +def test_file1_method1(): + """Test file 1 method 1.""" + x = 5 + y = 6 + assert x + 1 == y, "test failed" + + +def test_file1_method2(): + """Test file 1 method 2.""" + x = 5 + y = 6 + assert x + 1 == y, "test failed" diff --git a/kubeinit/galaxy.yml b/kubeinit/galaxy.yml index cd16f3a6d..a9d889353 100644 --- a/kubeinit/galaxy.yml +++ b/kubeinit/galaxy.yml @@ -1,7 +1,7 @@ --- namespace: kubeinit name: kubeinit -version: 0.6.4 +version: 0.6.5 readme: README.md authors: - Carlos Camacho diff --git a/tools/check_version.sh b/tools/check_version.sh new file mode 100755 index 000000000..e689cb2c5 --- /dev/null +++ b/tools/check_version.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +KUBEINIT_AGENT_VERSION=$(cat ./agent/setup.py | grep "_REVISION = '" | cut -d"'" -f2) +KUBEINIT_COLLECTION_VERSION=$(cat ./kubeinit/galaxy.yml | grep version | cut -d' ' -f2) +#KUBEINIT_UI_VERSION=$(cat ./ui/package.json | grep version | cut -d'"' -f4) + +if [[ "$KUBEINIT_AGENT_VERSION" == "$KUBEINIT_COLLECTION_VERSION" ]]; then + echo "Version match" +else + echo "Version do not match" + exit 1 +fi diff --git a/tools/pypi_publish.sh b/tools/pypi_publish.sh new file mode 100755 index 000000000..81fda931c --- /dev/null +++ b/tools/pypi_publish.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +PARAMS="" +while (( "$#" )); do + case "$1" in + -k|--pypi-key) + KARG=$2 + shift 2 + ;; + --) # end argument parsing + shift + break + ;; + -*|--*=) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + echo "Please use ./pypi_publish.sh [-k | --pypi-key ]" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# +# Initial variables +# + +all_published_versions=$(curl -L https://pypi.python.org/pypi/kubeinit/json | jq -r '.releases' | jq 'keys[]') +current_kubeinit_version=$(cat agent/setup.py | grep "_REVISION = '" | cut -d "'" -f 2) + +publish="1" + +# +# Check all the current published versions and if the +# packaged to be created has a different version, then +# we publish it to Galaxy Ansible +# + +for ver in $all_published_versions; do + echo "--" + echo "Published: "$ver + echo "Built: "$current_kubeinit_version + echo "" + if [[ $ver == \"$current_kubeinit_version\" ]]; then + echo "The current version $current_kubeinit_version is already published" + echo "Proceed to update the setup.py file with a newer version" + echo "After the version change, when the commit is merged, then the package" + echo "will be published automatically." + publish="0" + fi +done + +cd ./agent +python3 setup.py sdist +twine check dist/* + +if [ "$publish" == "1" ]; then + echo 'This version is not published, publishing!...' + +cat < ~/.pypirc +[distutils] +index-servers = + pypi + testpypi + kubeinit + +[pypi] +username = __token__ + +[testpypi] +username = __token__ + +[kubeinit] +repository = https://upload.pypi.org/legacy/ +username = __token__ +password = $KARG +EOF + + twine upload --verbose --repository kubeinit dist/* +fi diff --git a/tox.ini b/tox.ini index 8a832e982..3dd20c5b5 100644 --- a/tox.ini +++ b/tox.ini @@ -73,10 +73,15 @@ deps = {[testenv:linters]deps} commands = bash -c './tools/sanity.sh' +[testenv:checktools] +commands = + {toxinidir}/tools/check_version.sh + [testenv:linters] deps = -r{toxinidir}/test-requirements.txt commands = + {[testenv:checktools]commands} {[testenv:flake8]commands} {[testenv:bashate]commands} {[testenv:yamlfind]commands}