Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rewrote pr_list_database.py to use wxflow's SQLiteDB Class #2376

Merged
Merged
Changes from 12 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
460c8ec
new pr database code
Mar 4, 2024
b9dcab0
made if else into match with functions
Mar 5, 2024
aff1d57
made each input a functional call
Mar 5, 2024
b9974e8
got mostly working
Mar 5, 2024
f7dc2d6
Merge branch 'NOAA-EMC:develop' into update_database
TerrenceMcGuinness-NOAA Mar 5, 2024
bf51d0d
rewrote pr_list_database.py using wxflow import SQLiteDB
Mar 5, 2024
2519ba2
Merge branch 'update_database' of https://github.com/TerrenceMcGuinne…
Mar 5, 2024
eb66922
Merge branch 'NOAA-EMC:develop' into update_database
TerrenceMcGuinness-NOAA Mar 5, 2024
8374797
removed pygithub work
Mar 5, 2024
ba785a0
Merge branch 'update_database' of https://github.com/TerrenceMcGuinne…
Mar 5, 2024
4f85832
added pylint spaces
Mar 5, 2024
45c80fb
more pycodestyle spaces
Mar 5, 2024
d2866a6
Update ci/scripts/pr_list_database.py
TerrenceMcGuinness-NOAA Mar 7, 2024
79836e2
Merge branch 'NOAA-EMC:develop' into update_database
TerrenceMcGuinness-NOAA Mar 8, 2024
64f7441
replaced search for uniquness in adding pr to state database with UNI…
Mar 8, 2024
9a9858c
Merge branch 'update_database' of https://github.com/TerrenceMcGuinne…
Mar 8, 2024
37cad0a
added error exception class for SQLiteDB
Mar 8, 2024
e11df59
updated hash for wxflow to include SQLite class and its latest updates
Mar 8, 2024
0862970
added better condtion on checkout error code string for when user tri…
Mar 8, 2024
c2e6473
simplyfied test for uniqueness to account for different versions of s…
Mar 8, 2024
fe3a007
improved args to function calls and their types
Mar 8, 2024
c15d457
remove orion from gefs case for testing non system build bash ci
Mar 8, 2024
ac49e30
took orion skip back out of gefs case
Mar 8, 2024
cdc69ff
updated SQLiteDB.Error to SQLiteDBError
Mar 8, 2024
4464385
Merge branch 'update_database' of https://github.com/TerrenceMcGuinne…
Mar 8, 2024
56df6de
added gitmodule dir for wxflow with some last minute changes to SQLit…
Mar 8, 2024
03cb16d
Merge branch 'develop' into update_database
TerrenceMcGuinness-NOAA Mar 8, 2024
966e288
remove function was redundent so was removed
Mar 10, 2024
32c3ce2
Update pr_list_database.py
TerrenceMcGuinness-NOAA Mar 11, 2024
e3fdde5
pynorms space added
Mar 11, 2024
b12fcb2
Update ci/scripts/pr_list_database.py
TerrenceMcGuinness-NOAA Mar 11, 2024
a95bd1f
Update ci/scripts/pr_list_database.py
TerrenceMcGuinness-NOAA Mar 11, 2024
c6b0a16
updated few names and added docsting for fullpath
Mar 11, 2024
cb33781
removed whitespace per pycodestyle
Mar 11, 2024
d44e68e
Update ci/scripts/pr_list_database.py
TerrenceMcGuinness-NOAA Mar 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 73 additions & 129 deletions ci/scripts/pr_list_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,154 +2,121 @@

import sys
import os
from pathlib import Path
from wxflow import SQLiteDB
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER, ZERO_OR_MORE
import sqlite3


def full_path(string):
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved
"""
Gets the absolute path of the given file and confirms the directory exists

Parameters
----------
string : str
Path to a file

Returns
--------
str
Absolute path of input path

Raises
-------
NotADirectoryError
If the target directory for the file does not exist.

"""

if os.path.isfile(string) or os.path.isdir(os.path.dirname(string)):
return os.path.abspath(string)
else:
raise NotADirectoryError(string)


def sql_connection(filename: os.path) -> sqlite3.Connection:
def create(db):
"""
Returns an Sqlite3 Cursor object from a given path to a sqlite3 database file
Create a new database.
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved

Parameters
----------
filename : Path
Full path to a sqlite3 database file

Returns
-------
sqlite3.Connection
Sqlite3 Connection object for updating table

ci_database : SQLiteDB
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved
The database to create.
"""
try:
return sqlite3.connect(filename)
except sqlite3.Error:
print(sqlite3.Error)
sys.exit(-1)
db.create_table('pr_list', ['pr INTEGER PRIMARY KEY', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT'])


def sql_table(obj: sqlite3.Cursor) -> None:
def add_pr(db, pr):
"""
Creates the initial sqlite3 table for PR states and status
Add a pull request to the database.

Parameters
----------
obj : sqlite3.Cursor
Cursor object for Sqlite3

ci_database : SQLiteDB
The database to add the pull request to.
pr : str
The pull request to add.
"""
rows = db.fetch_data('pr_list')
for row in rows:
if str(row[0]) == str(args.add_pr[0]):
print(f"pr {row[0]} already is in list: nothing added")
sys.exit(0)
aerorahul marked this conversation as resolved.
Show resolved Hide resolved

obj.execute("CREATE TABLE processing(pr integer PRIMARY KEY, state text, status text, reset_id integer, cases text)")
entities = (args.add_pr[0], 'Open', 'Ready', 0, 'ci_repo')
db.insert_data('pr_list', entities)


def sql_insert(obj: sqlite3.Cursor, entities: list) -> None:
def update_pr(db, args):
"""
Inserts a new row in sqlite3 table with PR, state, and status
Update a pull request in the database.

Parameters
----------
obj : sqlite3.Cursor
Cursor object for Sqlite3
entities : list
A list of four string values that go into sqlite table (pr, state, status, reset_id, cases)
pr: pull request number
state: The new value for the state (Open, Closed)
status: The new value for the status (Ready, Running, Failed)
reset_id: The value for number of times reset_id to Ready
cases: String containing case selection information

"""

obj.execute('INSERT INTO processing(pr, state, status, reset_id, cases) VALUES(?, ?, ?, ?, ?)', entities)


def sql_update(obj: sqlite3.Cursor, pr: str, updates: dict) -> None:
"""Updates table for a given pr with new values for state and status

Parameters
----------
obj : sqlite.sql_connection
sqlite3 Cursor Object
pr : str
The given pr number to update in the table
updates : dict
Dictionary of values to update for a given PR to include by postion
state, The new value for the state (Open, Closed)
status, The new value for the status (Ready, Running, Failed)
reset_id, The value for number of times reset_id to Ready
cases, Information regarding which cases are used (i.e. self PR)

ci_database : SQLiteDB
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved
The database to update the pull request in.
args : argparse.Namespace
The command line arguments.
"""
if len(args.update_pr) < 2:
print(f"update_pr must have at least one vaule to update")
sys.exit(0)

update_list = ['state', 'status', 'reset_id', 'cases']
rows = sql_fetch(obj)
for value in updates:
for value in args.update_pr[1:]:
update = update_list.pop(0)
obj.execute(f'UPDATE processing SET "{update}" = "{value}" WHERE pr = {pr}')
db.update_data('pr_list', update, value, 'pr', args.update_pr[0])


def sql_fetch(obj: sqlite3.Cursor) -> list:
""" Gets list of all rows in table
def remove_pr(db, args):
"""
Remove a pull request from the database.

Parameters
----------
obj : sqlite.sql_connection
sqlite3 Cursor Object

ci_database : SQLiteDB
The database to remove the pull request from.
pr : str
The pull request to remove.
"""
db.remove_column('pr_list', args.remove_pr[0])

obj.execute('SELECT * FROM processing')
return obj.fetchall()


def sql_remove(obj: sqlite3.Cursor, pr: str) -> None:
""" Removes the row from table with given pr number
def display(db, args):
"""
Display the database.

Parameters
----------
obj : sqlite.sql_connection
sqlite3 Connection Object
pr : str
pr number acting as key for removing the row with in it
ci_database : SQLiteDB
The database to display.
args : argparse.Namespace
The command line arguments.
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved

Returns
-------
list
The rows of the database.
"""
values = []
if len(args.display) == 1:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f'pr = {args.display[0]}')
else:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
for row in rows:
values.append(' '.join(map(str, row)))

obj.execute(f'DELETE FROM processing WHERE pr = {pr}').rowcount
return values


def input_args():

description = """Arguments for creating and updating db file for pr states
"""
Parse command line arguments.

Returns
-------
argparse.Namespace
The parsed command line arguments.
"""
parser = ArgumentParser(description=description,
formatter_class=ArgumentDefaultsHelpFormatter)

Expand All @@ -174,42 +141,19 @@ def input_args():
print(f'Error: {args.dbfile} does not exsist')
sys.exit(-1)

con = sql_connection(args.dbfile)
obj = con.cursor()
ci_database = SQLiteDB(args.dbfile)
ci_database.connect()

if args.create:
sql_table(obj)

create(ci_database)
if args.add_pr:
rows = sql_fetch(obj)
for row in rows:
if str(row[0]) == str(args.add_pr[0]):
print(f"pr {row[0]} already is in list: nothing added")
sys.exit(0)

entities = (args.add_pr[0], 'Open', 'Ready', 0, 'ci_repo')
sql_insert(obj, entities)

add_pr(ci_database, args.add_pr[0])
if args.update_pr:
if len(args.update_pr) < 2:
print(f"update_pr must have at least one vaule to update")
sys.exit(0)
pr = args.update_pr[0]

sql_update(obj, pr, args.update_pr[1:])

update_pr(ci_database, args)
if args.remove_pr:
sql_remove(obj, args.remove_pr[0])

remove_pr(ci_database, args.remove_pr[0])
if args.display is not None:
rows = sql_fetch(obj)
if len(args.display) == 1:
for row in rows:
if int(args.display[0]) == int(row[0]):
print(' '.join(map(str, row)))
else:
for row in rows:
print(' '.join(map(str, row)))

con.commit()
con.close()
for rows in display(ci_database, args):
print(rows)
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved Hide resolved

ci_database.disconnect()
Loading