Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pluginupdate.py: fix bugs and add improvements; vimPlugins: sort properly #353786

Merged
merged 3 commits into from
Nov 9, 2024
Merged

pluginupdate.py: fix bugs and add improvements; vimPlugins: sort properly #353786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b503ec
pluginupdate.py: reformat with ruff
PerchunPak Nov 8, 2024
a6fe798
pluginupdate.py: fix bugs and add improvements
PerchunPak Nov 5, 2024
e9b1d2d
vimPlugins: sort properly
PerchunPak Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 52 additions & 49 deletions maintainers/scripts/pluginupdate-py/pluginupdate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# - pkgs/development/lua-modules/updater/updater.py

# format:
# $ nix run nixpkgs#black maintainers/scripts/pluginupdate.py
# $ nix run nixpkgs#ruff maintainers/scripts/pluginupdate.py
# type-check:
# $ nix run nixpkgs#python3.pkgs.mypy maintainers/scripts/pluginupdate.py
# linted:
Expand Down Expand Up @@ -142,7 +142,7 @@ def _prefetch(self, ref: Optional[str]):
return loaded

def prefetch(self, ref: Optional[str]) -> str:
print("Prefetching %s", self.uri)
log.info("Prefetching %s", self.uri)
loaded = self._prefetch(ref)
return loaded["sha256"]

Expand Down Expand Up @@ -195,7 +195,7 @@ def latest_commit(self) -> Tuple[str, datetime]:
xml = req.read()

# Filter out illegal XML characters
illegal_xml_regex = re.compile(b"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]")
illegal_xml_regex = re.compile(b"[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]")
xml = illegal_xml_regex.sub(b"", xml)

root = ET.fromstring(xml)
Expand Down Expand Up @@ -256,21 +256,20 @@ class PluginDesc:

@property
def name(self):
if self.alias is None:
return self.repo.name
else:
return self.alias

def __lt__(self, other):
return self.repo.name < other.repo.name
return self.alias or self.repo.name

@staticmethod
def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> "PluginDesc":
log.debug("Loading row %s", row)
branch = row["branch"]
repo = make_repo(row["repo"], branch.strip())
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), row["alias"])
return PluginDesc(
repo,
branch.strip(),
# alias is usually an empty string
row["alias"] if row["alias"] else None,
)

@staticmethod
def load_from_string(config: FetchConfig, line: str) -> "PluginDesc":
Expand Down Expand Up @@ -328,12 +327,11 @@ def load_plugins_from_csv(
return plugins



def run_nix_expr(expr, nixpkgs: str, **args):
'''
"""
:param expr nix expression to fetch current plugins
:param nixpkgs Path towards a nixpkgs checkout
'''
"""
with CleanEnvironment(nixpkgs) as nix_path:
cmd = [
"nix",
Expand Down Expand Up @@ -382,16 +380,14 @@ def add(self, args):
fetch_config = FetchConfig(args.proc, args.github_token)
editor = self
for plugin_line in args.add_plugins:
log.debug("using plugin_line", plugin_line)
log.debug("using plugin_line %s", plugin_line)
pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
log.debug("loaded as pdesc", pdesc)
log.debug("loaded as pdesc %s", pdesc)
append = [pdesc]
editor.rewrite_input(
fetch_config, args.input_file, editor.deprecated, append=append
)
plugin, _ = prefetch_plugin(
pdesc,
)
plugin, _ = prefetch_plugin(pdesc)
autocommit = not args.no_commit
if autocommit:
commit(
Expand All @@ -406,9 +402,9 @@ def add(self, args):
# Expects arguments generated by 'update' subparser
def update(self, args):
"""CSV spec"""
print("the update member function should be overriden in subclasses")
print("the update member function should be overridden in subclasses")

def get_current_plugins(self, nixpkgs) -> List[Plugin]:
def get_current_plugins(self, nixpkgs: str) -> List[Plugin]:
"""To fill the cache"""
data = run_nix_expr(self.get_plugins, nixpkgs)
plugins = []
Expand Down Expand Up @@ -440,6 +436,7 @@ def update() -> dict:

plugins, redirects = check_results(results)

plugins = sorted(plugins, key=lambda v: v[1].normalized_name)
self.generate_nix(plugins, outfile)

return redirects
Expand Down Expand Up @@ -559,6 +556,7 @@ def run(
parser = self.create_parser()
args = parser.parse_args()
command = args.command or "update"
logging.basicConfig()
log.setLevel(LOG_LEVELS[args.debug])
log.info("Chose to run command: %s", command)
self.nixpkgs = args.nixpkgs
Expand Down Expand Up @@ -591,25 +589,24 @@ def prefetch_plugin(
p: PluginDesc,
cache: "Optional[Cache]" = None,
) -> Tuple[Plugin, Optional[Repo]]:
repo, branch, alias = p.repo, p.branch, p.alias
name = alias or p.repo.name
commit = None
log.info(f"Fetching last commit for plugin {name} from {repo.uri}@{branch}")
commit, date = repo.latest_commit()
log.info(f"Fetching last commit for plugin {p.name} from {p.repo.uri}@{p.branch}")
commit, date = p.repo.latest_commit()

cached_plugin = cache[commit] if cache else None
if cached_plugin is not None:
log.debug("Cache hit !")
cached_plugin.name = name
log.debug(f"Cache hit for {p.name}!")
cached_plugin.name = p.name
cached_plugin.date = date
return cached_plugin, repo.redirect
return cached_plugin, p.repo.redirect

has_submodules = repo.has_submodules()
log.debug(f"prefetch {name}")
sha256 = repo.prefetch(commit)
has_submodules = p.repo.has_submodules()
log.debug(f"prefetch {p.name}")
sha256 = p.repo.prefetch(commit)

return (
Plugin(name, commit, has_submodules, sha256, date=date),
repo.redirect,
Plugin(p.name, commit, has_submodules, sha256, date=date),
p.repo.redirect,
)


Expand All @@ -624,7 +621,7 @@ def print_download_error(plugin: PluginDesc, ex: Exception):


def check_results(
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]]
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]],
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
""" """
failures: List[Tuple[PluginDesc, Exception]] = []
Expand All @@ -642,10 +639,9 @@ def check_results(

print(f"{len(results) - len(failures)} plugins were checked", end="")
if len(failures) == 0:
print()
return plugins, redirects
else:
print(f", {len(failures)} plugin(s) could not be downloaded:\n")
log.error(f", {len(failures)} plugin(s) could not be downloaded:\n")

for plugin, exception in failures:
print_download_error(plugin, exception)
Expand Down Expand Up @@ -738,10 +734,7 @@ def rewrite_input(
append: List[PluginDesc] = [],
):
log.info("Rewriting input file %s", input_file)
plugins = load_plugins_from_csv(
config,
input_file,
)
plugins = load_plugins_from_csv(config, input_file)

plugins.extend(append)

Expand All @@ -753,15 +746,25 @@ def rewrite_input(
deprecations = json.load(f)
# TODO parallelize this step
for pdesc, new_repo in redirects.items():
log.info("Rewriting input file %s", input_file)
log.info("Resolving deprecated plugin %s -> %s", pdesc.name, new_repo.name)
new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias)

old_plugin, _ = prefetch_plugin(pdesc)
new_plugin, _ = prefetch_plugin(new_pdesc)

if old_plugin.normalized_name != new_plugin.normalized_name:
deprecations[old_plugin.normalized_name] = {
"new": new_plugin.normalized_name,
"date": cur_date_iso,
}

# remove plugin from index file, so we won't add it to deprecations again
for i, plugin in enumerate(plugins):
if plugin.name == pdesc.name:
plugins.pop(i)
break
plugins.append(new_pdesc)

with open(deprecated, "w") as f:
json.dump(deprecations, f, indent=4, sort_keys=True)
f.write("\n")
Expand All @@ -772,7 +775,7 @@ def rewrite_input(
fieldnames = ["repo", "branch", "alias"]
writer = csv.DictWriter(f, fieldnames, dialect="unix", quoting=csv.QUOTE_NONE)
writer.writeheader()
for plugin in sorted(plugins):
for plugin in sorted(plugins, key=lambda x: x.name):
writer.writerow(asdict(plugin))


Expand All @@ -792,9 +795,11 @@ def update_plugins(editor: Editor, args):

log.info("Start updating plugins")
if args.proc > 1 and args.github_token == None:
log.warning("You have enabled parallel updates but haven't set a github token.\n"
"You may be hit with `HTTP Error 429: too many requests` as a consequence."
"Either set --proc=1 or --github-token=YOUR_TOKEN. ")
log.warning(
"You have enabled parallel updates but haven't set a github token.\n"
"You may be hit with `HTTP Error 429: too many requests` as a consequence."
"Either set --proc=1 or --github-token=YOUR_TOKEN. "
)

fetch_config = FetchConfig(args.proc, args.github_token)
update = editor.get_update(args.input_file, args.outfile, fetch_config)
Expand All @@ -810,11 +815,9 @@ def update_plugins(editor: Editor, args):
if autocommit:
try:
repo = git.Repo(os.getcwd())
updated = datetime.now(tz=UTC).strftime('%Y-%m-%d')
updated = datetime.now(tz=UTC).strftime("%Y-%m-%d")
print(args.outfile)
commit(repo,
f"{editor.attr_path}: update on {updated}", [args.outfile]
)
commit(repo, f"{editor.attr_path}: update on {updated}", [args.outfile])
except git.InvalidGitRepositoryError as e:
print(f"Not in a git repository: {e}", file=sys.stderr)
sys.exit(1)
Expand Down
58 changes: 36 additions & 22 deletions pkgs/applications/editors/kakoune/plugins/update.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,63 @@
#!nix-shell update-shell.nix -i python3

# format:
# $ nix run nixpkgs.python3Packages.black -c black update.py
# $ nix run nixpkgs#python3Packages.ruff -- update.py
# type-check:
# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
# $ nix run nixpkgs#python3Packages.mypy -- update.py
# linted:
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265,E402 update.py
# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 update.py

import inspect
import os
import sys
from typing import List, Tuple
from pathlib import Path
from typing import List, Tuple

# Import plugin update library from maintainers/scripts/pluginupdate.py
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) # type: ignore
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) # type: ignore
sys.path.insert(
0, os.path.join(ROOT.parent.parent.parent.parent.parent, "maintainers", "scripts")
)
import pluginupdate

GET_PLUGINS = f"""(with import <localpkgs> {{}};
GET_PLUGINS = f"""(
with import <localpkgs> {{ }};
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also manually formatted this with nixfmt-rfc-style

let
inherit (kakouneUtils.override {{}}) buildKakounePluginFrom2Nix;
inherit (kakouneUtils.override {{ }}) buildKakounePluginFrom2Nix;
generated = callPackage {ROOT}/generated.nix {{
inherit buildKakounePluginFrom2Nix;
}};
hasChecksum = value: lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value;
getChecksum = name: value:
if hasChecksum value then {{
submodules = value.src.fetchSubmodules or false;
sha256 = value.src.outputHash;
rev = value.src.rev;
}} else null;
hasChecksum =
value:
lib.isAttrs value
&& lib.hasAttrByPath [
"src"
"outputHash"
] value;
getChecksum =
name: value:
if hasChecksum value then
{{
submodules = value.src.fetchSubmodules or false;
sha256 = value.src.outputHash;
rev = value.src.rev;
}}
else
null;
checksums = lib.mapAttrs getChecksum generated;
in lib.filterAttrs (n: v: v != null) checksums)"""

HEADER = "# This file has been generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!"

class KakouneEditor(pluginupdate.Editor):
in
lib.filterAttrs (n: v: v != null) checksums
)"""

HEADER = "# This file has been @generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why this @ ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just saw that the same does poetry

python-poetry/poetry#2773

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, fine by me.


def generate_nix(self, plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[1].name.lower())

class KakouneEditor(pluginupdate.Editor):
def generate_nix(
self,
plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate.Plugin]],
outfile: str,
):
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
Expand All @@ -54,7 +68,7 @@ def generate_nix(self, plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate
packages = ( self:
{"""
)
for pluginDesc, plugin in sorted_plugins:
for pluginDesc, plugin in plugins:
f.write(
f"""
{plugin.normalized_name} = buildKakounePluginFrom2Nix {{
Expand Down
Loading