diff --git a/src/python/pants/base/file_system_project_tree.py b/src/python/pants/base/file_system_project_tree.py deleted file mode 100644 index 2285ead834c..00000000000 --- a/src/python/pants/base/file_system_project_tree.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import os -from glob import glob1 - -from pants.base.project_tree import ProjectTree -from pants.util.dirutil import fast_relpath, safe_walk - - -class FileSystemProjectTree(ProjectTree): - def _join(self, relpath): - if relpath.startswith(os.sep): - raise ValueError('Absolute path "{}" not legal in {}.'.format(relpath, self)) - return os.path.join(self.build_root, relpath) - - def _glob1_raw(self, dir_relpath, glob): - return glob1(self._join(dir_relpath), glob) - - def _isdir_raw(self, relpath): - return os.path.isdir(self._join(relpath)) - - def _isfile_raw(self, relpath): - return os.path.isfile(self._join(relpath)) - - def _exists_raw(self, relpath): - return os.path.exists(self._join(relpath)) - - def _content_raw(self, file_relpath): - with open(self._join(file_relpath), "rb") as source: - return source.read() - - def _relative_readlink_raw(self, relpath): - return os.readlink(self._join(relpath)) - - def _walk_raw(self, relpath, topdown=True): - def onerror(error): - raise OSError( - getattr(error, "errno", None), "Failed to walk below {}".format(relpath), error - ) - - for root, dirs, files in safe_walk(self._join(relpath), topdown=topdown, onerror=onerror): - yield fast_relpath(root, self.build_root), dirs, files - - def __eq__(self, other): - return other and (type(other) == type(self)) and (self.build_root == other.build_root) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.build_root) - - def __repr__(self): - return "{}({})".format(self.__class__.__name__, self.build_root) diff --git a/src/python/pants/base/project_tree.py b/src/python/pants/base/project_tree.py deleted file mode 100644 index 28561f225ea..00000000000 --- a/src/python/pants/base/project_tree.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import logging -import os -from abc import ABC, abstractmethod -from typing import List, Optional - -from pathspec.pathspec import PathSpec -from pathspec.patterns.gitwildmatch import GitWildMatchPattern - -from pants.util.dirutil import fast_relpath - -logger = logging.getLogger(__name__) - - -class ProjectTree(ABC): - """Represents project tree which is used to locate and read build files. - - Has two implementations: one backed by file system and one backed by SCM. - """ - - class InvalidBuildRootError(Exception): - """Raised when the build_root specified to a ProjectTree is not valid.""" - - class AccessIgnoredPathError(Exception): - """Raised when accessing a path which is ignored by pants.""" - - def __init__(self, build_root: str, ignore_patterns: Optional[List[str]] = None): - if not os.path.isabs(build_root): - raise self.InvalidBuildRootError( - "ProjectTree build_root {} must be an absolute path.".format(build_root) - ) - self.build_root = os.path.realpath(build_root) - logger.debug("ProjectTree ignore_patterns: %s", ignore_patterns) - self.ignore_patterns = ignore_patterns if ignore_patterns else [] - self.ignore = PathSpec.from_lines(GitWildMatchPattern, self.ignore_patterns) - - @abstractmethod - def _glob1_raw(self, dir_relpath, glob): - """Returns a list of paths in path that match glob.""" - - @abstractmethod - def _isdir_raw(self, relpath): - """Returns True if path is a directory.""" - - @abstractmethod - def _isfile_raw(self, relpath): - """Returns True if path is a file.""" - - @abstractmethod - def _exists_raw(self, relpath): - """Returns True if path exists.""" - - @abstractmethod - def _content_raw(self, file_relpath): - """Returns the content for file at path.""" - - @abstractmethod - def _relative_readlink_raw(self, relpath): - """Execute `readlink` for the given path, which may result in a relative path.""" - - @abstractmethod - def _walk_raw(self, relpath, topdown=True): - """Walk the file tree rooted at `path`. - - Works like os.walk but returned root value is relative path. - """ - - def glob1(self, dir_relpath, glob): - """Returns a list of paths in path that match glob and are not ignored.""" - if self.isignored(dir_relpath, directory=True): - return [] - - matched_files = self._glob1_raw(dir_relpath, glob) - prefix = self._relpath_no_dot(dir_relpath) - return self._filter_ignored(matched_files, selector=lambda p: os.path.join(prefix, p)) - - def isdir(self, relpath): - """Returns True if path is a directory and is not ignored.""" - if self._isdir_raw(relpath): - if not self.isignored(relpath, directory=True): - return True - - return False - - def isfile(self, relpath): - """Returns True if path is a file and is not ignored.""" - if self.isignored(relpath): - return False - return self._isfile_raw(relpath) - - def exists(self, relpath): - """Returns True if path exists and is not ignored.""" - if self.isignored(self._append_slash_if_dir_path(relpath)): - return False - return self._exists_raw(relpath) - - def content(self, file_relpath): - """Returns the content for file at path. Raises exception if path is ignored. - - Raises exception if path is ignored. - """ - if self.isignored(file_relpath): - self._raise_access_ignored(file_relpath) - - return self._content_raw(file_relpath) - - def relative_readlink(self, relpath): - """Execute `readlink` for the given path, which may result in a relative path. - - Raises exception if path is ignored. - """ - if self.isignored(self._append_slash_if_dir_path(relpath)): - self._raise_access_ignored(relpath) - return self._relative_readlink_raw(relpath) - - def walk(self, relpath, topdown=True): - """Walk the file tree rooted at `path`. - - Works like os.walk but returned root value is relative path. Ignored paths will not be - returned. - """ - for root, dirs, files in self._walk_raw(relpath, topdown): - matched_dirs = self.ignore.match_files( - [os.path.join(root, "{}/".format(d)) for d in dirs] - ) - matched_files = self.ignore.match_files([os.path.join(root, f) for f in files]) - for matched_dir in matched_dirs: - dirs.remove(fast_relpath(matched_dir, root).rstrip("/")) - - for matched_file in matched_files: - files.remove(fast_relpath(matched_file, root)) - - yield root, dirs, files - - def readlink(self, relpath): - link_path = self.relative_readlink(relpath) - if os.path.isabs(link_path): - raise IOError( - "Absolute symlinks not supported in {}: {} -> {}".format(self, relpath, link_path) - ) - # In order to enforce that this link does not escape the build_root, we join and - # then remove it. - abs_normpath = os.path.normpath( - os.path.join(self.build_root, os.path.dirname(relpath), link_path) - ) - return fast_relpath(abs_normpath, self.build_root) - - def isignored(self, relpath, directory=False): - """Returns True if path matches pants ignore pattern.""" - relpath = self._relpath_no_dot(relpath) - if directory: - relpath = self._append_trailing_slash(relpath) - return self.ignore.match_file(relpath) - - def _filter_ignored(self, entries, selector=None): - """Given an opaque entry list, filter any ignored entries. - - :param entries: A list or generator that produces entries to filter. - :param selector: A function that computes a path for an entry relative to the root of the - ProjectTree, or None to use identity. - """ - selector = selector or (lambda x: x) - prefixed_entries = [ - (self._append_slash_if_dir_path(selector(entry)), entry) for entry in entries - ] - ignored_paths = set(self.ignore.match_files(path for path, _ in prefixed_entries)) - return [entry for path, entry in prefixed_entries if path not in ignored_paths] - - def _relpath_no_dot(self, relpath): - return relpath.lstrip("./") if relpath != "." else "" - - def _raise_access_ignored(self, relpath): - """Raises exception when accessing ignored path.""" - raise self.AccessIgnoredPathError("The path {} is ignored in {}".format(relpath, self)) - - def _append_trailing_slash(self, relpath): - """Add a trailing slash if not already has one.""" - return relpath if relpath.endswith("/") or len(relpath) == 0 else relpath + "/" - - def _append_slash_if_dir_path(self, relpath): - """For a dir path return a path that has a trailing slash.""" - if self._isdir_raw(relpath): - return self._append_trailing_slash(relpath) - - return relpath diff --git a/src/python/pants/engine/internals/scheduler.py b/src/python/pants/engine/internals/scheduler.py index f1348e69633..1a8bee10d11 100644 --- a/src/python/pants/engine/internals/scheduler.py +++ b/src/python/pants/engine/internals/scheduler.py @@ -47,7 +47,6 @@ from pants.option.global_options import ExecutionOptions from pants.util.contextutil import temporary_file_path from pants.util.logging import LogLevel -from pants.util.ordered_set import FrozenOrderedSet from pants.util.strutil import pluralize logger = logging.getLogger(__name__) @@ -97,7 +96,7 @@ def __init__( local_execution_root_dir: str, named_caches_dir: str, ca_certs_path: Optional[str], - rules: FrozenOrderedSet[Rule], + rules: Iterable[Rule], union_membership: UnionMembership, execution_options: ExecutionOptions, executor: PyExecutor, diff --git a/src/python/pants/engine/internals/scheduler_test_base.py b/src/python/pants/engine/internals/scheduler_test_base.py index ddb273d1119..cff8803f877 100644 --- a/src/python/pants/engine/internals/scheduler_test_base.py +++ b/src/python/pants/engine/internals/scheduler_test_base.py @@ -2,15 +2,12 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import shutil -from dataclasses import asdict -from pants.base.file_system_project_tree import FileSystemProjectTree from pants.engine.internals.native import Native from pants.engine.internals.native_engine import PyExecutor from pants.engine.internals.scheduler import Scheduler, SchedulerSession from pants.engine.unions import UnionMembership -from pants.option.global_options import DEFAULT_EXECUTION_OPTIONS, ExecutionOptions +from pants.option.global_options import DEFAULT_EXECUTION_OPTIONS from pants.util.contextutil import temporary_file_path from pants.util.dirutil import safe_mkdtemp, safe_rmtree @@ -29,53 +26,33 @@ def _create_work_dir(self): self.addCleanup(safe_rmtree, work_dir) return work_dir - def mk_fs_tree(self, build_root_src=None, ignore_patterns=None, work_dir=None): - """Create a temporary FilesystemProjectTree. - - :param build_root_src: Optional directory to pre-populate from; otherwise, empty. - :returns: A FilesystemProjectTree. - """ - work_dir = work_dir or self._create_work_dir() - build_root = os.path.join(work_dir, "build_root") - if build_root_src is not None: - shutil.copytree(build_root_src, build_root, symlinks=True) - else: - os.makedirs(build_root) - return FileSystemProjectTree(build_root, ignore_patterns=ignore_patterns) - def mk_scheduler( self, - rules=None, - project_tree=None, - work_dir=None, - include_trace_on_error=True, - execution_options=None, - ca_certs_path=None, + rules, + include_trace_on_error: bool = True, ) -> SchedulerSession: """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" - rules = rules or [] - work_dir = work_dir or self._create_work_dir() - project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) + work_dir = self._create_work_dir() + + build_root = os.path.join(work_dir, "build_root") + os.makedirs(build_root) + local_store_dir = os.path.realpath(safe_mkdtemp()) local_execution_root_dir = os.path.realpath(safe_mkdtemp()) named_caches_dir = os.path.realpath(safe_mkdtemp()) - if execution_options is not None: - eo = asdict(DEFAULT_EXECUTION_OPTIONS) - eo.update(execution_options) - execution_options = ExecutionOptions(**eo) scheduler = Scheduler( native=self._native, - ignore_patterns=project_tree.ignore_patterns, + ignore_patterns=[], use_gitignore=False, - build_root=project_tree.build_root, + build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, - ca_certs_path=ca_certs_path, + ca_certs_path=None, rules=rules, union_membership=UnionMembership({}), executor=self._executor, - execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS, + execution_options=DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session( diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 11dc9bb9db6..557ef1bb1d3 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -468,7 +468,7 @@ class RuleIndex: union_rules: FrozenOrderedSet[UnionRule] @classmethod - def create(cls, rule_entries) -> RuleIndex: + def create(cls, rule_entries: Iterable[Rule]) -> RuleIndex: """Creates a RuleIndex with tasks indexed by their output type.""" rules: OrderedSet[TaskRule] = OrderedSet() queries: OrderedSet[QueryRule] = OrderedSet()