Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement local reporter mode #126

Merged
merged 6 commits into from
May 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/ahriman/application/ahriman.py
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ def _set_patch_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
parser = root.add_parser("patch-list", help="list patch sets",
description="list available patches for the package", formatter_class=_formatter)
parser.add_argument("package", help="package base", nargs="?")
parser.add_argument("package", help="package base")
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("-v", "--variable", help="if set, show only patches for specified PKGBUILD variables",
action="append")
Expand Down
3 changes: 1 addition & 2 deletions src/ahriman/application/application/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,7 @@ def missing_dependencies(source: Iterable[Package]) -> dict[str, str | None]:
package = Package.from_aur(package_name, username)
with_dependencies[package.base] = package

# register package in local database
self.database.package_base_update(package)
# register package in the database
self.repository.reporter.set_unknown(package)

return list(with_dependencies.values())
4 changes: 2 additions & 2 deletions src/ahriman/application/application/application_packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def _add_aur(self, source: str, username: str | None) -> None:
"""
package = Package.from_aur(source, username)
self.database.build_queue_insert(package)
self.database.package_base_update(package)
self.reporter.set_unknown(package)

def _add_directory(self, source: str, *_: Any) -> None:
"""
Expand Down Expand Up @@ -139,7 +139,7 @@ def _add_repository(self, source: str, username: str | None) -> None:
"""
package = Package.from_official(source, self.repository.pacman, username)
self.database.build_queue_insert(package)
self.database.package_base_update(package)
self.reporter.set_unknown(package)

def add(self, names: Iterable[str], source: PackageSource, username: str | None = None) -> None:
"""
Expand Down
11 changes: 11 additions & 0 deletions src/ahriman/application/application/application_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from ahriman.core.database import SQLite
from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository
from ahriman.core.status import Client
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId

Expand Down Expand Up @@ -63,3 +64,13 @@ def architecture(self) -> str:
str: repository architecture
"""
return self.repository_id.architecture

@property
def reporter(self) -> Client:
"""
instance of the web/database client

Returns:
Client: repository reposter
"""
return self.repository.reporter
6 changes: 2 additions & 4 deletions src/ahriman/application/application/application_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,13 @@ def changes(self, packages: Iterable[Package]) -> None:
Args:
packages(Iterable[Package]): list of packages to retrieve changes
"""
last_commit_hashes = self.database.hashes_get()

for package in packages:
last_commit_sha = last_commit_hashes.get(package.base)
last_commit_sha = self.reporter.package_changes_get(package.base).last_commit_sha
if last_commit_sha is None:
continue # skip check in case if we can't calculate diff

changes = self.repository.package_changes(package, last_commit_sha)
self.repository.reporter.package_changes_set(package.base, changes)
self.repository.reporter.package_changes_update(package.base, changes)

def clean(self, *, cache: bool, chroot: bool, manual: bool, packages: bool, pacman: bool) -> None:
"""
Expand Down
3 changes: 2 additions & 1 deletion src/ahriman/application/handlers/add.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuratio
application.add(args.package, args.source, args.username)
patches = [PkgbuildPatch.from_env(patch) for patch in args.variable] if args.variable is not None else []
for package in args.package: # for each requested package insert patch
application.database.patches_insert(package, patches)
for patch in patches:
application.reporter.package_patches_update(package, patch)

if not args.now:
return
Expand Down
2 changes: 1 addition & 1 deletion src/ahriman/application/handlers/change.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,4 @@ def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuratio
ChangesPrinter(changes)(verbose=True, separator="")
Change.check_if_empty(args.exit_code, changes.is_empty)
case Action.Remove:
client.package_changes_set(args.package, Changes())
client.package_changes_update(args.package, Changes())
21 changes: 14 additions & 7 deletions src/ahriman/application/handlers/patch.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,25 +116,28 @@ def patch_set_create(application: Application, package_base: str, patch: Pkgbuil
package_base(str): package base
patch(PkgbuildPatch): patch descriptor
"""
application.database.patches_insert(package_base, [patch])
application.reporter.package_patches_update(package_base, patch)

@staticmethod
def patch_set_list(application: Application, package_base: str | None, variables: list[str] | None,
def patch_set_list(application: Application, package_base: str, variables: list[str] | None,
exit_code: bool) -> None:
"""
list patches available for the package base

Args:
application(Application): application instance
package_base(str | None): package base
package_base(str): package base
variables(list[str] | None): extract patches only for specified PKGBUILD variables
exit_code(bool): exit with error on empty search result
"""
patches = application.database.patches_list(package_base, variables)
patches = [
patch
for patch in application.reporter.package_patches_get(package_base, None)
if variables is None or patch.key in variables
]
Patch.check_if_empty(exit_code, not patches)

for base, patch in patches.items():
PatchPrinter(base, patch)(verbose=True, separator=" = ")
PatchPrinter(package_base, patches)(verbose=True, separator=" = ")

@staticmethod
def patch_set_remove(application: Application, package_base: str, variables: list[str] | None) -> None:
Expand All @@ -146,4 +149,8 @@ def patch_set_remove(application: Application, package_base: str, variables: lis
package_base(str): package base
variables(list[str] | None): remove patches only for specified PKGBUILD variables
"""
application.database.patches_remove(package_base, variables)
if variables is not None:
for variable in variables: # iterate over single variable
application.reporter.package_patches_remove(package_base, variable)
else:
application.reporter.package_patches_remove(package_base, None) # just pass as is
2 changes: 1 addition & 1 deletion src/ahriman/application/handlers/rebuild.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def extract_packages(application: Application, status: BuildStatusEnum | None, *
if from_database:
return [
package
for (package, last_status) in application.database.packages_get()
for (package, last_status) in application.reporter.package_get(None)
if status is None or last_status.status == status
]

Expand Down
8 changes: 2 additions & 6 deletions src/ahriman/application/handlers/status_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,8 @@ def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuratio
match args.action:
case Action.Update if args.package:
# update packages statuses
packages = application.repository.packages()
for base in args.package:
if (local := next((package for package in packages if package.base == base), None)) is not None:
client.package_add(local, args.status)
else:
client.package_update(base, args.status)
for package in args.package:
client.package_update(package, args.status)
case Action.Update:
# update service status
client.status_update(args.status)
Expand Down
2 changes: 1 addition & 1 deletion src/ahriman/application/lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateRunError
from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.status import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.repository_id import RepositoryId
Expand Down
10 changes: 5 additions & 5 deletions src/ahriman/core/alpm/pacman.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,31 +177,31 @@ def database_sync(self, handle: Handle, *, force: bool) -> None:
PacmanDatabase(database, self.configuration).sync(force=force)
transaction.release()

def files(self, packages: Iterable[str] | None = None) -> dict[str, set[Path]]:
def files(self, packages: Iterable[str] | None = None) -> dict[str, set[str]]:
"""
extract list of known packages from the databases

Args:
packages(Iterable[str] | None, optional): filter by package names (Default value = None)

Returns:
dict[str, set[Path]]: map of package name to its list of files
dict[str, set[str]]: map of package name to its list of files
"""
packages = packages or []

def extract(tar: tarfile.TarFile) -> Generator[tuple[str, set[Path]], None, None]:
def extract(tar: tarfile.TarFile) -> Generator[tuple[str, set[str]], None, None]:
for descriptor in filter(lambda info: info.path.endswith("/files"), tar.getmembers()):
package, *_ = str(Path(descriptor.path).parent).rsplit("-", 2)
if packages and package not in packages:
continue # skip unused packages
content = tar.extractfile(descriptor)
if content is None:
continue
files = {Path(filename.decode("utf8").rstrip()) for filename in content.readlines()}
files = {filename.decode("utf8").rstrip() for filename in content.readlines()}

yield package, files

result: dict[str, set[Path]] = {}
result: dict[str, set[str]] = {}
for database in self.handle.get_syncdbs():
database_file = self.repository_paths.pacman / "sync" / f"{database.name}.files.tar.gz"
if not database_file.is_file():
Expand Down
7 changes: 3 additions & 4 deletions src/ahriman/core/build_tools/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import BuildError
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output
Expand Down Expand Up @@ -116,20 +115,20 @@ def build(self, sources_dir: Path, **kwargs: str | None) -> list[Path]:
# e.g. in some cases packagelist command produces debug packages which were not actually built
return list(filter(lambda path: path.is_file(), map(Path, packages)))

def init(self, sources_dir: Path, database: SQLite, local_version: str | None) -> str | None:
def init(self, sources_dir: Path, patches: list[PkgbuildPatch], local_version: str | None) -> str | None:
"""
fetch package from git

Args:
sources_dir(Path): local path to fetch
database(SQLite): database instance
patches(list[PkgbuildPatch]): list of patches for the package
local_version(str | None): local version of the package. If set and equal to current version, it will
automatically bump pkgrel

Returns:
str | None: current commit sha if available
"""
last_commit_sha = Sources.load(sources_dir, self.package, database.patches_get(self.package.base), self.paths)
last_commit_sha = Sources.load(sources_dir, self.package, patches, self.paths)
if local_version is None:
return last_commit_sha # there is no local package or pkgrel increment is disabled

Expand Down
24 changes: 0 additions & 24 deletions src/ahriman/core/database/operations/changes_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,27 +117,3 @@ def run(connection: Connection) -> None:
})

return self.with_connection(run, commit=True)

def hashes_get(self, repository_id: RepositoryId | None = None) -> dict[str, str]:
"""
extract last commit hashes if available

Args:
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)

Returns:
dict[str, str]: map of package base to its last commit hash
"""

repository_id = repository_id or self._repository_id

def run(connection: Connection) -> dict[str, str]:
return {
row["package_base"]: row["last_commit_sha"]
for row in connection.execute(
"""select package_base, last_commit_sha from package_changes where repository = :repository""",
{"repository": repository_id.id}
)
}

return self.with_connection(run)
28 changes: 10 additions & 18 deletions src/ahriman/core/database/operations/dependencies_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from sqlite3 import Connection

from ahriman.core.database.operations.operations import Operations
Expand All @@ -31,7 +30,7 @@ class DependenciesOperations(Operations):
"""

def dependencies_get(self, package_base: str | None = None,
repository_id: RepositoryId | None = None) -> list[Dependencies]:
repository_id: RepositoryId | None = None) -> dict[str, Dependencies]:
"""
get dependencies for the specific package base if available

Expand All @@ -44,15 +43,9 @@ def dependencies_get(self, package_base: str | None = None,
"""
repository_id = repository_id or self._repository_id

def run(connection: Connection) -> list[Dependencies]:
return [
Dependencies(
row["package_base"],
{
Path(path): packages
for path, packages in row["dependencies"].items()
}
)
def run(connection: Connection) -> dict[str, Dependencies]:
return {
row["package_base"]: Dependencies(row["dependencies"])
for row in connection.execute(
"""
select package_base, dependencies from package_dependencies
Expand All @@ -64,15 +57,17 @@ def run(connection: Connection) -> list[Dependencies]:
"repository": repository_id.id,
}
)
]
}

return self.with_connection(run)

def dependencies_insert(self, dependencies: Dependencies, repository_id: RepositoryId | None = None) -> None:
def dependencies_insert(self, package_base: str, dependencies: Dependencies,
repository_id: RepositoryId | None = None) -> None:
"""
insert package dependencies

Args:
package_base(str): package base
dependencies(Dependencies): package dependencies
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
"""
Expand All @@ -89,12 +84,9 @@ def run(connection: Connection) -> None:
dependencies = :dependencies
""",
{
"package_base": dependencies.package_base,
"package_base": package_base,
"repository": repository_id.id,
"dependencies": {
str(path): packages
for path, packages in dependencies.paths.items()
}
"dependencies": dependencies.paths,
})

return self.with_connection(run, commit=True)
Expand Down
4 changes: 3 additions & 1 deletion src/ahriman/core/database/operations/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths


T = TypeVar("T")
Expand All @@ -38,7 +39,7 @@ class Operations(LazyLogging):
path(Path): path to the database file
"""

def __init__(self, path: Path, repository_id: RepositoryId) -> None:
def __init__(self, path: Path, repository_id: RepositoryId, repository_paths: RepositoryPaths) -> None:
"""
default constructor

Expand All @@ -48,6 +49,7 @@ def __init__(self, path: Path, repository_id: RepositoryId) -> None:
"""
self.path = path
self._repository_id = repository_id
self._repository_paths = repository_paths

@staticmethod
def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]:
Expand Down
Loading
Loading