Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DepsTask with Click #6260

Merged
merged 9 commits into from
Nov 22, 2022
9 changes: 8 additions & 1 deletion core/dbt/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from dbt.profiler import profiler
from dbt.tracking import initialize_from_flags, track_run
from dbt.config.runtime import load_project
from dbt.task.deps import DepsTask


def cli_runner():
Expand Down Expand Up @@ -228,7 +229,13 @@ def debug(ctx, **kwargs):
def deps(ctx, **kwargs):
"""Pull the most recent version of the dependencies listed in packages.yml"""
flags = Flags()
click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
project = ctx.obj['project']

task = DepsTask.from_project(project, flags.VARS)

results = task.run()
success = task.interpret_results(results)
return results, success


# dbt init
Expand Down
14 changes: 7 additions & 7 deletions core/dbt/deps/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,19 +94,19 @@ def __iter__(self) -> Iterator[UnpinnedPackage]:

def _check_for_duplicate_project_names(
final_deps: List[PinnedPackage],
config: Project,
project: Project,
renderer: DbtProjectYamlRenderer,
):
seen: Set[str] = set()
for package in final_deps:
project_name = package.get_project_name(config, renderer)
project_name = package.get_project_name(project, renderer)
if project_name in seen:
raise_dependency_error(
f'Found duplicate project "{project_name}". This occurs when '
"a dependency has the same project name as some other "
"dependency."
)
elif project_name == config.project_name:
elif project_name == project.project_name:
raise_dependency_error(
"Found a dependency with the same name as the root project "
f'"{project_name}". Package names must be unique in a project.'
Expand All @@ -116,21 +116,21 @@ def _check_for_duplicate_project_names(


def resolve_packages(
packages: List[PackageContract], config: RuntimeConfig
packages: List[PackageContract], project: Project, profile, cli_vars
) -> List[PinnedPackage]:
pending = PackageListing.from_contracts(packages)
final = PackageListing()
renderer = DbtProjectYamlRenderer(config, config.cli_vars)
renderer = DbtProjectYamlRenderer(profile, cli_vars)
MichelleArk marked this conversation as resolved.
Show resolved Hide resolved

while pending:
next_pending = PackageListing()
# resolve the dependency in question
for package in pending:
final.incorporate(package)
target = final[package].resolved().fetch_metadata(config, renderer)
target = final[package].resolved().fetch_metadata(project, renderer)
next_pending.update_from(target.packages)
pending = next_pending

resolved = final.resolved()
_check_for_duplicate_project_names(resolved, config, renderer)
_check_for_duplicate_project_names(resolved, project, renderer)
return resolved
15 changes: 8 additions & 7 deletions core/dbt/task/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ class BaseTask(metaclass=ABCMeta):

def __init__(self, args, config):
self.args = args
self.args.single_threaded = False
# TODO
# self.args.single_threaded = False
self.config = config

@classmethod
Expand Down Expand Up @@ -139,14 +140,14 @@ def run(self):
def interpret_results(self, results):
return True


def get_nearest_project_dir(args):
from pathlib import Path
def get_nearest_project_dir(project_dir: Optional[Path]):
# If the user provides an explicit project directory, use that
# but don't look at parent directories.
if args.project_dir:
project_file = os.path.join(args.project_dir, "dbt_project.yml")
if project_dir:
project_file = os.path.join(project_dir, "dbt_project.yml")
if os.path.exists(project_file):
return args.project_dir
return project_dir
else:
raise dbt.exceptions.RuntimeException(
"fatal: Invalid --project-dir flag. Not a dbt project. "
Expand All @@ -169,7 +170,7 @@ def get_nearest_project_dir(args):


def move_to_nearest_project_dir(args):
nearest_project_dir = get_nearest_project_dir(args)
nearest_project_dir = get_nearest_project_dir(args.project_dir)
MichelleArk marked this conversation as resolved.
Show resolved Hide resolved
os.chdir(nearest_project_dir)
return nearest_project_dir

Expand Down
30 changes: 21 additions & 9 deletions core/dbt/task/deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,17 @@
from dbt.task.base import BaseTask, move_to_nearest_project_dir


from dbt.config import RuntimeConfig, Project
from dbt.task.base import NoneConfig

class DepsTask(BaseTask):
ChenyuLInx marked this conversation as resolved.
Show resolved Hide resolved
ConfigType = UnsetProfileConfig
# NoneConfig is the default, just setting it explicitly here for now.
ConfigType = NoneConfig

def __init__(self, args, config: UnsetProfileConfig):
def __init__(self, args, config: NoneConfig, project: Project, cli_vars):
super().__init__(args=args, config=config)
self.project = project
self.cli_vars = cli_vars

def track_package_install(self, package_name: str, source_type: str, version: str) -> None:
# Hub packages do not need to be hashed, as they are public
Expand All @@ -39,22 +45,22 @@ def track_package_install(self, package_name: str, source_type: str, version: st
package_name = dbt.utils.md5(package_name)
version = dbt.utils.md5(version)
dbt.tracking.track_package_install(
self.config,
self.config.args,
"deps", #flags.WHICH alternatively
self.project.hashed_name(),
{"name": package_name, "source": source_type, "version": version},
)

def run(self):
system.make_directory(self.config.packages_install_path)
packages = self.config.packages.packages
system.make_directory(self.project.packages_install_path)
packages = self.project.packages.packages
if not packages:
fire_event(DepsNoPackagesFound())
return

with downloads_directory():
final_deps = resolve_packages(packages, self.config)
final_deps = resolve_packages(packages, self.project, None, self.cli_vars)

renderer = DbtProjectYamlRenderer(self.config, self.config.cli_vars)
renderer = DbtProjectYamlRenderer(None, self.cli_vars)

packages_to_upgrade = []
for package in final_deps:
Expand All @@ -63,7 +69,7 @@ def run(self):
version = package.get_version()

fire_event(DepsStartPackageInstall(package_name=package_name))
package.install(self.config, renderer)
package.install(self.project, renderer)
fire_event(DepsInstallInfo(version_name=package.nice_version_name()))
if source_type == "hub":
version_latest = package.get_version_latest()
Expand All @@ -88,3 +94,9 @@ def from_args(cls, args):
# into the modules directory
move_to_nearest_project_dir(args)
return super().from_args(args)

@classmethod
def from_project(cls, project: Project, cli_vars):
move_to_nearest_project_dir(project.project_root)
# TODO: consider args, need for UnsetProfile
return cls(None, NoneConfig(), project, cli_vars)
6 changes: 3 additions & 3 deletions core/dbt/tracking.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,15 +287,15 @@ def get_base_invocation_context():
}


def track_package_install(config, args, options):
def track_package_install(command_name: str, project_hashed_name: Optional[str], options):
assert active_user is not None, "Cannot track package installs when active user is None"

invocation_data = get_base_invocation_context()

invocation_data.update(
{
"project_id": None if config is None else config.hashed_name(),
"command": args.which,
"project_id": project_hashed_name,
"command": command_name
}
)

Expand Down