Merge pull request #5 from blooop/feature/empy_apt

Feature/empy apt

README.md

@@ -3,24 +3,34 @@
 This is a [rocker](https://github.com/tfoote/rocker) extension for automating dependency installation.  The aim is to allow a projects to define its development dependencies in a deps.yaml file which are added to the rocker container. The extension will recursivly search for deps.yaml files and run the install commands in several layers.  
 
 Layer order:
-- script_tools
+- env
+
+- scripts_tools
 - apt_tools
 - pip_tools
 
-- script_base
+- scripts_base
 - apt_base
 - pip_base
 
-- script
+- scripts
 - apt
 - pip
-- script_post
+
+- pyproject_toml #reads from all pyproject.toml files
+
+- scripts_post
 
 If rocker is used to launch from a folder that contains multple projects with deps.yaml it will create a container to enable development of all of them combined together. 
 
+
 example deps.yaml
 
 ```
+
+env:
+  - BASIC_ENV_VAR=1
+
 apt_tools: #install basic development tools which almost never change
   - git
   - git-lfs
@@ -48,7 +58,3 @@ rocker --deps-dependencies ubuntu:22.04
 ## limitations/TODO
 
 This has only been tested on the ubuntu base image. It assumes you have access to apt-get.
-
-all the pip tags must have an entry for the moment.  Will improve the dockerfile logic to allow empty pip layers
-
-

deps.yaml

@@ -1,17 +1,14 @@
 env:
   - DEPS_ROCKER=1  
 
+scripts_tools:
+  - add_robotpkg.sh
+
 apt_tools:
   - git
   - git-lfs
   - python3-pip
 
 pip_tools:
   - flit
-  - pip #updates to latest pip
-
-scripts:
-  - scripts_base.sh
-
-pip:
-  - pip
+  - pip #updates to latest pip

deps_rocker/dependencies.py

@@ -12,6 +12,8 @@ class Dependencies(RockerExtension):
 
     def __init__(self) -> None:
         self.dependencies= defaultdict(set)
+        self.empy_args={}
+        self.all_files={}
         self.read_dependencies()
         print("dependencies dictionary")
         for k,v in self.dependencies.items():
@@ -50,41 +52,36 @@ def get_deps(self,key:str,as_list:bool = False)->str:
                 return dep_list
             return " ".join(dep_list)
         return ""
-
-    def get_pips_deps(self,key:str)->str:
-        pip_deps = self.get_deps(key)
-        print(pip_deps)
-
-        #todo remove this hack
-        if pip_deps =="":
-            pip_deps = "pip"
-        return pip_deps
+
 
     def get_files(self, cliargs)->dict:
-        all_files = {}
-
         self.read_dependencies()
-        #apt deps
-        deps_names = ["apt_tools","apt_base","apt"]
+        #apt and pip deps
+        deps_names = ["apt_tools","apt_base","apt","pip_tools","pip_base", "pip"]
         for dep in deps_names:
-            all_files[f"{dep}.deps"]= self.get_deps(dep)
+            self.add_file(dep, self.get_deps(dep))
 
-        #pip deps
-        deps_names = ["pip_tools","pip_base"]
-        for dep in deps_names:
-            all_files[f"{dep}.deps"]=self.get_pips_deps(dep) 
-
-        all_files["pip.deps"] =  self.get_pips_deps("pip") +" "+ self.get_pyproject_toml_deps()
+        #pyproject.toml deps
+        self.add_file("pyproject_toml", self.get_pyproject_toml_deps())
 
         #setup custom scripts
-        all_files["scripts_tools.sh"] = self.get_scripts("scripts_tools")
-        all_files["scripts_base.sh"] = self.get_scripts("scripts_base")
-        all_files["scripts.sh"] = self.get_scripts("scripts")
-        all_files["scripts_post.sh"] = self.get_scripts("scripts_post")
-
-        all_files["env_vars"] = self.get_deps("env")
+        for s in ["scripts_tools","scripts_base","scripts","scripts_post"]:
+            self.add_file(s,self.get_scripts(s))
+
+        return self.all_files
+
+    def add_file(self,filename:str,content:str)->None:
+        """Create a file on the users host machine to be copied into the docker context. This also updates the empy_args dictionary with True if that file is generated.  The empy_args are used to determine if a snipped related to that file should be generated or not.
 
-        return all_files
+        Args:
+            filename (str): name of file to create
+            content (str): the contents of the file
+        """
+        valid = len(content)>0
+        print(f"adding file {filename}, {valid}")
+        self.empy_args[filename] =valid
+        if valid:
+            self.all_files[filename] = content
 
     def get_scripts(self,name:str)->str:
         """collect all scripts files into a single script
@@ -102,7 +99,9 @@ def get_scripts(self,name:str)->str:
                 with open(s,encoding="utf-8") as f:
                     scripts.extend(f.readlines())
 
-        return "\n".join(scripts) 
+        if len(scripts)>1:
+            return "\n".join(scripts) 
+        return ""
 
 
     def get_pyproject_toml_deps(self)->str:
@@ -133,10 +132,13 @@ def get_snippet(self, cliargs):
             'deps_rocker',
             'templates/dependencies_snippet.Dockerfile').decode('utf-8')            
 
-        empy_args={}
-        empy_args["env_vars"] = self.get_deps("env",as_list=True)
-        print("empy_args",empy_args)
-        return em.expand(snippet,empy_args)
+        self.get_files(None)
+        self.empy_args["env_vars"] = self.get_deps("env",as_list=True)
+        print("all files")
+        for k,v in self.all_files.items():
+            print(k,v)
+        print("empy_args",self.empy_args)
+        return em.expand(snippet,self.empy_args)
 
 
     @staticmethod

deps_rocker/templates/dependencies_snippet.Dockerfile

@@ -1,44 +1,56 @@
+@# DEFINE EMPY MACROS FOR GENERATING DOCKERFILE
+
+@# DEFINE EMPY FUNCTION FOR RUNNING SCRIPTS
+@[def define_script(filename,file_exists)]@
+@[if file_exists]@
+COPY @filename /@filename
+RUN chmod +x /@filename; /@filename
+@[end if]@
+@[end def]@
+
+@# DEFINE EMPY FUNCTION FOR INSTALLING APT DEPS
+@[def define_apt_deps(filename,file_exists)]@
+@[if file_exists]@
+COPY @filename /@filename
+RUN apt-get update \ 
+ && apt-get install -y --no-install-recommends $(cat /@filename) \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+@[end if]@
+@[end def]@
+
+@# DEFINE EMPY FUNCTION FOR PIP INSTALLING
+@[def define_pip_install(filename,file_exists)]@
+@[if file_exists]@
+COPY @filename /@filename
+RUN pip3 install -U $(cat /@filename)
+@[end if]@
+@[end def]@
+
+@# END OF EMPY MACROS
+
+
 #SET UP ENVIRONMENT VARIABLES
 @[for x in env_vars]@
 ENV @x
 @[end for]@
 
 #INSTALL DEVELOPMENT TOOLS
-COPY scripts_tools.sh /scripts_tools.sh
-RUN chmod +x /scripts_tools.sh; /scripts_tools.sh
-
-COPY apt_tools.deps /apt_tools.deps
-RUN apt-get update \ 
- && apt-get install -y --no-install-recommends $(cat /apt_tools.deps) \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-COPY pip_tools.deps /pip_tools.deps
-RUN pip3 install -U $(cat /pip_tools.deps)
+@define_script("scripts_tools",scripts_tools)
+@define_apt_deps("apt_tools",apt_tools)
+@define_pip_install("pip_tools",pip_tools)
 
 #INSTALL EXPENSIVE BASE DEPENDENCIES
-COPY scripts_base.sh /scripts_base.sh
-RUN chmod +x /scripts_base.sh; /scripts_base.sh
-
-COPY apt_base.deps /apt_base.deps
-RUN apt-get update \ 
- && apt-get install -y --no-install-recommends $(cat /apt_base.deps) \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-COPY pip_base.deps /pip_base.deps
-RUN pip3 install -U $(cat /pip_base.deps)
+@define_script("scripts_base",scripts_base)
+@define_apt_deps("apt_base",apt_base)
+@define_pip_install("pip_base",pip_base)
 
 #INSTALL DEVELOPMENT DEPENDENCIES
-COPY scripts.sh /scripts.sh
-RUN chmod +x /scripts.sh; /scripts.sh
-
-COPY apt.deps /apt.deps
-RUN apt-get update \ 
- && apt-get install -y --no-install-recommends $(cat /apt.deps) \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
+@define_script("scripts",scripts)
+@define_apt_deps("apt",apt)
+@define_pip_install("pip",pip)
 
-COPY pip.deps /pip.deps
-RUN pip3 install -U $(cat /pip.deps)
+#INSTALL FROM PYPROJECT.TOML
+@define_pip_install("pyproject_toml",pyproject_toml)
 
 #POST SETUP
-COPY scripts_post.sh /scripts_post.sh
-RUN chmod +x /scripts_post.sh; /scripts_post.sh
+@define_script("scripts_post",scripts_post)