Add SkillEntry.from_directory

ovos_skills_manager/appstores/local.py

@@ -1,21 +1,16 @@
+from os import listdir
+from os.path import join, isdir
+from typing import Optional
+
+from ovos_utils.skills import get_skills_folder
+
+from ovos_skills_manager.local_skill import get_skill_data_from_directory
 from ovos_skills_manager.skill_entry import SkillEntry
 from ovos_skills_manager.appstores import AbstractAppstore
-from ovos_skills_manager.licenses import parse_license_type
-from ovos_skills_manager.github.utils import GITHUB_README_FILES, \
-    GITHUB_ICON_FILES, GITHUB_JSON_FILES, GITHUB_DESKTOP_FILES, \
-    GITHUB_LOGO_FILES, GITHUB_REQUIREMENTS_FILES, \
-    GITHUB_SKILL_REQUIREMENTS_FILES, GITHUB_LICENSE_FILES, \
-    GITHUB_MANIFEST_FILES, author_repo_from_github_url
-from ovos_skills_manager.utils import readme_to_json
-from ovos_utils.skills import get_skills_folder
-from ovos_skills_manager.requirements import validate_manifest
-from ovos_utils.json_helper import merge_dict
-import json
-from os import listdir, walk
-from os.path import join, isdir, isfile
 
 
-def get_local_skills(parse_github:bool=False, skiplist=None):
+def get_local_skills(parse_github: bool = False,
+                     skiplist: Optional[list] = None):
     try:
         skills = get_skills_folder()
     except FileNotFoundError:
@@ -30,72 +25,8 @@ def get_local_skills(parse_github:bool=False, skiplist=None):
         if not isdir(path) or fold in skiplist:
             continue
 
-        skill = {
-            "appstore": "InstalledSkills",
-            "appstore_url": skills,
-            "skill_id": fold,
-            "foldername": fold,
-            "requirements": {"python": [], "system": [], "skill": []}
-        }
-
-        # if installed by msm/osm will obey this convention
-        if "." in fold:
-            try:
-                repo, author = fold.split(".")
-                skill["skillname"] = repo
-                skill["authorname"] = author
-                skill["url"] = f'https://github.com/{author}/{repo}'
-            except: # TODO replace with some clever check ?
-                pass
-
-        # parse git info
-        gitinfo = join(path, ".git/config")
-        if isfile(gitinfo):
-            with open(gitinfo) as f:
-                for l in f.readlines():
-                    if l.strip().startswith("url ="):
-                        skill["url"] = l.split("url =")[-1].strip()
-                        skill["authorname"], skill["skillname"] = \
-                            author_repo_from_github_url(skill["url"])
-                    if l.strip().startswith("[branch "):
-                        skill["branch"] = l.split("branch")[-1]\
-                            .replace('"', "").strip()
-
-        for rtdir, foldrs, files in walk(join(skills, fold)):
-            for f in files:
-                if f in GITHUB_JSON_FILES:
-                    with open(join(rtdir, f)) as fi:
-                        skill_meta = json.load(fi)
-                    skill = merge_dict(skill, skill_meta, merge_lists=True)
-                elif f in GITHUB_README_FILES:
-                    with open(join(rtdir, f)) as fi:
-                        readme = readme_to_json(fi.read())
-                    skill = merge_dict(skill, readme,
-                                       new_only=True, merge_lists=True)
-                elif f in GITHUB_DESKTOP_FILES:
-                    skill['desktopFile'] = True
-                elif f in GITHUB_ICON_FILES:
-                    skill["icon"] = join(rtdir, f)
-                elif f in GITHUB_LICENSE_FILES:
-                    with open(join(rtdir, f)) as fi:
-                        lic = fi.read()
-                    skill["license"] = parse_license_type(lic)
-                elif f in GITHUB_LOGO_FILES:
-                    skill["logo"] = join(rtdir, f)
-                elif f in GITHUB_MANIFEST_FILES:
-                    with open(join(rtdir, f)) as fi:
-                        manifest = validate_manifest(fi.read())
-                    skill["requirements"]["python"] += manifest.get("python") or []
-                    skill["requirements"]["system"] += manifest.get("system") or []
-                    skill["requirements"]["skill"] += manifest.get("skill") or []
-                elif f in GITHUB_REQUIREMENTS_FILES:
-                    with open(join(rtdir, f)) as fi:
-                        reqs = [r for r in fi.read().split("\n") if r.strip()]
-                    skill["requirements"]["python"] += reqs
-                elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
-                    with open(join(rtdir, f)) as fi:
-                        reqs = [r for r in fi.read().split("\n") if r.strip()]
-                    skill["requirements"]["skill"] += reqs
+        skill_dir = join(skills, fold)
+        skill = get_skill_data_from_directory(skill_dir)
         yield SkillEntry.from_json(skill, parse_github=parse_github)
 
 

ovos_skills_manager/github/utils.py

@@ -118,7 +118,7 @@ def validate_branch(branch:str, url:str):
     return requests.get(url).status_code == 200
 
 
-def download_url_from_github_url(url:str, branch:str=None):
+def download_url_from_github_url(url: str, branch: str = None):
     # specific file
     try:
         url = blob2raw(url)

ovos_skills_manager/local_skill/__init__.py

@@ -0,0 +1,111 @@
+import json
+
+from os import walk
+from os.path import join, isfile
+
+from ovos_utils.json_helper import merge_dict
+
+from ovos_skills_manager.licenses import parse_license_type
+from ovos_skills_manager.requirements import validate_manifest
+from ovos_skills_manager.github.utils import (
+    GITHUB_README_FILES,
+    GITHUB_JSON_FILES,
+    GITHUB_DESKTOP_FILES,
+    GITHUB_ICON_FILES,
+    GITHUB_LICENSE_FILES,
+    GITHUB_LOGO_FILES,
+    GITHUB_REQUIREMENTS_FILES,
+    GITHUB_SKILL_REQUIREMENTS_FILES,
+    GITHUB_MANIFEST_FILES, author_repo_from_github_url
+)
+from ovos_skills_manager.utils import readme_to_json
+
+
+def get_skill_data_from_directory(skill_dir: str):
+    """
+    Parse the specified skill directory and return a dict representation of a
+    SkillEntry.
+    @param skill_dir: path to skill directory
+    @return: dict parsed skill data
+    """
+    skills, fold = skill_dir.rsplit('/', 1)
+    skill_data = {
+        "appstore": "InstalledSkills",
+        "appstore_url": skills,
+        "skill_id": fold,
+        "requirements": {"python": [], "system": {}, "skill": []}
+    }
+
+    # if installed by msm/osm will obey this convention
+    if "." in fold:
+        try:
+            repo, author = fold.split(".")
+            skill_data["skillname"] = repo
+            skill_data["authorname"] = author
+            skill_data["url"] = f'https://github.com/{author}/{repo}'
+        except:  # TODO replace with some clever check ?
+            pass
+
+    # parse git info
+    gitinfo = join(skill_dir, ".git/config")
+    if isfile(gitinfo):
+        with open(gitinfo) as f:
+            for l in f.readlines():
+                if l.strip().startswith("url ="):
+                    skill_data["url"] = l.split("url =")[-1].strip()
+                    skill_data["authorname"], skill_data["skillname"] = \
+                        author_repo_from_github_url(skill_data["url"])
+                if l.strip().startswith("[branch "):
+                    skill_data["branch"] = l.split("branch")[-1] \
+                        .replace('"', "").strip()
+
+    # parse skill files
+    for root_dir, _, files in walk(skill_dir):
+        for f in files:
+            if f in GITHUB_JSON_FILES:  # skill.json
+                with open(join(root_dir, f)) as fi:
+                    skill_meta = json.load(fi)
+                skill_data = merge_dict(skill_data, skill_meta,
+                                        merge_lists=True)
+            elif f in GITHUB_README_FILES:
+                with open(join(root_dir, f)) as fi:
+                    readme = readme_to_json(fi.read())
+                skill_data = merge_dict(skill_data, readme,
+                                        new_only=True, merge_lists=True)
+            elif f in GITHUB_DESKTOP_FILES:
+                skill_data['desktopFile'] = True
+            elif f in GITHUB_ICON_FILES:
+                skill_data["icon"] = join(root_dir, f)
+            elif f in GITHUB_LICENSE_FILES:
+                with open(join(root_dir, f)) as fi:
+                    lic = fi.read()
+                skill_data["license"] = parse_license_type(lic)
+            elif f in GITHUB_LOGO_FILES:
+                skill_data["logo"] = join(root_dir, f)
+            elif f in GITHUB_MANIFEST_FILES:
+                with open(join(root_dir, f)) as fi:
+                    manifest = validate_manifest(fi.read()).get("dependencies", {})
+                skill_data["requirements"]["python"] += \
+                    manifest.get("python") or []
+                skill_data["requirements"]["system"] = \
+                    merge_dict(skill_data["requirements"]["system"],
+                               manifest.get("system") or {}, merge_lists=True)
+
+                skill_data["requirements"]["skill"] += \
+                    manifest.get("skill") or []
+            elif f in GITHUB_REQUIREMENTS_FILES:
+                with open(join(root_dir, f)) as fi:
+                    reqs = [r for r in fi.read().split("\n") if r.strip()]
+                skill_data["requirements"]["python"] += reqs
+            elif f in GITHUB_SKILL_REQUIREMENTS_FILES:
+                with open(join(root_dir, f)) as fi:
+                    reqs = [r for r in fi.read().split("\n") if r.strip()]
+                skill_data["requirements"]["skill"] += reqs
+    # de-dupe requirements
+    skill_data["requirements"]["python"] = \
+        list(set(skill_data["requirements"]["python"]))
+    skill_data["requirements"]["skill"] = \
+        list(set(skill_data["requirements"]["skill"]))
+    skill_data['foldername'] = fold  # Override what the config specifies
+    skill_data['authorname'] = skill_data.get('authorname') or "local"
+    return skill_data

ovos_skills_manager/skill_entry.py

@@ -5,6 +5,7 @@
 from os.path import isfile, exists, expanduser, join, isdir
 from typing import Optional, Union
 
+from ovos_skills_manager.local_skill import get_skill_data_from_directory
 from ovos_skills_manager.session import SESSION as requests
 from ovos_skills_manager.exceptions import GithubInvalidUrl, \
     JSONDecodeError, GithubFileNotFound, SkillEntryError, GithubInvalidBranch
@@ -52,7 +53,7 @@ def json(self):
 
     # constructors
     @staticmethod
-    def from_json(data: Union[str, dict], parse_github:bool=True):
+    def from_json(data: Union[str, dict], parse_github: bool = True):
         if isinstance(data, str):
             if data.startswith("http"):
                 url = data
@@ -90,7 +91,7 @@ def from_json(data: Union[str, dict], parse_github:bool=True):
         return SkillEntry(data)
 
     @staticmethod
-    def from_github_url(url, branch:str=None, parse_github:bool=True):
+    def from_github_url(url, branch: str = None, parse_github: bool = True):
         if not branch:
             try:
                 branch = get_branch_from_github_url(url)
@@ -100,10 +101,27 @@ def from_github_url(url, branch:str=None, parse_github:bool=True):
         return SkillEntry.from_json({"url": url, "branch": branch},
                                     parse_github=parse_github)
 
+    @staticmethod
+    def from_directory(skill_dir: str, github_token: Optional[str] = None):
+        """
+        Build a SkillEntry for a local skill directory
+        @param skill_dir: path to skill
+        @param github_token: optional Github token for private dependencies
+        @return: SkillEntry representation of the specified skill
+        """
+        skill_dir = expanduser(skill_dir)
+        if not isdir(skill_dir):
+            raise ValueError(f"{skill_dir} is not a valid directory")
+
+        data = get_skill_data_from_directory(skill_dir)
+        parse_python_dependencies(data["requirements"].get("python"),
+                                  github_token)
+        return SkillEntry.from_json(data, False)
+
     # properties
     @property
-    def url(self):
-        return self.json.get("url")
+    def url(self) -> str:
+        return self.json.get("url") or ""
 
     @property
     def appstore(self):
@@ -146,7 +164,7 @@ def skill_icon(self):
 
     @property
     def skill_author(self):
-        return self.json.get("authorname") or self.url.split("/")[-2] if self.url and "/" in self.url else ""
+        return self.json.get("authorname") or (self.url.split("/")[-2] if self.url and "/" in self.url else "")
 
     @property
     def skill_tags(self):
@@ -163,17 +181,23 @@ def homescreen_msg(self):
                                       author=self.skill_author)
 
     @property
-    def branch(self):
-        return self.json.get("branch") or get_branch(self.url)
+    def branch(self) -> str:
+        try:
+            return self.json.get("branch") or get_branch(self.url)
+        except GithubInvalidUrl:
+            return ""
 
     @property
     def branch_overrides(self):
         return self.json.get("branch_overrides") or {}
 
     @property
-    def download_url(self):
+    def download_url(self) -> str:
         """ generated from github url directly"""
-        return download_url_from_github_url(self.url, self.branch)
+        try:
+            return download_url_from_github_url(self.url, self.branch)
+        except GithubInvalidUrl:
+            return ""
 
     @property
     def default_download_url(self):

setup.py

@@ -13,7 +13,8 @@ def _get_version():
               'ovos_skills_manager.github',
               'ovos_skills_manager.appstores',
               'ovos_skills_manager.scripts',
-              'ovos_skills_manager.versioning'],
+              'ovos_skills_manager.versioning',
+              'ovos_skills_manager.local_skill'],
     url='https://github.com/OpenVoiceOS/ovos_skill_manager',
     license='Apache-2.0',
     author='JarbasAI',

tests/skill_dirs/tskill-osm_parsing-complete/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2021, Daniel McKnight
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

tests/skill_dirs/tskill-osm_parsing-complete/README.md

@@ -0,0 +1,34 @@
+# <img src='https://0000.us/klatchat/app/files/neon_images/icons/neon_skill.png' card_color="#FF8600" width="50" style="vertical-align:bottom">OSM Test Skill
+
+## Summary
+
+Skill used to test OSM Parsing
+
+## Requirements
+
+datetime
+
+## Description
+
+This is a skill description
+
+## Examples
+
+Here are some examples
+
+- "Do something cool."
+
+## Category
+**Daily**
+Productivity
+
+## Credits
+@neongeckocom
+@neondaniel
+@reginaneon
+
+## Tags
+#NeonGecko
+#OVOS
+#Test
+#NotARealSkill