update license and cluster outmapping

.gitignore

@@ -1,4 +1,11 @@
 *py[cod]
+*.c
+__pycache__/
 example/
-*job
+*.job
 *log
+tmp/
+build/
+test/
+dist/
+*.egg-info/

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022~2023 yodeng <yougdeng@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

runjob/__init__.py

@@ -0,0 +1 @@
+from ._version import __version__

setup.py

@@ -1,46 +1,124 @@
 import os
+import sys
+import sysconfig
 
 from setuptools import setup
-from src.version import __version__
-
-
-def getdes():
-    des = ""
-    with open(os.path.join(os.getcwd(), "README.md")) as fi:
-        des = fi.read()
-    return des
-
-
-def listdir(path):
-    df = []
-    for a, b, c in os.walk(path):
-        df.append((a, [os.path.join(a, i) for i in c]))
-    return df
-
-
-setup(
-    name="runjob",
-    version=__version__,
-    packages=["runjob"],
-    package_data={"runjob": ["*config", "*.json"]},
-    package_dir={"runjob": "src"},
-    author="Deng Yong",
-    author_email="yodeng@tju.edu.cn",
-    url="https://github.com/yodeng/runjob",
-    license="BSD",
-    install_requires=["psutil", "ratelimiter",
-                      "batchcompute", "prettytable", "configparser"],
-    python_requires='>=2.7.10, <3.10',
-    long_description=getdes(),
-    long_description_content_type='text/markdown',
-    entry_points={
-        'console_scripts': [
-            'runjob = runjob.run:main',
-            'qs = runjob.jobstat:main',
-            'qcs = runjob.jobstat:batchStat',
-            'runsge = runjob.sge_run:main',
-            'runbatch = runjob.sge_run:main',
-            'runsge0 = runjob.sge:main',
+from functools import partial
+from setuptools.extension import Extension
+
+
+class Packages(object):
+
+    def __init__(self, pkg_name=""):
+        self.name = pkg_name
+        self.base_dir = os.path.dirname(__file__)
+        basejoin = partial(self._join, self.base_dir)
+        self.source_dir = basejoin("src")
+        self.version_file = basejoin("src/_version.py")
+        self.des_file = basejoin("README.md")
+        self.req_file = basejoin("requirements.txt")
+
+    def _join(self, *args):
+        return os.path.join(*args)
+
+    @property
+    def listdir(self):
+        df, nc = [], []
+        if os.path.isdir(self.name):
+            nc = os.listdir(self.name)
+        for a, b, c in os.walk(self.source_dir):
+            if os.path.basename(a).startswith("__"):
+                continue
+            for i in c:
+                if i.startswith("__") or i.endswith(".ini") or not i.endswith(".py") or i in nc:
+                    continue
+                p = os.path.join(a[len(self.source_dir)+1:], i)
+                df.append(p)
+        return df
+
+    @property
+    def description(self):
+        des = ""
+        with open(self.des_file) as fi:
+            des = fi.read()
+        return des
+
+    @property
+    def version(self):
+        v = {}
+        with open(self.version_file) as fi:
+            c = fi.read()
+        exec(compile(c, self.version_file, "exec"), v)
+        return v["__version__"]
+
+    @property
+    def requirements(self):
+        requires = []
+        with open(self.req_file) as fi:
+            for line in fi:
+                line = line.strip()
+                requires.append(line)
+        return requires
+
+    @property
+    def _extensions(self):
+        exts = []
+        for f in self.listdir:
+            e = Extension(self.name + "." + os.path.splitext(f)[0].replace("/", "."),
+                          [os.path.join(self.source_dir, f), ], extra_compile_args=["-O3", ],)
+            e.cython_directives = {
+                'language_level': sysconfig._PY_VERSION_SHORT_NO_DOT[0]}
+            exts.append(e)
+        return exts
+
+    @property
+    def _package_dir(self):
+        pd = {}
+        for a, b, v in os.walk(self.source_dir):
+            p = a.replace(self.source_dir, self.name).replace("/", ".")
+            pd[p] = a.replace(self.source_dir, "src")
+        return pd
+
+    def install(self, ext=False):
+        kwargs = {}
+        kwargs.update(
+            name=self.name,
+            version=self.version,
+            license="MIT",
+            packages=list(self._package_dir.keys()),
+            package_dir=self._package_dir,
+            package_data={self.name: ["*config", "*.json"], },
+            install_requires=self.requirements,
+            python_requires='>=2.7.10, <3.10',
+            long_description=self.description,
+            long_description_content_type='text/markdown',
+            entry_points={'console_scripts': self._entrys},
+            author="yodeng",
+            author_email="yodeng@tju.edu.cn",
+            url="https://github.com/yodeng/runjob",
+        )
+        if ext:
+            kwargs.pop("package_dir")
+            kwargs["ext_modules"] = self._extensions
+        setup(**kwargs)
+
+    @property
+    def _entrys(self):
+        eps = [
+            '%s = %s.run:main' % (self.name, self.name),
+            '%s = %s.jobstat:main' % ("qs", self.name),
+            '%s = %s.jobstat:batchStat' % ("qcs", self.name),
+            '%s = %s.sge_run:main' % ("runsge", self.name),
+            '%s = %s.sge_run:main' % ("runbatch", self.name),
+            '%s = %s.sge:main' % ("runsge0", self.name),
         ]
-    }
-)
+        return eps
+
+
+def main():
+    pkgs = Packages("runjob")
+    pkgs.install(ext=False)
+
+
+if __name__ == "__main__":
+    main()

src/__init__.py

@@ -1 +1 @@
-from .version import __version__
+from ._version import __version__

src/cluster.py

@@ -130,7 +130,7 @@ def AddOneTask(self, job=None, outdir=""):
         self.client = self.cluster.client
         job.client = self.client
 
-    def modifyTaskOutMapping(self, mapping=""):
+    def modifyTaskOutMapping(self, job=None, mapping=""):
         if mapping:
             if self.cluster.oss_mount_entry["Destination"] in mapping:
                 # if not mapping.endswith("/"):
@@ -142,7 +142,7 @@ def modifyTaskOutMapping(self, mapping=""):
                     "^"+des, src, mapping)
             else:
                 self.cluster.conf.logger.debug(
-                    "Only oss mount path support(%s), %s", outdir, job.name)
+                    "Only oss mount path support(%s), %s", mapping, job.name)
 
     def get_instance_type(self, cpu=2, mem=4):
         e = []

src/jobstat.py

@@ -22,7 +22,7 @@
 from .bc_stat import *
 from .cluster import *
 from .job import Jobfile
-from .version import __version__
+from ._version import __version__
 from .config import load_config, print_config
 
 

src/run.py

@@ -16,7 +16,7 @@
 
 from .utils import *
 from .qsub import qsub
-from .version import __version__
+from ._version import __version__
 
 
 class ParseSingal(Thread):

src/sge_run.py

@@ -16,7 +16,7 @@
 from .utils import *
 from .cluster import *
 from .sge import ParseSingal
-from .version import __version__
+from ._version import __version__
 from .qsub import myQueue, QsubError
 from .config import load_config, print_config
 
@@ -368,7 +368,7 @@ def submit(self, job):
                 task.AddOneTask(
                     job=job, outdir=self.conf.get("args", "out_maping"))
                 if job.out_maping:
-                    task.modifyTaskOutMapping(mapping=job.out_maping)
+                    task.modifyTaskOutMapping(job=job, mapping=job.out_maping)
                 task.Submit()
                 info = "Your job (%s) has been submitted in batchcompute (%s) %d times\n" % (
                     task.name, task.id, job.subtimes+1)

src/utils.py

@@ -10,7 +10,7 @@
 from subprocess import call, PIPE
 from ratelimiter import RateLimiter
 
-from .version import __version__
+from ._version import __version__
 
 if sys.version_info[0] < 3:
     from Queue import Queue