diff --git a/.copyright.hook b/.copyright.hook new file mode 100644 index 000000000..09afff207 --- /dev/null +++ b/.copyright.hook @@ -0,0 +1,121 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import io, re +import sys, os +import subprocess +import platform + +COPYRIGHT = ''' + Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +LANG_COMMENT_MARK = None + +NEW_LINE_MARK = None + +COPYRIGHT_HEADER = None + +if platform.system() == "Windows": + NEW_LINE_MARK = "\r\n" +else: + NEW_LINE_MARK = '\n' + COPYRIGHT_HEADER = COPYRIGHT.split(NEW_LINE_MARK)[1] + p = re.search('(\d{4})', COPYRIGHT_HEADER).group(0) + process = subprocess.Popen(["date", "+%Y"], stdout=subprocess.PIPE) + date, err = process.communicate() + date = date.decode("utf-8").rstrip("\n") + COPYRIGHT_HEADER = COPYRIGHT_HEADER.replace(p, date) + + +def generate_copyright(template, lang='C'): + if lang == 'Python': + LANG_COMMENT_MARK = '#' + else: + LANG_COMMENT_MARK = "//" + + lines = template.split(NEW_LINE_MARK) + BLANK = " " + ans = LANG_COMMENT_MARK + BLANK + COPYRIGHT_HEADER + NEW_LINE_MARK + for lino, line in enumerate(lines): + if lino == 0 or lino == 1 or lino == len(lines) - 1: continue + if len(line) == 0: + BLANK = "" + else: + BLANK = " " + ans += LANG_COMMENT_MARK + BLANK + line + NEW_LINE_MARK + + return ans + "\n" + + +def lang_type(filename): + if filename.endswith(".py"): + return "Python" + elif filename.endswith(".h"): + return "C" + elif filename.endswith(".c"): + return "C" + elif filename.endswith(".hpp"): + return "C" + elif filename.endswith(".cc"): + return "C" + elif filename.endswith(".cpp"): + return "C" + elif filename.endswith(".cu"): + return "C" + elif filename.endswith(".cuh"): + return "C" + elif filename.endswith(".go"): + return "C" + elif filename.endswith(".proto"): + return "C" + else: + print("Unsupported filetype %s", filename) + exit(0) + + +PYTHON_ENCODE = re.compile("^[ \t\v]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Checker for copyright declaration.') + parser.add_argument('filenames', nargs='*', help='Filenames to check') + args = parser.parse_args(argv) + + retv = 0 + for filename in args.filenames: + fd = io.open(filename, encoding="utf-8") + first_line = fd.readline() + second_line = fd.readline() + if "COPYRIGHT (C)" in first_line.upper(): continue + if first_line.startswith("#!") or PYTHON_ENCODE.match( + second_line) != None or PYTHON_ENCODE.match(first_line) != None: + continue + original_contents = io.open(filename, encoding="utf-8").read() + new_contents = generate_copyright( + COPYRIGHT, lang_type(filename)) + original_contents + print('Auto Insert Copyright Header {}'.format(filename)) + retv = 1 + with io.open(filename, 'w') as output_file: + output_file.write(new_contents) + + return retv + + +if __name__ == '__main__': + exit(main()) diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..bd156ddda --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.DS_Store + +*~ + +*.pyc +.pydevproject diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..29d28d677 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +repos: +- repo: https://github.com/PaddlePaddle/mirrors-yapf.git + sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37 + hooks: + - id: yapf + files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ +- repo: https://github.com/pre-commit/pre-commit-hooks + sha: 5bf6c09bfa1297d3692cadd621ef95f1284e33c0 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + files: (?!.*paddle)^.*$ + - id: end-of-file-fixer + files: \.md$ + - id: trailing-whitespace + files: \.md$ +- repo: https://github.com/Lucas-C/pre-commit-hooks + sha: v1.0.1 + hooks: + - id: forbid-crlf + files: \.md$ + - id: remove-crlf + files: \.md$ + - id: forbid-tabs + files: \.md$ + - id: remove-tabs + files: \.md$ +- repo: local + hooks: + - id: copyright_checker + name: copyright_checker + entry: python ./.copyright.hook + language: system + files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$ diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 000000000..4741fb4f3 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,3 @@ +[style] +based_on_style = pep8 +column_limit = 80 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..75c2c1351 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,34 @@ +language: cpp +cache: ccache +sudo: required +dist: trusty +services: + - docker +os: + - linux +env: + - JOB=PRE_COMMIT + +addons: + apt: + packages: + - git + - python + - python-pip + - python2.7-dev + +before_install: + - sudo pip install -U virtualenv pre-commit pip + - docker pull paddlepaddle/paddle:latest + +script: + - exit_code=0 + - .travis/precommit.sh || exit_code=$(( exit_code | $? )) + - docker run -i --rm -v "$PWD:/py_unittest" paddlepaddle/paddle:latest /bin/bash -c + 'cd /py_unittest; sh .travis/unittest.sh' || exit_code=$(( exit_code | $? )) + exit $exit_code + +notifications: + email: + on_success: change + on_failure: always diff --git a/.travis/precommit.sh b/.travis/precommit.sh new file mode 100755 index 000000000..bcbfb2bb5 --- /dev/null +++ b/.travis/precommit.sh @@ -0,0 +1,21 @@ +#!/bin/bash +function abort(){ + echo "Your commit not fit PaddlePaddle code style" 1>&2 + echo "Please use pre-commit scripts to auto-format your code" 1>&2 + exit 1 +} + +trap 'abort' 0 +set -e +cd `dirname $0` +cd .. +export PATH=/usr/bin:$PATH +pre-commit install + +if ! pre-commit run -a ; then + ls -lh + git diff --exit-code + exit 1 +fi + +trap : 0 diff --git a/.travis/unittest.sh b/.travis/unittest.sh new file mode 100755 index 000000000..f27dc481a --- /dev/null +++ b/.travis/unittest.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +abort(){ + echo "Run unittest failed" 1>&2 + echo "Please check your code" 1>&2 + exit 1 +} + +unittest(){ + cd $1 > /dev/null + if [ -f "setup.sh" ]; then + sh setup.sh + export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + fi + if [ $? != 0 ]; then + exit 1 + fi + find . -name 'tests' -type d -print0 | \ + xargs -0 -I{} -n1 bash -c \ + 'python -m unittest discover -v -s {}' + cd - > /dev/null +} + +trap 'abort' 0 +set -e + +unittest . + +trap : 0 diff --git a/convert.py b/convert.py index bebb0acfc..dd09b1490 100644 --- a/convert.py +++ b/convert.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,14 +30,16 @@ def convert(dirname): inference_scope = fluid.core.Scope() with fluid.scope_guard(inference_scope): [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(dirname, exe) + fetch_targets] = fluid.io.load_inference_model(dirname, exe) # Using blocks in programs, create nodes using: onnx_nodes = [] all_inputs = [] for block in inference_program.blocks: - all_inputs += [paddle_variable_to_onnx_tensor( - v, block) for v in block.vars if v not in ['feed', 'fetch']] + all_inputs += [ + paddle_variable_to_onnx_tensor(v, block) for v in block.vars + if v not in ['feed', 'fetch'] + ] for op in block.ops: if op.type in ops.PADDLE_TO_ONNX: @@ -45,8 +47,8 @@ def convert(dirname): # TODO(varunarora): Use the modifier function to make the # transformation. node_proto = helper.make_node( - ops.PADDLE_TO_ONNX[op.type][0], - op.input_arg_names, op.output_arg_names) + ops.PADDLE_TO_ONNX[op.type][0], op.input_arg_names, + op.output_arg_names) onnx_nodes.append(node_proto) else: @@ -59,8 +61,9 @@ def convert(dirname): # Nodes, name of graph, inputs, outputs. if dirname[-1] == '/': dirname = dirname[:-1] - graph = helper.make_graph(onnx_nodes, os.path.basename( - dirname).split('.')[0], all_inputs, []) + graph = helper.make_graph(onnx_nodes, + os.path.basename(dirname).split('.')[0], + all_inputs, []) print graph @@ -70,7 +73,7 @@ def convert(dirname): if __name__ == "__main__": # Read arguments: path to model. parser = argparse.ArgumentParser() - parser.add_argument("--modeldir", required=True, - help="Input PaddlePaddle model") + parser.add_argument( + "--modeldir", required=True, help="Input PaddlePaddle model") args = parser.parse_args() convert(args.modeldir) diff --git a/ops.py b/ops.py index 8c6785166..bba0b7da3 100644 --- a/ops.py +++ b/ops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Priority of ops (uniques) to figure out support for. @@ -460,7 +459,6 @@ def xor_op(): # 'Ceil', NEEDS ATTENTION. 'cast': ('Clip', clip_op), 'concat': ('Concat', concat_op), - ',': ('Constant', constant_op), 'conv': ('Conv', conv_op), diff --git a/setup.sh b/setup.sh new file mode 100644 index 000000000..8427d6404 --- /dev/null +++ b/setup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# install python dependencies +if [ -f "requirements.txt" ]; then + pip install -r requirements.txt +fi +if [ $? != 0 ]; then + echo "Install python dependencies failed !!!" + exit 1 +fi diff --git a/variables.py b/variables.py index d337166dd..a87b21547 100644 --- a/variables.py +++ b/variables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ from onnx import helper, onnx_pb2, TensorProto import paddle.fluid.core as core + def paddle_variable_to_onnx_tensor(paddle_var_name, block): # TODO(varunarora): Need to do this only in the case of VarType.LOD_TENSOR. paddle_var = block.var(paddle_var_name) @@ -36,5 +37,5 @@ def paddle_variable_to_onnx_tensor(paddle_var_name, block): # '': onnx_pb2.TensorProto.STRING, # '': onnx_pb2.TensorProto.COMPLEX64, # '': onnx_pb2.TensorProto.COMPLEX128, - core.VarDesc.VarType.BOOL: onnx_pb2.TensorProto.BOOL, + core.VarDesc.VarType.BOOL: onnx_pb2.TensorProto.BOOL }