fix file preambles for Python scripts (#357)

* fix file preambles for Python scripts

* respond to review comments

integration-tests/test_184.py

@@ -1,3 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 import sys
 import time
 

integration-tests/test_207.py

@@ -1,3 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 import os
 import sys
 import tempfile

integration-tests/test_209.py

@@ -1,3 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
+import io
 import json
 import logging
 import os

integration-tests/test_hdfs.py

@@ -1,7 +1,14 @@
+# -*- coding: utf-8 -*-
 #
-# Sample code for HDFS integration tests.
-# Requires hadoop to be running on localhost, at the moment.
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
+"""
+Sample code for HDFS integration tests.
+Requires hadoop to be running on localhost, at the moment.
+"""
 import smart_open
 
 with smart_open.smart_open("hdfs://user/root/input/core-site.xml") as fin:

integration-tests/test_http.py

@@ -1,6 +1,10 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 from __future__ import unicode_literals
 
 import logging

integration-tests/test_minio.py

@@ -1,3 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 import logging
 import boto3
 

integration-tests/test_s3.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 
 from __future__ import unicode_literals
 import io

integration-tests/test_webhdfs.py

@@ -1,7 +1,14 @@
+# -*- coding: utf-8 -*-
 #
-# Sample code for WebHDFS integration tests.
-# Requires hadoop to be running on localhost, at the moment.
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
+"""
+Sample code for WebHDFS integration tests.
+Requires hadoop to be running on localhost, at the moment.
+"""
 import smart_open
 
 with smart_open.smart_open("webhdfs://localhost:50070/user/root/input/core-site.xml") as fin:

release/check_preamble.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
+
+"""Checks preambles of Python script files.
+
+We want to ensure they all contain the appropriate license and copyright.
+
+For the purposes of this script, the *preamble* is defined as the first
+lines of the file starting with a hash (#).  Any line that does not start
+with a hash ends the preamble.
+
+Usage::
+
+    python check_preamble.py --replace /path/to/template.py script.py
+
+The above command reads the preamble from ``template.py``, and then copies
+that preamble into ``script.py``.  If ``script.py`` already contains a
+preamble, then the existing preamble will be replaced **entirely**.
+
+Processing entire subdirectories with one command::
+
+    find subdir1 subdir2 -iname "*.py" | xargs -n 1 python check_preamble.py --replace template.py
+
+"""
+import argparse
+import logging
+import os
+import sys
+
+
+def extract_preamble(fin):
+    end_preamble = False
+    preamble, body = [], []
+
+    for line in fin:
+        if end_preamble:
+            body.append(line)
+        elif line.startswith('#'):
+            preamble.append(line)
+        else:
+            end_preamble = True
+            body.append(line)
+
+    return preamble, body
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('path', help='the path of the file to check')
+    parser.add_argument('--replace', help='replace the preamble with the one from this file')
+    parser.add_argument('--loglevel', default=logging.INFO)
+    args = parser.parse_args()
+
+    logging.basicConfig(level=args.loglevel)
+
+    with open(args.path) as fin:
+        preamble, body = extract_preamble(fin)
+
+    for line in preamble:
+        logging.info('%s: %s', args.path, line.rstrip())
+
+    if not args.replace:
+        sys.exit(0)
+
+    with open(args.replace) as fin:
+        preamble, _ = extract_preamble(fin)
+
+    if os.access(args.path, os.X_OK):
+        preamble.insert(0, '#!/usr/bin/env python\n')
+
+    with open(args.path, 'w') as fout:
+        for line in preamble + body:
+            fout.write(line)
+
+
+if __name__ == '__main__':
+    main()

smart_open/__init__.py

@@ -1,4 +1,3 @@
-#
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>

smart_open/bytebuffer.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 """Implements ByteBuffer class for amortizing network transfer overhead."""
 
 import io

smart_open/doctools.py

@@ -1,9 +1,9 @@
-#
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
-# This code is distributed under the terms and conditions from the MIT License (MIT).
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
 #
 
 """Common functions for working with docstrings.

smart_open/hdfs.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
-# This code is distributed under the terms and conditions from the MIT License (MIT).
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
 #
 
 """Implements reading and writing to/from HDFS.

smart_open/http.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 """Implements file-like objects for reading from http."""
 
 import io

smart_open/s3.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 """Implements file-like objects for reading and writing from/to S3."""
 
 import io

smart_open/smart_open_lib.py

@@ -1,7 +1,6 @@
-#
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2015 Radim Rehurek <me@radimrehurek.com>
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
 # This code is distributed under the terms and conditions
 # from the MIT License (MIT).

smart_open/ssh.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
-# This code is distributed under the terms and conditions from the MIT License (MIT).
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
 #
 
 """Implements I/O streams over SSH.

smart_open/tests/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#

smart_open/tests/test_bytebuffer.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 import random
 import unittest
 

smart_open/tests/test_hdfs.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 from __future__ import print_function
 from __future__ import unicode_literals
 

smart_open/tests/test_http.py

@@ -1,3 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 import unittest
 
 import responses

smart_open/tests/test_s3.py

@@ -1,4 +1,10 @@
 # -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
+#
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
+#
 import gzip
 import io
 import logging

smart_open/tests/test_smart_open.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2015 Radim Rehurek <me@radimrehurek.com>
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
 # This code is distributed under the terms and conditions
 # from the MIT License (MIT).
+#
 
 import bz2
 import io

smart_open/tests/test_smart_open_old.py

@@ -1,15 +1,16 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2015 Radim Rehurek <me@radimrehurek.com>
+# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
 # This code is distributed under the terms and conditions
 # from the MIT License (MIT).
 #
-# These are tests that test the deprecated smart_open.smart_open function.
-# They mostly duplicate tests in test_smart_open.py and are here to guarantee
-# backwards compatibility.
-#
+"""
+These are tests that test the deprecated smart_open.smart_open function.
+They mostly duplicate tests in test_smart_open.py and are here to guarantee
+backwards compatibility.
+"""
+
 
 import io
 import logging

smart_open/webhdfs.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
 #
-# This code is distributed under the terms and conditions from the MIT License (MIT).
+# This code is distributed under the terms and conditions
+# from the MIT License (MIT).
 #
 
 """Implements reading and writing to/from WebHDFS.