Add kfctl third party licenses

Dockerfile

@@ -1,6 +1,5 @@
 #**********************************************************************
 # Builder
-# 
 # Create a go runtime suitable for building and testing kfctl
 ARG GOLANG_VERSION=1.12.7
 FROM golang:$GOLANG_VERSION as builder
@@ -66,5 +65,8 @@ WORKDIR /opt/kubeflow
 FROM barebones_base as kfctl
 
 COPY --from=kfctl_base /go/src/github.com/kubeflow/kfctl/bin/kfctl /usr/local/bin
+COPY --from=kfctl_base /go/src/github.com/kubeflow/kfctl/third_party /third_party
+COPY --from=kfctl_base /go/pkg/mod /third_party/vendor
+
 
 CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]

Makefile

@@ -22,10 +22,10 @@ TAG ?= $(eval TAG := $(shell git describe --tags --long --always))$(TAG)
 REPO ?= $(shell echo $$(cd ../kubeflow && git config --get remote.origin.url) | sed 's/git@\(.*\):\(.*\).git$$/https:\/\/\1\/\2/')
 BRANCH ?= $(shell cd ../kubeflow && git branch | grep '^*' | awk '{print $$2}')
 KFCTL_TARGET ?= kfctl
-MOUNT_KUBE ?=  -v $(HOME)/.kube:/root/.kube 
-MOUNT_GCP ?=  -v $(HOME)/.config:/root/.config 
+MOUNT_KUBE ?=  -v $(HOME)/.kube:/root/.kube
+MOUNT_GCP ?=  -v $(HOME)/.config:/root/.config
 # set to -V
-VERBOSE ?= 
+VERBOSE ?=
 PLUGINS_ENVIRONMENT ?= $(GOPATH)/src/github.com/kubeflow/kfctl/bin
 export GO111MODULE = on
 export GO = go
@@ -59,7 +59,7 @@ auth:
 
 # Run go fmt against code
 fmt:
-	@${GO} fmt ./config ./cmd/... ./pkg/... 
+	@${GO} fmt ./config ./cmd/... ./pkg/...
 
 # Run go vet against code
 vet:
@@ -222,17 +222,20 @@ build-builder-container-gcb:
 
 #***************************************************************************************************
 
-clean: 
+clean:
 	rm -rf test && mkdir test
 
 doc:
 	doctoc ./cmd/kfctl/README.md README.md k8sSpec/README.md developer_guide.md
 
 
 #**************************************************************************************************
+# checks licenses
+check-licenses:
+	./third_party/check-license.sh
 # rules to run unittests
 #
-test: build-kfctl
+test: build-kfctl check-licenses
 	go test ./... -v
 
 

third_party/README.md

@@ -0,0 +1,7 @@
+This folder contains all licenses and source code of third-party dependencies for distributing kfctl tools.
+
+For any dependency changes, please follow [this instruction](https://github.com/kubeflow/testing/blob/master/py/kubeflow/testing/go-license-tools/README.md) to update the content of this folder.
+
+- `dep.txt` contains third party Go dependencies based `go.mod`.
+- `license.txt` contains all license content of dependencies.
+- `license_info.csv` contains license URLs.

third_party/check-license.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -ex
+# Copyright 2018 The Kubeflow Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script to autoformat libsonnet files.
+# Assumes jsonnet is on the path.
+
+go list -m all | cut -d ' ' -f 1 > /tmp/generated_dep.txt
+
+cd third_party
+
+if ! diff /tmp/generated_dep.txt dep.txt; then
+    echo "Please update the license file for changed dependencies."
+    exit 1
+fi
+
+python3 concatenate_license.py --output=/tmp/generated_license.txt
+
+if ! diff /tmp/generated_license.txt license.txt; then
+    echo "Please regenerate third_party/license.txt."
+    exit 1
+fi

third_party/concatenate_license.py

@@ -0,0 +1,82 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import requests
+import sys
+import traceback
+
+parser = argparse.ArgumentParser(
+  description='Generate dependencies json from license.csv file.')
+parser.add_argument(
+  'license_info_file',
+  nargs='?',
+  default='license_info.csv',
+  help='CSV file with license info fetched from github using get-github-license-info CLI tool.'
+    +'(default: %(default)s)',
+)
+parser.add_argument(
+  '-o',
+  '--output',
+  dest='output_file',
+  nargs='?',
+  default='license.txt',
+  help=
+  'Concatenated license file path this command generates. (default: %(default)s)'
+)
+args = parser.parse_args()
+
+
+def fetch_license_text(download_link):
+  response = requests.get(download_link)
+  assert response.ok, 'Fetching {} failed with {} {}'.format(
+    download_link, response.status_code, response.reason)
+  return response.text
+
+
+def main():
+  with open(args.license_info_file,
+            'r') as license_info_file, open(args.output_file,
+                                            'w') as output_file:
+    repo_failed = []
+    for line in license_info_file:
+      line = line.strip()
+      [repo, license_link, license_name,
+        license_download_link] = line.split(',')
+      try:
+        print('Repo {} has license download link {}'.format(
+            repo, license_download_link),
+              file=sys.stderr)
+        license_text = fetch_license_text(license_download_link)
+        print(
+            '--------------------------------------------------------------------------------',
+            file=output_file,
+        )
+        print('{}  {}  {}'.format(repo, license_name, license_link),
+              file=output_file)
+        print(
+            '--------------------------------------------------------------------------------',
+            file=output_file,
+        )
+        print(license_text, file=output_file)
+      except Exception as e: # pylint: disable=broad-except
+        print('[failed]', e, file=sys.stderr)
+        traceback.print_exc(file=sys.stderr)
+        repo_failed.append(repo)
+    print('Failed to download license file for {} repos.'.format(len(repo_failed)), file=sys.stderr)
+    for repo in repo_failed:
+      print(repo, file=sys.stderr)
+
+
+main()