From 06098f166ce24c890d4b18402293a050242e885b Mon Sep 17 00:00:00 2001 From: xichen1 Date: Wed, 8 Nov 2023 02:36:31 -0800 Subject: [PATCH 1/4] Refactor old cc pkg upload API Refactor old chaincode package upload API according to the new design doc. Signed-off-by: xichen1 --- src/api-engine/api/lib/peer/chaincode.py | 22 ++++ src/api-engine/api/models.py | 12 ++- .../api/routes/chaincode/serializers.py | 23 ++-- src/api-engine/api/routes/chaincode/views.py | 100 +++++++++--------- 4 files changed, 90 insertions(+), 67 deletions(-) diff --git a/src/api-engine/api/lib/peer/chaincode.py b/src/api-engine/api/lib/peer/chaincode.py index 97ca76df8..103a05492 100644 --- a/src/api-engine/api/lib/peer/chaincode.py +++ b/src/api-engine/api/lib/peer/chaincode.py @@ -373,3 +373,25 @@ def query(self, orderer_url, orderer_tls_rootcert, channel_name, cc_name, args): except Exception as e: err_msg = "query failed for {}!".format(e) raise Exception(err_msg) + + def lifecycle_calculatepackageid(self, cc_path): + """ + calculate the chaincode packageid. + :param cc_path: where the chaincode package is + :return: calculated packageid + """ + try: + res = subprocess.Popen("{} lifecycle chaincode calculatepackageid {} " + .format(self.peer, cc_path), + shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = res.communicate() + return_code = res.returncode + if return_code == 0: + content = str(stdout, encoding="utf-8") + return return_code, content + else: + stderr = str(stderr, encoding="utf-8") + return return_code, stderr + except Exception as e: + err_msg = "calculated chaincode packageid failed for {}!".format(e) + raise Exception(err_msg) \ No newline at end of file diff --git a/src/api-engine/api/models.py b/src/api-engine/api/models.py index fa7d0dcce..4bf81418e 100644 --- a/src/api-engine/api/models.py +++ b/src/api-engine/api/models.py @@ -804,9 +804,17 @@ class ChainCode(models.Model): editable=False, unique=True ) + package_id = models.CharField( + help_text="package_id of chainCode", max_length=128, + editable=False, + unique=True + ) name = models.CharField( help_text="name of chainCode", max_length=128 ) + label = models.CharField( + help_text="label of chainCode", max_length=128 + ) version = models.CharField( help_text="version of chainCode", max_length=128 ) @@ -816,8 +824,8 @@ class ChainCode(models.Model): language = models.CharField( help_text="language of chainCode", max_length=128 ) - md5 = models.CharField( - help_text="md5 of chainCode", max_length=128 + description = models.CharField( + help_text="description of chainCode", max_length=128, blank=True, null=True ) create_ts = models.DateTimeField( help_text="Create time of chainCode", auto_now_add=True diff --git a/src/api-engine/api/routes/chaincode/serializers.py b/src/api-engine/api/routes/chaincode/serializers.py index 6440fcf10..3125e4bbb 100644 --- a/src/api-engine/api/routes/chaincode/serializers.py +++ b/src/api-engine/api/routes/chaincode/serializers.py @@ -6,7 +6,7 @@ from api.models import ChainCode from api.common.serializers import ListResponseSerializer -import hashlib +import os def upload_to(instance, filename): @@ -18,25 +18,20 @@ class ChainCodeIDSerializer(serializers.Serializer): class ChainCodePackageBody(serializers.Serializer): - name = serializers.CharField(max_length=128, required=True) - version = serializers.CharField(max_length=128, required=True) - language = serializers.CharField(max_length=128, required=True) - md5 = serializers.CharField(max_length=128, required=True) file = serializers.FileField() + description = serializers.CharField(max_length=128, required=False) + def validate(self, attrs): - md5_get = self.md5_for_file(attrs["file"]) - if md5_get != attrs["md5"]: - raise serializers.ValidationError("md5 not same.") + extension_get = self.extension_for_file(attrs["file"]) + if not extension_get: + raise serializers.ValidationError("unsupported package type") return super().validate(attrs) @staticmethod - def md5_for_file(chunks): - md5 = hashlib.md5() - for data in chunks: - md5.update(data) - return md5.hexdigest() - + def extension_for_file(file): + extension = file.name.endswith('.tar.gz') + return extension class ChainCodeNetworkSerializer(serializers.Serializer): id = serializers.UUIDField(help_text="Network ID") diff --git a/src/api-engine/api/routes/chaincode/views.py b/src/api-engine/api/routes/chaincode/views.py index 16e2ad61b..82dc78bb2 100644 --- a/src/api-engine/api/routes/chaincode/views.py +++ b/src/api-engine/api/routes/chaincode/views.py @@ -6,7 +6,7 @@ from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated import os -import zipfile +import tempfile, shutil, tarfile, json from drf_yasg.utils import swagger_auto_schema from api.config import FABRIC_CHAINCODE_STORE @@ -92,76 +92,74 @@ def list(self, request): def package(self, request): serializer = ChainCodePackageBody(data=request.data) if serializer.is_valid(raise_exception=True): - name = serializer.validated_data.get("name") - version = serializer.validated_data.get("version") - language = serializer.validated_data.get("language") - md5 = serializer.validated_data.get("md5") file = serializer.validated_data.get("file") - id = make_uuid() - + description = serializer.validated_data.get("description") + uuid = make_uuid() + fd, temp_path = tempfile.mkstemp() try: - file_path = os.path.join(FABRIC_CHAINCODE_STORE, id) - if not os.path.exists(file_path): - os.makedirs(file_path) - fileziped = os.path.join(file_path, file.name) - with open(fileziped, 'wb') as f: + # try to calculate packageid + with open(fd, 'wb') as f: for chunk in file.chunks(): f.write(chunk) - f.close() - zipped_file = zipfile.ZipFile(fileziped) - for filename in zipped_file.namelist(): - zipped_file.extract(filename, file_path) - - # When there is go.mod in the chain code, execute the go mod vendor command to obtain dependencies. - chaincode_path = file_path - found = False - for _, dirs, _ in os.walk(file_path): - if found: - break - elif dirs: - for each in dirs: - chaincode_path += "/" + each - if os.path.exists(chaincode_path + "/go.mod"): - cwd = os.getcwd() - print("cwd:", cwd) - os.chdir(chaincode_path) - os.system("go mod vendor") - found = True - os.chdir(cwd) - break - # if can not find go.mod, use the dir after extract zipped_file - if not found: - for _, dirs, _ in os.walk(file_path): - chaincode_path = file_path + "/" + dirs[0] - break org = request.user.organization qs = Node.objects.filter(type="peer", organization=org) if not qs.exists(): - raise ResourceNotFound + return Response( + err("At least 1 peer node is required for the chaincode package upload."), + status=status.HTTP_400_BAD_REQUEST + ) peer_node = qs.first() envs = init_env_vars(peer_node, org) - peer_channel_cli = PeerChainCode("v2.2.0", **envs) - res = peer_channel_cli.lifecycle_package( - name, version, chaincode_path, language) - os.system("rm -rf {}/*".format(file_path)) - os.system("mv {}.tar.gz {}".format(name, file_path)) - if res != 0: - return Response(err("package chaincode failed."), status=status.HTTP_400_BAD_REQUEST) + return_code, content = peer_channel_cli.lifecycle_calculatepackageid(temp_path) + if (return_code != 0): + return Response( + err("Calculate packageid failed for {}.".format(content)), + status=status.HTTP_400_BAD_REQUEST + ) + packageid = content.strip() + + # check if packageid exists + cc = ChainCode.objects.filter(package_id=packageid) + if cc.exists(): + return Response( + err("Packageid {} already exists.".format(packageid)), + status=status.HTTP_400_BAD_REQUEST + ) + + # try to save chaincode package + ccpackage_path = os.path.join(FABRIC_CHAINCODE_STORE, packageid) + if not os.path.exists(ccpackage_path): + os.makedirs(ccpackage_path) + # tared_file = tarfile.TarFile(temp_path) + with tarfile.open(temp_path) as tared_file: + tared_file.extractall(ccpackage_path) + + with open(os.path.join(ccpackage_path, "metadata.json"), 'r') as f: + metadata = json.load(f) + language = metadata["type"] + label = metadata["label"] + + os.system("rm -rf {}/*".format(ccpackage_path)) + + ccpackage = os.path.join(ccpackage_path, file.name) + shutil.copy(temp_path, ccpackage) chaincode = ChainCode( - id=id, - name=name, - version=version, + id=uuid, + package_id=packageid, language=language, creator=org.name, - md5=md5 + label=label, + description=description, ) chaincode.save() except Exception as e: return Response( err(e.args), status=status.HTTP_400_BAD_REQUEST ) + finally: + os.remove(temp_path) return Response( ok("success"), status=status.HTTP_200_OK ) From 404536a2a6fec44256c59b06c04b5281f5df851d Mon Sep 17 00:00:00 2001 From: xichen1 Date: Mon, 27 Nov 2023 19:31:01 -0800 Subject: [PATCH 2/4] Simplify cc read process Extract only necessary info from cc package. Optimize performance. Add postman testcase for cc upload. Signed-off-by: xichen1 --- src/api-engine/api/models.py | 6 -- src/api-engine/api/routes/chaincode/views.py | 79 ++++++++++++------- ...r Cello Api Engine.postman_collection.json | 56 +++++++++++++ 3 files changed, 106 insertions(+), 35 deletions(-) diff --git a/src/api-engine/api/models.py b/src/api-engine/api/models.py index 4bf81418e..8a99bb52b 100644 --- a/src/api-engine/api/models.py +++ b/src/api-engine/api/models.py @@ -809,15 +809,9 @@ class ChainCode(models.Model): editable=False, unique=True ) - name = models.CharField( - help_text="name of chainCode", max_length=128 - ) label = models.CharField( help_text="label of chainCode", max_length=128 ) - version = models.CharField( - help_text="version of chainCode", max_length=128 - ) creator = models.CharField( help_text="creator of chainCode", max_length=128 ) diff --git a/src/api-engine/api/routes/chaincode/views.py b/src/api-engine/api/routes/chaincode/views.py index 82dc78bb2..4f35504d0 100644 --- a/src/api-engine/api/routes/chaincode/views.py +++ b/src/api-engine/api/routes/chaincode/views.py @@ -31,12 +31,37 @@ ChaincodeListResponse ) from api.common import ok, err +import threading class ChainCodeViewSet(viewsets.ViewSet): """Class represents Channel related operations.""" permission_classes = [IsAuthenticated, ] + def _read_cc_pkg(self, pk, filename, ccpackage_path): + try: + meta_path = os.path.join(ccpackage_path, "metadata.json") + # extract metadata file + with tarfile.open(os.path.join(ccpackage_path, filename)) as tared_file: + metadata_file = tared_file.getmember("metadata.json") + tared_file.extract(metadata_file, path=ccpackage_path) + + with open(meta_path, 'r') as f: + metadata = json.load(f) + language = metadata["type"] + label = metadata["label"] + + if os.path.exists(meta_path): + os.remove(meta_path) + + chaincode = ChainCode.objects.get(id=pk) + chaincode.language = language + chaincode.label = label + chaincode.save() + + except Exception as e: + raise e + @swagger_auto_schema( query_serializer=PageQuerySerializer, responses=with_common_response( @@ -88,15 +113,15 @@ def list(self, request): {status.HTTP_201_CREATED: ChainCodeIDSerializer} ), ) - @action(detail=False, methods=['post']) + @action(detail=False, methods=['post'], url_path="chaincodeRepo") def package(self, request): serializer = ChainCodePackageBody(data=request.data) if serializer.is_valid(raise_exception=True): file = serializer.validated_data.get("file") description = serializer.validated_data.get("description") uuid = make_uuid() - fd, temp_path = tempfile.mkstemp() try: + fd, temp_cc_path = tempfile.mkstemp() # try to calculate packageid with open(fd, 'wb') as f: for chunk in file.chunks(): @@ -106,16 +131,16 @@ def package(self, request): qs = Node.objects.filter(type="peer", organization=org) if not qs.exists(): return Response( - err("At least 1 peer node is required for the chaincode package upload."), + err("at least 1 peer node is required for the chaincode package upload."), status=status.HTTP_400_BAD_REQUEST ) peer_node = qs.first() envs = init_env_vars(peer_node, org) peer_channel_cli = PeerChainCode("v2.2.0", **envs) - return_code, content = peer_channel_cli.lifecycle_calculatepackageid(temp_path) + return_code, content = peer_channel_cli.lifecycle_calculatepackageid(temp_cc_path) if (return_code != 0): return Response( - err("Calculate packageid failed for {}.".format(content)), + err("calculate packageid failed for {}.".format(content)), status=status.HTTP_400_BAD_REQUEST ) packageid = content.strip() @@ -124,45 +149,41 @@ def package(self, request): cc = ChainCode.objects.filter(package_id=packageid) if cc.exists(): return Response( - err("Packageid {} already exists.".format(packageid)), + err("package with id {} already exists.".format(packageid)), status=status.HTTP_400_BAD_REQUEST ) - # try to save chaincode package - ccpackage_path = os.path.join(FABRIC_CHAINCODE_STORE, packageid) - if not os.path.exists(ccpackage_path): - os.makedirs(ccpackage_path) - # tared_file = tarfile.TarFile(temp_path) - with tarfile.open(temp_path) as tared_file: - tared_file.extractall(ccpackage_path) - - with open(os.path.join(ccpackage_path, "metadata.json"), 'r') as f: - metadata = json.load(f) - language = metadata["type"] - label = metadata["label"] - - os.system("rm -rf {}/*".format(ccpackage_path)) - - ccpackage = os.path.join(ccpackage_path, file.name) - shutil.copy(temp_path, ccpackage) chaincode = ChainCode( id=uuid, package_id=packageid, - language=language, creator=org.name, - label=label, description=description, ) chaincode.save() + + # save chaincode package locally + ccpackage_path = os.path.join(FABRIC_CHAINCODE_STORE, packageid) + if not os.path.exists(ccpackage_path): + os.makedirs(ccpackage_path) + ccpackage = os.path.join(ccpackage_path, file.name) + shutil.copy(temp_cc_path, ccpackage) + + # start thread to read package meta info, update db + try: + threading.Thread(target=self._read_cc_pkg, + args=(uuid, file.name, ccpackage_path)).start() + except Exception as e: + raise e + + return Response( + ok("success"), status=status.HTTP_200_OK + ) except Exception as e: return Response( err(e.args), status=status.HTTP_400_BAD_REQUEST ) finally: - os.remove(temp_path) - return Response( - ok("success"), status=status.HTTP_200_OK - ) + os.remove(temp_cc_path) @swagger_auto_schema( method="post", diff --git a/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json b/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json index 567e4e1af..b3887fe61 100644 --- a/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json +++ b/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json @@ -2817,6 +2817,57 @@ } }, "response": [] + }, + { + "name": "Upload Chaincode Package", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + }, + { + "key": "Authorization", + "value": "JWT {{token}}", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "type": "file", + "src": "mycc.tar.gz" + }, + { + "key": "description", + "value": "CC Description", + "type": "text" + } + ] + }, + "url": { + "raw": "http://127.0.0.1:8080/api/v1/chaincodes/chaincodeRepo", + "protocol": "http", + "host": [ + "127", + "0", + "0", + "1" + ], + "port": "8080", + "path": [ + "api", + "v1", + "chaincodes", + "chaincodeRepo" + ] + } + }, + "response": [] } ], "event": [ @@ -2847,6 +2898,11 @@ { "key": "webRoot", "value": "api/v1" + }, + { + "key": "token", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzAxMTQ0NjIyLCJpYXQiOjE3MDExNDEwMjIsImp0aSI6ImQyOGZjNTAzZWQxODRkYTg4MWNjMTRhYjA4ZWI1MTIwIiwidXNlcl9pZCI6IjRmNjViYjNhLTdhZmQtNDdjZi1iODE1LTk2ZDY0NWQ2OGYxNyJ9.gAMBXqJwtPcO2tCKJ_4wgl34qAEddiGSsJ5r5CRluU8", + "type": "string" } ] } \ No newline at end of file From 85e5251ee56020901ffb3b3d4df2ccbc29202fba Mon Sep 17 00:00:00 2001 From: xichen1 Date: Mon, 27 Nov 2023 19:34:08 -0800 Subject: [PATCH 3/4] Remove personal token Signed-off-by: xichen1 --- .../Hyperledger Cello Api Engine.postman_collection.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json b/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json index b3887fe61..9814fdeb7 100644 --- a/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json +++ b/tests/postman/test/Hyperledger Cello Api Engine.postman_collection.json @@ -2898,11 +2898,6 @@ { "key": "webRoot", "value": "api/v1" - }, - { - "key": "token", - "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzAxMTQ0NjIyLCJpYXQiOjE3MDExNDEwMjIsImp0aSI6ImQyOGZjNTAzZWQxODRkYTg4MWNjMTRhYjA4ZWI1MTIwIiwidXNlcl9pZCI6IjRmNjViYjNhLTdhZmQtNDdjZi1iODE1LTk2ZDY0NWQ2OGYxNyJ9.gAMBXqJwtPcO2tCKJ_4wgl34qAEddiGSsJ5r5CRluU8", - "type": "string" } ] } \ No newline at end of file From 137cac43e7b7f779458db69e09380cb39e0cd8f3 Mon Sep 17 00:00:00 2001 From: xichen1 Date: Mon, 27 Nov 2023 19:36:34 -0800 Subject: [PATCH 4/4] Add help func doc Signed-off-by: xichen1 --- src/api-engine/api/routes/chaincode/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/api-engine/api/routes/chaincode/views.py b/src/api-engine/api/routes/chaincode/views.py index 4f35504d0..b1693637a 100644 --- a/src/api-engine/api/routes/chaincode/views.py +++ b/src/api-engine/api/routes/chaincode/views.py @@ -39,6 +39,12 @@ class ChainCodeViewSet(viewsets.ViewSet): permission_classes = [IsAuthenticated, ] def _read_cc_pkg(self, pk, filename, ccpackage_path): + """ + read and extract chaincode package meta info + :pk: chaincode id + :filename: uploaded chaincode package filename + :ccpackage_path: chaincode package path + """ try: meta_path = os.path.join(ccpackage_path, "metadata.json") # extract metadata file