Rush logs files (#13)

- update the pipeline file to include more comments, and the future cron script - uploads log files, unit, fv and coverage outputs - uploads the jsdoc API html - uploads the tgz and docker image - pulls the latest docker images for the master branch Signed-off-by: Matthew B White <whitemat@uk.ibm.com> Change-Id: I5051443c57a290dd6b895191b806319e8aa9fd13
hyperledger · Oct 29, 2019 · f821b78 · f821b78
1 parent bf2ace9
commit f821b78
Show file tree

Hide file tree

Showing 20 changed files with 12,245 additions and 128 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,6 @@
 coverage
 node_modules
 **/node_modules/*
-npm-shrinkwrap.json
 npm-debug.log
 .DS_Store
 fabric-shim/.DS_Store
@@ -32,3 +31,4 @@ common/temp/**
 **/*.build.log
 
 package-deps.json
+test-results.xml
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -27,6 +27,10 @@ The following [Rush categories](https://rushjs.io/pages/maintainer/add_to_repo/)
 
 > Note that npm v6 has some bugs that mean adding new dependencies etc are not properly picked up. Longer term we should consider moving to yarn or pnpm. However in practice this isn't a serious problem and has been possible to be worked around by issuing `rm ./common/config/rush/npm-shrinkwrap.json` and then `rush update`
 
+The fv and e2e tests require a set of docker images of Fabric Peers, Orderers and CAs. To ensure you have the correct images ensure these have been dowloaded and tagged.  `rush edge-docker` will do this for you. 
+
+They also need to have the `nodeenv` image present - this is build as part of the `rush rebuild` so please ensure this has been run first.  It is advisable to clean up the docker containers and images between test runs to avoid any odd behaviour. Commands to help do this are below.
+
 ## Using the repo
 
 * Clone the repo, and ensure you are using nove v10, and have rush installed
@@ -35,8 +39,8 @@ The following [Rush categories](https://rushjs.io/pages/maintainer/add_to_repo/)
 At this point the repo is fully ready for use and running tests, etc. A full sequence of build-test that is equivalent to the CI pipeline is
 
 * `rush rebuild` will run the linting, and unit tests across the codebase, as well as building the docker images, and jsdoc API docs
-* `rush start-verdaccio` & `npm stop-verdaccio` will start/stop verdaccio (used for local hosting of NPM modules)
-* `rush start-fabric` & `npm stop-fabric` will start/stop the test fabric ready for running FV tests
+* `rush start-verdaccio` & `rush stop-verdaccio` will start/stop verdaccio (used for local hosting of NPM modules)
+* `rush start-fabric` & `rush stop-fabric` will start/stop the test fabric ready for running FV tests
 * `rush test:fv` will run the fv tests, ensure that both the fabric and verdaccio have already been started
 * `rush test:e2e` to run e2e tests across the repos
 
@@ -46,10 +50,19 @@ For more specific purposes during development the following are useful:
 If you want to get a set of `.tar.gz` files of the node modules to use for local testing this command will put them into the `tarballs` directory
 * `rush rebuild --to fvtests` to run the unit tests for the core modules, but not the docker or jsdoc
 * `rush rebuild --to fabric-contract-api` to build, lint and run just the `fabric-contract-api` 
+* `rush logs` will show the location of all the log files 
+
+To clean up docker
+
+* `docker kill $(docker ps -q) && docker rm $(docker ps -aq)` will remove the running containers
+* `docker rmi $(docker images 'dev-*' -q) --force` will remove the images for the chaincode containers
+
+## Mechanics of Contributing
+
+The codebase is maintained in [github](https://github.com/hyperledger/fabric-chaincode-node), with a CI pileline run with [Azure Devlops](https://dev.azure.com/Hyperledger/Fabric-Chaincode-Node/_build?definitionId=33&_a=summary). Issues are handling in [Jira](https://jira.hyperledger.org/issues/?jql=project%20%3D%20FAB%20AND%20component%20%3D%20fabric-chaincode-node)   (please use the component `fabric-chaincode-node` in jira as this is shared project with other Fabric components).
+
 
-## Where are the log files?
 
-Each individual module will write out logs files to it's own directory. 
 
 ## Code of Conduct Guidelines <a name="conduct"></a>
 

diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml
@@ -6,6 +6,20 @@
 # 
 name: RELEASE-$(Date:yyyyMMdd)$(Rev:.rrr)
 
+# Daily build for final quality
+# cf https://crontab.guru/#0_23_*_*_*
+schedules:
+  - cron: "0 23 * * *"
+    displayName: 'Chaincode Node Nightly Driver'
+    branches:
+      include:
+        - master
+    always: true
+
+
+# Trigger on the master and the release-1.4 branches only
+# Stndard Pull Request and Merge builds
+# Any git tag will also trigger a build that should do publishing
 trigger:
   branches:
     include:
@@ -15,25 +29,38 @@ trigger:
     include:
       - '*'
 
+
 # These are custom defined variables, the pipeline one is currently used for the build scripts
 # to know to produce tests results in XML format for Azure to consume, for developers
 # this isn't set so command line output is given
 # 
+# Chaincode_Node_Vars is the protected group of keys for publishing
 variables:
   - group: Chaincode_Node_Vars
   - name: component
     value: fabric-chaincode-node
   - name: pipeline
     value: ci 
 
-# Build on Ubutu
+# Build on Ubuntu
 pool:
   vmImage: 'ubuntu-latest'
 
 # 
 # The stages and jobs, potential for rationalization and optimization
 # Keeping it simple and explict whilst we gain experience
 stages:
+
+    # output some basic information for reference
+    # Not for use every time, but this can be useful template to keep
+    # - stage: info
+    #   jobs:
+    #     - job: env_info
+    #       steps:
+    #         - script: |
+    #             env | sort
+
+    # Primary stage of building, linting and unit tests
     - stage: Build_and_Test
       jobs:
         - job: main
@@ -43,12 +70,14 @@ stages:
               inputs:
                 versionSpec: '10.x' 
             - script: |
+                set -ev
                 node common/scripts/install-run-rush.js install
-                node common/scripts/install-run-rush.js update
-              displayName: 'Setup the node environment'
+              displayName: 'Setup the rush environment'
             - script: |
+                set -ev 
                 node common/scripts/install-run-rush.js rebuild
               displayName: 'Full rebuild'              
+
             - task: PublishTestResults@2
               condition: or(succeeded(), failed())   # publish either way
               inputs:
@@ -60,67 +89,115 @@ stages:
               inputs:
                 codeCoverageTool: 'cobertura'
                 summaryFileLocation: '**/cobertura-coverage.xml'        
-                artifactName: 'Unit Test Coverage'
+                artifactName: 'Unit Test Coverage' 
+            # CopyFiles can follow symlinks hence be careful with node_modules           
+            - task: CopyFiles@2
+              condition: or(succeeded(), failed())   # publish either way
+              inputs:
+                contents: |
+                  **/*.build*.log                    
+                  !**/node_modules/**                 
+                targetFolder: $(Build.ArtifactStagingDirectory)/logs
+            - task: PublishBuildArtifacts@1    
+              condition: or(succeeded(), failed())   # publish either way
+              inputs:
+                pathToPublish:  $(Build.ArtifactStagingDirectory)/logs
+                artifactName: 'Build logs'                 
+            - task: PublishBuildArtifacts@1    
+              condition: or(succeeded(), failed())   # publish either way
+              inputs:
+                pathToPublish: docs/apidocs/_out
+                artifactName: 'JS API Doc'     
+
+            # Copy the built artifacts to the staging directory, tgz, and the docker image    
             - script: |
+                set -ev
                 node common/scripts/install-run-rush.js publish --include-all --pack --release-folder $(Build.ArtifactStagingDirectory)/tgz --publish   
-              displayName: 'Package as tgz'                  
+                docker image save hyperledger/fabric-nodeenv | gzip > $(Build.ArtifactStagingDirectory)/fabric-nodeenv.tar.gz
+              displayName: 'Package tgz and docker image'    
+            - task: PublishBuildArtifacts@1    
+              inputs:
+                pathToPublish:  $(Build.ArtifactStagingDirectory)/tgz
+                artifactName: node-tgz        
+            - task: PublishBuildArtifacts@1    
+              inputs:
+                pathToPublish:  $(Build.ArtifactStagingDirectory)/fabric-nodeenv.tar.gz
+                artifactName: nodeenv-docker-image        
+
+        # Run the FV tests but using the built material        
         - job: fvt
           displayName: 'FV Tests'
           dependsOn: main
           steps:         
             - task: NodeTool@0
               inputs:
                 versionSpec: '10.x' 
+            - task: DownloadPipelineArtifact@2
+              inputs:
+                artifact: nodeenv-docker-image
+                path: $(Build.SourcesDirectory)/build
+            - task: DownloadPipelineArtifact@2
+              inputs:
+                artifact: node-tgz 
+                path: $(Build.SourcesDirectory)/build
             - script: |
+                set -ev
+                ./tools/getEdgeDocker.sh                                # essential to get master docker images of peer etc.
+                docker image load --input build/fabric-nodeenv.tar.gz         # gets the build image of nodeenv
+                docker images
                 node common/scripts/install-run-rush.js install
-                node common/scripts/install-run-rush.js update
-                node common/scripts/install-run-rush.js rebuild  # replace this with pull from the artifacts staging
-              displayName: 'Setup the node environment'                 
+                node common/scripts/install-run-rush.js update          # should the tests need 'building' this will need to go here
+              displayName: 'Get the latest master docker images'              
             - script: |
+                set -ev
                 node common/scripts/install-run-rush.js start-fabric
-                node common/scripts/install-run-rush.js start-verdaccio
+                node common/scripts/install-run-rush.js start-verdaccio # script will check for the ci variable and use built images
                 docker images | grep hyperledger && docker ps -a
                 node common/scripts/install-run-rush.js test:fv
                 node common/scripts/install-run-rush.js test:e2e
               displayName: 'FV Tests'  
+
             - task: PublishTestResults@2
               condition: or(succeeded(), failed())   # publish either way
               inputs:
                 testResultsFiles: '**/test-results.xml'
-                artifactName: 'FV Test results'
-                testRunTitle: 'FV Test results'                      
-        - job: apidocs
-          steps:
-            - task: NodeTool@0
+                testRunTitle: 'FV Test results'
+            - task: CopyFiles@2
+              condition: or(succeeded(), failed())   # publish either way
               inputs:
-                versionSpec: '8.x' 
-            - script: |
-                npm install
-                npm install -g gulp-cli
-              displayName: 'Setup the node environment'            
-            - script: |
-                gulp docs
-              displayName: 'Produce API docs'
-            - publish: $(System.DefaultWorkingDirectory)/docs/gen/
-              artifact: 'jsdocs'
+                sourceFolder: $(Build.SourcesDirectory)/test
+                contents: |
+                  **/*.build*.log
+                  !**/node_modules/**
+                targetFolder: $(Build.ArtifactStagingDirectory)/testlogs
+            - task: PublishBuildArtifacts@1    
+              condition: or(succeeded(), failed())   # publish either way
+              inputs:
+                pathToPublish:  $(Build.ArtifactStagingDirectory)/testlogs
+                artifactName: 'Test logs'           
+
+    # Publish tag for the Merge build of a regular PRi.e. w
     - stage: Publish_tag
       condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags'))
       jobs:
         - job: update_version
           steps:
             - script: |
+                set -ev
                 env | sort
                 echo "Update the version number"
                 echo "Make sure release notes are present"
                 echo "Make sure change history is present"
         - job: npm_publish
           steps:
             - script: |
+                set -ev
                 echo "Setup .npmrc"
                 echo "Use pipeline secret to login"
                 echo "publish"
         - job: jsdoc_publish    
           steps:
             - script: |
+                set -ev
                 echo "checkout docs branch"
                 echo "checking"
diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json
@@ -68,6 +68,20 @@
             "summary": "Starts local Fabric test network ",
             "description": "Run this command to stop local Fabric network created for testing",
             "shellCommand": "gulp -f ./tools/toolchain/fabric.js stopFabric"
+        },
+        {
+            "commandKind": "global",
+            "name": "logs",
+            "summary": "Lists build log files created",
+            "description": "Run this command to list the build logs created after rush rebuild",
+            "shellCommand": "./tools/logfiles.sh"
+        },
+        {
+            "commandKind": "global",
+            "name": "edge-docker",
+            "summary": "Gets the latest master docker images for Fabric ",
+            "description": "Run this command before doing anything to get the master branch Fabric docker images",
+            "shellCommand": "./tools/getEdgeDocker.sh"
         }
     ]
 }