Issue343 roundtrip fixes (#410)

- Implemented Round Trip testing of XML->JSON->XML on content in CI/CD workflow.
- Repaired error in catalog and profile metaschemas, with Schematron checks to detect these errors.
- Removing whitespace stripping in converted content
- Repairing bug in round trip handling of quote marks " (#7)

.circleci/config.yml

@@ -50,6 +50,20 @@ commands:
             git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "$SCHEMATRON_HOME"
             cd "$SCHEMATRON_HOME"
             git checkout master -- trunk/schematron/code
+  install-maven:
+    description: "Installs Maven"
+    steps:
+      - run:
+          name: Get Maven
+          command: |
+            sudo apt-get install maven
+  install-lxml:
+    description: "Installs Python lxml package"
+    steps:
+      - run:
+          name: Get lxml
+          command: |
+            sudo pip install lxml
   install-xmllint:
     description: "Retrieve xmllint and setup the environment to run it"
     steps:
@@ -92,14 +106,6 @@ commands:
           name: Get hub
           command: |
             sudo brew install hub
-  install-python-libs:
-    description: "Retrieve python libs and setup the environment to run it"
-    steps:
-      - run:
-          name: Get Python Libs
-          command: |
-            sudo pip install 'jsonschema>=3.0.1'
-            sudo pip install simplejson
   install-bundler:
     description: "Install bundler and html-proofer"
     steps:
@@ -121,16 +127,16 @@ commands:
             fi
 executors:
   java-executor:
-    docker: # run the steps with Docker
+    docker: # run the java jobs with Docker
       - image: circleci/openjdk:8-jdk-node-browsers
     <<: *common_environment 
   ruby-node-executor:
-    docker: # run the steps with Docker
+    docker: # run the ruby jobs with Docker
       - image: circleci/ruby:2.4.6-node-browsers
     <<: *common_environment 
   python-executor:
-    docker:
-      - image: circleci/python:3.7.3-node-browser
+    docker: #run the python jobs with Docker
+      - image: circleci/python:3.7.3-node-browsers
     <<: *common_environment 
 jobs: # a collection of 
   job-ci-skip-check:
@@ -246,6 +252,20 @@ jobs: # a collection of
           name: Validate content instances
           command: |
             bash "$CICD_DIR/validate-content.sh" "$OSCAL_BUILD_DIR"
+  job-roundtrip-conversions:
+    executor: python-executor
+    steps: 
+      - *checkout
+      - *attach_build_workspace
+      - install-maven
+      - install-saxon
+      - install-lxml
+      - install-xmllint
+      - run:
+          name: Generate round trip XML and JSON conversions
+          command: |
+            bash "${CICD_DIR}/validate-round-trips.sh" "$OSCAL_BUILD_DIR"
+      - *publish_artifacts
   job-generate-docs:
     executor: java-executor
     steps:
@@ -382,6 +402,10 @@ workflows:
       - job-validate-content:
           requires:
             - job-generate-schema
+      - job-roundtrip-conversions:
+          requires:
+            - job-generate-converters
+            - job-validate-content
       - job-copy-and-convert-content:
            requires:
             - job-generate-converters
@@ -411,4 +435,4 @@ workflows:
             - job-deploy-artifacts
           filters:
             branches:
-              only: master
+              only: master

.gitignore

@@ -1,26 +1,26 @@
-.DS_Store
-
 # Proprietary files not to be propagated
 vault
 
 # IDE files
 *.xpr
 /OSCAL-dev.xpr
 .vscode
+.DS_Store
 
 # Working files
 scratch
 wendell
 backup*
 working/
+__pycache__
 
 # Production outputs
 pub/
-roundtripped/
 target/
 xspec/*.html
 
 # ignore generated content
+roundtrip
 content
 xml
 json

build/ci-cd/.gitignore

@@ -0,0 +1,3 @@
+/python/*.pyc
+/temp/*
+/python/*.jar

build/ci-cd/README.md

@@ -9,19 +9,19 @@ Using a CI/CD process ensures that all artifacts in [pull requests](https://gith
 
 # Provided Scripts
 
-The primary CircleCI supported CI/CD workflow, ```build```, is uses a series of bash scripts. The following scripts are used in this workflow:
+The primary CircleCI supported CI/CD workflow, ```build```, is uses a series of bash scripts. The following scripts are used in this workflow for all PRs that are submitted:
 
 1. [run-all.sh](run-all.sh): Runs all of the scripts in workflow order.
 1. [validate-metaschema.sh](validate-metaschema.sh): Ensures that all [metaschema](https://github.com/usnistgov/OSCAL/tree/master/src/metaschema) are valid according to the [metaschema XML schema](https://github.com/usnistgov/OSCAL/blob/master/build/metaschema/lib/metaschema.xsd).
 1. [generate-schema.sh](generate-schema.sh): Generates [XML schema](https://github.com/usnistgov/OSCAL/tree/master/xml/schema) and [JSON schema](https://github.com/usnistgov/OSCAL/tree/master/json/schema) files for each OSCAL model based on their [respective metaschemas](https://github.com/usnistgov/OSCAL/tree/master/src/metaschema).
 1. [validate-content.sh](validate-content.sh): Validates OSCAL content in the repository's [src](https://github.com/usnistgov/OSCAL/tree/master/src) against the content's respective OSCAL model and format.
 1. [generate-content-converters.sh](generate-content-converters.sh): Generates [JSON to XML](https://github.com/usnistgov/OSCAL/tree/master/xml/convert) and [XML to JSON](https://github.com/usnistgov/OSCAL/tree/master/json/convert) content conversion scripts for each OSCAL model based on their [respective metaschemas](https://github.com/usnistgov/OSCAL/tree/master/src/metaschema).
 1. [copy-and-convert-content.sh](copy-and-convert-content.sh): Copies selective content from the [src/content](https://github.com/usnistgov/OSCAL/tree/master/src/content) directory, and generates instance of this content in alternate OSCAL formats using the appropriate content converters. Note __This script requires that the process have write access to the Git repository branch that the CI/CD is operating on to work completely.__
+1. [validate-round-trips.sh](validate-round-trips.sh): Performs round trip validations of the content from XML->JSON->XML to confirm that the conversions are lossless and that resulting files comply with their schemas.
 
-Additional scripts are provided that generate content for the OSCAL project website, supported by the CircleCI ```build-and-deploy-site``` workflow.
+Additional scripts are provided that generate content for the OSCAL project website, supported by the CircleCI ```build``` workflow on the ```master``` branch.
 1. [generate-schema-documentation.sh](generate-schema-documentation.sh): This is used as part of the site generation pipeline to build HTML documentation for the OSCAL models.
 
-
 # Running the Scripts Locally
 
 The bash scripts used in the CI/CD workflow can be run locally in the correct environment. In addition to a bash shell, to run these scripts the following must be installed:
@@ -38,13 +38,10 @@ NodeJS Packages:
 
 ```
 npm install -g ajv-cli
-```
 
-Python packages:
+Python Packages:
 
-```
-pip install 'jsonschema>=3.0.1'
-pip install simplejson
+pip install lxml
 ```
 
 The Saxon jar file needs to be retrieved. This can be two ways:

build/ci-cd/config/content

@@ -1,6 +1,4 @@
 # path to source|format of source|model of source|format(s) to convert to
 src/content/fedramp.gov/xml/*profile.xml|xml|profile|json
-src/content/nist.gov/**/xml/*catalog.xml|xml|catalog|json
-src/content/nist.gov/**/xml/*profile.xml|xml|profile|json
-#content/**/*catalog.json|json|catalog|xml
-
+src/content/nist.gov/SP800-53/rev4/xml/*catalog.xml|xml|catalog|json
+src/content/nist.gov/SP800-53/rev4/xml/*profile.xml|xml|profile|json

build/ci-cd/generate-content-converters.sh

@@ -76,4 +76,4 @@ done < $OSCALDIR/build/ci-cd/config/metaschema
 shopt -u nullglob
 shopt -u globstar
 
-exit $exitcode
+exit $exitcode