From e3505c584f9bed1c634f31d8e8950d1f5b3234c4 Mon Sep 17 00:00:00 2001 From: Harry Gibson Date: Tue, 24 Nov 2020 14:52:22 +0000 Subject: [PATCH] updated paths and db parameters --- .../DHS_PG_Database_Add_Indexes.ipynb | 510 ++++++++-------- .../Load_Value_Specs_To_DB.fmw | 578 +++++++++--------- .../Load_survey_metadata_to_db.fmw | 102 ++-- .../Load_All_Cluster_Shapes_To_PG.fmw | 224 ++++--- DHS_To_Database/run_processing.py | 4 +- 5 files changed, 727 insertions(+), 691 deletions(-) diff --git a/DHS_DB_Creation/DHS_PG_Database_Add_Indexes.ipynb b/DHS_DB_Creation/DHS_PG_Database_Add_Indexes.ipynb index b65d213..6812df0 100644 --- a/DHS_DB_Creation/DHS_PG_Database_Add_Indexes.ipynb +++ b/DHS_DB_Creation/DHS_PG_Database_Add_Indexes.ipynb @@ -1,259 +1,251 @@ -{ - "metadata": { - "name": "", - "signature": "sha256:b5bbd6206e8d5f405c190fde308bc31598bf0b78c563a71df7f0fbae1c94d3d5" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ - { - "cells": [ - { - "cell_type": "heading", - "level": 3, - "metadata": {}, - "source": [ - "Simple script to go through all the REC* tables in the DHS database and add indexes to all the key columns " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Rationale - we loaded all the parsed .DHS tables to a postgres database with a straightforward dump and now want to automate the process of indexing the fields we are likely to want to join on" - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "import psycopg2\n" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 1 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "conn_string = \"host='map-fs1.ndph.ox.ac.uk' port='5433' dbname='dhs_data_pit' \\\n", - " user='admin' password='##PASSWORD##'\"" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 2 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "conn = psycopg2.connect(conn_string)" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 3 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "crs = conn.cursor()" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 4 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "# Figure out the table names we want to look at, they are those starting with REC or MREC\n", - "crs.execute(\"select relname from pg_class where relkind='r' and relname ~ 'REC';\")\n", - "tblNames = [i[0] for i in crs.fetchall()]" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 5 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "crs.execute(\"select relname from pg_class where relkind='i'\")\n", - "existingIndexes = [i[0] for i in crs.fetchall()]" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 6 - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Identifying the columns to index is based on knowledge of the content of the database: they all have \"id\" in the names, in lowercase. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We're not doing anything more sophisticated than that for now (such as, for example trying to be intelligent with the \"relationships\" part of the .dcf), partly because it doesn't necessarily cover all possible joins and partly because we haven't really needed to!\n", - "\n", - "However there are a few columns we ought to index that get missed by this simple strategy or for joins that are outside the normal hierarchy but would nonetheless be possible - for example REC01.V034 represents \"line number of husband\" so it might be desirable to join based on this field to the husband's record to create some kind of couple's dataset. For now, we haven't addressed this." - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "runIt = 0\n", - "allUpper = 0\n", - "replaceExisting = 0" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 7 - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We're just going to make an index on each *ID* column, and covering indice(s) across multiple of them for tables which have more than one." - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "cleanSQLTemplate = \"DROP INDEX IF EXISTS dhs_data_tables.{0};\"\n", - "idxSQLTemplate = \"CREATE INDEX {0} ON {1}({2});\"\n", - "idxNameTemplate = \"{0}_{1}\"\n", - "allSQL = []\n", - "# Create an index for each column with \"id\" in the title, and if there \n", - "# is more than one (e.g. caseid, bidx) then also create a composite index\n", - "for tblName in tblNames:\n", - " print tblName\n", - " crs.execute('SELECT * FROM dhs_data_tables.\"{0}\" LIMIT 1'.format(tblName))\n", - " \n", - " if allUpper:\n", - " tblName = str.upper(tblName)\n", - " # note the lowercase will exclude the OGC_FID column which is already indexed\n", - " idxfields = [c[0] for c in crs.description if c[0].find(\"id\") != -1]\n", - " \n", - " # create a separate index on each joining column\n", - " #dropidxStmts = [cleanSQLTemplate.format(idxNameTemplate.format(c, str.lower(tblName)))\n", - " # for c in idxfields]\n", - " dropidxStmts = []\n", - " idxStmts = []\n", - " print idxfields\n", - " \n", - " for c in idxfields:\n", - " idxName = idxNameTemplate.format(c, str.lower(tblName))\n", - " idxStmt = idxSQLTemplate.format(idxName, 'dhs_data_tables.\"'+ tblName + '\"', c)\n", - " if idxName in existingIndexes:\n", - " if replaceExisting:\n", - " dropidxStmt = cleanSQLTemplate.format(idxName)\n", - " dropidxStmts.append(dropidxStmt)\n", - " idxStmts.append(idxStmt)\n", - " print \"Replacing idx \"+idxName\n", - " else:\n", - " print \"Skipped existing idx \"+idxName\n", - " else:\n", - " idxStmts.append(idxStmt)\n", - " print \"Adding idx \"+idxName\n", - " #idxStmts = [idxSQLTemplate.format(idxNameTemplate.format(c, str.lower(tblName)),\n", - " # 'dhs_data_tables.\"'+tblName+'\"', c) for c in idxfields]\n", - " \n", - " # also create a single covering index on all joining columns\n", - " if len(idxfields) > 1:\n", - " idxName = idxNameTemplate.format(\"allidx\", str.lower(tblName))\n", - " allStmt = idxSQLTemplate.format(idxName,\n", - " 'dhs_data_tables.\"'+tblName+'\"', \",\".join(idxfields))\n", - " if idxName in existingIndexes:\n", - " if replaceExisting:\n", - " dropAllStmt = cleanSQLTemplate.format(idxName)\n", - " dropidxStmts.append(dropAllStmt)\n", - " idxStmts.append(allStmt)\n", - " print \"Replacing idx \"+idxName\n", - " else:\n", - " print \"Skipped existing idx \"+idxName\n", - " else:\n", - " idxStmts.append(allStmt)\n", - " print \"Adding idx \"+idxName\n", - " \n", - " # also create a covering index on the first two joining columns if there are three \n", - " # (or all except the last one, if there's more)\n", - " # e.g. surveyid and caseid but not bidx (the cols are in the appropriate order in the CSVs)\n", - " if len(idxfields) > 2:\n", - " idxName = idxNameTemplate.format(\"twoidx\", str.lower(tblName))\n", - " allStmt = idxSQLTemplate.format(idxName,\n", - " 'dhs_data_tables.\"'+tblName+'\"', \",\".join(idxfields[:-1]))\n", - " if idxName in existingIndexes:\n", - " if replaceExisting:\n", - " dropAllStmt = cleanSQLTemplate.format(idxName)\n", - " dropidxStmts.append(dropAllStmt)\n", - " idxStmts.append(allStmt)\n", - " print \"Replacing idx \"+idxName\n", - " else:\n", - " print \"Skipped existing idx \"+idxName\n", - " else:\n", - " idxStmts.append(allStmt)\n", - " print \"Adding idx \"+idxName\n", - " \n", - "\n", - " dropIndexSQL = \"\\n\".join(dropidxStmts)\n", - " indexSQL = \"\\n\".join(idxStmts)\n", - " if runIt:\n", - " if len(dropIndexSQL) > 0:\n", - " crs.executescript(dropIndexSQL)\n", - " if len(indexSQL) > 0:\n", - " crs.executescript(indexSQL)\n", - " if len(dropIndexSQL)>0: \n", - " allSQL.append(dropIndexSQL)\n", - " if len(indexSQL) > 0:\n", - " allSQL.append(indexSQL)\n", - "if runIt:\n", - " dbconn.commit()\n", - " dbconn.close()" - ], - "language": "python", - "metadata": {}, - "outputs": [] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "print \"\\n\".join(allSQL)" - ], - "language": "python", - "metadata": {}, - "outputs": [] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "for l in allSQL:\n", - " if len(l) > 0:\n", - " print l\n", - " crs.execute(l)\n", - "conn.commit()" - ], - "language": "python", - "metadata": {}, - "outputs": [] - } - ], - "metadata": {} - } - ] -} \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Simple script to go through all the REC* tables in the DHS database and add indexes to all the key columns " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Rationale - we loaded all the parsed .DHS tables to a postgres database with a straightforward dump and now want to automate the process of indexing the fields we are likely to want to join on" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import psycopg2\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "conn_string = \"host='map-fs1.ndph.ox.ac.uk' port='5433' dbname='dhs_data_pit' \\\n", + " user='admin' password='##PASSWORD##'\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "conn = psycopg2.connect(conn_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "crs = conn.cursor()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Figure out the table names we want to look at, they are those starting with REC or MREC\n", + "crs.execute(\"select relname from pg_class where relkind='r' and relname ~ 'REC';\")\n", + "tblNames = [i[0] for i in crs.fetchall()]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "crs.execute(\"select relname from pg_class where relkind='i'\")\n", + "existingIndexes = [i[0] for i in crs.fetchall()]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Identifying the columns to index is based on knowledge of the content of the database: they all have \"id\" in the names, in lowercase. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We're not doing anything more sophisticated than that for now (such as, for example trying to be intelligent with the \"relationships\" part of the .dcf), partly because it doesn't necessarily cover all possible joins and partly because we haven't really needed to!\n", + "\n", + "However there are a few columns we ought to index that get missed by this simple strategy or for joins that are outside the normal hierarchy but would nonetheless be possible - for example REC01.V034 represents \"line number of husband\" so it might be desirable to join based on this field to the husband's record to create some kind of couple's dataset. For now, we haven't addressed this." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "runIt = 0\n", + "allUpper = 0\n", + "replaceExisting = 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We're just going to make an index on each *ID* column, and covering indice(s) across multiple of them for tables which have more than one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cleanSQLTemplate = \"DROP INDEX IF EXISTS dhs_data_tables.{0};\"\n", + "idxSQLTemplate = \"CREATE INDEX {0} ON {1}({2});\"\n", + "idxNameTemplate = \"{0}_{1}\"\n", + "allSQL = []\n", + "# Create an index for each column with \"id\" in the title, and if there \n", + "# is more than one (e.g. caseid, bidx) then also create a composite index\n", + "for tblName in tblNames:\n", + " print tblName\n", + " crs.execute('SELECT * FROM dhs_data_tables.\"{0}\" LIMIT 1'.format(tblName))\n", + " \n", + " if allUpper:\n", + " tblName = str.upper(tblName)\n", + " # note the lowercase will exclude the OGC_FID column which is already indexed\n", + " idxfields = [c[0] for c in crs.description if c[0].find(\"id\") != -1]\n", + " \n", + " # create a separate index on each joining column\n", + " #dropidxStmts = [cleanSQLTemplate.format(idxNameTemplate.format(c, str.lower(tblName)))\n", + " # for c in idxfields]\n", + " dropidxStmts = []\n", + " idxStmts = []\n", + " print idxfields\n", + " \n", + " for c in idxfields:\n", + " idxName = idxNameTemplate.format(c, str.lower(tblName))\n", + " idxStmt = idxSQLTemplate.format(idxName, 'dhs_data_tables.\"'+ tblName + '\"', c)\n", + " if idxName in existingIndexes:\n", + " if replaceExisting:\n", + " dropidxStmt = cleanSQLTemplate.format(idxName)\n", + " dropidxStmts.append(dropidxStmt)\n", + " idxStmts.append(idxStmt)\n", + " print \"Replacing idx \"+idxName\n", + " else:\n", + " print \"Skipped existing idx \"+idxName\n", + " else:\n", + " idxStmts.append(idxStmt)\n", + " print \"Adding idx \"+idxName\n", + " #idxStmts = [idxSQLTemplate.format(idxNameTemplate.format(c, str.lower(tblName)),\n", + " # 'dhs_data_tables.\"'+tblName+'\"', c) for c in idxfields]\n", + " \n", + " # also create a single covering index on all joining columns\n", + " if len(idxfields) > 1:\n", + " idxName = idxNameTemplate.format(\"allidx\", str.lower(tblName))\n", + " allStmt = idxSQLTemplate.format(idxName,\n", + " 'dhs_data_tables.\"'+tblName+'\"', \",\".join(idxfields))\n", + " if idxName in existingIndexes:\n", + " if replaceExisting:\n", + " dropAllStmt = cleanSQLTemplate.format(idxName)\n", + " dropidxStmts.append(dropAllStmt)\n", + " idxStmts.append(allStmt)\n", + " print \"Replacing idx \"+idxName\n", + " else:\n", + " print \"Skipped existing idx \"+idxName\n", + " else:\n", + " idxStmts.append(allStmt)\n", + " print \"Adding idx \"+idxName\n", + " \n", + " # also create a covering index on the first two joining columns if there are three \n", + " # (or all except the last one, if there's more)\n", + " # e.g. surveyid and caseid but not bidx (the cols are in the appropriate order in the CSVs)\n", + " if len(idxfields) > 2:\n", + " idxName = idxNameTemplate.format(\"twoidx\", str.lower(tblName))\n", + " allStmt = idxSQLTemplate.format(idxName,\n", + " 'dhs_data_tables.\"'+tblName+'\"', \",\".join(idxfields[:-1]))\n", + " if idxName in existingIndexes:\n", + " if replaceExisting:\n", + " dropAllStmt = cleanSQLTemplate.format(idxName)\n", + " dropidxStmts.append(dropAllStmt)\n", + " idxStmts.append(allStmt)\n", + " print \"Replacing idx \"+idxName\n", + " else:\n", + " print \"Skipped existing idx \"+idxName\n", + " else:\n", + " idxStmts.append(allStmt)\n", + " print \"Adding idx \"+idxName\n", + " \n", + "\n", + " dropIndexSQL = \"\\n\".join(dropidxStmts)\n", + " indexSQL = \"\\n\".join(idxStmts)\n", + " if runIt:\n", + " if len(dropIndexSQL) > 0:\n", + " crs.executescript(dropIndexSQL)\n", + " if len(indexSQL) > 0:\n", + " crs.executescript(indexSQL)\n", + " if len(dropIndexSQL)>0: \n", + " allSQL.append(dropIndexSQL)\n", + " if len(indexSQL) > 0:\n", + " allSQL.append(indexSQL)\n", + "if runIt:\n", + " dbconn.commit()\n", + " dbconn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print \"\\n\".join(allSQL)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for l in allSQL:\n", + " if len(l) > 0:\n", + " print l\n", + " crs.execute(l)\n", + "conn.commit()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_Value_Specs_To_DB.fmw b/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_Value_Specs_To_DB.fmw index 46368f3..ec0fbff 100644 --- a/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_Value_Specs_To_DB.fmw +++ b/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_Value_Specs_To_DB.fmw @@ -2,10 +2,10 @@ #! Dataadmin" -# --RecordSpecCSVs "C:\Users\zool1301\Documents\02_Update_Staging_Template\parsed\*.FlatRecordSpec.csv" -# --DestDataset_POSTGRES_3 "DHSDataadmin" +# --ValueSpecCSVs "D:\InformalCities\DHS\staging\parsed_specs\*.FlatValuesSpec.csv" +# --DestDataset_POSTGRES "DHSadminlocaldocker" +# --RecordSpecCSVs "D:\InformalCities\DHS\staging\parsed_specs\*.FlatRecordSpec.csv" +# --DestDataset_POSTGRES_5 "DHSadminlocaldocker" # #! ATTR_TYPE_ENCODING="SDF" #! BEGIN_PYTHON="" @@ -19,9 +19,9 @@ #! END_PYTHON="" #! END_TCL="" #! EXPLICIT_BOOKMARK_ORDER="false" -#! FME_BUILD_NUM="20238" -#! FME_DOCUMENT_GUID="9ad56795-0d35-4f98-b4a7-f51bae30b0d5" -#! FME_DOCUMENT_PRIORGUID="1768a313-5d92-4800-9d34-791ff3e85d51,84392675-a625-4311-bf66-2944216d01ed,24d83111-81c9-423d-85e4-b11b3616d2bf,e6236a3d-657c-497e-b16b-2bb44f3f56b7,9b11cad5-a301-404d-97c8-380ff1c25005,26223ab7-256d-4b8b-be84-e0c976791b6d,9c0b8be5-23c4-4c12-a1b9-25c339d175e4,d626059c-9681-45d8-831d-d5ee63b4f0e6,41a5e54b-ee12-42c0-8aec-db52d93e2b0e,a2e021a0-4ea2-4127-9a4c-4cfb6234a691,8dfd5293-a105-465b-ac9a-9ceaa921db3e,70644038-5ede-4a5a-84cb-b5540abb0d26,74c8b584-376c-4787-b939-2dff457c9657,ad13c8df-327e-42c7-82a4-db0353229eb7,02c414fb-a9e3-4f2f-9c32-4334a9838584,51ccaf29-e0c2-4c0c-82a4-70bb56ac9da9" +#! FME_BUILD_NUM="20624" +#! FME_DOCUMENT_GUID="9165ee9c-a080-4618-8b0b-c592d214f8d9" +#! FME_DOCUMENT_PRIORGUID="1768a313-5d92-4800-9d34-791ff3e85d51,84392675-a625-4311-bf66-2944216d01ed,24d83111-81c9-423d-85e4-b11b3616d2bf,e6236a3d-657c-497e-b16b-2bb44f3f56b7,9b11cad5-a301-404d-97c8-380ff1c25005,26223ab7-256d-4b8b-be84-e0c976791b6d,9c0b8be5-23c4-4c12-a1b9-25c339d175e4,d626059c-9681-45d8-831d-d5ee63b4f0e6,41a5e54b-ee12-42c0-8aec-db52d93e2b0e,a2e021a0-4ea2-4127-9a4c-4cfb6234a691,8dfd5293-a105-465b-ac9a-9ceaa921db3e,70644038-5ede-4a5a-84cb-b5540abb0d26,74c8b584-376c-4787-b939-2dff457c9657,ad13c8df-327e-42c7-82a4-db0353229eb7,02c414fb-a9e3-4f2f-9c32-4334a9838584,51ccaf29-e0c2-4c0c-82a4-70bb56ac9da9,9ad56795-0d35-4f98-b4a7-f51bae30b0d5,68494f41-2f2d-4506-ad82-c63912a2da5b,570c0c40-0d3e-4c01-b508-0c5394882064" #! FME_GEOMETRY_HANDLING="Enhanced" #! FME_IMPLICIT_CSMAP_REPROJECTION_MODE="Auto" #! FME_REPROJECTION_ENGINE="FME" @@ -29,8 +29,8 @@ #! FME_STROKE_MAX_DEVIATION="0" #! HISTORY="" #! IGNORE_READER_FAILURE="No" -#! LAST_SAVE_BUILD="FME(R) 2020.0.2.1 (20200511 - Build 20238 - WIN64)" -#! LAST_SAVE_DATE="2020-05-22T14:11:12" +#! LAST_SAVE_BUILD="FME(R) 2020.1.2.1 (20200909 - Build 20624 - WIN64)" +#! LAST_SAVE_DATE="2020-10-08T10:03:07" #! LOG_FILE="" #! LOG_MAX_RECORDED_FEATURES="200" #! MARKDOWN_DESCRIPTION="" @@ -49,7 +49,7 @@ #! TITLE="" #! USAGE="" #! USE_MARKDOWN="" -#! VIEW_POSITION="-2472.68 -226.565" +#! VIEW_POSITION="-4011.76 203.127" #! WARN_INVALID_XFORM_PARAM="Yes" #! WORKSPACE_VERSION="1" #! ZOOM_SCALE="80" @@ -60,7 +60,7 @@ #! ROLE="READER" #! FORMAT="CSV2" #! DATASET="$(ValueSpecCSVs)" -#! KEYWORD="CSV2_2" +#! KEYWORD="CSV2_1" #! MULTI_GEOM_ALLOWED="false" #! ATTR_MAX_LENGTH="0" #! ATTR_CASE="ANY" @@ -68,7 +68,7 @@ #! WRITE_DEFS="true" #! DEFLINE_TEMPLATE="" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="csv_line_number uint64 fme_color string fme_type string fme_text_size string multi_reader_id string fme_secondary_axis string csv_total_number_of_columns uint32 fme_feature_type string fme_primary_axis string multi_reader_full_id string multi_reader_type string csv_type string fme_geometry fme_char(50) fme_rotation string fme_text_string string fme_start_angle string fme_sweep_angle string fme_basename string fme_dataset string multi_reader_keyword string fme_fill_color string csv_skipped_line string csv_number_of_fields uint32" +#! EXPOSABLE_ATTRS="fme_sweep_angle string fme_rotation string csv_type string csv_total_number_of_columns uint32 csv_line_number uint64 fme_type string csv_number_of_fields uint32 csv_skipped_line string fme_start_angle string fme_primary_axis string multi_reader_type string multi_reader_keyword string fme_fill_color string multi_reader_full_id string fme_geometry fme_char(50) fme_text_string string fme_basename string fme_text_size string fme_secondary_axis string fme_color string multi_reader_id string fme_dataset string fme_feature_type string" #! DEFLINE_PARMS="" #! ATTR_INDEX_TYPES="" #! ATTR_NAME_INVALID_CHARS="" @@ -119,7 +119,7 @@ #! ROLE="READER" #! FORMAT="CSV2" #! DATASET="$(RecordSpecCSVs)" -#! KEYWORD="CSV2_1" +#! KEYWORD="CSV2_2" #! MULTI_GEOM_ALLOWED="false" #! ATTR_MAX_LENGTH="0" #! ATTR_CASE="ANY" @@ -127,7 +127,7 @@ #! WRITE_DEFS="true" #! DEFLINE_TEMPLATE="" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="csv_line_number uint64 fme_color string fme_type string fme_text_size string multi_reader_id string fme_secondary_axis string csv_total_number_of_columns uint32 fme_feature_type string fme_primary_axis string multi_reader_full_id string multi_reader_type string csv_type string fme_geometry fme_char(50) fme_rotation string fme_text_string string fme_start_angle string fme_sweep_angle string fme_basename string fme_dataset string multi_reader_keyword string fme_fill_color string csv_skipped_line string csv_number_of_fields uint32" +#! EXPOSABLE_ATTRS="fme_sweep_angle string fme_rotation string csv_type string csv_total_number_of_columns uint32 csv_line_number uint64 fme_type string csv_number_of_fields uint32 csv_skipped_line string fme_start_angle string fme_primary_axis string multi_reader_type string multi_reader_keyword string fme_fill_color string multi_reader_full_id string fme_geometry fme_char(50) fme_text_string string fme_basename string fme_text_size string fme_secondary_axis string fme_color string multi_reader_id string fme_dataset string fme_feature_type string" #! DEFLINE_PARMS="" #! ATTR_INDEX_TYPES="" #! ATTR_NAME_INVALID_CHARS="" @@ -184,23 +184,32 @@ #! ATTR_CASE="LOWER" #! ALLOWED_FEAT_TYPES="" #! WRITE_DEFS="true" -#! DEFLINE_TEMPLATE="postgres_type {FME_GEN_GEOMETRY} postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING fme_selection_method MATCH_COLUMNS fme_match_columns "" fme_where_builder_clause "" postgres_create_with_oids NO postgres_vacuum_analyze NO postgres_allow_serial_writing NO" +#! DEFLINE_TEMPLATE="postgres_type {FME_GEN_GEOMETRY} postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING fme_selection_method MATCH_COLUMNS fme_match_columns "" fme_where_builder_clause "" postgres_vacuum_analyze NO postgres_allow_serial_writing NO" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="fme_type char(50) fme_start_angle float8 fme_feature_type char(50) fme_secondary_axis float8 fme_text_string char(50) fme_text_size float8 fme_sweep_angle float8 fme_fill_color char(50) fme_primary_axis float8 fme_basename char(50) fme_db_operation char(8) fme_color char(50) fme_rotation float8 fme_dataset char(50) postgres_type char(50) fme_geometry fme_char(50)" -#! DEFLINE_PARMS=""GUI OPTIONAL NAMEDGROUP fme_configuration_group fme_configuration_common_group%fme_spatial_group%fme_advanced_group%oracle_advanced_group Table Settings" "" "GUI OPTIONAL NAMEDGROUP fme_configuration_common_group fme_feature_operation%fme_table_handling%oracle_model%fme_update_geometry%fme_selection_group%fme_table_creation_group General" "" "GUI ACTIVECHOICE_LOOKUP fme_feature_operation Insert,INSERT,fme_update_geometry,fme_selection_group%Update,UPDATE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN%Delete,DELETE,++fme_table_handling+USE_EXISTING,fme_update_geometry,++fme_selection_group+FME_DISCLOSURE_OPEN,fme_spatial_group,fme_advanced_group,oracle_sequenced_cols%<at>Value<openparen>fme_db_operation<closeparen>,MULTIPLE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN Feature Operation" INSERT "GUI ACTIVECHOICE_LOOKUP fme_table_handling Use<space>Existing,USE_EXISTING,fme_table_creation_group%Create<space>If<space>Needed,CREATE_IF_MISSING%Drop<space>and<space>Create,DROP_CREATE%Truncate<space>Existing,TRUNCATE_EXISTING,fme_table_creation_group Table Handling" CREATE_IF_MISSING "GUI OPTIONAL DISCLOSUREGROUP fme_selection_group fme_selection_method Row Selection" "" "GUI OPTIONAL WHOLE_LINE RADIOPARAMETERGROUP fme_selection_method fme_match_columns,MATCH_COLUMNS%fme_where_builder_clause,BUILDER Row Selection Method" MATCH_COLUMNS "GUI WHOLE_LINE ATTRLIST_COMMAS fme_match_columns \" \" Match Columns" "" "GUI WHOLE_LINE TEXT_EDIT_SQL_CFG_OR_ATTR fme_where_builder_clause MODE,WHERE WHERE Clause" "" "GUI OPTIONAL DISCLOSUREGROUP fme_table_creation_group postgres_create_with_oids Table Creation Parameters" "" "GUI CHOICE postgres_create_with_oids YES%NO Create With OIDs" NO "GUI OPTIONAL DISCLOSUREGROUP fme_advanced_group postgres_vacuum_analyze%postgres_allow_serial_writing Advanced" "" "GUI CHOICE postgres_vacuum_analyze YES%NO Vacuum Analyze Table" NO "GUI CHOICE postgres_allow_serial_writing YES%NO Allow Serial Column Writing" NO " -#! ATTR_INDEX_TYPES="BTREE HASH PRIKEY RTREE" +#! EXPOSABLE_ATTRS="fme_fill_color char(50) fme_color char(50) fme_type char(50) fme_basename char(50) fme_feature_type char(50) fme_db_operation char(8) fme_secondary_axis float8 fme_sweep_angle float8 fme_primary_axis float8 fme_start_angle float8 fme_dataset char(50) fme_geometry char(50) fme_rotation float8 fme_text_string char(50) fme_text_size float8 postgres_type char(50)" +#! DEFLINE_PARMS=""GUI OPTIONAL NAMEDGROUP fme_configuration_group fme_configuration_common_group%fme_spatial_group%fme_advanced_group%oracle_advanced_group Table" "" "GUI OPTIONAL NAMEDGROUP fme_configuration_common_group fme_feature_operation%fme_table_handling%mie_pack%oracle_model%fme_update_geometry%fme_selection_group%fme_table_creation_group General" "" "GUI ACTIVECHOICE_LOOKUP fme_feature_operation Insert,INSERT,fme_update_geometry,fme_selection_group,mie_pack%Update,UPDATE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN%Delete,DELETE,++fme_table_handling+USE_EXISTING,fme_update_geometry,++fme_selection_group+FME_DISCLOSURE_OPEN,fme_spatial_group,fme_advanced_group,oracle_sequenced_cols%<at>Value<openparen>fme_db_operation<closeparen>,MULTIPLE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN Feature Operation" INSERT "GUI ACTIVECHOICE_LOOKUP fme_table_handling Use<space>Existing,USE_EXISTING,fme_table_creation_group%Create<space>If<space>Needed,CREATE_IF_MISSING%Drop<space>and<space>Create,DROP_CREATE%Truncate<space>Existing,TRUNCATE_EXISTING,fme_table_creation_group Table Handling" CREATE_IF_MISSING "GUI OPTIONAL DISCLOSUREGROUP fme_selection_group fme_selection_method Row Selection" "" "GUI OPTIONAL WHOLE_LINE RADIOPARAMETERGROUP fme_selection_method fme_match_columns,MATCH_COLUMNS%fme_where_builder_clause,BUILDER Row Selection Method" MATCH_COLUMNS "GUI WHOLE_LINE ATTRLIST_COMMAS fme_match_columns \" \" Match Columns" "" "GUI WHOLE_LINE TEXT_EDIT_SQL_CFG_OR_ATTR fme_where_builder_clause MODE,WHERE WHERE Clause" "" "GUI OPTIONAL DISCLOSUREGROUP fme_advanced_group postgres_vacuum_analyze%postgres_allow_serial_writing Advanced" "" "GUI CHOICE postgres_vacuum_analyze YES%NO Vacuum Analyze Table" NO "GUI CHOICE postgres_allow_serial_writing YES%NO Allow Serial Column Writing" NO " +#! ATTR_INDEX_TYPES="BTree RTree Hash PrimaryKey" #! ATTR_NAME_INVALID_CHARS=". %-#[]"()!?*'&+\/" #! SUPPORTS_FEATURE_TYPE_FANOUT="true" #! ENABLED="true" #! DYNAMIC_FEATURE_TYPES_LIST_ON_MERGE="true" #! DATASET_TYPE="DATABASE" #! FTTR_TYPE="" -#! GENERATE_FME_BUILD_NUM="16492" +#! GENERATE_FME_BUILD_NUM="20624" +#! NAME="DHS admin @ local docker [POSTGRES] - 2" #! COORDSYS="" #! > #! +#! +#! #! #! +#! #! #! #! #! #! -#! #! #! #! -#! #! #! #! #! #! -#! +#! #! #! #! #! #! #! #! #! @@ -1161,7 +1173,7 @@ #! BOUNDING_RECT="-908.51499999999999 -268.12700000000001 633.51199999999994 624.12599999999998" #! STICKY="true" #! COLOUR="0.42352941176470588,0.56470588235294117,0.40000000000000002,0.39215686274509803" -#! CONTENTS="8 10 12 2 11 " +#! CONTENTS="10 2 12 8 11 " #! > #! #! #! #! #! #! #! #! #! #! #! #! #! #! #! #! #! #! @@ -1405,8 +1417,8 @@ #! IDENTIFIER="13" #! TYPE="NullAttributeMapper" #! VERSION="1" -#! POSITION="867.87800000000004 -920.75800000000004" -#! BOUNDING_RECT="867.87800000000004 -920.75800000000004 431 71" +#! POSITION="821.75199999999984 -1065.2906953269533" +#! BOUNDING_RECT="821.75199999999984 -1065.2906953269533 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1535,7 +1547,7 @@ #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" -#! LAST_PARM_EDIT="17271" +#! LAST_PARM_EDIT="20624" #! > #! #! @@ -1578,16 +1590,18 @@ #! #! #! -#! +#! #! #! #! #! #! -#! +#! #! -#! +#! +#! #! +#! #! #! #! @@ -1597,8 +1611,8 @@ #! IDENTIFIER="27" #! TYPE="Tester" #! VERSION="2" -#! POSITION="1216.3797499974996 -1477.8900000000001" -#! BOUNDING_RECT="1216.3797499974996 -1477.8900000000001 431 71" +#! POSITION="1267.1615078150776 -1417.8900000000001" +#! BOUNDING_RECT="1267.1615078150776 -1417.8900000000001 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1655,8 +1669,8 @@ #! IDENTIFIER="26" #! TYPE="AttributeCreator" #! VERSION="8" -#! POSITION="1695.8900000000001 -1477.8900000000001" -#! BOUNDING_RECT="1695.8900000000001 -1477.8900000000001 431 71" +#! POSITION="1884.6470000000002 -1378.77" +#! BOUNDING_RECT="1884.6470000000002 -1378.77 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1736,13 +1750,13 @@ #! #! #! @@ -1783,11 +1797,12 @@ #! #! #! -#! +#! #! #! #! #! +#! #! #! #! @@ -1797,7 +1812,7 @@ #! #! #! -#! +#! #! #! #! @@ -1808,11 +1823,11 @@ #! IDENTIFIER="36" #! TYPE="Inspector" #! VERSION="4" -#! POSITION="1929.4004375043753 -33.749499999999998" -#! BOUNDING_RECT="1929.4004375043753 -33.749499999999998 431 71" +#! POSITION="2167.6840703407038 -92.715554685546863" +#! BOUNDING_RECT="2167.6840703407038 -92.715554685546863 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" -#! ENABLED="true" +#! ENABLED="false" #! LAST_PARM_EDIT="17271" #! > #! @@ -1881,8 +1896,10 @@ #! #! #! +#! #! #! +#! #! #! #! @@ -1896,7 +1913,7 @@ #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" -#! LAST_PARM_EDIT="17271" +#! LAST_PARM_EDIT="20624" #! > #! #! @@ -1914,17 +1931,19 @@ #! #! #! -#! +#! #! #! #! #! #! -#! +#! #! +#! #! -#! +#! #! +#! #! #! #! @@ -1934,13 +1953,13 @@ #! #! @@ -2005,11 +2024,12 @@ #! #! #! -#! +#! #! #! #! #! +#! #! #! #! @@ -2019,7 +2039,7 @@ #! #! #! -#! +#! #! #! #! @@ -2035,7 +2055,7 @@ #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" -#! LAST_PARM_EDIT="17271" +#! LAST_PARM_EDIT="20624" #! > #! #! @@ -2045,16 +2065,18 @@ #! #! #! -#! +#! #! #! #! #! #! -#! +#! #! -#! +#! +#! #! +#! #! #! #! @@ -2064,12 +2086,12 @@ #! IDENTIFIER="57" #! TYPE="SQLCreator" #! VERSION="3" -#! POSITION="-200.13303125031251 -651.87699999999995" -#! BOUNDING_RECT="-200.13303125031251 -651.87699999999995 431 71" +#! POSITION="-200.13303125031251 -686.25300000000004" +#! BOUNDING_RECT="-200.13303125031251 -686.25300000000004 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" -#! LAST_PARM_EDIT="17271" +#! LAST_PARM_EDIT="20624" #! > #! #! @@ -2079,16 +2101,18 @@ #! #! #! -#! +#! #! #! #! #! #! -#! +#! #! -#! +#! +#! #! +#! #! #! #! @@ -2097,13 +2121,13 @@ #! #! #! @@ -2140,11 +2164,12 @@ #! #! #! -#! +#! #! #! #! #! +#! #! #! #! @@ -2154,7 +2179,7 @@ #! #! #! -#! +#! #! #! #! @@ -2165,8 +2190,8 @@ #! IDENTIFIER="77" #! TYPE="Tester" #! VERSION="2" -#! POSITION="-1431.27 -1485.7622187221875" -#! BOUNDING_RECT="-1431.27 -1485.7622187221875 431 71" +#! POSITION="-1411.7385546855467 -1537.8900000000001" +#! BOUNDING_RECT="-1411.7385546855467 -1537.8900000000001 431 71" #! ORDER="500000000000004.13" #! PARMS_EDITED="true" #! ENABLED="true" @@ -2432,7 +2457,7 @@ #! TARGET_NODE="36" #! SOURCE_PORT_DESC="fo 0 MERGED" #! TARGET_PORT_DESC="fi 0 " -#! ENABLED="true" +#! ENABLED="false" #! EXECUTION_IDX="0" #! HIDDEN="false" #! COLOUR="0,0,0,1" @@ -2633,27 +2658,27 @@ #! FME_PYTHON_VERSION 37 -GUI IGNORE SourceDataset_CSV2_2,CSV2_IN_DATASET_GROUP_CSV2_2,CSV2_IN_FIELDS_GROUP_CSV2_2,CSV2_IN_DELIMITER_CSV2_2,CSV2_IN_FIELD_NAMES_LINE_CSV2_2,CSV2_IN_DATA_START_LINE_CSV2_2,CSV2_IN_ADVANCED_GROUP_CSV2_2,CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_2,CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2,CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2,CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2,CSV2_IN_FIELD_NAMES_GROUP_CSV2_2,CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2,CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2,CSV2_IN_FIELD_VALUES_GROUP_CSV2_2,CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2,CSV2_IN_ENCODING_GROUP_CSV2_2,CSV2_IN_ENCODING_CSV2_2,CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2,CSV2_IN_SKIPPED_LINES_GROUP_CSV2_2,CSV2_IN_READ_SKIPPED_LINES_CSV2_2,CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_2,CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2,CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2,SourceDataset_CSV2_1,CSV2_IN_DATASET_GROUP_CSV2_1,CSV2_IN_FIELDS_GROUP_CSV2_1,CSV2_IN_DELIMITER_CSV2_1,CSV2_IN_FIELD_NAMES_LINE_CSV2_1,CSV2_IN_DATA_START_LINE_CSV2_1,CSV2_IN_ADVANCED_GROUP_CSV2_1,CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_1,CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_1,CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_1,CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_1,CSV2_IN_FIELD_NAMES_GROUP_CSV2_1,CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_1,CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_1,CSV2_IN_FIELD_VALUES_GROUP_CSV2_1,CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_1,CSV2_IN_ENCODING_GROUP_CSV2_1,CSV2_IN_ENCODING_CSV2_1,CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_1,CSV2_IN_SKIPPED_LINES_GROUP_CSV2_1,CSV2_IN_READ_SKIPPED_LINES_CSV2_1,CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_1,CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_1,CSV2_IN_NETWORK_AUTHENTICATION_CSV2_1,DestDataset_POSTGRES_1,POSTGRES_OUT_START_TRANSACTION_POSTGRES_1,POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1,POSTGRES_OUT_BULK_COPY_POSTGRES_1,POSTGRES_OUT_BEGIN_SQL_POSTGRES_1,POSTGRES_OUT_END_SQL_POSTGRES_1,DestDataset_POSTGRES_2,POSTGRES_OUT_FME_CONNECTION_GROUP_POSTGRES_2,POSTGRES_OUT_ADVANCED_POSTGRES_2,POSTGRES_OUT_BULK_COPY_POSTGRES_2,POSTGRES_OUT_START_TRANSACTION_POSTGRES_2,POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_2,POSTGRES_OUT_BEGIN_SQL_POSTGRES_2,POSTGRES_OUT_END_SQL_POSTGRES_2 -DEFAULT_MACRO ValueSpecCSVs C:\Users\zool1301\Documents\02_Update_Staging_Template\parsed\*.FlatValuesSpec.csv +GUI IGNORE SourceDataset_CSV2_1,CSV2_IN_DATASET_GROUP_CSV2_1,CSV2_IN_FIELDS_GROUP_CSV2_1,CSV2_IN_DELIMITER_CSV2_1,CSV2_IN_FIELD_NAMES_LINE_CSV2_1,CSV2_IN_DATA_START_LINE_CSV2_1,CSV2_IN_ADVANCED_GROUP_CSV2_1,CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_1,CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_1,CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_1,CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_1,CSV2_IN_FIELD_NAMES_GROUP_CSV2_1,CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_1,CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_1,CSV2_IN_FIELD_VALUES_GROUP_CSV2_1,CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_1,CSV2_IN_ENCODING_GROUP_CSV2_1,CSV2_IN_ENCODING_CSV2_1,CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_1,CSV2_IN_SKIPPED_LINES_GROUP_CSV2_1,CSV2_IN_READ_SKIPPED_LINES_CSV2_1,CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_1,CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_1,CSV2_IN_NETWORK_AUTHENTICATION_CSV2_1,SourceDataset_CSV2_2,CSV2_IN_DATASET_GROUP_CSV2_2,CSV2_IN_FIELDS_GROUP_CSV2_2,CSV2_IN_DELIMITER_CSV2_2,CSV2_IN_FIELD_NAMES_LINE_CSV2_2,CSV2_IN_DATA_START_LINE_CSV2_2,CSV2_IN_ADVANCED_GROUP_CSV2_2,CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_2,CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2,CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2,CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2,CSV2_IN_FIELD_NAMES_GROUP_CSV2_2,CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2,CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2,CSV2_IN_FIELD_VALUES_GROUP_CSV2_2,CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2,CSV2_IN_ENCODING_GROUP_CSV2_2,CSV2_IN_ENCODING_CSV2_2,CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2,CSV2_IN_SKIPPED_LINES_GROUP_CSV2_2,CSV2_IN_READ_SKIPPED_LINES_CSV2_2,CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_2,CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2,CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2,DestDataset_POSTGRES_1,POSTGRES_OUT_FME_CONNECTION_GROUP_POSTGRES_1,POSTGRES_OUT_ADVANCED_POSTGRES_1,POSTGRES_OUT_BULK_COPY_POSTGRES_1,POSTGRES_OUT_START_TRANSACTION_POSTGRES_1,POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1,POSTGRES_OUT_BEGIN_SQL_POSTGRES_1,POSTGRES_OUT_END_SQL_POSTGRES_1,DestDataset_POSTGRES_2,POSTGRES_OUT_FME_CONNECTION_GROUP_POSTGRES_2,POSTGRES_OUT_ADVANCED_POSTGRES_2,POSTGRES_OUT_BULK_COPY_POSTGRES_2,POSTGRES_OUT_START_TRANSACTION_POSTGRES_2,POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_2,POSTGRES_OUT_BEGIN_SQL_POSTGRES_2,POSTGRES_OUT_END_SQL_POSTGRES_2 +DEFAULT_MACRO ValueSpecCSVs D:\InformalCities\DHS\staging\parsed_specs\*.FlatValuesSpec.csv GUI MULTIFILE ValueSpecCSVs CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|All_Files|* Parsed value spec csv file(s): -DEFAULT_MACRO DestDataset_POSTGRES DHSDataadmin +DEFAULT_MACRO DestDataset_POSTGRES DHSadminlocaldocker GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGRES FMT:POSTGRES%FAMILY:PostgreSQL Connection: -DEFAULT_MACRO RecordSpecCSVs C:\Users\zool1301\Documents\02_Update_Staging_Template\parsed\*.FlatRecordSpec.csv +DEFAULT_MACRO RecordSpecCSVs D:\InformalCities\DHS\staging\parsed_specs\*.FlatRecordSpec.csv GUI MULTIFILE RecordSpecCSVs CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|All_Files|* Parsed record spec csv file(s): -DEFAULT_MACRO DestDataset_POSTGRES_3 DHSDataadmin -GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGRES_3 FMT:POSTGRES%FAMILY:PostgreSQL Connection: +DEFAULT_MACRO DestDataset_POSTGRES_5 DHSadminlocaldocker +GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGRES_5 FMT:POSTGRES%FAMILY:PostgreSQL Connection: INCLUDE [ if {{$(ValueSpecCSVs)} == {}} { puts_real {Parameter 'ValueSpecCSVs' must be given a value.}; exit 1; }; ] INCLUDE [ if {{$(DestDataset_POSTGRES)} == {}} { puts_real {Parameter 'DestDataset_POSTGRES' must be given a value.}; exit 1; }; ] INCLUDE [ if {{$(RecordSpecCSVs)} == {}} { puts_real {Parameter 'RecordSpecCSVs' must be given a value.}; exit 1; }; ] -INCLUDE [ if {{$(DestDataset_POSTGRES_3)} == {}} { puts_real {Parameter 'DestDataset_POSTGRES_3' must be given a value.}; exit 1; }; ] +INCLUDE [ if {{$(DestDataset_POSTGRES_5)} == {}} { puts_real {Parameter 'DestDataset_POSTGRES_5' must be given a value.}; exit 1; }; ] #! START_HEADER #! START_WB_HEADER READER_TYPE MULTI_READER MULTI_READER_TYPE{0} CSV2 -MULTI_READER_KEYWORD{0} CSV2_2 +MULTI_READER_KEYWORD{0} CSV2_1 MULTI_READER_GEN_DIRECTIVES{0} DELIMITER,comma,TRIM_FIELD_NAME_QUALIFIERS,Yes,ENCODING,,DATASET_GROUP,,ENCODING_GROUP,,TRIM_FIELD_VALUE_QUALIFIERS,Yes,FIELD_VALUES_GROUP,,_MERGE_SCHEMAS,YES,FEATURE_TYPE_NAME,FORMAT_NAME,SCHEMA_GROUP,,CASE_SENSITIVE_FIELD_NAMES,Yes,DATA_START_LINE,2,FIELD_SEPARATION_GROUP,,SKIPPED_LINES_GROUP,,USE_CUSTOM_SCHEMA,NO,ADVANCED_GROUP,,EXPOSE_ATTRS_GROUP,,CSV2_EXPOSE_FORMAT_ATTRS,,FIELDS_GROUP,,SKIP_DUPLICATE_DELIMITERS,No,FIELD_NAMES_GROUP,,PRESERVE_SOURCE_ENCODING,No,PREVIEW_GROUP,,FIELD_NAMES_LINE,1,READ_SKIPPED_LINES,No,ENFORCE_STRICT_SCHEMA,No,FIELD_QUALIFIER_CHARACTER,quote,SCAN_ADDITIONAL_FIELDS,No,PREVIEW_PLACEHOLDER,,SCHEMA_GENERATION_GROUP,,SCAN_MAX_ROWS,1000,SCAN_TYPES,No,FIELD_QUALIFIER_ESCAPE_CHARACTER,quote,SCHEMA, MULTI_READER_TYPE{1} CSV2 -MULTI_READER_KEYWORD{1} CSV2_1 +MULTI_READER_KEYWORD{1} CSV2_2 MULTI_READER_GEN_DIRECTIVES{1} DELIMITER,comma,TRIM_FIELD_NAME_QUALIFIERS,Yes,ENCODING,,DATASET_GROUP,,ENCODING_GROUP,,TRIM_FIELD_VALUE_QUALIFIERS,Yes,FIELD_VALUES_GROUP,,_MERGE_SCHEMAS,YES,FEATURE_TYPE_NAME,FORMAT_NAME,SCHEMA_GROUP,,CASE_SENSITIVE_FIELD_NAMES,Yes,DATA_START_LINE,2,FIELD_SEPARATION_GROUP,,SKIPPED_LINES_GROUP,,USE_CUSTOM_SCHEMA,NO,ADVANCED_GROUP,,EXPOSE_ATTRS_GROUP,,CSV2_EXPOSE_FORMAT_ATTRS,,FIELDS_GROUP,,SKIP_DUPLICATE_DELIMITERS,No,FIELD_NAMES_GROUP,,PRESERVE_SOURCE_ENCODING,No,PREVIEW_GROUP,,FIELD_NAMES_LINE,1,READ_SKIPPED_LINES,No,ENFORCE_STRICT_SCHEMA,No,FIELD_QUALIFIER_CHARACTER,quote,SCAN_ADDITIONAL_FIELDS,No,PREVIEW_PLACEHOLDER,,SCHEMA_GENERATION_GROUP,,SCAN_MAX_ROWS,1000,SCAN_TYPES,No,FIELD_QUALIFIER_ESCAPE_CHARACTER,quote,SCHEMA, WRITER_TYPE MULTI_WRITER MULTI_WRITER_DATASET_ORDER BY_ID @@ -2664,102 +2689,6 @@ MULTI_WRITER_TYPE{1} POSTGRES MULTI_WRITER_KEYWORD{1} POSTGRES_2 #! END_WB_HEADER #! START_WB_HEADER -MACRO WB_KEYWORD "CSV2_2" -#! END_WB_HEADER -#! START_SOURCE_HEADER CSV2 CSV2_2 -# ===================================================================== -# The following GUI line prompts for a file to be used as the source CSV file. -# The user input is stored in a macro, which is then used to define -# the dataset to be read. -# The dataset this mapping file was generated from was: -#! END_SOURCE_HEADER -#! START_WB_HEADER -DEFAULT_MACRO SourceDataset -INCLUDE [ if {{$(SourceDataset)} != ""} { \ - puts {DEFAULT_MACRO SourceDataset_CSV2_2 $(SourceDataset)} \ - } ] -#! END_WB_HEADER -#! START_SOURCE_HEADER CSV2 CSV2_2 -DEFAULT_MACRO SourceDataset_CSV2_2 $(ValueSpecCSVs) -GUI MULTIFILE SourceDataset_CSV2_2 CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|All_Files|* Source CSV (Comma Separated Value) File(s): -DEFAULT_MACRO CSV2_IN_DATASET_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_DATASET_GROUP_CSV2_2 FME_DISCLOSURE_OPEN%FEATURE_TYPE_NAME Dataset Parameters -DEFAULT_MACRO CSV2_IN_FEATURE_TYPE_NAME_CSV2_2 FORMAT_NAME -CSV2_2_FEATURE_TYPE_NAME "$(CSV2_IN_FEATURE_TYPE_NAME_CSV2_2)" -DEFAULT_MACRO CSV2_IN_FIELDS_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELDS_GROUP_CSV2_2 FME_DISCLOSURE_OPEN%DELIMITER%FIELD_NAMES_LINE%DATA_START_LINE Fields -DEFAULT_MACRO CSV2_IN_DELIMITER_CSV2_2 -CSV2_2_DELIMITER "$(CSV2_IN_DELIMITER_CSV2_2)" -GUI STRING_OR_CHOICE_ENCODED CSV2_IN_DELIMITER_CSV2_2 ,%;%|%space%tab Delimiter Character: -DEFAULT_MACRO CSV2_IN_FIELD_NAMES_LINE_CSV2_2 1 -CSV2_2_FIELD_NAMES_LINE "$(CSV2_IN_FIELD_NAMES_LINE_CSV2_2)" -GUI OPTIONAL RANGE_SLIDER CSV2_IN_FIELD_NAMES_LINE_CSV2_2 1%MAX%0 Field Names Line: -DEFAULT_MACRO CSV2_IN_DATA_START_LINE_CSV2_2 2 -CSV2_2_DATA_START_LINE "$(CSV2_IN_DATA_START_LINE_CSV2_2)" -GUI RANGE_SLIDER CSV2_IN_DATA_START_LINE_CSV2_2 1%MAX%0 Data Start Line: -DEFAULT_MACRO CSV2_IN_ADVANCED_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_ADVANCED_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%FIELD_SEPARATION_GROUP%FIELD_NAMES_GROUP%FIELD_VALUES_GROUP%ENCODING_GROUP%SKIPPED_LINES_GROUP%SCHEMA_GENERATION_GROUP Advanced -DEFAULT_MACRO CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%SKIP_DUPLICATE_DELIMITERS%FIELD_QUALIFIER_CHARACTER%FIELD_QUALIFIER_ESCAPE_CHARACTER Field Separation -DEFAULT_MACRO CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2 No -CSV2_2_SKIP_DUPLICATE_DELIMITERS "$(CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2)" -GUI CHOICE CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2 Yes%No Skip Duplicate Delimiters: -DEFAULT_MACRO CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2 -CSV2_2_FIELD_QUALIFIER_CHARACTER "$(CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2)" -GUI OPTIONAL STRING_ENCODED CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2 Field Qualifier Character: -DEFAULT_MACRO CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2 -CSV2_2_FIELD_QUALIFIER_ESCAPE_CHARACTER "$(CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2)" -GUI OPTIONAL STRING_ENCODED CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2 Field Qualifier Escape Character: -DEFAULT_MACRO CSV2_IN_FIELD_NAMES_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELD_NAMES_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%CASE_SENSITIVE_FIELD_NAMES%ENFORCE_STRICT_SCHEMA Field Names -DEFAULT_MACRO CSV2_IN_TRIM_FIELD_NAME_QUALIFIERS_CSV2_2 Yes -CSV2_2_TRIM_FIELD_NAME_QUALIFIERS "$(CSV2_IN_TRIM_FIELD_NAME_QUALIFIERS_CSV2_2)" -DEFAULT_MACRO CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2 Yes -CSV2_2_CASE_SENSITIVE_FIELD_NAMES "$(CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2)" -GUI CHOICE CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2 Yes%No Case Sensitive Field Names: -DEFAULT_MACRO CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2 No -CSV2_2_ENFORCE_STRICT_SCHEMA "$(CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2)" -GUI CHOICE CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2 Yes%No Enforce Strict Schema: -DEFAULT_MACRO CSV2_IN_FIELD_VALUES_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELD_VALUES_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%TRIM_FIELD_VALUE_QUALIFIERS Field Values -DEFAULT_MACRO CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2 Yes -CSV2_2_TRIM_FIELD_VALUE_QUALIFIERS "$(CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2)" -GUI CHOICE CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2 Yes%No Trim Field Qualifier Characters: -DEFAULT_MACRO CSV2_IN_ENCODING_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_ENCODING_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%ENCODING%PRESERVE_SOURCE_ENCODING Encoding -DEFAULT_MACRO CSV2_IN_ENCODING_CSV2_2 -CSV2_2_ENCODING "$(CSV2_IN_ENCODING_CSV2_2)" -GUI OPTIONAL STRING_OR_ENCODING CSV2_IN_ENCODING_CSV2_2 SYSTEM%* Character Encoding: -DEFAULT_MACRO CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2 No -CSV2_2_PRESERVE_SOURCE_ENCODING "$(CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2)" -GUI CHOICE CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2 Yes%No Preserve Source Encoding: -DEFAULT_MACRO CSV2_IN_SKIPPED_LINES_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_SKIPPED_LINES_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%READ_SKIPPED_LINES Skipped Lines -DEFAULT_MACRO CSV2_IN_READ_SKIPPED_LINES_CSV2_2 No -CSV2_2_READ_SKIPPED_LINES "$(CSV2_IN_READ_SKIPPED_LINES_CSV2_2)" -GUI CHOICE CSV2_IN_READ_SKIPPED_LINES_CSV2_2 Yes%No Read Skipped Lines: -DEFAULT_MACRO CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_2 -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%SCAN_MAX_ROWS%SCAN_TYPES%SCAN_ADDITIONAL_FIELDS Schema Generation -DEFAULT_MACRO CSV2_IN_SCAN_MAX_ROWS_CSV2_2 1000 -CSV2_2_SCAN_MAX_ROWS "$(CSV2_IN_SCAN_MAX_ROWS_CSV2_2)" -DEFAULT_MACRO CSV2_IN_SCAN_TYPES_CSV2_2 No -CSV2_2_SCAN_TYPES "$(CSV2_IN_SCAN_TYPES_CSV2_2)" -DEFAULT_MACRO CSV2_IN_SCAN_ADDITIONAL_FIELDS_CSV2_2 No -CSV2_2_SCAN_ADDITIONAL_FIELDS "$(CSV2_IN_SCAN_ADDITIONAL_FIELDS_CSV2_2)" -DEFAULT_MACRO CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2 -CSV2_2_EXPOSE_ATTRS_GROUP "$(CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2)" -GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2 CSV2_EXPOSE_FORMAT_ATTRS Schema Attributes -# =========================================================================== -DEFAULT_MACRO CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2 -CSV2_2_NETWORK_AUTHENTICATION "$(CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2)" -GUI OPTIONAL AUTHENTICATOR CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2 CONTAINER%GROUP%CONTAINER_TITLE%"Network Authentication"%PROMPT_TYPE%NETWORK Network Authentication -# =========================================================================== -DEFAULT_MACRO CSV2_IN_ATTRIBUTE_READING_CSV2_2 DEFLINE -CSV2_2_ATTRIBUTE_READING "$(CSV2_IN_ATTRIBUTE_READING_CSV2_2)" -CSV2_2_DATASET "$(SourceDataset_CSV2_2)" -CSV2_2_GENERATE_FME_BUILD_NUM 17271 -#! END_SOURCE_HEADER -#! START_WB_HEADER MACRO WB_KEYWORD "CSV2_1" #! END_WB_HEADER #! START_SOURCE_HEADER CSV2 CSV2_1 @@ -2776,7 +2705,7 @@ INCLUDE [ if {{$(SourceDataset)} != ""} { \ } ] #! END_WB_HEADER #! START_SOURCE_HEADER CSV2 CSV2_1 -DEFAULT_MACRO SourceDataset_CSV2_1 $(RecordSpecCSVs) +DEFAULT_MACRO SourceDataset_CSV2_1 $(ValueSpecCSVs) GUI MULTIFILE SourceDataset_CSV2_1 CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|All_Files|* Source CSV (Comma Separated Value) File(s): DEFAULT_MACRO CSV2_IN_DATASET_GROUP_CSV2_1 GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_DATASET_GROUP_CSV2_1 FME_DISCLOSURE_OPEN%FEATURE_TYPE_NAME Dataset Parameters @@ -2856,6 +2785,102 @@ CSV2_1_DATASET "$(SourceDataset_CSV2_1)" CSV2_1_GENERATE_FME_BUILD_NUM 17271 #! END_SOURCE_HEADER #! START_WB_HEADER +MACRO WB_KEYWORD "CSV2_2" +#! END_WB_HEADER +#! START_SOURCE_HEADER CSV2 CSV2_2 +# ===================================================================== +# The following GUI line prompts for a file to be used as the source CSV file. +# The user input is stored in a macro, which is then used to define +# the dataset to be read. +# The dataset this mapping file was generated from was: +#! END_SOURCE_HEADER +#! START_WB_HEADER +DEFAULT_MACRO SourceDataset +INCLUDE [ if {{$(SourceDataset)} != ""} { \ + puts {DEFAULT_MACRO SourceDataset_CSV2_2 $(SourceDataset)} \ + } ] +#! END_WB_HEADER +#! START_SOURCE_HEADER CSV2 CSV2_2 +DEFAULT_MACRO SourceDataset_CSV2_2 $(RecordSpecCSVs) +GUI MULTIFILE SourceDataset_CSV2_2 CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|All_Files|* Source CSV (Comma Separated Value) File(s): +DEFAULT_MACRO CSV2_IN_DATASET_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_DATASET_GROUP_CSV2_2 FME_DISCLOSURE_OPEN%FEATURE_TYPE_NAME Dataset Parameters +DEFAULT_MACRO CSV2_IN_FEATURE_TYPE_NAME_CSV2_2 FORMAT_NAME +CSV2_2_FEATURE_TYPE_NAME "$(CSV2_IN_FEATURE_TYPE_NAME_CSV2_2)" +DEFAULT_MACRO CSV2_IN_FIELDS_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELDS_GROUP_CSV2_2 FME_DISCLOSURE_OPEN%DELIMITER%FIELD_NAMES_LINE%DATA_START_LINE Fields +DEFAULT_MACRO CSV2_IN_DELIMITER_CSV2_2 +CSV2_2_DELIMITER "$(CSV2_IN_DELIMITER_CSV2_2)" +GUI STRING_OR_CHOICE_ENCODED CSV2_IN_DELIMITER_CSV2_2 ,%;%|%space%tab Delimiter Character: +DEFAULT_MACRO CSV2_IN_FIELD_NAMES_LINE_CSV2_2 1 +CSV2_2_FIELD_NAMES_LINE "$(CSV2_IN_FIELD_NAMES_LINE_CSV2_2)" +GUI OPTIONAL RANGE_SLIDER CSV2_IN_FIELD_NAMES_LINE_CSV2_2 1%MAX%0 Field Names Line: +DEFAULT_MACRO CSV2_IN_DATA_START_LINE_CSV2_2 2 +CSV2_2_DATA_START_LINE "$(CSV2_IN_DATA_START_LINE_CSV2_2)" +GUI RANGE_SLIDER CSV2_IN_DATA_START_LINE_CSV2_2 1%MAX%0 Data Start Line: +DEFAULT_MACRO CSV2_IN_ADVANCED_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_ADVANCED_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%FIELD_SEPARATION_GROUP%FIELD_NAMES_GROUP%FIELD_VALUES_GROUP%ENCODING_GROUP%SKIPPED_LINES_GROUP%SCHEMA_GENERATION_GROUP Advanced +DEFAULT_MACRO CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%SKIP_DUPLICATE_DELIMITERS%FIELD_QUALIFIER_CHARACTER%FIELD_QUALIFIER_ESCAPE_CHARACTER Field Separation +DEFAULT_MACRO CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2 No +CSV2_2_SKIP_DUPLICATE_DELIMITERS "$(CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2)" +GUI CHOICE CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_2 Yes%No Skip Duplicate Delimiters: +DEFAULT_MACRO CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2 +CSV2_2_FIELD_QUALIFIER_CHARACTER "$(CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2)" +GUI OPTIONAL STRING_ENCODED CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_2 Field Qualifier Character: +DEFAULT_MACRO CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2 +CSV2_2_FIELD_QUALIFIER_ESCAPE_CHARACTER "$(CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2)" +GUI OPTIONAL STRING_ENCODED CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_2 Field Qualifier Escape Character: +DEFAULT_MACRO CSV2_IN_FIELD_NAMES_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELD_NAMES_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%CASE_SENSITIVE_FIELD_NAMES%ENFORCE_STRICT_SCHEMA Field Names +DEFAULT_MACRO CSV2_IN_TRIM_FIELD_NAME_QUALIFIERS_CSV2_2 Yes +CSV2_2_TRIM_FIELD_NAME_QUALIFIERS "$(CSV2_IN_TRIM_FIELD_NAME_QUALIFIERS_CSV2_2)" +DEFAULT_MACRO CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2 Yes +CSV2_2_CASE_SENSITIVE_FIELD_NAMES "$(CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2)" +GUI CHOICE CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_2 Yes%No Case Sensitive Field Names: +DEFAULT_MACRO CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2 No +CSV2_2_ENFORCE_STRICT_SCHEMA "$(CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2)" +GUI CHOICE CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_2 Yes%No Enforce Strict Schema: +DEFAULT_MACRO CSV2_IN_FIELD_VALUES_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_FIELD_VALUES_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%TRIM_FIELD_VALUE_QUALIFIERS Field Values +DEFAULT_MACRO CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2 Yes +CSV2_2_TRIM_FIELD_VALUE_QUALIFIERS "$(CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2)" +GUI CHOICE CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_2 Yes%No Trim Field Qualifier Characters: +DEFAULT_MACRO CSV2_IN_ENCODING_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_ENCODING_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%ENCODING%PRESERVE_SOURCE_ENCODING Encoding +DEFAULT_MACRO CSV2_IN_ENCODING_CSV2_2 +CSV2_2_ENCODING "$(CSV2_IN_ENCODING_CSV2_2)" +GUI OPTIONAL STRING_OR_ENCODING CSV2_IN_ENCODING_CSV2_2 SYSTEM%* Character Encoding: +DEFAULT_MACRO CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2 No +CSV2_2_PRESERVE_SOURCE_ENCODING "$(CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2)" +GUI CHOICE CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_2 Yes%No Preserve Source Encoding: +DEFAULT_MACRO CSV2_IN_SKIPPED_LINES_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_SKIPPED_LINES_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%READ_SKIPPED_LINES Skipped Lines +DEFAULT_MACRO CSV2_IN_READ_SKIPPED_LINES_CSV2_2 No +CSV2_2_READ_SKIPPED_LINES "$(CSV2_IN_READ_SKIPPED_LINES_CSV2_2)" +GUI CHOICE CSV2_IN_READ_SKIPPED_LINES_CSV2_2 Yes%No Read Skipped Lines: +DEFAULT_MACRO CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_2 +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_SCHEMA_GENERATION_GROUP_CSV2_2 FME_DISCLOSURE_CLOSED%SCAN_MAX_ROWS%SCAN_TYPES%SCAN_ADDITIONAL_FIELDS Schema Generation +DEFAULT_MACRO CSV2_IN_SCAN_MAX_ROWS_CSV2_2 1000 +CSV2_2_SCAN_MAX_ROWS "$(CSV2_IN_SCAN_MAX_ROWS_CSV2_2)" +DEFAULT_MACRO CSV2_IN_SCAN_TYPES_CSV2_2 No +CSV2_2_SCAN_TYPES "$(CSV2_IN_SCAN_TYPES_CSV2_2)" +DEFAULT_MACRO CSV2_IN_SCAN_ADDITIONAL_FIELDS_CSV2_2 No +CSV2_2_SCAN_ADDITIONAL_FIELDS "$(CSV2_IN_SCAN_ADDITIONAL_FIELDS_CSV2_2)" +DEFAULT_MACRO CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2 +CSV2_2_EXPOSE_ATTRS_GROUP "$(CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2)" +GUI OPTIONAL DISCLOSUREGROUP CSV2_IN_EXPOSE_ATTRS_GROUP_CSV2_2 CSV2_EXPOSE_FORMAT_ATTRS Schema Attributes +# =========================================================================== +DEFAULT_MACRO CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2 +CSV2_2_NETWORK_AUTHENTICATION "$(CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2)" +GUI OPTIONAL AUTHENTICATOR CSV2_IN_NETWORK_AUTHENTICATION_CSV2_2 CONTAINER%GROUP%CONTAINER_TITLE%"Network Authentication"%PROMPT_TYPE%NETWORK Network Authentication +# =========================================================================== +DEFAULT_MACRO CSV2_IN_ATTRIBUTE_READING_CSV2_2 DEFLINE +CSV2_2_ATTRIBUTE_READING "$(CSV2_IN_ATTRIBUTE_READING_CSV2_2)" +CSV2_2_DATASET "$(SourceDataset_CSV2_2)" +CSV2_2_GENERATE_FME_BUILD_NUM 17271 +#! END_SOURCE_HEADER +#! START_WB_HEADER MACRO WB_KEYWORD "POSTGRES_1" #! END_WB_HEADER #! START_DEST_HEADER POSTGRES POSTGRES_1 @@ -2869,24 +2894,32 @@ INCLUDE [ if {"$(DestDataset)" != ""} { \ #! START_DEST_HEADER POSTGRES POSTGRES_1 DEFAULT_MACRO DestDataset_POSTGRES_1 $(DestDataset_POSTGRES) GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGRES_1 FMT:POSTGRES Connection: -DEFAULT_MACRO POSTGRES_OUT_WRITER_MODE_POSTGRES_1 INSERT +DEFAULT_MACRO POSTGRES_OUT_STRICT_TEMPORAL_TYPE_HANDLING_POSTGRES_1 Yes +POSTGRES_1_STRICT_TEMPORAL_TYPE_HANDLING "$(POSTGRES_OUT_STRICT_TEMPORAL_TYPE_HANDLING_POSTGRES_1)" +DEFAULT_MACRO POSTGRES_OUT_FME_CONNECTION_GROUP_POSTGRES_1 +GUI OPTIONAL DISCLOSUREGROUP POSTGRES_OUT_FME_CONNECTION_GROUP_POSTGRES_1 POSTGRES_OUT_NAMED_CONNECTION Database Connection +DEFAULT_MACRO POSTGRES_OUT_ADVANCED_POSTGRES_1 +GUI OPTIONAL DISCLOSUREGROUP POSTGRES_OUT_ADVANCED_POSTGRES_1 POSTGRES_OUT_BULK_COPY_POSTGRES_1%POSTGRES_OUT_START_TRANSACTION_POSTGRES_1%POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1%POSTGRES_OUT_BEGIN_SQL_POSTGRES_1%POSTGRES_OUT_END_SQL_POSTGRES_1 Advanced +DEFAULT_MACRO POSTGRES_OUT_BULK_COPY_POSTGRES_1 YES +POSTGRES_1_BULK_COPY "$(POSTGRES_OUT_BULK_COPY_POSTGRES_1)" +GUI CHOICE POSTGRES_OUT_BULK_COPY_POSTGRES_1 YES%NO Bulk Insert DEFAULT_MACRO POSTGRES_OUT_START_TRANSACTION_POSTGRES_1 0 +POSTGRES_1_START_TRANSACTION "$(POSTGRES_OUT_START_TRANSACTION_POSTGRES_1)" GUI TEXT POSTGRES_OUT_START_TRANSACTION_POSTGRES_1 Starting Feature: -POSTGRES_1_START_TRANSACTION $(POSTGRES_OUT_START_TRANSACTION_POSTGRES_1) DEFAULT_MACRO POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1 1000 -GUI TEXT POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1 Features Per Transaction: -POSTGRES_1_TRANSACTION_INTERVAL $(POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1) -DEFAULT_MACRO POSTGRES_OUT_BULK_COPY_POSTGRES_1 YES -GUI CHOICE POSTGRES_OUT_BULK_COPY_POSTGRES_1 YES%NO Bulk Insert: -POSTGRES_1_BULK_COPY $(POSTGRES_OUT_BULK_COPY_POSTGRES_1) +POSTGRES_1_TRANSACTION_INTERVAL "$(POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1)" +GUI TEXT POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1 Features Per Transaction DEFAULT_MACRO POSTGRES_OUT_BEGIN_SQL_POSTGRES_1 -GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGRES_OUT_BEGIN_SQL_POSTGRES_1 MODE,SQL;FORMAT,POSTGRES SQL Statement to Execute Before Translation: POSTGRES_1_BEGIN_SQL "$(POSTGRES_OUT_BEGIN_SQL_POSTGRES_1)" +GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGRES_OUT_BEGIN_SQL_POSTGRES_1 MODE,SQL;FORMAT,POSTGRES SQL To Run Before Write DEFAULT_MACRO POSTGRES_OUT_END_SQL_POSTGRES_1 -GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGRES_OUT_END_SQL_POSTGRES_1 MODE,SQL;FORMAT,POSTGRES SQL Statement to Execute After Translation: POSTGRES_1_END_SQL "$(POSTGRES_OUT_END_SQL_POSTGRES_1)" +GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGRES_OUT_END_SQL_POSTGRES_1 MODE,SQL;FORMAT,POSTGRES SQL To Run After Write +DEFAULT_MACRO POSTGRES_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGRES_1 FEATURE +POSTGRES_1_COORDINATE_SYSTEM_GRANULARITY "$(POSTGRES_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGRES_1)" +# ============================================================================ +POSTGRES_1_GENERATE_FME_BUILD_NUM 20624 POSTGRES_1_DATASET "$(DestDataset_POSTGRES_1)" -POSTGRES_1_GENERATE_FME_BUILD_NUM 16492 #! END_DEST_HEADER #! START_WB_HEADER MACRO WB_KEYWORD "POSTGRES_2" @@ -2900,7 +2933,7 @@ INCLUDE [ if {"$(DestDataset)" != ""} { \ } ] #! END_WB_HEADER #! START_DEST_HEADER POSTGRES POSTGRES_2 -DEFAULT_MACRO DestDataset_POSTGRES_2 $(DestDataset_POSTGRES_3) +DEFAULT_MACRO DestDataset_POSTGRES_2 $(DestDataset_POSTGRES_5) GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGRES_2 FMT:POSTGRES Connection: DEFAULT_MACRO POSTGRES_OUT_STRICT_TEMPORAL_TYPE_HANDLING_POSTGRES_2 Yes POSTGRES_2_STRICT_TEMPORAL_TYPE_HANDLING "$(POSTGRES_OUT_STRICT_TEMPORAL_TYPE_HANDLING_POSTGRES_2)" @@ -2926,7 +2959,7 @@ GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGRES_OUT_END_SQL_POSTGRES_2 MODE,SQL;FORMAT,P DEFAULT_MACRO POSTGRES_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGRES_2 FEATURE POSTGRES_2_COORDINATE_SYSTEM_GRANULARITY "$(POSTGRES_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGRES_2)" # ============================================================================ -POSTGRES_2_GENERATE_FME_BUILD_NUM 20238 +POSTGRES_2_GENERATE_FME_BUILD_NUM 20624 POSTGRES_2_DATASET "$(DestDataset_POSTGRES_2)" #! END_DEST_HEADER #! START_WB_HEADER @@ -2942,17 +2975,17 @@ FME_REPROJECTION_ENGINE FME FME_IMPLICIT_CSMAP_REPROJECTION_MODE Auto FME_GEOMETRY_HANDLING Enhanced FME_STROKE_MAX_DEVIATION 0 -DEFAULT_MACRO DATASET_KEYWORD_CSV2_2 CSV2_2 DEFAULT_MACRO DATASET_KEYWORD_CSV2_1 CSV2_1 +DEFAULT_MACRO DATASET_KEYWORD_CSV2_2 CSV2_2 DEFAULT_MACRO DATASET_KEYWORD_POSTGRES_1 POSTGRES_1 DEFAULT_MACRO DATASET_KEYWORD_POSTGRES_2 POSTGRES_2 # ------------------------------------------------------------------------- -CSV2_2_READER_META_ATTRIBUTES fme_feature_type +CSV2_1_READER_META_ATTRIBUTES fme_feature_type # ------------------------------------------------------------------------- -CSV2_1_READER_META_ATTRIBUTES fme_feature_type +CSV2_2_READER_META_ATTRIBUTES fme_feature_type # ------------------------------------------------------------------------- @@ -2963,20 +2996,14 @@ MULTI_READER_CONTINUE_ON_READER_FAILURE No MACRO WORKSPACE_NAME Load_Value_Specs_To_DB MACRO FME_VIEWER_APP fmedatainspector # ------------------------------------------------------------------------- - -INCLUDE [ if {[info exists env(FME_TEMP)] && [file isdirectory $env(FME_TEMP)]} {set gVisualizerTemp $env(FME_TEMP)} elseif {[info exists env(TEMP)] && [file isdirectory $env(TEMP)]} {set gVisualizerTemp $env(TEMP)} elseif { $tcl_platform(platform) == "unix" } {set gVisualizerTemp "/tmp"} else {set gVisualizerTemp c:}; regsub -all {[ ,&]} {$(WORKSPACE_NAME)} {_} wsname; set gVisualizerTemp $gVisualizerTemp/${wsname}_[clock format [clock seconds] -format %H%M%S]; puts "MACRO WORKSPACE_TEMP_DIR $gVisualizerTemp"; -MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/inspector.ffs -MACRO VISUALIZER_SCHEMA_FILE $(WORKSPACE_TEMP_DIR)/inspector.fsc -MACRO VISUALIZER_CREATE_SPATIAL_INDEX YES -# ------------------------------------------------------------------------- -CSV2_2_DEF CSV fme_attribute_reading defined FileCode string Name string Value string ValueDesc string ValueType string +CSV2_1_DEF CSV fme_attribute_reading defined FileCode string Name string Value string ValueDesc string ValueType string # ------------------------------------------------------------------------- -CSV2_1_DEF CSV fme_attribute_reading defined ItemType string FileCode string RecordName string RecordTypeValue string RecordLabel string Name string Label string Start string Len string Occurrences string ZeroFill string DecimalChar string Decimal string FMETYPE string +CSV2_2_DEF CSV fme_attribute_reading defined ItemType string FileCode string RecordName string RecordTypeValue string RecordLabel string Name string Label string Start string Len string Occurrences string ZeroFill string DecimalChar string Decimal string FMETYPE string # ------------------------------------------------------------------------- -FACTORY_DEF * RoutingFactory FACTORY_NAME "Router and Unexpected Input Remover" COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE * ROUTE CSV2 CSV2_2::CSV multi_reader_keyword,$(DATASET_KEYWORD_CSV2_2) TO FME_GENERIC ::CSV ALIAS_GEOMETRY ROUTE CSV2 CSV2_1::CSV multi_reader_keyword,$(DATASET_KEYWORD_CSV2_1) TO FME_GENERIC ::CSV ALIAS_GEOMETRY MERGE_INPUT Yes OUTPUT ROUTED FEATURE_TYPE * -CSV2_2_MERGE_DEF CSV2_2::CSV EXACT CSV +FACTORY_DEF * RoutingFactory FACTORY_NAME "Router and Unexpected Input Remover" COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE * ROUTE CSV2 CSV2_1::CSV multi_reader_keyword,$(DATASET_KEYWORD_CSV2_1) TO FME_GENERIC ::CSV ALIAS_GEOMETRY ROUTE CSV2 CSV2_2::CSV multi_reader_keyword,$(DATASET_KEYWORD_CSV2_2) TO FME_GENERIC ::CSV ALIAS_GEOMETRY MERGE_INPUT Yes OUTPUT ROUTED FEATURE_TYPE * CSV2_1_MERGE_DEF CSV2_1::CSV EXACT CSV +CSV2_2_MERGE_DEF CSV2_2::CSV EXACT CSV # ------------------------------------------------------------------------- FACTORY_DEF * TeeFactory FACTORY_NAME "CSV_CSV2_2 Splitter" INPUT FEATURE_TYPE CSV multi_reader_keyword $(DATASET_KEYWORD_CSV2_2) OUTPUT FEATURE_TYPE CSV_CSV2_2 @@ -2985,15 +3012,15 @@ FACTORY_DEF * TeeFactory FACTORY_NAME "CSV_CSV2_2 Splitter" INPUT FEATURE_ FACTORY_DEF * TeeFactory FACTORY_NAME "CSV_CSV2_1 Splitter" INPUT FEATURE_TYPE CSV multi_reader_keyword $(DATASET_KEYWORD_CSV2_1) OUTPUT FEATURE_TYPE CSV_CSV2_1 DEFAULT_MACRO WB_CURRENT_CONTEXT # ------------------------------------------------------------------------- -MACRO SQLCreator_DIRECTIVES NAMED_CONNECTION DHSDataReadonly QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes +MACRO SQLCreator_DIRECTIVES QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes NAMED_CONNECTION DHSadminlocaldocker CREATE_FEATURE_TABLES_FROM_DATA Yes READ_TIME_WITH_TIMEZONE Yes FACTORY_DEF * CreationFactory FACTORY_NAME SQLCreator_Creator OUTPUT FEATURE_TYPE _____CREATED______ -FACTORY_DEF * QueryFactory FACTORY_NAME SQLCreator INPUT FEATURE_TYPE _____CREATED______ QUERY_SQL selectdistinctsurveyidfromdhs_survey_specs.dhs_table_specs_flat COMBINE_ATTRIBUTES RESULT_ONLY COMBINE_GEOMETRY RESULT_ONLY READER_TYPE POSTGRES READER_DATASET "DHSDataReadonly" READER_PARAMS $(SQLCreator_DIRECTIVES) QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES CONTINUE_ON_READER_ERROR YES OUTPUT RESULT FEATURE_TYPE SQLCreator_RESULT @SupplyAttributes(fme_feature_type,SQLCreator) OUTPUT READER_ERROR FEATURE_TYPE SQLCreator_ +FACTORY_DEF * QueryFactory FACTORY_NAME SQLCreator INPUT FEATURE_TYPE _____CREATED______ QUERY_SQL selectdistinctsurveyidfromdhs_survey_specs.dhs_table_specs_flat COMBINE_ATTRIBUTES RESULT_ONLY COMBINE_GEOMETRY RESULT_ONLY READER_TYPE POSTGRES READER_DATASET "DHSadminlocaldocker" READER_PARAMS $(SQLCreator_DIRECTIVES) QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES CONTINUE_ON_READER_ERROR YES OUTPUT RESULT FEATURE_TYPE SQLCreator_RESULT @SupplyAttributes(fme_feature_type,SQLCreator) OUTPUT READER_ERROR FEATURE_TYPE SQLCreator_ FACTORY_DEF * TeeFactory FACTORY_NAME "SQLCreator Transformer Output Nuker" INPUT FEATURE_TYPE SQLCreator_ FACTORY_DEF * BranchingFactory FACTORY_NAME "SQLCreator_RESULT Brancher -1 53" INPUT FEATURE_TYPE SQLCreator_RESULT TARGET_FACTORY "$(WB_CURRENT_CONTEXT)_CREATOR_BRANCH_TARGET" MAXIMUM_COUNT None OUTPUT PASSED FEATURE_TYPE * # ------------------------------------------------------------------------- -MACRO SQLCreator_2_DIRECTIVES NAMED_CONNECTION DHSDataReadonly QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes +MACRO SQLCreator_2_DIRECTIVES QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes NAMED_CONNECTION DHSadminlocaldocker CREATE_FEATURE_TABLES_FROM_DATA Yes READ_TIME_WITH_TIMEZONE Yes FACTORY_DEF * CreationFactory FACTORY_NAME SQLCreator_2_Creator OUTPUT FEATURE_TYPE _____CREATED______ -FACTORY_DEF * QueryFactory FACTORY_NAME SQLCreator_2 INPUT FEATURE_TYPE _____CREATED______ QUERY_SQL selectdistinctsurveyidfromdhs_survey_specs.dhs_value_descs COMBINE_ATTRIBUTES RESULT_ONLY COMBINE_GEOMETRY RESULT_ONLY READER_TYPE POSTGRES READER_DATASET "DHSDataReadonly" READER_PARAMS $(SQLCreator_2_DIRECTIVES) QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES CONTINUE_ON_READER_ERROR YES OUTPUT RESULT FEATURE_TYPE SQLCreator_2_RESULT @SupplyAttributes(fme_feature_type,SQLCreator_2) OUTPUT READER_ERROR FEATURE_TYPE SQLCreator_2_ +FACTORY_DEF * QueryFactory FACTORY_NAME SQLCreator_2 INPUT FEATURE_TYPE _____CREATED______ QUERY_SQL selectdistinctsurveyidfromdhs_survey_specs.dhs_value_descs COMBINE_ATTRIBUTES RESULT_ONLY COMBINE_GEOMETRY RESULT_ONLY READER_TYPE POSTGRES READER_DATASET "DHSadminlocaldocker" READER_PARAMS $(SQLCreator_2_DIRECTIVES) QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES CONTINUE_ON_READER_ERROR YES OUTPUT RESULT FEATURE_TYPE SQLCreator_2_RESULT @SupplyAttributes(fme_feature_type,SQLCreator_2) OUTPUT READER_ERROR FEATURE_TYPE SQLCreator_2_ FACTORY_DEF * TeeFactory FACTORY_NAME "SQLCreator_2 Transformer Output Nuker" INPUT FEATURE_TYPE SQLCreator_2_ FACTORY_DEF * BranchingFactory FACTORY_NAME "SQLCreator_2_RESULT Brancher -1 62" INPUT FEATURE_TYPE SQLCreator_2_RESULT TARGET_FACTORY "$(WB_CURRENT_CONTEXT)_CREATOR_BRANCH_TARGET" MAXIMUM_COUNT None OUTPUT PASSED FEATURE_TYPE * # ------------------------------------------------------------------------- @@ -3001,19 +3028,19 @@ FACTORY_DEF * TeeFactory FACTORY_NAME "$(WB_CURRENT_CONTEXT)_CREATOR_BRANCH_TA # ------------------------------------------------------------------------- MACRO casemac INCLUDE [ if { {no} == {no} } { puts {MACRO casemac -nocase}} ] -Tcl2 proc StringSearcher_236af038_0d88_4883_b1b1_e6c68f28d98e5_doGrep { stringToSearch regExp matchedChars matches matchedParts } { set subMatches 1; set matches [FME_DecodeText $matches]; set matchedParts [FME_DecodeText $matchedParts]; set stringToSearch [FME_ReplaceRegex "\015\012?" [FME_DecodeText $stringToSearch] "\012"]; set results [FME_GetRegex $(casemac) [FME_DecodeText $regExp] $stringToSearch]; if {[llength [lindex $results 0]] > 0} { set attrName [FME_DecodeText $matchedChars]; FME_SetAttribute $attrName [lindex [lindex [lindex $results 0] 0] 0]; set index 0; set parts [lindex $results 1]; foreach part $parts { if {[llength $matchedParts] > 0} { set attrName "$matchedParts{$index}.part"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matchedParts{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; incr index; }; set index 0; set regexMatches [lindex $results 0]; foreach part $regexMatches { if {[llength $matches] > 0} { set attrName "$matches{$index}.match"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matches{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; set isFirstMatch 0; incr index; }; return $index; }; return -1; } -FACTORY_DEF * TestFactory FACTORY_NAME StringSearcher INPUT FEATURE_TYPE CSV_CSV2_2 TEST @Tcl2("StringSearcher_236af038_0d88_4883_b1b1_e6c68f28d98e5_doGrep {@EvaluateExpression(FDIV,STRING_ENCODED,ValueFileCode,StringSearcher)} {^d+} {surveyid} {} {}") > 0 OUTPUT PASSED FEATURE_TYPE StringSearcher_MATCHED +Tcl2 proc StringSearcher_144759a2_a059_4288_91d1_823acf1753c13_doGrep { stringToSearch regExp matchedChars matches matchedParts } { set subMatches 1; set matches [FME_DecodeText $matches]; set matchedParts [FME_DecodeText $matchedParts]; set stringToSearch [FME_ReplaceRegex "\015\012?" [FME_DecodeText $stringToSearch] "\012"]; set results [FME_GetRegex $(casemac) [FME_DecodeText $regExp] $stringToSearch]; if {[llength [lindex $results 0]] > 0} { set attrName [FME_DecodeText $matchedChars]; FME_SetAttribute $attrName [lindex [lindex [lindex $results 0] 0] 0]; set index 0; set parts [lindex $results 1]; foreach part $parts { if {[llength $matchedParts] > 0} { set attrName "$matchedParts{$index}.part"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matchedParts{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; incr index; }; set index 0; set regexMatches [lindex $results 0]; foreach part $regexMatches { if {[llength $matches] > 0} { set attrName "$matches{$index}.match"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matches{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; set isFirstMatch 0; incr index; }; return $index; }; return -1; } +FACTORY_DEF * TestFactory FACTORY_NAME StringSearcher INPUT FEATURE_TYPE CSV_CSV2_1 TEST @Tcl2("StringSearcher_144759a2_a059_4288_91d1_823acf1753c13_doGrep {@EvaluateExpression(FDIV,STRING_ENCODED,ValueFileCode,StringSearcher)} {^d+} {surveyid} {} {}") > 0 OUTPUT PASSED FEATURE_TYPE StringSearcher_MATCHED # ------------------------------------------------------------------------- MACRO casemac INCLUDE [ if { {NO} == {NO} } { puts {MACRO casemac -nocase}} ] -Tcl2 proc StringReplacer_d0969c27_e80a_4c0a_a51c_fe5688a29cca5_replacer { findText replaceText invalidValue} { set attrs [split \"Value\" {,}]; set findValue [FME_DecodeText $findText]; set replaceValue [FME_DecodeText $replaceText]; foreach attr $attrs { set attrVal [FME_GetAttribute $attr]; set noMatch true; if {{NO} == {YES}} { set newAttrVal [FME_ReplaceRegex $(casemac) $findValue $attrVal $replaceValue]; } else { set newAttrVal [string map $(casemac) [list $findValue $replaceValue] $attrVal]; }; if { $newAttrVal ne $attrVal } { set noMatch false; FME_SetAttribute $attr $newAttrVal; }; if {$noMatch} { if { $invalidValue == {FME_NULL_VALUE} } { FME_SetAttributeNull $attr; } elseif { $invalidValue != {_FME_NO_OP_} } { FME_SetAttribute $attr [FME_DecodeText $invalidValue]; }; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME StringReplacer INPUT FEATURE_TYPE StringSearcher_MATCHED OUTPUT FEATURE_TYPE StringReplacer_OUTPUT @Tcl2("StringReplacer_d0969c27_e80a_4c0a_a51c_fe5688a29cca5_replacer {} {} {_FME_NO_OP_}") +Tcl2 proc StringReplacer_780dec02_a75e_499a_a6df_7f4b005691903_replacer { findText replaceText invalidValue} { set attrs [split \"Value\" {,}]; set findValue [FME_DecodeText $findText]; set replaceValue [FME_DecodeText $replaceText]; foreach attr $attrs { set attrVal [FME_GetAttribute $attr]; set noMatch true; if {{NO} == {YES}} { set newAttrVal [FME_ReplaceRegex $(casemac) $findValue $attrVal $replaceValue]; } else { set newAttrVal [string map $(casemac) [list $findValue $replaceValue] $attrVal]; }; if { $newAttrVal ne $attrVal } { set noMatch false; FME_SetAttribute $attr $newAttrVal; }; if {$noMatch} { if { $invalidValue == {FME_NULL_VALUE} } { FME_SetAttributeNull $attr; } elseif { $invalidValue != {_FME_NO_OP_} } { FME_SetAttribute $attr [FME_DecodeText $invalidValue]; }; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME StringReplacer INPUT FEATURE_TYPE StringSearcher_MATCHED OUTPUT FEATURE_TYPE StringReplacer_OUTPUT @Tcl2("StringReplacer_780dec02_a75e_499a_a6df_7f4b005691903_replacer {} {} {_FME_NO_OP_}") # ------------------------------------------------------------------------- -Tcl2 proc AttributeTrimmer_9a8d4074_69eb_44b7_9dc9_49fd07bb4af45_trimmer { trimChars} { set attrs [split {ValueDesc,Value} {,}]; set trimChars [FME_DecodeText $trimChars]; foreach attr $attrs { set attrVal [FME_GetAttribute $attr]; set length [string length $attrVal]; if {$length > 0} { if {[string length $trimChars] == 0} { set result [string trim $attrVal]; } else { set result [string trim $attrVal $trimChars]; }; if {[string length $result] == 0 && {Yes} == {No}} { if { {trim} == {trimleft}} { set lastPos [expr $length - 1]; set result [string range $attrVal $lastPos $lastPos ]; } elseif { {trim} == {trimright}} { set result [string range $attrVal 0 0]; } else { set midPos [expr $length / 2]; set result [string range $attrVal $midPos $midPos]; }; }; FME_SetAttribute $attr $result; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME AttributeTrimmer INPUT FEATURE_TYPE StringReplacer_OUTPUT OUTPUT FEATURE_TYPE AttributeTrimmer_OUTPUT @Tcl2("AttributeTrimmer_9a8d4074_69eb_44b7_9dc9_49fd07bb4af45_trimmer {} ") +Tcl2 proc AttributeTrimmer_5644f8b8_f227_427c_8613_004a26b5cc653_trimmer { trimChars} { set attrs [split {ValueDesc,Value} {,}]; set trimChars [FME_DecodeText $trimChars]; foreach attr $attrs { set attrVal [FME_GetAttribute $attr]; set length [string length $attrVal]; if {$length > 0} { if {[string length $trimChars] == 0} { set result [string trim $attrVal]; } else { set result [string trim $attrVal $trimChars]; }; if {[string length $result] == 0 && {Yes} == {No}} { if { {trim} == {trimleft}} { set lastPos [expr $length - 1]; set result [string range $attrVal $lastPos $lastPos ]; } elseif { {trim} == {trimright}} { set result [string range $attrVal 0 0]; } else { set midPos [expr $length / 2]; set result [string range $attrVal $midPos $midPos]; }; }; FME_SetAttribute $attr $result; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME AttributeTrimmer INPUT FEATURE_TYPE StringReplacer_OUTPUT OUTPUT FEATURE_TYPE AttributeTrimmer_OUTPUT @Tcl2("AttributeTrimmer_5644f8b8_f227_427c_8613_004a26b5cc653_trimmer {} ") # ------------------------------------------------------------------------- INCLUDE [if { {ATTRIBUTES} == {ATTRIBUTES} } { puts "MACRO FeatureMerger_3_REFERENCE_INFO ATTRIBUTES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_AND_ATTR_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_AND_ATTR_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} } { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} } { puts "MACRO FeatureMerger_3_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } else {}; ] -FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger_3 INPUT REFERENCER FEATURE_TYPE AttributeTrimmer_OUTPUT INPUT REFERENCEE FEATURE_TYPE SQLCreator_2_RESULT REFERENCE_INFO $(FeatureMerger_3_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_3) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_3) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No LIST_ATTRS_TO_INCLUDE { } LIST_ATTRS_TO_INCLUDE_MODE MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES No REFERENCEES_FIRST Yes REJECT_INVALID_GEOM YES CLEANING_TOLERANCE NONE OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_3_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_3_UNMERGED_REQUESTOR +FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger_3 FLUSH_WHEN_GROUPS_CHANGE No INPUT REFERENCER FEATURE_TYPE AttributeTrimmer_OUTPUT INPUT REFERENCEE FEATURE_TYPE SQLCreator_2_RESULT REFERENCE_INFO $(FeatureMerger_3_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_3) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_3) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No LIST_ATTRS_TO_INCLUDE { } LIST_ATTRS_TO_INCLUDE_MODE MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES NO REFERENCEES_FIRST Yes REJECT_INVALID_GEOM YES CLEANING_TOLERANCE OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_3_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_3_UNMERGED_REQUESTOR FACTORY_DEF * TeeFactory FACTORY_NAME "FeatureMerger_3 MERGED Splitter" INPUT FEATURE_TYPE FeatureMerger_3_MERGED OUTPUT FEATURE_TYPE FeatureMerger_3_MERGED_0_Mn9sgEf9BEc= OUTPUT FEATURE_TYPE FeatureMerger_3_MERGED_1_eZgB6jtr+Vw= # ------------------------------------------------------------------------- INCLUDE [ puts "MACRO REAL_RATE [expr 1 * -1 ]" ] @@ -3022,68 +3049,38 @@ FACTORY_DEF {*} SamplingFactory FACTORY_NAME Sampler SAMPLE_RATE $(REAL_RA # This way we behave nicely even if there are no output connections on the transformer. See PR#28886. FACTORY_DEF * TeeFactory FACTORY_NAME Sampler_Sampled INPUT FEATURE_TYPE ___SAMPLED___ OUTPUT FEATURE_TYPE Sampler_SAMPLED # ------------------------------------------------------------------------- -INCLUDE [ set macroLine "MACRO AttributeKeeper_e5e3f6fe_a391_4f83_a5b6_14e291fde31f5_LIST_EXP "; foreach attr [split ""] { set attr [FME_DecodeText $attr]; set attr [regsub "{}$" $attr "{}.*"]; set attr [regsub -all "{}" $attr "\\{\[0-9\]+\\}"]; append macroLine ",^$attr$"; }; puts $macroLine; ] -FACTORY_DEF {*} TeeFactory FACTORY_NAME AttributeKeeper INPUT FEATURE_TYPE Sampler_SAMPLED OUTPUT { FEATURE_TYPE AttributeKeeper_OUTPUT @KeepAttributes(fme_encoded,surveyid,fme_pcre_match,"^fme_$(AttributeKeeper_e5e3f6fe_a391_4f83_a5b6_14e291fde31f5_LIST_EXP)") } +FACTORY_DEF {*} AttributeKeeperFactory FACTORY_NAME AttributeKeeper INPUT FEATURE_TYPE Sampler_SAMPLED KEEP_ATTRS { surveyid } KEEP_LISTS { } KEEP_FME_ATTRIBUTES Yes BUILD_FEATURE_TABLES { No } OUTPUT_ON_ATTRIBUTE_CHANGE { } OUTPUT { OUTPUT FEATURE_TYPE AttributeKeeper_OUTPUT } # ------------------------------------------------------------------------- -MACRO SQLExecutor_2_DIRECTIVES NAMED_CONNECTION DHSDataReadonly QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes DISABLE_COLLECTION_SPLITTING yes USE_TRUE_POSTGIS_TYPES yes -FACTORY_DEF * QueryFactory FACTORY_NAME SQLExecutor_2 INPUT FEATURE_TYPE AttributeKeeper_OUTPUT QUERY_SQL "@EvaluateExpression(FDIV,STRING_ENCODED,SELECTdistinctcol_nameFROMdhs_survey_specs.dhs_value_descsWHEREsurveyid=Valuesurveyid,SQLExecutor_2)" READER_TYPE POSTGIS READER_DATASET "DHSDataReadonly" COMBINE_ATTRIBUTES PREFER_QUERY COMBINE_GEOMETRY QUERY_ONLY CONTINUE_ON_READER_ERROR YES QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES READER_PARAMS $(SQLExecutor_2_DIRECTIVES) OUTPUT RESULT FEATURE_TYPE SQLExecutor_2_RESULT OUTPUT QUERY FEATURE_TYPE SQLExecutor_2_INITIATOR OUTPUT READER_ERROR FEATURE_TYPE SQLExecutor_2_ +MACRO SQLExecutor_2_DIRECTIVES USE_TRUE_POSTGIS_TYPES yes QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes NAMED_CONNECTION DHSadminlocaldocker CREATE_FEATURE_TABLES_FROM_DATA Yes READ_TIME_WITH_TIMEZONE Yes DISABLE_COLLECTION_SPLITTING yes +FACTORY_DEF * QueryFactory FACTORY_NAME SQLExecutor_2 INPUT FEATURE_TYPE AttributeKeeper_OUTPUT QUERY_SQL "@EvaluateExpression(FDIV,STRING_ENCODED,SELECTdistinctcol_nameFROMdhs_survey_specs.dhs_value_descsWHEREsurveyid=Valuesurveyid,SQLExecutor_2)" READER_TYPE POSTGIS READER_DATASET "@EvaluateExpression(FDIV,STRING_ENCODED,DHSadminlocaldocker,SQLExecutor_2)" COMBINE_ATTRIBUTES PREFER_QUERY COMBINE_GEOMETRY QUERY_ONLY CONTINUE_ON_READER_ERROR YES QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES READER_PARAMS $(SQLExecutor_2_DIRECTIVES) OUTPUT RESULT FEATURE_TYPE SQLExecutor_2_RESULT OUTPUT QUERY FEATURE_TYPE SQLExecutor_2_INITIATOR OUTPUT READER_ERROR FEATURE_TYPE SQLExecutor_2_ FACTORY_DEF * TeeFactory FACTORY_NAME "SQLExecutor_2 INITIATOR Transformer Output Nuker" INPUT FEATURE_TYPE SQLExecutor_2_INITIATOR FACTORY_DEF * TeeFactory FACTORY_NAME "SQLExecutor_2 Transformer Output Nuker" INPUT FEATURE_TYPE SQLExecutor_2_ # ------------------------------------------------------------------------- INCLUDE [if { {ATTRIBUTES} == {ATTRIBUTES} } { puts "MACRO FeatureMerger_REFERENCE_INFO ATTRIBUTES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} } { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} } { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } else {}; ] -FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger GROUP_BY { surveyid } INPUT REFERENCER FEATURE_TYPE FeatureMerger_3_MERGED_1_eZgB6jtr+Vw= INPUT REFERENCEE FEATURE_TYPE SQLExecutor_2_RESULT REFERENCE_INFO $(FeatureMerger_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger) AUTO @EvaluateExpression(FDIV,STRING_ENCODED,ValueName,FeatureMerger) @EvaluateExpression(FDIV,STRING_ENCODED,Valuecol_name,FeatureMerger) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No LIST_ATTRS_TO_INCLUDE { } LIST_ATTRS_TO_INCLUDE_MODE MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES No REFERENCEES_FIRST No REJECT_INVALID_GEOM YES CLEANING_TOLERANCE NONE OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_UNMERGED_REQUESTOR +FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger GROUP_BY { surveyid } FLUSH_WHEN_GROUPS_CHANGE No INPUT REFERENCER FEATURE_TYPE FeatureMerger_3_MERGED_1_eZgB6jtr+Vw= INPUT REFERENCEE FEATURE_TYPE SQLExecutor_2_RESULT REFERENCE_INFO $(FeatureMerger_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger) AUTO @EvaluateExpression(FDIV,STRING_ENCODED,ValueName,FeatureMerger) @EvaluateExpression(FDIV,STRING_ENCODED,Valuecol_name,FeatureMerger) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No LIST_ATTRS_TO_INCLUDE { } LIST_ATTRS_TO_INCLUDE_MODE MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES NO REFERENCEES_FIRST No REJECT_INVALID_GEOM YES CLEANING_TOLERANCE OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_UNMERGED_REQUESTOR +FACTORY_DEF * TeeFactory FACTORY_NAME "FeatureMerger_MERGED Disabled Nuker -1 37" INPUT FEATURE_TYPE FeatureMerger_MERGED # ------------------------------------------------------------------------- FACTORY_DEF {*} AttrSetFactory FACTORY_NAME AttributeManager INPUT FEATURE_TYPE FeatureMerger_UNMERGED_REQUESTOR INPUT FEATURE_TYPE FeatureMerger_3_UNMERGED_REQUESTOR MULTI_FEATURE_MODE NO NULL_ATTR_MODE NO_OP ATTRSET_CREATE_DIRECTIVES _PROPAGATE_MISSING_FDIV ACTION_COLUMN 3 DEF_VAL_COLUMN 2 ATTR_ACTION { "FileCode" "FileCode" "" "REMOVE" } ATTR_ACTION { "Name" "col_name" "" "RENAME_SET_VALUE" } ATTR_ACTION { "Value" "value" "" "RENAME_SET_VALUE" } ATTR_ACTION { "ValueDesc" "value_desc" "" "RENAME_SET_VALUE" } ATTR_ACTION { "ValueType" "value_type" "" "RENAME_SET_VALUE" } OUTPUT OUTPUT FEATURE_TYPE AttributeManager_OUTPUT # ------------------------------------------------------------------------- -Tcl2 proc NullAttributeMapper_49fc2824_fdb6_4fe8_b2c1_ce9c62e1aad55_null_replacer { otherValue newValue regExp } { set attrs [split {value_desc value_type} { }]; set decode true; if { {SELECTED} == {ALL} } { set attrs [FME_AttributeNames]; set decode false; }; set decodedVal [FME_DecodeText $otherValue]; set decodedRegExp [FME_DecodeText $regExp]; foreach attr $attrs { set act {NOOP}; if {$decode} { set attr [FME_DecodeText $attr]; }; if {[FME_AttributeExists $attr]} { set attrVal [FME_GetAttribute $attr]; if {[FME_IsAttributeNull $attr] } { if {[string first {SETNULL} {SETNULL EMPTY}] != -1} { set act {SETNULL}; }; } elseif { [string first {EMPTY} {SETNULL EMPTY}] != -1 && $attrVal == {}} { set act {SETNULL}; } elseif { $otherValue != {} && [string equal $attrVal $decodedVal]} { set act {SETNULL}; } elseif { $decodedRegExp != {} && [regexp $decodedRegExp $attrVal]==1 } { set act {SETNULL}; }; } elseif {[string first {MISSINGATTR} {SETNULL EMPTY}] != -1} { set act {SETNULL}; }; if {$act == {DELETEATTR} } { FME_UnsetAttributes $attr; } elseif { $act == {SETVALUE} } { FME_SetAttribute $attr [FME_DecodeText $newValue]; } elseif { $act == {SETNULL} } { FME_SetAttributeNull $attr; } elseif { $act == {EMPTY} } { FME_SetAttribute $attr {}; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME NullAttributeMapper INPUT FEATURE_TYPE AttributeManager_OUTPUT OUTPUT FEATURE_TYPE NullAttributeMapper_OUTPUT @Tcl2("NullAttributeMapper_49fc2824_fdb6_4fe8_b2c1_ce9c62e1aad55_null_replacer {} {} {}") +Tcl2 proc NullAttributeMapper_8345acdc_ade8_4d8e_80b9_d4978703f8b83_null_replacer { otherValue newValue regExp } { set attrs [split {value_desc value_type} { }]; set decode true; if { {SELECTED} == {ALL} } { set attrs [FME_AttributeNames]; set decode false; }; set decodedVal [FME_DecodeText $otherValue]; set decodedRegExp [FME_DecodeText $regExp]; foreach attr $attrs { set act {NOOP}; if {$decode} { set attr [FME_DecodeText $attr]; }; if {[FME_AttributeExists $attr]} { set attrVal [FME_GetAttribute $attr]; if {[FME_IsAttributeNull $attr] } { if {[string first {SETNULL} {SETNULL EMPTY}] != -1} { set act {SETNULL}; }; } elseif { [string first {EMPTY} {SETNULL EMPTY}] != -1 && $attrVal == {}} { set act {SETNULL}; } elseif { $otherValue != {} && [string equal $attrVal $decodedVal]} { set act {SETNULL}; } elseif { $decodedRegExp != {} && [regexp $decodedRegExp $attrVal]==1 } { set act {SETNULL}; }; } elseif {[string first {MISSINGATTR} {SETNULL EMPTY}] != -1} { set act {SETNULL}; }; if {$act == {DELETEATTR} } { FME_UnsetAttributes $attr; } elseif { $act == {SETVALUE} } { FME_SetAttribute $attr [FME_DecodeText $newValue]; } elseif { $act == {SETNULL} } { FME_SetAttributeNull $attr; } elseif { $act == {EMPTY} } { FME_SetAttribute $attr {}; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME NullAttributeMapper INPUT FEATURE_TYPE AttributeManager_OUTPUT OUTPUT FEATURE_TYPE NullAttributeMapper_OUTPUT @Tcl2("NullAttributeMapper_8345acdc_ade8_4d8e_80b9_d4978703f8b83_null_replacer {} {} {}") # ------------------------------------------------------------------------- -# Assumption is that the workspace temp dir has been already set and created. See controldefs.cpp - ControlDefs::writeVisualizerTempDir -DEFAULT_MACRO WORKSPACE_TEMP_DIR -INCLUDE [ set grpByAttrs {}; if { [string length $grpByAttrs] > 0 } { if { [string first "@EvaluateExpression" $grpByAttrs] == 0 } { puts "MACRO CATMAC _$grpByAttrs" ; } else { set catter "" ; set comma "" ; foreach attr $grpByAttrs { set catter "$catter$comma@Value(\"$attr\")" ; set comma ",_," ; } ; puts "MACRO CATMAC _@Concatenate($catter)" ; } } else { puts "MACRO CATMAC " ; }; set safeName "[regsub -all {[^a-zA-Z0-9]} {ValuesAlreadyPresent} _]_[expr round(rand() * 1000000)]_[clock clicks -milliseconds]"; puts "MACRO SAFE_FFS_NAME $safeName"; ] -# Make the temporary directory that the FFS files will live in. -FACTORY_DEF * CreationFactory FACTORY_NAME ValuesAlreadyPresent_DirCreator OUTPUT FEATURE_TYPE __nukeme__ @Tcl2("catch {file mkdir {$(WORKSPACE_TEMP_DIR)}}") -FACTORY_DEF * TeeFactory FACTORY_NAME ValuesAlreadyPresent_DirCreatorCleanerUpper INPUT FEATURE_TYPE __nukeme__ -# Send raster, point cloud and vector features down different paths to prepare -# them for viewing -FACTORY_DEF * TestFactory FACTORY_NAME ValuesAlreadyPresent_FeatureDirector INPUT FEATURE_TYPE FeatureMerger_MERGED TEST &fme_type == "fme_raster" OUTPUT FAILED FEATURE_TYPE ValuesAlreadyPresent___NotRasterFeatures___ OUTPUT PASSED FEATURE_TYPE ValuesAlreadyPresent___RasterFeatures___ -FACTORY_DEF * TestFactory FACTORY_NAME ValuesAlreadyPresent___FeatureDirector1 INPUT FEATURE_TYPE ValuesAlreadyPresent___NotRasterFeatures___ TEST &fme_type == "fme_point_cloud" OUTPUT FAILED FEATURE_TYPE ValuesAlreadyPresent___VectorFeatures___ OUTPUT PASSED FEATURE_TYPE ValuesAlreadyPresent___PointCloudFeatures___ -# Vector features simply may get their colors overridden -FACTORY_DEF * TeeFactory FACTORY_NAME ValuesAlreadyPresent_VectorSetterUpper INPUT FEATURE_TYPE ValuesAlreadyPresent___VectorFeatures___ OUTPUT FEATURE_TYPE __viewme__ -# Point Cloud Features Might Need to be thinned -# Note that we don't call the ThinPointCloud function directly, but rather -# do it through TCL. This should allow this transformer to be used with a -# Desktop license, even though ThinPointCloud requires Professional. -Tcl2 proc ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME ValuesAlreadyPresent___PointCloudThinner INPUT FEATURE_TYPE ValuesAlreadyPresent___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_thinPointCloud") -Tcl2 proc ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME ValuesAlreadyPresent_RasterSetterUpper INPUT FEATURE_TYPE ValuesAlreadyPresent___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_prepareRaster {} {} {} {}") -# Now route all the features into the recorder, changing their -# feature type to the transformer name so that they view nicely -# ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_VIS_FEAT_TYPE changes the feature types -# of the features being visualized. -DEFAULT_MACRO ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_VIS_FEAT_TYPE ValuesAlreadyPresent$(CATMAC) -DEFAULT_MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/$(SAFE_FFS_NAME).ffs -DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO -# [PR#45549] The Data Inspector will limit the maximum number of features -# it reads on WIN32, so we will correspondingly limit the number of -# features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME ValuesAlreadyPresent_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(ValuesAlreadyPresent_f758f975_8735_4399_b105_189b8d795c425_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" STORE_SCANNED_SCHEMA YES CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA FileCode,buffer,Name,buffer,Value,buffer,ValueDesc,buffer,ValueType,buffer,surveyid,varchar200,col_name,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * TeeFactory FACTORY_NAME "ValuesAlreadyPresent (Disabled) Nuker" INPUT FEATURE_TYPE FeatureMerger_MERGED # ------------------------------------------------------------------------- -FACTORY_DEF * TestFactory FACTORY_NAME Tester_2 INPUT FEATURE_TYPE CSV_CSV2_1 TEST "@EvaluateExpression(FDIV,STRING_ENCODED,ValueRecordLabel,Tester_2)" != * ENCODED TEST "@EvaluateExpression(FDIV,STRING_ENCODED,ValueRecordName,Tester_2)" != * ENCODED BOOLEAN_OPERATOR AND COMPOSITE_TEST_EXPR "" OUTPUT PASSED FEATURE_TYPE Tester_2_PASSED +FACTORY_DEF * TestFactory FACTORY_NAME Tester_2 INPUT FEATURE_TYPE CSV_CSV2_2 TEST "@EvaluateExpression(FDIV,STRING_ENCODED,ValueRecordLabel,Tester_2)" != * ENCODED TEST "@EvaluateExpression(FDIV,STRING_ENCODED,ValueRecordName,Tester_2)" != * ENCODED BOOLEAN_OPERATOR AND COMPOSITE_TEST_EXPR "" OUTPUT PASSED FEATURE_TYPE Tester_2_PASSED # ------------------------------------------------------------------------- FACTORY_DEF {*} TeeFactory FACTORY_NAME BulkAttributeRenamer INPUT FEATURE_TYPE Tester_2_PASSED OUTPUT { FEATURE_TYPE BulkAttributeRenamer_OUTPUT @RenameAttributes(PATTERN_REPLACE, CASECHANGE, , "", "", "lower", "", "-_", "RENAME", "NONFME") } # ------------------------------------------------------------------------- MACRO casemac INCLUDE [ if { {no} == {no} } { puts {MACRO casemac -nocase}} ] -Tcl2 proc StringSearcher_2_8eac1dc0_2101_41bb_a70a_0387f336d2fb5_doGrep { stringToSearch regExp matchedChars matches matchedParts } { set subMatches 1; set matches [FME_DecodeText $matches]; set matchedParts [FME_DecodeText $matchedParts]; set stringToSearch [FME_ReplaceRegex "\015\012?" [FME_DecodeText $stringToSearch] "\012"]; set results [FME_GetRegex $(casemac) [FME_DecodeText $regExp] $stringToSearch]; if {[llength [lindex $results 0]] > 0} { set attrName [FME_DecodeText $matchedChars]; FME_SetAttribute $attrName [lindex [lindex [lindex $results 0] 0] 0]; set index 0; set parts [lindex $results 1]; foreach part $parts { if {[llength $matchedParts] > 0} { set attrName "$matchedParts{$index}.part"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matchedParts{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; incr index; }; set index 0; set regexMatches [lindex $results 0]; foreach part $regexMatches { if {[llength $matches] > 0} { set attrName "$matches{$index}.match"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matches{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; set isFirstMatch 0; incr index; }; return $index; }; return -1; } -FACTORY_DEF * TestFactory FACTORY_NAME StringSearcher_2 INPUT FEATURE_TYPE BulkAttributeRenamer_OUTPUT TEST @Tcl2("StringSearcher_2_8eac1dc0_2101_41bb_a70a_0387f336d2fb5_doGrep {@EvaluateExpression(FDIV,STRING_ENCODED,Valuefilecode,StringSearcher_2)} {^d+} {surveyid} {} {}") > 0 OUTPUT PASSED FEATURE_TYPE StringSearcher_2_MATCHED +Tcl2 proc StringSearcher_2_e9a9fc82_c6a4_435f_9d98_4262255c34863_doGrep { stringToSearch regExp matchedChars matches matchedParts } { set subMatches 1; set matches [FME_DecodeText $matches]; set matchedParts [FME_DecodeText $matchedParts]; set stringToSearch [FME_ReplaceRegex "\015\012?" [FME_DecodeText $stringToSearch] "\012"]; set results [FME_GetRegex $(casemac) [FME_DecodeText $regExp] $stringToSearch]; if {[llength [lindex $results 0]] > 0} { set attrName [FME_DecodeText $matchedChars]; FME_SetAttribute $attrName [lindex [lindex [lindex $results 0] 0] 0]; set index 0; set parts [lindex $results 1]; foreach part $parts { if {[llength $matchedParts] > 0} { set attrName "$matchedParts{$index}.part"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matchedParts{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; incr index; }; set index 0; set regexMatches [lindex $results 0]; foreach part $regexMatches { if {[llength $matches] > 0} { set attrName "$matches{$index}.match"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matches{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; set isFirstMatch 0; incr index; }; return $index; }; return -1; } +FACTORY_DEF * TestFactory FACTORY_NAME StringSearcher_2 INPUT FEATURE_TYPE BulkAttributeRenamer_OUTPUT TEST @Tcl2("StringSearcher_2_e9a9fc82_c6a4_435f_9d98_4262255c34863_doGrep {@EvaluateExpression(FDIV,STRING_ENCODED,Valuefilecode,StringSearcher_2)} {^d+} {surveyid} {} {}") > 0 OUTPUT PASSED FEATURE_TYPE StringSearcher_2_MATCHED # ------------------------------------------------------------------------- INCLUDE [if { {ATTRIBUTES} == {ATTRIBUTES} } { puts "MACRO FeatureMerger_2_REFERENCE_INFO ATTRIBUTES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_AND_ATTR_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_AND_ATTR_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} } { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} } { puts "MACRO FeatureMerger_2_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } else {}; ] -FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger_2 INPUT REFERENCER FEATURE_TYPE StringSearcher_2_MATCHED INPUT REFERENCEE FEATURE_TYPE SQLCreator_RESULT REFERENCE_INFO $(FeatureMerger_2_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_2) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_2) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No LIST_ATTRS_TO_INCLUDE { } LIST_ATTRS_TO_INCLUDE_MODE MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES No REFERENCEES_FIRST Yes REJECT_INVALID_GEOM YES CLEANING_TOLERANCE NONE OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_2_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_2_UNMERGED_REQUESTOR +FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger_2 FLUSH_WHEN_GROUPS_CHANGE No INPUT REFERENCER FEATURE_TYPE StringSearcher_2_MATCHED INPUT REFERENCEE FEATURE_TYPE SQLCreator_RESULT REFERENCE_INFO $(FeatureMerger_2_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_2) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger_2) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No LIST_ATTRS_TO_INCLUDE { } LIST_ATTRS_TO_INCLUDE_MODE MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES NO REFERENCEES_FIRST Yes REJECT_INVALID_GEOM YES CLEANING_TOLERANCE OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_2_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_2_UNMERGED_REQUESTOR # ------------------------------------------------------------------------- -MACRO SQLExecutor_DIRECTIVES NAMED_CONNECTION DHSDataReadonly QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes -FACTORY_DEF * QueryFactory FACTORY_NAME SQLExecutor INPUT FEATURE_TYPE FeatureMerger_2_MERGED QUERY_SQL "@EvaluateExpression(FDIV,STRING_ENCODED,selectnamefromdhs_survey_specs.dhs_table_specs_flatwheresurveyid=ValuesurveyidANDrecordname=ValuerecordnameANDname=Valuename,SQLExecutor)" READER_TYPE POSTGRES READER_DATASET "DHSDataReadonly" COMBINE_ATTRIBUTES RESULT_ONLY COMBINE_GEOMETRY QUERY_ONLY CONTINUE_ON_READER_ERROR YES QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES READER_PARAMS $(SQLExecutor_DIRECTIVES) OUTPUT RESULT FEATURE_TYPE SQLExecutor_RESULT OUTPUT QUERY FEATURE_TYPE SQLExecutor_INITIATOR OUTPUT READER_ERROR FEATURE_TYPE SQLExecutor_ +MACRO SQLExecutor_DIRECTIVES QUERY_FEATURE_TYPES_FOR_MERGE_FILTERS Yes NAMED_CONNECTION DHSadminlocaldocker CREATE_FEATURE_TABLES_FROM_DATA Yes READ_TIME_WITH_TIMEZONE Yes +FACTORY_DEF * QueryFactory FACTORY_NAME SQLExecutor INPUT FEATURE_TYPE FeatureMerger_2_MERGED QUERY_SQL "@EvaluateExpression(FDIV,STRING_ENCODED,selectnamefromdhs_survey_specs.dhs_table_specs_flatwheresurveyid=ValuesurveyidANDrecordname=ValuerecordnameANDname=Valuename,SQLExecutor)" READER_TYPE POSTGRES READER_DATASET "@EvaluateExpression(FDIV,STRING_ENCODED,DHSadminlocaldocker,SQLExecutor)" COMBINE_ATTRIBUTES RESULT_ONLY COMBINE_GEOMETRY QUERY_ONLY CONTINUE_ON_READER_ERROR YES QUERYFCT_PROPAGATE_IDS_ON_SQL_EXECUTE YES READER_PARAMS $(SQLExecutor_DIRECTIVES) OUTPUT RESULT FEATURE_TYPE SQLExecutor_RESULT OUTPUT QUERY FEATURE_TYPE SQLExecutor_INITIATOR OUTPUT READER_ERROR FEATURE_TYPE SQLExecutor_ FACTORY_DEF * TeeFactory FACTORY_NAME "SQLExecutor RESULT Transformer Output Nuker" INPUT FEATURE_TYPE SQLExecutor_RESULT FACTORY_DEF * TeeFactory FACTORY_NAME "SQLExecutor Transformer Output Nuker" INPUT FEATURE_TYPE SQLExecutor_ # ------------------------------------------------------------------------- @@ -3094,17 +3091,12 @@ FACTORY_DEF {*} AttrSetFactory FACTORY_NAME AttributeCreator_2 INPUT FEAT FACTORY_DEF {*} AttrSetFactory FACTORY_NAME AttributeCreator INPUT FEATURE_TYPE Tester_FAILED MULTI_FEATURE_MODE NO NULL_ATTR_MODE NO_OP ATTRSET_CREATE_DIRECTIVES _PROPAGATE_MISSING_FDIV ATTR_ACTION { "" "fme_db_operation" "SET_TO" "UPDATE" } OUTPUT OUTPUT FEATURE_TYPE AttributeCreator_OUTPUT # ------------------------------------------------------------------------- -FACTORY_DEF * RoutingFactory FACTORY_NAME "Destination Feature Type Routing Correlator" COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE * ROUTE FME_GENERIC NullAttributeMapper_OUTPUT TO POSTGRES __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,0,SupplyAttributesENCODED__wb_out_feat_type__dhs_survey_specs.dhs_value_descs GEOMETRY ROUTE FME_GENERIC AttributeCreator_2_OUTPUT TO POSTGRES __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,1,SupplyAttributesENCODED__wb_out_feat_type__dhs_survey_specs.dhs_table_specs_flat GEOMETRY ROUTE FME_GENERIC AttributeCreator_OUTPUT TO POSTGRES __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,1,SupplyAttributesENCODED__wb_out_feat_type__dhs_survey_specs.dhs_table_specs_flat GEOMETRY FEATURE_TYPE_ATTRIBUTE __wb_out_feat_type__ OUTPUT ROUTED FEATURE_TYPE * OUTPUT NOT_ROUTED FEATURE_TYPE __nuke_me__ @Tcl2("FME_StatMessage 818059 [FME_GetAttribute fme_template_feature_type] 818060 818061 fme_warn") +FACTORY_DEF * RoutingFactory FACTORY_NAME "Destination Feature Type Routing Correlator" COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE * ROUTE FME_GENERIC AttributeCreator_OUTPUT TO POSTGRES __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,1,SupplyAttributesENCODED__wb_out_feat_type__dhs_survey_specs.dhs_table_specs_flat GEOMETRY ROUTE FME_GENERIC AttributeCreator_2_OUTPUT TO POSTGRES __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,1,SupplyAttributesENCODED__wb_out_feat_type__dhs_survey_specs.dhs_table_specs_flat GEOMETRY ROUTE FME_GENERIC NullAttributeMapper_OUTPUT TO POSTGRES __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,0,SupplyAttributesENCODED__wb_out_feat_type__dhs_survey_specs.dhs_value_descs GEOMETRY FEATURE_TYPE_ATTRIBUTE __wb_out_feat_type__ OUTPUT ROUTED FEATURE_TYPE * OUTPUT NOT_ROUTED FEATURE_TYPE __nuke_me__ @Tcl2("FME_StatMessage 818059 [FME_GetAttribute fme_template_feature_type] 818060 818061 fme_warn") # ------------------------------------------------------------------------- FACTORY_DEF * TeeFactory FACTORY_NAME "Final Output Nuker" INPUT FEATURE_TYPE __nuke_me__ # ------------------------------------------------------------------------- - -DEFAULT_MACRO FME_LAUNCH_VIEWER_APP NO -FACTORY_DEF * CreationFactory FACTORY_NAME VisualizerProcessor CREATE_AT_END OUTPUT FEATURE_TYPE __NUKEME__ @TCL("if {{$(FME_LAUNCH_VIEWER_APP)} == {YES}} { if [file exists {$(WORKSPACE_TEMP_DIR)}] {set files [glob -nocomplain -directory {$(WORKSPACE_TEMP_DIR)} -tails *.ffs]; if {[llength $files] == 1} { set theFile {$(WORKSPACE_TEMP_DIR)/}; append theFile [lindex $files 0]; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -f FFS \042$theFile\042 & \175; } elseif { [llength $files] == 0 } { catch { file delete -force {$(WORKSPACE_TEMP_DIR)} }; } else { set filesNoExt {}; set lastfile {}; foreach {fileName} [lsort $files] { set fileNoExt [string trimright $fileName {.ffs}]; if [regexp {(.*)_[0-9]*$} $fileNoExt wholething fileNoExtBase] { if { $fileNoExtBase == $lastfile } { continue } }; set lastfile $fileNoExt; lappend filesNoExt $fileNoExt }; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -d {$(WORKSPACE_TEMP_DIR)} -a ffs -f FFS $filesNoExt & \175; }; }; }; ") -FACTORY_DEF * TeeFactory FACTORY_NAME VisualizerCleaner INPUT FEATURE_TYPE __NUKEME__ -# ------------------------------------------------------------------------- -POSTGRES_1_DEF dhs_survey_specs.dhs_value_descs postgres_type All postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING postgres_create_with_oids YES postgres_vacuum_analyze YES postgres_allow_serial_writing NO col_name varchar(20) value varchar(20) value_desc varchar(200) value_type varchar(20) surveyid varchar(4) +POSTGRES_1_DEF dhs_survey_specs.dhs_value_descs postgres_type All postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING postgres_vacuum_analyze YES postgres_allow_serial_writing NO col_name varchar(20) value varchar(20) value_desc varchar(200) value_type varchar(20) surveyid varchar(4) # ------------------------------------------------------------------------- POSTGRES_2_DEF dhs_survey_specs.dhs_table_specs_flat postgres_type All postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation MULTIPLE fme_table_handling USE_EXISTING fme_selection_method MATCH_COLUMNS fme_match_columns name,recordname,surveyid postgres_vacuum_analyze YES postgres_allow_serial_writing NO itemtype varchar(17) recordname varchar(20) recordtypevalue varchar(4) recordlabel varchar(85) name varchar(20) label varchar(200) start int2 len int2 fmetype varchar(13) surveyid varchar(4) diff --git a/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_survey_metadata_to_db.fmw b/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_survey_metadata_to_db.fmw index 4c3e51a..5c878fc 100644 --- a/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_survey_metadata_to_db.fmw +++ b/DHS_Preprocess_And_Ingestion/07A_Load_New_Metadata/Load_survey_metadata_to_db.fmw @@ -1,8 +1,8 @@ #! #! onMAP-FS1DHSinstance" +# "C:\Program Files\FME2020\fme.exe" C:\Users\zool1301\Documents\GitHub\DHS-DataExtraction\DHS_Preprocess_And_Ingestion\07A_Load_New_Metadata\Load_survey_metadata_to_db.fmw +# --DestDataset_POSTGRES "DHSadminlocaldocker" # #! ATTR_TYPE_ENCODING="SDF" #! BEGIN_PYTHON="" @@ -11,14 +11,14 @@ #! DESCRIPTION="" #! DESTINATION="NONE" #! DESTINATION_ROUTING_FILE="" -#! DOC_EXTENTS="2093.84 1154.09" -#! DOC_TOP_LEFT="-652.348 -716.963" +#! DOC_EXTENTS="2543.59 1225.09" +#! DOC_TOP_LEFT="-652.348 -787.963" #! END_PYTHON="" #! END_TCL="" #! EXPLICIT_BOOKMARK_ORDER="false" -#! FME_BUILD_NUM="18547" -#! FME_DOCUMENT_GUID="abb20828-0247-4083-8678-5ce8a1e88e66" -#! FME_DOCUMENT_PRIORGUID="b244887d-d76d-47fc-8588-2ccdc984525d" +#! FME_BUILD_NUM="20608" +#! FME_DOCUMENT_GUID="39870a71-0e53-4dcd-8026-e859c9c9a555" +#! FME_DOCUMENT_PRIORGUID="b244887d-d76d-47fc-8588-2ccdc984525d,abb20828-0247-4083-8678-5ce8a1e88e66" #! FME_GEOMETRY_HANDLING="Enhanced" #! FME_IMPLICIT_CSMAP_REPROJECTION_MODE="Auto" #! FME_REPROJECTION_ENGINE="FME" @@ -26,8 +26,8 @@ #! FME_STROKE_MAX_DEVIATION="0" #! HISTORY="" #! IGNORE_READER_FAILURE="No" -#! LAST_SAVE_BUILD="FME(R) 2018.1.0.2 (20180903 - Build 18547 - WIN64)" -#! LAST_SAVE_DATE="2019-05-13T17:21:07" +#! LAST_SAVE_BUILD="FME(R) 2020.1.1.0 (20200801 - Build 20608 - WIN64)" +#! LAST_SAVE_DATE="2020-09-17T12:05:57" #! LOG_FILE="" #! LOG_MAX_RECORDED_FEATURES="200" #! MARKDOWN_DESCRIPTION="" @@ -35,8 +35,9 @@ #! MAX_LOG_FEATURES="200" #! MULTI_WRITER_DATASET_ORDER="BY_ID" #! PASSWORD="" -#! PYTHON_COMPATIBILITY="27" +#! PYTHON_COMPATIBILITY="37" #! REDIRECT_TERMINATORS="NONE" +#! SAVE_ON_PROMPT_AND_RUN="Yes" #! SHOW_ANNOTATIONS="true" #! SHOW_INFO_NODES="true" #! SOURCE="NONE" @@ -45,7 +46,7 @@ #! TITLE="" #! USAGE="" #! USE_MARKDOWN="" -#! VIEW_POSITION="-1437.51 718.757" +#! VIEW_POSITION="-1434.39 300.003" #! WARN_INVALID_XFORM_PARAM="Yes" #! WORKSPACE_VERSION="1" #! ZOOM_SCALE="100" @@ -64,7 +65,7 @@ #! WRITE_DEFS="true" #! DEFLINE_TEMPLATE="" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="fme_feature_type string fme_sweep_angle string multi_reader_full_id string csv_type string csv_number_of_fields uint32 fme_type string csv_total_number_of_columns uint32 csv_skipped_line string fme_start_angle string fme_basename string fme_fill_color string fme_text_string string multi_reader_type string fme_rotation string csv_line_number uint64 fme_dataset string fme_secondary_axis string fme_primary_axis string fme_text_size string fme_geometry string multi_reader_id string fme_color string multi_reader_keyword string" +#! EXPOSABLE_ATTRS="fme_sweep_angle string fme_text_string string csv_total_number_of_columns uint32 csv_line_number uint64 fme_fill_color string fme_color string multi_reader_id string multi_reader_full_id string fme_dataset string csv_number_of_fields uint32 csv_type string fme_primary_axis string fme_secondary_axis string fme_feature_type string fme_type string fme_rotation string fme_basename string fme_start_angle string multi_reader_keyword string fme_text_size string multi_reader_type string csv_skipped_line string fme_geometry string" #! DEFLINE_PARMS="" #! ATTR_INDEX_TYPES="" #! ATTR_NAME_INVALID_CHARS="" @@ -72,6 +73,7 @@ #! ENABLED="true" #! DYNAMIC_FEATURE_TYPES_LIST_ON_MERGE="true" #! DATASET_TYPE="FILEDIR" +#! FTTR_TYPE="LISTBOX_ENCODED" #! GENERATE_FME_BUILD_NUM="18547" #! COORDSYS="" #! FEATURE_TYPES="" @@ -122,7 +124,7 @@ #! WRITE_DEFS="true" #! DEFLINE_TEMPLATE="postgres_type {FME_GEN_GEOMETRY} postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING fme_selection_method MATCH_COLUMNS fme_match_columns "" fme_where_builder_clause "" postgres_create_with_oids NO postgres_vacuum_analyze NO postgres_allow_serial_writing NO" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="fme_db_operation char(8) fme_primary_axis float8 fme_secondary_axis float8 fme_rotation float8 fme_fill_color char(50) fme_feature_type char(50) fme_sweep_angle float8 fme_text_string char(50) fme_geometry char(50) fme_type char(50) postgres_type char(50) fme_text_size float8 fme_color char(50) fme_basename char(50) fme_start_angle float8 fme_dataset char(50)" +#! EXPOSABLE_ATTRS="fme_secondary_axis float8 fme_db_operation char(8) fme_dataset char(50) fme_sweep_angle float8 fme_color char(50) fme_text_string char(50) fme_text_size float8 fme_start_angle float8 fme_type char(50) postgres_type char(50) fme_rotation float8 fme_feature_type char(50) fme_basename char(50) fme_fill_color char(50) fme_geometry char(50) fme_primary_axis float8" #! DEFLINE_PARMS=""GUI OPTIONAL NAMEDGROUP fme_configuration_group fme_configuration_common_group%fme_spatial_group%fme_advanced_group%oracle_advanced_group Table" "" "GUI OPTIONAL NAMEDGROUP fme_configuration_common_group fme_feature_operation%fme_table_handling%oracle_model%fme_update_geometry%fme_selection_group%fme_table_creation_group General" "" "GUI ACTIVECHOICE_LOOKUP fme_feature_operation Insert,INSERT,fme_update_geometry,fme_selection_group%Update,UPDATE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN%Delete,DELETE,++fme_table_handling+USE_EXISTING,fme_update_geometry,++fme_selection_group+FME_DISCLOSURE_OPEN,fme_spatial_group,fme_advanced_group,oracle_sequenced_cols%<at>Value<openparen>fme_db_operation<closeparen>,MULTIPLE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN Feature Operation" INSERT "GUI ACTIVECHOICE_LOOKUP fme_table_handling Use<space>Existing,USE_EXISTING,fme_table_creation_group%Create<space>If<space>Needed,CREATE_IF_MISSING%Drop<space>and<space>Create,DROP_CREATE%Truncate<space>Existing,TRUNCATE_EXISTING,fme_table_creation_group Table Handling" CREATE_IF_MISSING "GUI OPTIONAL DISCLOSUREGROUP fme_selection_group fme_selection_method Row Selection" "" "GUI OPTIONAL WHOLE_LINE RADIOPARAMETERGROUP fme_selection_method fme_match_columns,MATCH_COLUMNS%fme_where_builder_clause,BUILDER Row Selection Method" MATCH_COLUMNS "GUI WHOLE_LINE ATTRLIST_COMMAS fme_match_columns \" \" Match Columns" "" "GUI WHOLE_LINE TEXT_EDIT_SQL_CFG_OR_ATTR fme_where_builder_clause MODE,WHERE WHERE Clause" "" "GUI OPTIONAL DISCLOSUREGROUP fme_table_creation_group postgres_create_with_oids Table Creation Parameters" "" "GUI CHOICE postgres_create_with_oids YES%NO Create With OIDs" NO "GUI OPTIONAL DISCLOSUREGROUP fme_advanced_group postgres_vacuum_analyze%postgres_allow_serial_writing Advanced" "" "GUI CHOICE postgres_vacuum_analyze YES%NO Vacuum Analyze Table" NO "GUI CHOICE postgres_allow_serial_writing YES%NO Allow Serial Column Writing" NO " #! ATTR_INDEX_TYPES="BTree RTree Hash PrimaryKey" #! ATTR_NAME_INVALID_CHARS=". %-#[]"()!?*'&+\/" @@ -130,6 +132,7 @@ #! ENABLED="true" #! DYNAMIC_FEATURE_TYPES_LIST_ON_MERGE="true" #! DATASET_TYPE="DATABASE" +#! FTTR_TYPE="" #! GENERATE_FME_BUILD_NUM="18547" #! COORDSYS="" #! > @@ -852,8 +855,8 @@ #! IS_EDITABLE="true" #! IDENTIFIER="3" #! FEAT_GEOMTYPE="All" -#! POSITION="1441.4883748837487 167.41726416265624" -#! BOUNDING_RECT="1441.4883748837487 167.41726416265624 -1 -1" +#! POSITION="1460.2385623856237 167.41726416265624" +#! BOUNDING_RECT="1460.2385623856237 167.41726416265624 431 71" #! ORDER="500000000000002" #! COLLAPSED="false" #! KEYWORD="POSTGRES_1" @@ -907,12 +910,12 @@ #! #! #! -#! -#! #! #! @@ -942,7 +945,8 @@ #! ORDER="500000000000004" #! FOLLOW_ANCHOR="true" #! INFO_NODE="false" -#! COLOUR="1,1,0.78431372549019607,0.19607843137254902" +#! CUSTOM_USER_COLOR="false" +#! COLOUR="1,1,0.95686274509803926,0.78431372549019607" #! SIZE_POLICY="10" #! ANCHORED_NODE="0" #! /> @@ -956,7 +960,8 @@ #! ORDER="500000000000000" #! FOLLOW_ANCHOR="true" #! INFO_NODE="false" -#! COLOUR="1,1,0.78431372549019607,0.19215686274509805" +#! CUSTOM_USER_COLOR="false" +#! COLOUR="1,1,0.95686274509803926,0.78431372549019607" #! SIZE_POLICY="10" #! ANCHORED_NODE="0" #! /> @@ -970,7 +975,8 @@ #! ORDER="500000000000010" #! FOLLOW_ANCHOR="true" #! INFO_NODE="false" -#! COLOUR="1,1,0.78431372549019607,0.19607843137254902" +#! CUSTOM_USER_COLOR="false" +#! COLOUR="1,1,0.95686274509803926,0.78431372549019607" #! SIZE_POLICY="10" #! ANCHORED_NODE="17" #! /> @@ -985,7 +991,7 @@ #! TYPE="BulkAttributeRenamer" #! VERSION="4" #! POSITION="243.48560137091806 -307.58717335171877" -#! BOUNDING_RECT="243.48560137091806 -307.58717335171877 -1 -1" +#! BOUNDING_RECT="243.48560137091806 -307.58717335171877 431 71" #! ORDER="500000000000003" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1011,7 +1017,7 @@ #! TYPE="Creator" #! VERSION="5" #! POSITION="-332.38961738126955 167.41726416265624" -#! BOUNDING_RECT="-332.38961738126955 167.41726416265624 -1 -1" +#! BOUNDING_RECT="-332.38961738126955 167.41726416265624 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1036,7 +1042,7 @@ #! TYPE="FeatureReader" #! VERSION="9" #! POSITION="-332.38961738126955 -351.33761085609376" -#! BOUNDING_RECT="-332.38961738126955 -351.33761085609376 -1 -1" +#! BOUNDING_RECT="-332.38961738126955 -351.33761085609376 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1139,7 +1145,7 @@ #! TYPE="AttributeCreator" #! VERSION="8" #! POSITION="-332.38961738126955 -189.58264210640618" -#! BOUNDING_RECT="-332.38961738126955 -189.58264210640618 -1 -1" +#! BOUNDING_RECT="-332.38961738126955 -189.58264210640618 431 71" #! ORDER="500000000000000" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1165,7 +1171,7 @@ #! TYPE="Counter" #! VERSION="2" #! POSITION="-332.38961738126955 13.91756103062491" -#! BOUNDING_RECT="-332.38961738126955 13.91756103062491 -1 -1" +#! BOUNDING_RECT="-332.38961738126955 13.91756103062491 431 71" #! ORDER="500000000000007" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1191,7 +1197,7 @@ #! TYPE="Inspector" #! VERSION="5" #! POSITION="243.48560137091806 -716.96326711265624" -#! BOUNDING_RECT="243.48560137091806 -716.96326711265624 -1 -1" +#! BOUNDING_RECT="243.48560137091806 -716.96326711265624 431 71" #! ORDER="500000000000008" #! PARMS_EDITED="true" #! ENABLED="true" @@ -1225,6 +1231,10 @@ #! ENABLED="true" #! EXECUTION_IDX="0" #! HIDDEN="false" +#! COLOUR="0,0,0,1" +#! PEN_WIDTH="4" +#! PEN_STYLE="1" +#! LINK_STYLE_NAME="<Default>" #! EXTRA_POINTS="" #! /> #! -FME_PYTHON_VERSION 27 +FME_PYTHON_VERSION 37 GUI IGNORE SourceDataset_CSV2_1,CSV2_IN_FIELDS_GROUP_CSV2_1,CSV2_IN_DELIMITER_CSV2_1,CSV2_IN_FIELD_NAMES_LINE_CSV2_1,CSV2_IN_DATA_START_LINE_CSV2_1,CSV2_IN_ADVANCED_GROUP_CSV2_1,CSV2_IN_FIELD_SEPARATION_GROUP_CSV2_1,CSV2_IN_SKIP_DUPLICATE_DELIMITERS_CSV2_1,CSV2_IN_FIELD_QUALIFIER_CHARACTER_CSV2_1,CSV2_IN_FIELD_QUALIFIER_ESCAPE_CHARACTER_CSV2_1,CSV2_IN_FIELD_NAMES_GROUP_CSV2_1,CSV2_IN_CASE_SENSITIVE_FIELD_NAMES_CSV2_1,CSV2_IN_ENFORCE_STRICT_SCHEMA_CSV2_1,CSV2_IN_FIELD_VALUES_GROUP_CSV2_1,CSV2_IN_TRIM_FIELD_VALUE_QUALIFIERS_CSV2_1,CSV2_IN_ENCODING_GROUP_CSV2_1,CSV2_IN_ENCODING_CSV2_1,CSV2_IN_PRESERVE_SOURCE_ENCODING_CSV2_1,CSV2_IN_SKIPPED_LINES_GROUP_CSV2_1,CSV2_IN_READ_SKIPPED_LINES_CSV2_1,CSV2_IN_NETWORK_AUTHENTICATION_CSV2_1,DestDataset_POSTGRES_1,POSTGRES_OUT_FME_CONNECTION_GROUP_POSTGRES_1,POSTGRES_OUT_ADVANCED_POSTGRES_1,POSTGRES_OUT_BULK_COPY_POSTGRES_1,POSTGRES_OUT_START_TRANSACTION_POSTGRES_1,POSTGRES_OUT_TRANSACTION_INTERVAL_POSTGRES_1,POSTGRES_OUT_BEGIN_SQL_POSTGRES_1,POSTGRES_OUT_END_SQL_POSTGRES_1 -DEFAULT_MACRO DestDataset_POSTGRES AdminonMAP-FS1DHSinstance +DEFAULT_MACRO DestDataset_POSTGRES DHSadminlocaldocker GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGRES FMT:POSTGRES%FAMILY:PostgreSQL Connection: DEFAULT_MACRO SourceDataset_CSV2 http://api.dhsprogram.com/rest/dhs/surveys?f=csv&surveyStatus=all&perpage=2000 -GUI IGNORE MULTIFILE SourceDataset_CSV2 CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|All_Files|* Source CSV (Comma Separated Value) File(s): +GUI IGNORE OPTIONAL MULTIFILE SourceDataset_CSV2 CSV/Compressed_CSV/TEXT_Files(*.csv;*.gz;*.txt)|*.csv;*.gz;*.txt|CSV_Files(*.csv)|*.csv|Compressed_CSV_Files(*.gz)|*.gz|Text_Files(*.txt)|*.txt|Compressed_Files(*.bz2;*.gz)|*.bz2;*.gz|Archive_Files(*.7z;*.7zip;*.rar;*.rvz;*.tar;*.tar.bz2;*.tar.gz;*.tgz;*.zip;*.zipx)|*.7z;*.7zip;*.rar;*.rvz;*.tar;*.tar.bz2;*.tar.gz;*.tgz;*.zip;*.zipx|All_Files|* Source CSV (Comma Separated Value) File(s): INCLUDE [ if {{$(DestDataset_POSTGRES)} == {}} { puts_real {Parameter 'DestDataset_POSTGRES' must be given a value.}; exit 1; }; ] -INCLUDE [ if {{$(SourceDataset_CSV2)} == {}} { puts_real {Parameter 'SourceDataset_CSV2' must be given a value.}; exit 1; }; ] #! START_HEADER #! START_WB_HEADER READER_TYPE MULTI_READER @@ -1489,7 +1518,7 @@ FACTORY_DEF * RoutingFactory FACTORY_NAME "Router and Unexpected Input Remover CSV2_1_MERGE_DEF CSV2_1::CSV EXACT CSV # ------------------------------------------------------------------------- -FACTORY_DEF * TeeFactory FACTORY_NAME "CSV (CSV2_1) Splitter" INPUT FEATURE_TYPE CSV +FACTORY_DEF * TeeFactory FACTORY_NAME "CSV_CSV2_1 Splitter" INPUT FEATURE_TYPE CSV DEFAULT_MACRO WB_CURRENT_CONTEXT # ------------------------------------------------------------------------- Tcl2 proc Creator_CoordSysRemover {} { global FME_CoordSys; set FME_CoordSys {}; } @@ -1512,7 +1541,7 @@ FACTORY_DEF * TeeFactory FACTORY_NAME "$(WB_CURRENT_CONTEXT)_CREATOR_BRANCH_TA # Because @Count itself will evalute the expression to calculate the # domain, we don't need to do anything in Tcl per feature going by INCLUDE [ set domainPrefix {}; if { {Global} == {Local} } { set domainPrefix [FME_EncodeText {Counter_}]; }; puts "MACRO Counter_FULL_DOMAIN ${domainPrefix}counter"; ] -FACTORY_DEF * ExecuteFunctionFactory FACTORY_NAME Counter_Rejector COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE Creator_CREATED FUNCTION_DEFINITION @Count(REJECTABLE_WITH_FLAG,fme_encoded,$(Counter_FULL_DOMAIN),"1") RESULT_ATTRIBUTE "pagenum" OUTPUT COMPLETE FEATURE_TYPE Counter_OUTPUT OUTPUT REJECTED FEATURE_TYPE Counter_ +FACTORY_DEF * ExecuteFunctionFactory FACTORY_NAME Counter_Rejector COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE Creator_CREATED FUNCTION_DEFINITION @Count(REJECTABLE_WITH_FLAG,fme_encoded,$(Counter_FULL_DOMAIN),"1") RESULT_ATTRIBUTE "pagenum" FEATURE_TABLE_SHIM_SUPPORT Yes OUTPUT COMPLETE FEATURE_TYPE Counter_OUTPUT OUTPUT REJECTED FEATURE_TYPE Counter_ DEFAULT_MACRO _WB_BYPASS_TERMINATION No FACTORY_DEF * TeeFactory FACTORY_NAME Counter_ INPUT FEATURE_TYPE Counter_ OUTPUT FAILED FEATURE_TYPE * @Abort(ENCODED, CounteroutputaRejectedfeature.TocontinuetranslationwhenfeaturesarerejectedchangeWorkspaceParametersTranslationRejectedFeatureHandlingtoContinueTranslation) # ------------------------------------------------------------------------- @@ -1539,7 +1568,7 @@ DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO # [PR#45549] The Data Inspector will limit the maximum number of features # it reads on WIN32, so we will correspondingly limit the number of # features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME blank_pages_Recorder COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType(@Value(__inspector_feature_type_attr__)) @RemoveAttributes(__inspector_feature_type_attr__) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" STORE_SCANNED_SCHEMA YES CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA _csvurl,varchar255,pagenum,varchar255,_reader_error,varchar255 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * RecorderFactory FACTORY_NAME blank_pages_Recorder COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType(@Value(__inspector_feature_type_attr__)) @RemoveAttributes(__inspector_feature_type_attr__) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" STORE_SCANNED_SCHEMA YES NORMALIZE_LIST_ATTRIBUTES_ON_SCHEMA YES CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA _csvurl,varchar200,pagenum,uint64,_reader_error,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 # ------------------------------------------------------------------------- FACTORY_DEF {*} TeeFactory FACTORY_NAME BulkAttributeRenamer INPUT FEATURE_TYPE FeatureReader_2_ OUTPUT { FEATURE_TYPE BulkAttributeRenamer_OUTPUT @RenameAttributes(PATTERN_REPLACE, CASECHANGE, , "", "", "lower", "", "-_", "RENAME", "NONFME") } # ------------------------------------------------------------------------- @@ -1551,7 +1580,8 @@ FACTORY_DEF * TeeFactory FACTORY_NAME "Final Output Nuker" INPUT FEATURE_TYP # ------------------------------------------------------------------------- -FACTORY_DEF * CreationFactory FACTORY_NAME VisualizerProcessor CREATE_AT_END OUTPUT FEATURE_TYPE __NUKEME__ @TCL("if [file exists {$(WORKSPACE_TEMP_DIR)}] {set files [glob -nocomplain -directory {$(WORKSPACE_TEMP_DIR)} -tails *.ffs]; if {[llength $files] == 1} { set theFile {$(WORKSPACE_TEMP_DIR)/}; append theFile [lindex $files 0]; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -f FFS \042$theFile\042 & \175; } elseif { [llength $files] == 0 } { catch { file delete -force {$(WORKSPACE_TEMP_DIR)} }; } else { set filesNoExt {}; set lastfile {}; foreach {fileName} [lsort $files] { set fileNoExt [string trimright $fileName {.ffs}]; if [regexp {(.*)_[0-9]*$} $fileNoExt wholething fileNoExtBase] { if { $fileNoExtBase == $lastfile } { continue } }; set lastfile $fileNoExt; lappend filesNoExt $fileNoExt }; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -d {$(WORKSPACE_TEMP_DIR)} -a ffs -f FFS $filesNoExt & \175; }; }; ") +DEFAULT_MACRO FME_LAUNCH_VIEWER_APP NO +FACTORY_DEF * CreationFactory FACTORY_NAME VisualizerProcessor CREATE_AT_END OUTPUT FEATURE_TYPE __NUKEME__ @TCL("if {{$(FME_LAUNCH_VIEWER_APP)} == {YES}} { if [file exists {$(WORKSPACE_TEMP_DIR)}] {set files [glob -nocomplain -directory {$(WORKSPACE_TEMP_DIR)} -tails *.ffs]; if {[llength $files] == 1} { set theFile {$(WORKSPACE_TEMP_DIR)/}; append theFile [lindex $files 0]; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -f FFS \042$theFile\042 & \175; } elseif { [llength $files] == 0 } { catch { file delete -force {$(WORKSPACE_TEMP_DIR)} }; } else { set filesNoExt {}; set lastfile {}; foreach {fileName} [lsort $files] { set fileNoExt [string trimright $fileName {.ffs}]; if [regexp {(.*)_[0-9]*$} $fileNoExt wholething fileNoExtBase] { if { $fileNoExtBase == $lastfile } { continue } }; set lastfile $fileNoExt; lappend filesNoExt $fileNoExt }; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -d {$(WORKSPACE_TEMP_DIR)} -a ffs -f FFS $filesNoExt & \175; }; }; }; ") FACTORY_DEF * TeeFactory FACTORY_NAME VisualizerCleaner INPUT FEATURE_TYPE __NUKEME__ # ------------------------------------------------------------------------- POSTGRES_1_DEF dhs_survey_specs.dhs_survey_listing postgres_type All postgres_sql_key_columns "" postgres_drop_table "" postgres_truncate_table "" postgres_mode "" fme_feature_operation INSERT fme_table_handling TRUNCATE_EXISTING postgres_create_with_oids NO postgres_vacuum_analyze NO postgres_allow_serial_writing NO surveyid text dhs_countrycode text surveyyear text surveytype text surveystatus text countryname text surveyyearlabel text surveynum text indicatordata text regionname text subregionname text publicationdate text releasedate text surveycharacteristicids text footnotes text fieldworkstart text fieldworkend text implementingorg text numberofsamplepoints text numberofhouseholds text universeofwomen text numberofwomen text minagewomen text maxagewomen text universeofmen text numberofmen text minagemen text maxagemen text numberoffacilities text diff --git a/DHS_Preprocess_And_Ingestion/07C_Load_New_Data/Load_All_Cluster_Shapes_To_PG.fmw b/DHS_Preprocess_And_Ingestion/07C_Load_New_Data/Load_All_Cluster_Shapes_To_PG.fmw index 4d45740..9d8e7f0 100644 --- a/DHS_Preprocess_And_Ingestion/07C_Load_New_Data/Load_All_Cluster_Shapes_To_PG.fmw +++ b/DHS_Preprocess_And_Ingestion/07C_Load_New_Data/Load_All_Cluster_Shapes_To_PG.fmw @@ -2,7 +2,7 @@ #! Dataadmin" # #! ATTR_TYPE_ENCODING="SDF" @@ -17,9 +17,9 @@ #! END_PYTHON="" #! END_TCL="" #! EXPLICIT_BOOKMARK_ORDER="false" -#! FME_BUILD_NUM="20238" -#! FME_DOCUMENT_GUID="fd536b2f-0c99-4570-86db-11a01dd555c6" -#! FME_DOCUMENT_PRIORGUID="91187cca-a6b9-471f-9281-e4f8e83fcb95,06fb4c19-788e-4b9f-97c8-9e36d2b14d75,61c3a51b-b719-4049-9c22-f25e76d01e88,fd676752-563c-4f8b-8570-fa0f05d53c9a,0205ef3e-a4e5-475a-be8e-b55a6e63aa0a,0bffcacf-9f89-4385-b118-f7ebd86e277e,1d7622ec-66a1-4956-b2d1-d9891cc66d95,19524b40-069a-4915-aa04-1a38d4f5c0aa,f6a3a578-1290-475e-990d-2dfafb8a9d3e,a10a720e-40b8-44f9-bd82-7c467b131bd9,5b4ec1de-6967-47ca-948a-8a1ab791a4fd,2e85591f-5256-4351-8179-9eae7bb549fa,6ac41170-b3e0-4bb7-a9d5-73caf83592db" +#! FME_BUILD_NUM="20608" +#! FME_DOCUMENT_GUID="3295bb10-2f64-472f-85a3-1f7252119708" +#! FME_DOCUMENT_PRIORGUID="91187cca-a6b9-471f-9281-e4f8e83fcb95,06fb4c19-788e-4b9f-97c8-9e36d2b14d75,61c3a51b-b719-4049-9c22-f25e76d01e88,fd676752-563c-4f8b-8570-fa0f05d53c9a,0205ef3e-a4e5-475a-be8e-b55a6e63aa0a,0bffcacf-9f89-4385-b118-f7ebd86e277e,1d7622ec-66a1-4956-b2d1-d9891cc66d95,19524b40-069a-4915-aa04-1a38d4f5c0aa,f6a3a578-1290-475e-990d-2dfafb8a9d3e,a10a720e-40b8-44f9-bd82-7c467b131bd9,5b4ec1de-6967-47ca-948a-8a1ab791a4fd,2e85591f-5256-4351-8179-9eae7bb549fa,6ac41170-b3e0-4bb7-a9d5-73caf83592db,fd536b2f-0c99-4570-86db-11a01dd555c6,db6356d8-b071-4343-9e18-fe96594726f4" #! FME_GEOMETRY_HANDLING="Enhanced" #! FME_IMPLICIT_CSMAP_REPROJECTION_MODE="Auto" #! FME_REPROJECTION_ENGINE="FME" @@ -27,8 +27,8 @@ #! FME_STROKE_MAX_DEVIATION="0" #! HISTORY="" #! IGNORE_READER_FAILURE="No" -#! LAST_SAVE_BUILD="FME(R) 2020.0.2.1 (20200511 - Build 20238 - WIN64)" -#! LAST_SAVE_DATE="2020-05-22T14:36:46" +#! LAST_SAVE_BUILD="FME(R) 2020.1.1.0 (20200801 - Build 20608 - WIN64)" +#! LAST_SAVE_DATE="2020-09-11T17:17:59" #! LOG_FILE="" #! LOG_MAX_RECORDED_FEATURES="200" #! MARKDOWN_DESCRIPTION="" @@ -47,7 +47,7 @@ #! TITLE="" #! USAGE="" #! USE_MARKDOWN="" -#! VIEW_POSITION="-2753.15 1609.39" +#! VIEW_POSITION="-2928.15 1003.14" #! WARN_INVALID_XFORM_PARAM="Yes" #! WORKSPACE_VERSION="1" #! ZOOM_SCALE="100" @@ -66,7 +66,7 @@ #! WRITE_DEFS="true" #! DEFLINE_TEMPLATE="SHAPE_GEOMETRY {FME_GEN_GEOMETRY}" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="fme_rotation double fme_feature_type char(50) SHAPE_GEOMETRY char(50) fme_color char(50) multi_reader_type char(50) fme_geometry fme_char(50) fme_basename char(50) fme_secondary_axis double fme_primary_axis double fme_type char(50) fme_start_angle double fme_text_string char(50) shape_geometry_error{} char(254) multi_reader_full_id long fme_sweep_angle double fme_fill_color char(50) fme_dataset char(50) shape_measures char(254) fme_text_size double multi_reader_id long multi_reader_keyword char(50)" +#! EXPOSABLE_ATTRS="fme_geometry fme_char(50) fme_text_size double multi_reader_full_id long fme_fill_color char(50) shape_measures char(254) fme_sweep_angle double multi_reader_type char(50) fme_text_string char(50) fme_feature_type char(50) fme_dataset char(50) fme_color char(50) fme_primary_axis double SHAPE_GEOMETRY char(50) fme_start_angle double multi_reader_id long fme_type char(50) shape_geometry_error{} char(254) fme_basename char(50) multi_reader_keyword char(50) fme_secondary_axis double fme_rotation double" #! DEFLINE_PARMS="" #! ATTR_INDEX_TYPES="INDEX" #! ATTR_NAME_INVALID_CHARS="" @@ -117,29 +117,37 @@ #! ROLE="WRITER" #! FORMAT="POSTGIS" #! DATASET="$(Points_DB_Connection)" -#! KEYWORD="POSTGIS_1" +#! KEYWORD="POSTGIS_2" #! MULTI_GEOM_ALLOWED="true" #! ATTR_MAX_LENGTH="63" #! ATTR_CASE="LOWER" #! ALLOWED_FEAT_TYPES="" #! WRITE_DEFS="true" -#! DEFLINE_TEMPLATE="postgis_type postgis_geometry postgis_mode "" postgis_sql_key_columns "" postgis_drop_table "" postgis_truncate_table "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING fme_selection_method MATCH_COLUMNS fme_match_columns "" fme_where_builder_clause "" fme_update_geometry YES postgis_create_with_oids NO postgis_create_gist_index YES postgis_spatial_column_type geometry postgis_spatial_column geom postgis_srid "" postgis_vacuum_analyze no postgis_allow_serial_writing NO" +#! DEFLINE_TEMPLATE="postgis_type postgis_geometry postgis_mode "" postgis_sql_key_columns "" postgis_drop_table "" postgis_truncate_table "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING fme_selection_method MATCH_COLUMNS fme_match_columns "" fme_where_builder_clause "" fme_update_geometry YES postgis_create_gist_index YES postgis_spatial_column_type geometry postgis_spatial_column geom postgis_srid "" postgis_vacuum_analyze no postgis_allow_serial_writing NO" #! DEFLINE_ATTRS="true" -#! EXPOSABLE_ATTRS="fme_rotation float8 fme_feature_type char(50) fme_color char(50) postgis_collection_id int4 fme_db_operation char(8) fme_geometry fme_char(50) fme_basename char(50) fme_secondary_axis float8 fme_primary_axis float8 fme_type char(50) fme_start_angle float8 fme_text_string char(50) fme_sweep_angle float8 fme_fill_color char(50) fme_dataset char(50) fme_text_size float8 postgis_collection_part_id int4 postgis_type char(50)" -#! DEFLINE_PARMS=""GUI OPTIONAL NAMEDGROUP fme_configuration_group fme_configuration_common_group%fme_spatial_group%fme_advanced_group%oracle_advanced_group Table Settings" "" "GUI OPTIONAL NAMEDGROUP fme_configuration_common_group fme_feature_operation%fme_table_handling%oracle_model%fme_update_geometry%fme_selection_group%fme_table_creation_group General" "" "GUI ACTIVECHOICE_LOOKUP fme_feature_operation Insert,INSERT,fme_update_geometry,fme_selection_group%Update,UPDATE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN%Delete,DELETE,++fme_table_handling+USE_EXISTING,fme_update_geometry,++fme_selection_group+FME_DISCLOSURE_OPEN,fme_spatial_group,fme_advanced_group,oracle_sequenced_cols%<at>Value<openparen>fme_db_operation<closeparen>,MULTIPLE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN Feature Operation" INSERT "GUI ACTIVECHOICE_LOOKUP fme_table_handling Use<space>Existing,USE_EXISTING,fme_table_creation_group%Create<space>If<space>Needed,CREATE_IF_MISSING%Drop<space>and<space>Create,DROP_CREATE%Truncate<space>Existing,TRUNCATE_EXISTING,fme_table_creation_group Table Handling" CREATE_IF_MISSING "GUI WHOLE_LINE LOOKUP_CHOICE fme_update_geometry Yes,YES%No,NO Update Spatial Column(s)" YES "GUI OPTIONAL DISCLOSUREGROUP fme_selection_group fme_selection_method Row Selection" "" "GUI OPTIONAL WHOLE_LINE RADIOPARAMETERGROUP fme_selection_method fme_match_columns,MATCH_COLUMNS%fme_where_builder_clause,BUILDER Row Selection Method" MATCH_COLUMNS "GUI WHOLE_LINE ATTRLIST_COMMAS fme_match_columns \" \" Match Columns" "" "GUI WHOLE_LINE TEXT_EDIT_SQL_CFG_OR_ATTR fme_where_builder_clause MODE,WHERE WHERE Clause" "" "GUI OPTIONAL DISCLOSUREGROUP fme_table_creation_group postgis_create_with_oids%postgis_create_gist_index Table Creation Parameters" "" "GUI CHOICE postgis_create_with_oids YES%NO Create With OIDs" NO "GUI CHOICE postgis_create_gist_index YES%NO Create GiST Index" YES "GUI OPTIONAL NAMEDGROUP fme_spatial_group postgis_spatial_column_type%postgis_spatial_column%postgis_srid Spatial" "" "GUI LOOKUP_CHOICE postgis_spatial_column_type Geometry,geometry%Geography,geography Spatial Type" geometry "GUI OPTIONAL WHOLE_LINE TEXT postgis_spatial_column Spatial Column" geom "GUI OPTIONAL FLOAT postgis_srid Spatial Column SRID" "" "GUI OPTIONAL DISCLOSUREGROUP fme_advanced_group postgis_vacuum_analyze%postgis_allow_serial_writing Advanced" "" "GUI CHOICE postgis_vacuum_analyze YES%NO Vacuum Analyze Table" NO "GUI CHOICE postgis_allow_serial_writing YES%NO Allow Serial Column Writing" NO " -#! ATTR_INDEX_TYPES="BTREE HASH PRIKEY RTREE" +#! EXPOSABLE_ATTRS="fme_geometry char(50) fme_text_size float8 postgis_collection_part_id int4 fme_fill_color char(50) postgis_collection_id int4 postgis_type char(50) fme_sweep_angle float8 fme_text_string char(50) fme_db_operation char(8) fme_feature_type char(50) fme_dataset char(50) fme_color char(50) fme_primary_axis float8 fme_start_angle float8 fme_type char(50) fme_basename char(50) fme_secondary_axis float8 fme_rotation float8" +#! DEFLINE_PARMS=""GUI OPTIONAL NAMEDGROUP fme_configuration_group fme_configuration_common_group%fme_spatial_group%fme_advanced_group%oracle_advanced_group Table" "" "GUI OPTIONAL NAMEDGROUP fme_configuration_common_group fme_feature_operation%fme_table_handling%mie_pack%oracle_model%fme_update_geometry%fme_selection_group%fme_table_creation_group General" "" "GUI ACTIVECHOICE_LOOKUP fme_feature_operation Insert,INSERT,fme_update_geometry,fme_selection_group,mie_pack%Update,UPDATE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN%Delete,DELETE,++fme_table_handling+USE_EXISTING,fme_update_geometry,++fme_selection_group+FME_DISCLOSURE_OPEN,fme_spatial_group,fme_advanced_group,oracle_sequenced_cols%<at>Value<openparen>fme_db_operation<closeparen>,MULTIPLE,++fme_table_handling+USE_EXISTING,++fme_selection_group+FME_DISCLOSURE_OPEN Feature Operation" INSERT "GUI ACTIVECHOICE_LOOKUP fme_table_handling Use<space>Existing,USE_EXISTING,fme_table_creation_group%Create<space>If<space>Needed,CREATE_IF_MISSING%Drop<space>and<space>Create,DROP_CREATE%Truncate<space>Existing,TRUNCATE_EXISTING,fme_table_creation_group Table Handling" CREATE_IF_MISSING "GUI WHOLE_LINE LOOKUP_CHOICE fme_update_geometry Yes,YES%No,NO Update Spatial Column(s)" YES "GUI OPTIONAL DISCLOSUREGROUP fme_selection_group fme_selection_method Row Selection" "" "GUI OPTIONAL WHOLE_LINE RADIOPARAMETERGROUP fme_selection_method fme_match_columns,MATCH_COLUMNS%fme_where_builder_clause,BUILDER Row Selection Method" MATCH_COLUMNS "GUI WHOLE_LINE ATTRLIST_COMMAS fme_match_columns \" \" Match Columns" "" "GUI WHOLE_LINE TEXT_EDIT_SQL_CFG_OR_ATTR fme_where_builder_clause MODE,WHERE WHERE Clause" "" "GUI OPTIONAL DISCLOSUREGROUP fme_table_creation_group postgis_create_gist_index Table Creation Parameters" "" "GUI CHOICE postgis_create_gist_index YES%NO Create GiST Index" YES "GUI OPTIONAL NAMEDGROUP fme_spatial_group postgis_spatial_column_type%postgis_spatial_column%postgis_srid Spatial" "" "GUI LOOKUP_CHOICE postgis_spatial_column_type Geometry,geometry%Geography,geography Spatial Type" geometry "GUI OPTIONAL WHOLE_LINE TEXT postgis_spatial_column Spatial Column" geom "GUI OPTIONAL FLOAT postgis_srid Spatial Column SRID" "" "GUI OPTIONAL DISCLOSUREGROUP fme_advanced_group postgis_vacuum_analyze%postgis_allow_serial_writing Advanced" "" "GUI CHOICE postgis_vacuum_analyze YES%NO Vacuum Analyze Table" NO "GUI CHOICE postgis_allow_serial_writing YES%NO Allow Serial Column Writing" NO " +#! ATTR_INDEX_TYPES="BTree RTree Hash PrimaryKey" #! ATTR_NAME_INVALID_CHARS=". %-#[]"()!?*'&+\/" #! SUPPORTS_FEATURE_TYPE_FANOUT="true" #! ENABLED="true" #! DYNAMIC_FEATURE_TYPES_LIST_ON_MERGE="true" #! DATASET_TYPE="DATABASE" #! FTTR_TYPE="" -#! GENERATE_FME_BUILD_NUM="16492" +#! GENERATE_FME_BUILD_NUM="20608" #! COORDSYS="" #! > #! +#! +#! #! +#! +#! #! #! #! @@ -879,11 +895,11 @@ #! IS_EDITABLE="true" #! IDENTIFIER="18" #! FEAT_GEOMTYPE="All" -#! POSITION="1169.6099999999999 500.50900000000001" -#! BOUNDING_RECT="1169.6099999999999 500.50900000000001 431 71" +#! POSITION="1166.4849687496876 500.50900000000001" +#! BOUNDING_RECT="1166.4849687496876 500.50900000000001 431 71" #! ORDER="500000000000000" #! COLLAPSED="false" -#! KEYWORD="POSTGIS_1" +#! KEYWORD="POSTGIS_2" #! PARMS_EDITED="true" #! ENABLED="true" #! SCHEMA_ATTRIBUTE_SOURCE="1" @@ -919,12 +935,11 @@ #! #! #! -#! +#! #! #! #! #! -#! #! #! #! @@ -943,7 +958,7 @@ #! #! #! #! @@ -2021,13 +2036,13 @@ #! FME_PYTHON_VERSION 37 -GUI IGNORE SourceDataset_ESRISHAPE_1,ESRISHAPE_IN_SIMPLE_DONUT_GEOMETRY_ESRISHAPE_1,ESRISHAPE_IN_ENCODING_ESRISHAPE_1,ESRISHAPE_IN_EXPOSE_ATTRS_GROUP_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MINX_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MINY_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MAXX_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MAXY_ESRISHAPE_1,ESRISHAPE_IN_CLIP_TO_ENVELOPE_ESRISHAPE_1,ESRISHAPE_IN_NETWORK_AUTHENTICATION_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_COORDINATE_SYSTEM_ESRISHAPE_1,_MEASURES_AS_Z_ESRISHAPE_1,ESRISHAPE_IN_DISSOLVE_HOLES_ESRISHAPE_1,ESRISHAPE_IN_REPORT_BAD_GEOMETRY_ESRISHAPE_1,ESRISHAPE_IN_TRIM_PRECEDING_SPACES_ESRISHAPE_1,DestDataset_POSTGIS_1,POSTGIS_OUT_START_TRANSACTION_POSTGIS_1,POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_1,POSTGIS_OUT_BULK_COPY_POSTGIS_1,POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_1,POSTGIS_OUT_BEGIN_SQL_POSTGIS_1,POSTGIS_OUT_END_SQL_POSTGIS_1 -DEFAULT_MACRO SourceDataset_ESRISHAPE C:\Users\zool1301\Documents\02_Update_Staging_Template\download\**\*.shp +GUI IGNORE SourceDataset_ESRISHAPE_1,ESRISHAPE_IN_SIMPLE_DONUT_GEOMETRY_ESRISHAPE_1,ESRISHAPE_IN_ENCODING_ESRISHAPE_1,ESRISHAPE_IN_EXPOSE_ATTRS_GROUP_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MINX_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MINY_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MAXX_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_MAXY_ESRISHAPE_1,ESRISHAPE_IN_CLIP_TO_ENVELOPE_ESRISHAPE_1,ESRISHAPE_IN_NETWORK_AUTHENTICATION_ESRISHAPE_1,ESRISHAPE_IN_SEARCH_ENVELOPE_COORDINATE_SYSTEM_ESRISHAPE_1,_MEASURES_AS_Z_ESRISHAPE_1,ESRISHAPE_IN_DISSOLVE_HOLES_ESRISHAPE_1,ESRISHAPE_IN_REPORT_BAD_GEOMETRY_ESRISHAPE_1,ESRISHAPE_IN_TRIM_PRECEDING_SPACES_ESRISHAPE_1,DestDataset_POSTGIS_2,POSTGIS_OUT_FME_CONNECTION_GROUP_POSTGIS_2,POSTGIS_OUT_ADVANCED_POSTGIS_2,POSTGIS_OUT_BULK_COPY_POSTGIS_2,POSTGIS_OUT_START_TRANSACTION_POSTGIS_2,POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_2,POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_2,POSTGIS_OUT_BEGIN_SQL_POSTGIS_2,POSTGIS_OUT_END_SQL_POSTGIS_2 +DEFAULT_MACRO SourceDataset_ESRISHAPE D:\InformalCities\DHS\staging\downloaded\**\*.shp GUI MULTIFILE SourceDataset_ESRISHAPE Shapefiles(*.shp)|*.shp|Compressed_Shapefiles(*.shz)|*.shz|Compressed_Files(*.bz2;*.gz)|*.bz2;*.gz|Archive_Files(*.7z;*.7zip;*.rar;*.rvz;*.tar;*.tar.bz2;*.tar.gz;*.tgz;*.zip;*.zipx)|*.7z;*.7zip;*.rar;*.rvz;*.tar;*.tar.bz2;*.tar.gz;*.tgz;*.zip;*.zipx|All_Files(*)|* Source Esri Shapefile(s): DEFAULT_MACRO Points_DB_Connection DHSDataadmin GUI NAMED_DB_CONNECTION_ENCODED Points_DB_Connection FMT:POSTGRES%FAMILY:PostgreSQL DB for survey points: DEFAULT_MACRO FEATURE_TYPES -GUI IGNORE OPTIONAL FEATURE_TYPES_ENCODED FEATURE_TYPES C:Userszool1301Documents02_Update_Staging_Templatedownload***.shp,_FORMAT_OVERRIDE_,ESRISHAPE,_DATASET_OVERRIDE_,SourceDataset_ESRISHAPE,SIMPLE_DONUT_GEOMETRY,simple,SEARCH_ENVELOPE_MINX,0,SEARCH_ENVELOPE_MINY,0,SEARCH_ENVELOPE_MAXX,0,SEARCH_ENVELOPE_MAXY,0,CLIP_TO_ENVELOPE,NO,_MEASURES_AS_Z,no,DISSOLVE_HOLES,no,REPORT_BAD_GEOMETRY,no,TRIM_PRECEDING_SPACES,YES,UPPER_CASE_ATTR_NAMES,No,USE_SEARCH_ENVELOPE,NO,DYNAMIC_WORKFLOW,yes,CHECK_NUMERIC_FIELDS,yes,_MERGE_SCHEMAS,YES,_MERGE_SCHEMAS,YES Feature Types to Read: +GUI IGNORE OPTIONAL FEATURE_TYPES_ENCODED FEATURE_TYPES D:InformalCitiesDHSstagingdownloaded***.shp,_FORMAT_OVERRIDE_,ESRISHAPE,_DATASET_OVERRIDE_,SourceDataset_ESRISHAPE,SIMPLE_DONUT_GEOMETRY,simple,SEARCH_ENVELOPE_MINX,0,SEARCH_ENVELOPE_MINY,0,SEARCH_ENVELOPE_MAXX,0,SEARCH_ENVELOPE_MAXY,0,CLIP_TO_ENVELOPE,NO,_MEASURES_AS_Z,no,DISSOLVE_HOLES,no,REPORT_BAD_GEOMETRY,no,TRIM_PRECEDING_SPACES,YES,UPPER_CASE_ATTR_NAMES,No,USE_SEARCH_ENVELOPE,NO,DYNAMIC_WORKFLOW,yes,CHECK_NUMERIC_FIELDS,yes,_MERGE_SCHEMAS,YES,_MERGE_SCHEMAS,YES Feature Types to Read: INCLUDE [ if {{$(SourceDataset_ESRISHAPE)} == {}} { puts_real {Parameter 'SourceDataset_ESRISHAPE' must be given a value.}; exit 1; }; ] INCLUDE [ if {{$(Points_DB_Connection)} == {}} { puts_real {Parameter 'Points_DB_Connection' must be given a value.}; exit 1; }; ] #! START_HEADER @@ -2040,7 +2055,7 @@ WRITER_TYPE MULTI_WRITER MULTI_WRITER_DATASET_ORDER BY_ID MULTI_WRITER_FIRST_WRITER_ID 0 MULTI_WRITER_TYPE{0} POSTGIS -MULTI_WRITER_KEYWORD{0} POSTGIS_1 +MULTI_WRITER_KEYWORD{0} POSTGIS_2 #! END_WB_HEADER #! START_WB_HEADER MACRO WB_KEYWORD "ESRISHAPE_1" @@ -2155,52 +2170,59 @@ ESRISHAPE_1_DATASET "$(SourceDataset_ESRISHAPE_1)" ESRISHAPE_1_GENERATE_FME_BUILD_NUM 16492 #! END_SOURCE_HEADER #! START_WB_HEADER -MACRO WB_KEYWORD "POSTGIS_1" +MACRO WB_KEYWORD "POSTGIS_2" #! END_WB_HEADER -#! START_DEST_HEADER POSTGIS POSTGIS_1 +#! START_DEST_HEADER POSTGIS POSTGIS_2 #! END_DEST_HEADER #! START_WB_HEADER DEFAULT_MACRO DestDataset INCLUDE [ if {"$(DestDataset)" != ""} { \ - puts {DEFAULT_MACRO DestDataset_POSTGIS_1 $(DestDataset)} \ + puts {DEFAULT_MACRO DestDataset_POSTGIS_2 $(DestDataset)} \ } ] #! END_WB_HEADER -#! START_DEST_HEADER POSTGIS POSTGIS_1 -DEFAULT_MACRO DestDataset_POSTGIS_1 $(Points_DB_Connection) -GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGIS_1 FMT:POSTGIS Connection: +#! START_DEST_HEADER POSTGIS POSTGIS_2 +DEFAULT_MACRO DestDataset_POSTGIS_2 $(Points_DB_Connection) +GUI NAMED_DB_CONNECTION_ENCODED DestDataset_POSTGIS_2 FMT:POSTGIS Connection: +POSTGIS_2_PLUGIN_MAX_CHANNELS 1 +DEFAULT_MACRO POSTGIS_OUT_STRICT_TEMPORAL_TYPE_HANDLING_POSTGIS_2 Yes +POSTGIS_2_STRICT_TEMPORAL_TYPE_HANDLING "$(POSTGIS_OUT_STRICT_TEMPORAL_TYPE_HANDLING_POSTGIS_2)" +DEFAULT_MACRO POSTGIS_OUT_FME_CONNECTION_GROUP_POSTGIS_2 +GUI OPTIONAL DISCLOSUREGROUP POSTGIS_OUT_FME_CONNECTION_GROUP_POSTGIS_2 POSTGIS_OUT_NAMED_CONNECTION Database Connection # ============================================================================ # Write geometry (planar) or geography (geodetic) column -DEFAULT_MACRO POSTGIS_OUT_SPATIAL_COLUMN_TYPE_POSTGIS_1 geometry -POSTGIS_1_SPATIAL_COLUMN_TYPE "$(POSTGIS_OUT_SPATIAL_COLUMN_TYPE_POSTGIS_1)" +DEFAULT_MACRO POSTGIS_OUT_SPATIAL_COLUMN_TYPE_POSTGIS_2 geometry +POSTGIS_2_SPATIAL_COLUMN_TYPE "$(POSTGIS_OUT_SPATIAL_COLUMN_TYPE_POSTGIS_2)" # ============================================================================ # Default spatial column name -DEFAULT_MACRO POSTGIS_OUT_SPATIAL_COLUMN_NAME_POSTGIS_1 geom -POSTGIS_1_SPATIAL_COLUMN_NAME "$(POSTGIS_OUT_SPATIAL_COLUMN_NAME_POSTGIS_1)" -DEFAULT_MACRO POSTGIS_OUT_GENERIC_GEOMETRY_POSTGIS_1 yes -POSTGIS_1_GENERIC_GEOMETRY "$(POSTGIS_OUT_GENERIC_GEOMETRY_POSTGIS_1)" -DEFAULT_MACRO POSTGIS_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGIS_1 FEATURE_TYPE -POSTGIS_1_COORDINATE_SYSTEM_GRANULARITY "$(POSTGIS_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGIS_1)" -DEFAULT_MACRO POSTGIS_OUT_WRITER_MODE_POSTGIS_1 INSERT -DEFAULT_MACRO POSTGIS_OUT_START_TRANSACTION_POSTGIS_1 0 -GUI TEXT POSTGIS_OUT_START_TRANSACTION_POSTGIS_1 Starting Feature: -POSTGIS_1_START_TRANSACTION $(POSTGIS_OUT_START_TRANSACTION_POSTGIS_1) -DEFAULT_MACRO POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_1 1000 -GUI TEXT POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_1 Features Per Transaction: -POSTGIS_1_TRANSACTION_INTERVAL $(POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_1) -DEFAULT_MACRO POSTGIS_OUT_BULK_COPY_POSTGIS_1 YES -GUI CHOICE POSTGIS_OUT_BULK_COPY_POSTGIS_1 YES%NO Bulk Insert: -POSTGIS_1_BULK_COPY $(POSTGIS_OUT_BULK_COPY_POSTGIS_1) -DEFAULT_MACRO POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_1 NO -GUI CHOICE POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_1 YES%NO Insert WKT: -POSTGIS_1_INSERT_TEXT_GEOMETRY $(POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_1) -DEFAULT_MACRO POSTGIS_OUT_BEGIN_SQL_POSTGIS_1 -GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGIS_OUT_BEGIN_SQL_POSTGIS_1 MODE,SQL;FORMAT,POSTGIS SQL Statement to Execute Before Translation: -POSTGIS_1_BEGIN_SQL "$(POSTGIS_OUT_BEGIN_SQL_POSTGIS_1)" -DEFAULT_MACRO POSTGIS_OUT_END_SQL_POSTGIS_1 -GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGIS_OUT_END_SQL_POSTGIS_1 MODE,SQL;FORMAT,POSTGIS SQL Statement to Execute After Translation: -POSTGIS_1_END_SQL "$(POSTGIS_OUT_END_SQL_POSTGIS_1)" -POSTGIS_1_DATASET "$(DestDataset_POSTGIS_1)" -POSTGIS_1_GENERATE_FME_BUILD_NUM 16492 +DEFAULT_MACRO POSTGIS_OUT_SPATIAL_COLUMN_NAME_POSTGIS_2 geom +POSTGIS_2_SPATIAL_COLUMN_NAME "$(POSTGIS_OUT_SPATIAL_COLUMN_NAME_POSTGIS_2)" +DEFAULT_MACRO POSTGIS_OUT_GENERIC_GEOMETRY_POSTGIS_2 yes +POSTGIS_2_GENERIC_GEOMETRY "$(POSTGIS_OUT_GENERIC_GEOMETRY_POSTGIS_2)" +DEFAULT_MACRO POSTGIS_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGIS_2 FEATURE_TYPE +POSTGIS_2_COORDINATE_SYSTEM_GRANULARITY "$(POSTGIS_OUT_COORDINATE_SYSTEM_GRANULARITY_POSTGIS_2)" +DEFAULT_MACRO POSTGIS_OUT_ADVANCED_POSTGIS_2 +GUI OPTIONAL DISCLOSUREGROUP POSTGIS_OUT_ADVANCED_POSTGIS_2 POSTGIS_OUT_BULK_COPY_POSTGIS_2%POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_2%POSTGIS_OUT_START_TRANSACTION_POSTGIS_2%POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_2%POSTGIS_OUT_BEGIN_SQL_POSTGIS_2%POSTGIS_OUT_END_SQL_POSTGIS_2 Advanced +DEFAULT_MACRO POSTGIS_OUT_BULK_COPY_POSTGIS_2 YES +POSTGIS_2_BULK_COPY "$(POSTGIS_OUT_BULK_COPY_POSTGIS_2)" +GUI CHOICE POSTGIS_OUT_BULK_COPY_POSTGIS_2 YES%NO Bulk Insert +DEFAULT_MACRO POSTGIS_OUT_START_TRANSACTION_POSTGIS_2 0 +POSTGIS_2_START_TRANSACTION "$(POSTGIS_OUT_START_TRANSACTION_POSTGIS_2)" +GUI TEXT POSTGIS_OUT_START_TRANSACTION_POSTGIS_2 Starting Feature: +DEFAULT_MACRO POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_2 1000 +POSTGIS_2_TRANSACTION_INTERVAL "$(POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_2)" +GUI TEXT POSTGIS_OUT_TRANSACTION_INTERVAL_POSTGIS_2 Features Per Transaction +DEFAULT_MACRO POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_2 NO +POSTGIS_2_INSERT_TEXT_GEOMETRY "$(POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_2)" +GUI CHOICE POSTGIS_OUT_INSERT_TEXT_GEOMETRY_POSTGIS_2 YES%NO Insert WKT +DEFAULT_MACRO POSTGIS_OUT_BEGIN_SQL_POSTGIS_2 +POSTGIS_2_BEGIN_SQL "$(POSTGIS_OUT_BEGIN_SQL_POSTGIS_2)" +GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGIS_OUT_BEGIN_SQL_POSTGIS_2 MODE,SQL;FORMAT,POSTGIS SQL To Run Before Write +DEFAULT_MACRO POSTGIS_OUT_END_SQL_POSTGIS_2 +POSTGIS_2_END_SQL "$(POSTGIS_OUT_END_SQL_POSTGIS_2)" +GUI OPTIONAL TEXT_EDIT_SQL_CFG POSTGIS_OUT_END_SQL_POSTGIS_2 MODE,SQL;FORMAT,POSTGIS SQL To Run After Write +# ============================================================================ +POSTGIS_2_GENERATE_FME_BUILD_NUM 20608 +POSTGIS_2_DATASET "$(DestDataset_POSTGIS_2)" #! END_DEST_HEADER #! START_WB_HEADER #! END_WB_HEADER @@ -2216,7 +2238,7 @@ FME_IMPLICIT_CSMAP_REPROJECTION_MODE Auto FME_GEOMETRY_HANDLING Enhanced FME_STROKE_MAX_DEVIATION 0 DEFAULT_MACRO DATASET_KEYWORD_ESRISHAPE_1 ESRISHAPE_1 -DEFAULT_MACRO DATASET_KEYWORD_POSTGIS_1 POSTGIS_1 +DEFAULT_MACRO DATASET_KEYWORD_POSTGIS_2 POSTGIS_2 # ------------------------------------------------------------------------- ESRISHAPE_1_READER_META_ATTRIBUTES fme_feature_type @@ -2277,21 +2299,21 @@ FACTORY_DEF * TeeFactory FACTORY_NAME Existing_Survey_IDs_VectorSetterUpper # Note that we don't call the ThinPointCloud function directly, but rather # do it through TCL. This should allow this transformer to be used with a # Desktop license, even though ThinPointCloud requires Professional. -Tcl2 proc Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Existing_Survey_IDs___PointCloudThinner INPUT FEATURE_TYPE Existing_Survey_IDs___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_thinPointCloud") -Tcl2 proc Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Existing_Survey_IDs_RasterSetterUpper INPUT FEATURE_TYPE Existing_Survey_IDs___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_prepareRaster {} {} {} {}") +Tcl2 proc Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Existing_Survey_IDs___PointCloudThinner INPUT FEATURE_TYPE Existing_Survey_IDs___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_thinPointCloud") +Tcl2 proc Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Existing_Survey_IDs_RasterSetterUpper INPUT FEATURE_TYPE Existing_Survey_IDs___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_prepareRaster {} {} {} {}") # Now route all the features into the recorder, changing their # feature type to the transformer name so that they view nicely -# Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_VIS_FEAT_TYPE changes the feature types +# Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_VIS_FEAT_TYPE changes the feature types # of the features being visualized. -DEFAULT_MACRO Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_VIS_FEAT_TYPE Existing_Survey_IDs$(CATMAC) +DEFAULT_MACRO Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_VIS_FEAT_TYPE Existing_Survey_IDs$(CATMAC) DEFAULT_MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/$(SAFE_FFS_NAME).ffs DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO # [PR#45549] The Data Inspector will limit the maximum number of features # it reads on WIN32, so we will correspondingly limit the number of # features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME Existing_Survey_IDs_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Existing_Survey_IDs_377532e6_c8c4_49ab_9328_7cac6d2a061e1_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA surveyid,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * RecorderFactory FACTORY_NAME Existing_Survey_IDs_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Existing_Survey_IDs_6bd93e57_9e23_4150_9786_36b09779b5164_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA surveyid,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 # ------------------------------------------------------------------------- # Expose all attributes. This ensures that if you expose fme_dataset, you will get it. READER_META_ATTRIBUTES fme_feature_type fme_dataset fme_basename @@ -2299,8 +2321,8 @@ FACTORY_DEF * TeeFactory FACTORY_NAME AttributeExposer INPUT FEATURE_TYPE # ------------------------------------------------------------------------- MACRO casemac INCLUDE [ if { {no} == {no} } { puts {MACRO casemac -nocase}} ] -Tcl2 proc StringSearcher_17331842_b517_4d1c_bfd0_9c0bc5a2ba271_doGrep { stringToSearch regExp matchedChars matches matchedParts } { set subMatches 1; set matches [FME_DecodeText $matches]; set matchedParts [FME_DecodeText $matchedParts]; set stringToSearch [FME_ReplaceRegex "\015\012?" [FME_DecodeText $stringToSearch] "\012"]; set results [FME_GetRegex $(casemac) [FME_DecodeText $regExp] $stringToSearch]; if {[llength [lindex $results 0]] > 0} { set attrName [FME_DecodeText $matchedChars]; FME_SetAttribute $attrName [lindex [lindex [lindex $results 0] 0] 0]; set index 0; set parts [lindex $results 1]; foreach part $parts { if {[llength $matchedParts] > 0} { set attrName "$matchedParts{$index}.part"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matchedParts{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; incr index; }; set index 0; set regexMatches [lindex $results 0]; foreach part $regexMatches { if {[llength $matches] > 0} { set attrName "$matches{$index}.match"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matches{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; set isFirstMatch 0; incr index; }; return $index; }; return -1; } -FACTORY_DEF * TestFactory FACTORY_NAME StringSearcher INPUT FEATURE_TYPE AttributeExposer_OUTPUT TEST @Tcl2("StringSearcher_17331842_b517_4d1c_bfd0_9c0bc5a2ba271_doGrep {@EvaluateExpression(FDIV,STRING_ENCODED,Valuefme_basename,StringSearcher)} {^d+} {SurveyID} {} {}") > 0 OUTPUT PASSED FEATURE_TYPE StringSearcher_MATCHED OUTPUT FAILED FEATURE_TYPE StringSearcher_NOT_MATCHED +Tcl2 proc StringSearcher_0533f88c_15ff_46c3_b29c_5f98076f4b2a4_doGrep { stringToSearch regExp matchedChars matches matchedParts } { set subMatches 1; set matches [FME_DecodeText $matches]; set matchedParts [FME_DecodeText $matchedParts]; set stringToSearch [FME_ReplaceRegex "\015\012?" [FME_DecodeText $stringToSearch] "\012"]; set results [FME_GetRegex $(casemac) [FME_DecodeText $regExp] $stringToSearch]; if {[llength [lindex $results 0]] > 0} { set attrName [FME_DecodeText $matchedChars]; FME_SetAttribute $attrName [lindex [lindex [lindex $results 0] 0] 0]; set index 0; set parts [lindex $results 1]; foreach part $parts { if {[llength $matchedParts] > 0} { set attrName "$matchedParts{$index}.part"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matchedParts{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; incr index; }; set index 0; set regexMatches [lindex $results 0]; foreach part $regexMatches { if {[llength $matches] > 0} { set attrName "$matches{$index}.match"; FME_SetAttribute $attrName [lindex $part 0]; set attrName "$matches{$index}.startIndex"; FME_SetAttribute $attrName [lindex $part 1]; }; set isFirstMatch 0; incr index; }; return $index; }; return -1; } +FACTORY_DEF * TestFactory FACTORY_NAME StringSearcher INPUT FEATURE_TYPE AttributeExposer_OUTPUT TEST @Tcl2("StringSearcher_0533f88c_15ff_46c3_b29c_5f98076f4b2a4_doGrep {@EvaluateExpression(FDIV,STRING_ENCODED,Valuefme_basename,StringSearcher)} {^d+} {SurveyID} {} {}") > 0 OUTPUT PASSED FEATURE_TYPE StringSearcher_MATCHED OUTPUT FAILED FEATURE_TYPE StringSearcher_NOT_MATCHED # ------------------------------------------------------------------------- # Set the directive to add all the meta-attributes to the features. # We do this so users see all possible information in the viewed @@ -2322,21 +2344,21 @@ FACTORY_DEF * TeeFactory FACTORY_NAME Survey_ID_Not_Parsed_VectorSetterUpper # Note that we don't call the ThinPointCloud function directly, but rather # do it through TCL. This should allow this transformer to be used with a # Desktop license, even though ThinPointCloud requires Professional. -Tcl2 proc Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Survey_ID_Not_Parsed___PointCloudThinner INPUT FEATURE_TYPE Survey_ID_Not_Parsed___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_thinPointCloud") -Tcl2 proc Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Survey_ID_Not_Parsed_RasterSetterUpper INPUT FEATURE_TYPE Survey_ID_Not_Parsed___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_prepareRaster {} {} {} {}") +Tcl2 proc Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Survey_ID_Not_Parsed___PointCloudThinner INPUT FEATURE_TYPE Survey_ID_Not_Parsed___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_thinPointCloud") +Tcl2 proc Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Survey_ID_Not_Parsed_RasterSetterUpper INPUT FEATURE_TYPE Survey_ID_Not_Parsed___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_prepareRaster {} {} {} {}") # Now route all the features into the recorder, changing their # feature type to the transformer name so that they view nicely -# Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_VIS_FEAT_TYPE changes the feature types +# Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_VIS_FEAT_TYPE changes the feature types # of the features being visualized. -DEFAULT_MACRO Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_VIS_FEAT_TYPE Survey_ID_Not_Parsed$(CATMAC) +DEFAULT_MACRO Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_VIS_FEAT_TYPE Survey_ID_Not_Parsed$(CATMAC) DEFAULT_MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/$(SAFE_FFS_NAME).ffs DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO # [PR#45549] The Data Inspector will limit the maximum number of features # it reads on WIN32, so we will correspondingly limit the number of # features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME Survey_ID_Not_Parsed_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Survey_ID_Not_Parsed_ff6a723b_97ff_4535_83ec_1ccf022caa7f1_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * RecorderFactory FACTORY_NAME Survey_ID_Not_Parsed_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Survey_ID_Not_Parsed_6f7ab5e3_13f5_4092_bd45_2b8047a650404_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 # ------------------------------------------------------------------------- INCLUDE [if { {ATTRIBUTES} == {ATTRIBUTES} } { puts "MACRO FeatureMerger_REFERENCE_INFO ATTRIBUTES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {POLYGONS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_POLYS"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {AGGREGATES}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} && {} == {LINESFROMPOINTS}} { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_LINES_FROM_POINTS"; } elseif { {ATTRIBUTES} == {GEOM_BUILD} } { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_BUILD_AGGREGATES"; } elseif { {ATTRIBUTES} == {GEOM_AND_ATTRS} } { puts "MACRO FeatureMerger_REFERENCE_INFO GEOM_AND_ATTR_BUILD_AGGREGATES"; } else {}; ] FACTORY_DEF {*} ReferenceFactory FACTORY_NAME FeatureMerger INPUT REFERENCER FEATURE_TYPE StringSearcher_MATCHED INPUT REFERENCEE FEATURE_TYPE SQLCreator_RESULT_1_LV2HdoY65lk= REFERENCE_INFO $(FeatureMerger_REFERENCE_INFO) REFERENCE_TABLE @EvaluateExpression(FDIV,STRING_ENCODED,ValueSurveyID,FeatureMerger) @EvaluateExpression(FDIV,STRING_ENCODED,Valuesurveyid,FeatureMerger) AUTO ATTR_ACCUM_MODE "HANDLE_CONFLICT" ATTR_CONFLICT_RES "REQUESTOR_IF_CONFLICT" IGNORE_NULLS "No" HANDLE_NULL_MISSING_KEYS_LIKE_FME2013 No MERGE_ATTRIBUTES Yes MANAGE_FME_TYPE Yes MODE COMPLETE PROCESS_DUPLICATE_REFERENCEES No REFERENCEES_FIRST No OUTPUT COMPLETE FEATURE_TYPE FeatureMerger_MERGED OUTPUT INCOMPLETE FEATURE_TYPE FeatureMerger_NOT_MERGED @@ -2364,23 +2386,23 @@ FACTORY_DEF * TeeFactory FACTORY_NAME Points_Not_Present_VectorSetterUpper # Note that we don't call the ThinPointCloud function directly, but rather # do it through TCL. This should allow this transformer to be used with a # Desktop license, even though ThinPointCloud requires Professional. -Tcl2 proc Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Points_Not_Present___PointCloudThinner INPUT FEATURE_TYPE Points_Not_Present___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_thinPointCloud") -Tcl2 proc Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Points_Not_Present_RasterSetterUpper INPUT FEATURE_TYPE Points_Not_Present___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_prepareRaster {} {} {} {}") +Tcl2 proc Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Points_Not_Present___PointCloudThinner INPUT FEATURE_TYPE Points_Not_Present___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_thinPointCloud") +Tcl2 proc Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Points_Not_Present_RasterSetterUpper INPUT FEATURE_TYPE Points_Not_Present___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_prepareRaster {} {} {} {}") # Now route all the features into the recorder, changing their # feature type to the transformer name so that they view nicely -# Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_VIS_FEAT_TYPE changes the feature types +# Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_VIS_FEAT_TYPE changes the feature types # of the features being visualized. -DEFAULT_MACRO Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_VIS_FEAT_TYPE Points_Not_Present$(CATMAC) +DEFAULT_MACRO Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_VIS_FEAT_TYPE Points_Not_Present$(CATMAC) DEFAULT_MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/$(SAFE_FFS_NAME).ffs DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO # [PR#45549] The Data Inspector will limit the maximum number of features # it reads on WIN32, so we will correspondingly limit the number of # features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME Points_Not_Present_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Points_Not_Present_3d53bc66_e9e5_4bb0_a2a4_bcaccb5c89571_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200,SurveyID,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * RecorderFactory FACTORY_NAME Points_Not_Present_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Points_Not_Present_1f8094ea_4cc0_4578_8c8f_cd3ca0672e834_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200,SurveyID,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 # ------------------------------------------------------------------------- -Tcl2 set AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitDelim [FME_DecodeText {.}]; if { [regexp {^([1-9][0-9]*s)+$} [set AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitDelim]] } { set AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitWidths [split [regsub -all {s$} {.} {}] s]; proc AttributeSplitter_2_doSplit {} { global AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitWidths; set source [FME_GetAttribute [FME_DecodeText {fme_feature_type}]]; set attrNum 0; set listName [FME_DecodeText {_list}]; set attrPos 0; set keepEmptyParts [string equal {No} {No}]; foreach width [set AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitWidths] { set endPos [expr $attrPos + $width - 1]; set bit [string range $source $attrPos $endPos]; set part [string trim $bit]; if { $keepEmptyParts || $part != \"\" } { FME_SetAttribute "$listName{$attrNum}" $part; incr attrNum; }; incr attrPos $width; }; }; } else { proc AttributeSplitter_2_doSplit {} { global AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitDelim; set delim [set AttributeSplitter_2_a326500f_a938_4f47_8c04_c9dac66ad9d51_splitDelim]; set delimLength [string length $delim]; set source [FME_GetAttribute [FME_DecodeText {fme_feature_type}]]; set keepEmptyParts [string equal {No} {No}]; set bits {}; set startIndex 0; set nextIndex [string first $delim $source $startIndex]; while {$nextIndex >= 0} { lappend bits [string range $source $startIndex [expr $nextIndex-1]]; set startIndex [expr $nextIndex + $delimLength]; set nextIndex [string first $delim $source $startIndex]; }; lappend bits [string range $source $startIndex end]; set listName [FME_DecodeText {_list}]; set attrNum 0; foreach bit $bits { set trimmedPart [string trim $bit]; if { $keepEmptyParts || $trimmedPart != \"\" } { FME_SetAttribute "$listName{$attrNum}" $trimmedPart; incr attrNum; }; } } } +Tcl2 set AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitDelim [FME_DecodeText {.}]; if { [regexp {^([1-9][0-9]*s)+$} [set AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitDelim]] } { set AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitWidths [split [regsub -all {s$} {.} {}] s]; proc AttributeSplitter_2_doSplit {} { global AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitWidths; set source [FME_GetAttribute [FME_DecodeText {fme_feature_type}]]; set attrNum 0; set listName [FME_DecodeText {_list}]; set attrPos 0; set keepEmptyParts [string equal {No} {No}]; foreach width [set AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitWidths] { set endPos [expr $attrPos + $width - 1]; set bit [string range $source $attrPos $endPos]; set part [string trim $bit]; if { $keepEmptyParts || $part != \"\" } { FME_SetAttribute "$listName{$attrNum}" $part; incr attrNum; }; incr attrPos $width; }; }; } else { proc AttributeSplitter_2_doSplit {} { global AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitDelim; set delim [set AttributeSplitter_2_364c81e2_9a64_4822_8698_6f3dc91bfc974_splitDelim]; set delimLength [string length $delim]; set source [FME_GetAttribute [FME_DecodeText {fme_feature_type}]]; set keepEmptyParts [string equal {No} {No}]; set bits {}; set startIndex 0; set nextIndex [string first $delim $source $startIndex]; while {$nextIndex >= 0} { lappend bits [string range $source $startIndex [expr $nextIndex-1]]; set startIndex [expr $nextIndex + $delimLength]; set nextIndex [string first $delim $source $startIndex]; }; lappend bits [string range $source $startIndex end]; set listName [FME_DecodeText {_list}]; set attrNum 0; foreach bit $bits { set trimmedPart [string trim $bit]; if { $keepEmptyParts || $trimmedPart != \"\" } { FME_SetAttribute "$listName{$attrNum}" $trimmedPart; incr attrNum; }; } } } FACTORY_DEF * TeeFactory FACTORY_NAME AttributeSplitter_2 INPUT FEATURE_TYPE FeatureMerger_NOT_MERGED_0_mc6xVklKjb8= OUTPUT FEATURE_TYPE AttributeSplitter_2_OUTPUT @Tcl2(AttributeSplitter_2_doSplit) # ------------------------------------------------------------------------- Tcl2 proc ListIndexer_2_indexer {index numElements prefix suffix} { if {$index < 0 } { set index [expr $numElements + $index]; }; regsub \"{}\" {_list{}} \"{$index}\" listWithIndex; regsub \"{}\" {_list{}} {} listBase; set complexListExp "$listWithIndex\."; set prefixLength [expr [string length $complexListExp]-1]; set prefix [FME_DecodeText $prefix]; set suffix [FME_DecodeText $suffix]; foreach attr [FME_AttributeNames] { set newAttr {}; if {$attr == $listWithIndex} { set newAttr $listBase; } elseif {[string first $complexListExp $attr] == 0} { set newAttr [string replace $attr 0 $prefixLength]; }; if {$newAttr != {}} { if {$prefix != {} } { set newAttr $prefix$newAttr; }; if {$suffix != {} } { set newAttr $newAttr$suffix; }; if {[string first {fme_} $newAttr] != 0} { set first [string first {fme_} $newAttr]; lappend attrsToCopy $newAttr $attr; }; }; }; if {[info exists attrsToCopy] && [llength $attrsToCopy] > 0} { eval FME_Execute CopyAttributes $attrsToCopy; }; } @@ -2407,21 +2429,21 @@ FACTORY_DEF * TeeFactory FACTORY_NAME ListIndexer_2__Rejected__VectorSetterUp # Note that we don't call the ThinPointCloud function directly, but rather # do it through TCL. This should allow this transformer to be used with a # Desktop license, even though ThinPointCloud requires Professional. -Tcl2 proc ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME ListIndexer_2__Rejected____PointCloudThinner INPUT FEATURE_TYPE ListIndexer_2__Rejected____PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_thinPointCloud") -Tcl2 proc ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME ListIndexer_2__Rejected__RasterSetterUpper INPUT FEATURE_TYPE ListIndexer_2__Rejected____RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_prepareRaster {} {} {} {}") +Tcl2 proc ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME ListIndexer_2__Rejected____PointCloudThinner INPUT FEATURE_TYPE ListIndexer_2__Rejected____PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_thinPointCloud") +Tcl2 proc ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME ListIndexer_2__Rejected__RasterSetterUpper INPUT FEATURE_TYPE ListIndexer_2__Rejected____RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_prepareRaster {} {} {} {}") # Now route all the features into the recorder, changing their # feature type to the transformer name so that they view nicely -# ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_VIS_FEAT_TYPE changes the feature types +# ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_VIS_FEAT_TYPE changes the feature types # of the features being visualized. -DEFAULT_MACRO ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_VIS_FEAT_TYPE ListIndexer_2__Rejected_$(CATMAC) +DEFAULT_MACRO ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_VIS_FEAT_TYPE ListIndexer_2__Rejected_$(CATMAC) DEFAULT_MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/$(SAFE_FFS_NAME).ffs DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO # [PR#45549] The Data Inspector will limit the maximum number of features # it reads on WIN32, so we will correspondingly limit the number of # features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME ListIndexer_2__Rejected__Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(ListIndexer_2__Rejected__ac3c7ce7_877f_46d6_825a_cd5b83acc6551_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200,SurveyID,varchar200,_list,varchar200,fme_rejection_code,buffer MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * RecorderFactory FACTORY_NAME ListIndexer_2__Rejected__Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(ListIndexer_2__Rejected__12ff7b07_c06f_439d_ba9d_3cf8691823df4_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200,SurveyID,varchar200,_list,varchar200,fme_rejection_code,buffer MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 # ------------------------------------------------------------------------- FACTORY_DEF {*} AttrSetFactory COMMAND_PARM_EVALUATION SINGLE_PASS FACTORY_NAME AttributeRenamer INPUT FEATURE_TYPE ListIndexer_2_OUTPUT ATTRSET_CREATE_DIRECTIVES _PROPAGATE_MISSING_FDIV ATTR_ACTION { "_list" "ShapefileName" "RENAME_SET_DEFAULT" "" } OUTPUT OUTPUT FEATURE_TYPE AttributeRenamer_OUTPUT # ------------------------------------------------------------------------- @@ -2449,21 +2471,21 @@ FACTORY_DEF * TeeFactory FACTORY_NAME Survey_Pts_Already_Present_VectorSetter # Note that we don't call the ThinPointCloud function directly, but rather # do it through TCL. This should allow this transformer to be used with a # Desktop license, even though ThinPointCloud requires Professional. -Tcl2 proc Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Survey_Pts_Already_Present___PointCloudThinner INPUT FEATURE_TYPE Survey_Pts_Already_Present___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_thinPointCloud") -Tcl2 proc Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } -FACTORY_DEF * TeeFactory FACTORY_NAME Survey_Pts_Already_Present_RasterSetterUpper INPUT FEATURE_TYPE Survey_Pts_Already_Present___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_prepareRaster {} {} {} {}") +Tcl2 proc Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_thinPointCloud {} { if {[string equal {NO_THINNING} {KEEPNPOINT}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {MAXNUMPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {FIRSTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; } elseif {[string equal {NO_THINNING} {LASTNPOINTS}]} { FME_Execute ThinPointCloud NO_THINNING "\"\""; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Survey_Pts_Already_Present___PointCloudThinner INPUT FEATURE_TYPE Survey_Pts_Already_Present___PointCloudFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_thinPointCloud") +Tcl2 proc Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_prepareRaster {subsetStartRow subsetNumRows subsetStartCol subsetNumCols} { set useDI [string equal -nocase {$(FME_VIEWER_APP)} {fmedatainspector}]; if {!$useDI} { FME_Execute ApplyRasterRotation BILINEAR; }; set doResample [string equal {NoReduction} {Resample}]; set doSubset [string equal {NoReduction} {Subset}]; set doBounds [string equal {NoReduction} {BoundingBoxOnly}]; if {$doBounds} { FME_Execute GeometryType fme_polygon; } elseif {$doResample || $doSubset} { set comment { First, we need to prefix all the attributes. This is done because we are going to call @RasterProperties, and we want to make sure it doesn't overwrite any existing attributes. }; set kAttrPrefix "."; set allNames [FME_AttributeNames]; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $newName $oldName; }; FME_Execute RasterProperties RASTER; if {$doResample} { set oldNumCols [FME_GetAttribute _num_columns]; set oldNumRows [FME_GetAttribute _num_rows]; set oldSpacingX [FME_GetAttribute _spacing_x]; set oldSpacingY [FME_GetAttribute _spacing_y]; set kMinNumCells 32; set kMaxNumCells 512; set comment { We will only resample if both dimensions are greater than the minimum num cells, and at least one is greater than the maximum. }; if {$oldNumRows > $kMinNumCells && $oldNumCols > $kMinNumCells && ($oldNumRows > $kMaxNumCells || $oldNumCols > $kMaxNumCells)} { set newSpacingX 1.0; set newSpacingY 1.0; if {$oldNumRows > $oldNumCols} { set comment { If we have more rows than columns, we need to choose a spacing that reduces the number of rows to the maximum we've chosen. }; set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMaxNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; set comment { Now we need to check if doing this will result in the columns going below the minimum. If so, choose the spacing such that we only go down to the minimum (and not past). }; set newNumCols [expr $oldNumCols * $oldSpacingX / $newSpacingX]; if {$newNumCols < $kMinNumCells} { set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMinNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; }; } else { set comment { This is identical to the above case, except the roles of rows and cols are reversed. }; set newSpacingX [expr 1.0 * $oldNumCols * $oldSpacingX / $kMaxNumCells]; set newSpacingY [expr 1.0 * $newSpacingX / $oldSpacingX * $oldSpacingY]; set newNumRows [expr $oldNumRows * $oldSpacingY / $newSpacingY]; if {$newNumRows < $kMinNumCells} { set newSpacingY [expr 1.0 * $oldNumRows * $oldSpacingY / $kMinNumCells]; set newSpacingX [expr 1.0 * $newSpacingY / $oldSpacingY * $oldSpacingX]; }; }; FME_Execute ResampleRaster CELL_SIZE $newSpacingX $newSpacingY NearestNeighbor; }; } elseif {$doSubset} { set comment { Get the value of some strings that may be attributes or constants. Note that we pass in the attribute prefix we're using, because if one does turn out to be an attribute, it won't actually be the exact name in the string (since we renamed all our attributes above). }; set rasterNumRows [FME_GetAttribute _num_rows]; set rasterNumCols [FME_GetAttribute _num_columns]; set comment { Only do subsetting if the specified start position is actually within the raster. }; if {$subsetStartRow < $rasterNumRows && $subsetStartCol < $rasterNumCols} { set comment { Now bound the subset to the size of the raster, so that we don't create padding. }; if {[expr $subsetStartRow + $subsetNumRows] > $rasterNumRows} { set subsetNumRows [expr $rasterNumRows - $subsetStartRow]; }; if {[expr $subsetStartCol + $subsetNumCols] > $rasterNumCols} { set subsetNumCols [expr $rasterNumCols - $subsetStartCol]; }; FME_Execute SubsetRaster $subsetStartRow $subsetNumRows $subsetStartCol $subsetNumCols; }; }; set comment { Remove all the attributes added by @RasterProperties and remove the prefix from the real attributes. We assume they all start with an underscore. }; FME_Execute RemoveAttributes fme_regexp_match {^_}; foreach oldName $allNames { set newName $kAttrPrefix; append newName $oldName; FME_RenameAttribute $oldName $newName; }; }; } +FACTORY_DEF * TeeFactory FACTORY_NAME Survey_Pts_Already_Present_RasterSetterUpper INPUT FEATURE_TYPE Survey_Pts_Already_Present___RasterFeatures___ OUTPUT FEATURE_TYPE __viewme__ @Tcl2("Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_prepareRaster {} {} {} {}") # Now route all the features into the recorder, changing their # feature type to the transformer name so that they view nicely -# Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_VIS_FEAT_TYPE changes the feature types +# Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_VIS_FEAT_TYPE changes the feature types # of the features being visualized. -DEFAULT_MACRO Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_VIS_FEAT_TYPE Survey_Pts_Already_Present$(CATMAC) +DEFAULT_MACRO Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_VIS_FEAT_TYPE Survey_Pts_Already_Present$(CATMAC) DEFAULT_MACRO VISUALIZER_FEATURE_FILE $(WORKSPACE_TEMP_DIR)/$(SAFE_FFS_NAME).ffs DEFAULT_MACRO VISUALIZER_CREATE_SPATIAL_INDEX NO # [PR#45549] The Data Inspector will limit the maximum number of features # it reads on WIN32, so we will correspondingly limit the number of # features we record here using the MAX_FEATURES_TO_RECORD directive. -FACTORY_DEF * RecorderFactory FACTORY_NAME Survey_Pts_Already_Present_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Survey_Pts_Already_Present_ac41f4bd_43a7_4d6f_a457_549ffe96e8a41_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200,SurveyID,varchar200,surveyid,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 +FACTORY_DEF * RecorderFactory FACTORY_NAME Survey_Pts_Already_Present_Recorder INPUT FEATURE_TYPE __viewme__ _wb_termination_feature Yes @FeatureType(TERMINATOR_@Value(_wb_termination_xformer)) INPUT FEATURE_TYPE __viewme__ @FeatureType($(Survey_Pts_Already_Present_a76e08a6_f2be_4d70_a9d0_f9384e87e2154_VIS_FEAT_TYPE)) FEATURE_FILE "$(VISUALIZER_FEATURE_FILE)" CREATE_SPATIAL_INDEX "$(VISUALIZER_CREATE_SPATIAL_INDEX)" INSPECTOR "$(VISUALIZER_CREATE_SPATIAL_INDEX)" FSC_SCHEMA DHSID,varchar254,DHSCC,varchar254,DHSYEAR,number160,DHSCLUST,number160,CCFIPS,varchar254,ADM1FIPS,varchar254,ADM1FIPSNA,varchar254,ADM1SALBNA,varchar254,ADM1SALBCO,varchar254,ADM1DHS,number160,ADM1NAME,varchar254,DHSREGCO,number160,DHSREGNA,varchar254,SOURCE,varchar254,URBAN_RURA,varchar254,LATNUM,number166,LONGNUM,number166,ALT_GPS,number160,ALT_DEM,number160,DATUM,varchar254,fme_feature_type,varchar50,fme_basename,varchar200,fme_dataset,varchar200,SurveyID,varchar200,surveyid,varchar200 MODE RECORD STORE_SCANNED_METADATA RECORD_DIRECTLY_TO_DISK YES MAX_FEATURES_TO_RECORD WIN32 2000000 # ------------------------------------------------------------------------- FACTORY_DEF * RoutingFactory FACTORY_NAME "Destination Feature Type Routing Correlator" COMMAND_PARM_EVALUATION SINGLE_PASS INPUT FEATURE_TYPE * ROUTE FME_GENERIC BulkAttributeRenamer_OUTPUT TO POSTGIS __GO_TO_FINAL_OUTPUT_ROUTER__ multi_writer_id,0,SupplyAttributesENCODED__wb_out_feat_type__dhs_data_locations.dhs_cluster_locs GEOMETRY FEATURE_TYPE_ATTRIBUTE __wb_out_feat_type__ OUTPUT ROUTED FEATURE_TYPE * OUTPUT NOT_ROUTED FEATURE_TYPE __nuke_me__ @Tcl2("FME_StatMessage 818059 [FME_GetAttribute fme_template_feature_type] 818060 818061 fme_warn") @@ -2477,4 +2499,4 @@ DEFAULT_MACRO FME_LAUNCH_VIEWER_APP NO FACTORY_DEF * CreationFactory FACTORY_NAME VisualizerProcessor CREATE_AT_END OUTPUT FEATURE_TYPE __NUKEME__ @TCL("if {{$(FME_LAUNCH_VIEWER_APP)} == {YES}} { if [file exists {$(WORKSPACE_TEMP_DIR)}] {set files [glob -nocomplain -directory {$(WORKSPACE_TEMP_DIR)} -tails *.ffs]; if {[llength $files] == 1} { set theFile {$(WORKSPACE_TEMP_DIR)/}; append theFile [lindex $files 0]; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -f FFS \042$theFile\042 & \175; } elseif { [llength $files] == 0 } { catch { file delete -force {$(WORKSPACE_TEMP_DIR)} }; } else { set filesNoExt {}; set lastfile {}; foreach {fileName} [lsort $files] { set fileNoExt [string trimright $fileName {.ffs}]; if [regexp {(.*)_[0-9]*$} $fileNoExt wholething fileNoExtBase] { if { $fileNoExtBase == $lastfile } { continue } }; set lastfile $fileNoExt; lappend filesNoExt $fileNoExt }; eval FME_Execute System \173\042$(FME_HOME_UNIX)/fmedatainspector\042 --visualizer --single-application --delete-on-close --no-source-prompt -d {$(WORKSPACE_TEMP_DIR)} -a ffs -f FFS $filesNoExt & \175; }; }; }; ") FACTORY_DEF * TeeFactory FACTORY_NAME VisualizerCleaner INPUT FEATURE_TYPE __NUKEME__ # ------------------------------------------------------------------------- -POSTGIS_1_DEF dhs_data_locations.dhs_cluster_locs postgis_type postgis_geometry postgis_mode "" postgis_sql_key_columns "" postgis_drop_table "" postgis_truncate_table "" fme_feature_operation INSERT fme_table_handling USE_EXISTING postgis_create_with_oids YES postgis_create_gist_index YES postgis_spatial_column_type geometry postgis_spatial_column geom postgis_srid 4326 postgis_vacuum_analyze YES postgis_allow_serial_writing NO dhsid varchar(80) surveyid int4 dhscc varchar(6) dhsyear int8 dhsclust int8 ccfips varchar(80) adm1fips varchar(80) adm1fipsna varchar(80) adm1salbna varchar(80) adm1salbco varchar(80) adm1dhs int8 adm1name varchar(80) dhsregco int8 dhsregna varchar(80) source varchar(20) urban_rura varchar(20) latnum numeric(16,6) longnum numeric(16,6) alt_gps int8 alt_dem int8 datum varchar(80) shapefilename varchar(10) +POSTGIS_2_DEF dhs_data_locations.dhs_cluster_locs postgis_type postgis_geometry postgis_mode "" postgis_sql_key_columns "" postgis_drop_table "" postgis_truncate_table "" fme_feature_operation INSERT fme_table_handling CREATE_IF_MISSING postgis_create_gist_index YES postgis_spatial_column_type geometry postgis_spatial_column geom postgis_srid 4326 postgis_vacuum_analyze YES postgis_allow_serial_writing NO dhsid varchar(80) surveyid int4 dhscc varchar(6) dhsyear int8 dhsclust int8 ccfips varchar(80) adm1fips varchar(80) adm1fipsna varchar(80) adm1salbna varchar(80) adm1salbco varchar(80) adm1dhs int8 adm1name varchar(80) dhsregco int8 dhsregna varchar(80) source varchar(20) urban_rura varchar(20) latnum numeric(16,6) longnum numeric(16,6) alt_gps int8 alt_dem int8 datum varchar(80) shapefilename varchar(10) diff --git a/DHS_To_Database/run_processing.py b/DHS_To_Database/run_processing.py index e8f3173..a180904 100644 --- a/DHS_To_Database/run_processing.py +++ b/DHS_To_Database/run_processing.py @@ -96,5 +96,5 @@ def run(download_urls_list, staging_folder, parse_dcfs=False, parse_data=False): parse_dat_file(dat_file, spec_file, parsed_data_folder) -if __name__ == '__main__': - run(r"C:\Users\zool1301\Documents\Data\DHS\dhs_hierarchical\download_urls.txt", r"C:\Users\zool1301\Documents\Data\DHS\staging", False, True) \ No newline at end of file +#if __name__ == '__main__': +# run(r"D:\InformalCities\DHS\dhs_hierarchical\download_urls.txt", r"D:\InformalCities\DHS\staging", False, False) \ No newline at end of file