From fafe48bcdb9845015ce43fb1e0d861650e1ccde0 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 15:51:02 -0800 Subject: [PATCH 01/13] updated transformers intro page --- docs/docs/transformers/introduction.mdx | 13 ++++++++----- .../transformers/{custom.mdx => user-defined.mdx} | 4 ++-- docs/sidebars.ts | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) rename docs/docs/transformers/{custom.mdx => user-defined.mdx} (97%) diff --git a/docs/docs/transformers/introduction.mdx b/docs/docs/transformers/introduction.mdx index 30e853b565..a2ae53a25a 100644 --- a/docs/docs/transformers/introduction.mdx +++ b/docs/docs/transformers/introduction.mdx @@ -19,16 +19,19 @@ import { DocsImage } from '@site/src/CustomComponents/DocsImage'; ## Introduction -Transformers are data-type specific modules that anonymize or generate data. Tranformers are defined in the job workflow and are applied to every piece of data in the column they are assigned. Neosync ships with a number of transformers that are already built to handle common data types such as email, addresses, ssn, strings, integers and more. +Transformers are data-type specific modules that anonymize or generate data. For example, an email transformer can create a synthetic email address or anonymize an existing email address. - +Tranformers are defined in the job workflow and are applied to every piece of data in the column they are assigned. Neosync ships with over 40 pre-defined transformers that make it fast and easy to get started. + + ## Transformer Types There are two types of transformers: 1. **System Transformers** : Pre-built and configured transformers that automatically come with Neosync out of the box. These cannot be deleted but they can be cloned. -2. **Custom Transformers** : System transformers that have been cloned, re-configured and renamed. These transformers can be edited and deleted. + +2. **User Defined Transformers** : These are system transformers that have been cloned, re-configured and renamed. These transformers can be edited and deleted. ## Transformers @@ -43,9 +46,9 @@ You can browse our list of system transformers which come out of the box or you icon: , }, { - title: 'Custom Transformers', + title: 'User Defined Transformers', description: 'Define your own transformers', - link: '/transformers/custom', + link: '/transformers/user-defined', icon: , }, ]} diff --git a/docs/docs/transformers/custom.mdx b/docs/docs/transformers/user-defined.mdx similarity index 97% rename from docs/docs/transformers/custom.mdx rename to docs/docs/transformers/user-defined.mdx index d760fea69c..df8103755b 100644 --- a/docs/docs/transformers/custom.mdx +++ b/docs/docs/transformers/user-defined.mdx @@ -1,8 +1,8 @@ --- title: Custom -id: custom +id: user-defined hide_title: true -slug: /transformers/custom +slug: /transformers/user-defined --- import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx'; diff --git a/docs/sidebars.ts b/docs/sidebars.ts index c5d6c9e8ea..13efbd81f1 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -288,8 +288,8 @@ const sidebars: SidebarsConfig = { }, { type: 'doc', - id: 'transformers/custom', - label: 'Custom', + id: 'transformers/user-defined', + label: 'User Defined', }, ], }; From 830fb51424cc625df45c4fb5c1f433173a48c654 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 18:37:48 -0800 Subject: [PATCH 02/13] init transformer table with references --- docs/docs/transformers/system/reference.mdx | 59 +++++++++ docs/sidebars.ts | 114 +----------------- .../CustomComponents/TransformersTable.tsx | 70 +++++++++++ docs/src/css/custom.css | 4 + 4 files changed, 135 insertions(+), 112 deletions(-) create mode 100644 docs/docs/transformers/system/reference.mdx create mode 100644 docs/src/CustomComponents/TransformersTable.tsx diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx new file mode 100644 index 0000000000..9cadb34935 --- /dev/null +++ b/docs/docs/transformers/system/reference.mdx @@ -0,0 +1,59 @@ +--- +title: References +id: reference +hide_title: true +slug: /transformers/system/reference +--- + +import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx'; + +import { TransformersTable } from '@site/src/CustomComponents/TransformersTable.tsx'; + + + +## Introduction + +System transformers ship our of the box with Neosync and give you an easy way to get started with anonymizing or generating data. + +Below is a reference of all of the system transformers that Neosync ships with as well as some relevant links. + +## Reference + + + +## Generate Email{#generate-email} + +TransformersTabletestest + +## Generate Realistic Email{#generate-realistic-email} + +TransformersTabletestest + +## Transform Email{#transform-email} + +TransformersTabletestest diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 13efbd81f1..4028f78d12 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -171,118 +171,8 @@ const sidebars: SidebarsConfig = { items: [ { type: 'doc', - id: 'transformers/system/email', - label: 'Email', - }, - { - type: 'doc', - id: 'transformers/system/int-phone', - label: 'Phone (integer)', - }, - { - type: 'doc', - id: 'transformers/system/string-phone', - label: 'Phone (string)', - }, - { - type: 'doc', - id: 'transformers/system/street-address', - label: 'Street Address', - }, - { - type: 'doc', - id: 'transformers/system/city', - label: 'City', - }, - { - type: 'doc', - id: 'transformers/system/full-address', - label: 'Full Address', - }, - { - type: 'doc', - id: 'transformers/system/state', - label: 'State', - }, - { - type: 'doc', - id: 'transformers/system/card-number', - label: 'Card Number', - }, - { - type: 'doc', - id: 'transformers/system/first-name', - label: 'First Name', - }, - { - type: 'doc', - id: 'transformers/system/last-name', - label: 'Last Name', - }, - { - type: 'doc', - id: 'transformers/system/full-name', - label: 'Full Name', - }, - { - type: 'doc', - id: 'transformers/system/gender', - label: 'Gender', - }, - { - type: 'doc', - id: 'transformers/system/hash', - label: 'Hash', - }, - { - type: 'doc', - id: 'transformers/system/null-t', - label: 'Null Value', - }, - { - type: 'doc', - id: 'transformers/system/ssn', - label: 'SSN', - }, - { - type: 'doc', - id: 'transformers/system/random-bool', - label: 'Random Boolean', - }, - { - type: 'doc', - id: 'transformers/system/random-float', - label: 'Random Float', - }, - { - type: 'doc', - id: 'transformers/system/random-int', - label: 'Random Integer', - }, - { - type: 'doc', - id: 'transformers/system/random-string', - label: 'Random String', - }, - { - type: 'doc', - id: 'transformers/system/unix-ts', - label: 'Unix Timestamp', - }, - { - type: 'doc', - id: 'transformers/system/utc-ts', - label: 'UTC Timestamp', - }, - { - type: 'doc', - id: 'transformers/system/uuid', - label: 'UUID', - }, - { - type: 'doc', - id: 'transformers/system/zipcode', - label: 'Zipcode', + id: 'transformers/system/reference', + label: 'Reference', }, ], }, diff --git a/docs/src/CustomComponents/TransformersTable.tsx b/docs/src/CustomComponents/TransformersTable.tsx new file mode 100644 index 0000000000..4f9c843460 --- /dev/null +++ b/docs/src/CustomComponents/TransformersTable.tsx @@ -0,0 +1,70 @@ +import Link from '@docusaurus/Link'; +import { ArrowRightIcon } from '@radix-ui/react-icons'; +import React, { ReactElement } from 'react'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '../components/CustomTable'; + +interface TableData { + headers: string[]; + rowData: RowData[]; +} + +interface RowData { + title: string; + type: string; + description: string; + link: string; +} + +export function TransformersTable(props: TableData): ReactElement { + const { headers, rowData } = props; + + return ( + + + + {headers.map((header) => ( + {header} + ))} + + + + {rowData.map((tableData) => ( + + + +
+
{tableData.title}
+
+ +
+
+ + + + {tableData.type} + + + {tableData.description} + + + ))} + +
+ ); +} diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index 69c6c78aec..59c5706011 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -141,6 +141,10 @@ this is the class for the sidebar categories that section headers @apply text-xs; } + .menu__link--sublist-caret { + @apply text-gray-900; + } + .docsImage { @apply border border-gray-300 shadow-lg rounded-lg overflow-hidden my-6; } From 9bbcf401f2419f70a5e03dbb6065b4f6d5436d8d Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 21:53:53 -0800 Subject: [PATCH 03/13] updated transformer table and completed first few examples --- docs/docs/transformers/system/reference.mdx | 496 +++++++++++++++++- .../TransformerPageHeader.tsx | 1 - .../CustomComponents/TransformersTable.tsx | 24 +- docs/src/components/Badge.tsx | 2 +- 4 files changed, 498 insertions(+), 25 deletions(-) diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx index 9cadb34935..96cac6a433 100644 --- a/docs/docs/transformers/system/reference.mdx +++ b/docs/docs/transformers/system/reference.mdx @@ -6,54 +6,514 @@ slug: /transformers/system/reference --- import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx'; - +import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; import { TransformersTable } from '@site/src/CustomComponents/TransformersTable.tsx'; ## Introduction -System transformers ship our of the box with Neosync and give you an easy way to get started with anonymizing or generating data. +Neosync comes out of the box with 40+ system transformers in order to give you an easy way to get started with anonymizing or generating data. There are two kinds of system transformers: + +1. **Generators** - these begin with `generate_` and do **not** anonymize existing data. They only generate net-new synthetic data. -Below is a reference of all of the system transformers that Neosync ships with as well as some relevant links. +2. **Transformers** - these begin with `transform_` and **can** anonymize existing data or generate new-new synthetic data based on how you configure it. ## Reference ', + pagelink: 'generate-ssn', + codeRef: '', + }, + { + title: 'Generate State', + type: 'string', + description: + 'Randomly selects a US state and returns the two-character state code.', + pagelink: 'generate-state', + codeRef: '', + }, + { + title: 'Generate Street Address', + type: 'string', + description: + 'Randomly generates a street address in the format: {street_num} {street_addresss} {street_descriptor}. For example, 123 Main Street.', + pagelink: 'generate-street-address', + codeRef: '', + }, + { + title: 'Generate String Phone Number', + type: 'string', + description: + 'Generates a Generate phone number and returns it as a string.', + pagelink: 'generate-string-phone-number', + codeRef: '', + }, + { + title: 'Generate Random String', + type: 'string', + description: + 'Creates a randomly ordered alphanumeric string with a default length of 10 unless the String Length parameter are defined.', + pagelink: 'generate-random-string', + codeRef: '', + }, + { + title: 'Generate Unix Timestamp', + type: 'int64', + description: 'Randomly generates a Unix timestamp', + pagelink: 'generate-unix-timestamp', + codeRef: '', + }, + { + title: 'Generate Username', + type: 'string', + description: + 'Randomly generates a username in the format.', + pagelink: 'generate-username', + codeRef: '', + }, + { + title: 'Generate UTC Timestamp', + type: 'time', + description: 'Randomly generates a UTC timestamp.', + pagelink: 'generate-utc-timestamp', + codeRef: '', + }, + { + title: 'Generate UUID', + type: 'uuid', + description: 'Generates a new UUIDv4 id.', + pagelink: 'generate-uuid', + codeRef: '', + }, + { + title: 'Generate Zipcode', + type: 'string', + description: + 'Randomly selects a zip code from a list of predefined US zipcodes.', + pagelink: 'generate-zipcode', + codeRef: '', + }, + { + title: 'Transform E164 Phone Number', + type: 'string', + description: 'Transforms an existing E164 formatted phone number.', + pagelink: 'transform-e164-phone-number', + codeRef: '', + }, + { + title: 'Transform First Name', + type: 'string', + description: 'Transforms an existing first name', + pagelink: 'transform-first-name', + codeRef: '', + }, + { + title: 'Transform Float64', + type: 'float64', + description: 'Transforms an existing float value.', + pagelink: 'transform-float64', + codeRef: '', + }, + { + title: 'Transform Full Name', + type: 'string', + description: 'Transforms an existing full name.', + pagelink: 'transform-full-name', + codeRef: '', + }, + { + title: 'Transform Int64 Phone Number', + type: 'int64', + description: + 'Transforms an existing phone number that is typed as an integer', + pagelink: 'transform-int64-phone-number', + codeRef: '', + }, + { + title: 'Transform Int64', + type: 'int64', + description: 'Transforms an existing integer value.', + pagelink: 'transform-int64', + codeRef: '', + }, + { + title: 'Transform Last Name', + type: 'string', + description: 'Transforms an existing last name.', + pagelink: 'transform-last-name', + codeRef: '', + }, + { + title: 'Transform Phone Number', + type: 'string', + description: + 'Transforms an existing phone number that is typed as a string.', + pagelink: 'transform-phone-number', + codeRef: '', + }, + { + title: 'Transform String', + type: 'string', + description: 'Transforms an existing string value.', + pagelink: 'transform-string', + codeRef: '', + }, + { + title: 'Passthrough', + type: 'string', + description: + 'Passes the input value through to the desination with no changes.', + pagelink: 'passthrough', + codeRef: '', + }, + { + title: 'Null', + type: 'string', + description: 'Inserts a string instead of the source value.', + pagelink: 'null', + codeRef: '', + }, + ]} +/> + +### Generate Email{#generate-email} + +The generate email transformer generates a new randomized email address in the format: + +`@.` + +By default, the transformer randomizes the username, domain and top-level domain while always preserving the email format by retaining the `@` and `.` characters. + +For example, the following input value: +`john@acme.com` + +Would produce the following ouput value: +`ytvub873@ksh0293.com` + +**Configurations** + +There are no configurations for this transformer. + +**Examples** + + + +### Generate Realistic Email{#generate-realistic-email} + +The generate realistic email transformer generates a new randomized email address that "looks" realistis in the format: + +`@.` + +By "looks realistic", we mean that the email will look like an email that a normal person will have and won't be randomized letters and numbers. The generated email will either have a first letter initial and last name as a username and then a standardized domain name such as gmail, yahoo, hotmail, etc. + +**Configurations** + +There are no configurations for this transformer. + +**Examples** + + + +### Transform Email{#transform-email} + +The transform email transformer can anonymize an existing email address or generate a new randomized email address in the format: + +`@.` + +By default, the transformer randomizes the username, domain and top-level domain while always preserving the email format by retaining the `@` and `.` characters. + +For example, the following input value: +`john@acme.com` + +Would produce the following ouput value: +`ytvub873@ksh0293.com` + +**Configurations** + +Depending on your logic, you may want to configure the output email. The transform email transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different potential email formats. Here are some possible combinations: + + -## Generate Email{#generate-email} +### Generate Boolean{#generate-boolean} + +### Generate Card Number{#generate-card-number} + +### Generate City{#generate-city} + +### Generate E164 Phone Number{#generate-e164-phone-number} + +### Generate First Name{#generate-first-name} + +### Generate Float64{#generate-float64} + +### Generate Full Address{#generate-full-address} + +### Generate Full Name{#generate-full-name} + +### Generate Gender{#generate-gender} + +### Generate int64 Phone Number{#generate-int64-phone-number} + +### Generate Random Int64{#generate-random-int64} + +### Generate Last Name{#generate-last-name} + +### Generate SHA256 Hash{#generate-sha256-hash} + +### Generate SSN{#generate-ssn} + +### Generate State{#generate-state} + +### Generate Street Address{#generate-street-address} + +### Generate String Phone Number{#generate-string-phone-number} + +### Generate Random String{#generate-random-string} + +### Generate Unix Timestamp{#generate-unix-timestamp} + +### Generate Username{#generate-username} + +### Generate UTC Timestamp{#generate-utc-timestamp} + +### Generate UUID{#generate-uuid} + +### Generate Zipcode{#generate-zipcode} + +### Transform E164 Phone Number{#transform-e164-phone-number} + +### Transform First Name{#transform-first-name} + +### Transform Float64{#transform-float64} + +### Transform Full Name{#transform-full-name} + +### Transform Int64 Phone Number{#transform-int64-phone-number} + +### Transform Int64{#transform-int64} -TransformersTabletestest +### Transform Last Name{#transform-last-name} -## Generate Realistic Email{#generate-realistic-email} +### Transform Phone Number{#transform-phone-number} -TransformersTabletestest +### Transform String{#transform-string} -## Transform Email{#transform-email} +### Passthrough{#passthrough} -TransformersTabletestest +### Null{#null} diff --git a/docs/src/CustomComponents/TransformerPageHeader.tsx b/docs/src/CustomComponents/TransformerPageHeader.tsx index ecc8aba771..c92aa81817 100644 --- a/docs/src/CustomComponents/TransformerPageHeader.tsx +++ b/docs/src/CustomComponents/TransformerPageHeader.tsx @@ -12,7 +12,6 @@ export const TransformerPageHeader = (props: Props) => { const { title, type, apiRef } = props; return (
-
{title}
{type} {rowData.map((tableData) => ( - +
@@ -54,7 +56,19 @@ export function TransformersTable(props: TableData): ReactElement { key={tableData.type} className="font-medium bg-[#FFFFFF]" > - {tableData.type} + {tableData.type} + + + +
+
+ +
+
+ Date: Mon, 4 Dec 2023 22:02:28 -0800 Subject: [PATCH 04/13] updated all code links --- docs/docs/transformers/system/reference.mdx | 102 +++++++++++++------- 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx index 96cac6a433..d6b9706dcd 100644 --- a/docs/docs/transformers/system/reference.mdx +++ b/docs/docs/transformers/system/reference.mdx @@ -53,14 +53,16 @@ Neosync comes out of the box with 40+ system transformers in order to give you a type: 'boolean', description: 'Generates a boolean value at random.', pagelink: 'generate-boolean', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_bool.go', }, { title: 'Generate Card Number', type: 'int64', description: 'Generates a card number.', pagelink: 'generate-card-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_card_number.go', }, { title: 'Generate City', @@ -68,28 +70,32 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly selects a city from a list of predfined US cities.', pagelink: 'generate-city', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_city.go', }, { title: 'Generate E164 Phone Number', type: 'string', description: 'Generates a Generate phone number in e164 format.', pagelink: 'generate-e164-phone-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_e164_phone.go', }, { title: 'Generate First Name', type: 'string', description: 'Generates a random first name.', pagelink: 'generate-first-name', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_first_name.go', }, { title: 'Generate Float64', type: 'float64', description: 'Generates a random float64 value.', pagelink: 'generate-float64', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_float.go', }, { title: 'Generate Full Address', @@ -97,7 +103,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly generates a street address in the format: {street_num} {street_addresss} {street_descriptor} {city}, {state} {zipcode}. For example, 123 Main Street Boston, Massachusetts 02169.', pagelink: 'generate-full-address', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_full_address.go', }, { title: 'Generate Full Name', @@ -105,7 +112,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Generates a new full name consisting of a first and last name', pagelink: 'generate-full-name', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_full_name.go', }, { title: 'Generate Gender', @@ -113,7 +121,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly generates one of the following genders: female, male, undefined, nonbinary.', pagelink: 'generate-gender', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_gender.go', }, { title: 'Generate int64 Phone Number', @@ -121,7 +130,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Generates a new phone number of type int64 with a default length of 10.', pagelink: 'generate-int64-phone-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_164_phone.go', }, { title: 'Generate Random Int64', @@ -129,21 +139,24 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Generates a random integer value with a default length of 4 unless the Integer Length or Preserve Length parameters are defined.', pagelink: 'generate-random-int64', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_int.go', }, { title: 'Generate Last Name', type: 'int64', description: 'Generates a random last name.', pagelink: 'generate-last-name', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_last_name.go', }, { title: 'Generate SHA256 Hash', type: 'string', description: 'SHA256 hashes a randomly generated value.', pagelink: 'generate-sha256-hash', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_sha256hash.go', }, { title: 'Generate SSN', @@ -151,7 +164,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Generates a completely random social security numbers including the hyphens in the format ', pagelink: 'generate-ssn', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_ssn.go', }, { title: 'Generate State', @@ -159,7 +173,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly selects a US state and returns the two-character state code.', pagelink: 'generate-state', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_state.go', }, { title: 'Generate Street Address', @@ -167,7 +182,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly generates a street address in the format: {street_num} {street_addresss} {street_descriptor}. For example, 123 Main Street.', pagelink: 'generate-street-address', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_street_address.go', }, { title: 'Generate String Phone Number', @@ -175,7 +191,8 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Generates a Generate phone number and returns it as a string.', pagelink: 'generate-string-phone-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_string_phone.go', }, { title: 'Generate Random String', @@ -183,14 +200,16 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Creates a randomly ordered alphanumeric string with a default length of 10 unless the String Length parameter are defined.', pagelink: 'generate-random-string', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_string.go', }, { title: 'Generate Unix Timestamp', type: 'int64', description: 'Randomly generates a Unix timestamp', pagelink: 'generate-unix-timestamp', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_unixtimestamp.go', }, { title: 'Generate Username', @@ -198,21 +217,24 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly generates a username in the format.', pagelink: 'generate-username', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_username.go', }, { title: 'Generate UTC Timestamp', type: 'time', description: 'Randomly generates a UTC timestamp.', pagelink: 'generate-utc-timestamp', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_utctimestamp.go', }, { title: 'Generate UUID', type: 'uuid', description: 'Generates a new UUIDv4 id.', pagelink: 'generate-uuid', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_uuid.go', }, { title: 'Generate Zipcode', @@ -220,35 +242,40 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Randomly selects a zip code from a list of predefined US zipcodes.', pagelink: 'generate-zipcode', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_zipcode.go', }, { title: 'Transform E164 Phone Number', type: 'string', description: 'Transforms an existing E164 formatted phone number.', pagelink: 'transform-e164-phone-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_e164_phone.go', }, { title: 'Transform First Name', type: 'string', description: 'Transforms an existing first name', pagelink: 'transform-first-name', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_first_name.go', }, { title: 'Transform Float64', type: 'float64', description: 'Transforms an existing float value.', pagelink: 'transform-float64', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_float.go', }, { title: 'Transform Full Name', type: 'string', description: 'Transforms an existing full name.', pagelink: 'transform-full-name', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_full_name.go', }, { title: 'Transform Int64 Phone Number', @@ -256,21 +283,24 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Transforms an existing phone number that is typed as an integer', pagelink: 'transform-int64-phone-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_int_phone.go', }, { title: 'Transform Int64', type: 'int64', description: 'Transforms an existing integer value.', pagelink: 'transform-int64', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_int.go', }, { title: 'Transform Last Name', type: 'string', description: 'Transforms an existing last name.', pagelink: 'transform-last-name', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_last_name.go', }, { title: 'Transform Phone Number', @@ -278,14 +308,16 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Transforms an existing phone number that is typed as a string.', pagelink: 'transform-phone-number', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_phone.go', }, { title: 'Transform String', type: 'string', description: 'Transforms an existing string value.', pagelink: 'transform-string', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_string.go', }, { title: 'Passthrough', @@ -293,14 +325,16 @@ Neosync comes out of the box with 40+ system transformers in order to give you a description: 'Passes the input value through to the desination with no changes.', pagelink: 'passthrough', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/4e459151080109ffa0c5b0d9937f4f20fecfa6c6/worker/pkg/workflows/datasync/activities/activities.go', }, { title: 'Null', type: 'string', description: 'Inserts a string instead of the source value.', pagelink: 'null', - codeRef: '', + codeRef: + 'https://github.com/nucleuscloud/neosync/blob/4e459151080109ffa0c5b0d9937f4f20fecfa6c6/worker/pkg/workflows/datasync/activities/activities.go', }, ]} /> From 3879d50a082d574ab275973b674f18ab093a6af9 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 4 Dec 2023 23:20:16 -0800 Subject: [PATCH 05/13] updated like 20 of the transformers --- docs/docs/transformers/system/card-number.mdx | 63 - docs/docs/transformers/system/city.mdx | 42 - docs/docs/transformers/system/email.mdx | 93 -- docs/docs/transformers/system/first-name.mdx | 66 - .../docs/transformers/system/full-address.mdx | 43 - docs/docs/transformers/system/full-name.mdx | 66 - docs/docs/transformers/system/gender.mdx | 84 - docs/docs/transformers/system/int-phone.mdx | 73 - docs/docs/transformers/system/last-name.mdx | 66 - docs/docs/transformers/system/null.mdx | 37 - docs/docs/transformers/system/random-bool.mdx | 40 - .../docs/transformers/system/random-float.mdx | 93 -- docs/docs/transformers/system/reference.mdx | 1349 ++++++++++++++++- docs/docs/transformers/system/sha256hash.mdx | 52 - docs/docs/transformers/system/ssn.mdx | 43 - docs/docs/transformers/system/state.mdx | 42 - .../transformers/system/street-address.mdx | 43 - .../docs/transformers/system/string-phone.mdx | 106 -- docs/docs/transformers/system/unix-ts.mdx | 63 - docs/docs/transformers/system/utc-ts.mdx | 71 - docs/docs/transformers/system/uuid.mdx | 68 - docs/docs/transformers/system/zipcode.mdx | 42 - 22 files changed, 1310 insertions(+), 1335 deletions(-) delete mode 100644 docs/docs/transformers/system/card-number.mdx delete mode 100644 docs/docs/transformers/system/city.mdx delete mode 100644 docs/docs/transformers/system/email.mdx delete mode 100644 docs/docs/transformers/system/first-name.mdx delete mode 100644 docs/docs/transformers/system/full-address.mdx delete mode 100644 docs/docs/transformers/system/full-name.mdx delete mode 100644 docs/docs/transformers/system/gender.mdx delete mode 100644 docs/docs/transformers/system/int-phone.mdx delete mode 100644 docs/docs/transformers/system/last-name.mdx delete mode 100644 docs/docs/transformers/system/null.mdx delete mode 100644 docs/docs/transformers/system/random-bool.mdx delete mode 100644 docs/docs/transformers/system/random-float.mdx delete mode 100644 docs/docs/transformers/system/sha256hash.mdx delete mode 100644 docs/docs/transformers/system/ssn.mdx delete mode 100644 docs/docs/transformers/system/state.mdx delete mode 100644 docs/docs/transformers/system/street-address.mdx delete mode 100644 docs/docs/transformers/system/string-phone.mdx delete mode 100644 docs/docs/transformers/system/unix-ts.mdx delete mode 100644 docs/docs/transformers/system/utc-ts.mdx delete mode 100644 docs/docs/transformers/system/uuid.mdx delete mode 100644 docs/docs/transformers/system/zipcode.mdx diff --git a/docs/docs/transformers/system/card-number.mdx b/docs/docs/transformers/system/card-number.mdx deleted file mode 100644 index 43d6c9dd6e..0000000000 --- a/docs/docs/transformers/system/card-number.mdx +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Card Number -id: card-number -hide_title: true -slug: /transformers/system/card-number ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The card number transformer can anonymize an existing email address or completely generate a new one. A card number can be quite complex but we make it easy to generate valid luhn-check card numbers. - -By default, the card number transformer generates a random 16 digit card number that is _not_ luhn valid. If you want luhn validation, please set the luhn-check config to `true`. - -## Configurations - -Depending on your validations, you may want to configure the output card number. The card number transformer has the following configurations: - - - -## Examples - -Here are some examples of an output card number: - - diff --git a/docs/docs/transformers/system/city.mdx b/docs/docs/transformers/system/city.mdx deleted file mode 100644 index eb59addb93..0000000000 --- a/docs/docs/transformers/system/city.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: City -id: city -hide_title: true -slug: /transformers/system/city ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The city transformer generates a randomly selected US city. You can see the complete list of cities that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). - -If you'd rather not get back a real city value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value. - -## Configurations - -There are no configurations for the city transformer. - -## Examples - -Here are some examples of what an output City value may look like. - - diff --git a/docs/docs/transformers/system/email.mdx b/docs/docs/transformers/system/email.mdx deleted file mode 100644 index 6159324876..0000000000 --- a/docs/docs/transformers/system/email.mdx +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: Email -id: email -hide_title: true -slug: /transformers/system/email ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The email transformer can anonymize an existing email address or completely generate a new one. An email is made up of three parts: - -`@.` - -By default, the transformer randomizes the username, domain and top-level domain while always preserving the email format by retaining the @ and . characters. - -For example, the following input value: -`john@acme.com` - -Would produce the following ouput value: -`ytvub873@ksh0293.com` - -You can see we generated new privacy-safe values for the username and the domain. You can update this structure using the configurations below. Lastly, the email transformer takes in a `string` and returns a `string`. - -## Configurations - -Depending on your validations, you may want to configure the output email. The email transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different potential email formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/first-name.mdx b/docs/docs/transformers/system/first-name.mdx deleted file mode 100644 index 6908eab3e2..0000000000 --- a/docs/docs/transformers/system/first-name.mdx +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: First Name -id: first-name -hide_title: true -slug: /transformers/system/first-name ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json). - -By default, the first name transformer generates a first name of random length. To preserve the length of the input first name, you can set the `preserveLength` config. - -## Configurations - -Depending on your validations, you may want to configure the output first name. The first name transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different first name formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/full-address.mdx b/docs/docs/transformers/system/full-address.mdx deleted file mode 100644 index eaa2e25885..0000000000 --- a/docs/docs/transformers/system/full-address.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: Full Address -id: full-address -hide_title: true -slug: /transformers/system/full-address ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The full address transformer generates a randomly selects a real full address that exists in the United States. You can see the complete list of full addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). - -The full address transformer returns a valid United States address. For example: -`123 Main Street Boston, Massachusetts, 02169` - -## Configurations - -There are no configurations for the full adddress transformer. - -## Examples - -Here are some examples of what an output full address value may look like. - - diff --git a/docs/docs/transformers/system/full-name.mdx b/docs/docs/transformers/system/full-name.mdx deleted file mode 100644 index 38ac2b9e91..0000000000 --- a/docs/docs/transformers/system/full-name.mdx +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: Full Name -id: full-name -hide_title: true -slug: /transformers/system/full-name ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. - -By default, the full name transformer generates a full name of random length. To preserve the length of the input full name, you can set the `preserveLength` config. - -## Configurations - -Depending on your validations, you may want to configure the output full name. The full name transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/gender.mdx b/docs/docs/transformers/system/gender.mdx deleted file mode 100644 index 6df48089b6..0000000000 --- a/docs/docs/transformers/system/gender.mdx +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Gender -id: gender -hide_title: true -slug: /transformers/system/gender ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The gender transformer randomly selects a gender value from a predefined list of genders. Here is the list: - - - -By default, the gender transformer does not abbreviate the gender. If you'd like to return an abbreviated gender, pass in the `abbreviate` config. - -## Configurations - -Depending on your validations, you may want to configure the output gender. The gender transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/int-phone.mdx b/docs/docs/transformers/system/int-phone.mdx deleted file mode 100644 index 7b1ef2f9c6..0000000000 --- a/docs/docs/transformers/system/int-phone.mdx +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: Phone (integer) -id: int-phone -hide_title: true -slug: /transformers/system/int-phone ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes an integer value and returns an integer value. The [Phone (string) transformer](/transformers/system/string-phone) takes in a string value, returns a string and has more configurations available. - -By default, the phone (integer) transformer generates a random 10 digit phone number. - -For example, the following input value: -`7829828714` - -Would produce the following ouput value: -`5698437232` - -You can see we generated a new phone integer value that can be used as an integer phone number. Also, note that we don't include hyphens in this transformer since the output type is an `integer`. - -Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. - -## Configurations - -Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (integer) transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/last-name.mdx b/docs/docs/transformers/system/last-name.mdx deleted file mode 100644 index dff6c417b7..0000000000 --- a/docs/docs/transformers/system/last-name.mdx +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: Last Name -id: last-name -hide_title: true -slug: /transformers/system/last-name ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json). - -By default, the last name transformer generates a last name of random length. To preserve the length of the input last name, you can set the `preserveLength` config. - -## Configurations - -Depending on your validations, you may want to configure the output last name. The last name transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different last name formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/null.mdx b/docs/docs/transformers/system/null.mdx deleted file mode 100644 index fb5cf8dc74..0000000000 --- a/docs/docs/transformers/system/null.mdx +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Null Value -id: null-t -hide_title: true -slug: /transformers/system/null-t ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The null transformer simply returns a null value. This may be useful if you a column that can't be null but don't have a specific value that you want to insert. - -## Configurations - -There are no configurations for the null transformer. - -## Examples - -Here are some examples of what an output null value may look like. - - diff --git a/docs/docs/transformers/system/random-bool.mdx b/docs/docs/transformers/system/random-bool.mdx deleted file mode 100644 index c027452fa2..0000000000 --- a/docs/docs/transformers/system/random-bool.mdx +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: Random Boolean -id: random-bool -hide_title: true -slug: /transformers/system/random-bool ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The random boolean transformer randomly generates a boolean value. - -## Configurations - -There are no configurations for the random boolean transformer. - -## Examples - -Here are some examples of what an output random boolean value may look like. - - diff --git a/docs/docs/transformers/system/random-float.mdx b/docs/docs/transformers/system/random-float.mdx deleted file mode 100644 index 137072c354..0000000000 --- a/docs/docs/transformers/system/random-float.mdx +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: Random Float -id: random-float -hide_title: true -slug: /transformers/system/random-float ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The random float transformer generates a random floating point number. - -For example: -`32.2432` - -By default, the random float transformer generates a floating point number with 2 digits before the decimal point and 3 digits after the decimal point. - -## Configurations - -Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx index d6b9706dcd..9b193b8e34 100644 --- a/docs/docs/transformers/system/reference.mdx +++ b/docs/docs/transformers/system/reference.mdx @@ -360,19 +360,19 @@ There are no configurations for this transformer. **Examples** @@ -392,19 +392,19 @@ There are no configurations for this transformer. **Examples** @@ -457,7 +457,7 @@ Depending on your logic, you may want to configure the output email. The transfo ]} /> -## Examples +**Examples** There are several ways you can mix-and-match configurations to get different potential email formats. Here are some possible combinations: @@ -486,68 +486,1339 @@ There are several ways you can mix-and-match configurations to get different pot ### Generate Boolean{#generate-boolean} +The genereate boolean transformer randomly generates a boolean value. + +**Configurations** + +There are no configurations for the random boolean transformer. + +**Examples** + +Here are some examples of what an output random boolean value may look like. + + + ### Generate Card Number{#generate-card-number} +The generate card number transformer generates a new card number that is [luhn valid.](https://en.wikipedia.org/wiki/Luhn_algorithm) + +By default, the card number transformer generates a random 16 digit card number that is _not_ luhn valid. If you want luhn validation, please set the luhn-check config to `true`. + +**Configurations** + +Depending on your validations, you may want to configure the output card number. The card number transformer has the following configurations: + + + +**Examples** + +Here are some examples of an output card number: + + + ### Generate City{#generate-city} +The generate city transformers generates a randomly selected US city. You can see the complete list of cities that are available to be randomly selected [here](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). + +If you'd rather not get back a real city value, you can use the [Generate Random String Transformer](/transformers/system/reference#generate-random-string) to generate a random string value. + +**Configurations** + +There are no configurations for the city transformer. + +**Examples** + +Here are some examples of what an output City value may look like. + + + ### Generate E164 Phone Number{#generate-e164-phone-number} +The generate e164 phone transformer generates a new international phone number including the + sign. By default, the generate e164 transformer generates a random 12 digit number with no hyphens in the format: + +`+34567890123` + +TPhone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to be a certain length by passing in the `length` configuration. Here is more information on the [e164 format](https://www.twilio.com/docs/glossary/what-e164). This transformer also has a min of 9 and a max of 15. If you want to generate a number that is longer than that, you can use the [Generate Random int64 transformer](/transformers/system/reference#generate-random-int64) + +**Configurations** + +Depending on your validations, you may want to configure the output e164 phone number: + + + +**Examples** + +There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: + + + ### Generate First Name{#generate-first-name} +The generate first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json) + +By default, the generate first name transformer randomly picks a first name with a length between 2 and 12. + +**Configurations** + +There are no configurations for this transformer. + +**Examples** + +There are several ways you can mix-and-match configurations to get different first name formats. Here are some possible combinations: + + + ### Generate Float64{#generate-float64} -### Generate Full Address{#generate-full-address} +The generate float transformer generates a random floating point number. -### Generate Full Name{#generate-full-name} +For example: +`32.2432` -### Generate Gender{#generate-gender} +By default, the random float transformer generates a floating point number with 3 digits before the decimal point and 3 digits after the decimal point. -### Generate int64 Phone Number{#generate-int64-phone-number} +**Configurations** -### Generate Random Int64{#generate-random-int64} +Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations: -### Generate Last Name{#generate-last-name} + -### Generate SHA256 Hash{#generate-sha256-hash} +**Examples** -### Generate SSN{#generate-ssn} +There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations: -### Generate State{#generate-state} + -### Generate Street Address{#generate-street-address} +### Generate Full Address{#generate-full-address} -### Generate String Phone Number{#generate-string-phone-number} +The generate full address transformer generates a randomly selected real full address that exists in the United States. You can see the complete list of full addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json) -### Generate Random String{#generate-random-string} +The full address transformer returns a valid United States address. For example: +`123 Main Street Boston, Massachusetts, 02169` -### Generate Unix Timestamp{#generate-unix-timestamp} +**Configurations** -### Generate Username{#generate-username} +There are no configurations for the full adddress transformer. -### Generate UTC Timestamp{#generate-utc-timestamp} +**Examples** -### Generate UUID{#generate-uuid} +Here are some examples of what an output full address value may look like. -### Generate Zipcode{#generate-zipcode} + -### Transform E164 Phone Number{#transform-e164-phone-number} +### Generate Full Name{#generate-full-name} -### Transform First Name{#transform-first-name} +The generate full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. -### Transform Float64{#transform-float64} +**Configurations** -### Transform Full Name{#transform-full-name} +There are no configurations for this transformer. -### Transform Int64 Phone Number{#transform-int64-phone-number} +**Examples** -### Transform Int64{#transform-int64} +Here are some examples of what an output full name value may look like. -### Transform Last Name{#transform-last-name} + -### Transform Phone Number{#transform-phone-number} +### Generate Gender{#generate-gender} -### Transform String{#transform-string} +The gender transformer randomly selects a gender value from a predefined list of genders. Here is the list: -### Passthrough{#passthrough} + -### Null{#null} +By default, the gender transformer does not abbreviate the gender. If you'd like to return an abbreviated gender, pass in the `abbreviate` config. + +**Configurations** + +Depending on your validations, you may want to configure the output gender. The gender transformer has the following configurations: + + + +**Examples** + +There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations: + + + +### Generate int64 Phone Number{#generate-int64-phone-number} + +The generate int64 phone number generates a random 10 digit phone number and returns it as an `int64` type with no hyphens. If you want to return a `string` or want to include hyphens, check out the [Generate String Phone transformer](/transformers/system/reference#generate-string-phone-number). + +For example, the generate int64 phone number transformer could generate the following ouput value: +`5698437232` + +**Configurations** + +There are no configurations for this transformer. + +**Examples** + +Here are some examples of what the output values could look like: + + + +### Generate Random Int64{#generate-random-int64} + +The generate random int64 transformer generates a random integer and returns it as a int64 type. + +For example: +`6782` + +By default, the generate random int64 transformer generates an integer of 4 digits long. + +**Configurations** + +Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations: + + + +**Examples** + +There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: + + + +### Generate Last Name{#generate-last-name} + +The generate last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json). + +**Configurations** + +This transformer has no configurations. + +**Examples** + +Here are some possible output examples: + + + +### Generate SHA256 Hash{#generate-sha256-hash} + +The generate sha256 hash transformer generates a random SHA256 hash and hex encodes the resulting value and returns it as a string. + +**Configurations** + +There are no configurations for the hash transformer. + +**Examples** + +Here are some examples of what an output hash value may look like. + + + +### Generate SSN{#generate-ssn} + +The generate ssn transformer randomly generates a social security number and returns it with hyphens as a string. + +For, example: +`123-45-6789` + +**Configurations** + +There are no configurations for the ssn transformer. + +**Examples** + +Here are some examples of what an output street address value may look like. + + + +### Generate State{#generate-state} + +The generate state transformer generates a randomly selected US state. You can see the complete list of states that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). + +If you'd rather not get back a real state value, you can use the [Random String Transformer](/transformers/system/reference#generate-random-string) to generate a random string value. + +**Configurations** + +There are no configurations for the state transformer. + +**Examples** + +Here are some examples of what an output state value may look like. + + + +### Generate Street Address{#generate-street-address} + +The generate street address transformer generates a randomly selects a real street address that exists in the United States. You can see the complete list of street addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). + +The street address transformer returns a valid United States address. For example: +`123 Main Street Boston` + +**Configurations** + +There are no configurations for the street adddress transformer. + +**Examples** + +Here are some examples of what an output street address value may look like. + + + +### Generate String Phone Number{#generate-string-phone-number} + +### Generate Random String{#generate-random-string} + +### Generate Unix Timestamp{#generate-unix-timestamp} + +The generate unix timestamp transformer randomly generates a unix timestamp in UTC timezone and returns back an int64 representation of that timestamp. + +By default, the generated timestamp will always be in the **past**. + +**Configurations** + +There are no configurations for this transformer. + +**Examples** + +Here are some examples of what an output state value may look like. + + + +### Generate Username{#generate-username} + +The generate username transformer generates a random string in the format of ``. The last names are pulled from the last name transformer(/transformers/system/reference#generate-last-name). + +**Configurations** + +There are no configurations for the state transformer. + +**Examples** + +Here are some examples of what an output state value may look like. + + + +### Generate UTC Timestamp{#generate-utc-timestamp} + +The utc timestamp transformer randomly generates a utc timestamp in UTC timezone and returns back an time.Time representation of that timestamp. + +By default, the generated timestamp will always be in the **past**. + +**Configurations** + +There are no configurations for this transformer. + +**Examples** + +Here are some examples of what an output state value may look like. + + + +### Generate UUID{#generate-uuid} + +The UUID transformer generates a new UUID v4. + +For example: +`6d871028b072442c9ad9e6e4e223adfa` + +**Configurations** + +Depending on your validations, you may want to configure the output uuid. The uuid transformer has the following configurations: + + + +**Examples** + +Here are some example UUID values that the uuid transformer can generate: + + + +### Generate Zipcode{#generate-zipcode} + +The generate zipcode transformer generates a randomly selected US zipcode. You can see the complete list of zipcodes that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). + +If you'd rather not get back a real zipcode value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value. + +**Configurations** + +There are no configurations for the zipcode transformer. + +**Examples** + +Here are some examples of what an output zipcode value may look like. + + + +### Transform E164 Phone Number{#transform-e164-phone-number} + +The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes a string value and returns a string value. The [Phone (integer) transformer](/transformers/system/string-phone) takes in an integer value and returns an integer value. + +By default, the phone (string) transformer generates a random 10 digit phone number with no hyphens. + +For example, the following input value: +`7829828714` + +Would produce the following ouput value: +`5698437232` + +You can see we generated a new phone string value that can be used as an string phone number. + +Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. In fact, you can generate a specific international "looking" phone number by specifying the [e164 format](https://www.twilio.com/docs/glossary/what-e164) config. + +## Configurations + +Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (string) transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: + + + +### Transform First Name{#transform-first-name} + +The generate first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json). + +By default, the first name transformer generates a first name of random length. To preserve the length of the input first name, you can set the `preserveLength` config. + +**Configurations** + +Depending on your validations, you may want to configure the output first name. The first name transformer has the following configurations: + + + +**Examples** + +There are several ways you can mix-and-match configurations to get different first name formats. Here are some possible combinations: + + + +### Transform Float64{#transform-float64} + +The random float transformer generates a random floating point number. + +For example: +`32.2432` + +By default, the random float transformer generates a floating point number with 2 digits before the decimal point and 3 digits after the decimal point. + +## Configurations + +Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations: + + + +### Transform Full Name{#transform-full-name} + +The full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. + +By default, the full name transformer generates a full name of random length. To preserve the length of the input full name, you can set the `preserveLength` config. + +**Configurations** + +Depending on your validations, you may want to configure the output full name. The full name transformer has the following configurations: + + + +**Examples** + +There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations: + + + +### Transform Int64 Phone Number{#transform-int64-phone-number} + +The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes an integer value and returns an integer value. The [Phone (string) transformer](/transformers/system/string-phone) takes in a string value, returns a string and has more configurations available. + +By default, the phone (integer) transformer generates a random 10 digit phone number. + +For example, the following input value: +`7829828714` + +Would produce the following ouput value: +`5698437232` + +You can see we generated a new phone integer value that can be used as an integer phone number. Also, note that we don't include hyphens in this transformer since the output type is an `integer`. + +Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. + +## Configurations + +Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (integer) transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: + + + +### Transform Int64{#transform-int64} + +The random integer transformer generates a random integer. + +For example: +`6782` + +By default, the random int transformer generates an integer of 4 digits long. + +## Configurations + +Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: + + + +### Transform Last Name{#transform-last-name} + +The last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json). + +By default, the last name transformer generates a last name of random length. To preserve the length of the input last name, you can set the `preserveLength` config. + +## Configurations + +Depending on your validations, you may want to configure the output last name. The last name transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different last name formats. Here are some possible combinations: + + + +### Transform Phone Number{#transform-phone-number} + +The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes a string value and returns a string value. The [Phone (integer) transformer](/transformers/system/string-phone) takes in an integer value and returns an integer value. + +By default, the phone (string) transformer generates a random 10 digit phone number with no hyphens. + +For example, the following input value: +`7829828714` + +Would produce the following ouput value: +`5698437232` + +You can see we generated a new phone string value that can be used as an string phone number. + +Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. In fact, you can generate a specific international "looking" phone number by specifying the [e164 format](https://www.twilio.com/docs/glossary/what-e164) config. + +## Configurations + +Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (string) transformer has the following configurations: + + + +## Examples + +There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: + + + +### Transform String{#transform-string} + +### Passthrough{#passthrough} + +### Null{#null} + +The null transformer simply returns a null value. This may be useful if you a column that can't be null but don't have a specific value that you want to insert. + +## Configurations + +There are no configurations for the null transformer. + +## Examples + +Here are some examples of what an output null value may look like. + + diff --git a/docs/docs/transformers/system/sha256hash.mdx b/docs/docs/transformers/system/sha256hash.mdx deleted file mode 100644 index d83e247bd3..0000000000 --- a/docs/docs/transformers/system/sha256hash.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Hash -id: hash -hide_title: true -slug: /transformers/system/hash ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The hash transformer creates a SHA256 hash and hex encodes the resulting value and returns it as a string. Before the value is hashed, it's first transformed into a byte array to account for flexible input types. - -## Configurations - -There are no configurations for the hash transformer. - -## Examples - -Here are some examples of what an output hash value may look like. - - diff --git a/docs/docs/transformers/system/ssn.mdx b/docs/docs/transformers/system/ssn.mdx deleted file mode 100644 index d5910a18a4..0000000000 --- a/docs/docs/transformers/system/ssn.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: SSN -id: ssn -hide_title: true -slug: /transformers/system/ssn ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The ssn transformer randomly generates a social security number and returns it with hyphens as a string. - -For, example: -`123-45-6789` - -## Configurations - -There are no configurations for the ssn transformer. - -## Examples - -Here are some examples of what an output street address value may look like. - - diff --git a/docs/docs/transformers/system/state.mdx b/docs/docs/transformers/system/state.mdx deleted file mode 100644 index fb29bfba8f..0000000000 --- a/docs/docs/transformers/system/state.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: State -id: state -hide_title: true -slug: /transformers/system/state ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The state transformer generates a randomly selected US state. You can see the complete list of states that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). - -If you'd rather not get back a real state value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value. - -## Configurations - -There are no configurations for the state transformer. - -## Examples - -Here are some examples of what an output state value may look like. - - diff --git a/docs/docs/transformers/system/street-address.mdx b/docs/docs/transformers/system/street-address.mdx deleted file mode 100644 index 835163bf03..0000000000 --- a/docs/docs/transformers/system/street-address.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: Street Address -id: street-address -hide_title: true -slug: /transformers/system/street-address ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The street address transformer generates a randomly selects a real street address that exists in the United States. You can see the complete list of street addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). - -The street address transformer returns a valid United States address. For example: -`123 Main Street Boston` - -## Configurations - -There are no configurations for the street adddress transformer. - -## Examples - -Here are some examples of what an output street address value may look like. - - diff --git a/docs/docs/transformers/system/string-phone.mdx b/docs/docs/transformers/system/string-phone.mdx deleted file mode 100644 index f6400f77a7..0000000000 --- a/docs/docs/transformers/system/string-phone.mdx +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Phone (string) -id: string-phone -hide_title: true -slug: /transformers/system/string-phone ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes a string value and returns a string value. The [Phone (integer) transformer](/transformers/system/string-phone) takes in an integer value and returns an integer value. - -By default, the phone (string) transformer generates a random 10 digit phone number with no hyphens. - -For example, the following input value: -`7829828714` - -Would produce the following ouput value: -`5698437232` - -You can see we generated a new phone string value that can be used as an string phone number. - -Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. In fact, you can generate a specific international "looking" phone number by specifying the [e164 format](https://www.twilio.com/docs/glossary/what-e164) config. - -## Configurations - -Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (string) transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/unix-ts.mdx b/docs/docs/transformers/system/unix-ts.mdx deleted file mode 100644 index 8cd2dd4652..0000000000 --- a/docs/docs/transformers/system/unix-ts.mdx +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Unix Timestamp -id: unix-ts -hide_title: true -slug: /transformers/system/unix-ts ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The unix timestamp transformer randomly generates a unix timestamp in UTC timezone and returns back an int64 representation of that timestamp. - -By default, the generated timestamp will always be in the **past**. To generate a timestamp that is in the future, set the `Future` config to true. - -## Configurations - -Depending on your validations, you may want to configure the output unix timestamp. The unix timestamp transformer has the following configuration options: - - - -## Examples - -Here are some examples of what an output state value may look like. - - diff --git a/docs/docs/transformers/system/utc-ts.mdx b/docs/docs/transformers/system/utc-ts.mdx deleted file mode 100644 index 34d2ae368f..0000000000 --- a/docs/docs/transformers/system/utc-ts.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: UTC Timestamp -id: utc-ts -hide_title: true -slug: /transformers/system/utc-ts ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The utc timestamp transformer randomly generates a utc timestamp in UTC timezone and returns back an tiem.Time representation of that timestamp. - -By default, the generated timestamp will always be in the **past**. To generate a timestamp that is in the future, set the `future` config to true. - -## Configurations - -Depending on your validations, you may want to configure the output utc timestamp. The utc timestamp transformer has the following configuration options: - - - -## Examples - -Here are some examples of what an output state value may look like. - - diff --git a/docs/docs/transformers/system/uuid.mdx b/docs/docs/transformers/system/uuid.mdx deleted file mode 100644 index d3492b1572..0000000000 --- a/docs/docs/transformers/system/uuid.mdx +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: UUID -id: uuid -hide_title: true -slug: /transformers/system/uuid ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The UUID transformer generates a new UUID v4. - -For example: -`6d871028b072442c9ad9e6e4e223adfa` - -## Configurations - -Depending on your validations, you may want to configure the output uuid. The uuid transformer has the following configurations: - - - -## Examples - -Here are some example UUID values that the uuid transformer can generate: - - diff --git a/docs/docs/transformers/system/zipcode.mdx b/docs/docs/transformers/system/zipcode.mdx deleted file mode 100644 index 574f95b474..0000000000 --- a/docs/docs/transformers/system/zipcode.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Zipcode -id: zipcode -hide_title: true -slug: /transformers/system/zipcode ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The zipcode transformer generates a randomly selected US zipcode. You can see the complete list of zipcodes that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). - -If you'd rather not get back a real zipcode value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value. - -## Configurations - -There are no configurations for the zipcode transformer. - -## Examples - -Here are some examples of what an output zipcode value may look like. - - From be2048af106b733813ed15b9282470d4d3b7a6dc Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 11:18:25 -0800 Subject: [PATCH 06/13] finished up tranformer docs --- .../transformers/system/random-string.mdx | 83 ---- docs/docs/transformers/system/reference.mdx | 372 +++++++++++------- 2 files changed, 226 insertions(+), 229 deletions(-) delete mode 100644 docs/docs/transformers/system/random-string.mdx diff --git a/docs/docs/transformers/system/random-string.mdx b/docs/docs/transformers/system/random-string.mdx deleted file mode 100644 index daa9d4fb8f..0000000000 --- a/docs/docs/transformers/system/random-string.mdx +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Random String -id: random-string -hide_title: true -slug: /transformers/system/random-string ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The random string transformer generates a random string of alphanumeric characters. - -For example: -`h2983hf28h92` - -By default, the random string transformer generates a string of 10 characters long. - -## Configurations - -Depending on your validations, you may want to configure the output string. The random string transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: - - diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx index 9b193b8e34..09178aa1a6 100644 --- a/docs/docs/transformers/system/reference.mdx +++ b/docs/docs/transformers/system/reference.mdx @@ -125,7 +125,7 @@ Neosync comes out of the box with 40+ system transformers in order to give you a 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_gender.go', }, { - title: 'Generate int64 Phone Number', + title: 'Generate Int64 Phone Number', type: 'int64', description: 'Generates a new phone number of type int64 with a default length of 10.', @@ -804,7 +804,7 @@ Here are some examples of what an output full name value may look like. ### Generate Gender{#generate-gender} -The gender transformer randomly selects a gender value from a predefined list of genders. Here is the list: +The generate gender transformer randomly selects a gender value from a predefined list of genders. Here is the list: -### Generate int64 Phone Number{#generate-int64-phone-number} +### Generate Int64 Phone Number{#generate-int64-phone-number} The generate int64 phone number generates a random 10 digit phone number and returns it as an `int64` type with no hyphens. If you want to return a `string` or want to include hyphens, check out the [Generate String Phone transformer](/transformers/system/reference#generate-string-phone-number). @@ -908,7 +908,7 @@ By default, the generate random int64 transformer generates an integer of 4 digi **Configurations** -Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations: +Depending on your validations, you may want to configure the output float. The random integer transformer has the following configurations: + +**Examples** + +There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: + + + ### Generate Random String{#generate-random-string} +The generate random string transformer generates a random string of alphanumeric characters. + +For example: +`h2983hf28h92` + +By default, the random string transformer generates a string of 10 characters long. + +**Configurations** + +Depending on your validations, you may want to configure the output string. The generate string transformer has the following configurations: + + + +**Examples** + +There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: + + + ### Generate Unix Timestamp{#generate-unix-timestamp} The generate unix timestamp transformer randomly generates a unix timestamp in UTC timezone and returns back an int64 representation of that timestamp. @@ -1161,7 +1262,7 @@ Here are some examples of what an output state value may look like. ### Generate UTC Timestamp{#generate-utc-timestamp} -The utc timestamp transformer randomly generates a utc timestamp in UTC timezone and returns back an time.Time representation of that timestamp. +The generate utc timestamp transformer randomly generates a utc timestamp in UTC timezone and returns back an time.Time representation of that timestamp. By default, the generated timestamp will always be in the **past**. @@ -1195,7 +1296,7 @@ Here are some examples of what an output state value may look like. ### Generate UUID{#generate-uuid} -The UUID transformer generates a new UUID v4. +The generate UUID transformer generates a new UUID v4. For example: `6d871028b072442c9ad9e6e4e223adfa` @@ -1269,23 +1370,19 @@ Here are some examples of what an output zipcode value may look like. ### Transform E164 Phone Number{#transform-e164-phone-number} -The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes a string value and returns a string value. The [Phone (integer) transformer](/transformers/system/string-phone) takes in an integer value and returns an integer value. - -By default, the phone (string) transformer generates a random 10 digit phone number with no hyphens. +The transform e164 phone transformer can anonymize an existing e164 phone number or completely generate a new one. It returns a string value with the format `+`. For example, the following input value: -`7829828714` - -Would produce the following ouput value: -`5698437232` +`+7829828714` -You can see we generated a new phone string value that can be used as an string phone number. +Could produce the following ouput value: +`+5698437232` -Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. In fact, you can generate a specific international "looking" phone number by specifying the [e164 format](https://www.twilio.com/docs/glossary/what-e164) config. +E164 Phone numbers vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value in order maintain the same shape as the input value. -## Configurations +**Configurations** -Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (string) transformer has the following configurations: +Depending on your validations, you may want to configure the output e164 phone number. The transform e164 phone number transformer has the following configurations: -## Examples +**Examples** -There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: +Here are some examples of what the transform e164 phone number tranformer can output: ### Transform First Name{#transform-first-name} -The generate first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json). +The transform first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json). By default, the first name transformer generates a first name of random length. To preserve the length of the input first name, you can set the `preserveLength` config. @@ -1381,7 +1445,7 @@ Depending on your validations, you may want to configure the output first name. 'PreserveLength', 'Preserve Length will ensure that the output first name is the same length as the input first name. The preserveLength config only preserves names up to 12 characters long.', 'false', - 'John', + 'Johnathan', 'Bill', ], }, @@ -1390,7 +1454,7 @@ Depending on your validations, you may want to configure the output first name. **Examples** -There are several ways you can mix-and-match configurations to get different first name formats. Here are some possible combinations: +Here are some examples of what the transform first name tranformer can output: ### Transform Float64{#transform-float64} -The random float transformer generates a random floating point number. +The transform float transformer can anonymize an existing float64 value or generate a compeletely random floating point number. For example: `32.2432` -By default, the random float transformer generates a floating point number with 2 digits before the decimal point and 3 digits after the decimal point. +By default, the random float transformer generates a floating point number with 3 digits before the decimal point and 3 digits after the decimal point. -## Configurations +**Configurations** Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations: @@ -1440,53 +1501,43 @@ Depending on your validations, you may want to configure the output float. The r }, { data: [ - 'Digits before Decimal', - 'Specifies how many digits you want before the decimal place. This has a max of 9 digits. ', - '2', - '23.24', - '94.17', - ], - }, - { - data: [ - 'Digits after Decimal', - 'Specifies how many digits you want after the decimal place. This has a max of 9 digits. ', - '3', - '87923.4', - '34.8', + 'PreserveSign', + 'Preserves the sign of the value. If false, will always return a positive value. ', + 'true', + '-67.2', + '-71.523', ], }, ]} /> -## Examples +**Examples** There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations: ### Transform Full Name{#transform-full-name} -The full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. +The transform full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. By default, the full name transformer generates a full name of random length. To preserve the length of the input full name, you can set the `preserveLength` config. @@ -1536,9 +1587,7 @@ There are several ways you can mix-and-match configurations to get different ful ### Transform Int64 Phone Number{#transform-int64-phone-number} -The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes an integer value and returns an integer value. The [Phone (string) transformer](/transformers/system/string-phone) takes in a string value, returns a string and has more configurations available. - -By default, the phone (integer) transformer generates a random 10 digit phone number. +The transform int64 phone transformer can anonymize an existing phone number or completely generate a new one. By default, the transform int64 phone number transformer generates a random 10 digit phone number. For example, the following input value: `7829828714` @@ -1548,11 +1597,9 @@ Would produce the following ouput value: You can see we generated a new phone integer value that can be used as an integer phone number. Also, note that we don't include hyphens in this transformer since the output type is an `integer`. -Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. - -## Configurations +**#Configurations** -Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (integer) transformer has the following configurations: +Depending on your validations, you may want to configure the output phone number. -## Examples +**Examples** -There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: +Here are some possible example output values: @@ -1600,7 +1647,7 @@ For example: By default, the random int transformer generates an integer of 4 digits long. -## Configurations +**Configurations** Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations: @@ -1616,55 +1663,55 @@ Depending on your validations, you may want to configure the output integer. The { data: [ 'Preserve Length', - 'Preserves the length of the input integer to the output integer.', + 'Preserves the length of the input float to the output float while maintaining the placement of the decimal.', 'false', - '12334', - '78223', + '1.34', + '873.923', ], }, { data: [ - 'Integer Length', - 'Specifies how many digits you want the integer to be. This has a max of 18 digits. ', - '4', - '7', - '2782736', + 'PreserveSign', + 'Preserves the sign of the value. If false, will always return a positive value. ', + 'true', + '-672', + '-71523', ], }, ]} /> -## Examples +**Examples** -There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: +There are several ways you can mix-and-match configurations to get different potential random int64 values. Here are some possible combinations: ### Transform Last Name{#transform-last-name} -The last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json). +The transform last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json). By default, the last name transformer generates a last name of random length. To preserve the length of the input last name, you can set the `preserveLength` config. -## Configurations +**Configurations** Depending on your validations, you may want to configure the output last name. The last name transformer has the following configurations: @@ -1689,7 +1736,7 @@ Depending on your validations, you may want to configure the output last name. T ]} /> -## Examples +**Examples** There are several ways you can mix-and-match configurations to get different last name formats. Here are some possible combinations: @@ -1710,9 +1757,9 @@ There are several ways you can mix-and-match configurations to get different las ### Transform Phone Number{#transform-phone-number} -The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes a string value and returns a string value. The [Phone (integer) transformer](/transformers/system/string-phone) takes in an integer value and returns an integer value. +The transform phone transformer can anonymize an existing phone number or completely generate a new one. This transformer specifically takes a string value and returns a string value. -By default, the phone (string) transformer generates a random 10 digit phone number with no hyphens. +By default, the transform phone transformer generates a random 10 string value phone number with no hyphens. For example, the following input value: `7829828714` @@ -1720,13 +1767,9 @@ For example, the following input value: Would produce the following ouput value: `5698437232` -You can see we generated a new phone string value that can be used as an string phone number. - -Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. In fact, you can generate a specific international "looking" phone number by specifying the [e164 format](https://www.twilio.com/docs/glossary/what-e164) config. - -## Configurations +**Configurations** -Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (string) transformer has the following configurations: +Depending on your validations, you may want to configure the output phone number. Here are the configurations for the transform string phone number transformer. -## Examples +**Examples** There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations: @@ -1775,42 +1809,88 @@ There are several ways you can mix-and-match configurations to get different pot headers={[ 'PreserveLength', 'Include Hyphens', - 'e164 Format', 'Example Input', 'Example Output', ]} rowData={[ { - data: ['false', 'false', 'false', '2890923784', '5209273239'], + data: ['false', 'false', '2890923784', '5209273239'], }, { - data: ['false', 'true', 'false', '2890923784', '520-927-3239'], + data: ['false', 'true', '2890923784', '520-927-3239'], }, { - data: ['true', 'false', 'false', '289092378423', '520927323239'], + data: ['true', 'false', '289092378423', '520927323239'], }, { - data: ['false', 'false', 'true', '2890923784', '+520922339'], + data: ['true', 'true', '2890923784', '520-922-3539'], }, + ]} +/> + +### Transform String{#transform-string} + +The random string transformer generates a random string of alphanumeric characters. + +For example: +`h2983hf28h92` + +By default, the random string transformer generates a string of 10 characters long. + +**Configurations** + +Depending on your validations, you may want to configure the output string. The random string transformer has the following configurations: + + -### Transform String{#transform-string} +**Examples** + +There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: + + ### Passthrough{#passthrough} +The passthrough tranformer simplfy passes the input data out to the output without making any modications to it. This is useful in many circumstances but cautious of accidently leaking sensitive data through this transformer. + ### Null{#null} The null transformer simply returns a null value. This may be useful if you a column that can't be null but don't have a specific value that you want to insert. -## Configurations +**Configurations** There are no configurations for the null transformer. -## Examples +**Examples** Here are some examples of what an output null value may look like. From 6025b7147848f54089f625607150d0cefd7fceec Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 13:06:24 -0800 Subject: [PATCH 07/13] fixed table key warning --- docs/src/CustomComponents/DocsTable.tsx | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/src/CustomComponents/DocsTable.tsx b/docs/src/CustomComponents/DocsTable.tsx index d7ddb798ef..ffd892e86e 100644 --- a/docs/src/CustomComponents/DocsTable.tsx +++ b/docs/src/CustomComponents/DocsTable.tsx @@ -24,16 +24,19 @@ export function DocsTable(props: TableData): ReactElement { - {headers.map((header) => ( - {header} + {headers.map((header, headerIndex) => ( + {header} ))} - {rowData.map((tableData) => ( - - {tableData.data.map((item) => ( - + {rowData.map((tableData, rowIndex) => ( + + {tableData.data.map((item, columnIndex) => ( + {item} ))} From 6a9c99037705357f79b65fa2a9bea50d045923b3 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 13:09:59 -0800 Subject: [PATCH 08/13] fixed page link --- docs/src/CustomComponents/TransformersTable.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/CustomComponents/TransformersTable.tsx b/docs/src/CustomComponents/TransformersTable.tsx index c5f3039d89..0145e3ac17 100644 --- a/docs/src/CustomComponents/TransformersTable.tsx +++ b/docs/src/CustomComponents/TransformersTable.tsx @@ -20,7 +20,7 @@ interface RowData { title: string; type: string; description: string; - pageLink: string; + pagelink: string; codeRef: string; } @@ -41,7 +41,7 @@ export function TransformersTable(props: TableData): ReactElement {
From 088047d8d857590f9e2db6f2d57fee040474e839 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 13:44:46 -0800 Subject: [PATCH 09/13] updated caret for sidebar items --- docs/docs/transformers/system/random-int.mdx | 83 ------------------- docs/sidebars.ts | 2 +- docs/src/CustomComponents/IconHandler.tsx | 21 +++-- docs/src/css/custom.css | 4 +- .../theme/DocSidebarItem/Category/index.tsx | 20 +++-- 5 files changed, 34 insertions(+), 96 deletions(-) delete mode 100644 docs/docs/transformers/system/random-int.mdx diff --git a/docs/docs/transformers/system/random-int.mdx b/docs/docs/transformers/system/random-int.mdx deleted file mode 100644 index b2cdeaca75..0000000000 --- a/docs/docs/transformers/system/random-int.mdx +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Random Integer -id: random-int -hide_title: true -slug: /transformers/system/random-int ---- - -import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx'; - -import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; - - - -## Definition - -The random integer transformer generates a random integer. - -For example: -`6782` - -By default, the random int transformer generates an integer of 4 digits long. - -## Configurations - -Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations: - - - -## Examples - -There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations: - - diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 4028f78d12..a349b32b65 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -165,7 +165,7 @@ const sidebars: SidebarsConfig = { }, { type: 'category', - label: 'System', + label: 'System Transformers', collapsible: true, collapsed: true, items: [ diff --git a/docs/src/CustomComponents/IconHandler.tsx b/docs/src/CustomComponents/IconHandler.tsx index d1086cd9fb..109491ce46 100644 --- a/docs/src/CustomComponents/IconHandler.tsx +++ b/docs/src/CustomComponents/IconHandler.tsx @@ -11,7 +11,12 @@ import { AiOutlineMail, AiOutlinePhone, } from 'react-icons/ai'; -import { BiLogoPostgresql, BiSolidCity, BiTimeFive } from 'react-icons/bi'; +import { + BiLogoPostgresql, + BiSolidCity, + BiTerminal, + BiTimeFive, +} from 'react-icons/bi'; import { BsCalendarDate, BsFillKeyFill, @@ -27,7 +32,7 @@ import { FaStreetView, } from 'react-icons/fa'; import { GiTexas } from 'react-icons/gi'; -import { GoCode } from 'react-icons/go'; +import { GoCode, GoTable } from 'react-icons/go'; import { GrMysql, GrSecure } from 'react-icons/gr'; import { IoBuildOutline } from 'react-icons/io5'; import { MdPassword } from 'react-icons/md'; @@ -38,7 +43,7 @@ import { RxLetterCaseCapitalize, } from 'react-icons/rx'; import { SiKubernetes } from 'react-icons/si'; -import { TbDecimal } from 'react-icons/tb'; +import { TbDecimal, TbVariable } from 'react-icons/tb'; import { TiSortNumerically } from 'react-icons/ti'; export function IconHandler(name: string): ReactElement { @@ -67,9 +72,9 @@ export function IconHandler(name: string): ReactElement { return ; case 'SSN': return ; - case 'Custom': + case 'User Defined': return ; - case 'System': + case 'System Transformers': return ; case 'Use cases': return ; @@ -117,6 +122,12 @@ export function IconHandler(name: string): ReactElement { return ; case 'Hash': return ; + case 'Neosync CLI': + return ; + case 'Environment Variables': + return ; + case 'Reference': + return ; default: return ; } diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index 59c5706011..2f5413678d 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -141,8 +141,8 @@ this is the class for the sidebar categories that section headers @apply text-xs; } - .menu__link--sublist-caret { - @apply text-gray-900; + .menu__link--sublist { + @apply text-gray-900 font-[10px]; } .docsImage { diff --git a/docs/src/theme/DocSidebarItem/Category/index.tsx b/docs/src/theme/DocSidebarItem/Category/index.tsx index e3644a066a..00e9d8d783 100644 --- a/docs/src/theme/DocSidebarItem/Category/index.tsx +++ b/docs/src/theme/DocSidebarItem/Category/index.tsx @@ -21,6 +21,8 @@ import { useDocSidebarItemsExpandedState, } from '@docusaurus/theme-common/internal'; import useIsBrowser from '@docusaurus/useIsBrowser'; +import { ChevronDownIcon, ChevronRightIcon } from '@radix-ui/react-icons'; +import { IconHandler } from '@site/src/CustomComponents/IconHandler'; import type { Props } from '@theme/DocSidebarItem/Category'; import DocSidebarItems from '@theme/DocSidebarItems'; import clsx from 'clsx'; @@ -169,14 +171,19 @@ export default function DocSidebarItemCategory({ )} >
+ {IconHandler(item.label)} - {label} +
+ {label} + {collapsed ? : } +
{href && collapsible && ( Date: Tue, 5 Dec 2023 13:49:07 -0800 Subject: [PATCH 10/13] updated icons --- docs/src/CustomComponents/IconHandler.tsx | 95 ++++++----------------- 1 file changed, 23 insertions(+), 72 deletions(-) diff --git a/docs/src/CustomComponents/IconHandler.tsx b/docs/src/CustomComponents/IconHandler.tsx index 109491ce46..87a9cbdd1b 100644 --- a/docs/src/CustomComponents/IconHandler.tsx +++ b/docs/src/CustomComponents/IconHandler.tsx @@ -1,50 +1,25 @@ import { + AvatarIcon, HomeIcon, LayersIcon, LinkBreak1Icon, + ListBulletIcon, Share1Icon, + StackIcon, TokensIcon, } from '@radix-ui/react-icons'; import React, { ReactElement } from 'react'; -import { - AiOutlineCreditCard, - AiOutlineMail, - AiOutlinePhone, -} from 'react-icons/ai'; -import { - BiLogoPostgresql, - BiSolidCity, - BiTerminal, - BiTimeFive, -} from 'react-icons/bi'; -import { - BsCalendarDate, - BsFillKeyFill, - BsFunnel, - BsGenderAmbiguous, - BsPinMap, - BsShieldCheck, -} from 'react-icons/bs'; -import { - FaAws, - FaDocker, - FaRegAddressBook, - FaStreetView, -} from 'react-icons/fa'; -import { GiTexas } from 'react-icons/gi'; -import { GoCode, GoTable } from 'react-icons/go'; -import { GrMysql, GrSecure } from 'react-icons/gr'; +import { AiOutlineMail, AiOutlinePhone } from 'react-icons/ai'; +import { BiLogInCircle, BiLogoPostgresql, BiTerminal } from 'react-icons/bi'; +import { BsFunnel, BsShieldCheck } from 'react-icons/bs'; +import { FaAws, FaDocker } from 'react-icons/fa'; +import { GoCode, GoTable, GoVersions } from 'react-icons/go'; +import { GrMysql } from 'react-icons/gr'; import { IoBuildOutline } from 'react-icons/io5'; -import { MdPassword } from 'react-icons/md'; +import { MdPassword, MdStart } from 'react-icons/md'; import { PiArrowsSplitLight, PiFlaskLight } from 'react-icons/pi'; -import { - RxAvatar, - RxComponentBoolean, - RxLetterCaseCapitalize, -} from 'react-icons/rx'; import { SiKubernetes } from 'react-icons/si'; -import { TbDecimal, TbVariable } from 'react-icons/tb'; -import { TiSortNumerically } from 'react-icons/ti'; +import { TbVariable } from 'react-icons/tb'; export function IconHandler(name: string): ReactElement { switch (name) { @@ -86,48 +61,24 @@ export function IconHandler(name: string): ReactElement { return ; case 'Subset Data': return ; - case 'City': - return ; - case 'Card Number': - return ; - case 'First Name': - return ; - case 'Full Name': - return ; - case 'Last Name': - return ; - case 'Full Address': - return ; - case 'Gender': - return ; - case 'Random Boolean': - return ; - case 'Random Float': - return ; - case 'Random Integer': - return ; - case 'Random String': - return ; - case 'State': - return ; - case 'Street Address': - return ; - case 'Unix Timestamp': - return ; - case 'UTC Timestamp': - return ; - case 'UUID': - return ; - case 'Zipcode': - return ; - case 'Hash': - return ; case 'Neosync CLI': return ; case 'Environment Variables': return ; case 'Reference': return ; + case 'login': + return ; + case 'whoami': + return ; + case 'jobs': + return ; + case 'list': + return ; + case 'trigger': + return ; + case 'version': + return ; default: return ; } From 54452a782f857d7293ed60eb8e60b84c93f78167 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 14:22:32 -0800 Subject: [PATCH 11/13] updated docs --- docs/docs/transformers/user-defined.mdx | 38 ++++++++++------------- docs/sidebars.ts | 2 +- docs/src/CustomComponents/IconHandler.tsx | 2 +- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/docs/docs/transformers/user-defined.mdx b/docs/docs/transformers/user-defined.mdx index df8103755b..2ae0e8e9d3 100644 --- a/docs/docs/transformers/user-defined.mdx +++ b/docs/docs/transformers/user-defined.mdx @@ -1,5 +1,5 @@ --- -title: Custom +title: User Defined id: user-defined hide_title: true slug: /transformers/user-defined @@ -9,43 +9,39 @@ import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx'; import { DocsImage } from '@site/src/CustomComponents/DocsImage'; - + ## Introduction -Custom Transfomers are a great way to configure a system transformer with your own presets and publish it everywhere. Custom transformers are saved at the account level and +User defined Transfomers are a great way to configure a system transformer with your own presets and publish it everywhere. User defined transformers are saved at the account level and can be used across multiple jobs, saving your time during the schema configuration process. - + -## Creating a Custom Transformer +## Creating a User Defined Transformer -In order to create a custom transformer, follow these steps: +In order to create a user defined transformer, follow these steps: 1. Navigate to the **Transformers** page and click on **+ New Transformer**. -2. You'll be brought to the new transformer page where you can select a base transformer. A base transformer serves as the blueprint for the custom transformer. Select the base transformer for your custom transformer. +2. You'll be brought to the new transformer page where you can select a base transformer. A base transformer serves as the blueprint for the user defined transformer. Select the base transformer for your user defined transformer. - + -3. Once you've selected a base transformer, you'll be prompted to give the transformer a name and description. Additionally, you can preset custom default configurations depending on the transformer. +3. Once you've selected a base transformer, you'll be prompted to give the transformer a name and description. Additionally, you can preset custom default configurations depending on the transformer. Fill out the details and click **save**. - + -4. Once you've filled out the form, you can click on **save** and then you will be routed to the newly created transformer. +## Using a User Defined Transformer - +Once you've created a user defined transformer, you'll see it appear in the transformer list in Transformers main page as well as the Schema configuration page. Above, we created a user defined transformer +called `custom-float-transfomer`, we can now see it both places. -## Using a Custom Transformer +In the transformers table under the User Defined Transformer tab. -Once you've created a custom transformer, you'll see it appear in the transformer list in Transformers main page as well as the Schema configuration page. Above, we created a custom transformer -called `custom-string-transformer`, we can now see it both places. - -In the transformers table under the Custom Transformer tab. - - + In the Schema configuration page in the transformer select. - + -Now we can finsih the rest of our job configuration and the newly created custom transformer will be used in the column that it's mapped to. +Now we can finsih the rest of our job configuration and the newly created user defined transformer will be used in the column that it's mapped to. diff --git a/docs/sidebars.ts b/docs/sidebars.ts index a349b32b65..4028f78d12 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -165,7 +165,7 @@ const sidebars: SidebarsConfig = { }, { type: 'category', - label: 'System Transformers', + label: 'System', collapsible: true, collapsed: true, items: [ diff --git a/docs/src/CustomComponents/IconHandler.tsx b/docs/src/CustomComponents/IconHandler.tsx index 87a9cbdd1b..c33cb81979 100644 --- a/docs/src/CustomComponents/IconHandler.tsx +++ b/docs/src/CustomComponents/IconHandler.tsx @@ -49,7 +49,7 @@ export function IconHandler(name: string): ReactElement { return ; case 'User Defined': return ; - case 'System Transformers': + case 'System': return ; case 'Use cases': return ; From 05b14eab4b7ea1cac17829d35c7e77ba6c9b42a7 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 14:30:17 -0800 Subject: [PATCH 12/13] updated broken links --- docs/docs/overview/use-cases/anonymization.mdx | 2 +- docs/docs/transformers/introduction.mdx | 2 +- docs/docs/transformers/system/reference.mdx | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs/overview/use-cases/anonymization.mdx b/docs/docs/overview/use-cases/anonymization.mdx index 5cb6b5c05b..8338281b2a 100644 --- a/docs/docs/overview/use-cases/anonymization.mdx +++ b/docs/docs/overview/use-cases/anonymization.mdx @@ -27,7 +27,7 @@ We've mapped each transformer to the right column which means that every time th //screenshot of the output here -Since the transformers are customizable, you can anonymize or mask any source data by writing your own custom tranformer. Check out the [custom transformers](/transformers/custom) guide for more information. This gives you ultimate flexibility to control your data. +Since the transformers are customizable, you can anonymize or mask any source data by writing your own custom tranformer. Check out the [custom transformers](/transformers/user-defined) guide for more information. This gives you ultimate flexibility to control your data. Lastly, you can decide to subset this data to reduce the size of it to make it fit in lower level environments like your local database. diff --git a/docs/docs/transformers/introduction.mdx b/docs/docs/transformers/introduction.mdx index a2ae53a25a..3fd02d9249 100644 --- a/docs/docs/transformers/introduction.mdx +++ b/docs/docs/transformers/introduction.mdx @@ -42,7 +42,7 @@ You can browse our list of system transformers which come out of the box or you { title: 'System Transformers', description: 'Pre-built transformers that are ready to use', - link: '/transformers/system/email', + link: '/transformers/system/reference', icon: , }, { diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx index 09178aa1a6..acd51ae0c6 100644 --- a/docs/docs/transformers/system/reference.mdx +++ b/docs/docs/transformers/system/reference.mdx @@ -777,7 +777,7 @@ Here are some examples of what an output full address value may look like. ### Generate Full Name{#generate-full-name} -The generate full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. +The generate full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/reference#generate-first-name) and [last](/transformers/system/last-name) names transformers. **Configurations** @@ -1346,7 +1346,7 @@ Here are some example UUID values that the uuid transformer can generate: The generate zipcode transformer generates a randomly selected US zipcode. You can see the complete list of zipcodes that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json). -If you'd rather not get back a real zipcode value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value. +If you'd rather not get back a real zipcode value, you can use the [Random String Transformer](/transformers/system/reference#generate-random-string) to generate a random string value. **Configurations** @@ -1537,7 +1537,7 @@ There are several ways you can mix-and-match configurations to get different pot ### Transform Full Name{#transform-full-name} -The transform full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers. +The transform full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/reference#generate-first-name) and [last](/transformers/system/reference#generate-last-name) names transformers. By default, the full name transformer generates a full name of random length. To preserve the length of the input full name, you can set the `preserveLength` config. From 9ed59d0b2878f10104d9e16aeeec975f6e1012ae Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 5 Dec 2023 14:33:38 -0800 Subject: [PATCH 13/13] updated broken link --- docs/docs/transformers/system/reference.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx index acd51ae0c6..fc1eded192 100644 --- a/docs/docs/transformers/system/reference.mdx +++ b/docs/docs/transformers/system/reference.mdx @@ -777,7 +777,7 @@ Here are some examples of what an output full address value may look like. ### Generate Full Name{#generate-full-name} -The generate full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/reference#generate-first-name) and [last](/transformers/system/last-name) names transformers. +The generate full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/reference#generate-first-name) and [last](/transformers/system/reference#generate-last-name) names transformers. **Configurations**