diff --git a/docs/docs/overview/use-cases/anonymization.mdx b/docs/docs/overview/use-cases/anonymization.mdx
index 5cb6b5c05b..8338281b2a 100644
--- a/docs/docs/overview/use-cases/anonymization.mdx
+++ b/docs/docs/overview/use-cases/anonymization.mdx
@@ -27,7 +27,7 @@ We've mapped each transformer to the right column which means that every time th
//screenshot of the output here
-Since the transformers are customizable, you can anonymize or mask any source data by writing your own custom tranformer. Check out the [custom transformers](/transformers/custom) guide for more information. This gives you ultimate flexibility to control your data.
+Since the transformers are customizable, you can anonymize or mask any source data by writing your own custom tranformer. Check out the [custom transformers](/transformers/user-defined) guide for more information. This gives you ultimate flexibility to control your data.
Lastly, you can decide to subset this data to reduce the size of it to make it fit in lower level environments like your local database.
diff --git a/docs/docs/transformers/custom.mdx b/docs/docs/transformers/custom.mdx
deleted file mode 100644
index d760fea69c..0000000000
--- a/docs/docs/transformers/custom.mdx
+++ /dev/null
@@ -1,51 +0,0 @@
----
-title: Custom
-id: custom
-hide_title: true
-slug: /transformers/custom
----
-
-import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx';
-
-import { DocsImage } from '@site/src/CustomComponents/DocsImage';
-
-
-
-## Introduction
-
-Custom Transfomers are a great way to configure a system transformer with your own presets and publish it everywhere. Custom transformers are saved at the account level and
-can be used across multiple jobs, saving your time during the schema configuration process.
-
-
-
-## Creating a Custom Transformer
-
-In order to create a custom transformer, follow these steps:
-
-1. Navigate to the **Transformers** page and click on **+ New Transformer**.
-2. You'll be brought to the new transformer page where you can select a base transformer. A base transformer serves as the blueprint for the custom transformer. Select the base transformer for your custom transformer.
-
-
-
-3. Once you've selected a base transformer, you'll be prompted to give the transformer a name and description. Additionally, you can preset custom default configurations depending on the transformer.
-
-
-
-4. Once you've filled out the form, you can click on **save** and then you will be routed to the newly created transformer.
-
-
-
-## Using a Custom Transformer
-
-Once you've created a custom transformer, you'll see it appear in the transformer list in Transformers main page as well as the Schema configuration page. Above, we created a custom transformer
-called `custom-string-transformer`, we can now see it both places.
-
-In the transformers table under the Custom Transformer tab.
-
-
-
-In the Schema configuration page in the transformer select.
-
-
-
-Now we can finsih the rest of our job configuration and the newly created custom transformer will be used in the column that it's mapped to.
diff --git a/docs/docs/transformers/introduction.mdx b/docs/docs/transformers/introduction.mdx
index 30e853b565..3fd02d9249 100644
--- a/docs/docs/transformers/introduction.mdx
+++ b/docs/docs/transformers/introduction.mdx
@@ -19,16 +19,19 @@ import { DocsImage } from '@site/src/CustomComponents/DocsImage';
## Introduction
-Transformers are data-type specific modules that anonymize or generate data. Tranformers are defined in the job workflow and are applied to every piece of data in the column they are assigned. Neosync ships with a number of transformers that are already built to handle common data types such as email, addresses, ssn, strings, integers and more.
+Transformers are data-type specific modules that anonymize or generate data. For example, an email transformer can create a synthetic email address or anonymize an existing email address.
-
+Tranformers are defined in the job workflow and are applied to every piece of data in the column they are assigned. Neosync ships with over 40 pre-defined transformers that make it fast and easy to get started.
+
+
## Transformer Types
There are two types of transformers:
1. **System Transformers** : Pre-built and configured transformers that automatically come with Neosync out of the box. These cannot be deleted but they can be cloned.
-2. **Custom Transformers** : System transformers that have been cloned, re-configured and renamed. These transformers can be edited and deleted.
+
+2. **User Defined Transformers** : These are system transformers that have been cloned, re-configured and renamed. These transformers can be edited and deleted.
## Transformers
@@ -39,13 +42,13 @@ You can browse our list of system transformers which come out of the box or you
{
title: 'System Transformers',
description: 'Pre-built transformers that are ready to use',
- link: '/transformers/system/email',
+ link: '/transformers/system/reference',
icon: ,
},
{
- title: 'Custom Transformers',
+ title: 'User Defined Transformers',
description: 'Define your own transformers',
- link: '/transformers/custom',
+ link: '/transformers/user-defined',
icon: ,
},
]}
diff --git a/docs/docs/transformers/system/card-number.mdx b/docs/docs/transformers/system/card-number.mdx
deleted file mode 100644
index 43d6c9dd6e..0000000000
--- a/docs/docs/transformers/system/card-number.mdx
+++ /dev/null
@@ -1,63 +0,0 @@
----
-title: Card Number
-id: card-number
-hide_title: true
-slug: /transformers/system/card-number
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The card number transformer can anonymize an existing email address or completely generate a new one. A card number can be quite complex but we make it easy to generate valid luhn-check card numbers.
-
-By default, the card number transformer generates a random 16 digit card number that is _not_ luhn valid. If you want luhn validation, please set the luhn-check config to `true`.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output card number. The card number transformer has the following configurations:
-
-
-
-## Examples
-
-Here are some examples of an output card number:
-
-
diff --git a/docs/docs/transformers/system/city.mdx b/docs/docs/transformers/system/city.mdx
deleted file mode 100644
index eb59addb93..0000000000
--- a/docs/docs/transformers/system/city.mdx
+++ /dev/null
@@ -1,42 +0,0 @@
----
-title: City
-id: city
-hide_title: true
-slug: /transformers/system/city
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The city transformer generates a randomly selected US city. You can see the complete list of cities that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
-
-If you'd rather not get back a real city value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value.
-
-## Configurations
-
-There are no configurations for the city transformer.
-
-## Examples
-
-Here are some examples of what an output City value may look like.
-
-
diff --git a/docs/docs/transformers/system/email.mdx b/docs/docs/transformers/system/email.mdx
deleted file mode 100644
index 6159324876..0000000000
--- a/docs/docs/transformers/system/email.mdx
+++ /dev/null
@@ -1,93 +0,0 @@
----
-title: Email
-id: email
-hide_title: true
-slug: /transformers/system/email
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The email transformer can anonymize an existing email address or completely generate a new one. An email is made up of three parts:
-
-`@.`
-
-By default, the transformer randomizes the username, domain and top-level domain while always preserving the email format by retaining the @ and . characters.
-
-For example, the following input value:
-`john@acme.com`
-
-Would produce the following ouput value:
-`ytvub873@ksh0293.com`
-
-You can see we generated new privacy-safe values for the username and the domain. You can update this structure using the configurations below. Lastly, the email transformer takes in a `string` and returns a `string`.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output email. The email transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different potential email formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/first-name.mdx b/docs/docs/transformers/system/first-name.mdx
deleted file mode 100644
index 6908eab3e2..0000000000
--- a/docs/docs/transformers/system/first-name.mdx
+++ /dev/null
@@ -1,66 +0,0 @@
----
-title: First Name
-id: first-name
-hide_title: true
-slug: /transformers/system/first-name
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json).
-
-By default, the first name transformer generates a first name of random length. To preserve the length of the input first name, you can set the `preserveLength` config.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output first name. The first name transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different first name formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/full-address.mdx b/docs/docs/transformers/system/full-address.mdx
deleted file mode 100644
index eaa2e25885..0000000000
--- a/docs/docs/transformers/system/full-address.mdx
+++ /dev/null
@@ -1,43 +0,0 @@
----
-title: Full Address
-id: full-address
-hide_title: true
-slug: /transformers/system/full-address
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The full address transformer generates a randomly selects a real full address that exists in the United States. You can see the complete list of full addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
-
-The full address transformer returns a valid United States address. For example:
-`123 Main Street Boston, Massachusetts, 02169`
-
-## Configurations
-
-There are no configurations for the full adddress transformer.
-
-## Examples
-
-Here are some examples of what an output full address value may look like.
-
-
diff --git a/docs/docs/transformers/system/full-name.mdx b/docs/docs/transformers/system/full-name.mdx
deleted file mode 100644
index 38ac2b9e91..0000000000
--- a/docs/docs/transformers/system/full-name.mdx
+++ /dev/null
@@ -1,66 +0,0 @@
----
-title: Full Name
-id: full-name
-hide_title: true
-slug: /transformers/system/full-name
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/first-name) and [last](/transformers/system/last-name) names transformers.
-
-By default, the full name transformer generates a full name of random length. To preserve the length of the input full name, you can set the `preserveLength` config.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output full name. The full name transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/gender.mdx b/docs/docs/transformers/system/gender.mdx
deleted file mode 100644
index 6df48089b6..0000000000
--- a/docs/docs/transformers/system/gender.mdx
+++ /dev/null
@@ -1,84 +0,0 @@
----
-title: Gender
-id: gender
-hide_title: true
-slug: /transformers/system/gender
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The gender transformer randomly selects a gender value from a predefined list of genders. Here is the list:
-
-
-
-By default, the gender transformer does not abbreviate the gender. If you'd like to return an abbreviated gender, pass in the `abbreviate` config.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output gender. The gender transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/int-phone.mdx b/docs/docs/transformers/system/int-phone.mdx
deleted file mode 100644
index 7b1ef2f9c6..0000000000
--- a/docs/docs/transformers/system/int-phone.mdx
+++ /dev/null
@@ -1,73 +0,0 @@
----
-title: Phone (integer)
-id: int-phone
-hide_title: true
-slug: /transformers/system/int-phone
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes an integer value and returns an integer value. The [Phone (string) transformer](/transformers/system/string-phone) takes in a string value, returns a string and has more configurations available.
-
-By default, the phone (integer) transformer generates a random 10 digit phone number.
-
-For example, the following input value:
-`7829828714`
-
-Would produce the following ouput value:
-`5698437232`
-
-You can see we generated a new phone integer value that can be used as an integer phone number. Also, note that we don't include hyphens in this transformer since the output type is an `integer`.
-
-Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (integer) transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/last-name.mdx b/docs/docs/transformers/system/last-name.mdx
deleted file mode 100644
index dff6c417b7..0000000000
--- a/docs/docs/transformers/system/last-name.mdx
+++ /dev/null
@@ -1,66 +0,0 @@
----
-title: Last Name
-id: last-name
-hide_title: true
-slug: /transformers/system/last-name
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json).
-
-By default, the last name transformer generates a last name of random length. To preserve the length of the input last name, you can set the `preserveLength` config.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output last name. The last name transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different last name formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/null.mdx b/docs/docs/transformers/system/null.mdx
deleted file mode 100644
index fb5cf8dc74..0000000000
--- a/docs/docs/transformers/system/null.mdx
+++ /dev/null
@@ -1,37 +0,0 @@
----
-title: Null Value
-id: null-t
-hide_title: true
-slug: /transformers/system/null-t
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The null transformer simply returns a null value. This may be useful if you a column that can't be null but don't have a specific value that you want to insert.
-
-## Configurations
-
-There are no configurations for the null transformer.
-
-## Examples
-
-Here are some examples of what an output null value may look like.
-
-
diff --git a/docs/docs/transformers/system/random-bool.mdx b/docs/docs/transformers/system/random-bool.mdx
deleted file mode 100644
index c027452fa2..0000000000
--- a/docs/docs/transformers/system/random-bool.mdx
+++ /dev/null
@@ -1,40 +0,0 @@
----
-title: Random Boolean
-id: random-bool
-hide_title: true
-slug: /transformers/system/random-bool
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The random boolean transformer randomly generates a boolean value.
-
-## Configurations
-
-There are no configurations for the random boolean transformer.
-
-## Examples
-
-Here are some examples of what an output random boolean value may look like.
-
-
diff --git a/docs/docs/transformers/system/random-float.mdx b/docs/docs/transformers/system/random-float.mdx
deleted file mode 100644
index 137072c354..0000000000
--- a/docs/docs/transformers/system/random-float.mdx
+++ /dev/null
@@ -1,93 +0,0 @@
----
-title: Random Float
-id: random-float
-hide_title: true
-slug: /transformers/system/random-float
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The random float transformer generates a random floating point number.
-
-For example:
-`32.2432`
-
-By default, the random float transformer generates a floating point number with 2 digits before the decimal point and 3 digits after the decimal point.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/random-int.mdx b/docs/docs/transformers/system/random-int.mdx
deleted file mode 100644
index b2cdeaca75..0000000000
--- a/docs/docs/transformers/system/random-int.mdx
+++ /dev/null
@@ -1,83 +0,0 @@
----
-title: Random Integer
-id: random-int
-hide_title: true
-slug: /transformers/system/random-int
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The random integer transformer generates a random integer.
-
-For example:
-`6782`
-
-By default, the random int transformer generates an integer of 4 digits long.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/random-string.mdx b/docs/docs/transformers/system/random-string.mdx
deleted file mode 100644
index daa9d4fb8f..0000000000
--- a/docs/docs/transformers/system/random-string.mdx
+++ /dev/null
@@ -1,83 +0,0 @@
----
-title: Random String
-id: random-string
-hide_title: true
-slug: /transformers/system/random-string
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The random string transformer generates a random string of alphanumeric characters.
-
-For example:
-`h2983hf28h92`
-
-By default, the random string transformer generates a string of 10 characters long.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output string. The random string transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/reference.mdx b/docs/docs/transformers/system/reference.mdx
new file mode 100644
index 0000000000..fc1eded192
--- /dev/null
+++ b/docs/docs/transformers/system/reference.mdx
@@ -0,0 +1,1904 @@
+---
+title: References
+id: reference
+hide_title: true
+slug: /transformers/system/reference
+---
+
+import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx';
+import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
+import { TransformersTable } from '@site/src/CustomComponents/TransformersTable.tsx';
+
+
+
+## Introduction
+
+Neosync comes out of the box with 40+ system transformers in order to give you an easy way to get started with anonymizing or generating data. There are two kinds of system transformers:
+
+1. **Generators** - these begin with `generate_` and do **not** anonymize existing data. They only generate net-new synthetic data.
+
+2. **Transformers** - these begin with `transform_` and **can** anonymize existing data or generate new-new synthetic data based on how you configure it.
+
+## Reference
+
+',
+ pagelink: 'generate-ssn',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_ssn.go',
+ },
+ {
+ title: 'Generate State',
+ type: 'string',
+ description:
+ 'Randomly selects a US state and returns the two-character state code.',
+ pagelink: 'generate-state',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_state.go',
+ },
+ {
+ title: 'Generate Street Address',
+ type: 'string',
+ description:
+ 'Randomly generates a street address in the format: {street_num} {street_addresss} {street_descriptor}. For example, 123 Main Street.',
+ pagelink: 'generate-street-address',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_street_address.go',
+ },
+ {
+ title: 'Generate String Phone Number',
+ type: 'string',
+ description:
+ 'Generates a Generate phone number and returns it as a string.',
+ pagelink: 'generate-string-phone-number',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_string_phone.go',
+ },
+ {
+ title: 'Generate Random String',
+ type: 'string',
+ description:
+ 'Creates a randomly ordered alphanumeric string with a default length of 10 unless the String Length parameter are defined.',
+ pagelink: 'generate-random-string',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_string.go',
+ },
+ {
+ title: 'Generate Unix Timestamp',
+ type: 'int64',
+ description: 'Randomly generates a Unix timestamp',
+ pagelink: 'generate-unix-timestamp',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_unixtimestamp.go',
+ },
+ {
+ title: 'Generate Username',
+ type: 'string',
+ description:
+ 'Randomly generates a username in the format.',
+ pagelink: 'generate-username',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_username.go',
+ },
+ {
+ title: 'Generate UTC Timestamp',
+ type: 'time',
+ description: 'Randomly generates a UTC timestamp.',
+ pagelink: 'generate-utc-timestamp',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_utctimestamp.go',
+ },
+ {
+ title: 'Generate UUID',
+ type: 'uuid',
+ description: 'Generates a new UUIDv4 id.',
+ pagelink: 'generate-uuid',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_uuid.go',
+ },
+ {
+ title: 'Generate Zipcode',
+ type: 'string',
+ description:
+ 'Randomly selects a zip code from a list of predefined US zipcodes.',
+ pagelink: 'generate-zipcode',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/generate_zipcode.go',
+ },
+ {
+ title: 'Transform E164 Phone Number',
+ type: 'string',
+ description: 'Transforms an existing E164 formatted phone number.',
+ pagelink: 'transform-e164-phone-number',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_e164_phone.go',
+ },
+ {
+ title: 'Transform First Name',
+ type: 'string',
+ description: 'Transforms an existing first name',
+ pagelink: 'transform-first-name',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_first_name.go',
+ },
+ {
+ title: 'Transform Float64',
+ type: 'float64',
+ description: 'Transforms an existing float value.',
+ pagelink: 'transform-float64',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_float.go',
+ },
+ {
+ title: 'Transform Full Name',
+ type: 'string',
+ description: 'Transforms an existing full name.',
+ pagelink: 'transform-full-name',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_full_name.go',
+ },
+ {
+ title: 'Transform Int64 Phone Number',
+ type: 'int64',
+ description:
+ 'Transforms an existing phone number that is typed as an integer',
+ pagelink: 'transform-int64-phone-number',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_int_phone.go',
+ },
+ {
+ title: 'Transform Int64',
+ type: 'int64',
+ description: 'Transforms an existing integer value.',
+ pagelink: 'transform-int64',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_int.go',
+ },
+ {
+ title: 'Transform Last Name',
+ type: 'string',
+ description: 'Transforms an existing last name.',
+ pagelink: 'transform-last-name',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_last_name.go',
+ },
+ {
+ title: 'Transform Phone Number',
+ type: 'string',
+ description:
+ 'Transforms an existing phone number that is typed as a string.',
+ pagelink: 'transform-phone-number',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_phone.go',
+ },
+ {
+ title: 'Transform String',
+ type: 'string',
+ description: 'Transforms an existing string value.',
+ pagelink: 'transform-string',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/transform_string.go',
+ },
+ {
+ title: 'Passthrough',
+ type: 'string',
+ description:
+ 'Passes the input value through to the desination with no changes.',
+ pagelink: 'passthrough',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/4e459151080109ffa0c5b0d9937f4f20fecfa6c6/worker/pkg/workflows/datasync/activities/activities.go',
+ },
+ {
+ title: 'Null',
+ type: 'string',
+ description: 'Inserts a string instead of the source value.',
+ pagelink: 'null',
+ codeRef:
+ 'https://github.com/nucleuscloud/neosync/blob/4e459151080109ffa0c5b0d9937f4f20fecfa6c6/worker/pkg/workflows/datasync/activities/activities.go',
+ },
+ ]}
+/>
+
+### Generate Email{#generate-email}
+
+The generate email transformer generates a new randomized email address in the format:
+
+`@.`
+
+By default, the transformer randomizes the username, domain and top-level domain while always preserving the email format by retaining the `@` and `.` characters.
+
+For example, the following input value:
+`john@acme.com`
+
+Would produce the following ouput value:
+`ytvub873@ksh0293.com`
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+
+
+### Generate Realistic Email{#generate-realistic-email}
+
+The generate realistic email transformer generates a new randomized email address that "looks" realistis in the format:
+
+`@.`
+
+By "looks realistic", we mean that the email will look like an email that a normal person will have and won't be randomized letters and numbers. The generated email will either have a first letter initial and last name as a username and then a standardized domain name such as gmail, yahoo, hotmail, etc.
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+
+
+### Transform Email{#transform-email}
+
+The transform email transformer can anonymize an existing email address or generate a new randomized email address in the format:
+
+`@.`
+
+By default, the transformer randomizes the username, domain and top-level domain while always preserving the email format by retaining the `@` and `.` characters.
+
+For example, the following input value:
+`john@acme.com`
+
+Would produce the following ouput value:
+`ytvub873@ksh0293.com`
+
+**Configurations**
+
+Depending on your logic, you may want to configure the output email. The transform email transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential email formats. Here are some possible combinations:
+
+
+
+### Generate Boolean{#generate-boolean}
+
+The genereate boolean transformer randomly generates a boolean value.
+
+**Configurations**
+
+There are no configurations for the random boolean transformer.
+
+**Examples**
+
+Here are some examples of what an output random boolean value may look like.
+
+
+
+### Generate Card Number{#generate-card-number}
+
+The generate card number transformer generates a new card number that is [luhn valid.](https://en.wikipedia.org/wiki/Luhn_algorithm)
+
+By default, the card number transformer generates a random 16 digit card number that is _not_ luhn valid. If you want luhn validation, please set the luhn-check config to `true`.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output card number. The card number transformer has the following configurations:
+
+
+
+**Examples**
+
+Here are some examples of an output card number:
+
+
+
+### Generate City{#generate-city}
+
+The generate city transformers generates a randomly selected US city. You can see the complete list of cities that are available to be randomly selected [here](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
+
+If you'd rather not get back a real city value, you can use the [Generate Random String Transformer](/transformers/system/reference#generate-random-string) to generate a random string value.
+
+**Configurations**
+
+There are no configurations for the city transformer.
+
+**Examples**
+
+Here are some examples of what an output City value may look like.
+
+
+
+### Generate E164 Phone Number{#generate-e164-phone-number}
+
+The generate e164 phone transformer generates a new international phone number including the + sign. By default, the generate e164 transformer generates a random 12 digit number with no hyphens in the format:
+
+`+34567890123`
+
+TPhone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to be a certain length by passing in the `length` configuration. Here is more information on the [e164 format](https://www.twilio.com/docs/glossary/what-e164). This transformer also has a min of 9 and a max of 15. If you want to generate a number that is longer than that, you can use the [Generate Random int64 transformer](/transformers/system/reference#generate-random-int64)
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output e164 phone number:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations:
+
+
+
+### Generate First Name{#generate-first-name}
+
+The generate first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json)
+
+By default, the generate first name transformer randomly picks a first name with a length between 2 and 12.
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different first name formats. Here are some possible combinations:
+
+
+
+### Generate Float64{#generate-float64}
+
+The generate float transformer generates a random floating point number.
+
+For example:
+`32.2432`
+
+By default, the random float transformer generates a floating point number with 3 digits before the decimal point and 3 digits after the decimal point.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations:
+
+
+
+### Generate Full Address{#generate-full-address}
+
+The generate full address transformer generates a randomly selected real full address that exists in the United States. You can see the complete list of full addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json)
+
+The full address transformer returns a valid United States address. For example:
+`123 Main Street Boston, Massachusetts, 02169`
+
+**Configurations**
+
+There are no configurations for the full adddress transformer.
+
+**Examples**
+
+Here are some examples of what an output full address value may look like.
+
+
+
+### Generate Full Name{#generate-full-name}
+
+The generate full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/reference#generate-first-name) and [last](/transformers/system/reference#generate-last-name) names transformers.
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+Here are some examples of what an output full name value may look like.
+
+
+
+### Generate Gender{#generate-gender}
+
+The generate gender transformer randomly selects a gender value from a predefined list of genders. Here is the list:
+
+
+
+By default, the gender transformer does not abbreviate the gender. If you'd like to return an abbreviated gender, pass in the `abbreviate` config.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output gender. The gender transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations:
+
+
+
+### Generate Int64 Phone Number{#generate-int64-phone-number}
+
+The generate int64 phone number generates a random 10 digit phone number and returns it as an `int64` type with no hyphens. If you want to return a `string` or want to include hyphens, check out the [Generate String Phone transformer](/transformers/system/reference#generate-string-phone-number).
+
+For example, the generate int64 phone number transformer could generate the following ouput value:
+`5698437232`
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+Here are some examples of what the output values could look like:
+
+
+
+### Generate Random Int64{#generate-random-int64}
+
+The generate random int64 transformer generates a random integer and returns it as a int64 type.
+
+For example:
+`6782`
+
+By default, the generate random int64 transformer generates an integer of 4 digits long.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output float. The random integer transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations:
+
+
+
+### Generate Last Name{#generate-last-name}
+
+The generate last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json).
+
+**Configurations**
+
+This transformer has no configurations.
+
+**Examples**
+
+Here are some possible output examples:
+
+
+
+### Generate SHA256 Hash{#generate-sha256-hash}
+
+The generate sha256 hash transformer generates a random SHA256 hash and hex encodes the resulting value and returns it as a string.
+
+**Configurations**
+
+There are no configurations for the hash transformer.
+
+**Examples**
+
+Here are some examples of what an output hash value may look like.
+
+
+
+### Generate SSN{#generate-ssn}
+
+The generate ssn transformer randomly generates a social security number and returns it with hyphens as a string.
+
+For, example:
+`123-45-6789`
+
+**Configurations**
+
+There are no configurations for the ssn transformer.
+
+**Examples**
+
+Here are some examples of what an output street address value may look like.
+
+
+
+### Generate State{#generate-state}
+
+The generate state transformer generates a randomly selected US state. You can see the complete list of states that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
+
+If you'd rather not get back a real state value, you can use the [Random String Transformer](/transformers/system/reference#generate-random-string) to generate a random string value.
+
+**Configurations**
+
+There are no configurations for the state transformer.
+
+**Examples**
+
+Here are some examples of what an output state value may look like.
+
+
+
+### Generate Street Address{#generate-street-address}
+
+The generate street address transformer generates a randomly selects a real street address that exists in the United States. You can see the complete list of street addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
+
+The street address transformer returns a valid United States address. For example:
+`123 Main Street Boston`
+
+**Configurations**
+
+There are no configurations for the street adddress transformer.
+
+**Examples**
+
+Here are some examples of what an output street address value may look like.
+
+
+
+### Generate String Phone Number{#generate-string-phone-number}
+
+The generate string phone transformer generates a random string phone number. By default, the string phone transformer generates a random 10 digit phone number with no hyphens.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output phone number. The generate string phone number transformer has the following configuration:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations:
+
+
+
+### Generate Random String{#generate-random-string}
+
+The generate random string transformer generates a random string of alphanumeric characters.
+
+For example:
+`h2983hf28h92`
+
+By default, the random string transformer generates a string of 10 characters long.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output string. The generate string transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations:
+
+
+
+### Generate Unix Timestamp{#generate-unix-timestamp}
+
+The generate unix timestamp transformer randomly generates a unix timestamp in UTC timezone and returns back an int64 representation of that timestamp.
+
+By default, the generated timestamp will always be in the **past**.
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+Here are some examples of what an output state value may look like.
+
+
+
+### Generate Username{#generate-username}
+
+The generate username transformer generates a random string in the format of ``. The last names are pulled from the last name transformer(/transformers/system/reference#generate-last-name).
+
+**Configurations**
+
+There are no configurations for the state transformer.
+
+**Examples**
+
+Here are some examples of what an output state value may look like.
+
+
+
+### Generate UTC Timestamp{#generate-utc-timestamp}
+
+The generate utc timestamp transformer randomly generates a utc timestamp in UTC timezone and returns back an time.Time representation of that timestamp.
+
+By default, the generated timestamp will always be in the **past**.
+
+**Configurations**
+
+There are no configurations for this transformer.
+
+**Examples**
+
+Here are some examples of what an output state value may look like.
+
+
+
+### Generate UUID{#generate-uuid}
+
+The generate UUID transformer generates a new UUID v4.
+
+For example:
+`6d871028b072442c9ad9e6e4e223adfa`
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output uuid. The uuid transformer has the following configurations:
+
+
+
+**Examples**
+
+Here are some example UUID values that the uuid transformer can generate:
+
+
+
+### Generate Zipcode{#generate-zipcode}
+
+The generate zipcode transformer generates a randomly selected US zipcode. You can see the complete list of zipcodes that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
+
+If you'd rather not get back a real zipcode value, you can use the [Random String Transformer](/transformers/system/reference#generate-random-string) to generate a random string value.
+
+**Configurations**
+
+There are no configurations for the zipcode transformer.
+
+**Examples**
+
+Here are some examples of what an output zipcode value may look like.
+
+
+
+### Transform E164 Phone Number{#transform-e164-phone-number}
+
+The transform e164 phone transformer can anonymize an existing e164 phone number or completely generate a new one. It returns a string value with the format `+`.
+
+For example, the following input value:
+`+7829828714`
+
+Could produce the following ouput value:
+`+5698437232`
+
+E164 Phone numbers vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value in order maintain the same shape as the input value.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output e164 phone number. The transform e164 phone number transformer has the following configurations:
+
+
+
+**Examples**
+
+Here are some examples of what the transform e164 phone number tranformer can output:
+
+
+
+### Transform First Name{#transform-first-name}
+
+The transform first name transformer generates a valid first name from a list of predefined first name values. You can see the entire list of first name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/first-names.json).
+
+By default, the first name transformer generates a first name of random length. To preserve the length of the input first name, you can set the `preserveLength` config.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output first name. The first name transformer has the following configurations:
+
+
+
+**Examples**
+
+Here are some examples of what the transform first name tranformer can output:
+
+
+
+### Transform Float64{#transform-float64}
+
+The transform float transformer can anonymize an existing float64 value or generate a compeletely random floating point number.
+
+For example:
+`32.2432`
+
+By default, the random float transformer generates a floating point number with 3 digits before the decimal point and 3 digits after the decimal point.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential random float formats. Here are some possible combinations:
+
+
+
+### Transform Full Name{#transform-full-name}
+
+The transform full name transformer generates a valid full name from a list of predefined full name values. The generated full name is made from a combination of the [first](/transformers/system/reference#generate-first-name) and [last](/transformers/system/reference#generate-last-name) names transformers.
+
+By default, the full name transformer generates a full name of random length. To preserve the length of the input full name, you can set the `preserveLength` config.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output full name. The full name transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different full name formats. Here are some possible combinations:
+
+
+
+### Transform Int64 Phone Number{#transform-int64-phone-number}
+
+The transform int64 phone transformer can anonymize an existing phone number or completely generate a new one. By default, the transform int64 phone number transformer generates a random 10 digit phone number.
+
+For example, the following input value:
+`7829828714`
+
+Would produce the following ouput value:
+`5698437232`
+
+You can see we generated a new phone integer value that can be used as an integer phone number. Also, note that we don't include hyphens in this transformer since the output type is an `integer`.
+
+**#Configurations**
+
+Depending on your validations, you may want to configure the output phone number.
+
+
+
+**Examples**
+
+Here are some possible example output values:
+
+
+
+### Transform Int64{#transform-int64}
+
+The random integer transformer generates a random integer.
+
+For example:
+`6782`
+
+By default, the random int transformer generates an integer of 4 digits long.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential random int64 values. Here are some possible combinations:
+
+
+
+### Transform Last Name{#transform-last-name}
+
+The transform last name transformer generates a valid last name from a list of predefined last name values. You can see the entire list of last name value [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/last-names.json).
+
+By default, the last name transformer generates a last name of random length. To preserve the length of the input last name, you can set the `preserveLength` config.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output last name. The last name transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different last name formats. Here are some possible combinations:
+
+
+
+### Transform Phone Number{#transform-phone-number}
+
+The transform phone transformer can anonymize an existing phone number or completely generate a new one. This transformer specifically takes a string value and returns a string value.
+
+By default, the transform phone transformer generates a random 10 string value phone number with no hyphens.
+
+For example, the following input value:
+`7829828714`
+
+Would produce the following ouput value:
+`5698437232`
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output phone number. Here are the configurations for the transform string phone number transformer.
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations:
+
+
+
+### Transform String{#transform-string}
+
+The random string transformer generates a random string of alphanumeric characters.
+
+For example:
+`h2983hf28h92`
+
+By default, the random string transformer generates a string of 10 characters long.
+
+**Configurations**
+
+Depending on your validations, you may want to configure the output string. The random string transformer has the following configurations:
+
+
+
+**Examples**
+
+There are several ways you can mix-and-match configurations to get different potential random integer formats. Here are some possible combinations:
+
+
+
+### Passthrough{#passthrough}
+
+The passthrough tranformer simplfy passes the input data out to the output without making any modications to it. This is useful in many circumstances but cautious of accidently leaking sensitive data through this transformer.
+
+### Null{#null}
+
+The null transformer simply returns a null value. This may be useful if you a column that can't be null but don't have a specific value that you want to insert.
+
+**Configurations**
+
+There are no configurations for the null transformer.
+
+**Examples**
+
+Here are some examples of what an output null value may look like.
+
+
diff --git a/docs/docs/transformers/system/sha256hash.mdx b/docs/docs/transformers/system/sha256hash.mdx
deleted file mode 100644
index d83e247bd3..0000000000
--- a/docs/docs/transformers/system/sha256hash.mdx
+++ /dev/null
@@ -1,52 +0,0 @@
----
-title: Hash
-id: hash
-hide_title: true
-slug: /transformers/system/hash
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The hash transformer creates a SHA256 hash and hex encodes the resulting value and returns it as a string. Before the value is hashed, it's first transformed into a byte array to account for flexible input types.
-
-## Configurations
-
-There are no configurations for the hash transformer.
-
-## Examples
-
-Here are some examples of what an output hash value may look like.
-
-
diff --git a/docs/docs/transformers/system/ssn.mdx b/docs/docs/transformers/system/ssn.mdx
deleted file mode 100644
index d5910a18a4..0000000000
--- a/docs/docs/transformers/system/ssn.mdx
+++ /dev/null
@@ -1,43 +0,0 @@
----
-title: SSN
-id: ssn
-hide_title: true
-slug: /transformers/system/ssn
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The ssn transformer randomly generates a social security number and returns it with hyphens as a string.
-
-For, example:
-`123-45-6789`
-
-## Configurations
-
-There are no configurations for the ssn transformer.
-
-## Examples
-
-Here are some examples of what an output street address value may look like.
-
-
diff --git a/docs/docs/transformers/system/state.mdx b/docs/docs/transformers/system/state.mdx
deleted file mode 100644
index fb29bfba8f..0000000000
--- a/docs/docs/transformers/system/state.mdx
+++ /dev/null
@@ -1,42 +0,0 @@
----
-title: State
-id: state
-hide_title: true
-slug: /transformers/system/state
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The state transformer generates a randomly selected US state. You can see the complete list of states that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
-
-If you'd rather not get back a real state value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value.
-
-## Configurations
-
-There are no configurations for the state transformer.
-
-## Examples
-
-Here are some examples of what an output state value may look like.
-
-
diff --git a/docs/docs/transformers/system/street-address.mdx b/docs/docs/transformers/system/street-address.mdx
deleted file mode 100644
index 835163bf03..0000000000
--- a/docs/docs/transformers/system/street-address.mdx
+++ /dev/null
@@ -1,43 +0,0 @@
----
-title: Street Address
-id: street-address
-hide_title: true
-slug: /transformers/system/street-address
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The street address transformer generates a randomly selects a real street address that exists in the United States. You can see the complete list of street addresses that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
-
-The street address transformer returns a valid United States address. For example:
-`123 Main Street Boston`
-
-## Configurations
-
-There are no configurations for the street adddress transformer.
-
-## Examples
-
-Here are some examples of what an output street address value may look like.
-
-
diff --git a/docs/docs/transformers/system/string-phone.mdx b/docs/docs/transformers/system/string-phone.mdx
deleted file mode 100644
index f6400f77a7..0000000000
--- a/docs/docs/transformers/system/string-phone.mdx
+++ /dev/null
@@ -1,106 +0,0 @@
----
-title: Phone (string)
-id: string-phone
-hide_title: true
-slug: /transformers/system/string-phone
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The phone transformer can anonymize an existing phone number or completely generate a new one. There are two phone number transformers that differ by type and the configurations that are available to customize the transformer. This transformer specifically takes a string value and returns a string value. The [Phone (integer) transformer](/transformers/system/string-phone) takes in an integer value and returns an integer value.
-
-By default, the phone (string) transformer generates a random 10 digit phone number with no hyphens.
-
-For example, the following input value:
-`7829828714`
-
-Would produce the following ouput value:
-`5698437232`
-
-You can see we generated a new phone string value that can be used as an string phone number.
-
-Phone numbers also vary in length with some international phone numbers reaching up to 15 digits in length. You can set this transformer to respect the length of the input value if you're working with phone numbers longer than 10 digits in length. In fact, you can generate a specific international "looking" phone number by specifying the [e164 format](https://www.twilio.com/docs/glossary/what-e164) config.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output phone number. As we discussed above, some phone numbers may be longer than the default 10 digits in length. The phone number (string) transformer has the following configurations:
-
-
-
-## Examples
-
-There are several ways you can mix-and-match configurations to get different potential phone number formats. Here are some possible combinations:
-
-
diff --git a/docs/docs/transformers/system/unix-ts.mdx b/docs/docs/transformers/system/unix-ts.mdx
deleted file mode 100644
index 8cd2dd4652..0000000000
--- a/docs/docs/transformers/system/unix-ts.mdx
+++ /dev/null
@@ -1,63 +0,0 @@
----
-title: Unix Timestamp
-id: unix-ts
-hide_title: true
-slug: /transformers/system/unix-ts
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The unix timestamp transformer randomly generates a unix timestamp in UTC timezone and returns back an int64 representation of that timestamp.
-
-By default, the generated timestamp will always be in the **past**. To generate a timestamp that is in the future, set the `Future` config to true.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output unix timestamp. The unix timestamp transformer has the following configuration options:
-
-
-
-## Examples
-
-Here are some examples of what an output state value may look like.
-
-
diff --git a/docs/docs/transformers/system/utc-ts.mdx b/docs/docs/transformers/system/utc-ts.mdx
deleted file mode 100644
index 34d2ae368f..0000000000
--- a/docs/docs/transformers/system/utc-ts.mdx
+++ /dev/null
@@ -1,71 +0,0 @@
----
-title: UTC Timestamp
-id: utc-ts
-hide_title: true
-slug: /transformers/system/utc-ts
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The utc timestamp transformer randomly generates a utc timestamp in UTC timezone and returns back an tiem.Time representation of that timestamp.
-
-By default, the generated timestamp will always be in the **past**. To generate a timestamp that is in the future, set the `future` config to true.
-
-## Configurations
-
-Depending on your validations, you may want to configure the output utc timestamp. The utc timestamp transformer has the following configuration options:
-
-
-
-## Examples
-
-Here are some examples of what an output state value may look like.
-
-
diff --git a/docs/docs/transformers/system/uuid.mdx b/docs/docs/transformers/system/uuid.mdx
deleted file mode 100644
index d3492b1572..0000000000
--- a/docs/docs/transformers/system/uuid.mdx
+++ /dev/null
@@ -1,68 +0,0 @@
----
-title: UUID
-id: uuid
-hide_title: true
-slug: /transformers/system/uuid
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The UUID transformer generates a new UUID v4.
-
-For example:
-`6d871028b072442c9ad9e6e4e223adfa`
-
-## Configurations
-
-Depending on your validations, you may want to configure the output uuid. The uuid transformer has the following configurations:
-
-
-
-## Examples
-
-Here are some example UUID values that the uuid transformer can generate:
-
-
diff --git a/docs/docs/transformers/system/zipcode.mdx b/docs/docs/transformers/system/zipcode.mdx
deleted file mode 100644
index 574f95b474..0000000000
--- a/docs/docs/transformers/system/zipcode.mdx
+++ /dev/null
@@ -1,42 +0,0 @@
----
-title: Zipcode
-id: zipcode
-hide_title: true
-slug: /transformers/system/zipcode
----
-
-import { TransformerPageHeader } from '@site/src/CustomComponents/TransformerPageHeader.tsx';
-
-import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx';
-
-
-
-## Definition
-
-The zipcode transformer generates a randomly selected US zipcode. You can see the complete list of zipcodes that are available to be randomly selected [here.](https://github.com/nucleuscloud/neosync/blob/main/worker/internal/benthos/transformers/data-sets/addresses.json).
-
-If you'd rather not get back a real zipcode value, you can use the [Random String Transformer](/transformers/system/random-string) to generate a random string value.
-
-## Configurations
-
-There are no configurations for the zipcode transformer.
-
-## Examples
-
-Here are some examples of what an output zipcode value may look like.
-
-
diff --git a/docs/docs/transformers/user-defined.mdx b/docs/docs/transformers/user-defined.mdx
new file mode 100644
index 0000000000..2ae0e8e9d3
--- /dev/null
+++ b/docs/docs/transformers/user-defined.mdx
@@ -0,0 +1,47 @@
+---
+title: User Defined
+id: user-defined
+hide_title: true
+slug: /transformers/user-defined
+---
+
+import { DocPageHeader } from '@site/src/CustomComponents/DocPageHeader.tsx';
+
+import { DocsImage } from '@site/src/CustomComponents/DocsImage';
+
+
+
+## Introduction
+
+User defined Transfomers are a great way to configure a system transformer with your own presets and publish it everywhere. User defined transformers are saved at the account level and
+can be used across multiple jobs, saving your time during the schema configuration process.
+
+
+
+## Creating a User Defined Transformer
+
+In order to create a user defined transformer, follow these steps:
+
+1. Navigate to the **Transformers** page and click on **+ New Transformer**.
+2. You'll be brought to the new transformer page where you can select a base transformer. A base transformer serves as the blueprint for the user defined transformer. Select the base transformer for your user defined transformer.
+
+
+
+3. Once you've selected a base transformer, you'll be prompted to give the transformer a name and description. Additionally, you can preset custom default configurations depending on the transformer. Fill out the details and click **save**.
+
+
+
+## Using a User Defined Transformer
+
+Once you've created a user defined transformer, you'll see it appear in the transformer list in Transformers main page as well as the Schema configuration page. Above, we created a user defined transformer
+called `custom-float-transfomer`, we can now see it both places.
+
+In the transformers table under the User Defined Transformer tab.
+
+
+
+In the Schema configuration page in the transformer select.
+
+
+
+Now we can finsih the rest of our job configuration and the newly created user defined transformer will be used in the column that it's mapped to.
diff --git a/docs/sidebars.ts b/docs/sidebars.ts
index c5d6c9e8ea..4028f78d12 100644
--- a/docs/sidebars.ts
+++ b/docs/sidebars.ts
@@ -171,125 +171,15 @@ const sidebars: SidebarsConfig = {
items: [
{
type: 'doc',
- id: 'transformers/system/email',
- label: 'Email',
- },
- {
- type: 'doc',
- id: 'transformers/system/int-phone',
- label: 'Phone (integer)',
- },
- {
- type: 'doc',
- id: 'transformers/system/string-phone',
- label: 'Phone (string)',
- },
- {
- type: 'doc',
- id: 'transformers/system/street-address',
- label: 'Street Address',
- },
- {
- type: 'doc',
- id: 'transformers/system/city',
- label: 'City',
- },
- {
- type: 'doc',
- id: 'transformers/system/full-address',
- label: 'Full Address',
- },
- {
- type: 'doc',
- id: 'transformers/system/state',
- label: 'State',
- },
- {
- type: 'doc',
- id: 'transformers/system/card-number',
- label: 'Card Number',
- },
- {
- type: 'doc',
- id: 'transformers/system/first-name',
- label: 'First Name',
- },
- {
- type: 'doc',
- id: 'transformers/system/last-name',
- label: 'Last Name',
- },
- {
- type: 'doc',
- id: 'transformers/system/full-name',
- label: 'Full Name',
- },
- {
- type: 'doc',
- id: 'transformers/system/gender',
- label: 'Gender',
- },
- {
- type: 'doc',
- id: 'transformers/system/hash',
- label: 'Hash',
- },
- {
- type: 'doc',
- id: 'transformers/system/null-t',
- label: 'Null Value',
- },
- {
- type: 'doc',
- id: 'transformers/system/ssn',
- label: 'SSN',
- },
- {
- type: 'doc',
- id: 'transformers/system/random-bool',
- label: 'Random Boolean',
- },
- {
- type: 'doc',
- id: 'transformers/system/random-float',
- label: 'Random Float',
- },
- {
- type: 'doc',
- id: 'transformers/system/random-int',
- label: 'Random Integer',
- },
- {
- type: 'doc',
- id: 'transformers/system/random-string',
- label: 'Random String',
- },
- {
- type: 'doc',
- id: 'transformers/system/unix-ts',
- label: 'Unix Timestamp',
- },
- {
- type: 'doc',
- id: 'transformers/system/utc-ts',
- label: 'UTC Timestamp',
- },
- {
- type: 'doc',
- id: 'transformers/system/uuid',
- label: 'UUID',
- },
- {
- type: 'doc',
- id: 'transformers/system/zipcode',
- label: 'Zipcode',
+ id: 'transformers/system/reference',
+ label: 'Reference',
},
],
},
{
type: 'doc',
- id: 'transformers/custom',
- label: 'Custom',
+ id: 'transformers/user-defined',
+ label: 'User Defined',
},
],
};
diff --git a/docs/src/CustomComponents/DocsTable.tsx b/docs/src/CustomComponents/DocsTable.tsx
index d7ddb798ef..ffd892e86e 100644
--- a/docs/src/CustomComponents/DocsTable.tsx
+++ b/docs/src/CustomComponents/DocsTable.tsx
@@ -24,16 +24,19 @@ export function DocsTable(props: TableData): ReactElement {
- {headers.map((header) => (
- {header}
+ {headers.map((header, headerIndex) => (
+ {header}
))}
- {rowData.map((tableData) => (
-
- {tableData.data.map((item) => (
-
+ {rowData.map((tableData, rowIndex) => (
+
+ {tableData.data.map((item, columnIndex) => (
+
{item}
))}
diff --git a/docs/src/CustomComponents/IconHandler.tsx b/docs/src/CustomComponents/IconHandler.tsx
index d1086cd9fb..c33cb81979 100644
--- a/docs/src/CustomComponents/IconHandler.tsx
+++ b/docs/src/CustomComponents/IconHandler.tsx
@@ -1,45 +1,25 @@
import {
+ AvatarIcon,
HomeIcon,
LayersIcon,
LinkBreak1Icon,
+ ListBulletIcon,
Share1Icon,
+ StackIcon,
TokensIcon,
} from '@radix-ui/react-icons';
import React, { ReactElement } from 'react';
-import {
- AiOutlineCreditCard,
- AiOutlineMail,
- AiOutlinePhone,
-} from 'react-icons/ai';
-import { BiLogoPostgresql, BiSolidCity, BiTimeFive } from 'react-icons/bi';
-import {
- BsCalendarDate,
- BsFillKeyFill,
- BsFunnel,
- BsGenderAmbiguous,
- BsPinMap,
- BsShieldCheck,
-} from 'react-icons/bs';
-import {
- FaAws,
- FaDocker,
- FaRegAddressBook,
- FaStreetView,
-} from 'react-icons/fa';
-import { GiTexas } from 'react-icons/gi';
-import { GoCode } from 'react-icons/go';
-import { GrMysql, GrSecure } from 'react-icons/gr';
+import { AiOutlineMail, AiOutlinePhone } from 'react-icons/ai';
+import { BiLogInCircle, BiLogoPostgresql, BiTerminal } from 'react-icons/bi';
+import { BsFunnel, BsShieldCheck } from 'react-icons/bs';
+import { FaAws, FaDocker } from 'react-icons/fa';
+import { GoCode, GoTable, GoVersions } from 'react-icons/go';
+import { GrMysql } from 'react-icons/gr';
import { IoBuildOutline } from 'react-icons/io5';
-import { MdPassword } from 'react-icons/md';
+import { MdPassword, MdStart } from 'react-icons/md';
import { PiArrowsSplitLight, PiFlaskLight } from 'react-icons/pi';
-import {
- RxAvatar,
- RxComponentBoolean,
- RxLetterCaseCapitalize,
-} from 'react-icons/rx';
import { SiKubernetes } from 'react-icons/si';
-import { TbDecimal } from 'react-icons/tb';
-import { TiSortNumerically } from 'react-icons/ti';
+import { TbVariable } from 'react-icons/tb';
export function IconHandler(name: string): ReactElement {
switch (name) {
@@ -67,7 +47,7 @@ export function IconHandler(name: string): ReactElement {
return ;
case 'SSN':
return ;
- case 'Custom':
+ case 'User Defined':
return ;
case 'System':
return ;
@@ -81,42 +61,24 @@ export function IconHandler(name: string): ReactElement {
return ;
case 'Subset Data':
return ;
- case 'City':
- return ;
- case 'Card Number':
- return ;
- case 'First Name':
- return ;
- case 'Full Name':
- return ;
- case 'Last Name':
- return ;
- case 'Full Address':
- return ;
- case 'Gender':
- return ;
- case 'Random Boolean':
- return ;
- case 'Random Float':
- return ;
- case 'Random Integer':
- return ;
- case 'Random String':
- return ;
- case 'State':
- return ;
- case 'Street Address':
- return ;
- case 'Unix Timestamp':
- return ;
- case 'UTC Timestamp':
- return ;
- case 'UUID':
- return ;
- case 'Zipcode':
- return ;
- case 'Hash':
- return ;
+ case 'Neosync CLI':
+ return ;
+ case 'Environment Variables':
+ return ;
+ case 'Reference':
+ return ;
+ case 'login':
+ return ;
+ case 'whoami':
+ return ;
+ case 'jobs':
+ return ;
+ case 'list':
+ return ;
+ case 'trigger':
+ return ;
+ case 'version':
+ return ;
default:
return ;
}
diff --git a/docs/src/CustomComponents/TransformerPageHeader.tsx b/docs/src/CustomComponents/TransformerPageHeader.tsx
index ecc8aba771..c92aa81817 100644
--- a/docs/src/CustomComponents/TransformerPageHeader.tsx
+++ b/docs/src/CustomComponents/TransformerPageHeader.tsx
@@ -12,7 +12,6 @@ export const TransformerPageHeader = (props: Props) => {
const { title, type, apiRef } = props;
return (