Release v1.17.7

- Add [system defined constants](https://mithrandie.github.io/csvq/reference/system-defined-constant.html). - Handle NaN and Infinity. - Add several numeric and datetime functions. - Fix some bugs.
mithrandie · Jul 3, 2022 · d9d8655 · d9d8655
2 parents 4442b67 + cb9dec2
commit d9d8655
Show file tree

Hide file tree

Showing 110 changed files with 6,423 additions and 4,495 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # Change Log
 
+## Version 1.17.7
+
+Released on Jul 3, 2022
+
+- Add [system defined constants](https://mithrandie.github.io/csvq/reference/system-defined-constant.html).
+- Handle NaN and Infinity.
+- Add several numeric and datetime functions.
+- Fix some bugs.
+
 ## Version 1.17.6
 
 Released on Jun 30, 2022

diff --git a/Makefile b/Makefile
@@ -2,6 +2,7 @@ GOPATH := $(shell pwd)/build
 BINARY := csvq
 RELEASE_ARCH := darwin/amd64 darwin/arm64 linux/amd64 linux/386 linux/arm linux/arm64 freebsd/amd64 freebsd/386 freebsd/arm netbsd/amd64 netbsd/386 netbsd/arm openbsd/amd64 openbsd/386 windows/amd64 windows/386
 PRERELEASE_ARCH := darwin/amd64 darwin/arm64 linux/amd64 windows/amd64
+BUILD_TAGS := -tags urfave_cli_no_docs
 
 ifneq ($(shell command -v git && git remote -v 2>/dev/null | grep mithrandie/csvq.git && echo true),true)
 	VERSION := $(shell git describe --tags --always 2>/dev/null)
@@ -19,11 +20,11 @@ $(BINARY): build
 
 .PHONY: build
 build:
-	go build -trimpath $(LDFLAGS) -o $(GOPATH)/bin/
+	go build $(BUILD_TAGS) -trimpath $(LDFLAGS) -o $(GOPATH)/bin/
 
 .PHONY: install
 install:
-	go install $(LDFLAGS)
+	go install $(BUILD_TAGS) -trimpath $(LDFLAGS)
 
 .PHONY: clean
 clean:
@@ -35,7 +36,7 @@ build-all:
 	for TARGET in $(RELEASE_ARCH); \
 	do \
 		set -- $$TARGET; \
-		GOOS=$$1 GOARCH=$$2 go build -trimpath $(LDFLAGS) -o "dist/$(BINARY)-$(VERSION)-$${1}-$${2}/"; \
+		GOOS=$$1 GOARCH=$$2 go build $(BUILD_TAGS) -trimpath $(LDFLAGS) -o "dist/$(BINARY)-$(VERSION)-$${1}-$${2}/"; \
 	done
 
 .PHONY: build-pre-release
@@ -44,7 +45,7 @@ build-pre-release:
 	for TARGET in $(PRERELEASE_ARCH); \
 	do \
 		set -- $$TARGET; \
-		GOOS=$$1 GOARCH=$$2 go build -trimpath $(LDFLAGS) -o "dist/$(BINARY)-$(VERSION)-$${1}-$${2}/"; \
+		GOOS=$$1 GOARCH=$$2 go build $(BUILD_TAGS) -trimpath $(LDFLAGS) -o "dist/$(BINARY)-$(VERSION)-$${1}-$${2}/"; \
 	done
 
 .PHONY: dist -

diff --git a/docs/_includes/side_menu.html b/docs/_includes/side_menu.html
@@ -35,6 +35,7 @@
                   <li><a href="{{ '/reference/flag.html' | relative_url }}">Flag</a></li>
                   <li><a href="{{ '/reference/environment-variable.html' | relative_url }}">Environment Variable</a></li>
                   <li><a href="{{ '/reference/runtime-information.html' | relative_url }}">Runtime Information</a></li>
+                  <li><a href="{{ '/reference/system-defined-constant.html' | relative_url }}">System Defined Constant</a></li>
                   <li><a href="{{ '/reference/json.html' | relative_url }}">JSON</a></li>
                 </ul>
               </div>

diff --git a/docs/_posts/2006-01-02-cast-functions.md b/docs/_posts/2006-01-02-cast-functions.md
@@ -6,14 +6,14 @@ category: reference
 
 # Cast Functions
 
-| name | description |
-| :- | :- |
-| [STRING](#string) | Convert a value to a string |
-| [INTEGER](#integer) | Convert a value to an integer |
-| [FLOAT](#float) | Convert a value to a float |
+| name                  | description                   |
+|:----------------------|:------------------------------|
+| [STRING](#string)     | Convert a value to a string   |
+| [INTEGER](#integer)   | Convert a value to an integer |
+| [FLOAT](#float)       | Convert a value to a float    |
 | [DATETIME](#datetime) | Convert a value to a datetime |
-| [BOOLEAN](#boolean) | Convert a value to a boolean |
-| [TERNARY](#ternary) | Convert a value to a ternary |
+| [BOOLEAN](#boolean)   | Convert a value to a boolean  |
+| [TERNARY](#ternary)   | Convert a value to a ternary  |
 
 ## Definitions
 
@@ -32,14 +32,14 @@ _return_
 
 Convert _value_ to a string.
 
-| value type | description |
-| :- | :- |
-| Integer  | An integer value is converted to a string representing a decimal integer. |
-| Float    | A float value is converted to a string representing a floating-point decimal. |
-| Datetime | A datetime value is converted to a string formatted with RFC3339 with Nano Seconds. |
-| Boolean  | A boolean value is converted to either 'true' or 'false'. |
-| Ternary  | A ternaly value is converted to any one string of 'TRUE', 'FALSE' and 'UNKNOWN'. |
-| Null     | A null value is kept as it is. |
+| Type     | Value after conversion                          |
+|:---------|:------------------------------------------------|
+| Integer  | String representing a decimal integer           |
+| Float    | String representing a floating-point decimal    |
+| Datetime | String formatted with RFC3339 with Nano Seconds |
+| Boolean  | 'true' or 'false'                               |
+| Ternary  | 'TRUE', 'FALSE' and 'UNKNOWN'                   |
+| Null     | Null                                            |
 
 
 ### INTEGER
@@ -57,14 +57,17 @@ _return_
 
 Convert _value_ to an integer.
 
-| value type | description |
-| :- | :- |
-| String   | If a string is a representation of a decimal integer or its exponential notation, then it is converted to an integer. If a string is a representation of a floating-point decimal or its exponential notation, then it is converted and rounded to an integer. Otherwise it is converted to a null. |
-| Float    | A float value is rounded to an integer. |
-| Datetime | A datetime value is converted to an integer representing its unix time. |
-| Boolean  | A boolean value is converted to a null. |
-| Ternary  | A ternaly value is converted to a null. |
-| Null     | A null value is kept as it is. |
+| Type     | Value                                                                  | Value after conversion                   |
+|:---------|:-----------------------------------------------------------------------|:-----------------------------------------|
+| String   | Representation of a decimal integer                                    | Integer represented by the string        |
+|          | Representation of a floating-point decimal or its exponential notation | Integer with decimal places rounded down |
+|          | Other values                                                           | Null                                     |
+| Float    | +Inf, -Inf, NaN                                                        | Null                                     |
+|          | Other values                                                           | Integer with decimal places rounded down |
+| Datetime |                                                                        | Integer representing its unix time       |
+| Boolean  |                                                                        | Null                                     |
+| Ternary  |                                                                        | Null                                     |
+| Null     |                                                                        | Null                                     |
 
 ### FLOAT
 {: #float}
@@ -81,14 +84,18 @@ _return_
 
 Convert _value_ to a float.
 
-| value type | description |
-| :- | :- |
-| String   | If a string is a representation of a floating-point decimal or its exponential notation, then it is converted to a float. Otherwise it is converted to a null. |
-| Integer  | An integer value is converted to a float. |
-| Datetime | A datetime value is converted to a float representing its unix time. |
-| Boolean  | A boolean value is converted to a null. |
-| Ternary  | A ternary value is converted to a null. |
-| Null     | A null value is kept as it is. |
+| Type     | Value                                                                  | Value after conversion           |
+|:---------|:-----------------------------------------------------------------------|:---------------------------------|
+| String   | Representation of a floating-point decimal or its exponential notation | Float represented by the string  |
+|          | 'Inf', '+Inf'                                                          | +Inf                             |
+|          | '-Inf'                                                                 | -Inf                             |
+|          | 'NaN'                                                                  | NaN                              |
+|          | Other values                                                           | Null                             |
+| Integer  |                                                                        | Float equivalent to the integer  |
+| Datetime |                                                                        | Float representing its unix time |
+| Boolean  |                                                                        | Null                             |
+| Ternary  |                                                                        | Null                             |
+| Null     |                                                                        | Null                             |
 
 ### DATETIME
 {: #datetime}
@@ -113,41 +120,45 @@ _return_
 
 Convert _value_ to a datetime.
 
-| value type | description |
-| :- | :- |
-| String   | If a string value is a representation of an integer, or a float value, then it is converted to a datetime represented by the number as a unix time. If a string value is formatted as a datetime, then it is convered to a datetime. Otherwise, it is converted to a null. |
-| Integer  | An integer value is converted to a datetime represented by the integer value as a unix time. |
-| Float    | A float value is converted to a datetime represented by the float value as a unix time. |
-| Boolean  | A boolean value is converted to a null. |
-| Ternary  | A ternaly value is converted to a null. |
-| Null     | A null value is kept as it is. |
+| Type    | Value                                                                  | Value after conversion                                   |
+|:--------|:-----------------------------------------------------------------------|:---------------------------------------------------------|
+| String  | Datetime Formats                                                       | Datetime represented by the string                       |
+|         | Representation of a decimal integer                                    | Datetime represented by the integer value as a unix time |
+|         | Representation of a floating-point decimal or its exponential notation | Datetime represented by the float value as a unix time   |
+|         | Other values                                                           | Null                                                     |
+| Integer |                                                                        | Datetime represented by the integer value as a unix time |
+| Float   | +Inf, -Inf, NaN                                                        | Null                                                     |
+|         | Other values                                                           | Datetime represented by the float value as a unix time   |
+| Boolean |                                                                        | Null                                                     |
+| Ternary |                                                                        | Null                                                     |
+| Null    |                                                                        | Null                                                     |
 
 #### Format of string to be interpreted as datetime
 {: #format-of-string-as-datetime}
 
 Strings of the form passed by the [--datetime-format option]({{ '/reference/command.html#options' | relative_url }}) and defined in the [configuration files]({{ '/reference/command.html#configurations' | relative_url }}),  or the following forms can be converted to datetime values.
 
-| DateFormat | Example |
-| :- | :- |
+| DateFormat | Example    |
+|:-----------|:-----------|
 | YYYY-MM-DD | 2012-03-15 |
 | YYYY/MM/DD | 2012/03/15 |
-| YYYY-M-D   | 2012-3-15 |
-| YYYY/M/D   | 2012/3/15 |
+| YYYY-M-D   | 2012-3-15  |
+| YYYY/M/D   | 2012/3/15  |
 
 &nbsp;
 
-| DatetimeFormat | Example |
-| :- | :- |
-| DateFormat | 2012-03-15 |
-| DateFormat hh:mm:ss(.NanoSecods) | 2012-03-15 12:03:01<br />2012-03-15 12:03:01.123456789 |
-| DateFormat hh:mm:ss(.NanoSecods) ±hh:mm | 2012-03-15 12:03:01 -07:00 |
-| DateFormat hh:mm:ss(.NanoSecods) ±hhmm | 2012-03-15 12:03:01 -0700 |
-| DateFormat hh:mm:ss(.NanoSecods) TimeZone | 2012-03-15 12:03:01 PST |
-| YYYY-MM-DDThh:mm:ss(.NanoSeconds) | 2012-03-15T12:03:01 |
-| RFC3339 | 2012-03-15T12:03:01-07:00 |
-| RFC3339 with Nano Seconds | 2012-03-15T12:03:01.123456789-07:00 |
-| RFC822 | 03 Mar 12 12:03 PST |
-| RFC822 with Numeric Zone | 03 Mar 12 12:03 -0700 |
+| DatetimeFormat                            | Example                                                |
+|:------------------------------------------|:-------------------------------------------------------|
+| DateFormat                                | 2012-03-15                                             |
+| DateFormat hh:mm:ss(.NanoSecods)          | 2012-03-15 12:03:01<br />2012-03-15 12:03:01.123456789 |
+| DateFormat hh:mm:ss(.NanoSecods) ±hh:mm   | 2012-03-15 12:03:01 -07:00                             |
+| DateFormat hh:mm:ss(.NanoSecods) ±hhmm    | 2012-03-15 12:03:01 -0700                              |
+| DateFormat hh:mm:ss(.NanoSecods) TimeZone | 2012-03-15 12:03:01 PST                                |
+| YYYY-MM-DDThh:mm:ss(.NanoSeconds)         | 2012-03-15T12:03:01                                    |
+| RFC3339                                   | 2012-03-15T12:03:01-07:00                              |
+| RFC3339 with Nano Seconds                 | 2012-03-15T12:03:01.123456789-07:00                    |
+| RFC822                                    | 03 Mar 12 12:03 PST                                    |
+| RFC822 with Numeric Zone                  | 03 Mar 12 12:03 -0700                                  |
 
 > Timezone abbreviations such as "PST" may not work properly depending on your environment,
 > so you should use timezone offset such as "-07:00" as possible.
@@ -167,14 +178,22 @@ _boolean_
 
 Convert _value_ to a boolean.
 
-| value type | description |
-| :- | :- |
-| String   | If a string value is any of '1', 't', 'T', 'TRUE', 'true' and 'True', then it is converted to true. If a string value is any of '0', 'f', 'F', 'FALSE' and 'false', then it is converted to false. Otherwise it is converted to a null. |
-| Integer  | If an integer value is 1, then it is converted to true. If an integer value is 0, then it is converted to false. Otherwise it is converted to a null. |
-| Float    | If a float value is 1, then it is converted to true. If a float value is 0, then it is converted to false. Otherwise it is converted to a null. |
-| Datetime | A datetime value is converted to a null. |
-| Ternary  | If a ternary value is TRUE, then it is converted to true. If a ternary value is FALSE, then it is converted to false. Otherwise it is converted to a null. |
-| Null     | A null value is kept as it is. |
+| Type     | Value             | Value after conversion |
+|:---------|:------------------|:-----------------------|
+| String   | '1', 't', 'true'  | true                   |
+| String   | '0', 'f', 'false' | false                  |
+| String   | Other values      | Null                   |
+| Integer  | 1                 | true                   |
+| Integer  | 0                 | false                  |
+| Integer  | Other values      | Null                   |
+| Float    | 1                 | true                   |
+| Float    | 0                 | false                  |
+| Float    | Other values      | Null                   |
+| Datetime |                   | Null                   |
+| Ternary  | TRUE              | true                   |
+|          | FALSE             | false                  |
+|          | UNKNOWN           | Null                   |
+| Null     |                   | Null                   |
 
 ### TERNARY
 {: #ternary}
@@ -191,11 +210,18 @@ _return_
 
 Convert _value_ to a ternary.
 
-| value type | description |
-| :- | :- |
-| String   | If a string value is any of '1', 't', 'T', 'TRUE', 'true' and 'True', then it is converted to TRUE. If a string value is any of '0', 'f', 'F', 'FALSE' and 'false', then it is converted to FALSE. Otherwise it is converted to UNKNOWN. |
-| Integer  | If an integer value is 1, then it is converted to TRUE. If an integer value is 0, then it is converted to FALSE. Otherwise it is converted to UNKNOWN. |
-| Float    | If a float value is 1, then it is converted to TRUE. If a float value is 0, then it is converted to FALSE. Otherwise it is converted to UNKNOWN. |
-| Datetime | A datetime value is converted to UNKNOWN. |
-| Boolean  | If a boolean value is true, then it is converted to TRUE. If a boolean value is false, then it is converted to FALSE. |
-| Null     | A null value is converted to UNKNOWN. |
+| Type     | Value             | Value after conversion |
+|:---------|:------------------|:-----------------------|
+| String   | '1', 't', 'true'  | TRUE                   |
+| String   | '0', 'f', 'false' | FALSE                  |
+| String   | Other values      | UNKNOWN                |
+| Integer  | 1                 | TRUE                   |
+| Integer  | 0                 | FALSE                  |
+| Integer  | Other values      | UNKNOWN                |
+| Float    | 1                 | TRUE                   |
+| Float    | 0                 | FALSE                  |
+| Float    | Other values      | UNKNOWN                |
+| Datetime |                   | UNKNOWN                |
+| Boolean  | true              | TRUE                   |
+|          | false             | FALSE                  |
+| Null     |                   | UNKNOWN                |
diff --git a/docs/_posts/2006-01-02-command.md b/docs/_posts/2006-01-02-command.md
@@ -259,6 +259,9 @@ When this option is not specified, the file specified by the _--out_ option will
 --pretty-print, -P
 : Make JSON output easier to read in query results.
 
+--scientific-notation -SN
+: Use Scientific Notation for large exponents in output.
+
 --east-asian-encoding, -W
 : Count ambiguous characters as fullwidth. If not, then that characters are counted as halfwidth.
 

diff --git a/docs/_posts/2006-01-02-datetime-functions.md b/docs/_posts/2006-01-02-datetime-functions.md
@@ -46,6 +46,7 @@ category: reference
 | [TIME_DIFF](#time_diff) | Return the difference of time between two datetime values as seconds |
 | [TIME_NANO_DIFF](#time_nano_diff) | Return the difference of time between two datetime values as nanoseconds |
 | [UTC](#utc) | Return a datetime in UTC |
+| [MILLI_TO_DATETIME](#milli_to_datetime) | Convert an integer representing Unix milliseconds to a datetime |
 | [NANO_TO_DATETIME](#nano_to_datetime) | Convert an integer representing Unix nano time to a datetime |
 
 ## Definitions
@@ -701,6 +702,20 @@ _return_
 
 Returns the datetime value of _datetime_ in UTC.
 
+### MILLI_TO_DATETIME
+{: #milli_to_datetime}
+
+```
+MILLI_TO_DATETIME(unix_milliseconds)
+```
+
+_unix_milliseconds_
+: [integer]({{ '/reference/value.html#integer' | relative_url }})
+
+_return_
+: [datetime]({{ '/reference/value.html#datetime' | relative_url }})
+
+Converts an integer representing Unix milliseconds to a datetime.
 
 ### NANO_TO_DATETIME
 {: #nano_to_datetime}