From 8ad6f7181a71dd567188edeae112f89c08bde089 Mon Sep 17 00:00:00 2001 From: Anton-4 <17049058+Anton-4@users.noreply.github.com> Date: Mon, 8 Jul 2024 18:40:29 +0200 Subject: [PATCH 1/5] README, flake, improved examples --- README.md | 13 +- examples/{simple.roc => countCodePoints.roc} | 0 flake.lock | 186 +++++++++++++++++++ flake.nix | 39 ++++ 4 files changed, 236 insertions(+), 2 deletions(-) rename examples/{simple.roc => countCodePoints.roc} (100%) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/README.md b/README.md index 058a63c..0731105 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ -# Work in progress! +Convenient functions for working with unicode. -Someday this will be a useful collection of Unicode operations, but for right now it's extremely WIP. +⚠️ This package has only gone through limited testing. [Make an issue](https://github.com/roc-lang/unicode/issues) when you hit a bug. + +:eyes: [**examples**](https://github.com/roc-lang/unicode/tree/main/examples) + +:book: **documentation**: TODO + + +## Learning about Unicode + +The string/unicode rabbit hole goes deep, we have a [good overview](https://www.roc-lang.org/builtins/Str) (scroll to the unicode section). diff --git a/examples/simple.roc b/examples/countCodePoints.roc similarity index 100% rename from examples/simple.roc rename to examples/countCodePoints.roc diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..13eda32 --- /dev/null +++ b/flake.lock @@ -0,0 +1,186 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixgl": { + "inputs": { + "flake-utils": [ + "roc", + "flake-utils" + ], + "nixpkgs": [ + "roc", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1710868679, + "narHash": "sha256-V1o2bCZdeYKP/0zgVp4EN0KUjMItAMk6J7SvCXUI5IU=", + "owner": "guibou", + "repo": "nixGL", + "rev": "d709a8abcde5b01db76ca794280745a43c8662be", + "type": "github" + }, + "original": { + "owner": "guibou", + "repo": "nixGL", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1712163089, + "narHash": "sha256-Um+8kTIrC19vD4/lUCN9/cU9kcOsD1O1m+axJqQPyMM=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "fd281bd6b7d3e32ddfa399853946f782553163b5", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "rev": "fd281bd6b7d3e32ddfa399853946f782553163b5", + "type": "github" + } + }, + "roc": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils_2", + "nixgl": "nixgl", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1719383847, + "narHash": "sha256-MhKnV9IHpeMbiARX5e+6LLXYJenNNx0YNFGCVtmufu8=", + "owner": "roc-lang", + "repo": "roc", + "rev": "f8c6786502bc253ab202a55e2bccdcc693e549c8", + "type": "github" + }, + "original": { + "owner": "roc-lang", + "repo": "roc", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "roc", + "nixpkgs" + ], + "roc": "roc" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": [ + "roc", + "flake-utils" + ], + "nixpkgs": [ + "roc", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712369449, + "narHash": "sha256-tbWug3uXPlSm1j0xD80Y3xbP+otT6gLnQo1e/vQat48=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "41b3b080cc3e4b3a48e933b87fc15a05f1870779", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..b0830a1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,39 @@ +{ + description = "Unicode package devShell flake"; + + inputs = { + roc.url = "github:roc-lang/roc"; + nixpkgs.follows = "roc/nixpkgs"; + + # to easily make configs for multiple architectures + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils, roc }: + let supportedSystems = [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" "aarch64-linux" ]; + in flake-utils.lib.eachSystem supportedSystems (system: + let + pkgs = import nixpkgs { inherit system; }; + rocPkgs = roc.packages.${system}; + + linuxInputs = with pkgs; + lib.optionals stdenv.isLinux [ + ]; + + darwinInputs = with pkgs; + lib.optionals stdenv.isDarwin + (with pkgs.darwin.apple_sdk.frameworks; [ + ]); + + sharedInputs = (with pkgs; [ + rocPkgs.cli + ]); + in { + + devShell = pkgs.mkShell { + buildInputs = sharedInputs ++ darwinInputs ++ linuxInputs; + }; + + formatter = pkgs.nixpkgs-fmt; + }); +} From 191b22b6b415ab9fcb5121251b79c91da2f73eff Mon Sep 17 00:00:00 2001 From: Anton-4 <17049058+Anton-4@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:50:12 +0200 Subject: [PATCH 2/5] improved examples --- examples/countCodePoints.roc | 18 -------------- .../{visualWidth.roc => getVisualWidth.roc} | 7 +++--- examples/nrOfCodePoints.roc | 24 +++++++++++++++++++ 3 files changed, 28 insertions(+), 21 deletions(-) delete mode 100644 examples/countCodePoints.roc rename examples/{visualWidth.roc => getVisualWidth.roc} (82%) mode change 100644 => 100755 create mode 100755 examples/nrOfCodePoints.roc diff --git a/examples/countCodePoints.roc b/examples/countCodePoints.roc deleted file mode 100644 index 973ffd6..0000000 --- a/examples/countCodePoints.roc +++ /dev/null @@ -1,18 +0,0 @@ -app [main] { - pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.11.0/SY4WWMhWQ9NvQgvIthcv15AUeA7rAIJHAHgiaSHGhdY.tar.br", - unicode: "../package/main.roc", # use release URL (ends in tar.br) for local example, see github.com/roc/unicode/releases -} - -import pf.Stdout -import unicode.CodePoint - -word = "ẇ͓̞͒͟͡ǫ̠̠̉̏͠͡ͅr̬̺͚̍͛̔͒͢d̠͎̗̳͇͆̋̊͂͐" - -maybeLength : Result U64 CodePoint.Utf8ParseErr -maybeLength = word |> Str.toUtf8 |> CodePoint.parseUtf8 |> Result.map List.len - -main = - when maybeLength is - Ok count -> Stdout.line "\n\nThere are a total of $(Num.toStr count) code points in $(word)\n\n" - Err _ -> crash "ERROR: Unable to parse $(word)!" - diff --git a/examples/visualWidth.roc b/examples/getVisualWidth.roc old mode 100644 new mode 100755 similarity index 82% rename from examples/visualWidth.roc rename to examples/getVisualWidth.roc index 49b0b86..5549cf4 --- a/examples/visualWidth.roc +++ b/examples/getVisualWidth.roc @@ -8,9 +8,10 @@ import unicode.CodePoint word = "世界" -visualWidth : Result U32 CodePoint.Utf8ParseErr -visualWidth = - word +## Get the display width (in amount of characters) of a Str +getVisualWidth : Str -> Result U32 CodePoint.Utf8ParseErr +getVisualWidth = \str -> + str |> Str.toUtf8 |> CodePoint.parseUtf8 |> Result.map (\lst -> List.map lst CodePoint.visualWidth) diff --git a/examples/nrOfCodePoints.roc b/examples/nrOfCodePoints.roc new file mode 100755 index 0000000..12d1ed3 --- /dev/null +++ b/examples/nrOfCodePoints.roc @@ -0,0 +1,24 @@ +app [main] { + pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.11.0/SY4WWMhWQ9NvQgvIthcv15AUeA7rAIJHAHgiaSHGhdY.tar.br", + unicode: "../package/main.roc", # use release URL (ends in tar.br) for local example, see github.com/roc/unicode/releases +} + +import pf.Stdout +import pf.Task +import unicode.CodePoint exposing [Utf8ParseErr] + +## Get the number of code points for a given Str +nrOfCodePoints : Str -> Result U64 Utf8ParseErr +nrOfCodePoints = \str -> + str |> Str.toUtf8 |> CodePoint.parseUtf8 |> Result.map List.len + +main = + word = "ẇ͓̞͒͟͡ǫ̠̠̉̏͠͡ͅr̬̺͚̍͛̔͒͢d̠͎̗̳͇͆̋̊͂͐" + + when nrOfCodePoints word is + Ok nr -> + Stdout.line "String \"$(word)\" consists of $(Num.toStr nr) code points." + + Err _ -> + Task.err (Exit 1 "Failed to parse string $(word) as Utf8.") + From b96cb6182ba4656a662cb83e2d97e3edf56ff2be Mon Sep 17 00:00:00 2001 From: Anton-4 <17049058+Anton-4@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:53:19 +0200 Subject: [PATCH 3/5] Update getVisualWidth.roc --- examples/getVisualWidth.roc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/getVisualWidth.roc b/examples/getVisualWidth.roc index 5549cf4..326774f 100755 --- a/examples/getVisualWidth.roc +++ b/examples/getVisualWidth.roc @@ -22,4 +22,4 @@ main = Ok width -> Stdout.line "\n\nThe word $(word) will be displayed with the width of $(Num.toStr width) characters on most UIs.\n\n" Err _ -> crash "ERROR: Unable to parse $(word)!" -expect visualWidth == Ok 4 +expect getVisualWidth == Ok 4 From 4339143f8431f186ca25d3af24018c9af2c0daa6 Mon Sep 17 00:00:00 2001 From: Anton-4 <17049058+Anton-4@users.noreply.github.com> Date: Fri, 12 Jul 2024 16:19:28 +0200 Subject: [PATCH 4/5] fixed errors, aaded nix tests --- .github/workflows/tests-nix.yaml | 30 ++++++++++++++++++++++++++++++ .github/workflows/tests.yaml | 4 ++-- examples/getVisualWidth.roc | 4 ++-- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/tests-nix.yaml diff --git a/.github/workflows/tests-nix.yaml b/.github/workflows/tests-nix.yaml new file mode 100644 index 0000000..60bb92e --- /dev/null +++ b/.github/workflows/tests-nix.yaml @@ -0,0 +1,30 @@ +on: + pull_request: + workflow_dispatch: + + # this cancels workflows currently in progress if you start a new one +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-examples-nix: + runs-on: [ubuntu-22.04] + steps: + - uses: actions/checkout@v3 + + # install nix + - uses: cachix/install-nix-action@v23 + with: + nix_path: nixpkgs=channel:nixos-unstable + + # Run all tests + - name: Run all tests + run: nix develop -c sh -c 'export ROC=roc ./ci/all_tests.sh' + + - name: Run all tests and capture output to check for crash later + run: nix develop -c sh -c 'export ROC=roc && ./ci/all_tests.sh 2>&1 | tee all_tests_output.log' + + # Workaround for https://github.com/roc-lang/roc/issues/6688 + - name: Check if crash occurred + run: grep -qv "crashed" all_tests_output.log \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a2bad23..919a650 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -9,8 +9,8 @@ concurrency: jobs: - test-examples: - runs-on: [ubuntu-20.04] + test-examples-ubuntu: + runs-on: [ubuntu-22.04] steps: - uses: actions/checkout@v3 diff --git a/examples/getVisualWidth.roc b/examples/getVisualWidth.roc index 326774f..69c6fdd 100755 --- a/examples/getVisualWidth.roc +++ b/examples/getVisualWidth.roc @@ -18,8 +18,8 @@ getVisualWidth = \str -> |> Result.map List.sum main = - when visualWidth is + when (getVisualWidth word) is Ok width -> Stdout.line "\n\nThe word $(word) will be displayed with the width of $(Num.toStr width) characters on most UIs.\n\n" Err _ -> crash "ERROR: Unable to parse $(word)!" -expect getVisualWidth == Ok 4 +expect (getVisualWidth word) == Ok 4 From f86828e9824289ea2cdb62e507d1680c82ef118d Mon Sep 17 00:00:00 2001 From: Anton-4 <17049058+Anton-4@users.noreply.github.com> Date: Fri, 12 Jul 2024 18:06:37 +0200 Subject: [PATCH 5/5] forgot && --- .github/workflows/tests-nix.yaml | 2 +- .gitignore | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-nix.yaml b/.github/workflows/tests-nix.yaml index 60bb92e..933c66d 100644 --- a/.github/workflows/tests-nix.yaml +++ b/.github/workflows/tests-nix.yaml @@ -20,7 +20,7 @@ jobs: # Run all tests - name: Run all tests - run: nix develop -c sh -c 'export ROC=roc ./ci/all_tests.sh' + run: nix develop -c sh -c 'export ROC=roc && ./ci/all_tests.sh' - name: Run all tests and capture output to check for crash later run: nix develop -c sh -c 'export ROC=roc && ./ci/all_tests.sh 2>&1 | tee all_tests_output.log' diff --git a/.gitignore b/.gitignore index c60a4ab..a8d48c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Ignore the example binaries -examples/simple -examples/visualWidth +examples/nrOfCodePoints +examples/getVisualWidth # Ignore the generated files generated-docs