Skip to content

Commit

Permalink
Add unit tests and separate library commands from cli tool
Browse files Browse the repository at this point in the history
  • Loading branch information
aw committed Jul 13, 2018
1 parent 1586e0a commit 3158357
Show file tree
Hide file tree
Showing 10 changed files with 286 additions and 108 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.modules/
26 changes: 26 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
language: bash
sudo: false
cache: apt
addons:
apt:
packages:
- libc6-dev-i386
- libc6-i386
- linux-libc-dev
- gcc-multilib

env:
matrix:
- BUILD_TYPE=src64 BUILD_VER=18.6
- BUILD_TYPE=src64 BUILD_VER=3.1.11
- BUILD_TYPE=src BUILD_VER=18.6
- BUILD_TYPE=src BUILD_VER=3.1.11

before_script:
- wget http://software-lab.de/picoLisp-${BUILD_VER}.tgz -O /tmp/picolisp.tgz
- cd /tmp; tar -xf /tmp/picolisp.tgz
- cd /tmp/picoLisp/${BUILD_TYPE} && make

script:
- export PATH=$PATH:/tmp/picoLisp
- cd ${TRAVIS_BUILD_DIR} && make check
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## 0.7.0 (2018-07-13)

* Replace (++ L) with (pop 'L) for backward compat with older PicoLisp
* Add unit tests for awscurl library. #2
* Add Travis-CI automated testing

## 0.6.0 (2018-06-19)

* Handle file uploads by accepting '--data @filename' parameter. #4
Expand Down
24 changes: 24 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# picolisp-json Makefile

PIL_MODULE_DIR ?= .modules
REPO_PREFIX ?= https://github.com/aw

# Unit testing
TEST_REPO = $(REPO_PREFIX)/picolisp-unit.git
TEST_DIR = $(PIL_MODULE_DIR)/picolisp-unit/HEAD
TEST_REF = v2.1.0

.PHONY: all

all: check

$(TEST_DIR):
mkdir -p $(TEST_DIR) && \
git clone $(TEST_REPO) $(TEST_DIR) && \
cd $(TEST_DIR) && \
git checkout $(TEST_REF)

check: $(TEST_DIR) run-tests

run-tests:
PIL_NAMESPACES=false ./test.l
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

This command line tool can be used to sign [AWS Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html) requests and make calls to various AWS APIs.

[![GitHub release](https://img.shields.io/github/release/aw/picolisp-awscurl.svg)](https://github.com/aw/picolisp-awscurl) [![Build Status](https://travis-ci.org/aw/picolisp-awscurl.svg?branch=master)](https://travis-ci.org/aw/picolisp-awscurl) [![Dependency](https://img.shields.io/badge/[deps] picolisp--unit-v2.1.0-ff69b4.svg)](https://github.com/aw/picolisp-unit.git)

1. [Requirements](#requirements)
2. [Usage](#usage)
3. [Options](#options)
Expand All @@ -13,9 +15,14 @@ This command line tool can be used to sign [AWS Signature Version 4](https://doc
# Requirements

* `picolisp`: 32-bit or 64-bit v3.1.11+ (tested up to v18.6.5)
* `picolisp-unit`: v2.1.0+ for testing the library
* `openssl`: v1.0.0+ for signing and hashing strings
* `curl`: for sending requests to the AWS APIs

### Notes

The file `libawscurl.l` can be included as a library, rather than using the command line `awscurl.l`. Function are prefixed with `awscurl-` and variables are prefixed with `*Aws_`.

# Usage

## WARNING
Expand Down
110 changes: 2 additions & 108 deletions awscurl.l
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

[de APP_INFO
("name" "awscurl")
("version" "0.6.1")
("version" "0.7.0")
("summary" "PicoLisp AWS CLI tool using OpenSSL and Curl")
("source" "https://github.com/aw/picolisp-awscurl")
("author" "Alexander Williams")
Expand All @@ -26,115 +26,9 @@
()
("Environment variables: AWS_PROFILE, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN/AWS_SECURITY_TOKEN^J Reads ~/.aws/credentials if an environment variable isn't set." ]

(setq
*Aws_method "GET"
*Aws_region "us-east-1"
*Aws_service "ec2"
*Aws_host "ec2.amazonaws.com"
*Aws_endpoint "/"
*Aws_profile (if (sys "AWS_PROFILE") @ "default")
*Aws_access_key (sys "AWS_ACCESS_KEY_ID")
*Aws_secret_key (sys "AWS_SECRET_ACCESS_KEY")
*Aws_session_token (if (sys "AWS_SESSION_TOKEN") @ (sys "AWS_SECURITY_TOKEN")) # AWS_SECURITY_TOKEN is deprecated
*Aws_request_date (dat$ (date T))
*Aws_request_time (pack (mapcar '((S) (pad 2 S)) (time (time T)))) )

### main
[de awscurl-make-curl (Cmd)
(prin (in Cmd (till NIL T) ]

[de awscurl-make-request (Auth Headers)
(make
(link 'curl "--silent" "--request" *Aws_method (pack "https://" *Aws_host *Aws_endpoint (when *Aws_query (pack "?" @))) "--header" Auth)
(mapc '((S) (link "--header" (pack (car S) ": " (cdr S)))) Headers)
(when *Aws_data (link "--data" (cdr *Aws_data)))
(when *Aws_verbose (link "--verbose") ]

[de awscurl-make-auth-header (Headers Signature)
(pack "Authorization: AWS4-HMAC-SHA256 Credential=" *Aws_access_key "/" *Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request, SignedHeaders=" (glue ";" (mapcar car Headers)) ", Signature=" Signature) ]

[de awscurl-make-hmac (Key Payload)
(awscurl-parse-openssl Payload "-mac" "HMAC" "-macopt" Key) ]

[de awscurl-make-signature (Str2sign)
(let (A (awscurl-make-hmac (pack "key:AWS4" *Aws_secret_key) *Aws_request_date)
B (awscurl-make-hmac (pack "hexkey:" A) *Aws_region)
C (awscurl-make-hmac (pack "hexkey:" B) *Aws_service)
D (awscurl-make-hmac (pack "hexkey:" C) "aws4_request") )

(awscurl-make-hmac (pack "hexkey:" D) Str2sign) ]

[de awscurl-make-canonical-hash (Canonical)
(pack
"AWS4-HMAC-SHA256" "^J"
*Aws_request_date "T" *Aws_request_time "Z" "^J"
*Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request" "^J"
(awscurl-parse-openssl Canonical) ]

[de awscurl-make-canonical (Hash Headers)
(pack
*Aws_method "^J"
*Aws_endpoint "^J"
*Aws_query "^J"
(mapcar '((S) (pack (car S) ":" (cdr S) "^J")) Headers) "^J"
(glue ";" (mapcar car Headers)) "^J"
Hash ]

[de awscurl-sort-headers (Sha256)
(by car sort
(make
(link
(cons "host" *Aws_host)
(cons "x-amz-content-sha256" Sha256)
(cons "x-amz-date" (pack *Aws_request_date "T" *Aws_request_time "Z")) )
(when *Aws_data (link (cons "content-length" (if (= "file" (car *Aws_data)) (car (info (cdr *Aws_data))) (length (cdr *Aws_data))))))
(when *Aws_session_token (link (cons "x-amz-security-token" @)))
(mapc link *Aws_headers) ]

[de awscurl-file-openssl (Filename)
(pack (clip (cadr (split (in (make (link 'openssl "dgst" "-sha256" Filename)) (line)) "=") ]

[de awscurl-parse-openssl (Payload . @)
(pack (clip (cadr (split (tc-call-msg (make (link 'openssl "dgst" "-sha256") (when (rest) (chain @))) Payload) "=") ]

[de awscurl-get-credentials ()
(let Credentials (pack (sys "HOME") "/.aws/credentials") # ~/.aws/credentials
(when (info Credentials)
(in Credentials (while (and (from (pack "[" *Aws_profile "]")) (till "["))
(let Keys (clip (split @ "^J" "=")) # remove newlines and split into key=value pairs
(for (L Keys L)
(case (pack (clip (++ L)))
# use the values in the config if they weren't set from environment variables
["aws_access_key_id" (prog1 (clip (++ L)) (unless *Aws_access_key (setq *Aws_access_key (pack @) ]
["aws_secret_access_key" (prog1 (clip (++ L)) (unless *Aws_secret_key (setq *Aws_secret_key (pack @) ]
["aws_session_token" (prog1 (clip (++ L)) (unless *Aws_session_token (setq *Aws_session_token (pack @) ]
(T (++ L) ]

[de awscurl-start ()
(awscurl-get-credentials)
(let (Sha256 (if (= "file" (car *Aws_data)) (awscurl-file-openssl (cdr *Aws_data)) (awscurl-parse-openssl (cdr *Aws_data)))
Headers (awscurl-sort-headers Sha256)
Canonical (awscurl-make-canonical Sha256 Headers)
Str2sign (awscurl-make-canonical-hash Canonical)
Signature (awscurl-make-signature Str2sign)
Auth (awscurl-make-auth-header Headers Signature) )

(awscurl-make-curl (awscurl-make-request Auth Headers)) ]

[de awscurl-data (Value)
(let Data (chop Value)
(setq *Aws_data
(if (and (= "@" (car Data)) (info (pack (cdr Data))))
(cons "file" (pack (cdr Data)))
(cons "data" Value) ]

[de awscurl-headers (Key Value)
(push '*Aws_headers (cons (lowc Key) (pack (clip (chop Value) ]
(load (pack (car (file)) "libawscurl.l"))

### helpers
[de tc-call-msg (Cmd Msg)
(pipe (out Cmd (prin Msg)) (line) ]

[de tc-options (N)
(tab (2 -22 5) " " (car N) (cdr N) ]

Expand Down
113 changes: 113 additions & 0 deletions libawscurl.l
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# picolisp-awscurl - https://github.com/aw/picolisp-awscurl
#
# Copyright (c) 2018 Alexander Williams, Unscramble <license@unscramble.jp>
# License MIT

(setq
*Aws_method "GET"
*Aws_region "us-east-1"
*Aws_service "ec2"
*Aws_host "ec2.amazonaws.com"
*Aws_endpoint "/"
*Aws_profile (if (sys "AWS_PROFILE") @ "default")
*Aws_access_key (sys "AWS_ACCESS_KEY_ID")
*Aws_secret_key (sys "AWS_SECRET_ACCESS_KEY")
*Aws_session_token (if (sys "AWS_SESSION_TOKEN") @ (sys "AWS_SECURITY_TOKEN")) # AWS_SECURITY_TOKEN is deprecated
*Aws_request_date (dat$ (date T))
*Aws_request_time (pack (mapcar '((S) (pad 2 S)) (time (time T)))) )

### main
[de awscurl-make-curl (Cmd)
(prin (in Cmd (till NIL T) ]

[de awscurl-make-request (Auth Headers)
(make
(link 'curl "--silent" "--request" *Aws_method (pack "https://" *Aws_host *Aws_endpoint (when *Aws_query (pack "?" @))) "--header" Auth)
(mapc '((S) (link "--header" (pack (car S) ": " (cdr S)))) Headers)
(when *Aws_data (link "--data" (cdr *Aws_data)))
(when *Aws_verbose (link "--verbose") ]

[de awscurl-make-auth-header (Headers Signature)
(pack "Authorization: AWS4-HMAC-SHA256 Credential=" *Aws_access_key "/" *Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request, SignedHeaders=" (glue ";" (mapcar car Headers)) ", Signature=" Signature) ]

[de awscurl-make-hmac (Key Payload)
(awscurl-parse-openssl Payload "-mac" "HMAC" "-macopt" Key) ]

[de awscurl-make-signature (Str2sign)
(let (A (awscurl-make-hmac (pack "key:AWS4" *Aws_secret_key) *Aws_request_date)
B (awscurl-make-hmac (pack "hexkey:" A) *Aws_region)
C (awscurl-make-hmac (pack "hexkey:" B) *Aws_service)
D (awscurl-make-hmac (pack "hexkey:" C) "aws4_request") )

(awscurl-make-hmac (pack "hexkey:" D) Str2sign) ]

[de awscurl-make-canonical-hash (Canonical)
(pack
"AWS4-HMAC-SHA256" "^J"
*Aws_request_date "T" *Aws_request_time "Z" "^J"
*Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request" "^J"
(awscurl-parse-openssl Canonical) ]

[de awscurl-make-canonical (Hash Headers)
(pack
*Aws_method "^J"
*Aws_endpoint "^J"
*Aws_query "^J"
(mapcar '((S) (pack (car S) ":" (cdr S) "^J")) Headers) "^J"
(glue ";" (mapcar car Headers)) "^J"
Hash ]

[de awscurl-sort-headers (Sha256)
(by car sort
(make
(link
(cons "host" *Aws_host)
(cons "x-amz-content-sha256" Sha256)
(cons "x-amz-date" (pack *Aws_request_date "T" *Aws_request_time "Z")) )
(when *Aws_data (link (cons "content-length" (if (= "file" (car *Aws_data)) (car (info (cdr *Aws_data))) (length (cdr *Aws_data))))))
(when *Aws_session_token (link (cons "x-amz-security-token" @)))
(mapc link *Aws_headers) ]

[de awscurl-file-openssl (Filename)
(pack (clip (cadr (split (in (make (link 'openssl "dgst" "-sha256" Filename)) (line)) "=") ]

[de awscurl-parse-openssl (Payload . @)
(pack (clip (cadr (split (tc-call-msg (make (link 'openssl "dgst" "-sha256") (when (rest) (chain @))) Payload) "=") ]

[de awscurl-get-credentials ()
(let Credentials (pack (sys "HOME") "/.aws/credentials") # ~/.aws/credentials
(when (info Credentials)
(in Credentials (while (and (from (pack "[" *Aws_profile "]")) (till "["))
(let Keys (clip (split @ "^J" "=")) # remove newlines and split into key=value pairs
(for (L Keys L)
(case (pack (clip (pop 'L)))
# use the values in the config if they weren't set from environment variables
["aws_access_key_id" (prog1 (clip (pop 'L)) (unless *Aws_access_key (setq *Aws_access_key (pack @) ]
["aws_secret_access_key" (prog1 (clip (pop 'L)) (unless *Aws_secret_key (setq *Aws_secret_key (pack @) ]
["aws_session_token" (prog1 (clip (pop 'L)) (unless *Aws_session_token (setq *Aws_session_token (pack @) ]
(T (pop 'L) ]

[de awscurl-start ()
(awscurl-get-credentials)
(let (Sha256 (if (= "file" (car *Aws_data)) (awscurl-file-openssl (cdr *Aws_data)) (awscurl-parse-openssl (cdr *Aws_data)))
Headers (awscurl-sort-headers Sha256)
Canonical (awscurl-make-canonical Sha256 Headers)
Str2sign (awscurl-make-canonical-hash Canonical)
Signature (awscurl-make-signature Str2sign)
Auth (awscurl-make-auth-header Headers Signature) )

(awscurl-make-curl (awscurl-make-request Auth Headers)) ]

[de awscurl-data (Value)
(let Data (chop Value)
(setq *Aws_data
(if (and (= "@" (car Data)) (info (pack (cdr Data))))
(cons "file" (pack (cdr Data)))
(cons "data" Value) ]

[de awscurl-headers (Key Value)
(push '*Aws_headers (cons (lowc Key) (pack (clip (chop Value) ]

### helpers
[de tc-call-msg (Cmd Msg)
(pipe (out Cmd (prin Msg)) (line) ]
Loading

0 comments on commit 3158357

Please sign in to comment.