From a0ff2b7ca170691ad8c5e2e3606b9ac6914b3b4b Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Mon, 21 Dec 2015 22:14:45 +1100 Subject: [PATCH] (untested WIP) support for creating ec2 instance to build Julia runtime --- src/OCAWS.jl | 2 +- src/ec2.jl | 111 ++++++++++++++++++++++++++++++++ src/lambda.jl | 174 +++++++++++++++++++++++++++++++++++++++++++++----- src/mime.jl | 27 ++++++++ src/ocdict.jl | 35 ++++++++-- src/s3.jl | 17 +++-- src/trap.jl | 2 + 7 files changed, 339 insertions(+), 29 deletions(-) create mode 100644 src/ec2.jl create mode 100644 src/mime.jl diff --git a/src/OCAWS.jl b/src/OCAWS.jl index b7c31a1..a4b32bf 100644 --- a/src/OCAWS.jl +++ b/src/OCAWS.jl @@ -73,7 +73,7 @@ function post_request(aws::AWSConfig, url = aws_endpoint(service, aws[:region]) * resource content = format_query_str(merge(query, "Version" => version)) - merge(aws, @symdict(verb = "POST", service, resource, url, content)) + @symdict(verb = "POST", service, resource, url, content, aws...) end diff --git a/src/ec2.jl b/src/ec2.jl new file mode 100644 index 0000000..bdd3d94 --- /dev/null +++ b/src/ec2.jl @@ -0,0 +1,111 @@ +#==============================================================================# +# ec2.jl +# +# EC2 API. See http://aws.amazon.com/documentation/ec2/ +# +# Copyright Sam O'Connor 2015 - All rights reserved +#==============================================================================# + + +export + +include("mime.jl") + + +function ec2(aws; query) + + do_request(post_request(aws, "ec2", "2014-02-01", query)) +end + + +function ec2_id(aws, name) + + r = ec2(aws, @symdict(Action = "DescribeTags", + "Filter.1.Name" = "key", + "Filter.1.Value.1" = "Name", + "Filter.2.Name" = "value", + "Filter.2.Value.1" = name)) + println(r) + exir(0) +end + + +function create_ec2(aws, name; ImageId="ami-1ecae776", + UserData="", + Policy="", + args...) + + if isa(UserData,Array{Tuple,1}) + UserData=mime_multipart(UserData) + end + + # Delete old instance... + old_id = ec2_id(aws, name) + if old_id != nothing + + ec2(aws, @symdict(Action = "DeleteTags", + "ResourceId.1" = old_id + "Tag.1.Key" = "Name")) + + ec2(aws, @symdict(Action = "TerminateInstances", + "InstanceId.1" = old_id)) + end + + # Set up InstanceProfile Policy... + if Policy != "" + + iam(aws, Action = "CreateRole", + Path = "/", + RoleName = name, + AssumeRolePolicyDocument = """{ + "Version": "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } ] + }""") + + iam(aws, Action = "PutRolePolicy", + RoleName = "$name-role", + PolicyName = "$name-policy", + PolicyDocument = Policy) + + iam(aws, Action = "CreateInstanceProfile", + InstanceProfileName = "$name-role", + Path = "/") + + iam(aws, Action = "AddRoleToInstanceProfile", + InstanceProfileName = "$name-role", + RoleName = "$name-role") + end + + # Deploy instance... + r = ec2(aws, @symdict(Action="RunInstances", + ImageID, + UserData, + "IamInstanceProfile.Name" = "$name-role", + MinCount=1, + MaxCount=1, + args...)) + + println(r) +exit(0) +#FIXME +# dset ec2 id [get $response instancesSet item instanceId] + + ec2(aws, StrDict("Action" => "CreateTags", + "ResourceId.1" => r[:id], + "Tag.1.Key" => "Name", + "Tag.1.Value" => name)) + + return r +end + + + +#==============================================================================# +# End of file. +#==============================================================================# diff --git a/src/lambda.jl b/src/lambda.jl index 320fa7f..225b0fd 100644 --- a/src/lambda.jl +++ b/src/lambda.jl @@ -27,13 +27,14 @@ function lambda(aws, verb; path="", query="") resource = "/2015-03-31/functions/$path" - r = merge(aws, @symdict( + r = @symdict( service = "lambda", url = aws_endpoint("lambda", aws[:region]) * resource, content = query == "" ? "" : json(query), resource, - verb - )) + verb, + aws... + ) r = do_request(r) @@ -76,13 +77,13 @@ function create_lambda(aws, name, S3Key; Timeout=30, args...) - query = merge!(SymDict(args), - @symdict(FunctionName = name, - Code = @symdict(S3Key, S3Bucket), - Handler, - Role, - Runtime, - Timeout)) + query = @symdict(FunctionName = name, + Code = @symdict(S3Key, S3Bucket), + Handler, + Role, + Runtime, + Timeout, + args...) lambda(aws, "POST", query=query) end @@ -122,13 +123,25 @@ end function invoke_lambda(aws, name, args) - r = lambda(aws, "POST", path="$name/invocations", query=args) + @safe try + + r = lambda(aws, "POST", path="$name/invocations", query=args) - if isa(r, Dict) && haskey(r, :errorMessage) - throw(AWSLambdaException(string(name), r[:errorMessage])) + if isa(r, Dict) && haskey(r, :errorMessage) + throw(AWSLambdaException(string(name), r[:errorMessage])) + end + + return r + + catch e + @trap e if e.code == "429" + message = "HTTP 429 $(e.message)\nSee " * + "http://docs.aws.amazon.com/lambda/latest/dg/limits.html" + throw(AWSLambdaException(string(name), message)) + end end - - return r + + @assert false # Unreachable end @@ -418,6 +431,137 @@ end +#------------------------------------------------------------------------------- +# Julia Runtime Build Script +#------------------------------------------------------------------------------- + + +# Build the Julia runtime using a temporary EC2 server. +# Upload the Julia runtime to "aws[:lambda_bucket]/jl_lambda_base.zip". + +function create_jl_lambda_base(;pkg_list::Array{AbstractString,1}=["JSON"], + release = "release-0.4") + server_config = [( + + "cloud_config.txt", "text/cloud-config", + + """packages: + - git + - cmake + - m4 + - patch + - gcc + - gcc-c++ + - gcc-gfortran + - libgfortran + - openssl-devel + """ + + ),( + + "build_julia.sh", "text/x-shellscript", + + """#!/bin/bash + + # Download Julia source code... + git clone git://github.com/JuliaLang/julia.git + cd julia + git checkout $release + + # Configure Julia for the Xeon E5-2680 CPU used by AWS Lambda... + cp Make.inc Make.inc.orig + find='OPENBLAS_TARGET_ARCH=.*$' + repl='OPENBLAS_TARGET_ARCH=SANDYBRIDGE\nMARCH=core-avx-i' + sed s/\$find/\$repl/ < Make.inc.orig > Make.inc + + # Set up /var/task Lambda staging dir... + mkdir -p /var/task/julia + export HOME=/var/task + export JULIA_PKGDIR=/var/task/julia + + # Build and install Julia under /var/task... + make -j2 prefix= DESTDIR=/var/task all + make prefix= DESTDIR=/var/task install + """ + + ),( + + "precomplie_julia.jl", "text/x-shellscript", + + """#!/var/task/bin/julia + Pkg.init() + $(join(["Pkg.add(\"$p\")\nusing $p\n" for p in pkg_list])) + """ + + ),( + + "package_julia.sh", "text/x-shellscript", + + """#!/bin/bash + + # Copy minimal set of files to /task-staging... + mkdir -p /task-staging/bin + mkdir -p /task-staging/lib/julia + cd /task-staging + cp /var/task/bin/julia bin/ + cp -a /var/task/lib/julia/*.so* lib/julia + rm -f lib/julia/*-debug.so + cp -a /usr/lib64/libgfortran.so* lib/julia + cp -a /usr/lib64/libquadmath.so* lib/julia + + # Copy pre-compiled modules to /tmp/task... + cp -a /var/task/julia . + chmod -R a+r julia/lib/ + find julia -name '.git' \ + -o -name '.cache' \ + -o -name '.travis.yml' \ + -o -name '.gitignore' \ + -o -name 'REQUIRE' \ + -o -name 'test' \ + -o -path '*/deps/downloads' \ + -o -path '*/deps/builds' \ + -o -path '*/deps/src' \ + -o -path '*/deps/usr/include' \ + -o -path '*/deps/usr/bin' \ + -o -path '*/deps/usr/lib/*.a' \ + -o -name 'doc' \ + -o -name '*.md' \ + -o -name 'METADATA' \ + | xargs rm -rf + + # Create .zip file... + zip -u --symlinks -r -9 /jl_lambda_base.zip * + + # Upload .zip file to S3... + aws --region $(aws[:region]) \ + s3 cp /jl_lambda_base.zip \ + s3://$(aws[:lambda_bucket])/jl_lambda_base.zip + + # Suspend the build server... + shutdown -h now + """ + )] + + policy = """{ + "Version": "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "s3:PutObject", + "Resource": [ + "arn:aws:s3:::$(aws[:lambda_bucket])/jl_lambda_base.zip", + ] + } ] + }""" + + create_ec2(aws, "ocaws_jl_lambda_build_server", + ImageId = "ami-1ecae776", + InstanceType = "c3.large", + KeyName = "ssh-ec2", + UserData = server_config + Policy = policy) +end + + #==============================================================================# # End of file. diff --git a/src/mime.jl b/src/mime.jl new file mode 100644 index 0000000..f6dcd18 --- /dev/null +++ b/src/mime.jl @@ -0,0 +1,27 @@ +function mime_multipart(parts::Array{Tuple{Any,Any,Any},1}) + + boundary = "=PRZLn8Nm1I82df0Dtj4ZvJi=" + + mime = + """ + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="$boundary" + + --$boundary + """ + + for (filename, content_type, content) in d + + mime *= + """ + Content-Disposition: attachment; filename=$filename + Content-Type: $content_type + Content-Transfer-Encoding: binary + + $content + --$boundary + """ + end + + return mime +end diff --git a/src/ocdict.jl b/src/ocdict.jl index 7b43319..c6cf0b0 100644 --- a/src/ocdict.jl +++ b/src/ocdict.jl @@ -47,16 +47,43 @@ symdict(;args...) = SymDict(args) # b = 2 # @symdict(a,b,c=3,d=4) # Dict(:a=>1,:b=>2,:c=>4,:d=>4) +# +# function f(a; args...) +# b = 2 +# @symdict(a, b, c=3, d=0, args...) +# end +# f(1, d=4) +# Dict(:a=>1,:b=>2,:c=>4,:d=>4) macro symdict(args...) @assert !isa(args[1], Expr) || args[1].head != :tuple - args = [esc(isa(a, Expr) ? a : :($a=$a)) for a in args] - for i in 1:length(args) - args[i].args[1].head = :kw + # Check for "args..." at end... + extra = nothing + if isa(args[end], Expr) && args[end].head == symbol("...") + extra = :(SymDict($(args[end].args[1]))) + args = args[1:end-1] + end + + # Ensure that all args are keyword arg Exprs... + new_args = [] + for a in args + if !isa(a, Expr) + a = :($a=$a) + end + if !isa(a.args[1], Symbol) + a.args[1] = eval(:(symbol($(a.args[1])))) + end + a.head = :kw + push!(new_args, a) + end + + if extra != nothing + :(merge!(symdict($(new_args...)), $extra)) + else + :(symdict($(new_args...))) end - :(symdict($(args...))) end diff --git a/src/s3.jl b/src/s3.jl index de41e46..7a3661d 100644 --- a/src/s3.jl +++ b/src/s3.jl @@ -35,15 +35,14 @@ function s3(aws, verb, bucket=""; resource = "/$path$(query == "" ? "" : "?$query")" url = aws_endpoint("s3", aws[:region], bucket) * resource - r = merge(aws, @symdict(service = "s3", - verb, - url, - resource, - headers, - content, - return_stream)) - - do_request(r) + do_request(@symdict(service = "s3", + verb, + url, + resource, + headers, + content, + return_stream, + aws...)) end diff --git a/src/trap.jl b/src/trap.jl index 566f695..34646f5 100644 --- a/src/trap.jl +++ b/src/trap.jl @@ -36,6 +36,8 @@ macro safe(try_expr::Expr) (try_block, exception, catch_block) = try_expr.args + @assert isa(exception, Symbol) + # Build safe try expression... safe_try_expr = quote try