Skip to content

Instantly share code, notes, and snippets.

@Gooseus
Forked from Varriount/s3_get_bucket_obj.nim
Last active October 25, 2017 06:57
Show Gist options
  • Save Gooseus/0ac36280d7e0420360d781058093f882 to your computer and use it in GitHub Desktop.
Save Gooseus/0ac36280d7e0420360d781058093f882 to your computer and use it in GitHub Desktop.
Testing async dispatch and http requests in Nim
# testing async dispatch in pursuit of an efficient async s3 interface
import os, times, math, httpclient, asyncdispatch, asyncfile
let t0 = epochTime()
proc dt() : float =
round(epochTime() - t0,6)
const aws_url = "http://localhost:1234/"
proc s3_get_bucket_object(aws_key:string,bucket:string,obj:string) : Future[string] {.async.} =
var
client = newAsyncHttpClient()
c_prog = false
# do the HMAC here
client.headers = newHttpHeaders({ "Authorization": "TEST KEY:SIG-"&obj })
client.onProgressChanged = proc(total, progress, speed: BiggestInt) {.async.} =
#echo("S3 download progress: ", progress, " of ", total)
#echo("S3 transfer rate: ", speed div 1000, "kb/s")
c_prog = true
try:
echo "get_content for ", obj, " ", dt()
result = await client.getContent(aws_url)
while not c_prog:
poll()
except HttpRequestError:
echo "http error: "
echo getCurrentExceptionMsg()
except:
echo "unknown exception"
echo getCurrentExceptionMsg()
proc download_s3_file(aws_key, bucket, obj, output_path: string): Future[bool] {.async.} =
let data = s3_get_bucket_object(aws_key, bucket, obj)
var file = openAsync(output_path, fmReadWrite)
echo "start file write ", output_path, " ", dt()
await file.write(await data)
echo "finish file write ", output_path, " ", dt()
result = true
let bucket = "my_bucket"
var files: seq[Future[bool]] = @[]
for f in 0..9:
let file_name = "./some_file_" & $(f+1) & ".txt"
files.add(download_s3_file("blah", bucket, file_name, file_name))
var i=0
for file in files:
echo "waiting for file ", i+1, " ", dt()
discard waitFor(file)
echo "finished for file ", i+1, " ", dt()
inc i
echo "Transfer Complete ", dt()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment