Created
March 23, 2010 15:35
-
-
Save FND/341302 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
*.pyc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import csv | |
def dictify(filename): | |
""" | |
turn data from CSV file into a list of dictionaries | |
""" | |
f = open(filename, "rU") | |
items = [] | |
for row in unicode_csv_reader(f): | |
try: | |
item = dict([(k, row[i]) for i, k in enumerate(keys)]) | |
item["_source"] = filename # XXX: not a generic requirement | |
items.append(item) | |
except NameError: # first row | |
keys = row | |
f.close() | |
return items | |
# Unicode fixes as per documentation | |
def unicode_csv_reader(data, dialect=csv.excel, **kwargs): | |
""" | |
expects Unicode input, temporarily encoding it as UTF-8 | |
""" | |
reader = csv.reader(utf8_encoder(data), dialect=dialect, **kwargs) | |
for row in reader: | |
yield [unicode(cell, "utf-8", "replace") for cell in row] | |
def utf8_encoder(data): | |
for line in data: | |
yield line |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
""" | |
Usage: | |
$ ./csv2tiddlers.py <host> <bag> <filename> <titlefields> <textfields> <fieldmap> <tagsmap> | |
Example: | |
$ ./csv2tiddlers.py http://localhost:8080 common sample.csv "colA;colC" "colD;colE" "colB=foo;colF=bar" | |
""" | |
import sys | |
from csv2dict import dictify | |
from tiddler import put | |
def main(args): | |
""" | |
arguments are host, bag, filename, titlefields, textfields and fieldmap | |
fields are a list of semicolon-separated column names whose values will be | |
concatenated for the respective tiddler attribute | |
fieldmap provides a way to translate column names to tiddler fields | |
tagsmap provides a way to translate column values into tags | |
""" | |
args = [unicode(arg, "utf-8", "replace") for arg in args] | |
try: | |
host = args[1] | |
bag = args[2] | |
filename = args[3] | |
titlefields = args[4].split(";") | |
textfields = args[5].split(";") | |
except IndexError, exc: | |
print "ERROR: missing argument", exc | |
return False | |
try: | |
fieldmap = dict([i.split("=") for i in args[6].split(";")]) | |
except IndexError, exc: | |
fieldmap = {} | |
try: | |
tagsmap = args[7].split(";") | |
except IndexError, exc: | |
tagsmap = [] | |
items = dictify(filename) | |
put_tiddlers(items, host, bag, titlefields, textfields, fieldmap, tagsmap) | |
return True | |
def put_tiddlers(items, host, bag, titlefields, textfields, fieldmap, tagsmap): | |
for item in items: | |
values = [] | |
for field in titlefields: | |
values.append(item[field]) | |
title = "_".join(values) | |
try: | |
values = [] | |
for field in textfields: # TODO: DRY | |
values.append(item[field]) | |
text = "\n\n".join(values) | |
except KeyError: | |
text = None # XXX: invalid? | |
for column, field in fieldmap.items(): # XXX: inefficient!? | |
try: # XXX: temporary workaround | |
item[field] = item[column] | |
del item[column] | |
except KeyError: | |
pass | |
try: | |
tagsCol, tagsDelim = tagsmap | |
tags = item[tagsCol].split(tagsDelim) | |
except (ValueError, KeyError), exc: | |
tags = [] | |
tiddler = { | |
"text": text, | |
"tags": tags, | |
"fields": item | |
} | |
message = "sending tiddler: %s ..." % title.encode("utf-8") | |
sys.stdout.write(message) | |
status = put(host, bag, title, tiddler) | |
print "\t[%s]" % status | |
if __name__ == "__main__": | |
status = not main(sys.argv) | |
sys.exit(status) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
PUT a tiddler. | |
""" | |
import urllib2 | |
import httplib2 | |
try: | |
import json | |
except ImportError, exc: | |
import simplejson as json | |
def put(host, bag, title, tiddler): | |
""" | |
send a dictionary representing a tiddler to a TiddlyWeb server | |
""" | |
http = httplib2.Http() | |
bag = urllib2.quote(bag.encode("utf-8")) | |
title = urllib2.quote(title.encode("utf-8")) | |
uri = "%s/bags/%s/tiddlers/%s" % (host, bag, title) | |
response, content = http.request( | |
uri, | |
method="PUT", | |
headers={ "Content-Type": "application/json" }, | |
body=json.dumps(tiddler) | |
) | |
return response["status"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import sys | |
import urllib2 | |
import httplib2 | |
URI = "http://www.freethepostcode.org/geocode?postcode=%s" | |
DATA = """ | |
WC1N 1HT | |
SW7 4DN | |
SM1 2RF | |
E16 1EA | |
UB7 9NA | |
UB7 0DP | |
SW1E 6AF | |
E4 8ST | |
SE3 0RW | |
KT9 2NE | |
SW19 2BH | |
NW2 1LP | |
E16 1AL | |
W14 9NS | |
HA9 8HQ | |
SE1 9SG | |
W1W 5EE | |
WC1X 9HX | |
UB3 5AJ | |
W6 OQU | |
TW8 8AQ | |
E15 2NE | |
W1X 6NE | |
DA5 1ND | |
E16 2EE | |
EC4V 6DB | |
UB7 0JU | |
E1W 3HN | |
EC1V 9LN | |
SE10 8NA | |
SW18 1EG | |
SW19 5BA | |
UB4 0HF | |
E16 1SL | |
KT9 2NE | |
N12 0QZ | |
HA8 5AQ | |
E1 6BF | |
SE1 0JQ | |
NW1 2QE | |
IG11 7BA | |
SW1P 4DD | |
NW1 7BY | |
SE1 7PB | |
SW8 1RN | |
IG11 7BA | |
CR0 5RB | |
N1 0UY | |
TW6 3AF | |
W2 4RT | |
W6 9LU | |
SE1 0JA | |
UB7 9HB | |
RM13 9ED | |
NW7 3HU | |
TW2 6LS | |
SE16 5PR | |
WC2B 5RE | |
IG11 7BA | |
CR0 4RQ | |
UB3 1RW | |
SW19 2RD | |
SE1 9EF | |
W8 5SR | |
EN3 7NJ | |
E16 1AB | |
E1 7DB | |
NW2 3ED | |
NW1 2AJ | |
UB6 8TR | |
SE1 7BJ | |
CR0 9XY | |
SW6 3JD | |
SE1 9HH | |
IG1 1BA | |
KT6 7EL | |
W1J 7QY | |
E16 3BY | |
WC2B 4DD | |
UB3 5AL | |
SW1V 1EQ | |
SW6 1UD | |
TW13 4EX | |
E14 2AA | |
W1S 4BP | |
WC1H 0DG | |
WC1H 9EU | |
WC1N 1DB | |
WC1B 5BB | |
SM4 5PH | |
E14 9SH | |
EC1Y 1AE | |
RM1 1XJ | |
SW11 3SA | |
W1H 7EH | |
WC1H 9PJ | |
SW6 1HS | |
W2 1JU | |
E15 4LJ | |
N1 9LA | |
TW8 0BB | |
E16 2BZ | |
WC1X 8BH | |
TW1 4RB | |
RM1 3EN | |
SE1 7LS | |
W6 8DR | |
SM5 3PE | |
RM6 6QU | |
NW3 4RB | |
SE1 3LP | |
EC1V 8DS | |
NW1 2LW | |
E16 1AA | |
SE10 8JA | |
SW7 4QH | |
SW7 4LH | |
SW1X 9NU | |
W1K 2HP | |
W1C 1BY | |
W1T 5AY | |
WC2H 7LH | |
WC1B 3BL | |
WC2H 7HL | |
WC1B 3QD | |
W1J 8LT | |
WC2H 9HD | |
W1H 6PA | |
SW7 5BT | |
E14 8RS | |
EC3N 2BQ | |
WC1B 4AR | |
W1W 5NB | |
W1J 7BN | |
W2 4RJ | |
W2 1EE | |
SE1 2BY | |
W1K 1BE | |
SW1A 2TS | |
EC2M 7QN | |
W1H 7BH | |
W1H 7BY | |
W1K 1LB | |
W1B 1JA | |
SW1A 1NJ | |
SW6 2GA | |
W1J 7BJ | |
E16 1RH | |
W8 4PT | |
IG2 7RH | |
SE10 0GD | |
W3 6UP | |
E1 6JQ | |
W1G 8DN | |
WC1B 3NN | |
W1K 6JP | |
NW6 5RP | |
W1H 5DN | |
NW3 3ST | |
E14 4ED | |
W1K 7AA | |
N1 9AA | |
W1J 5EB | |
SW1W 0PS | |
W8 5DL | |
WC1B 4BH | |
W14 0AX | |
NW1 6JQ | |
SW1X 7RN | |
W1H 7DL | |
WC1A 2SD | |
WC2N 5HX | |
NW1 2LP | |
W2 3NR | |
W2 3HL | |
W1D 6BZ | |
SW1A 2EJ | |
E1W 1LD | |
WC2H 7HG | |
SW1W 0SJ | |
SW1W 0QT | |
WC1X 9QN | |
IG4 5DR | |
NW1 6SE | |
UB3 5AW | |
SE10 9JN | |
W1U 6LJ | |
SE16 5HW | |
E1 8BB | |
IG11 7BA | |
E14 9PE | |
SW1E 6DU | |
SW1V 2BJ | |
""" | |
def main(args): | |
args = [unicode(arg, "utf-8") for arg in args] | |
for postcode in DATA.strip().split("\n"): | |
print resolve(postcode) | |
def resolve(postcode): | |
uri = URI % urllib2.quote(postcode) | |
http = httplib2.Http() | |
response, content = http.request(uri, method="GET") | |
try: | |
loc = content.split("\n")[1] | |
loc = loc.split(" ") | |
lat, lon = loc[0:2] | |
except ValueError, exc: | |
print "ERROR:", response, content | |
return False | |
return "%s: %s, %s" % (postcode, loc[0], loc[1]) | |
if __name__ == "__main__": | |
status = not main(sys.argv) | |
sys.exit(status) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment