From 2d1e78361b42cbbf11bcae42526a13c3cef66d25 Mon Sep 17 00:00:00 2001 From: Floris van Enter Date: Sat, 2 Jun 2018 09:03:07 +0200 Subject: [PATCH] Add a BaseCamp export rework script --- .../process CSV with locations/1.Get_Data.py | 74 +++++++++++++++++++ .../2b.convert_Data.py | 69 +++++++++++++++++ Python/process CSV with locations/archive.log | 1 + Python/process CSV with locations/done.log | 1 + .../functions/convertGPS.py | 27 +++++++ .../functions/getLocationData.py | 35 +++++++++ .../process CSV with locations/locations.csv | 5 ++ 7 files changed, 212 insertions(+) create mode 100644 Python/process CSV with locations/1.Get_Data.py create mode 100644 Python/process CSV with locations/2b.convert_Data.py create mode 100644 Python/process CSV with locations/archive.log create mode 100644 Python/process CSV with locations/done.log create mode 100644 Python/process CSV with locations/functions/convertGPS.py create mode 100644 Python/process CSV with locations/functions/getLocationData.py create mode 100644 Python/process CSV with locations/locations.csv diff --git a/Python/process CSV with locations/1.Get_Data.py b/Python/process CSV with locations/1.Get_Data.py new file mode 100644 index 0000000..af09ea8 --- /dev/null +++ b/Python/process CSV with locations/1.Get_Data.py @@ -0,0 +1,74 @@ +# Get an export with GPS data, get more info and save result +# Tested in Python 2.7 +# I used this script to get more Google Maps data from an export with coordinates +# from BaseCamp. This way I can periodically export data from BaseCamp and import in a +# database so I can use it in Web Apps. +# After script 1, script 2 must be run +# +# You can contact me by e-mail at floris@entermi.nl. +# +# Last updated 02 June, 2018. +# +# Floris van Enter +# http://entermi.nl + +#!/bin/bash + +import random +import shutil +import time +from functions.VanlifelocationJunk import * +from functions.getLocationData import getplace +from functions.convertGPS import parse_dms + +todo = open('./locations.csv','r') +done = open('./done.log','a') +check = open('./done.log','r').read() +archived = open('./archive.log','r').read() + +cntError = 0 +cntDone = 0 +cntArchive = 0 +cntSuccess = 0 +cntShizzle = 0 + +tagsCheck = [] + +def rand_string(length, char_set='ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567889'): + return ''.join( random.choice(char_set) for _ in range(length) ) + +todoLine = todo.readline() + +while todoLine: + contents = todoLine.split(',') + lat = contents[1][:12] + lon = contents[2][:12] + + if (lat != 'lat'): + if (lat + ';' + lon) in check: + cntDone += 1 + elif (lat + ';' + lon) in archived: + cntArchive += 1 + else: + places = getplace(lat,lon) + if places != 'Google Shizzle' and places != None: + places = places.split(',') + date = contents[4].split('T')[0] + name = contents[7].replace('"','') + + done.write(lat + ';' + lon + ';' + date + ';' + name + ';' + places[0] + ';' + places[1] + ';' + places[2] + '\n') + cntSuccess += 1 + elif places == None: + cntShizzle += 1 + + todoLine = todo.readline() + +print("----------------------") +print(" result ") +print("----------------------") +print("Google shizzle: " + str(cntShizzle)) +print("success: " + str(cntSuccess)) +print("error: " + str(cntError)) +print("done: " + str(cntDone)) +print("archived: " + str(cntArchive)) +print("") \ No newline at end of file diff --git a/Python/process CSV with locations/2b.convert_Data.py b/Python/process CSV with locations/2b.convert_Data.py new file mode 100644 index 0000000..80246f3 --- /dev/null +++ b/Python/process CSV with locations/2b.convert_Data.py @@ -0,0 +1,69 @@ +# Rework the result of 1.Get_Data.py to Laravel migration +# Tested in Python 2.7 +# I used this script to create text files which content I past in a Laravel +# project, database migrations file. After reworking the data gets moved to an archive +# file to prevent duplicating data. +# +# You can contact me by e-mail at floris@entermi.nl. +# +# Last updated 02 June, 2018. +# +# Floris van Enter +# http://entermi.nl + +#!/bin/bash + +import random +import shutil +import time + +todo = open('./done.log','r') +archive = open('./archive.log','a') + +writeFile = open('./write/export_' + time.strftime("%Y%m%d-%H%M%S") + '.txt', 'w') +tagsCheck = [] + +todoLine = todo.readline() + +query = 'DB::statement("INSERT IGNORE INTO `sources` SET ' +query += "`name` = 'source name', `nickname` = 'source', `link` = 'http://example.com', `user_id`=1" +query += '");' + +writeFile.write(query + "\n\n") + +while todoLine: + contents = todoLine.replace('\n','').strip().split(';') + tags = [] + + lat = contents[0] + lon = contents[1] + date = contents[2] + name = contents[3] + if 'wild' in name: + wild = str(1) + else: + wild = str(0) + country = contents[4] + note = "Found at " + contents[5] + ", " + contents[6] + " export from Garmin Basecamp" + + query = 'DB::statement("INSERT INTO `locations` (`user_id`, `source_id`, `wild`, `name`, `country`, `lon`, `lat`, `note`, `visits`, `created_at`, `updated_at`) VALUES (' + query += "1, (SELECT `id` FROM `sources` WHERE `link`= 'http://example.com'), " + wild + ", '" + name + "', (SELECT `code` FROM `countries` WHERE `name` = '" + country + "'), '" + lon + "', '" + lat + "', '" + note + "', 0, NOW(), NOW())" + query += '");' + + writeFile.write(query + "\n") + + todoLine = todo.readline() + +writeFile.close() +todo.close() +todo = open('./done.log','r') +todoLine = todo.readline() +while todoLine: + archive.write(todoLine) + todoLine = todo.readline() + +todo.close() +archive.close() + +newFile = open('./done.log','w') +newFile.close() \ No newline at end of file diff --git a/Python/process CSV with locations/archive.log b/Python/process CSV with locations/archive.log new file mode 100644 index 0000000..91baaf8 --- /dev/null +++ b/Python/process CSV with locations/archive.log @@ -0,0 +1 @@ +48.445844985;5.1493910327;2017-05-26;Camperplaats Joinville;France;Joinville;Rue des Jardins \ No newline at end of file diff --git a/Python/process CSV with locations/done.log b/Python/process CSV with locations/done.log new file mode 100644 index 0000000..a5f8c33 --- /dev/null +++ b/Python/process CSV with locations/done.log @@ -0,0 +1 @@ +44.531172411;6.4302436541;2018-05-16;Camperplek bij Crote;France;Crots;N94 diff --git a/Python/process CSV with locations/functions/convertGPS.py b/Python/process CSV with locations/functions/convertGPS.py new file mode 100644 index 0000000..9c14df4 --- /dev/null +++ b/Python/process CSV with locations/functions/convertGPS.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import re + +def dms2dd(degrees, minutes, seconds, direction): + dd = float(degrees) + float(minutes)/60 + float(seconds)/(60*60); + + return dd; + +def dd2dms(deg): + d = int(deg) + md = abs(deg - d) * 60 + m = int(md) + sd = (md - m) * 60 + return [d, m, sd] + +def parse_dms(dms): + parts = re.split('[^\d\w]+', dms) + lat = dms2dd(parts[0], parts[1], parts[2], parts[3]) + + if 'W' in dms or 'S' in dms: + lat = lat * -1 + + return (lat) + + +#dd = parse_dms("5°38'38.5 W") +#print(dd) diff --git a/Python/process CSV with locations/functions/getLocationData.py b/Python/process CSV with locations/functions/getLocationData.py new file mode 100644 index 0000000..24f296c --- /dev/null +++ b/Python/process CSV with locations/functions/getLocationData.py @@ -0,0 +1,35 @@ +import urllib.request +import json + +def getplace(lat, lon): + lat = str(lat) + lon = str(lon) + url = "http://maps.googleapis.com/maps/api/geocode/json?" + url += "latlng=%s,%s&sensor=false" % (lat, lon) + + v = urllib.request.urlopen(url).read() + j = json.loads(v) + try: + components = j['results'][0]['address_components'] + except: + #print(lat + ' - ' + lon) + #print('## Google Shizzle ##') + return None + else: + country = town = street = None + for c in components: + if "route" in c['types']: + street = c['long_name'] + if "country" in c['types']: + country = c['long_name'] + if "locality" in c['types']: + town = c['long_name'] + + return (str(country) + "," + str(town) + "," + str(street)) + + +# print(getplace(47.591737, 2.759478)) +# print(getplace(51.2, 0.1)) +# print(getplace(51.3, 0.1)) + +#print(getplace("60°04'52.3N", "14°09'13.7E")) \ No newline at end of file diff --git a/Python/process CSV with locations/locations.csv b/Python/process CSV with locations/locations.csv new file mode 100644 index 0000000..67cab36 --- /dev/null +++ b/Python/process CSV with locations/locations.csv @@ -0,0 +1,5 @@ +ID,lat,lon,ele,time,magvar,geoidheight,name,cmt,desc,src,sym,type,fix,sat,hdop,vdop,pdop,ageofdgpsdata,dgpsid,Proximity,Temperature,Depth,DisplayMode,Samples,Expiration,CreationTime, +4,48.445844985544682,5.149391032755375,,2017-05-26T15:26:19Z,,,"Camperplaats Joinville",,,,"Campground","user",,,,,,,,,,,SymbolAndName,,,2017-05-26T15:26:19Z, +5,44.531172411516309,6.430243654176593,784.69921875,2018-05-16T11:49:02Z,,,"Camperplek bij Crote",,,,"Campground","user",,,,,,,,,,,SymbolAndName,,,2018-05-16T11:49:02Z, +6,49.790622694417834,5.047184955328703,,2018-05-01T11:47:57Z,,,"Camping Halliru",,,,"Campground","user",,,,,,,,,,,SymbolAndName,,,2018-05-01T11:47:57Z, +7,48.404876003041863,5.270125977694988,,2017-05-10T19:34:29Z,,,"Camping La Forge",,,,"Campground","user",,,,,,,,,,,SymbolAndName,,,2017-05-10T19:34:29Z, \ No newline at end of file