#!/usr/bin/env python3
import csv
import logging
import re
from collections import namedtuple
from decimal import Decimal
from io import StringIO
from urllib.error import URLError
from urllib.request import urlopen
from bs4 import BeautifulSoup
from geojson import dumps
from geojson import Feature
from geojson import FeatureCollection
from geojson import Point
from simplekml import featgeom
from simplekml import Kml
try:
from systemd import journal
systemd = True
except ImportError:
systemd = False
# ---------------------------------------------------------------------------------------- #
# All of the following default settings are overridden by the config file, if it exists. #
# ---------------------------------------------------------------------------------------- #
# Set the default config file location, this is overridden if the --config switch is used.
default_config = '/etc/archmap.conf'
# Define where to get the wiki list from.
# If a file path is supplied to 'default_file', it will be used instead of the URL
default_url = 'https://wiki.archlinux.org/index.php/ArchMap/List'
default_file = ''
# Set the output locations for the raw-text, GeoJSON, KML and CSV files.
# Setting any of the following to 'no' or leaving them blank will disable the output,
# use '-' to print the generated text to stdout.
default_text = '/tmp/archmap.txt'
default_geojson = '/tmp/archmap.geojson'
default_kml = '/tmp/archmap.kml'
default_csv = '/tmp/archmap.csv'
# Define the verbosity level:
# '-1' will disable all messages other than critical messages (same as '--quiet')
# '0' will disable all messages other than error messages
# '1' will enable info messages
# '2' will enable debugging
default_verbosity = 1
# Setting the following to 'True' will align the columns in the raw-text list
default_pretty = False
# -------------------------------------------------------------------------------------- #
logging.basicConfig(format='==> %(message)s')
log = logging.getLogger('archmap')
log.setLevel(logging.WARNING)
if systemd is not False:
log.addHandler(journal.JournalHandler(SYSLOG_IDENTIFIER='archmap'))
log.handlers[0].setFormatter(logging.Formatter('%(message)s.'))
# Define the namedtuple used to store each users details
Entry = namedtuple(typename='Entry', field_names=['latitude', 'longitude', 'name', 'comment'])
[docs]def get_users(url='https://wiki.archlinux.org/index.php/ArchMap/List', local=''):
"""This funtion parses the list of users from the ArchWiki and returns it as a string.
Args:
url (str): Link to a URL that points to a ArchWiki ArchMap list (default)
local (str): Path to a local copy of the ArchWiki ArchMap source
Returns:
str or None: The extracted raw-text list of users or None if not avaliable
"""
if local == '':
# Open and decode the page from the URL containing the list of users.
log.info('Getting users from the ArchWiki: {}'.format(url))
try:
wiki = urlopen(url)
wiki_source = wiki.read().decode()
except URLError:
log.critical("Can't connect to the ArchWiki")
return None
else:
# Open and decode the local page containing the list of users.
with open(local, 'r') as wiki:
log.info('Getting users from a local file: {}'.format(local))
wiki_source = wiki.read()
# Grab the user data between the last set of <pre> tags.
soup = BeautifulSoup(wiki_source, 'html.parser')
wiki_text = soup.find_all('pre')[-1].text.strip()
return wiki_text
[docs]def parse_users(users):
"""This function parses the raw-text list (``users``) that has been extracted from the wiki page
and splits it into a list of namedtuples containing the latitude, longitude, name and comment.
Args:
users (str): raw-text list from the ArchWiki
Returns:
:obj:`list` of :obj:`collections.namedtuple` \
(:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`)\
: A list of namedtuples, each namedtuple has 4 elements: ``(latitude, longitude, name, comment)``
"""
users = users.splitlines()
parsed = []
# Expression that matches one-half of a coordinate pair. Results in 3 groups: Full, before decimal and after decimal
re_coord = r'((\-?\d+)\.*(\d+)?)'
# Expression that matches the name inside the quote marks and puts that into a group
re_name = r'"(.*)"'
# Expression that puts the comment in a group
re_comment = r'(.*)'
# Compiled expression that combines all of the groups with the stuff inbetween. Results in 8 groups:
# 1. Full latitude
# 2. Latitude before decimal
# 3. Latitude after decimal
# 4. Full longitude
# 5. Longitude before decimal
# 6. Longitude after decimal
# 7. Name
# 8. Comment
re_whole = re.compile(str(re_coord + r'\s*,\s*' + re_coord + r'[^a-zA-Z]*' + re_name + r'\s*#*\s*' + re_comment))
log.info('Parsing ArchWiki list')
for line_number, line in enumerate(users, start=1):
# Retun None unless the line fully matches the RE
re_whole_result = re_whole.fullmatch(line)
if re_whole_result:
latitude = Decimal(re_whole_result.group(1))
longitude = Decimal(re_whole_result.group(4))
name = re_whole_result.group(7).strip()
comment = re_whole_result.group(8).strip()
parsed.append(Entry(latitude=latitude, longitude=longitude, name=name, comment=comment))
else:
log.error('Bad line ({}): {}'.format(line_number, line))
return parsed
[docs]def make_text(parsed_users, output_file='', pretty=False):
"""This function reads the user data supplied by ``parsed_users``, it then generates a raw-text list
according to the formatting specifications on the wiki and writes it to ``output_file``.
Args:
parsed_users (:obj:`list` of :obj:`collections.namedtuple` \
(:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\
: A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)``
output_file (str): Location to save the text output. If left empty, nothing will be output
pretty (bool): If set to True, the output "columns" will be aligned and expanded to match the longest element
Returns:
str: The text written to the output file
"""
text_str = ''
longest_latitude = 1
longest_longitude = 1
longest_name = 1
longest_comment = 1
if pretty:
log.debug('Finding longest strings for prettifying the raw-text')
# Go through all of the elements in each list and track the length of the longest string
for user in parsed_users:
if longest_latitude < len(str(user.latitude)):
longest_latitude = len(str(user.latitude))
if longest_longitude < len(str(user.longitude)):
longest_longitude = len(str(user.longitude))
if longest_name < len(str(user.name)):
longest_name = len(str(user.name))
if longest_comment < len(str(user.comment)):
longest_comment = len(str(user.comment))
log.debug('Making raw-text')
for user in parsed_users:
# This follows the formatting defined here:
# https://wiki.archlinux.org/index.php/ArchMap/List#Adding_yourself_to_the_list
#
# If pretty printing is enabled, the 'longest_' lengths are used to align the elements in the string
# Change the '<', '^' or '>' to change the justification (< = left, > = right, ^ = center)
text_str += '{:<{}},{:<{}} "{:^{}}" # {:>{}}\n'.format(user.latitude, longest_latitude,
user.longitude, longest_longitude,
user.name, longest_name,
user.comment, longest_comment)
# If the last user didnt have a comment, go back to that line
# and strip the trailing whitespace then replace the newline (prevents editor errors)
text_str = text_str.strip('\n').strip() + '\n'
if output_file == '-':
print(text_str)
elif output_file != '':
log.info('Writing raw-text to ' + output_file)
with open(output_file, 'w') as output:
output.write(text_str)
return text_str
[docs]def make_geojson(parsed_users, output_file=''):
"""This function reads the user data supplied by ``parsed_users``, it then generates
GeoJSON output and writes it to ``output_file``.
Args:
parsed_users (:obj:`list` of :obj:`collections.namedtuple` \
(:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\
: A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)``
output_file (str): Location to save the GeoJSON output. If left empty, nothing will be output
Returns:
str: The text written to the output file
"""
geojson = []
log.debug('Making GeoJSON')
for id, user in enumerate(parsed_users):
# Generate a GeoJSON point feature for the user and add it to 'geojson'.
point = Point((float(user.longitude), float(user.latitude)))
feature = Feature(geometry=point, properties={'Name': user.name, 'Comment': user.comment}, id=id)
geojson.append(feature)
# Make 'geojson_str' for output.
geojson_str = (dumps(FeatureCollection(geojson), sort_keys=True, indent=4)) + '\n'
if output_file == '-':
print(geojson_str)
elif output_file != '':
log.info('Writing GeoJSON to ' + output_file)
with open(output_file, 'w') as output:
output.write(geojson_str)
return geojson_str
[docs]def make_kml(parsed_users, output_file=''):
"""This function reads the user data supplied by ``parsed_users``, it then generates
KML output and writes it to ``output_file``.
Args:
parsed_users (:obj:`list` of :obj:`collections.namedtuple` \
(:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\
: A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)``
output_file (str): Location to save the KML output. If left empty, nothing will be output
Returns:
str: The text written to the output file
"""
kml = Kml()
log.debug('Making KML')
for user in parsed_users:
# Generate a KML point for the user.
kml.newpoint(coords=[(user.longitude, user.latitude)], name=user.name, description=user.comment)
# Reset the ID counters
featgeom.Feature._id = 0
featgeom.Geometry._id = 0
kml_str = kml.kml()
if output_file == '-':
print(kml_str)
elif output_file != '':
log.info('Writing KML to ' + output_file)
with open(output_file, 'w') as output:
output.write(kml_str)
return kml_str
[docs]def make_csv(parsed_users, output_file=''):
"""This function reads the user data supplied by ``parsed_users``, it then generates
CSV output and writes it to ``output_file``.
Args:
parsed_users (:obj:`list` of :obj:`collections.namedtuple` \
(:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\
: A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)``
output_file (str): Location to save the CSV output. If left empty, nothing will be output
Returns:
str: The text written to the output file
"""
log.debug('Making CSV')
csv_string = StringIO()
csv_string_writer = csv.writer(csv_string, quoting=csv.QUOTE_MINIMAL, dialect='unix')
csv_string_writer.writerow(('Latitude', 'Longitude', 'Name', 'Comment'))
for user in parsed_users:
csv_string_writer.writerow((user.latitude, user.longitude, user.name, user.comment))
csv_str = csv_string.getvalue()
csv_string.close()
if output_file == '-':
print(csv_str)
elif output_file != '':
log.info('Writing CSV to ' + output_file)
with open(output_file, 'w') as output:
output.write(csv_str)
return csv_str
def main():
from argparse import ArgumentParser
from configparser import ConfigParser
from pathlib import Path
# Define and parse arguments.
parser = ArgumentParser(description='ArchMap GeoJSON/KML generator')
parser.add_argument('-v', '--verbose', action='count',
help='Show info messages')
parser.add_argument('-q', '--quiet', action='store_true',
help='Disable info messages')
parser.add_argument('--config', metavar='FILE', default=default_config,
help='Use an alternative configuration file \
instead of /etc/archmap.conf')
parser.add_argument('--url', metavar='URL',
help='Use an alternative URL to parse the wiki list from')
parser.add_argument('--file', metavar='FILE',
help='Use a file to parse the wiki list from')
parser.add_argument('--pretty', action='store_true',
help='Prettify the raw-text. Only works if user output is enabled')
parser.add_argument('--text', metavar='FILE',
help="Output the raw-text to FILE, use 'no' to disable output or '-' to print to stdout")
parser.add_argument('--geojson', metavar='FILE',
help="Output the GeoJSON to FILE, use 'no' to disable output or '-' to print to stdout")
parser.add_argument('--kml', metavar='FILE',
help="Output the KML to FILE, use 'no' to disable output or '-' to print to stdout")
parser.add_argument('--csv', metavar='FILE',
help="Output the CSV to FILE, use 'no' to disable output or '-' to print to stdout")
args = parser.parse_args()
config_location = Path(args.config)
config = ConfigParser()
# Try to use the config file. If it doesn't exist, use the defaults.
if not config_location.is_file():
log.warning('Configuation file does not exist. Using defaults')
else:
config.read(str(config_location))
verbosity = config.getint('extras', 'verbosity', fallback=default_verbosity)
pretty = config.getboolean('extras', 'pretty', fallback=default_pretty)
input_url = config.get('files', 'url', fallback=default_url)
input_file = config.get('files', 'file', fallback=default_file)
output_file_text = config.get('files', 'text', fallback=default_text)
output_file_geojson = config.get('files', 'geojson', fallback=default_geojson)
output_file_kml = config.get('files', 'kml', fallback=default_kml)
output_file_csv = config.get('files', 'csv', fallback=default_csv)
# Finally, parse the command line arguments, anything passed to them will
# override both the defaults in this script and anything in the config file.
if args.verbose is not None:
verbosity = args.verbose
if verbosity == 1:
log.setLevel(logging.INFO)
if verbosity > 1:
log.setLevel(logging.DEBUG)
if args.quiet or verbosity == -1:
log.setLevel(logging.CRITICAL)
if args.pretty is not False:
pretty = True
if args.url is not None:
input_url = args.url
if args.file is not None:
input_file = args.file
if args.text is not None:
output_file_text = args.text
if args.geojson is not None:
output_file_geojson = args.geojson
if args.kml is not None:
output_file_kml = args.kml
if args.csv is not None:
output_file_csv = args.csv
# Do what's needed.
dont_run = ['', 'no']
if output_file_text in dont_run and \
output_file_geojson in dont_run and \
output_file_kml in dont_run and \
output_file_csv in dont_run:
log.warning('There is nothing to do')
else:
pipe_claims = []
if output_file_text == '-':
pipe_claims.append('Text')
if output_file_geojson == '-':
pipe_claims.append('GeoJSON')
if output_file_kml == '-':
pipe_claims.append('KML')
if output_file_csv == '-':
pipe_claims.append('CSV')
if len(pipe_claims) > 1:
log.warning('More than one format specified for printing. You probably want to disable one of the following: {}'
.format(', '.join(pipe_claims)))
users = get_users(url=input_url, local=input_file)
if users is None:
return None
parsed_users = parse_users(users)
if output_file_text not in dont_run:
make_text(parsed_users, output_file_text, pretty=pretty)
if output_file_geojson not in dont_run:
make_geojson(parsed_users, output_file_geojson)
if output_file_kml not in dont_run:
make_kml(parsed_users, output_file_kml)
if output_file_csv not in dont_run:
make_csv(parsed_users, output_file_csv)
# If the script is being run and not imported...
if __name__ == '__main__':
main()