Source code for archmap

#!/usr/bin/env python3
import csv
import logging
import re
from collections import namedtuple
from decimal import Decimal
from io import StringIO
from urllib.error import URLError
from urllib.request import urlopen

from bs4 import BeautifulSoup
from geojson import dumps
from geojson import Feature
from geojson import FeatureCollection
from geojson import Point
from simplekml import featgeom
from simplekml import Kml

try:
    from systemd import journal
    systemd = True
except ImportError:
    systemd = False


# ---------------------------------------------------------------------------------------- #
#  All of the following default settings are overridden by the config file, if it exists.  #
# ---------------------------------------------------------------------------------------- #

# Set the default config file location, this is overridden if the --config switch is used.
default_config = '/etc/archmap.conf'

# Define where to get the wiki list from.
# If a file path is supplied to 'default_file', it will be used instead of the URL
default_url = 'https://wiki.archlinux.org/index.php/ArchMap/List'
default_file = ''

# Set the output locations for the raw-text, GeoJSON, KML and CSV files.
# Setting any of the following to 'no' or leaving them blank will disable the output,
# use '-' to print the generated text to stdout.
default_text = '/tmp/archmap.txt'
default_geojson = '/tmp/archmap.geojson'
default_kml = '/tmp/archmap.kml'
default_csv = '/tmp/archmap.csv'

# Define the verbosity level:
# '-1' will disable all messages other than critical messages (same as '--quiet')
# '0' will disable all messages other than error messages
# '1' will enable info messages
# '2' will enable debugging
default_verbosity = 1

# Setting the following to 'True' will align the columns in the raw-text list
default_pretty = False

# -------------------------------------------------------------------------------------- #

logging.basicConfig(format='==> %(message)s')
log = logging.getLogger('archmap')
log.setLevel(logging.WARNING)

if systemd is not False:
    log.addHandler(journal.JournalHandler(SYSLOG_IDENTIFIER='archmap'))
    log.handlers[0].setFormatter(logging.Formatter('%(message)s.'))

# Define the namedtuple used to store each users details
Entry = namedtuple(typename='Entry', field_names=['latitude', 'longitude', 'name', 'comment'])


[docs]def get_users(url='https://wiki.archlinux.org/index.php/ArchMap/List', local=''): """This funtion parses the list of users from the ArchWiki and returns it as a string. Args: url (str): Link to a URL that points to a ArchWiki ArchMap list (default) local (str): Path to a local copy of the ArchWiki ArchMap source Returns: str or None: The extracted raw-text list of users or None if not avaliable """ if local == '': # Open and decode the page from the URL containing the list of users. log.info('Getting users from the ArchWiki: {}'.format(url)) try: wiki = urlopen(url) wiki_source = wiki.read().decode() except URLError: log.critical("Can't connect to the ArchWiki") return None else: # Open and decode the local page containing the list of users. with open(local, 'r') as wiki: log.info('Getting users from a local file: {}'.format(local)) wiki_source = wiki.read() # Grab the user data between the last set of <pre> tags. soup = BeautifulSoup(wiki_source, 'html.parser') wiki_text = soup.find_all('pre')[-1].text.strip() return wiki_text
[docs]def parse_users(users): """This function parses the raw-text list (``users``) that has been extracted from the wiki page and splits it into a list of namedtuples containing the latitude, longitude, name and comment. Args: users (str): raw-text list from the ArchWiki Returns: :obj:`list` of :obj:`collections.namedtuple` \ (:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`)\ : A list of namedtuples, each namedtuple has 4 elements: ``(latitude, longitude, name, comment)`` """ users = users.splitlines() parsed = [] # Expression that matches one-half of a coordinate pair. Results in 3 groups: Full, before decimal and after decimal re_coord = r'((\-?\d+)\.*(\d+)?)' # Expression that matches the name inside the quote marks and puts that into a group re_name = r'"(.*)"' # Expression that puts the comment in a group re_comment = r'(.*)' # Compiled expression that combines all of the groups with the stuff inbetween. Results in 8 groups: # 1. Full latitude # 2. Latitude before decimal # 3. Latitude after decimal # 4. Full longitude # 5. Longitude before decimal # 6. Longitude after decimal # 7. Name # 8. Comment re_whole = re.compile(str(re_coord + r'\s*,\s*' + re_coord + r'[^a-zA-Z]*' + re_name + r'\s*#*\s*' + re_comment)) log.info('Parsing ArchWiki list') for line_number, line in enumerate(users, start=1): # Retun None unless the line fully matches the RE re_whole_result = re_whole.fullmatch(line) if re_whole_result: latitude = Decimal(re_whole_result.group(1)) longitude = Decimal(re_whole_result.group(4)) name = re_whole_result.group(7).strip() comment = re_whole_result.group(8).strip() parsed.append(Entry(latitude=latitude, longitude=longitude, name=name, comment=comment)) else: log.error('Bad line ({}): {}'.format(line_number, line)) return parsed
[docs]def make_text(parsed_users, output_file='', pretty=False): """This function reads the user data supplied by ``parsed_users``, it then generates a raw-text list according to the formatting specifications on the wiki and writes it to ``output_file``. Args: parsed_users (:obj:`list` of :obj:`collections.namedtuple` \ (:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\ : A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)`` output_file (str): Location to save the text output. If left empty, nothing will be output pretty (bool): If set to True, the output "columns" will be aligned and expanded to match the longest element Returns: str: The text written to the output file """ text_str = '' longest_latitude = 1 longest_longitude = 1 longest_name = 1 longest_comment = 1 if pretty: log.debug('Finding longest strings for prettifying the raw-text') # Go through all of the elements in each list and track the length of the longest string for user in parsed_users: if longest_latitude < len(str(user.latitude)): longest_latitude = len(str(user.latitude)) if longest_longitude < len(str(user.longitude)): longest_longitude = len(str(user.longitude)) if longest_name < len(str(user.name)): longest_name = len(str(user.name)) if longest_comment < len(str(user.comment)): longest_comment = len(str(user.comment)) log.debug('Making raw-text') for user in parsed_users: # This follows the formatting defined here: # https://wiki.archlinux.org/index.php/ArchMap/List#Adding_yourself_to_the_list # # If pretty printing is enabled, the 'longest_' lengths are used to align the elements in the string # Change the '<', '^' or '>' to change the justification (< = left, > = right, ^ = center) text_str += '{:<{}},{:<{}} "{:^{}}" # {:>{}}\n'.format(user.latitude, longest_latitude, user.longitude, longest_longitude, user.name, longest_name, user.comment, longest_comment) # If the last user didnt have a comment, go back to that line # and strip the trailing whitespace then replace the newline (prevents editor errors) text_str = text_str.strip('\n').strip() + '\n' if output_file == '-': print(text_str) elif output_file != '': log.info('Writing raw-text to ' + output_file) with open(output_file, 'w') as output: output.write(text_str) return text_str
[docs]def make_geojson(parsed_users, output_file=''): """This function reads the user data supplied by ``parsed_users``, it then generates GeoJSON output and writes it to ``output_file``. Args: parsed_users (:obj:`list` of :obj:`collections.namedtuple` \ (:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\ : A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)`` output_file (str): Location to save the GeoJSON output. If left empty, nothing will be output Returns: str: The text written to the output file """ geojson = [] log.debug('Making GeoJSON') for id, user in enumerate(parsed_users): # Generate a GeoJSON point feature for the user and add it to 'geojson'. point = Point((float(user.longitude), float(user.latitude))) feature = Feature(geometry=point, properties={'Name': user.name, 'Comment': user.comment}, id=id) geojson.append(feature) # Make 'geojson_str' for output. geojson_str = (dumps(FeatureCollection(geojson), sort_keys=True, indent=4)) + '\n' if output_file == '-': print(geojson_str) elif output_file != '': log.info('Writing GeoJSON to ' + output_file) with open(output_file, 'w') as output: output.write(geojson_str) return geojson_str
[docs]def make_kml(parsed_users, output_file=''): """This function reads the user data supplied by ``parsed_users``, it then generates KML output and writes it to ``output_file``. Args: parsed_users (:obj:`list` of :obj:`collections.namedtuple` \ (:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\ : A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)`` output_file (str): Location to save the KML output. If left empty, nothing will be output Returns: str: The text written to the output file """ kml = Kml() log.debug('Making KML') for user in parsed_users: # Generate a KML point for the user. kml.newpoint(coords=[(user.longitude, user.latitude)], name=user.name, description=user.comment) # Reset the ID counters featgeom.Feature._id = 0 featgeom.Geometry._id = 0 kml_str = kml.kml() if output_file == '-': print(kml_str) elif output_file != '': log.info('Writing KML to ' + output_file) with open(output_file, 'w') as output: output.write(kml_str) return kml_str
[docs]def make_csv(parsed_users, output_file=''): """This function reads the user data supplied by ``parsed_users``, it then generates CSV output and writes it to ``output_file``. Args: parsed_users (:obj:`list` of :obj:`collections.namedtuple` \ (:obj:`decimal.Decimal`, :obj:`decimal.Decimal`, :obj:`str`, :obj:`str`))\ : A list of namedtuples, each namedtuple should have 4 elements: ``(latitude, longitude, name, comment)`` output_file (str): Location to save the CSV output. If left empty, nothing will be output Returns: str: The text written to the output file """ log.debug('Making CSV') csv_string = StringIO() csv_string_writer = csv.writer(csv_string, quoting=csv.QUOTE_MINIMAL, dialect='unix') csv_string_writer.writerow(('Latitude', 'Longitude', 'Name', 'Comment')) for user in parsed_users: csv_string_writer.writerow((user.latitude, user.longitude, user.name, user.comment)) csv_str = csv_string.getvalue() csv_string.close() if output_file == '-': print(csv_str) elif output_file != '': log.info('Writing CSV to ' + output_file) with open(output_file, 'w') as output: output.write(csv_str) return csv_str
def main(): from argparse import ArgumentParser from configparser import ConfigParser from pathlib import Path # Define and parse arguments. parser = ArgumentParser(description='ArchMap GeoJSON/KML generator') parser.add_argument('-v', '--verbose', action='count', help='Show info messages') parser.add_argument('-q', '--quiet', action='store_true', help='Disable info messages') parser.add_argument('--config', metavar='FILE', default=default_config, help='Use an alternative configuration file \ instead of /etc/archmap.conf') parser.add_argument('--url', metavar='URL', help='Use an alternative URL to parse the wiki list from') parser.add_argument('--file', metavar='FILE', help='Use a file to parse the wiki list from') parser.add_argument('--pretty', action='store_true', help='Prettify the raw-text. Only works if user output is enabled') parser.add_argument('--text', metavar='FILE', help="Output the raw-text to FILE, use 'no' to disable output or '-' to print to stdout") parser.add_argument('--geojson', metavar='FILE', help="Output the GeoJSON to FILE, use 'no' to disable output or '-' to print to stdout") parser.add_argument('--kml', metavar='FILE', help="Output the KML to FILE, use 'no' to disable output or '-' to print to stdout") parser.add_argument('--csv', metavar='FILE', help="Output the CSV to FILE, use 'no' to disable output or '-' to print to stdout") args = parser.parse_args() config_location = Path(args.config) config = ConfigParser() # Try to use the config file. If it doesn't exist, use the defaults. if not config_location.is_file(): log.warning('Configuation file does not exist. Using defaults') else: config.read(str(config_location)) verbosity = config.getint('extras', 'verbosity', fallback=default_verbosity) pretty = config.getboolean('extras', 'pretty', fallback=default_pretty) input_url = config.get('files', 'url', fallback=default_url) input_file = config.get('files', 'file', fallback=default_file) output_file_text = config.get('files', 'text', fallback=default_text) output_file_geojson = config.get('files', 'geojson', fallback=default_geojson) output_file_kml = config.get('files', 'kml', fallback=default_kml) output_file_csv = config.get('files', 'csv', fallback=default_csv) # Finally, parse the command line arguments, anything passed to them will # override both the defaults in this script and anything in the config file. if args.verbose is not None: verbosity = args.verbose if verbosity == 1: log.setLevel(logging.INFO) if verbosity > 1: log.setLevel(logging.DEBUG) if args.quiet or verbosity == -1: log.setLevel(logging.CRITICAL) if args.pretty is not False: pretty = True if args.url is not None: input_url = args.url if args.file is not None: input_file = args.file if args.text is not None: output_file_text = args.text if args.geojson is not None: output_file_geojson = args.geojson if args.kml is not None: output_file_kml = args.kml if args.csv is not None: output_file_csv = args.csv # Do what's needed. dont_run = ['', 'no'] if output_file_text in dont_run and \ output_file_geojson in dont_run and \ output_file_kml in dont_run and \ output_file_csv in dont_run: log.warning('There is nothing to do') else: pipe_claims = [] if output_file_text == '-': pipe_claims.append('Text') if output_file_geojson == '-': pipe_claims.append('GeoJSON') if output_file_kml == '-': pipe_claims.append('KML') if output_file_csv == '-': pipe_claims.append('CSV') if len(pipe_claims) > 1: log.warning('More than one format specified for printing. You probably want to disable one of the following: {}' .format(', '.join(pipe_claims))) users = get_users(url=input_url, local=input_file) if users is None: return None parsed_users = parse_users(users) if output_file_text not in dont_run: make_text(parsed_users, output_file_text, pretty=pretty) if output_file_geojson not in dont_run: make_geojson(parsed_users, output_file_geojson) if output_file_kml not in dont_run: make_kml(parsed_users, output_file_kml) if output_file_csv not in dont_run: make_csv(parsed_users, output_file_csv) # If the script is being run and not imported... if __name__ == '__main__': main()