#!python
"""Script to build database for search engine."""
import argparse
import csv
import ssl
from collections import deque
from datetime import datetime as dt
from multiprocessing.pool import Pool
from urllib.parse import urljoin

import numpy as np
import requests

#  import tqdm
from bs4 import BeautifulSoup as bs
from requests import get
from tqdm.contrib.concurrent import process_map

ssl._create_default_https_context = ssl._create_unverified_context
requests.packages.urllib3.disable_warnings()


def soup_from_url(url):
    return bs(get(url, verify=False).text, "lxml")


def get_month_urls():
    """Produce a list of urls, each referring to a webpage that
    contains all the job urls of a given month.
    """

    month_urls = deque()

    for month_link in soup_website.findAll("a"):
        month_href = month_link.get("href")

        if month_href and "date" in month_href:
            month_url = urljoin(URL_ROOT, month_href)
            month_urls.append(month_url)

    return month_urls


def run_on_month_url(month_url):
    """Produce a list of all the job urls for a given month."""

    out_urls = deque()

    month_soup = soup_from_url(month_url)

    for ad_link in month_soup.findAll("a"):
        ad_href = ad_link.get("href")

        if ad_href and "msg" in ad_href:

            ad_url = month_url.replace("date.html", ad_href)
            out_urls.append(ad_url)

    return out_urls


def run_on_ad_url(ad_url):
    """Get job ad information (title and date) from its url.
    Write them to csv file output.
    """

    ad_soup = soup_from_url(ad_url)

    try:
        title = ad_soup.find("h1").get_text()
    except AttributeError:
        return []

    split_str = "[Met-jobs] "
    if split_str in title:
        title = title.split(split_str)[1]

    dates = [
        a.get_text() for a in ad_soup.select("tr td") if a.get_text().count(":") == 2
    ]

    if dates:
        date = dates[0].rstrip()
        return [title, date, ad_url]

    return []


def parse_and_filter(urls, func):
    """Produce a list of return values of the function func, applied to
    datetime part of each input url."""
    return [func(dt.strptime(u.split("/")[-2], "%Y-%m")) for u in urls]


def filter_month_urls(urls, start, end):
    """Return a filtered list of month urls, where only urls withing start and
    end date are selected."""

    urls = np.array(urls)

    if start:
        start = dt.strptime(start, "%Y-%m")
        urls = urls[parse_and_filter(urls, lambda d: d >= start)]
    if end:
        end = dt.strptime(end, "%Y-%m")
        urls = urls[parse_and_filter(urls, lambda d: d <= end)]

    return urls.tolist()


if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description="Scrape the met-jobs website and create a database of job ads."
    )
    parser.add_argument(
        "path_csv",
        metavar="PATH_CSV",
        type=str,
        help="The output path for the database csv file",
    )
    parser.add_argument(
        "-s", "--start", type=str, help="Start date for database (format: YYYY-MM)"
    )
    parser.add_argument(
        "-e", "--end", type=str, help="End date for database (format: YYYY-MM)"
    )
    args = parser.parse_args()

    URL_ROOT = "https://www.lists.rdg.ac.uk/archives/met-jobs/"
    soup_website = soup_from_url(URL_ROOT)

    month_urls = filter_month_urls(get_month_urls(), args.start, args.end)

    print("Retrieve URLs of job ads...")
    ad_urls = Pool().map(run_on_month_url, month_urls)
    ad_urls = [url for url_list in ad_urls for url in url_list]

    kwargs = {}
    if len(ad_urls) > 1000:
        # default `chunksize=1` has poor performance for large iterables
        # (most time spent dispatching items to workers)
        kwargs = {"chunksize": 2}

    print("Extract information of job ads and write database to file...")
    job_infos = process_map(run_on_ad_url, ad_urls, **kwargs)

    with open(args.path_csv, "w") as csvfile:
        csv_writer = csv.writer(csvfile)
        csv_writer.writerow(["title", "date", "url"])
        csv_writer.writerows(job_infos)
