AO3-DL/python/metadata.py

#!/usr/bin/env python3
# This file is part of AOMO, "Archive Of My Own", a collection of Python and PHP
# scripts designed act as an improved local backup system
# for works published on https://archiveofourown.org
#
# Copyright (c) 2025  Cyberpro123, except where otherwise noted.
#
# This project is available under the GNU General Public License (GPL) v2.0,
# available at https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html
# or in the 'LICENSE.md' file that should be distributed alongside this one.
#
# Report issues to https://codeberg.org/Cyberpro123/AOMO
# Contact author at cyberpro123@posteo.com
import concurrent.futures
import datetime
import traceback

import requests

import common
import linkparser
from process import makeStr
import sql


def fetchSingle(
    id: int,
    session: requests.Session,
    logger: common.logging.Logger,
) -> dict:
    # Significant portions of this function's code are derrived from 'AO3 API',
    # https://github.com/wendytg/ao3_api/
    # as of commit 02e349985d927bd8693f905f440e1ef0539f1984
    # available under the MIT License Copyright (c) 2019 Francisco Patrício Rodrigues
    global countSeriesTotal
    global countSeriesStarted
    global countSeriesComplete
    global existingWorkIDs
    global incompleteSeriesIDs
    countSeriesStarted += 1
    logger.info(f"Starting series [{id}] - [{countSeriesStarted} / {countSeriesTotal}]")
    output = {
        "ID": id,
        "title": "",
        "dateBegun": 0,
        "dateUpdated": 0,
        "dateChecked": int(datetime.datetime.now().timestamp()),
        "description": "",
        "notes": "",
        "complete": None,
        "workIDs": [],
    }
    workIDs, soup = linkparser.getAllWorksFromSeries(
        seriesID=id,
        session=session,
        logger=logger,
        returnSoup=True,
    )
    for workID in workIDs:
        if workID not in existingWorkIDs:
            incompleteSeriesIDs.add(id)
    output["workIDs"] = list(workIDs)
    # Begin section derived from ao3_api
    dl = soup.find("div", {"class": "series-show region"})
    output["title"] = dl.h2.getText().replace("\t", "").replace("\n", "")
    info = dl.find_all(("dd", "dt"))
    last_dt = None
    for field in info:
        text = field.getText().strip()
        if field.name == "dt":
            last_dt = text
        else:
            match last_dt:
                case "Series Begun:":
                    output["dateBegun"] = int(
                        datetime.datetime.fromisoformat(text).timestamp()
                    )
                case "Series Updated:":
                    output["dateUpdated"] = int(
                        datetime.datetime.fromisoformat(text).timestamp()
                    )
                case "Description:":
                    output["description"] = makeStr(field.blockquote.contents)
                case "Notes:":
                    output["notes"] = makeStr(field.blockquote.contents)
                case "Complete:":
                    output["complete"] = int(text == "Yes")
    del dl
    del info
    del last_dt
    del text
    # End section derived from ao3_api
    countSeriesComplete += 1
    logger.info(
        f"Completed series [{id}] - [{countSeriesComplete} / {countSeriesTotal}]"
    )
    return output


if __name__ == "__main__":
    common.init(
        description="Fetches various information about the requested Series.",
        all="Fetch metadata for every Series in the database",
        auto="Fetch metadata for every Series in the database that doesn't have a title registered",
        secondOut="SeriesIDs of incomplete series",
    )
    seriesIDs = set(())
    if common.args.all or common.args.auto:
        allIDs = sql.getSeriesIDs(config=common.config, logger=common.logger)
        for id in allIDs:
            if common.args.all or (not allIDs[id]):
                seriesIDs.add(id)
        del allIDs
    else:
        inRaw = common.parseInfile(common.args.infile)
        for line in inRaw:
            try:
                seriesIDs.add(int(line))
            except ValueError:
                common.logger.error(f"Could not convert input [{line}] to int")
    countSeriesTotal = len(seriesIDs)
    countSeriesStarted = 0
    countSeriesComplete = 0
    incompleteSeriesIDs = set(())
    session = linkparser.download.login(
        config=common.config,
        logger=common.logger,
    )
    existingWorkIDs = sql.getWorkIDs(config=common.config, logger=common.logger)

    with concurrent.futures.ThreadPoolExecutor(
        max_workers=10, thread_name_prefix=common.threadNameBulk
    ) as pool:
        futures = {}
        for id in seriesIDs:
            futures[id] = pool.submit(
                fetchSingle,
                id=id,
                session=session,
                logger=common.logger,
            )
    common.logger.log(
        (20 + (10 * int(countSeriesTotal != countSeriesComplete))),
        f"Checked [{countSeriesComplete}] of [{countSeriesTotal}] series",
    )
    commands = []
    sqlMetaStr1 = "INSERT INTO series_meta(id) VALUES (?) ON CONFLICT DO NOTHING"
    sqlMetaStr2 = "UPDATE series_meta SET title = ?, date_begun = ?, date_updated = ?, date_checked = ?, description = ?, notes = ?, complete = ? WHERE id = ?"
    sqlWorksInsertStr = "INSERT INTO series_works VALUES (?, ?, ?)"
    sqlWorksDeleteStr = (
        "DELETE FROM series_works WHERE series_id = ? AND id = ? AND pos = ?"
    )
    for id in futures:
        try:
            result = futures[id].result()
        except Exception as ex:
            common.logger.error(
                f"Series [{id}] raised [{type(ex).__name__}]: [{ex.args}]"
            )
            traceback.print_exception(ex, file=common.args.outfile)
        else:
            sqlMetaTpl = [
                result["title"],
                result["dateBegun"],
                result["dateUpdated"],
                result["dateChecked"],
                result["description"],
                result["notes"],
                result["complete"],
                result["ID"],
            ]
            result["workIDs"]
            commands.append((sqlMetaStr1, tuple((id,))))
            commands.append((sqlMetaStr2, tuple(sqlMetaTpl)))

            existingWorks = sql.getWorksInSeriesWithPos(
                seriesID=id, config=common.config, logger=common.logger
            )
            existingWorks = set(existingWorks)
            newWorks = set(())
            for num, workID in enumerate(result["workIDs"]):
                newWorks.add((workID, (num + 1)))
            for i in newWorks.difference(existingWorks):
                commands.append((sqlWorksInsertStr, (id, i[0], i[1])))
            for i in existingWorks.difference(newWorks):
                commands.append((sqlWorksDeleteStr, (id, i[0], i[1])))
    sql.execute(commands=commands, config=common.config, logger=common.logger)
    common.logger.info(f"Found [{len(incompleteSeriesIDs)}] incomplete series:")
    for i in incompleteSeriesIDs:
        common.args.batch_out.write(f"{i}\n")

    common.logger.info("All operations complete, metadata.py exiting.")