#!/usr/bin/env python3 # This file is part of AOMO, "Archive Of My Own", a collection of Python and PHP # scripts designed act as an improved local backup system # for works published on https://archiveofourown.org # # Copyright (c) 2025 Cyberpro123, except where otherwise noted. # # This project is available under the GNU General Public License (GPL) v2.0, # available at https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html # or in the 'LICENSE.md' file that should be distributed alongside this one. # # Report issues to https://codeberg.org/Cyberpro123/AOMO # Contact author at cyberpro123@posteo.com import concurrent.futures import datetime import traceback import requests import common import linkparser from process import makeStr import sql def fetchSingle( id: int, session: requests.Session, logger: common.logging.Logger, ) -> dict: # Significant portions of this function's code are derrived from 'AO3 API', # https://github.com/wendytg/ao3_api/ # as of commit 02e349985d927bd8693f905f440e1ef0539f1984 # available under the MIT License Copyright (c) 2019 Francisco PatrĂ­cio Rodrigues global countSeriesTotal global countSeriesStarted global countSeriesComplete global existingWorkIDs global incompleteSeriesIDs countSeriesStarted += 1 logger.info(f"Starting series [{id}] - [{countSeriesStarted} / {countSeriesTotal}]") output = { "ID": id, "title": "", "dateBegun": 0, "dateUpdated": 0, "dateChecked": int(datetime.datetime.now().timestamp()), "description": "", "notes": "", "complete": None, "workIDs": [], } workIDs, soup = linkparser.getAllWorksFromSeries( seriesID=id, session=session, logger=logger, returnSoup=True, ) for workID in workIDs: if workID not in existingWorkIDs: incompleteSeriesIDs.add(id) output["workIDs"] = list(workIDs) # Begin section derived from ao3_api dl = soup.find("div", {"class": "series-show region"}) output["title"] = dl.h2.getText().replace("\t", "").replace("\n", "") info = dl.find_all(("dd", "dt")) last_dt = None for field in info: text = field.getText().strip() if field.name == "dt": last_dt = text else: match last_dt: case "Series Begun:": output["dateBegun"] = int( datetime.datetime.fromisoformat(text).timestamp() ) case "Series Updated:": output["dateUpdated"] = int( datetime.datetime.fromisoformat(text).timestamp() ) case "Description:": output["description"] = makeStr(field.blockquote.contents) case "Notes:": output["notes"] = makeStr(field.blockquote.contents) case "Complete:": output["complete"] = int(text == "Yes") del dl del info del last_dt del text # End section derived from ao3_api countSeriesComplete += 1 logger.info( f"Completed series [{id}] - [{countSeriesComplete} / {countSeriesTotal}]" ) return output if __name__ == "__main__": common.init( description="Fetches various information about the requested Series.", all="Fetch metadata for every Series in the database", auto="Fetch metadata for every Series in the database that doesn't have a title registered", secondOut="SeriesIDs of incomplete series", ) seriesIDs = set(()) if common.args.all or common.args.auto: allIDs = sql.getSeriesIDs(config=common.config, logger=common.logger) for id in allIDs: if common.args.all or (not allIDs[id]): seriesIDs.add(id) del allIDs else: inRaw = common.parseInfile(common.args.infile) for line in inRaw: try: seriesIDs.add(int(line)) except ValueError: common.logger.error(f"Could not convert input [{line}] to int") countSeriesTotal = len(seriesIDs) countSeriesStarted = 0 countSeriesComplete = 0 incompleteSeriesIDs = set(()) session = linkparser.download.login( config=common.config, logger=common.logger, ) existingWorkIDs = sql.getWorkIDs(config=common.config, logger=common.logger) with concurrent.futures.ThreadPoolExecutor( max_workers=10, thread_name_prefix=common.threadNameBulk ) as pool: futures = {} for id in seriesIDs: futures[id] = pool.submit( fetchSingle, id=id, session=session, logger=common.logger, ) common.logger.log( (20 + (10 * int(countSeriesTotal != countSeriesComplete))), f"Checked [{countSeriesComplete}] of [{countSeriesTotal}] series", ) commands = [] sqlMetaStr1 = "INSERT INTO series_meta(id) VALUES (?) ON CONFLICT DO NOTHING" sqlMetaStr2 = "UPDATE series_meta SET title = ?, date_begun = ?, date_updated = ?, date_checked = ?, description = ?, notes = ?, complete = ? WHERE id = ?" sqlWorksInsertStr = "INSERT INTO series_works VALUES (?, ?, ?)" sqlWorksDeleteStr = ( "DELETE FROM series_works WHERE series_id = ? AND work_id = ? AND pos = ?" ) for id in futures: try: result = futures[id].result() except Exception as ex: common.logger.error( f"Series [{id}] raised [{type(ex).__name__}]: [{ex.args}]" ) traceback.print_exception(ex, file=common.args.outfile) else: sqlMetaTpl = [ result["title"], result["dateBegun"], result["dateUpdated"], result["dateChecked"], result["description"], result["notes"], result["complete"], result["ID"], ] result["workIDs"] commands.append((sqlMetaStr1, tuple((id,)))) commands.append((sqlMetaStr2, tuple(sqlMetaTpl))) existingWorks = sql.getWorksInSeriesWithPos( seriesID=id, config=common.config, logger=common.logger ) existingWorks = set(existingWorks) newWorks = set(()) for num, workID in enumerate(result["workIDs"]): newWorks.add((workID, (num + 1))) for i in newWorks.difference(existingWorks): commands.append((sqlWorksInsertStr, (id, i[0], i[1]))) for i in existingWorks.difference(newWorks): commands.append((sqlWorksDeleteStr, (id, i[0], i[1]))) sql.execute(commands=commands, config=common.config, logger=common.logger) common.logger.info(f"Found [{len(incompleteSeriesIDs)}] incomplete series:") for i in incompleteSeriesIDs: common.args.batch_out.write(f"{i}\n") common.logger.info("All operations complete, metadata.py exiting.")