#!/usr/bin/env python3
# This file is part of AOMO, "Archive Of My Own", a collection of Python and PHP
# scripts designed act as an improved local backup system
# for works published on https://archiveofourown.org
#
# Copyright (c) 2025 Cyberpro123, except where otherwise noted.
#
# This project is available under the GNU General Public License (GPL) v2.0,
# available at https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html
# or in the 'LICENSE.md' file that should be distributed alongside this one.
#
# Report issues to https://codeberg.org/Cyberpro123/AOMO
# Contact author at cyberpro123@posteo.com
import concurrent.futures
import os.path
import shutil
import threading
import traceback
import bs4
import common
import sql
HTML_TEMPLATE = """
{title} - Work {workID} exported by AOMO
{skins}
"""
def renderUserstuff(
string: str,
imagePrefix: str,
) -> bs4.BeautifulSoup:
soup = bs4.BeautifulSoup(string, features="lxml")
for i in soup.find_all("img"):
if "src" in i.attrs:
i["src"] = i["src"].replace(common.imagePrefix, imagePrefix)
return soup
def makeChapterDiv(
workID: int,
chapterNo: int,
imagePrefix: str,
config: dict,
logger: common.logging.Logger,
showTitle: bool = True,
) -> bs4.Tag:
dbOut = sql.execute(
commands=[("SELECT title, summary, notes_start, notes_end, body FROM text WHERE id = ? AND chapter = ?", (workID, chapterNo))],
config=config,
logger=logger,
)
title, summary, notes_start, notes_end, body = dbOut[0][0]
outTag = bs4.Tag(name="div")
outTag["id"] = f"chapter-{chapterNo}"
outTag["class"] = "chapter"
if showTitle or summary or notes_start:
prefaceGroup = bs4.Tag(name="div", attrs={"class": "chapter preface group"})
if showTitle:
prefaceGroup.append(bs4.Tag(name="h3", attrs={"class": "title"}))
chaptNoStr = f"Chapter {chapterNo}"
if title == chaptNoStr:
prefaceGroup.h3.string = chaptNoStr
else:
prefaceGroup.h3.string = f"{chaptNoStr}: {title}"
if summary:
# putting tag in var instead of immediately appending to prefaceGroup to avoid confilct with startnotes
sumBlock = bs4.Tag(name="div", attrs={"id": "summary", "class": "summary module"})
sumBlock.append(bs4.Tag(name="h3", attrs={"class": "heading"}))
sumBlock.h3.string = "Summary:"
sumBQ = renderUserstuff(string=summary, imagePrefix=imagePrefix).html.body.extract()
sumBQ.attrs = {"class": "userstuff"}
sumBQ.name = "blockquote"
sumBlock.append(sumBQ)
prefaceGroup.append(sumBlock)
if notes_start:
nsBlock = bs4.Tag(name="div", attrs={"id": "notes", "class": "notes module"})
nsBlock.append(bs4.Tag(name="h3", attrs={"class": "heading"}))
nsBlock.h3.string = "Notes:"
nsBQ = renderUserstuff(string=notes_start, imagePrefix=imagePrefix).html.body.extract()
nsBQ.attrs = {"class": "userstuff"}
nsBQ.name = "blockquote"
nsBlock.append(nsBQ)
prefaceGroup.append(nsBlock)
outTag.append(prefaceGroup)
mainArticle = renderUserstuff(string=body, imagePrefix=imagePrefix).html.body.extract()
mainArticle.attrs = {"class": "userstuff module", "role": "article"}
mainArticle.name = "div"
outTag.append(mainArticle)
if notes_end:
afterwordGroup = bs4.Tag(name="div", attrs={"class": "chapter preface group"})
afterwordGroup.append(bs4.Tag(name="div", attrs={"class": "end notes module", "id": f"chapter_{chapterNo}_endnotes"}))
afterwordGroup.div.append(bs4.Tag(name="h3", attrs={"class": "heading"}))
afterwordGroup.div.h3.string = "Notes:"
endNoteBlock = renderUserstuff(string=notes_end, imagePrefix=imagePrefix).html.body.extract()
endNoteBlock.attrs = {"class": "userstuff"}
endNoteBlock.name = "blockquote"
afterwordGroup.append(endNoteBlock)
outTag.append(afterwordGroup)
return outTag
def makeSoupFromChapterDivs(
workID: int,
title: str,
chapters: list,
skins: str,
) -> bs4.BeautifulSoup:
soup = bs4.BeautifulSoup(HTML_TEMPLATE.format(title=title, workID=workID, skins=skins), features="lxml")
mainDiv = soup.find(id="main")
mainDiv.append(bs4.Tag(name="div", attrs={"id": "workskin"}))
chaptersDiv = bs4.Tag(name="div", attrs={"id": "chapters", "role": "article"})
for div in chapters:
chaptersDiv.append(div)
mainDiv.div.append(chaptersDiv)
return soup
def makeTagsBlurb(
workID: int,
) -> bs4.Tag:
pass
def makeSkinString(
workskin: str,
workskinName: str,
siteskin: int,
embedded: bool,
ao3css: str,
cssDir: str,
cssRelPath: str,
) -> str:
outStr = ""
for filename in sorted(os.listdir(ao3css)):
if filename[-4:] == ".css":
if embedded:
outStr += ""
if workskin:
if embedded:
outStr += ""
else:
if workskinName[-4:] != ".css":
workskinName += ".css"
with open(os.path.join(cssDir, workskinName), "w") as file:
file.write(workskin)
outStr += f""
return outStr
def mainSingle(
workID: int,
fileType: str,
chapters: list,
config: dict,
) -> bool:
threading.current_thread().name = f"ex_{workID}"
logger = common.logging.getLogger(__name__)
workingFolderPath = os.path.join(config["dirExport"], f"temp_{workID}")
imageFolderPath = os.path.join(workingFolderPath, "images")
cssFolderPath = os.path.join(workingFolderPath, "css")
for i in (imageFolderPath, cssFolderPath):
os.makedirs(i, exist_ok=True)
logger.info(workID)
dbOut = sql.execute(
commands=[
("SELECT count(chapter) FROM text WHERE id = ?", (workID,)),
("SELECT title, body FROM text WHERE id = ? AND chapter = 0", (workID,)),
],
config=config,
logger=logger,
)
chaptersCount = dbOut[0][0][0]
oneshot = chaptersCount <= 2
title, workskin = dbOut[1][0]
if fileType == "html":
imagePrefix = "https://"
else:
imagePrefix = "images/"
siteskin = 0
tagBlurb = makeTagsBlurb(workID=workID)
cssStr = makeSkinString(
workskin=workskin,
workskinName=f"{workID}.css",
siteskin=siteskin,
embedded=(fileType == "html"),
ao3css=config["dirAo3Css"],
cssDir=cssFolderPath,
cssRelPath=os.path.relpath(cssFolderPath, workingFolderPath),
)
chapters = {}
for chapterNo in range(1, chaptersCount):
div = makeChapterDiv(
workID=workID,
chapterNo=chapterNo,
imagePrefix=imagePrefix,
config=config,
logger=logger,
showTitle=(not oneshot),
)
chapters[chapterNo] = div
soup = makeSoupFromChapterDivs(workID=workID, title=title, chapters=chapters.values(), skins=cssStr)
with open(os.path.join(workingFolderPath, f"{workID}-0.html"), "w") as file:
# DO NOT write with prettify(), it turns ` ` into a linebreak and spaces, which messes up the literal first page of work 27665101
file.write(str(soup))
if __name__ == "__main__":
common.init(
description="Exports the requested Work(s) as ordinary files.",
all="Export every Work in the database",
auto="Export every Work in the database that has been modified since it was last exported",
chapters="Which chapters to export. 0 will export a 'view entire work' file and -1 will export all individual chapters",
fileType={
"help": "Which type of file to export as. All types support CSS, all but 'html' support downloaded images.",
"choices": ["tar", "gz", "bz2", "xz", "zst", "zip", "html", "zim"],
"default": "gz",
},
)
config = common.config
logger = common.logging.getLogger(__name__)
workIDs = set(())
if common.args.all or common.args.auto:
allIDs = sql.getAllEditTimePerWorkID(config=config, logger=logger)
for id in allIDs:
if common.args.all or (not allIDs[id]):
workIDs.add(id)
del allIDs
else:
inRaw = common.parseInfile(common.args.infile)
for line in inRaw:
try:
workIDs.add(int(line))
except ValueError:
common.logger.error(f"Could not convert input [{line}] to int")
futures = {}
with concurrent.futures.ProcessPoolExecutor() as pool1:
for workID in workIDs:
futures[workID] = pool1.submit(mainSingle, workID=workID, fileType=common.args.fileType, chapters=common.args.chapters, config=config)
for workID in futures:
try:
result = futures[workID].result()
except Exception as ex:
logger.error(f"Work [{workID}] raised [{type(ex).__name__}]: [{ex.args}]")
traceback.print_exception(ex, file=common.args.outfile)
else:
pass