mirror of
https://codeberg.org/Cyberpro123/AO3-DL.git
synced 2026-07-10 00:12:07 +00:00
284 lines
10 KiB
Python
Executable file
284 lines
10 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# This file is part of AOMO, "Archive Of My Own", a collection of Python and PHP
|
|
# scripts designed act as an improved local backup system
|
|
# for works published on https://archiveofourown.org
|
|
#
|
|
# Copyright (c) 2025 Cyberpro123, except where otherwise noted.
|
|
#
|
|
# This project is available under the GNU General Public License (GPL) v2.0,
|
|
# available at https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html
|
|
# or in the 'LICENSE.md' file that should be distributed alongside this one.
|
|
#
|
|
# Report issues to https://codeberg.org/Cyberpro123/AOMO
|
|
# Contact author at cyberpro123@posteo.com
|
|
import concurrent.futures
|
|
import os.path
|
|
import shutil
|
|
import threading
|
|
import traceback
|
|
|
|
import bs4
|
|
|
|
import common
|
|
import sql
|
|
|
|
|
|
HTML_TEMPLATE = """<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{title} - Work {workID} exported by AOMO</title>
|
|
<meta charset='UTF-8'>
|
|
{skins}
|
|
</head>
|
|
<body>
|
|
<div id='outer' class='wrapper'>
|
|
<div id='inner' class='wrapper'>
|
|
<div id='main' class='works-show region' role='main'>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def renderUserstuff(
|
|
string: str,
|
|
imagePrefix: str,
|
|
) -> bs4.BeautifulSoup:
|
|
soup = bs4.BeautifulSoup(string, features="lxml")
|
|
for i in soup.find_all("img"):
|
|
if "src" in i.attrs:
|
|
i["src"] = i["src"].replace(common.imagePrefix, imagePrefix)
|
|
return soup
|
|
|
|
|
|
def makeChapterDiv(
|
|
workID: int,
|
|
chapterNo: int,
|
|
imagePrefix: str,
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
showTitle: bool = True,
|
|
) -> bs4.Tag:
|
|
dbOut = sql.execute(
|
|
commands=[("SELECT title, summary, notes_start, notes_end, body FROM text WHERE id = ? AND chapter = ?", (workID, chapterNo))],
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
title, summary, notes_start, notes_end, body = dbOut[0][0]
|
|
outTag = bs4.Tag(name="div")
|
|
outTag["id"] = f"chapter-{chapterNo}"
|
|
outTag["class"] = "chapter"
|
|
if showTitle or summary or notes_start:
|
|
prefaceGroup = bs4.Tag(name="div", attrs={"class": "chapter preface group"})
|
|
if showTitle:
|
|
prefaceGroup.append(bs4.Tag(name="h3", attrs={"class": "title"}))
|
|
chaptNoStr = f"Chapter {chapterNo}"
|
|
if title == chaptNoStr:
|
|
prefaceGroup.h3.string = chaptNoStr
|
|
else:
|
|
prefaceGroup.h3.string = f"{chaptNoStr}: {title}"
|
|
if summary:
|
|
# putting tag in var instead of immediately appending to prefaceGroup to avoid confilct with startnotes
|
|
sumBlock = bs4.Tag(name="div", attrs={"id": "summary", "class": "summary module"})
|
|
sumBlock.append(bs4.Tag(name="h3", attrs={"class": "heading"}))
|
|
sumBlock.h3.string = "Summary:"
|
|
sumBQ = renderUserstuff(string=summary, imagePrefix=imagePrefix).html.body.extract()
|
|
sumBQ.attrs = {"class": "userstuff"}
|
|
sumBQ.name = "blockquote"
|
|
sumBlock.append(sumBQ)
|
|
prefaceGroup.append(sumBlock)
|
|
if notes_start:
|
|
nsBlock = bs4.Tag(name="div", attrs={"id": "notes", "class": "notes module"})
|
|
nsBlock.append(bs4.Tag(name="h3", attrs={"class": "heading"}))
|
|
nsBlock.h3.string = "Notes:"
|
|
nsBQ = renderUserstuff(string=notes_start, imagePrefix=imagePrefix).html.body.extract()
|
|
nsBQ.attrs = {"class": "userstuff"}
|
|
nsBQ.name = "blockquote"
|
|
nsBlock.append(nsBQ)
|
|
prefaceGroup.append(nsBlock)
|
|
outTag.append(prefaceGroup)
|
|
mainArticle = renderUserstuff(string=body, imagePrefix=imagePrefix).html.body.extract()
|
|
mainArticle.attrs = {"class": "userstuff module", "role": "article"}
|
|
mainArticle.name = "div"
|
|
outTag.append(mainArticle)
|
|
if notes_end:
|
|
afterwordGroup = bs4.Tag(name="div", attrs={"class": "chapter preface group"})
|
|
afterwordGroup.append(bs4.Tag(name="div", attrs={"class": "end notes module", "id": f"chapter_{chapterNo}_endnotes"}))
|
|
afterwordGroup.div.append(bs4.Tag(name="h3", attrs={"class": "heading"}))
|
|
afterwordGroup.div.h3.string = "Notes:"
|
|
endNoteBlock = renderUserstuff(string=notes_end, imagePrefix=imagePrefix).html.body.extract()
|
|
endNoteBlock.attrs = {"class": "userstuff"}
|
|
endNoteBlock.name = "blockquote"
|
|
afterwordGroup.append(endNoteBlock)
|
|
outTag.append(afterwordGroup)
|
|
return outTag
|
|
|
|
|
|
def makeSoupFromChapterDivs(
|
|
workID: int,
|
|
title: str,
|
|
chapters: list,
|
|
skins: str,
|
|
) -> bs4.BeautifulSoup:
|
|
soup = bs4.BeautifulSoup(HTML_TEMPLATE.format(title=title, workID=workID, skins=skins), features="lxml")
|
|
mainDiv = soup.find(id="main")
|
|
mainDiv.append(bs4.Tag(name="div", attrs={"id": "workskin"}))
|
|
chaptersDiv = bs4.Tag(name="div", attrs={"id": "chapters", "role": "article"})
|
|
for div in chapters:
|
|
chaptersDiv.append(div)
|
|
mainDiv.div.append(chaptersDiv)
|
|
return soup
|
|
|
|
|
|
def makeTagsBlurb(
|
|
workID: int,
|
|
) -> bs4.Tag:
|
|
pass
|
|
|
|
|
|
def makeSkinString(
|
|
workskin: str,
|
|
workskinName: str,
|
|
siteskin: int,
|
|
embedded: bool,
|
|
ao3css: str,
|
|
cssDir: str,
|
|
cssRelPath: str,
|
|
) -> str:
|
|
outStr = ""
|
|
for filename in sorted(os.listdir(ao3css)):
|
|
if filename[-4:] == ".css":
|
|
if embedded:
|
|
outStr += "<style type='text/css' media='"
|
|
else:
|
|
shutil.copyfile(
|
|
src=os.path.join(ao3css, filename),
|
|
dst=os.path.join(cssDir, filename),
|
|
)
|
|
outStr += f"<link rel='stylesheet' type='text/css' href='{os.path.join(cssRelPath, filename)}' media='"
|
|
match int(filename[:2]):
|
|
case 25:
|
|
outStr += "only screen and (max-width: 62em), handheld"
|
|
case 26:
|
|
outStr += "only screen and (max-width: 42em), handheld"
|
|
case 27:
|
|
outStr += "speech"
|
|
case 28:
|
|
outStr += "print"
|
|
case _:
|
|
outStr += "screen"
|
|
outStr += "'>"
|
|
if embedded:
|
|
with open(os.path.join(ao3css, filename), "r") as file:
|
|
outStr += file.read()
|
|
outStr += "</style>"
|
|
if workskin:
|
|
if embedded:
|
|
outStr += "<style type='text/css'>" + workskin + "</style>"
|
|
else:
|
|
if workskinName[-4:] != ".css":
|
|
workskinName += ".css"
|
|
with open(os.path.join(cssDir, workskinName), "w") as file:
|
|
file.write(workskin)
|
|
outStr += f"<link rel='stylesheet' type='text/css' href='{os.path.join(cssRelPath, workskinName)}'>"
|
|
return outStr
|
|
|
|
|
|
def mainSingle(
|
|
workID: int,
|
|
fileType: str,
|
|
chapters: list,
|
|
config: dict,
|
|
) -> bool:
|
|
threading.current_thread().name = f"ex_{workID}"
|
|
logger = common.logging.getLogger(__name__)
|
|
workingFolderPath = os.path.join(config["dirExport"], f"temp_{workID}")
|
|
imageFolderPath = os.path.join(workingFolderPath, "images")
|
|
cssFolderPath = os.path.join(workingFolderPath, "css")
|
|
for i in (imageFolderPath, cssFolderPath):
|
|
os.makedirs(i, exist_ok=True)
|
|
logger.info(workID)
|
|
dbOut = sql.execute(
|
|
commands=[
|
|
("SELECT count(chapter) FROM text WHERE id = ?", (workID,)),
|
|
("SELECT title, body FROM text WHERE id = ? AND chapter = 0", (workID,)),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
chaptersCount = dbOut[0][0][0]
|
|
oneshot = chaptersCount <= 2
|
|
title, workskin = dbOut[1][0]
|
|
if fileType == "html":
|
|
imagePrefix = "https://"
|
|
else:
|
|
imagePrefix = "images/"
|
|
siteskin = 0
|
|
tagBlurb = makeTagsBlurb(workID=workID)
|
|
cssStr = makeSkinString(
|
|
workskin=workskin,
|
|
workskinName=f"{workID}.css",
|
|
siteskin=siteskin,
|
|
embedded=(fileType == "html"),
|
|
ao3css=config["dirAo3Css"],
|
|
cssDir=cssFolderPath,
|
|
cssRelPath=os.path.relpath(cssFolderPath, workingFolderPath),
|
|
)
|
|
chapters = {}
|
|
for chapterNo in range(1, chaptersCount):
|
|
div = makeChapterDiv(
|
|
workID=workID,
|
|
chapterNo=chapterNo,
|
|
imagePrefix=imagePrefix,
|
|
config=config,
|
|
logger=logger,
|
|
showTitle=(not oneshot),
|
|
)
|
|
chapters[chapterNo] = div
|
|
soup = makeSoupFromChapterDivs(workID=workID, title=title, chapters=chapters.values(), skins=cssStr)
|
|
with open(os.path.join(workingFolderPath, f"{workID}-0.html"), "w") as file:
|
|
# DO NOT write with prettify(), it turns ` ` into a linebreak and spaces, which messes up the literal first page of work 27665101
|
|
file.write(str(soup))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
common.init(
|
|
description="Exports the requested Work(s) as ordinary files.",
|
|
all="Export every Work in the database",
|
|
auto="Export every Work in the database that has been modified since it was last exported",
|
|
chapters="Which chapters to export. 0 will export a 'view entire work' file and -1 will export all individual chapters",
|
|
fileType={
|
|
"help": "Which type of file to export as. All types support CSS, all but 'html' support downloaded images.",
|
|
"choices": ["tar", "gz", "bz2", "xz", "zst", "zip", "html", "zim"],
|
|
"default": "gz",
|
|
},
|
|
)
|
|
config = common.config
|
|
logger = common.logging.getLogger(__name__)
|
|
workIDs = set(())
|
|
if common.args.all or common.args.auto:
|
|
allIDs = sql.getAllEditTimePerWorkID(config=config, logger=logger)
|
|
for id in allIDs:
|
|
if common.args.all or (not allIDs[id]):
|
|
workIDs.add(id)
|
|
del allIDs
|
|
else:
|
|
inRaw = common.parseInfile(common.args.infile)
|
|
for line in inRaw:
|
|
try:
|
|
workIDs.add(int(line))
|
|
except ValueError:
|
|
common.logger.error(f"Could not convert input [{line}] to int")
|
|
futures = {}
|
|
with concurrent.futures.ProcessPoolExecutor() as pool1:
|
|
for workID in workIDs:
|
|
futures[workID] = pool1.submit(mainSingle, workID=workID, fileType=common.args.fileType, chapters=common.args.chapters, config=config)
|
|
for workID in futures:
|
|
try:
|
|
result = futures[workID].result()
|
|
except Exception as ex:
|
|
logger.error(f"Work [{workID}] raised [{type(ex).__name__}]: [{ex.args}]")
|
|
traceback.print_exception(ex, file=common.args.outfile)
|
|
else:
|
|
pass
|