mirror of
https://codeberg.org/Cyberpro123/AO3-DL.git
synced 2026-07-23 14:46:16 +00:00
375 lines
11 KiB
Python
375 lines
11 KiB
Python
# This file is part of AOMO, "Archive Of My Own", a collection of Python and PHP
|
|
# scripts designed act as an improved local backup system
|
|
# for works published on https://archiveofourown.org
|
|
#
|
|
# Copyright (c) 2025 Cyberpro123, except where otherwise noted.
|
|
#
|
|
# This project is available under the GNU General Public License (GPL) v2.0,
|
|
# available at https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html
|
|
# or in the 'LICENSE.md' file that should be distributed alongside this one.
|
|
#
|
|
# Report issues to https://codeberg.org/Cyberpro123/AOMO
|
|
# Contact author at cyberpro123@posteo.com
|
|
import datetime
|
|
import logging
|
|
|
|
import common
|
|
|
|
|
|
initStrWorks = """
|
|
CREATE TABLE IF NOT EXISTS works(
|
|
ID INTEGER PRIMARY KEY,
|
|
chaptersCount INTEGER,
|
|
chaptersExpected INTEGER,
|
|
dateLastDownloaded INTEGER,
|
|
dateLastEdited INTEGER,
|
|
dateLastUpdated INTEGER,
|
|
datePublished INTEGER,
|
|
dateProcessed INTEGER);"""
|
|
initStrTags = """
|
|
CREATE TABLE IF NOT EXISTS tags(
|
|
workID INTEGER,
|
|
tag VARCHAR(150),
|
|
category INTEGER,
|
|
sequence INTEGER)
|
|
"""
|
|
# values for tags table category column:
|
|
# 1: Fandom
|
|
# 2: Ships
|
|
# 3: Characters
|
|
# 4: Freeform
|
|
# 5: Languages
|
|
# 6: Ratings
|
|
# 7: Warnings
|
|
# 8: Categories
|
|
# 9: Authors
|
|
initStrSeries = """
|
|
CREATE TABLE IF NOT EXISTS seriesToWork(
|
|
ID INTEGER PRIMARY KEY,
|
|
title VARCHAR(255),
|
|
dateBegun INTEGER,
|
|
dateUpdated INTEGER,
|
|
dateChecked INTEGER,
|
|
description VARCHAR(1250),
|
|
notes VARCHAR(5000),
|
|
complete INTEGER
|
|
"""
|
|
for i in range(256):
|
|
initStrSeries += f", w{i + 1} INTEGER"
|
|
initStrSeries += ");"
|
|
initStrSeries2 = """
|
|
CREATE TABLE IF NOT EXISTS workToSeries(
|
|
ID INTEGER PRIMARY KEY
|
|
"""
|
|
for i in range(100):
|
|
initStrSeries2 += f", s{i + 1} INTEGER"
|
|
initStrSeries2 += f", p{i + 1} INTEGER"
|
|
initStrSeries2 += ");"
|
|
initStrText = """
|
|
CREATE TABLE IF NOT EXISTS text(
|
|
ID INTEGER,
|
|
CHAPTER INTEGER,
|
|
title VARCHAR(255),
|
|
summary VARCHAR(1250),
|
|
notesStart VARCHAR(5000),
|
|
notesEnd VARCHAR(5000),
|
|
body VARCHAR(500000),
|
|
PRIMARY KEY (ID, CHAPTER)
|
|
);
|
|
"""
|
|
|
|
|
|
def execute(
|
|
commands: list,
|
|
config: dict,
|
|
logger: logging.Logger,
|
|
) -> list:
|
|
config["sqlType"]
|
|
config["sqlLocation"]
|
|
output = []
|
|
match config["sqlType"]:
|
|
case "sqlite":
|
|
import contextlib
|
|
import os.path
|
|
import random
|
|
import sqlite3
|
|
import time
|
|
|
|
dbPath = os.path.join(config["dirWebUi"], config["sqlLocation"])
|
|
with contextlib.closing(sqlite3.connect(dbPath, timeout=10)) as con:
|
|
cur = con.cursor()
|
|
for command in commands:
|
|
logger.debug(command[0])
|
|
logger.debug(str(command[1])[:100])
|
|
loopNo = 1
|
|
while loopNo <= common.loopRetries:
|
|
try:
|
|
cur.execute(command[0], command[1])
|
|
except sqlite3.OperationalError as ex:
|
|
if "database is locked" in str(ex):
|
|
random.seed()
|
|
pauseMult = 0.5 + random.random()
|
|
pauseTime = loopNo * 5 * pauseMult
|
|
logger.log(
|
|
(10 + 20 * int(2 > loopNo)),
|
|
common.loopErrorTemplate.format(
|
|
"Doing database operation",
|
|
pauseTime,
|
|
type(ex).__name__,
|
|
ex.args,
|
|
),
|
|
)
|
|
loopNo += 1
|
|
time.sleep(pauseTime)
|
|
else:
|
|
raise ex
|
|
else:
|
|
loopNo += common.loopRetries
|
|
output.append(cur.fetchall())
|
|
con.commit()
|
|
case _:
|
|
exceptionStr = (
|
|
f"Invalid sqlType given to db.execute() [{config['sqlType']}]"
|
|
)
|
|
logger.critical(exceptionStr)
|
|
raise common.InvalidConfigurationError(exceptionStr)
|
|
return output
|
|
|
|
|
|
def init(
|
|
config: dict,
|
|
logger: logging.Logger,
|
|
) -> None:
|
|
execute(
|
|
commands=[
|
|
(initStrWorks, ()),
|
|
(initStrTags, ()),
|
|
(initStrSeries, ()),
|
|
(initStrSeries2, ()),
|
|
(initStrText, ()),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
logger.info("Initialized Database")
|
|
|
|
|
|
def addWork(
|
|
id: int,
|
|
info: dict,
|
|
config: dict,
|
|
logger: logging.Logger,
|
|
) -> None:
|
|
try:
|
|
for i in (
|
|
("nchapters", int),
|
|
("chaptersExpected", int),
|
|
("dateDl", int),
|
|
("dateEd", int),
|
|
("dateUp", int),
|
|
("datePb", int),
|
|
("rating", str),
|
|
("warnings", list),
|
|
("categories", list),
|
|
("authors", list),
|
|
("tagsFandom", list),
|
|
("tagsShips", list),
|
|
("tagsChara", list),
|
|
("tagsOther", list),
|
|
("serieses", list),
|
|
("text", dict),
|
|
):
|
|
if not isinstance(info[i[0]], i[1]):
|
|
raise KeyError
|
|
except KeyError:
|
|
exceptionStr = "Invalid info dict given to addWork() in db.py"
|
|
logger.critical(exceptionStr)
|
|
raise KeyError(exceptionStr)
|
|
sqlStrWorks = "INSERT OR REPLACE INTO works VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
sqlTplWorks = (
|
|
id,
|
|
info["nchapters"],
|
|
info["chaptersExpected"],
|
|
info["dateDl"],
|
|
info["dateEd"],
|
|
info["dateUp"],
|
|
info["datePb"],
|
|
int(datetime.datetime.now().astimezone().replace(microsecond=0).timestamp()),
|
|
)
|
|
|
|
def listToDesiredTagSets(
|
|
enumerable: list,
|
|
catNo: int,
|
|
) -> set:
|
|
outList = set(())
|
|
for num, i in enumerate(enumerable):
|
|
outList.add((i, catNo, (num + 1)))
|
|
return outList
|
|
|
|
desiredTags = set(())
|
|
desiredTags.add((info["rating"], 6, 1))
|
|
desiredTags.update(listToDesiredTagSets(info["warnings"], 7))
|
|
desiredTags.update(listToDesiredTagSets(info["categories"], 8))
|
|
desiredTags.update(listToDesiredTagSets(info["tagsFandom"], 1))
|
|
desiredTags.update(listToDesiredTagSets(info["tagsShips"], 2))
|
|
desiredTags.update(listToDesiredTagSets(info["tagsChara"], 3))
|
|
desiredTags.update(listToDesiredTagSets(info["tagsOther"], 4))
|
|
desiredTags.update(listToDesiredTagSets(info["authors"], 9))
|
|
|
|
existingTags = getTags(id, config, logger)
|
|
|
|
tagsToInsert = desiredTags.difference(existingTags)
|
|
tagsToDelete = existingTags.difference(desiredTags)
|
|
|
|
commands = [
|
|
(sqlStrWorks, sqlTplWorks),
|
|
]
|
|
for i in tagsToInsert:
|
|
commands.append(
|
|
("INSERT INTO tags VALUES (?, ?, ?, ?)", (id, i[0], i[1], i[2]))
|
|
)
|
|
for i in tagsToDelete:
|
|
commands.append(
|
|
(
|
|
"DELETE FROM tags WHERE workID = ? AND tag = ? AND category = ? AND sequence = ?",
|
|
(id, i[0], i[1], i[2]),
|
|
)
|
|
)
|
|
sqlStrSeries3 = "INSERT OR REPLACE INTO workToSeries VALUES (?"
|
|
for i in range(100):
|
|
sqlStrSeries3 += ", ?, ?"
|
|
sqlStrSeries3 += ")"
|
|
sqlTplSeries3 = [id]
|
|
# series[0] = series ID
|
|
# series[1] = work's position in that series
|
|
# series[2] = series title
|
|
for series in info["serieses"]:
|
|
sqlStrSeries1 = (
|
|
"INSERT OR IGNORE INTO seriesToWork VALUES (?, ?, ?, ?, ?, ?, ?, ?"
|
|
)
|
|
sqlTplSeries1 = [series[0], series[2], 0, 0, 0, "", "", 0]
|
|
for i in range(256):
|
|
sqlStrSeries1 += ", ?"
|
|
sqlTplSeries1.append(0)
|
|
sqlStrSeries1 += ")"
|
|
commands.append((sqlStrSeries1, sqlTplSeries1))
|
|
sqlStrSeries2 = f"UPDATE seriesToWork SET w{int(series[1])}=? WHERE ID = ?;"
|
|
sqlTplSeries2 = (id, series[0])
|
|
commands.append((sqlStrSeries2, sqlTplSeries2))
|
|
sqlTplSeries3.append(series[0])
|
|
sqlTplSeries3.append(series[1])
|
|
while len(sqlTplSeries3) < 201:
|
|
sqlTplSeries3.append(0)
|
|
commands.append((sqlStrSeries3, sqlTplSeries3))
|
|
|
|
for chaptNum in info["text"]:
|
|
sqlStrText = "INSERT OR REPLACE INTO text VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
for i in info["text"][chaptNum]:
|
|
info["text"][chaptNum][i].replace(
|
|
common.imagePrefix, common.imagePrefixSubstitute
|
|
)
|
|
sqlTplText = (
|
|
id,
|
|
chaptNum,
|
|
info["text"][chaptNum]["title"],
|
|
info["text"][chaptNum]["summary"],
|
|
info["text"][chaptNum]["notesStart"],
|
|
info["text"][chaptNum]["notesEnd"],
|
|
info["text"][chaptNum]["body"],
|
|
)
|
|
commands.append((sqlStrText, sqlTplText))
|
|
|
|
execute(
|
|
commands=commands,
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
|
|
|
|
def getWorkIDs(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> set:
|
|
output = set(())
|
|
for i in execute(
|
|
commands=[
|
|
("SELECT ID FROM works", ()),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output.add(i[0])
|
|
return output
|
|
|
|
|
|
def getCompleteWorkIDs(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> set:
|
|
output = set(())
|
|
for i in execute(
|
|
commands=[
|
|
("SELECT ID FROM works WHERE chaptersCount = chaptersExpected", ()),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output.add(i[0])
|
|
return output
|
|
|
|
|
|
def getSeriesIDs(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> dict:
|
|
output = {}
|
|
for i in execute(
|
|
commands=[("SELECT ID, title FROM seriesToWork", ())],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output[i[0]] = bool(i[1])
|
|
return output
|
|
|
|
|
|
def getAllEditTimePerWorkID(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> dict:
|
|
output = {}
|
|
for i in execute(
|
|
commands=[("SELECT ID, dateLastEdited FROM works", ())],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output[i[0]] = i[1]
|
|
return output
|
|
|
|
|
|
def getEditTimeFromWorkID(
|
|
id: int,
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> int:
|
|
output = execute(
|
|
commands=[("SELECT dateLastEdited FROM works WHERE ID = ?;", (id,))],
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
return int(output[0][0][0])
|
|
|
|
|
|
def getTags(
|
|
id: int,
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> set:
|
|
return set(
|
|
execute(
|
|
commands=[
|
|
("SELECT tag, category, sequence FROM tags WHERE workID = ?;", (id,))
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]
|
|
)
|