mirror of
https://codeberg.org/Cyberpro123/AO3-DL.git
synced 2026-07-26 08:06:16 +00:00
377 lines
10 KiB
Python
377 lines
10 KiB
Python
import datetime
|
|
import logging
|
|
|
|
import common
|
|
|
|
|
|
initStrWorks = """
|
|
CREATE TABLE IF NOT EXISTS works(
|
|
ID INTEGER PRIMARY KEY,
|
|
chaptersCount INTEGER,
|
|
chaptersExpected INTEGER,
|
|
dateLastDownloaded INTEGER,
|
|
dateLastEdited INTEGER,
|
|
dateLastUpdated INTEGER,
|
|
datePublished INTEGER,
|
|
dateProcessed INTEGER);"""
|
|
initStrTags = """
|
|
CREATE TABLE IF NOT EXISTS tags(
|
|
ID INTEGER PRIMARY KEY,
|
|
rating INTEGER,
|
|
warnings INTEGER,
|
|
category INTEGER
|
|
"""
|
|
for tagType in (
|
|
"Fandom",
|
|
"Ship",
|
|
"Character",
|
|
"Freeform",
|
|
):
|
|
for i in range(75):
|
|
initStrTags += f", tag{tagType}{i + 1} VARCHAR(100)"
|
|
for i in range(100):
|
|
initStrTags += f", author{i + 1} VARCHAR(40)"
|
|
initStrTags = initStrTags + ");"
|
|
initStrSeries = """
|
|
CREATE TABLE IF NOT EXISTS seriesToWork(
|
|
ID INTEGER PRIMARY KEY,
|
|
title VARCHAR(255),
|
|
dateBegun INTEGER,
|
|
dateUpdated INTEGER,
|
|
dateChecked INTEGER,
|
|
description VARCHAR(1250),
|
|
notes VARCHAR(5000),
|
|
complete INTEGER
|
|
"""
|
|
for i in range(256):
|
|
initStrSeries += f", w{i + 1} INTEGER"
|
|
initStrSeries += ");"
|
|
initStrSeries2 = """
|
|
CREATE TABLE IF NOT EXISTS workToSeries(
|
|
ID INTEGER PRIMARY KEY
|
|
"""
|
|
for i in range(100):
|
|
initStrSeries2 += f", s{i + 1} INTEGER"
|
|
initStrSeries2 += f", p{i + 1} INTEGER"
|
|
initStrSeries2 += ");"
|
|
initStrText = """
|
|
CREATE TABLE IF NOT EXISTS text(
|
|
ID INTEGER,
|
|
CHAPTER INTEGER,
|
|
title VARCHAR(255),
|
|
summary VARCHAR(1250),
|
|
notesStart VARCHAR(5000),
|
|
notesEnd VARCHAR(5000),
|
|
body VARCHAR(500000),
|
|
PRIMARY KEY (ID, CHAPTER)
|
|
);
|
|
"""
|
|
|
|
|
|
def execute(
|
|
commands: list,
|
|
config: dict,
|
|
logger: logging.Logger,
|
|
) -> list:
|
|
config["sqlType"]
|
|
config["sqlLocation"]
|
|
output = []
|
|
match config["sqlType"]:
|
|
case "sqlite":
|
|
import contextlib
|
|
import os.path
|
|
import random
|
|
import sqlite3
|
|
import time
|
|
|
|
dbPath = os.path.join(config["dirWebUi"], config["sqlLocation"])
|
|
with contextlib.closing(sqlite3.connect(dbPath, timeout=10)) as con:
|
|
cur = con.cursor()
|
|
for command in commands:
|
|
logger.debug(command[0])
|
|
logger.debug(str(command[1])[:100])
|
|
loopNo = 1
|
|
while loopNo <= common.loopRetries:
|
|
try:
|
|
cur.execute(command[0], command[1])
|
|
except sqlite3.OperationalError as ex:
|
|
if "database is locked" in str(ex):
|
|
random.seed()
|
|
pauseMult = 0.5 + random.random()
|
|
pauseTime = loopNo * 5 * pauseMult
|
|
logger.log(
|
|
(10 + 20 * int(2 > loopNo)),
|
|
common.loopErrorTemplate.format(
|
|
"Doing database operation",
|
|
pauseTime,
|
|
type(ex).__name__,
|
|
ex.args,
|
|
),
|
|
)
|
|
loopNo += 1
|
|
time.sleep(pauseTime)
|
|
else:
|
|
raise ex
|
|
else:
|
|
loopNo += common.loopRetries
|
|
output.append(cur.fetchall())
|
|
con.commit()
|
|
case _:
|
|
exceptionStr = (
|
|
f"Invalid sqlType given to db.execute() [{config['sqlType']}]"
|
|
)
|
|
logger.critical(exceptionStr)
|
|
raise common.InvalidConfigurationError(exceptionStr)
|
|
return output
|
|
|
|
|
|
def init(
|
|
config: dict,
|
|
logger: logging.Logger,
|
|
) -> None:
|
|
execute(
|
|
commands=[
|
|
(initStrWorks, ()),
|
|
(initStrTags, ()),
|
|
(initStrSeries, ()),
|
|
(initStrSeries2, ()),
|
|
(initStrText, ()),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
logger.info("Initialized Database")
|
|
|
|
|
|
def addWork(
|
|
id: int,
|
|
info: dict,
|
|
config: dict,
|
|
logger: logging.Logger,
|
|
) -> None:
|
|
try:
|
|
for i in (
|
|
("nchapters", int),
|
|
("chaptersExpected", int),
|
|
("dateDl", int),
|
|
("dateEd", int),
|
|
("dateUp", int),
|
|
("datePb", int),
|
|
("rating", str),
|
|
("warnings", list),
|
|
("categories", list),
|
|
("authors", list),
|
|
("tagsFandom", list),
|
|
("tagsShips", list),
|
|
("tagsChara", list),
|
|
("tagsOther", list),
|
|
("serieses", list),
|
|
("text", dict),
|
|
):
|
|
if not isinstance(info[i[0]], i[1]):
|
|
raise KeyError
|
|
except KeyError:
|
|
exceptionStr = "Invalid info dict given to addWork() in db.py"
|
|
logger.critical(exceptionStr)
|
|
raise KeyError(exceptionStr)
|
|
sqlStrWorks = "INSERT OR REPLACE INTO works VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
sqlTplWorks = (
|
|
id,
|
|
info["nchapters"],
|
|
info["chaptersExpected"],
|
|
info["dateDl"],
|
|
info["dateEd"],
|
|
info["dateUp"],
|
|
info["datePb"],
|
|
int(datetime.datetime.now().astimezone().replace(microsecond=0).timestamp()),
|
|
)
|
|
|
|
ratingInt = 0
|
|
ratings = [
|
|
"Not Rated",
|
|
"General Audiences",
|
|
"Teen And Up Audiences",
|
|
"Mature",
|
|
"Explicit",
|
|
]
|
|
ratingInt = ratings.index(info["rating"]) + 1
|
|
|
|
warningInt = 0
|
|
warnings = [
|
|
"No Archive Warnings Apply",
|
|
"Graphic Depictions Of Violence",
|
|
"Major Character Death",
|
|
"Rape/Non-Con",
|
|
"Underage Sex",
|
|
"Creator Chose Not To Use Archive Warnings",
|
|
]
|
|
for i in info["warnings"]:
|
|
warningInt += pow(2, warnings.index(i))
|
|
|
|
categoryInt = 0
|
|
categoriesList = [
|
|
"F/F",
|
|
"F/M",
|
|
"Gen",
|
|
"M/M",
|
|
"Multi",
|
|
"Other",
|
|
]
|
|
for i in info["categories"]:
|
|
categoryInt += pow(2, categoriesList.index(i))
|
|
|
|
sqlStrTags = "INSERT OR REPLACE INTO tags VALUES (?, ?, ?, ?"
|
|
for i in range((75 * 4) + 100):
|
|
sqlStrTags += ", ?"
|
|
sqlStrTags += ")"
|
|
sqlTplTags = (
|
|
id,
|
|
ratingInt,
|
|
warningInt,
|
|
categoryInt,
|
|
)
|
|
|
|
for i in (
|
|
info["tagsFandom"],
|
|
info["tagsShips"],
|
|
info["tagsChara"],
|
|
info["tagsOther"],
|
|
):
|
|
while len(i) > 75:
|
|
i.pop(75)
|
|
while len(i) < 75:
|
|
i.append("")
|
|
sqlTplTags += tuple(i)
|
|
|
|
while len(info["authors"]) > 100:
|
|
info["authors"].pop(100)
|
|
while len(info["authors"]) < 100:
|
|
info["authors"].append("")
|
|
sqlTplTags += tuple(info["authors"])
|
|
|
|
commands = [
|
|
(sqlStrWorks, sqlTplWorks),
|
|
(sqlStrTags, sqlTplTags),
|
|
]
|
|
sqlStrSeries3 = "INSERT OR REPLACE INTO workToSeries VALUES (?"
|
|
for i in range(100):
|
|
sqlStrSeries3 += ", ?, ?"
|
|
sqlStrSeries3 += ")"
|
|
sqlTplSeries3 = [id]
|
|
# series[0] = series ID
|
|
# series[1] = work's position in that series
|
|
# series[2] = series title
|
|
for series in info["serieses"]:
|
|
sqlStrSeries1 = (
|
|
"INSERT OR IGNORE INTO seriesToWork VALUES (?, ?, ?, ?, ?, ?, ?, ?"
|
|
)
|
|
sqlTplSeries1 = [series[0], series[2], 0, 0, 0, "", "", 0]
|
|
for i in range(256):
|
|
sqlStrSeries1 += ", ?"
|
|
sqlTplSeries1.append(0)
|
|
sqlStrSeries1 += ")"
|
|
commands.append((sqlStrSeries1, sqlTplSeries1))
|
|
sqlStrSeries2 = f"UPDATE seriesToWork SET w{int(series[1])}=? WHERE ID = ?;"
|
|
sqlTplSeries2 = (id, series[0])
|
|
commands.append((sqlStrSeries2, sqlTplSeries2))
|
|
sqlTplSeries3.append(series[0])
|
|
sqlTplSeries3.append(series[1])
|
|
while len(sqlTplSeries3) < 201:
|
|
sqlTplSeries3.append(0)
|
|
commands.append((sqlStrSeries3, sqlTplSeries3))
|
|
|
|
for chaptNum in info["text"]:
|
|
sqlStrText = "INSERT OR REPLACE INTO text VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
for i in info["text"][chaptNum]:
|
|
info["text"][chaptNum][i].replace(
|
|
common.imagePrefix, common.imagePrefixSubstitute
|
|
)
|
|
sqlTplText = (
|
|
id,
|
|
chaptNum,
|
|
info["text"][chaptNum]["title"],
|
|
info["text"][chaptNum]["summary"],
|
|
info["text"][chaptNum]["notesStart"],
|
|
info["text"][chaptNum]["notesEnd"],
|
|
info["text"][chaptNum]["body"],
|
|
)
|
|
commands.append((sqlStrText, sqlTplText))
|
|
|
|
execute(
|
|
commands=commands,
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
|
|
|
|
def getWorkIDs(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> set:
|
|
output = set(())
|
|
for i in execute(
|
|
commands=[
|
|
("SELECT ID FROM works", ()),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output.add(i[0])
|
|
return output
|
|
|
|
|
|
def getCompleteWorkIDs(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> set:
|
|
output = set(())
|
|
for i in execute(
|
|
commands=[
|
|
("SELECT ID FROM works WHERE chaptersCount = chaptersExpected", ()),
|
|
],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output.add(i[0])
|
|
return output
|
|
|
|
|
|
def getSeriesIDs(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> dict:
|
|
output = {}
|
|
for i in execute(
|
|
commands=[("SELECT ID, title FROM seriesToWork", ())],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output[i[0]] = bool(i[1])
|
|
return output
|
|
|
|
|
|
def getAllEditTimePerWorkID(
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> dict:
|
|
output = {}
|
|
for i in execute(
|
|
commands=[("SELECT ID, dateLastEdited FROM works", ())],
|
|
config=config,
|
|
logger=logger,
|
|
)[0]:
|
|
output[i[0]] = i[1]
|
|
return output
|
|
|
|
|
|
def getEditTimeFromWorkID(
|
|
id: int,
|
|
config: dict,
|
|
logger: common.logging.Logger,
|
|
) -> int:
|
|
output = execute(
|
|
commands=[("SELECT dateLastEdited FROM works WHERE ID = ?;", (id,))],
|
|
config=config,
|
|
logger=logger,
|
|
)
|
|
return int(output[0][0][0])
|