import datetime import logging import common initStrWorks = """ CREATE TABLE IF NOT EXISTS works( ID INTEGER PRIMARY KEY, chaptersCount INTEGER, chaptersExpected INTEGER, dateLastDownloaded INTEGER, dateLastEdited INTEGER, dateLastUpdated INTEGER, datePublished INTEGER, dateProcessed INTEGER);""" initStrTags = """ CREATE TABLE IF NOT EXISTS tags( ID INTEGER PRIMARY KEY, rating INTEGER, warnings INTEGER, category INTEGER """ for tagType in ( "Fandom", "Ship", "Character", "Freeform", ): for i in range(75): initStrTags += f", tag{tagType}{i + 1} VARCHAR(100)" for i in range(100): initStrTags += f", author{i + 1} VARCHAR(40)" initStrTags = initStrTags + ");" initStrSeries = """ CREATE TABLE IF NOT EXISTS seriesToWork( ID INTEGER PRIMARY KEY, title VARCHAR(255), dateBegun INTEGER, dateUpdated INTEGER, dateChecked INTEGER, description VARCHAR(1250), notes VARCHAR(5000), complete INTEGER """ for i in range(256): initStrSeries += f", w{i + 1} INTEGER" initStrSeries += ");" initStrSeries2 = """ CREATE TABLE IF NOT EXISTS workToSeries( ID INTEGER PRIMARY KEY """ for i in range(100): initStrSeries2 += f", s{i + 1} INTEGER" initStrSeries2 += f", p{i + 1} INTEGER" initStrSeries2 += ");" initStrText = """ CREATE TABLE IF NOT EXISTS text( ID INTEGER, CHAPTER INTEGER, title VARCHAR(255), summary VARCHAR(1250), notesStart VARCHAR(5000), notesEnd VARCHAR(5000), body VARCHAR(500000), PRIMARY KEY (ID, CHAPTER) ); """ def execute( commands: list, config: dict, logger: logging.Logger, ) -> list: config["sqlType"] config["sqlLocation"] output = [] match config["sqlType"]: case "sqlite": import contextlib import os.path import random import sqlite3 import time dbPath = os.path.join(config["dirWebUi"], config["sqlLocation"]) with contextlib.closing(sqlite3.connect(dbPath, timeout=10)) as con: cur = con.cursor() for command in commands: logger.debug(command[0]) logger.debug(str(command[1])[:100]) loopNo = 1 while loopNo <= common.loopRetries: try: cur.execute(command[0], command[1]) except sqlite3.OperationalError as ex: if "database is locked" in str(ex): random.seed() pauseMult = 0.5 + random.random() pauseTime = loopNo * 5 * pauseMult logger.log( (10 + 20 * int(2 > loopNo)), common.loopErrorTemplate.format( "Doing database operation", pauseTime, type(ex).__name__, ex.args, ), ) loopNo += 1 time.sleep(pauseTime) else: raise ex else: loopNo += common.loopRetries output.append(cur.fetchall()) con.commit() case _: exceptionStr = ( f"Invalid sqlType given to db.execute() [{config['sqlType']}]" ) logger.critical(exceptionStr) raise common.InvalidConfigurationError(exceptionStr) return output def init( config: dict, logger: logging.Logger, ) -> None: execute( commands=[ (initStrWorks, ()), (initStrTags, ()), (initStrSeries, ()), (initStrSeries2, ()), (initStrText, ()), ], config=config, logger=logger, ) logger.info("Initialized Database") def addWork( id: int, info: dict, config: dict, logger: logging.Logger, ) -> None: try: for i in ( ("nchapters", int), ("chaptersExpected", int), ("dateDl", int), ("dateEd", int), ("dateUp", int), ("datePb", int), ("rating", str), ("warnings", list), ("categories", list), ("authors", list), ("tagsFandom", list), ("tagsShips", list), ("tagsChara", list), ("tagsOther", list), ("serieses", list), ("text", dict), ): if not isinstance(info[i[0]], i[1]): raise KeyError except KeyError: exceptionStr = "Invalid info dict given to addWork() in db.py" logger.critical(exceptionStr) raise KeyError(exceptionStr) sqlStrWorks = "INSERT OR REPLACE INTO works VALUES (?, ?, ?, ?, ?, ?, ?, ?)" sqlTplWorks = ( id, info["nchapters"], info["chaptersExpected"], info["dateDl"], info["dateEd"], info["dateUp"], info["datePb"], int(datetime.datetime.now().astimezone().replace(microsecond=0).timestamp()), ) ratingInt = 0 ratings = [ "Not Rated", "General Audiences", "Teen And Up Audiences", "Mature", "Explicit", ] ratingInt = ratings.index(info["rating"]) + 1 warningInt = 0 warnings = [ "No Archive Warnings Apply", "Graphic Depictions Of Violence", "Major Character Death", "Rape/Non-Con", "Underage Sex", "Creator Chose Not To Use Archive Warnings", ] for i in info["warnings"]: warningInt += pow(2, warnings.index(i)) categoryInt = 0 categoriesList = [ "F/F", "F/M", "Gen", "M/M", "Multi", "Other", ] for i in info["categories"]: categoryInt += pow(2, categoriesList.index(i)) sqlStrTags = "INSERT OR REPLACE INTO tags VALUES (?, ?, ?, ?" for i in range((75 * 4) + 100): sqlStrTags += ", ?" sqlStrTags += ")" sqlTplTags = ( id, ratingInt, warningInt, categoryInt, ) for i in ( info["tagsFandom"], info["tagsShips"], info["tagsChara"], info["tagsOther"], ): while len(i) > 75: i.pop(75) while len(i) < 75: i.append("") sqlTplTags += tuple(i) while len(info["authors"]) > 100: info["authors"].pop(100) while len(info["authors"]) < 100: info["authors"].append("") sqlTplTags += tuple(info["authors"]) commands = [ (sqlStrWorks, sqlTplWorks), (sqlStrTags, sqlTplTags), ] sqlStrSeries3 = "INSERT OR REPLACE INTO workToSeries VALUES (?" for i in range(100): sqlStrSeries3 += ", ?, ?" sqlStrSeries3 += ")" sqlTplSeries3 = [id] # series[0] = series ID # series[1] = work's position in that series # series[2] = series title for series in info["serieses"]: sqlStrSeries1 = ( "INSERT OR IGNORE INTO seriesToWork VALUES (?, ?, ?, ?, ?, ?, ?, ?" ) sqlTplSeries1 = [series[0], series[2], 0, 0, 0, "", "", 0] for i in range(256): sqlStrSeries1 += ", ?" sqlTplSeries1.append(0) sqlStrSeries1 += ")" commands.append((sqlStrSeries1, sqlTplSeries1)) sqlStrSeries2 = f"UPDATE seriesToWork SET w{int(series[1])}=? WHERE ID = ?;" sqlTplSeries2 = (id, series[0]) commands.append((sqlStrSeries2, sqlTplSeries2)) sqlTplSeries3.append(series[0]) sqlTplSeries3.append(series[1]) while len(sqlTplSeries3) < 201: sqlTplSeries3.append(0) commands.append((sqlStrSeries3, sqlTplSeries3)) for chaptNum in info["text"]: sqlStrText = "INSERT OR REPLACE INTO text VALUES (?, ?, ?, ?, ?, ?, ?)" for i in info["text"][chaptNum]: info["text"][chaptNum][i].replace( common.imagePrefix, common.imagePrefixSubstitute ) sqlTplText = ( id, chaptNum, info["text"][chaptNum]["title"], info["text"][chaptNum]["summary"], info["text"][chaptNum]["notesStart"], info["text"][chaptNum]["notesEnd"], info["text"][chaptNum]["body"], ) commands.append((sqlStrText, sqlTplText)) execute( commands=commands, config=config, logger=logger, ) def getWorkIDs( config: dict, logger: common.logging.Logger, ) -> set: output = set(()) for i in execute( commands=[ ("SELECT ID FROM works", ()), ], config=config, logger=logger, )[0]: output.add(i[0]) return output def getCompleteWorkIDs( config: dict, logger: common.logging.Logger, ) -> set: output = set(()) for i in execute( commands=[ ("SELECT ID FROM works WHERE chaptersCount = chaptersExpected", ()), ], config=config, logger=logger, )[0]: output.add(i[0]) return output def getSeriesIDs( config: dict, logger: common.logging.Logger, ) -> dict: output = {} for i in execute( commands=[("SELECT ID, title FROM seriesToWork", ())], config=config, logger=logger, )[0]: output[i[0]] = bool(i[1]) return output def getAllEditTimePerWorkID( config: dict, logger: common.logging.Logger, ) -> dict: output = {} for i in execute( commands=[("SELECT ID, dateLastEdited FROM works", ())], config=config, logger=logger, )[0]: output[i[0]] = i[1] return output def getEditTimeFromWorkID( id: int, config: dict, logger: common.logging.Logger, ) -> int: output = execute( commands=[("SELECT dateLastEdited FROM works WHERE ID = ?;", (id,))], config=config, logger=logger, ) return int(output[0][0][0])