mirror of
https://github.com/yaroslaff/antifraud2gis.git
synced 2026-10-15 13:44:14 +00:00
happy_long_relations
This commit is contained in:
Yaroslav Polyakov
parent
605a38e1c1
commit
29f3d44f82
25 changed files with 682 additions and 430 deletions
|
|
@ -23,6 +23,8 @@ jq -s 'map(.score.WSS) | add / length' /tmp/export.jsonl
|
|||
jq -s 'map(select(.score.WSS > 0.5))' /tmp/export.jsonl
|
||||
jq -s '.[] | select(.score.WSS > 0.5)' /tmp/export.jsonl
|
||||
|
||||
# raw files
|
||||
zcat .af2gis-storage/companies/nnnnnn-reviews.json.gz | jq 'length'
|
||||
|
||||
~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ Repository = "https://github.com/yaroslaff/scrape2gis"
|
|||
|
||||
[project.scripts]
|
||||
af2gis = "antifraud2gis.cli.main:main"
|
||||
af2dev = "antifraud2gis.cli.dev:main"
|
||||
af2worker = "antifraud2gis.cli.af2worker:main"
|
||||
af2web = "antifraud2gis.cli.af2web:main"
|
||||
|
||||
|
|
|
|||
|
|
@ -86,3 +86,9 @@ aliases = {
|
|||
'alias': 'sheremetevo',
|
||||
},
|
||||
}
|
||||
|
||||
def resolve_alias(alias: str):
|
||||
for k, v in aliases.items():
|
||||
if v.get('alias') == alias:
|
||||
return k
|
||||
|
||||
|
|
@ -18,6 +18,8 @@ from rich import print_json
|
|||
from ..company import Company, CompanyList
|
||||
from ..exceptions import AFReportNotReady, AFNoCompany
|
||||
from ..tasks import fraud_task
|
||||
from ..settings import settings
|
||||
from ..const import REDIS_TASK_QUEUE_NAME, REDIS_TRUSTED_LIST, REDIS_UNTRUSTED_LIST, REDIS_WORKER_STATUS
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
|
@ -27,8 +29,6 @@ templates_path = importlib.resources.files("antifraud2gis") / "templates"
|
|||
templates = Jinja2Templates(directory=templates_path)
|
||||
app.mount("/static", StaticFiles(directory=static_path), name="static")
|
||||
|
||||
dqname="dramatiq:default"
|
||||
|
||||
r = redis.Redis(decode_responses=True)
|
||||
|
||||
|
||||
|
|
@ -39,8 +39,8 @@ class ReportRequest(BaseModel):
|
|||
async def home(request: Request):
|
||||
|
||||
|
||||
last_trusted = [json.loads(item) for item in r.lrange('af2gis:last_trusted', 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange('af2gis:last_untrusted', 0, -1)]
|
||||
last_trusted = [json.loads(item) for item in r.lrange(REDIS_TRUSTED_LIST, 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"index.html",
|
||||
|
|
@ -95,47 +95,37 @@ async def recent(request: Request):
|
|||
|
||||
return data
|
||||
|
||||
@app.post("/api/report")
|
||||
async def api_report(data: ReportRequest):
|
||||
print("report for", data.oid)
|
||||
c = Company(data.oid)
|
||||
print("report for", c)
|
||||
|
||||
try:
|
||||
print("read report from", c.report_path)
|
||||
with gzip.open(c.report_path, "rt") as fh:
|
||||
report = json.load(fh)
|
||||
return report
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(status_code=404, detail="Report not found")
|
||||
|
||||
return "OK"
|
||||
|
||||
@app.get("/report/{oid}", response_class=HTMLResponse)
|
||||
async def report(request: Request, oid: str):
|
||||
print("HTML report for", oid)
|
||||
try:
|
||||
c = Company(oid)
|
||||
except AFNoCompany:
|
||||
print("redirect.....")
|
||||
return RedirectResponse(request.url_for("miss", oid=oid))
|
||||
# raise HTTPException(status_code=404, detail="Company not found")
|
||||
print("report for", c)
|
||||
|
||||
try:
|
||||
print("read report from", c.report_path)
|
||||
with gzip.open(c.report_path, "rt") as fh:
|
||||
report = json.load(fh)
|
||||
# print_json(data=report)
|
||||
# print(report['relations'][0])
|
||||
|
||||
last_trusted = [json.loads(item) for item in r.lrange('af2gis:last_trusted', 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange('af2gis:last_untrusted', 0, -1)]
|
||||
for rel in report['relations']:
|
||||
rep_path = settings.company_storage / (rel['oid'] + '-report.json.gz')
|
||||
if rep_path.exists():
|
||||
with gzip.open(rep_path, "rt") as fh:
|
||||
rel_report = json.load(fh)
|
||||
rel['trusted'] = rel_report['score']['trusted']
|
||||
else:
|
||||
rel['trusted'] = None
|
||||
|
||||
last_trusted = [json.loads(item) for item in r.lrange(REDIS_TRUSTED_LIST, 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
|
||||
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"report.html", {
|
||||
"request": request,
|
||||
"request": request,
|
||||
"settings": settings,
|
||||
"c": c,
|
||||
"title": c.title,
|
||||
"score": report['score'],
|
||||
|
|
@ -145,7 +135,6 @@ async def report(request: Request, oid: str):
|
|||
}
|
||||
)
|
||||
|
||||
return report
|
||||
except FileNotFoundError:
|
||||
# return
|
||||
return RedirectResponse(request.url_for("miss", oid=oid))
|
||||
|
|
@ -156,12 +145,11 @@ async def report(request: Request, oid: str):
|
|||
@app.get("/miss/{oid}", response_class=HTMLResponse)
|
||||
async def miss(request: Request, oid: str):
|
||||
|
||||
last_trusted = [json.loads(item) for item in r.lrange('af2gis:last_trusted', 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange('af2gis:last_untrusted', 0, -1)]
|
||||
last_trusted = [json.loads(item) for item in r.lrange(REDIS_TRUSTED_LIST, 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
|
||||
|
||||
try:
|
||||
c = Company(oid)
|
||||
print("miss for", c)
|
||||
|
||||
except (AFNoCompany, AssertionError):
|
||||
return templates.TemplateResponse(
|
||||
|
|
@ -185,13 +173,11 @@ async def miss(request: Request, oid: str):
|
|||
|
||||
@app.post("/submit", response_class=HTMLResponse)
|
||||
async def submit(request: Request, oid: str = Form(...)):
|
||||
print("submit", oid)
|
||||
task_id = fraud_task.send(oid).message_id
|
||||
print("submit task_id", task_id)
|
||||
task_data = r.get(f"dramatiq:message:{task_id}")
|
||||
print("submit task data", task_data) # Shows serialized task details
|
||||
fraud_task.send(oid)
|
||||
|
||||
return RedirectResponse(request.url_for("progress", oid=oid, task_id=task_id), status_code=303)
|
||||
r.rpush('af2gis:queue', oid)
|
||||
|
||||
return RedirectResponse(request.url_for("progress", oid=oid), status_code=303)
|
||||
|
||||
|
||||
@app.post("/search", response_class=HTMLResponse)
|
||||
|
|
@ -200,8 +186,8 @@ async def search(request: Request, query: str = Form(...)):
|
|||
return RedirectResponse(request.url_for("report", oid=query), status_code=303)
|
||||
|
||||
|
||||
@app.get("/progress/{oid}/{task_id}", response_class=HTMLResponse)
|
||||
async def progress(request: Request, oid: str, task_id: str):
|
||||
@app.get("/progress/{oid}", response_class=HTMLResponse)
|
||||
async def progress(request: Request, oid: str):
|
||||
try:
|
||||
c = Company(oid)
|
||||
print("miss for", c)
|
||||
|
|
@ -215,28 +201,29 @@ async def progress(request: Request, oid: str, task_id: str):
|
|||
if c.report_path.exists():
|
||||
return RedirectResponse(request.url_for("report", oid=oid))
|
||||
|
||||
queue_size = r.llen(dqname)
|
||||
print(f"Queue size: {queue_size}")
|
||||
|
||||
wstatus = r.get('af2gis:worker_status')
|
||||
tasks = r.lrange(dqname, 0, -1)
|
||||
|
||||
wstatus = r.get(REDIS_WORKER_STATUS)
|
||||
tasks = r.lrange(REDIS_TASK_QUEUE_NAME, 0, -1) # возвращает list of bytes
|
||||
print("tasks:", tasks)
|
||||
queue_size = len(tasks)
|
||||
try:
|
||||
qpos = tasks.index(oid) + 1
|
||||
print(f"Position: {qpos}")
|
||||
except ValueError:
|
||||
print("Not found")
|
||||
qpos=None
|
||||
|
||||
task_data = r.get(f"dramatiq:message:{task_id}")
|
||||
print("TASK:", task_data)
|
||||
|
||||
position = next((i for i, task in enumerate(tasks) if task_id in str(task)), None)
|
||||
print("pos:", position)
|
||||
# position = next((i for i, task in enumerate(tasks) if task_id in str(task)), None)
|
||||
# print("pos:", position)
|
||||
|
||||
last_trusted = [json.loads(item) for item in r.lrange('af2gis:last_trusted', 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange('af2gis:last_untrusted', 0, -1)]
|
||||
last_trusted = [json.loads(item) for item in r.lrange(REDIS_TRUSTED_LIST, 0, -1)]
|
||||
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
|
||||
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"progress.html", {
|
||||
"request": request, "title": c.title, "oid": c.object_id,
|
||||
"qsize": queue_size, "position": position, "wstatus": wstatus,
|
||||
"qsize": queue_size, "wstatus": wstatus, "qpos": qpos,
|
||||
"trusted": last_trusted,
|
||||
"untrusted": last_untrusted
|
||||
}
|
||||
|
|
@ -246,7 +233,8 @@ async def progress(request: Request, oid: str, task_id: str):
|
|||
def main():
|
||||
global templates
|
||||
import uvicorn
|
||||
auto_reload = True
|
||||
auto_reload = bool(os.getenv("AUTO_RELOAD", False))
|
||||
print("AUTO_RELOAD:", auto_reload)
|
||||
load_dotenv()
|
||||
uvicorn.run("antifraud2gis.cli.af2web:app", host="0.0.0.0", port=8000, reload=auto_reload)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import dramatiq
|
||||
import sys
|
||||
from ..tasks import fraud_task
|
||||
from dramatiq.cli import main as dramatiq_main
|
||||
|
||||
|
|
@ -7,6 +8,7 @@ from dramatiq.cli import main as dramatiq_main
|
|||
af2worker antifraud2gis.tasks
|
||||
"""
|
||||
def main():
|
||||
sys.argv = ["dramatiq", "-p1", "-t1", "antifraud2gis.tasks"]
|
||||
dramatiq_main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import argparse
|
|||
import time
|
||||
import random
|
||||
import pkg_resources
|
||||
import redis
|
||||
from rich.text import Text
|
||||
|
||||
from ..company import CompanyList, Company
|
||||
from ..user import User
|
||||
|
|
@ -9,10 +11,8 @@ from ..settings import settings
|
|||
from ..fraud import detect, dump_report
|
||||
from ..exceptions import AFNoCompany
|
||||
from ..aliases import aliases
|
||||
|
||||
from .summary import printsummary
|
||||
|
||||
from rich.text import Text
|
||||
from ..const import REDIS_TASK_QUEUE_NAME, REDIS_TRUSTED_LIST, REDIS_UNTRUSTED_LIST, REDIS_WORKER_STATUS, REDIS_DRAMATIQ_QUEUE
|
||||
|
||||
def countdown(n=5):
|
||||
for i in range(n, 0, -1):
|
||||
|
|
@ -127,4 +127,62 @@ def handle_dev(args: argparse.Namespace):
|
|||
elif cmd == "tmp":
|
||||
# check users age for review
|
||||
u = User(args.args[0])
|
||||
print(u.birthday())
|
||||
print(u.birthday())
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("cmd", choices=['company-users', 'queue'])
|
||||
parser.add_argument("-v", "--verbose", default=False, action='store_true')
|
||||
parser.add_argument("--full", default=False, action='store_true')
|
||||
parser.add_argument("args", nargs='*', help='extra args')
|
||||
|
||||
g = parser.add_argument_group('Company selection')
|
||||
g.add_argument("-t", "--town", default=None, help="Filter by town")
|
||||
g.add_argument("-n", "--name", default=None, help="Filter by name (fnmatch)")
|
||||
g.add_argument("-l", "--limit", metavar='N', type=int, help="Limit to N companies")
|
||||
g.add_argument("-c", "--company", metavar='OID', help="Company ID")
|
||||
g.add_argument("--report", default=None, action='store_true', help="Company has antifraud report")
|
||||
g.add_argument("--noreport", default=None, action='store_true', help="Company has NO antifraud report")
|
||||
g.add_argument("--really", default=None, action='store_true', help="Really. (flag for dangerous commands like wipe)")
|
||||
|
||||
g = parser.add_argument_group('Fraud options')
|
||||
g.add_argument("-s", "--show", metavar='N', type=int, help="Show links with N hits")
|
||||
g.add_argument("--overwrite", default=None, action='store_true', help="Recalculate even if fraud report exists")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
cl = CompanyList()
|
||||
|
||||
cmd = args.cmd
|
||||
|
||||
if cmd == "company-users":
|
||||
c = cl[args.company]
|
||||
print(c)
|
||||
c.load_reviews()
|
||||
for u in c.users():
|
||||
print(u)
|
||||
|
||||
elif cmd == "queue":
|
||||
r = redis.Redis(decode_responses=True)
|
||||
|
||||
if 'reset' in args.args:
|
||||
print("RESET queue")
|
||||
r.delete(REDIS_TASK_QUEUE_NAME)
|
||||
r.delete(REDIS_TRUSTED_LIST)
|
||||
r.delete(REDIS_UNTRUSTED_LIST)
|
||||
|
||||
wstatus = r.get(REDIS_WORKER_STATUS)
|
||||
tasks = r.lrange(REDIS_TASK_QUEUE_NAME, 0, -1) # возвращает list of bytes
|
||||
trusted_len = r.llen(REDIS_TRUSTED_LIST)
|
||||
untrusted_len = r.llen(REDIS_UNTRUSTED_LIST)
|
||||
dqlen = r.llen(REDIS_DRAMATIQ_QUEUE)
|
||||
|
||||
print("Queue report")
|
||||
print(f"Worker status: {wstatus}")
|
||||
print(f"Dramatiq queue: {dqlen}")
|
||||
print(f"Tasks ({len(tasks)}): {tasks}")
|
||||
print(f"Trusted ({trusted_len})")
|
||||
print(f"Untrusted ({untrusted_len})")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import argparse
|
|||
import sys
|
||||
import time
|
||||
import json
|
||||
import builtins
|
||||
from builtins import print as _print
|
||||
|
||||
from argalias import ArgAlias
|
||||
|
||||
from rich import print_json, print
|
||||
|
|
@ -15,20 +16,23 @@ from pathlib import Path
|
|||
|
||||
from ..company import Company, CompanyList
|
||||
from ..user import User
|
||||
from ..fraud import detect, compare
|
||||
from ..fraud import detect, compare, dump_report
|
||||
from ..logger import loginit, logger, testlogger
|
||||
from ..db import db
|
||||
from ..const import SUMMARY_PERIOD
|
||||
from ..exceptions import AFNoCompany, AFReportNotReady, AFReportAlreadyExists
|
||||
from ..settings import settings
|
||||
from ..statistics import statistics
|
||||
|
||||
# CLI
|
||||
from .summary import add_summary_parser, handle_summary
|
||||
from .summary import add_summary_parser, handle_summary, printsummary
|
||||
from .company import add_company_parser, handle_company
|
||||
from .user import add_user_parser, handle_user
|
||||
from .dev import add_dev_parser, handle_dev
|
||||
|
||||
last_summary = 0
|
||||
|
||||
def get_args():
|
||||
def UNUSED_get_args():
|
||||
|
||||
aa = ArgAlias()
|
||||
aa.alias(["company"], "c")
|
||||
|
|
@ -75,28 +79,126 @@ def get_args():
|
|||
#return parser.parse_known_args()
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("cmd", choices=['list','stop','summary', 'fraud', 'delreport', 'wipe'])
|
||||
parser.add_argument("-v", "--verbose", default=False, action='store_true')
|
||||
parser.add_argument("--sleep", type=float, default=None, help='sleep N.M seconds after each processed company')
|
||||
parser.add_argument("--fmt", "-f", default="normal", choices=['brief', 'normal', 'full'])
|
||||
|
||||
g = parser.add_argument_group('Company selection')
|
||||
g.add_argument("-t", "--town", default=None, help="Filter by town")
|
||||
g.add_argument("-n", "--name", default=None, help="Filter by name (fnmatch)")
|
||||
g.add_argument("-l", "--limit", metavar='N', type=int, help="Limit to N companies")
|
||||
g.add_argument("-c", "--company", metavar='OID', help="Company ID")
|
||||
g.add_argument("--report", default=None, action='store_true', help="Company has antifraud report")
|
||||
g.add_argument("--noreport", default=None, action='store_true', help="Company has NO antifraud report")
|
||||
g.add_argument("--really", default=None, action='store_true', help="Really. (flag for dangerous commands like wipe)")
|
||||
|
||||
g = parser.add_argument_group('Fraud options')
|
||||
g.add_argument("-s", "--show", metavar='N', type=int, help="Show links with N hits")
|
||||
g.add_argument("--overwrite", default=None, action='store_true', help="Recalculate even if fraud report exists")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
#print(args)
|
||||
|
||||
stopfile = Path('.stop')
|
||||
|
||||
cl = CompanyList()
|
||||
|
||||
loginit("DEBUG" if args.verbose else "INFO")
|
||||
|
||||
if args.subparser is None:
|
||||
if args.stop:
|
||||
stopfile.touch()
|
||||
print("Stopfile created in current directory")
|
||||
|
||||
if args.subparser == "summary":
|
||||
handle_summary(args)
|
||||
if args.cmd == "stop":
|
||||
stopfile.touch()
|
||||
print("Stopfile created in current directory")
|
||||
|
||||
if args.subparser == "company":
|
||||
handle_company(args)
|
||||
elif args.cmd == "summary":
|
||||
if args.fmt == 'full':
|
||||
printsummary(cl=cl, full=True)
|
||||
else:
|
||||
printsummary(cl=cl, full=False)
|
||||
|
||||
elif args.subparser == "user":
|
||||
handle_user(args)
|
||||
elif args.cmd in ["list", "fraud", "delreport", "wipe"]:
|
||||
|
||||
elif args.subparser == "dev":
|
||||
handle_dev(args)
|
||||
# if company is given, create it first (if it's missing)
|
||||
if args.company and args.cmd not in ['wipe']:
|
||||
Company(args.company)
|
||||
|
||||
# PRE PROCESSING
|
||||
if args.cmd == "delreport":
|
||||
# force args.report
|
||||
args.report = True
|
||||
elif args.cmd == "fraud":
|
||||
if args.show:
|
||||
settings.show_hit_th = args.show
|
||||
|
||||
total_processed = 0
|
||||
effectively_processed = 0
|
||||
|
||||
for c in cl.companies(oid=args.company, name=args.name, town=args.town, report=args.report, noreport=args.noreport):
|
||||
|
||||
total_processed += 1
|
||||
|
||||
if args.cmd == "list":
|
||||
if args.fmt == "brief":
|
||||
_print(c.object_id)
|
||||
else:
|
||||
print(c)
|
||||
|
||||
elif args.cmd == "fraud":
|
||||
print("FF", c)
|
||||
|
||||
if args.show:
|
||||
settings.show_hit_th = args.show
|
||||
try:
|
||||
detect(c, cl, force=args.overwrite)
|
||||
effectively_processed += 1
|
||||
dump_report(c.object_id)
|
||||
except AFReportAlreadyExists as e:
|
||||
print(f"Report already exists for {c}")
|
||||
|
||||
elif args.cmd == "delreport":
|
||||
print(f"Delete report for {c}")
|
||||
c.report_path.unlink(missing_ok=True)
|
||||
|
||||
elif args.cmd == "wipe":
|
||||
if args.really:
|
||||
print(f"wipe {c}")
|
||||
c.wipe(c.object_id)
|
||||
else:
|
||||
print("[NOT REALLY] wipe", c)
|
||||
|
||||
|
||||
# Stop if stopfile
|
||||
if stopfile.exists():
|
||||
print("Stopfile found, exit")
|
||||
stopfile.unlink()
|
||||
sys.exit(0)
|
||||
|
||||
# Stop if processed enough
|
||||
if args.limit:
|
||||
if args.cmd == "fraud":
|
||||
if effectively_processed >= args.limit:
|
||||
print(f"Processed {args.limit} companies, exit")
|
||||
break
|
||||
else:
|
||||
if total_processed >= args.limit:
|
||||
print(f"Processed {args.limit} companies, exit")
|
||||
break
|
||||
|
||||
if args.sleep:
|
||||
time.sleep(args.sleep)
|
||||
|
||||
|
||||
if args.fmt == 'normal':
|
||||
# POST PROCESSING
|
||||
if args.cmd == "fraud":
|
||||
print(f'# Fraud reports calculated {effectively_processed}')
|
||||
|
||||
print(f'# Total processed {total_processed} comanies')
|
||||
print(statistics)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
import json
|
||||
from loguru import logger
|
||||
import time
|
||||
import fnmatch
|
||||
import sys
|
||||
import gzip
|
||||
import traceback
|
||||
import numpy as np
|
||||
from rich.progress import Progress
|
||||
from rich import print_json
|
||||
from rich.pretty import pretty_repr
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
from .settings import settings
|
||||
|
|
@ -15,24 +17,34 @@ from .user import User, get_user
|
|||
from .review import Review
|
||||
from .session import session
|
||||
from .exceptions import AFNoCompany
|
||||
from .aliases import resolve_alias
|
||||
from .statistics import statistics
|
||||
from .aliases import aliases
|
||||
|
||||
# to avoid circular import
|
||||
class RelationDict:
|
||||
pass
|
||||
#class RelationDict:
|
||||
# pass
|
||||
|
||||
|
||||
class Company:
|
||||
|
||||
relations: RelationDict
|
||||
relations: 'RelationDict'
|
||||
|
||||
def __init__(self, object_id):
|
||||
def __init__(self, object_id: str):
|
||||
assert object_id is not None
|
||||
assert object_id.isdigit()
|
||||
|
||||
if not object_id.isdigit():
|
||||
# try to resolve alias
|
||||
object_id = resolve_alias(object_id) or object_id
|
||||
|
||||
if not object_id.isdigit():
|
||||
raise AFNoCompany(f'Not an digital OID {object_id!r}')
|
||||
|
||||
self.object_id = object_id
|
||||
self.reviews_path = settings.company_storage / (object_id + '-reviews.json.gz')
|
||||
self.basic_path = settings.company_storage / (object_id + '-basic.json.gz')
|
||||
self.report_path = settings.company_storage / (object_id + '-report.json.gz')
|
||||
self.loaded_from_disk = False
|
||||
|
||||
self._reviews = list()
|
||||
|
||||
|
|
@ -48,6 +60,9 @@ class Company:
|
|||
|
||||
self.rate = None
|
||||
|
||||
self.total_count_2gis = None
|
||||
self.branch_count_2gis = None
|
||||
self.branch_rating_2gis = None
|
||||
|
||||
# ratings
|
||||
# self.score = dict()
|
||||
|
|
@ -79,7 +94,7 @@ class Company:
|
|||
|
||||
|
||||
def load_basic(self):
|
||||
if not self.load_basic_from_disk():
|
||||
if not self.load_basic_from_disk():
|
||||
self.load_basic_from_network()
|
||||
self.save_basic()
|
||||
|
||||
|
|
@ -111,12 +126,15 @@ class Company:
|
|||
self.error = _basic.get('error', None)
|
||||
self.frozen = _basic.get('frozen', False)
|
||||
self.tags = _basic.get('tags', None)
|
||||
self.total_count_2gis = _basic.get('total_count_2gis', None)
|
||||
self.branch_count_2gis = _basic.get('branch_count_2gis', None)
|
||||
self.branch_rating_2gis = _basic.get('branch_rating_2gis', None)
|
||||
if not self.title:
|
||||
# logger.debug(f"no title for {self.object_id}")
|
||||
pass
|
||||
self.loaded_from_disk = True
|
||||
return bool(self.title)
|
||||
else:
|
||||
logger.debug(f"no file on disk for {self.object_id}")
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
|
|
@ -130,12 +148,19 @@ class Company:
|
|||
'address': self.address,
|
||||
# 'score': self.score,
|
||||
'error': self.error,
|
||||
'frozen': self.frozen
|
||||
'frozen': self.frozen,
|
||||
'total_count_2gis': self.total_count_2gis,
|
||||
'branch_count_2gis': self.branch_count_2gis,
|
||||
'branch_rating_2gis': self.branch_rating_2gis
|
||||
}
|
||||
if self.tags:
|
||||
basic['tags'] = self.tags
|
||||
json.dump(basic, f)
|
||||
|
||||
if not self.loaded_from_disk:
|
||||
statistics.created_new_companies += 1
|
||||
|
||||
|
||||
|
||||
def load_reviews(self, local_only=False):
|
||||
if self._reviews:
|
||||
|
|
@ -154,9 +179,12 @@ class Company:
|
|||
def count_rate(self):
|
||||
self.ratings = list()
|
||||
for r in self._reviews:
|
||||
self.ratings.append(r['rating'])
|
||||
if r['rating'] is None:
|
||||
print_json(data=r)
|
||||
if r['rating'] is not None:
|
||||
self.ratings.append(r['rating'])
|
||||
|
||||
if self.ratings:
|
||||
if self.ratings:
|
||||
self.rate = round(float(np.mean(self.ratings)), 2)
|
||||
return(self.rate)
|
||||
|
||||
|
|
@ -196,9 +224,11 @@ class Company:
|
|||
url = f'https://public-api.reviews.2gis.com/2.0/branches/{self.object_id}/reviews?limit=50&fields=meta.providers,meta.branch_rating,meta.branch_reviews_count,meta.total_count,reviews.hiding_reason,reviews.is_verified&without_my_first_review=false&rated=true&sort_by=friends&key={REVIEWS_KEY}&locale=ru_RU'
|
||||
unused_geo = f'https://public-api.reviews.2gis.com/2.0/geo/141373143684284/reviews?limit=50&fields=meta.providers,meta.geo_rating,meta.geo_reviews_count,meta.total_count,reviews.hiding_reason&sort_by=friends&without_my_first_review=false&key={REVIEWS_KEY}&locale=ru_RU'
|
||||
|
||||
# print("LOAD NETWORK REVIEWS", self.object_id)
|
||||
|
||||
page=0
|
||||
while url:
|
||||
logger.debug(f".. load reviews p{page} for {self}: {url}")
|
||||
# logger.debug(f".. load reviews p{page} for {self}: {url}")
|
||||
r = None
|
||||
while r is None:
|
||||
|
||||
|
|
@ -219,23 +249,30 @@ class Company:
|
|||
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
total_count = data['meta']['total_count']
|
||||
branch_reviews_count = data['meta']['branch_reviews_count']
|
||||
if total_count == 0 or branch_reviews_count == 0:
|
||||
|
||||
if self.total_count_2gis is None:
|
||||
self.total_count_2gis = data['meta']['total_count']
|
||||
self.branch_count_2gis = data['meta']['branch_reviews_count']
|
||||
self.branch_rating_2gis = data['meta']['branch_rating']
|
||||
# print(f"Total/Branch reviews count: {self.total_count_2gis}/{self.branch_count_2gis}")
|
||||
|
||||
if self.total_count_2gis == 0 or self.branch_count_2gis == 0:
|
||||
raise AFNoCompany(f"No reviews for {self.object_id}")
|
||||
|
||||
# print(f"{self.object_id} page {page} first: {data['reviews'][0]['date_created']} last: {data['reviews'][-1]['date_created']}")
|
||||
|
||||
self._reviews.extend(data['reviews'])
|
||||
url = data['meta'].get('next_link')
|
||||
time.sleep(SLEEPTIME)
|
||||
|
||||
if len(self._reviews) >= LOAD_NREVIEWS:
|
||||
print(f"max reviews {len(self._reviews)} reached")
|
||||
# print(f"max reviews {len(self._reviews)} reached")
|
||||
break
|
||||
|
||||
page+=1
|
||||
|
||||
|
||||
logger.info(f"Company {self.object_id}: loaded {len(self._reviews)} reviews")
|
||||
logger.info(f"Company {self.object_id}: loaded from network {len(self._reviews)} reviews")
|
||||
# why we were called?
|
||||
# print("----------")
|
||||
# print("".join(traceback.format_stack(limit=10)))
|
||||
|
|
@ -245,6 +282,8 @@ class Company:
|
|||
with gzip.open(self.reviews_path, 'wt') as f:
|
||||
json.dump(self._reviews, f)
|
||||
|
||||
statistics.total_companies_loaded+=1
|
||||
statistics.total_companies_loaded_network+=1
|
||||
|
||||
def risk(self):
|
||||
if not self.score:
|
||||
|
|
@ -270,7 +309,7 @@ class Company:
|
|||
if self.frozen:
|
||||
tags += "[FROZEN]"
|
||||
|
||||
return f'Company({self.object_id} rate:{self.rate} {titlestr} addr: {self.address} reviews:{len(self._reviews) if self._reviews else "not loaded"}{tags})'
|
||||
return f'Company({self.object_id} rate: {self.branch_rating_2gis} {titlestr} addr: {self.address} reviews:{len(self._reviews) if self._reviews else "not loaded"}{tags})'
|
||||
|
||||
def get_title(self):
|
||||
return self.title or self.object_id
|
||||
|
|
@ -295,7 +334,7 @@ class Company:
|
|||
|
||||
def reviews(self):
|
||||
for r in self._reviews:
|
||||
yield Review(r)
|
||||
yield Review(r, company=self)
|
||||
|
||||
def nreviews(self):
|
||||
return len(self._reviews)
|
||||
|
|
@ -360,20 +399,24 @@ class CompanyList():
|
|||
raise KeyError(f"Company {index} not found")
|
||||
|
||||
|
||||
def companies(self, town = None):
|
||||
def companies(self, oid=None, name = None, town = None, report = None, noreport = None, limit=None):
|
||||
n = 0
|
||||
|
||||
if oid:
|
||||
c = Company(oid)
|
||||
if company_match(c, oid=None, name=name, town=town, report=report, noreport=noreport):
|
||||
yield c
|
||||
return
|
||||
|
||||
for f in self.path.glob('*-basic.json.gz'):
|
||||
oid = f.name.split('-')[0]
|
||||
c = Company(oid)
|
||||
if town:
|
||||
# filter by town
|
||||
town = town.lower()
|
||||
ctown = c.get_town()
|
||||
if ctown is None:
|
||||
continue
|
||||
ctown = ctown.lower()
|
||||
if town != ctown:
|
||||
continue
|
||||
yield c
|
||||
company_oid = f.name.split('-')[0]
|
||||
c = Company(company_oid)
|
||||
# print("created c", c.report_path)
|
||||
if company_match(c, oid=oid, name=name, town=town, report=report, noreport=noreport):
|
||||
yield c
|
||||
n+=1
|
||||
if limit and n==limit:
|
||||
return
|
||||
|
||||
|
||||
def company_exists(self, oid):
|
||||
|
|
@ -385,4 +428,37 @@ class CompanyList():
|
|||
return str(Company(oid))
|
||||
else:
|
||||
return oid
|
||||
|
||||
|
||||
def company_match(c: Company, oid: str, name: str = None, town: str = None, report = None, noreport = None):
|
||||
|
||||
if oid and c.object_id != oid:
|
||||
return False
|
||||
|
||||
if report:
|
||||
if not c.report_path.exists():
|
||||
return False
|
||||
|
||||
if noreport:
|
||||
if c.report_path.exists():
|
||||
return False
|
||||
|
||||
if name:
|
||||
name = name.lower()
|
||||
cname = c.get_title().lower()
|
||||
if not fnmatch.fnmatch(cname, name):
|
||||
return False
|
||||
|
||||
if town:
|
||||
# filter by town
|
||||
town = town.lower()
|
||||
ctown = c.get_town()
|
||||
if ctown is None:
|
||||
return False
|
||||
ctown = ctown.lower()
|
||||
if town != ctown:
|
||||
return False
|
||||
|
||||
|
||||
# all filters matched
|
||||
return True
|
||||
|
||||
|
|
|
|||
|
|
@ -12,3 +12,9 @@ SUMMARY_PERIOD = 10*60
|
|||
|
||||
# grep reviewApiKey in https://2gis.ru/ , see contrib/
|
||||
REVIEWS_KEY = '6e7e1929-4ea9-4a5d-8c05-d601860389bd'
|
||||
|
||||
REDIS_TASK_QUEUE_NAME="af2gis:queue"
|
||||
REDIS_TRUSTED_LIST="af2gis:last_trusted_list"
|
||||
REDIS_UNTRUSTED_LIST="af2gis:last_untrusted_list"
|
||||
REDIS_WORKER_STATUS="af2gis:worker_status"
|
||||
REDIS_DRAMATIQ_QUEUE="dramatiq:default"
|
||||
|
|
@ -1,8 +1,12 @@
|
|||
class AFException(Exception):
|
||||
pass
|
||||
|
||||
class AFReportNotReady(Exception):
|
||||
class AFReportNotReady(AFException):
|
||||
pass
|
||||
|
||||
class AFNoCompany(AFException):
|
||||
pass
|
||||
|
||||
class AFReportAlreadyExists(AFException):
|
||||
pass
|
||||
|
||||
class AFNoCompany(Exception):
|
||||
pass
|
||||
|
|
@ -19,7 +19,7 @@ from .company import Company, CompanyList
|
|||
from .user import User, get_user
|
||||
from .relation import RelationDict, _is_dangerous
|
||||
from .settings import settings
|
||||
from .exceptions import AFReportNotReady, AFNoCompany
|
||||
from .exceptions import AFReportNotReady, AFNoCompany, AFReportAlreadyExists
|
||||
|
||||
def compare(a: Company, b: Company):
|
||||
|
||||
|
|
@ -151,6 +151,7 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
debug_uids = os.getenv("DEBUG_UIDS", "").split(" ")
|
||||
|
||||
if c.report_path.exists() and not force:
|
||||
raise AFReportAlreadyExists(f"Report already exists: {c.report_path}")
|
||||
print(f"SKIP because exists {c.report_path}")
|
||||
# read
|
||||
with gzip.open(c.report_path, "rt", encoding="utf-8") as f:
|
||||
|
|
@ -212,7 +213,7 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
skipped_users_ratings.append(cr.rating)
|
||||
continue
|
||||
u = cr.user
|
||||
|
||||
|
||||
nrlist.append(u.nreviews())
|
||||
|
||||
if u.public_id in debug_uids:
|
||||
|
|
@ -229,6 +230,8 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
# print(f"Skip user {u} with {u.nreviews()} reviews")
|
||||
continue
|
||||
|
||||
user_ages.append((cr.created - u.birthday()).days)
|
||||
|
||||
# only for open profiles
|
||||
for r in u.reviews():
|
||||
|
||||
|
|
@ -237,14 +240,14 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
|
||||
if r.oid == c.object_id:
|
||||
continue
|
||||
user_ages.append((cr.created - u.birthday()).days)
|
||||
|
||||
|
||||
# print(" review:", r)
|
||||
|
||||
rel = c.relations[r.oid]
|
||||
rel.inc()
|
||||
rel.add_user(u, cr.rating, r.rating)
|
||||
rel.hit(cr.rating, r)
|
||||
## next two lines, + import addr/title from rel
|
||||
# rel.inc()
|
||||
# rel.add_user(u, cr.rating, r.rating)
|
||||
|
||||
c_hits[r.oid] += 1
|
||||
c_weight[r.oid] += 1/u.nreviews()
|
||||
|
|
@ -305,33 +308,18 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
|
||||
# print(f"{tr_c=}, {untr_c=}")
|
||||
|
||||
if settings.deep_check:
|
||||
nrel = 0
|
||||
nhits = 0
|
||||
nrel_high_ratings = 0
|
||||
nrel_high_hits = 0
|
||||
for rel in c.relations.relations.values():
|
||||
nhits+= rel.count
|
||||
if rel.check_high_hits():
|
||||
nrel += 1
|
||||
if rel.check_high_ratings():
|
||||
nrel_high_ratings += 1
|
||||
nrel_high_hits += rel.count
|
||||
# tmp: always on
|
||||
|
||||
|
||||
towns = set()
|
||||
hirel = 0
|
||||
hirelhihits = 0
|
||||
for rel in c.relations.relations.values():
|
||||
if rel.check_high_hits():
|
||||
towns = set()
|
||||
titles = set()
|
||||
hirel = 0
|
||||
for rel in c.relations.relations.values():
|
||||
if rel.check_high_hits():
|
||||
if rel.check_high_ratings():
|
||||
hirel += 1
|
||||
if rel.check_high_ratings():
|
||||
hirelhihits += 1
|
||||
try:
|
||||
b = Company(rel.b)
|
||||
except AFNoCompany:
|
||||
continue
|
||||
towns.add(b.get_town())
|
||||
towns.add(rel.get_btown())
|
||||
titles.add(rel.get_btitle())
|
||||
print(f"add title {rel.get_btitle()} for {rel.b}")
|
||||
|
||||
low_nrlist = list(filter(lambda x: x <= settings.risk_median_rpu, nrlist))
|
||||
|
||||
|
|
@ -349,14 +337,17 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
score['param_fp'] = settings.param_fp()
|
||||
score['median_reviews_per_user'] = round(float(np.median(nrlist)), 1)
|
||||
|
||||
if settings.deep_check:
|
||||
score['high_hit_relations'] = hirel
|
||||
score['high_hit_towns'] = len(towns)
|
||||
score['happy_long_rel'] = int(100*hirel/len(towns)) if len(towns) > 0 else 0
|
||||
score['high_hit_relations'] = hirel
|
||||
score['high_hit_towns'] = len(towns)
|
||||
score['high_hit_titles'] = len(titles)
|
||||
|
||||
score['happy_long_rel'] = int(100*len(towns)/hirel) if hirel > 0 else 0
|
||||
score['sametitle_rel'] = int(100*len(titles)/hirel) if hirel > 0 else 0
|
||||
|
||||
print(titles)
|
||||
print(f"{len(titles)} titles, hirel: {hirel}")
|
||||
print(score['sametitle_rel'])
|
||||
|
||||
score['nrel_high_ratings'] = nrel_high_ratings
|
||||
score['highrate_relations'] = int(100*nrel / nrel_high_ratings) if nrel_high_ratings > 0 else 0
|
||||
score['highrate_hits'] = int(nrel_high_hits / nrel) if nrel > 0 else 0
|
||||
|
||||
score['mean_user_age'] = int(np.median(user_ages)) if user_ages else None
|
||||
|
||||
|
|
@ -372,23 +363,34 @@ def detect(c: Company, cl: CompanyList, force=False):
|
|||
# c.score['full_rate'] = c.count_rate()
|
||||
# c.score['trusted_rate'] = round(float(np.mean(trusted_ratings)),2)
|
||||
|
||||
print(hirel >= settings.sametitle_rel and score['sametitle_rel'] >= settings.sametitle_ratio)
|
||||
print(hirel, settings.sametitle_rel, score['sametitle_rel'], settings.sametitle_ratio)
|
||||
|
||||
# make verdict
|
||||
if score['empty_user_ratio'] > settings.risk_empty_user_ratio:
|
||||
score['trusted'] = False
|
||||
score['reason'] = f"empty_user_ratio {score['empty_user_ratio']}% ({skipped_users}/{processed_users + skipped_users}) ({score['empty_user_avg_rate']})"
|
||||
|
||||
elif c.relations.nrisk_users > settings.risk_user_ratio:
|
||||
score['trusted'] = False
|
||||
score['reason'] = f"risk_users {c.relations.nrisk_users}"
|
||||
|
||||
elif score['median_reviews_per_user'] <= settings.risk_median_rpu:
|
||||
score['trusted'] = False
|
||||
score['reason'] = f"median_reviews_per_user {score['median_reviews_per_user']} <= {settings.risk_median_rpu} ({len(low_nrlist)} of {len(nrlist)} are <= {settings.risk_median_rpu})"
|
||||
elif settings.deep_check and len(towns) >= settings.happy_long_rel_min_towns_th \
|
||||
|
||||
elif len(towns) >= settings.happy_long_rel_min_towns_th \
|
||||
and score['happy_long_rel'] >= settings.happy_long_rel_th:
|
||||
score['trusted'] = False
|
||||
score['reason'] = f"happy_long_rel {score['happy_long_rel']}% ({hirel} of {len(towns)})"
|
||||
score['reason'] = f"happy_long_rel {score['happy_long_rel']}% ({len(towns)} of {hirel})"
|
||||
|
||||
elif hirel >= settings.sametitle_rel and score['sametitle_rel'] <= settings.sametitle_ratio:
|
||||
score['trusted'] = False
|
||||
score['reason'] = f"sametitle_rel {score['sametitle_rel']}% ({hirel} of {len(titles)})"
|
||||
|
||||
elif score['mean_user_age'] <= settings.mean_user_age:
|
||||
score['trusted'] = False
|
||||
score['reason'] = f"mean_user_age {score['mean_user_age']} <= {settings.mean_user_age}"
|
||||
score['reason'] = f"mean_user_age {score['mean_user_age']} <= {settings.mean_user_age} ({sum(a <= settings.mean_user_age for a in user_ages)} of {len(user_ages)})"
|
||||
else:
|
||||
score['trusted'] = True
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from .company import Company
|
|||
from .settings import settings
|
||||
from .exceptions import AFNoCompany
|
||||
from .logger import logger
|
||||
from .review import Review
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
|
|
@ -43,6 +44,8 @@ class Relation:
|
|||
median: float
|
||||
avg_arating: float
|
||||
avg_brating: float
|
||||
btitle: str
|
||||
baddr: str
|
||||
|
||||
count: int
|
||||
|
||||
|
|
@ -59,6 +62,8 @@ class Relation:
|
|||
self.median = None
|
||||
self.avg_arating = None
|
||||
self.avg_brating = None
|
||||
self.btitle = None
|
||||
self.baddr = None
|
||||
|
||||
def add_user(self, user, a_rating, b_rating):
|
||||
global users_added
|
||||
|
|
@ -88,8 +93,8 @@ class Relation:
|
|||
|
||||
self.mean = round(float(np.mean(user_reviews)), 3)
|
||||
self.median = int(np.median(user_reviews))
|
||||
self.avg_arating = round(float(np.mean(self._aratings)), 3)
|
||||
self.avg_brating = round(float(np.mean(self._bratings)), 3)
|
||||
self.avg_arating = round(float(np.mean(self._aratings)), 1)
|
||||
self.avg_brating = round(float(np.mean(self._bratings)), 1)
|
||||
|
||||
self._calculated = True
|
||||
|
||||
|
|
@ -154,13 +159,34 @@ class Relation:
|
|||
if self.count >= settings.risk_hit_th and self.median <= settings.risk_median_th:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_btown(self):
|
||||
if self.baddr is None:
|
||||
return None
|
||||
return self.baddr.split(',')[0].replace(u'\xa0', u' ')
|
||||
|
||||
def get_btitle(self):
|
||||
if self.btitle is None:
|
||||
return None
|
||||
return self.btitle.split(',')[0]
|
||||
|
||||
def __repr__(self):
|
||||
if not self._calculated:
|
||||
self.calc()
|
||||
bcompany = Company(self.b)
|
||||
return f"{bcompany.get_title()} ({bcompany.address}): hits: {len(self._users)}/{self.count} mean: {self.mean} median: {self.median})"
|
||||
|
||||
def hit(self, arating: int, r: Review):
|
||||
""" add review to relation """
|
||||
self.inc()
|
||||
self.add_user(r.user, arating, r.rating)
|
||||
|
||||
if self.btitle is None:
|
||||
self.btitle = r.title
|
||||
self.baddr = r.address
|
||||
# print(f'hit rel to {self.b} {self.btitle} ({self.baddr})')
|
||||
|
||||
|
||||
class RelationDict:
|
||||
|
||||
c: Company
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import datetime
|
|||
from rich import print_json
|
||||
|
||||
class Review():
|
||||
def __init__(self, data, user=None):
|
||||
def __init__(self, data, user=None, company=None):
|
||||
self._data = data
|
||||
self.review_id = data['id']
|
||||
self.rating = data['rating']
|
||||
|
|
@ -11,7 +11,10 @@ class Review():
|
|||
self.user_name = data['user']['name']
|
||||
self.text = data.get('text')
|
||||
self.nphotos = len(data['photos'])
|
||||
|
||||
self._user = user
|
||||
self._company = company
|
||||
|
||||
|
||||
self.created = datetime.datetime.strptime(data['date_created'].split('T')[0], "%Y-%m-%d")
|
||||
self.age = (datetime.datetime.now() - self.created).days
|
||||
|
|
@ -20,14 +23,20 @@ class Review():
|
|||
else:
|
||||
self.user_age = None
|
||||
|
||||
try:
|
||||
self.title = data['object']['name']
|
||||
except KeyError:
|
||||
self.title = None
|
||||
try:
|
||||
self.address = data['object']['address']
|
||||
except KeyError:
|
||||
self.address = None
|
||||
if self._company:
|
||||
self.title = self._company.title
|
||||
self.address = self._company.address
|
||||
|
||||
else:
|
||||
try:
|
||||
self.title = data['object']['name']
|
||||
except KeyError:
|
||||
self.title = None
|
||||
|
||||
try:
|
||||
self.address = data['object']['address']
|
||||
except KeyError:
|
||||
self.address = None
|
||||
|
||||
|
||||
def is_visible(self):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from dotenv import load_dotenv
|
|||
class Settings():
|
||||
def __init__(self):
|
||||
load_dotenv()
|
||||
self.storage = Path(".af2gis-storage")
|
||||
self.storage = Path("~/.af2gis-storage").expanduser()
|
||||
self.user_storage = self.storage / "users"
|
||||
self.private_user_storage = self.storage / "users" / "_private.json"
|
||||
self.company_storage = self.storage / "companies"
|
||||
|
|
@ -13,7 +13,8 @@ class Settings():
|
|||
self.deep_check = bool(int(os.getenv('DEEP_CHECK', '0')))
|
||||
|
||||
self.risk_hit_th = int(os.getenv('RISK_HIT_TH', '10'))
|
||||
self.sametitle_hit = int(os.getenv('SAMETITLE_HIT', '3'))
|
||||
self.sametitle_rel = int(os.getenv('SAMETITLE_HIT', '3'))
|
||||
self.sametitle_ratio = int(os.getenv('SAMETITLE_RATIO', '50'))
|
||||
|
||||
self.risk_median_th = int(os.getenv('RISK_MEDIAN_TH', '15'))
|
||||
self.show_hit_th = int(os.getenv('SHOW_HIT_TH', '1000'))
|
||||
|
|
@ -25,9 +26,11 @@ class Settings():
|
|||
self.risk_empty_user_ratio = float(os.getenv('RISK_EMPTY_USER_TH', '30'))
|
||||
self.risk_user_ratio = float(os.getenv('RISK_USER_TH', '20'))
|
||||
self.risk_median_rpu = float(os.getenv('RISK_MEDIAN_RPU', '2'))
|
||||
|
||||
self.happy_long_rel_th = int(os.getenv('HAPPY_LONG_REL_TH', '10'))
|
||||
self.happy_long_rel_min_towns_th = int(os.getenv('HAPPY_LONG_REL_MIN_TOWNS_TH', '3'))
|
||||
|
||||
|
||||
# 5 years old maximum 365*5=1825
|
||||
self.max_review_age = int(os.getenv('MAX_REVIEW_AGE', '1825'))
|
||||
|
||||
|
|
|
|||
BIN
src/antifraud2gis/static/fonts/CascadiaCode-Regular.woff2
Normal file
BIN
src/antifraud2gis/static/fonts/CascadiaCode-Regular.woff2
Normal file
Binary file not shown.
|
|
@ -1,139 +1,28 @@
|
|||
|
||||
|
||||
|
||||
document.getElementById('search-formQQQ').addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
const OID = document.getElementById('oid').value;
|
||||
const resultContainer = document.getElementById('result-container');
|
||||
|
||||
console.log("search", OID)
|
||||
|
||||
fetch("/report", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ oid: OID }),
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
console.log(data.relations);
|
||||
const resultDiv = document.getElementById("result");
|
||||
|
||||
// Create table for relations
|
||||
let tableHTML = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Town</th>
|
||||
<th>Hits</th>
|
||||
<th>Median</th>
|
||||
<th>A/B rating</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
`;
|
||||
|
||||
data.relations.forEach(rel => {
|
||||
const hitsClass = rel.hits > 20 ? 'red' : '';
|
||||
|
||||
if(rel.town === null)
|
||||
return;
|
||||
|
||||
|
||||
tableHTML += `
|
||||
<tr>
|
||||
<td>${rel.title}</td>
|
||||
<td>${rel.town}</td>
|
||||
<td class="${hitsClass}">${rel.hits}</td>
|
||||
<td>${rel.median}</td>
|
||||
<td>${rel.arating.toFixed(1)} ${rel.brating.toFixed(1)}</td>
|
||||
</tr>
|
||||
`;
|
||||
});
|
||||
tableHTML += `</tbody></table>`;
|
||||
|
||||
// Create score parameters list
|
||||
let scoreHTML = `<div class="score-container"><h3>Score Parameters</h3><ul>`;
|
||||
for (const [key, value] of Object.entries(data.score)) {
|
||||
scoreHTML += `<li><b>${key}:</b> ${value}</li>`;
|
||||
}
|
||||
scoreHTML += `</ul></div>`;
|
||||
|
||||
// Insert into div
|
||||
resultDiv.innerHTML = tableHTML + scoreHTML;
|
||||
|
||||
|
||||
})
|
||||
.catch(error => console.error("Error:", error));
|
||||
|
||||
|
||||
// Здесь будет логика обработки запроса
|
||||
resultContainer.innerHTML = `
|
||||
<h2>Результаты поиска: ${OID}</h2>
|
||||
<p>Информация о компании будет загружена здесь.</p>
|
||||
<div id="company-details"></div>
|
||||
`;
|
||||
});
|
||||
|
||||
|
||||
// Обработка кликов по ссылкам компаний
|
||||
document.querySelectorAll('.company-link').forEach(link => {
|
||||
link.addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
document.getElementById('company-input').value = this.textContent.split(' ')[0];
|
||||
document.getElementById('search-form').dispatchEvent(new Event('submit'));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
function drawRecent(){
|
||||
fetch("/recent", {
|
||||
method: "GET",
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
const trustedDiv = document.getElementById("trusted2");
|
||||
const oidInput = document.getElementById("oid");
|
||||
|
||||
data.trusted.forEach(company => {
|
||||
const companyDiv = document.createElement("div");
|
||||
companyDiv.classList.add("company");
|
||||
|
||||
companyDiv.innerHTML = `
|
||||
<div class="company-title">${company.title}
|
||||
<span class="company-tag trusted-tag">Проверено</span>
|
||||
</div>
|
||||
<div class="company-address">${company.address}</div>
|
||||
`;
|
||||
|
||||
companyDiv.addEventListener("click", () => {
|
||||
// oidInput.value = company.oid;
|
||||
window.location.href = `/report/${company.oid}`;
|
||||
});
|
||||
|
||||
trustedDiv.appendChild(companyDiv);
|
||||
});
|
||||
|
||||
})
|
||||
.catch(error => console.error("Error:", error));
|
||||
}
|
||||
|
||||
|
||||
function make_auto_refresh(){
|
||||
// if auto_refresh_btn exists, autorefresh every 10s
|
||||
const auto_refresh_btn = document.getElementById("auto_refresh_btn");
|
||||
if(auto_refresh_btn){
|
||||
let secondsLeft = 10;
|
||||
console.log(secondsLeft);
|
||||
|
||||
const countdownInterval = setInterval(function() {
|
||||
secondsLeft--;
|
||||
console.log(secondsLeft);
|
||||
auto_refresh_btn.textContent = `Обновить (${secondsLeft})`;
|
||||
|
||||
if (secondsLeft <= 0) {
|
||||
clearInterval(countdownInterval); // Останавливаем отсчет
|
||||
window.location.reload();
|
||||
}
|
||||
}, 1000); // Обновляем каждую секунду
|
||||
|
||||
|
||||
setInterval(function(){
|
||||
window.location.reload();
|
||||
}, 10000);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function main(){
|
||||
drawRecent()
|
||||
make_auto_refresh()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
@font-face {
|
||||
font-family: 'Cascadia Code';
|
||||
src: url('/static/fonts/CascadiaCode-Regular.woff2') format('truetype');
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Arial', sans-serif;
|
||||
font-family: 'PT Sans', sans-serif;
|
||||
line-height: 1.6;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
|
|
@ -12,14 +19,15 @@ h1 {
|
|||
color: #2c3e50;
|
||||
text-align: center;
|
||||
margin-bottom: 30px;
|
||||
font-family: 'Bitter';
|
||||
}
|
||||
|
||||
.search-container {
|
||||
background-color: #f9f9f9;
|
||||
padding: 20px;
|
||||
/*padding: 20px;*/
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
margin-bottom: 30px;
|
||||
margin-bottom: 1em;
|
||||
}
|
||||
|
||||
.search-form {
|
||||
|
|
@ -50,13 +58,14 @@ button:hover {
|
|||
background-color: #2980b9;
|
||||
}
|
||||
|
||||
.result-container {
|
||||
margin-top: 30px;
|
||||
padding: 20px;
|
||||
border: 1px solid #eee;
|
||||
border-radius: 8px;
|
||||
min-height: 200px;
|
||||
background-color: #f5f5f5;
|
||||
.companyinfo {
|
||||
display: flex;
|
||||
justify-content: space-around;
|
||||
}
|
||||
|
||||
.companyinfo > div {
|
||||
flex-grow: 1;
|
||||
margin: 1em;
|
||||
}
|
||||
|
||||
.company-lists {
|
||||
|
|
@ -77,15 +86,38 @@ button:hover {
|
|||
.company-list a:active { color: #333333; }
|
||||
|
||||
|
||||
.progress-div {
|
||||
text-align: left;
|
||||
display: inline-block;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.footnote {
|
||||
font-size: 0.8em;
|
||||
text-align: left;
|
||||
display: inline-block;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.trusted-companies {
|
||||
background-color: #e8f5e9;
|
||||
/* background-color: #e8f5e9; */
|
||||
background-color: #d8e5d9;
|
||||
border: 1px solid #a5d6a7;
|
||||
padding: 10px;
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
.suspicious-companies {
|
||||
background-color: #ffebee;
|
||||
/* background-color: #ffebee; */
|
||||
background-color: #efdbde;
|
||||
border: 1px solid #ef9a9a;
|
||||
padding: 10px;
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
.trusted-companies p,
|
||||
.suspicious-companies p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
h2 {
|
||||
|
|
@ -125,37 +157,44 @@ li {
|
|||
color: #b71c1c;
|
||||
}
|
||||
|
||||
#trusted {
|
||||
font-family: Arial, sans-serif; }
|
||||
|
||||
|
||||
|
||||
|
||||
.company { margin-bottom: 10px; cursor: pointer; }
|
||||
.company-title { font-size: 1.0em; font-weight: bold; }
|
||||
.company-address { font-size: 0.9em; color: gray; }
|
||||
|
||||
.red { color: red; }
|
||||
.green { color: green; }
|
||||
|
||||
.score {
|
||||
font-family: 'Cascadia Code';
|
||||
font-size: 0.8em;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* Result div, table */
|
||||
/* Modern Table Design */
|
||||
#result-container {
|
||||
font-family: 'Segoe UI', Roboto, -apple-system, sans-serif;
|
||||
font-family: 'PT Sans', -apple-system, sans-serif;
|
||||
max-width: 100%;
|
||||
overflow-x: auto;
|
||||
margin: 2rem auto;
|
||||
padding: 1.5rem;
|
||||
background-color: #cccccc; /* Light grey background */
|
||||
margin: 0em auto;
|
||||
padding: 0.5rem;
|
||||
/*background-color: #000; /* Light grey background */
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05);
|
||||
}
|
||||
|
||||
#result-container table {
|
||||
border: 1px solid #ddd;
|
||||
width: 100%;
|
||||
border-collapse: separate;
|
||||
border-spacing: 0;
|
||||
background-color: white;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05);
|
||||
/*box-shadow: 10px 10px 30px rgba(0, 0, 0, 0.05);*/
|
||||
box-shadow: 1px 5px 5px #666;
|
||||
margin-bottom: 1em;
|
||||
}
|
||||
|
||||
#result-container thead {
|
||||
|
|
@ -173,14 +212,20 @@ li {
|
|||
|
||||
#result-container td {
|
||||
padding: 0rem 1.0rem;
|
||||
border-bottom: 1px solid #edf2f7;
|
||||
|
||||
color: #4a5568;
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
||||
#result-container td.red {
|
||||
color: red;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
|
||||
/* Zebra striping with subtle grey shades */
|
||||
#result-container tbody tr:nth-child(odd) {
|
||||
background-color: #ffffff;
|
||||
background-color: #ccc;
|
||||
}
|
||||
|
||||
#result-container tbody tr:nth-child(even) {
|
||||
|
|
@ -188,31 +233,16 @@ li {
|
|||
}
|
||||
|
||||
|
||||
/* Zebra striping with subtle grey shades */
|
||||
#result-container tbody tr.risk:nth-child(odd) {
|
||||
background-color: #f00;
|
||||
color: forestgreen;
|
||||
}
|
||||
|
||||
#result-container tbody tr.risk:nth-child(even) {
|
||||
background-color: #f44;
|
||||
color: white;
|
||||
}
|
||||
|
||||
|
||||
#result-container tbody tr:hover {
|
||||
background-color: #444; /* Light blue-grey on hover */
|
||||
background-color: #aaa; /* Light blue-grey on hover */
|
||||
color: #0f0;
|
||||
}
|
||||
|
||||
#result-container tbody tr:last-child td {
|
||||
border-bottom: none;
|
||||
padding-bottom: 0.2em;
|
||||
}
|
||||
|
||||
#result-container tbody tr.risk a {
|
||||
text-decoration: none;
|
||||
color: white;
|
||||
}
|
||||
#result-container tbody tr a {
|
||||
text-decoration: none;
|
||||
}
|
||||
|
|
@ -236,5 +266,5 @@ li {
|
|||
#actionbox button {
|
||||
display: block;
|
||||
margin: 10px auto 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
17
src/antifraud2gis/statistics.py
Normal file
17
src/antifraud2gis/statistics.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import dataclasses
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Statistics:
|
||||
total_users_loaded: int
|
||||
total_companies_loaded: int
|
||||
total_users_loaded_network: int
|
||||
total_companies_loaded_network: int
|
||||
created_new_companies: int = 0
|
||||
|
||||
statistics = Statistics(
|
||||
total_users_loaded=0,
|
||||
total_companies_loaded=0,
|
||||
total_users_loaded_network=0,
|
||||
total_companies_loaded_network=0,
|
||||
created_new_companies=0
|
||||
)
|
||||
|
|
@ -6,9 +6,9 @@ import json
|
|||
# from filelock import FileLock, Timeout
|
||||
from .fraud import detect
|
||||
from .company import CompanyList, Company
|
||||
|
||||
|
||||
print("Antifraud2gis worker started")
|
||||
from .exceptions import AFNoCompany, AFReportAlreadyExists
|
||||
from .logger import logger
|
||||
from .const import REDIS_WORKER_STATUS, REDIS_TRUSTED_LIST, REDIS_UNTRUSTED_LIST, REDIS_TASK_QUEUE_NAME
|
||||
|
||||
broker = dramatiq.get_broker()
|
||||
|
||||
|
|
@ -18,20 +18,36 @@ r = redis.Redis(
|
|||
decode_responses=True
|
||||
)
|
||||
|
||||
r.set(REDIS_WORKER_STATUS, f'started...')
|
||||
|
||||
|
||||
@dramatiq.actor
|
||||
def fraud_task(oid: str):
|
||||
#lock = FileLock(lock_path)
|
||||
|
||||
#with lock.acquire():
|
||||
print(f"remove {oid} from {REDIS_TASK_QUEUE_NAME}")
|
||||
x = r.lrem(REDIS_TASK_QUEUE_NAME, count=1, value=oid)
|
||||
print(x)
|
||||
|
||||
c = Company(oid)
|
||||
cl = CompanyList()
|
||||
|
||||
|
||||
print(f"{os.getpid()} task STARTED {c}")
|
||||
r.set('af2gis:worker_status', f'processing {oid}')
|
||||
score = detect(c, cl)
|
||||
r.set('af2gis:worker_status', f'finished {oid}')
|
||||
r.set(REDIS_WORKER_STATUS, oid)
|
||||
try:
|
||||
score = detect(c, cl)
|
||||
except AFNoCompany:
|
||||
logger.warning(f"Worker: Company {oid!r} not found")
|
||||
return
|
||||
except AFReportAlreadyExists:
|
||||
logger.warning(f"Worker: Report for {oid!r} already exists")
|
||||
return
|
||||
|
||||
|
||||
|
||||
r.set(REDIS_WORKER_STATUS, f'finished {oid}')
|
||||
print(f"{os.getpid()} task FINISHED {c}")
|
||||
print("SCORE:", score)
|
||||
|
||||
|
|
@ -43,9 +59,9 @@ def fraud_task(oid: str):
|
|||
}
|
||||
|
||||
if score.get('trusted'):
|
||||
lname = 'af2gis:last_trusted'
|
||||
lname = REDIS_TRUSTED_LIST
|
||||
else:
|
||||
lname = 'af2gis:last_untrusted'
|
||||
lname = REDIS_UNTRUSTED_LIST
|
||||
|
||||
r.lpush(lname, json.dumps(res))
|
||||
r.ltrim(lname, 0, 19)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,12 @@
|
|||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{% block title %}{% endblock %}</title>
|
||||
<link href="/static/styles.css" rel="stylesheet">
|
||||
|
||||
<!-- fonts -->
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Bitter:wght@100..900&family=PT+Sans:ital,wght@0,400;0,700;1,400;1,700&family=Roboto:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<h1>{% block header %}{% endblock %}</h1>
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
{% block content %}
|
||||
|
||||
<div id="actionbox">
|
||||
На данный момент нет подготовленного отчета по компании <b>{{title}}</b>
|
||||
<p>На данный момент нет подготовленного отчета по компании <b>{{title}}</b>.</p>
|
||||
<p>Это можно легко исправить.</p>
|
||||
<form method="POST" action="/submit">
|
||||
<input type="hidden" name="oid" value="{{oid}}">
|
||||
<button>Рассчитать статистические аномалии</button>
|
||||
|
|
|
|||
|
|
@ -1,58 +1,16 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>В базе 2GIS не найдена компания {{oid}}</title>
|
||||
<link href="/static/styles.css" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
<h1>В базе 2GIS не найдена компания {{oid}}</h1>
|
||||
|
||||
<div class="search-container">
|
||||
<form class="search-form" id="search-form">
|
||||
<input type="text" id="oid" placeholder="Введите название компании" required>
|
||||
<button type="submit">Найти</button>
|
||||
</form>
|
||||
{% extends 'base.html' %}
|
||||
|
||||
{% block title %}{{oid}}{% endblock %}
|
||||
{% block header %}{{oid}} :: Нет в 2GIS{% endblock %}
|
||||
|
||||
|
||||
{% block content %}
|
||||
|
||||
<div id="actionbox">
|
||||
<h3>Нам не получилось взять данные о <i>компании</i> c id {{oid}} в 2GIS</h3>
|
||||
Вероятно, это не компания, а что-то ужасно интересное (географический объект или тайный штаб рептилоидов Верхней Пышмы).
|
||||
<p>
|
||||
Вы можете проверить сами по <a href="https://2gis.ru/af2gis/geo/{{oid}}">этой ссылке</a>. Только, если это штаб - не болтайте о нем! Мы вас не знаем, вы нас не знаете.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="company-lists">
|
||||
<div class="company-list trusted-companies" id="trusted">
|
||||
<h2>Trusted</h2>
|
||||
</div>
|
||||
|
||||
<div class="company-list suspicious-companies">
|
||||
<h2>Untrusted</h2>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="company-lists">
|
||||
<div class="company-list trusted-companies">
|
||||
<h2>Проверенные компании</h2>
|
||||
<ul>
|
||||
<li><a href="#" class="company-link">ООО "ГеоСтройПроект" <span class="company-tag trusted-tag">Проверено</span></a></li>
|
||||
<li><a href="#" class="company-link">АО "ТрансГеоТех" <span class="company-tag trusted-tag">Проверено</span></a></li>
|
||||
<li><a href="#" class="company-link">ЗАО "ГеоИнвест" <span class="company-tag trusted-tag">Проверено</span></a></li>
|
||||
<li><a href="#" class="company-link">ООО "ЭкоГеоСервис" <span class="company-tag trusted-tag">Проверено</span></a></li>
|
||||
<li><a href="#" class="company-link">НПО "ГеоАналитика" <span class="company-tag trusted-tag">Проверено</span></a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="company-list suspicious-companies">
|
||||
<h2>Подозрительные компании</h2>
|
||||
<ul>
|
||||
<li><a href="#" class="company-link">ООО "ГеоМираж" <span class="company-tag suspicious-tag">Подозрительная</span></a></li>
|
||||
<li><a href="#" class="company-link">АО "БыстрыйГеоСервис" <span class="company-tag suspicious-tag">Подозрительная</span></a></li>
|
||||
<li><a href="#" class="company-link">ЗАО "ГеоФастГрупп" <span class="company-tag suspicious-tag">Подозрительная</span></a></li>
|
||||
<li><a href="#" class="company-link">ООО "ГеоЭкспресс" <span class="company-tag suspicious-tag">Подозрительная</span></a></li>
|
||||
<li><a href="#" class="company-link">НПО "ГеоТехнологии" <span class="company-tag suspicious-tag">Подозрительная</span></a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="/static/js/app.js">
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
{% endblock %}
|
||||
|
|
@ -1,16 +1,31 @@
|
|||
{% extends 'base.html' %}
|
||||
|
||||
{% block title %}{{title}}{% endblock %}
|
||||
{% block header %}{{title}} - статистический анализ отзывов{% endblock %}
|
||||
{% block header %}{{title}} :: статистический анализ отзывов{% endblock %}
|
||||
|
||||
|
||||
{% block content %}
|
||||
|
||||
|
||||
<div id="actionbox">
|
||||
Обрабатывается.
|
||||
Позиция/размер очереди: {{position}}/{{qsize}}.
|
||||
Состояние обработчика: {{wstatus}}
|
||||
<button id="auto_refresh_btn" onclick="window.location.reload()">Обновить вручную</button>
|
||||
{% if wstatus == oid %}
|
||||
<div class="progress-div">
|
||||
<h3>Именно в предвкушении и есть главное наслаждение, не правда ли?</h3>
|
||||
Прямо сейчас наш обработчик уже обсчитывает Вашу задачу и уже скоро Вы увидите что-то ранее невидимое для Вас.
|
||||
|
||||
</div>
|
||||
{% else%}
|
||||
<div class="progress-div">
|
||||
<h3>Пришло время размышлений о смысле жизни!</h3>
|
||||
<p>
|
||||
Ваша задача стоит в очереди ({{qpos}}). Всего в очереди таких задач: {{qsize}}.
|
||||
</p>
|
||||
<p>
|
||||
А вы, добрый и веселый, сидите в удобном кресле.
|
||||
</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<button id="auto_refresh_btn" onclick="window.location.reload()">Обновить (10)</button>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -8,23 +8,40 @@
|
|||
{% block content %}
|
||||
|
||||
|
||||
<div class="result-container" id="result-container">
|
||||
{'tags': None, 'title': 'Harucha, Точки безалкогольных напитков', 'town': 'Новосибирск', 'alias': None, 'oid': '70000001080364339',
|
||||
'hits': 16, 'median': 7, 'arating': 4.5, 'brating': 5.0}
|
||||
{% if score.trusted %}
|
||||
<div class="trusted-companies">
|
||||
{% set divclass = "trusted-companies" %}
|
||||
{% set pclass = "green" %}
|
||||
{% else %}
|
||||
<div class="suspicious-companies">
|
||||
{% set divclass = "suspicious-companies" %}
|
||||
{% set pclass = "red" %}
|
||||
{% endif %}
|
||||
<h2>{{c.title}}</h2>
|
||||
<h3>{{c.address}}</h3>
|
||||
{% if score.trusted %}
|
||||
<span class="error">Аномалий в отзывах не обнаружено</span>
|
||||
{% else %}
|
||||
<span class="error">Аномалия: {{score.reason}}</span>
|
||||
{% endif %}
|
||||
|
||||
<div id="result-container">
|
||||
<div class="companyinfo">
|
||||
<div class="{{divclass}}">
|
||||
<p><b>{{c.title}} [ <a href="https://2gis.ru/af2gis/firm/{{c.object_id}}">2GIS</a> ]</b></p>
|
||||
<p>{{c.address}}</p>
|
||||
|
||||
{% if score.trusted %}
|
||||
<p class="green">Аномалий в отзывах не обнаружено</p>
|
||||
{% else %}
|
||||
<p class="red"><b>Отзывы не внушают доверия</b></p>
|
||||
<p class="red"><b>Аномалия:</b> {{score.reason}}</p>
|
||||
{% endif %}
|
||||
<p>Дата отчета: {{score.date}} </p>
|
||||
</div> <!-- company info div -->
|
||||
|
||||
<div class="score {{divclass}}">
|
||||
{% for key, value in score.items() %}
|
||||
{% if key != "param_fp" and key != "trusted" and key != "date" and key != "reason" %}
|
||||
<p>{{key}}: {{value}}</p>
|
||||
{% endif %}
|
||||
{% endfor%}
|
||||
</div> <!-- score div -->
|
||||
</div> <!-- companyinfo -->
|
||||
|
||||
{% if relations%}
|
||||
<h3>Значимые связи от {{title}}</h3>
|
||||
<table>
|
||||
<thead>
|
||||
<th>Компания</th>
|
||||
|
|
@ -38,11 +55,33 @@
|
|||
{% for rel in relations %}
|
||||
|
||||
<tr>
|
||||
<td>{% if rel.risk %}<span class="company-tag suspicious-tag">Подозрительная</span>{% endif%}<a href="/report/{{rel.oid}}">{{rel.title}}</a></td>
|
||||
|
||||
{% if rel.hits >= settings.risk_hit_th %}
|
||||
{% set hit_class = 'red' %}
|
||||
{% else %}
|
||||
{% set hit_class = '' %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if rel.arating >= settings.risk_highrate_th and rel.brating >= settings.risk_highrate_th %}
|
||||
{% set rate_class = 'red' %}
|
||||
{% else %}
|
||||
{% set rate_class = '' %}
|
||||
{% endif %}
|
||||
|
||||
<td>{% if rel.risk %}<span class="company-tag suspicious-tag">Подозрительная связь</span>{% endif%}<a href="/report/{{rel.oid}}"><b>{{rel.title}}</b></a>
|
||||
{% if rel.trusted is not none %}
|
||||
{% if rel.trusted %}
|
||||
<span class="company-tag trusted-tag">Проверено</span>
|
||||
{% else %}
|
||||
<span class="company-tag suspicious-tag">Подозрительная</span>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{rel.town}}</td>
|
||||
<td>{{rel.hits}}</td>
|
||||
<td class="{{hit_class}}">{{rel.hits}}</td>
|
||||
<td>{{rel.median}}</td>
|
||||
<td>{{rel.arating | round(1)}} {{rel.brating | round(1)}}</td>
|
||||
<td class="{{rate_class}}">{{rel.arating | round(1) }} {{rel.brating | round(1)}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
|
|
@ -54,20 +93,12 @@
|
|||
{%endif%}
|
||||
|
||||
<div>
|
||||
<details>
|
||||
<summary>
|
||||
Score details
|
||||
</summary>
|
||||
<div>
|
||||
{% for key, value in score.items() %}
|
||||
{{key}}: {{value}}<br>
|
||||
{% endfor%}
|
||||
</div>
|
||||
</details>
|
||||
Параметры расчета:
|
||||
<span class="score">{{score.param_fp}}</span>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div id="result">
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import gzip
|
|||
from .db import db
|
||||
from .const import WSS_THRESHOLD, LOAD_NREVIEWS, SLEEPTIME
|
||||
from .settings import settings
|
||||
from .statistics import statistics
|
||||
from .session import session
|
||||
from .review import Review
|
||||
from .logger import logger
|
||||
|
|
@ -160,6 +161,9 @@ class User:
|
|||
with gzip.open(self.reviews_path, 'wt') as f:
|
||||
json.dump(self._reviews, f)
|
||||
|
||||
statistics.total_users_loaded_network += 1
|
||||
statistics.total_users_loaded += 1
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return f"https://2gis.ru/af2gis/user/{self.public_id}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue