Gtag, better explain for risk_users

This commit is contained in:
Yaroslav Polyakov 2025-04-26 15:22:21 +07:00
parent 065d90188c
commit 93b5b8d863
12 changed files with 273 additions and 93 deletions

View file

@ -29,6 +29,7 @@ from ..settings import settings
from ..const import REDIS_TASK_QUEUE_NAME, REDIS_TRUSTED_LIST, REDIS_UNTRUSTED_LIST, REDIS_WORKER_STATUS
# from ..search import search
from ..companydb import dbsearch
from ..compare import compare
app = FastAPI()
@ -47,20 +48,30 @@ r = redis.Redis(decode_responses=True)
class ReportRequest(BaseModel):
oid: str
@app.get("/", response_class=HTMLResponse)
async def home(request: Request):
def render(request, template_name, context: dict):
# Добавляем общие данные
last_trusted = [json.loads(item) for item in r.lrange(REDIS_TRUSTED_LIST, 0, -1)]
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
return templates.TemplateResponse(
"index.html",
context.update({
"request": request,
"settings": settings,
"trusted": last_trusted,
"untrusted": last_untrusted,
})
return templates.TemplateResponse(template_name, context)
@app.get("/", response_class=HTMLResponse)
async def home(request: Request):
return render(request, "index.html",
{
"request": request,
"trusted": last_trusted,
"untrusted": last_untrusted
}
}
)
@ -76,6 +87,30 @@ async def explain(request: Request, oid: str):
return PlainTextResponse(content=explanation)
@app.get("/compare/{oida}/{oidb}", response_class=PlainTextResponse)
async def route_compare(request: Request, oida: str, oidb: str):
try:
ca = Company(oida)
except (AFNoCompany, AFNoTitle):
return RedirectResponse(app.url_path_for("miss", oid=oida))
try:
cb = Company(oidb)
except (AFNoCompany, AFNoTitle):
return RedirectResponse(app.url_path_for("miss", oid=oidb))
cmp_report = f"/tmp/af2gis-cmp-{oida}-{oidb}.txt"
# rewrite as with
with open(cmp_report, "w") as fh:
compare(ca, cb, fh=fh)
# return this file
data = open(cmp_report).read()
os.unlink(cmp_report)
return PlainTextResponse(content=data)
@app.get("/report/{oid}", response_class=HTMLResponse)
async def report(request: Request, oid: str):
try:
@ -103,7 +138,8 @@ async def report(request: Request, oid: str):
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
return templates.TemplateResponse(
return render(
request,
"report.html", {
"request": request,
"settings": settings,
@ -135,7 +171,8 @@ async def miss(request: Request, oid: str):
assert c.title is not None
except (AFNoCompany, AFNoTitle, AssertionError) as e:
return templates.TemplateResponse(
return render(
request,
"nocompany.html", {
"request": request, "oid": oid,
"trusted": last_trusted,
@ -146,7 +183,8 @@ async def miss(request: Request, oid: str):
return templates.TemplateResponse(
return render(
request,
"miss.html", {
"request": request, "title": c.title, "oid": c.object_id,
"settings": settings,
@ -163,7 +201,7 @@ async def submit(request: Request, oid: str = Form(...), force: bool = Form(Fals
try:
c = Company(oid)
except (AFNoCompany) as e:
return templates.TemplateResponse(
return render(request,
"nocompany.html", {
"request": request, "oid": oid,
}
@ -241,7 +279,8 @@ async def search_view(request: Request, query: str, detections: str):
last_trusted = [json.loads(item) for item in r.lrange(REDIS_TRUSTED_LIST, 0, -1)]
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
return templates.TemplateResponse(
return render(
request,
"search.html", {
"request": request,
"query": query,
@ -262,7 +301,8 @@ async def progress(request: Request, oid: str):
c = Company(oid)
print("miss for", c)
except (AFNoCompany, AssertionError):
return templates.TemplateResponse(
return render(
request,
"nocompany.html", {
"request": request, "oid": oid,
}
@ -287,7 +327,8 @@ async def progress(request: Request, oid: str):
last_untrusted = [json.loads(item) for item in r.lrange(REDIS_UNTRUSTED_LIST, 0, -1)]
return templates.TemplateResponse(
return render(
request,
"progress.html", {
"request": request, "title": c.title, "oid": c.object_id,
"qsize": queue_size, "wstatus": wstatus, "qpos": qpos,
@ -317,7 +358,7 @@ def md_page(request: Request, page: str):
return templates.TemplateResponse("markdown.html", {
return render(request, "markdown.html", {
"request": request,
"title": title,
"content": content_html,

View file

@ -139,10 +139,35 @@ def handle_dev(args: argparse.Namespace):
print(u.birthday())
def town_iterator(town: str, nreviews: int):
for crec in dbsearch(query='', addr=town, nreviews=nreviews, limit=0):
yield crec
def nreviews_for_oid(r, oid: str, provider:str):
key = f'af2gis:nreviews:{oid}:{provider}'
nr = r.get(key)
if nr is not None:
return int(nr)
# calculate and save
c = Company(oid)
c.load_reviews()
nr = c.nreviews(provider=provider)
r.set(key, nr)
return nr
def do_delkeys(prefix: str):
r = redis.Redis(decode_responses=True)
for key in r.scan_iter(f"{prefix}*"):
print("delete", key)
r.delete(key)
def do_provider(args, cl: CompanyList):
started = time.time()
provider = args.args[0]
redis_conn = redis.Redis(decode_responses=True)
try:
min_nprov_th = int(args.args[1])
@ -154,38 +179,45 @@ def do_provider(args, cl: CompanyList):
except IndexError:
th = 0
started = time.time()
provider = args.args[0]
print(f"# Analyse companies with {min_nprov_th}+ reviews from {provider}, show companies with more then {th}% reviews from {provider}")
processed = 0
provider_ratio = list()
over_th = 0
higher = 0
lower = 0
# max PROVIDER ratio among LO companies
maxlo = 0
all_providers = defaultdict(int)
for c in cl.companies(oid=args.company, name=args.name, town=args.town, report=args.report, noreport=args.noreport, limit=0):
for crec in town_iterator(args.town, nreviews=100):
if c.object_id in settings.skip_oids:
if crec['oid'] in settings.skip_oids:
continue
try:
crec['provider_nr'] = nreviews_for_oid(redis_conn, crec['oid'], provider=provider)
if crec['provider_nr'] < min_nprov_th:
continue
except AFCompanyError as e:
continue
# here we start processing
nprov = 0
total = 0
ratio = 0
pr = list()
r = list()
prov_rating = list()
rating = list()
skipped = 0
c = Company(crec['oid'])
c.load_reviews()
for rev in c.reviews():
if rev.age > settings.max_review_age:
skipped += 1
continue
for rev in c.reviews():
total += 1
@ -193,60 +225,48 @@ def do_provider(args, cl: CompanyList):
if rev.provider == provider:
nprov += 1
pr.append(rev.rating)
prov_rating.append(rev.rating)
else:
r.append(rev.rating)
rating.append(rev.rating)
if nprov > min_nprov_th:
# percent of this provider/total
ratio = int(100*nprov/total)
# percent of this provider/total
ratio = int(100*nprov/total)
# avg rating other providers
avg = np.mean(r) if r else 0
# avg rating this provider
avg_prov = np.mean(pr) if pr else 0
# avg rating other providers
avg = np.mean(rating) if rating else 0
# avg rating this provider
avg_prov = np.mean(prov_rating) if prov_rating else 0
if avg_prov > avg:
higher += 1
hl_str = "HI"
if avg_prov > avg + 1:
hl_str = "HI+"
else:
lower += 1
hl_str = "LO"
if ratio > maxlo:
maxlo = ratio
if ratio > th:
over_th += 1
print(f"{processed}: {c.object_id} {c.get_title()} (skip:{skipped}) {hl_str} ({avg:.1f}) {provider}: {nprov} / {total} = {ratio} ({avg_prov:.1f})")
else:
print(f"{processed}: {c.object_id} {c.get_title()} NORATINGS({nprov}) from {provider}")
pass
if avg_prov > avg:
higher += 1
hl_str = "HI"
if avg_prov > avg + settings.rating_diff:
hl_str = "HI+"
else:
lower += 1
hl_str = "LO"
processed += 1
provider_ratio.append(ratio)
reset_user_pool()
if ratio > th:
over_th += 1
print(f"{processed}: {c.object_id} {c.get_title()} (skip:{skipped}) {hl_str} ({avg:.1f}) {provider}: {nprov} / {total} = {ratio} ({avg_prov:.1f})")
print(f" rating ({len(rating)}): {rating}")
print(f" {provider} rating ({len(prov_rating)}): {prov_rating}")
print(f"processed {processed} companies in {int(time.time() - started)} sec. providers: {dict(all_providers)}")
if processed:
# calculage mean share of this provider among all companies and amont companies which has at least one review
mean_provider_ratio = np.mean(provider_ratio) if provider_ratio else 0
nz_provider_ratio = list(filter(None, provider_ratio))
nz_mean_provider_ratio = np.mean(nz_provider_ratio) if nz_provider_ratio else 0
print(f"companies with 0 reviews from {provider}: {len(provider_ratio) - len(nz_provider_ratio)}")
print(f"avg ratio of {provider}: {nz_mean_provider_ratio:.1f} ({np.mean(mean_provider_ratio):.1f} overall)")
print(f"over th ({th}): {over_th} ({100*over_th/processed:.1f}%)")
hilorate = higher/lower if lower else 0
print(f"hi: {higher} lo: {lower} hi/lo: {hilorate:.2f}")
print(f"maxlo: {maxlo}")
return
def lmdb_dump(args):
@ -283,7 +303,7 @@ def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("cmd", choices=['company-users', 'user-reviews', 'company-reviews', 'queue', 'explore', 'provider', 'sys', 'filldb', 'dev', 'lmdb', 'convert'])
parser.add_argument("cmd", choices=['company-users', 'user-reviews', 'company-reviews', 'queue', 'explore', 'provider', 'sys', 'filldb', 'dev', 'lmdb', 'convert', 'delkeys'])
parser.add_argument("-v", "--verbose", default=False, action='store_true')
parser.add_argument("--full", default=False, action='store_true')
parser.add_argument("args", nargs='*', help='extra args')
@ -317,13 +337,13 @@ def main():
for u in c.users():
print(u)
if cmd == "user-reviews":
elif cmd == "user-reviews":
u = User(args.args[0])
for r in u.reviews():
print(r)
if cmd == "company-reviews":
elif cmd == "company-reviews":
c = Company(args.company)
c.load_reviews()
@ -536,4 +556,9 @@ def main():
elif cmd == "lmdb":
lmdb_dump(args)
elif cmd == "delkeys":
do_delkeys(args.args[0])
else:
print(f"Unknown command {cmd!r}")

View file

@ -122,6 +122,7 @@ def main():
except (AFNoCompany, AFNoTitle):
print(f"Company {args.company} not found")
return
c.load_reviews()
basic = json.load(gzip.open(c.basic_path))
print(c)
print_json(data=basic)
@ -135,7 +136,7 @@ def main():
print(f"{len(report['relations'])} relations")
elif args.cmd == "search":
res = dbsearch(args.args[0], detection=args.detection, addr=args.town, extra=args.args[1:], limit=500)
res = dbsearch(args.args[0], detection=args.detection, addr=args.town, limit=500)
for rec in res:
print(rec)
if args.fmt == 'full':

View file

@ -7,6 +7,7 @@ import gzip
import traceback
import numpy as np
import lmdb
import redis
from rich.progress import Progress
from rich import print_json
from rich.pretty import pretty_repr
@ -194,9 +195,6 @@ class Company:
return len(self._reviews)
def count_rate(self):
self.ratings = list()
for r in self._reviews:
if r['rating'] is None:
@ -217,6 +215,10 @@ class Company:
continue
yield User(uid)
def users_ids(self):
for r in self._reviews:
yield r['user']['public_id']
def uids(self):
for r in self._reviews:
uid = r['user']['public_id']
@ -377,12 +379,29 @@ class Company:
def reviews(self):
for r in self._reviews:
yield Review(r, company=self)
def nreviews(self):
return len(self._reviews)
rev = Review(r, company=self)
if rev.age > settings.max_review_age:
continue
yield rev
def review_from(self, uid: str):
def nreviews(self, provider = None):
if provider is None:
return len(self._reviews)
# count reviews now myself
n = 0
for rev in self._reviews:
r = Review(rev, company=self)
if r.age > settings.max_review_age:
continue
if provider == '*' or rev['provider'] == provider:
n += 1
return n
def review_from(self, uid: str) -> Review:
self.load_reviews()
for r in self._reviews:
if r['user']['public_id'] == uid:

View file

@ -41,7 +41,7 @@ def get_by_oid(oid: str, conn = None) -> Optional[dict]:
col_names = [desc[0] for desc in cursor.description]
return dict(zip(col_names, row))
def dbsearch(query: str, addr: str = None, limit=None, detection=None, conn = None) -> list[dict]:
def dbsearch(query: str, addr: str = None, limit=None, nreviews=None, detection=None, conn = None) -> list[dict]:
limit = 100 if limit is None else int(limit)
@ -50,7 +50,7 @@ def dbsearch(query: str, addr: str = None, limit=None, detection=None, conn = No
conn = conn or make_connection()
if query and query not in ('.', 'ALL'):
if query and query not in ('', '.', 'ALL'):
words = query.strip().lower().split()
if not words:
return []
@ -73,6 +73,10 @@ def dbsearch(query: str, addr: str = None, limit=None, detection=None, conn = No
clauses += " AND detections LIKE ?"
params.append(f"%{detection}%")
if nreviews:
clauses += " AND nreviews >= ?"
params.append(int(nreviews))
if addr:
clauses += " AND address LIKE ?"
params.append(f"%{addr}%")

View file

@ -5,9 +5,74 @@ import numpy as np
from .company import Company
from .user import get_user
from .db import db
from .settings import settings
def quick_compare(a: Company, b: Company, fh = None):
""" new quick compare """
if fh is None:
fh = sys.stdout
a.load_reviews()
b.load_reviews()
print(settings.debug_uids)
print("A:", a, file=fh)
print("B:", b, file=fh)
print("--")
users_a = set(a.users_ids())
users_b = set(b.users_ids())
for uid in settings.debug_uids:
if uid in users_a:
print(f"User {uid} in A", file=fh)
if uid in users_b:
print(f"User {uid} in B", file=fh)
common_uids = users_a & users_b
a_first = None
a_last = None
b_first = None
b_last = None
ar = list()
br = list()
for idx, uid in enumerate(common_uids):
u = get_user(uid)
ua = a.review_from(uid)
ub = b.review_from(uid)
#if ua is None or ub is None:
# continue
### calc oldest/newest for a/b reviews
if a_first is None or a_first > ua.created:
a_first = ua.created
if a_last is None or a_last < ua.created:
a_last = ua.created
if b_first is None or b_first > ub.created:
b_first = ub.created
if b_last is None or b_last < ub.created:
b_last = ub.created
ar.append(ua.rating)
br.append(ub.rating)
print(f"{idx}: {u.url} {u.name!r} {u.nreviews()} A:{ua.rating} {ua.created_str} B:{ub.rating} {ub.created_str}", file=fh)
print(f"# Reviews for A: {a_first} .. {a_last} avg: {round(np.mean(ar), 1)}", file=fh)
print(f"# Reviews for B: {b_first} .. {b_last} avg: {round(np.mean(br), 1)}", file=fh)
def compare(a: Company, b: Company):
""" old and slow but reliable compare """
debug_oids = os.getenv("DEBUG_OIDS", "").split(" ")
debug_uids = os.getenv("DEBUG_UIDS", "").split(" ")

View file

@ -3,13 +3,15 @@ import numpy as np
from .fd import BaseFD
from ..user import User
from ..user import User, get_user
from ..company import Company
from ..companydb import get_by_oid
from ..review import Review
from ..settings import settings
from ..relation import RelationDict
"""
test companies for relation (detect):
2956015537022403
@ -117,8 +119,17 @@ class RelationFD(BaseFD):
if dname == 'risk_users':
print(f"Risk users ({len(self.risk_users)} / {self.processed_users} > {settings.risk_user_ratio}%)", file=fh)
for idx, public_id in enumerate(self.risk_users.keys(), start=1):
print(f"{idx}. {public_id} ({len(self.risk_users[public_id])}): {' '.join(self.risk_users[public_id])}", file=fh)
u = get_user(public_id)
print(f"user #{idx}. {public_id} {u.name} ({len(self.risk_users[public_id])}):", file=fh)
for oid in self.risk_users[public_id]:
crec = get_by_oid(oid)
if crec:
print(f" {oid} {crec['title']} {crec['address']}", file=fh)
else:
print(f" {oid} [special-not-a-company]", file=fh)
print(f"{len(self.risk_users)} / {self.processed_users} = {self.score['risk_users']}%", file=fh)
print("", file=fh)

View file

@ -98,4 +98,4 @@ class Review():
else:
title = "<NO TITLE>"
return f'Review({self.created_str} {self.provider} {self.uid} {user_str} {self.rating}) > {self.oid} {title})'
return f'Review({self.created_str} {self.provider} {self.uid} {user_str} {self.rating} > {self.oid} {title})'

View file

@ -65,13 +65,14 @@ class Settings():
# Other system parameters
self.proxy = os.getenv('HTTPS_PROXY', None)
# web UI
self.turnstile_sitekey = os.getenv('TURNSTILE_SITEKEY', None)
self.turnstile_secret = os.getenv('TURNSTILE_SECRET', None)
self.gtag = os.getenv('GTAG', None)
# Debugging
self.skip_oids = list(filter(None, os.getenv('SKIP_OIDS', '').split(' ')))
self.debug_uids = list(filter(None, os.getenv('DEBUG_UIDS', '').split(' ')))
def param_fp(self):
return f"risk_hit={self.risk_hit_th} risk_median_th={self.risk_median_th} risk_highrate_th={self.risk_highrate_th} " \
@ -81,7 +82,7 @@ class Settings():
f"median_user_age={self.median_user_age} median_rpu={self.median_rpu} " \
f"sametitle_rel={self.sametitle_rel} sametitle_ratio={self.sametitle_ratio} " \
f"apply_empty_user_min={self.apply_empty_user_min} apply_median_rpu={self.apply_median_rpu} apply_median_userage={self.apply_median_userage} " \
f"rating_diff={self.rating_diff}"
f"rating_diff={self.rating_diff} max_review_age={self.max_review_age}"
settings = Settings()

View file

@ -14,6 +14,19 @@
<!-- captcha -->
<script src="https://challenges.cloudflare.com/turnstile/v0/api.js" defer></script>
{% if settings.gtag %}
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id={{settings.gtag}}"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-MEH7Q9ZBRM');
</script>
{% endif %}
</head>
<body>

View file

@ -1,8 +1,7 @@
{% extends 'base.html' %}
{% block title %}2GIS антифрод - статистическая проверка отзывов{% endblock %}
{% block header %}2GIS антифрод - проверка отзывов{% endblock %}
{% block title %}Проверка отзывов 2GIS :: Антифрод{% endblock %}
{% block header %}Проверка отзывов 2GIS :: Антифрод{% endblock %}
{% block content %}
<div id="result-container">

View file

@ -98,7 +98,8 @@
<td>{{rel.town}}</td>
<td class="{{hit_class}}">{{rel.hits}}</td>
<td>{{rel.median}}</td>
<td class="{{rate_class}}">{{rel.arating | round(1) }} {{rel.brating | round(1)}}</td>
<td class="{{rate_class}}"><a href="/compare/{{oid}}/{{rel.oid}}">
{{rel.arating | round(1) }} {{rel.brating | round(1)}}</a></td>
</tr>
{% endfor %}
</tbody>