mirror of
https://github.com/yaroslaff/antifraud2gis.git
synced 2026-10-15 13:44:14 +00:00
provider ratings
This commit is contained in:
Yaroslav Polyakov
parent
00f2322a6a
commit
2bddda00db
8 changed files with 162 additions and 82 deletions
|
|
@ -14,7 +14,8 @@ dependencies = [
|
|||
"python-multipart",
|
||||
"dramatiq[redis, watch]",
|
||||
"markdown",
|
||||
"python-frontmatter"
|
||||
"python-frontmatter",
|
||||
"evalidate"
|
||||
]
|
||||
requires-python = ">=3.8"
|
||||
authors = [{ name = "Yaroslav Polyakov", email = "yaroslaff@gmail.com" }]
|
||||
|
|
|
|||
|
|
@ -85,6 +85,40 @@ aliases = {
|
|||
'4504127921282909': {
|
||||
'alias': 'sheremetevo',
|
||||
},
|
||||
|
||||
'141265770878134': {
|
||||
'alias': 'lenta'
|
||||
},
|
||||
'141265769737695': {
|
||||
'alias': 'lenta2'
|
||||
},
|
||||
'141265769524556': {
|
||||
'alias': 'lenta3'
|
||||
},
|
||||
'141265769640819': {
|
||||
'alias': 'auchan'
|
||||
},
|
||||
'141265770910298': {
|
||||
'alias': 'auchan2'
|
||||
},
|
||||
'141265770140530': {
|
||||
'alias': 'auchan3'
|
||||
},
|
||||
'70000001021506525': {
|
||||
'alias': 'hotdogmaster'
|
||||
},
|
||||
'70000001051303735': {
|
||||
'alias': 'hotdogmaster2'
|
||||
},
|
||||
'70000001017423547': {
|
||||
'alias': 'hotdogmaster3'
|
||||
},
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def resolve_alias(alias: str):
|
||||
|
|
|
|||
|
|
@ -135,6 +135,102 @@ def handle_dev(args: argparse.Namespace):
|
|||
print(u.birthday())
|
||||
|
||||
|
||||
def do_provider(args, cl):
|
||||
provider = args.args[0]
|
||||
try:
|
||||
th = int(args.args[1])
|
||||
except IndexError:
|
||||
th = 0
|
||||
|
||||
processed = 0
|
||||
provider_ratio = list()
|
||||
over_th = 0
|
||||
higher = 0
|
||||
lower = 0
|
||||
|
||||
# max PROVIDER ratio among LO companies
|
||||
maxlo = 0
|
||||
|
||||
all_providers = defaultdict(int)
|
||||
|
||||
for c in cl.companies(oid=args.company, name=args.name, town=args.town, report=args.report, noreport=args.noreport):
|
||||
nprov = 0
|
||||
total = 0
|
||||
ratio = 0
|
||||
pr = list()
|
||||
r = list()
|
||||
skipped = 0
|
||||
c.load_reviews()
|
||||
for rev in c.reviews():
|
||||
|
||||
|
||||
|
||||
if rev.age > settings.max_review_age:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
total += 1
|
||||
|
||||
all_providers[rev.provider] += 1
|
||||
|
||||
if rev.provider == provider:
|
||||
nprov += 1
|
||||
pr.append(rev.rating)
|
||||
else:
|
||||
r.append(rev.rating)
|
||||
|
||||
|
||||
|
||||
if nprov:
|
||||
ratio = int(100*nprov/total)
|
||||
|
||||
# avg rating other providers
|
||||
avg = np.mean(r) if r else 0
|
||||
# avg rating this provider
|
||||
avg_prov = np.mean(pr) if pr else 0
|
||||
|
||||
if avg_prov > avg:
|
||||
higher += 1
|
||||
hl_str = "HI"
|
||||
if avg_prov > avg + 1:
|
||||
hl_str = "HI+"
|
||||
else:
|
||||
lower += 1
|
||||
hl_str = "LO"
|
||||
if ratio > maxlo:
|
||||
maxlo = ratio
|
||||
|
||||
if ratio > th:
|
||||
over_th += 1
|
||||
print(f"{processed}: {c.object_id} {c.get_title()} (skip:{skipped}) {hl_str} ({avg:.1f}) {provider}: {nprov} / {total} = {ratio} ({avg_prov:.1f})")
|
||||
|
||||
else:
|
||||
print(f"{processed}: {c.object_id} {c.get_title()} NORATINGS from {provider}")
|
||||
pass
|
||||
|
||||
processed += 1
|
||||
provider_ratio.append(ratio)
|
||||
|
||||
|
||||
|
||||
print(f"processed {processed} company providers: {dict(all_providers)}")
|
||||
|
||||
if processed:
|
||||
# calculage mean share of this provider among all companies and amont companies which has at least one review
|
||||
mean_provider_ratio = np.mean(provider_ratio) if provider_ratio else 0
|
||||
nz_provider_ratio = list(filter(None, provider_ratio))
|
||||
nz_mean_provider_ratio = np.mean(nz_provider_ratio) if nz_provider_ratio else 0
|
||||
|
||||
|
||||
print(f"companies with 0 reviews from {provider}: {len(provider_ratio) - len(nz_provider_ratio)}")
|
||||
print(f"avg ratio of {provider}: {nz_mean_provider_ratio:.1f} ({np.mean(mean_provider_ratio):.1f} overall)")
|
||||
print(f"over th ({th}): {over_th} ({100*over_th/processed:.1f}%)")
|
||||
|
||||
hilorate = higher/lower if lower else 0
|
||||
print(f"hi: {higher} lo: {lower} hi/lo: {hilorate:.2f}")
|
||||
print(f"maxlo: {maxlo}")
|
||||
|
||||
|
||||
def get_args():
|
||||
|
||||
|
||||
|
|
@ -148,7 +244,7 @@ def get_args():
|
|||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("cmd", choices=['company-users', 'user-reviews', 'queue', 'explore', 'provider', 'sys', 'filldb', 'dev'])
|
||||
parser.add_argument("cmd", choices=['company-users', 'user-reviews', 'company-reviews', 'queue', 'explore', 'provider', 'sys', 'filldb', 'dev'])
|
||||
parser.add_argument("-v", "--verbose", default=False, action='store_true')
|
||||
parser.add_argument("--full", default=False, action='store_true')
|
||||
parser.add_argument("args", nargs='*', help='extra args')
|
||||
|
|
@ -183,11 +279,18 @@ def main():
|
|||
print(u)
|
||||
|
||||
if cmd == "user-reviews":
|
||||
|
||||
u = User(args.args[0])
|
||||
for r in u.reviews():
|
||||
print(r)
|
||||
|
||||
|
||||
if cmd == "company-reviews":
|
||||
|
||||
c = Company(args.company)
|
||||
c.load_reviews()
|
||||
for r in c.reviews():
|
||||
print(r)
|
||||
|
||||
elif cmd == "queue":
|
||||
r = redis.Redis(decode_responses=True)
|
||||
|
||||
|
|
@ -269,83 +372,7 @@ def main():
|
|||
print(f"Done. Inserted {inserted} records, already exists.")
|
||||
|
||||
elif cmd == "provider":
|
||||
provider = args.args[0]
|
||||
try:
|
||||
th = int(args.args[1])
|
||||
except IndexError:
|
||||
th = 0
|
||||
|
||||
processed = 0
|
||||
provider_ratio = list()
|
||||
over_th = 0
|
||||
higher = 0
|
||||
lower = 0
|
||||
|
||||
# max PROVIDER ratio among LO companies
|
||||
maxlo = 0
|
||||
|
||||
all_providers = defaultdict(int)
|
||||
|
||||
for c in cl.companies(oid=args.company, name=args.name, town=args.town, report=args.report, noreport=args.noreport):
|
||||
nprov = 0
|
||||
total = 0
|
||||
ratio = 0
|
||||
pr = list()
|
||||
r = list()
|
||||
c.load_reviews()
|
||||
for rev in c.reviews():
|
||||
total += 1
|
||||
|
||||
all_providers[rev.provider] += 1
|
||||
|
||||
if rev.provider == provider:
|
||||
nprov += 1
|
||||
pr.append(rev.rating)
|
||||
else:
|
||||
r.append(rev.rating)
|
||||
|
||||
|
||||
|
||||
if nprov:
|
||||
ratio = int(100*nprov/total)
|
||||
|
||||
# avg rating other providers
|
||||
avg = np.mean(r) if r else 0
|
||||
# avg rating this provider
|
||||
avg_prov = np.mean(pr) if pr else 0
|
||||
|
||||
if avg_prov > avg:
|
||||
higher += 1
|
||||
hl_str = "HI"
|
||||
if avg_prov > avg + 1:
|
||||
hl_str = "HI+"
|
||||
else:
|
||||
lower += 1
|
||||
hl_str = "LO"
|
||||
if ratio > maxlo:
|
||||
maxlo = ratio
|
||||
|
||||
if ratio > th:
|
||||
over_th += 1
|
||||
print(f"{c.object_id} {c.get_title()} {hl_str} ({avg:.1f}) {provider}: {nprov} / {total} = {ratio} ({avg_prov:.1f})")
|
||||
|
||||
else:
|
||||
# print(f"{c.object_id} {c.get_title()} no ratings from {provider}")
|
||||
pass
|
||||
|
||||
processed += 1
|
||||
provider_ratio.append(ratio)
|
||||
|
||||
print(f"processed {processed} companies: {dict(all_providers)}")
|
||||
nz_provider_ratio = list(filter(None, provider_ratio))
|
||||
|
||||
print(f"companies with 0 reviews from {provider}: {len(provider_ratio) - len(nz_provider_ratio)}")
|
||||
print(f"avg ratio of {provider}: {np.mean(nz_provider_ratio):.1f} ({np.mean(provider_ratio):.1f} overall)")
|
||||
print(f"over th ({th}): {over_th} ({100*over_th/processed:.1f}%)")
|
||||
|
||||
hilorate = higher/lower if lower else 0
|
||||
print(f"hi: {higher} lo: {lower} hi/lo: {hilorate:.2f}")
|
||||
print(f"maxlo: {maxlo}")
|
||||
do_provider(args, cl)
|
||||
|
||||
elif cmd == "explore":
|
||||
|
||||
|
|
|
|||
|
|
@ -112,7 +112,12 @@ def main():
|
|||
compare(c1, c2)
|
||||
|
||||
elif args.cmd == "info":
|
||||
c = Company(args.company)
|
||||
|
||||
try:
|
||||
c = Company(args.company)
|
||||
except AFNoCompany:
|
||||
print(f"Company {args.company} not found")
|
||||
return
|
||||
basic = json.load(gzip.open(c.basic_path))
|
||||
print(c)
|
||||
print_json(data=basic)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from .exceptions import AFNoCompany
|
|||
from .aliases import resolve_alias
|
||||
from .statistics import statistics
|
||||
from .aliases import aliases, resolve_alias
|
||||
from .db import db
|
||||
|
||||
# to avoid circular import
|
||||
#class RelationDict:
|
||||
|
|
@ -445,6 +446,7 @@ class CompanyList():
|
|||
|
||||
for f in self.path.glob('*-basic.json.gz'):
|
||||
company_oid = f.name.split('-')[0]
|
||||
|
||||
c = Company(company_oid)
|
||||
# print("created c", c.report_path)
|
||||
if company_match(c, oid=oid, name=name, town=town, detection=detection, report=report, noreport=noreport):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,16 @@ import json
|
|||
from loguru import logger
|
||||
|
||||
|
||||
|
||||
"""
|
||||
class to skip no-users (private users) and no-companies (e.g. geo objects, deleted companies)
|
||||
no-company examples:
|
||||
70030076824955120
|
||||
70000001026994387
|
||||
70000001052074829
|
||||
70000001037658506
|
||||
141373143684631
|
||||
"""
|
||||
class MyDB:
|
||||
def __init__(self, directory=Path("~/.cache").expanduser()):
|
||||
self.directory = directory
|
||||
|
|
|
|||
|
|
@ -102,4 +102,4 @@ class Review():
|
|||
else:
|
||||
title = "<NO TITLE>"
|
||||
|
||||
return f'Review({self.created_str} {self.uid} {user_str} (rating:{self.rating} {photo_str}) > {self.oid} {title})'
|
||||
return f'Review({self.created_str} {self.provider} {self.uid} {user_str} {self.rating}) > {self.oid} {title})'
|
||||
|
|
@ -39,6 +39,7 @@
|
|||
</div> <!-- company info div -->
|
||||
|
||||
<div class="score {{divclass}}">
|
||||
<p>ID: {{c.object_id}}</p>
|
||||
{% if score.detections %}
|
||||
<p>Detections:</p>
|
||||
<ul>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue