fix: scanner.py - parallel github_stat scan

update DB
This commit is contained in:
Dr.Lt.Data
2024-05-14 20:32:44 +09:00
parent 45972609a3
commit ef6b6ff690
7 changed files with 318 additions and 216 deletions

View File

@@ -317,9 +317,6 @@ def update_custom_nodes():
url, item = url_item
github_stats[url] = item
with open('github-stats-cache.json', 'w', encoding='utf-8') as file:
json.dump(github_stats, file, ensure_ascii=False, indent=4)
# renew outdated cache
outdated_urls = []
for k, v in github_stats.items():
@@ -327,8 +324,18 @@ def update_custom_nodes():
if elapsed > 60*60*12: # 12 hours
outdated_urls.append(k)
for url in outdated_urls:
renew_stat(url)
with concurrent.futures.ThreadPoolExecutor(11) as executor:
for url in outdated_urls:
futures.append(executor.submit(renew_stat, url))
for future in concurrent.futures.as_completed(futures):
url_item = future.result()
if url_item is not None:
url, item = url_item
github_stats[url] = item
with open('github-stats-cache.json', 'w', encoding='utf-8') as file:
json.dump(github_stats, file, ensure_ascii=False, indent=4)
with open(GITHUB_STATS_FILENAME, 'w', encoding='utf-8') as file:
for v in github_stats.values():