fix: scanner.py - parallel github_stat scan
update DB
This commit is contained in:
17
scanner.py
17
scanner.py
@@ -317,9 +317,6 @@ def update_custom_nodes():
|
||||
url, item = url_item
|
||||
github_stats[url] = item
|
||||
|
||||
with open('github-stats-cache.json', 'w', encoding='utf-8') as file:
|
||||
json.dump(github_stats, file, ensure_ascii=False, indent=4)
|
||||
|
||||
# renew outdated cache
|
||||
outdated_urls = []
|
||||
for k, v in github_stats.items():
|
||||
@@ -327,8 +324,18 @@ def update_custom_nodes():
|
||||
if elapsed > 60*60*12: # 12 hours
|
||||
outdated_urls.append(k)
|
||||
|
||||
for url in outdated_urls:
|
||||
renew_stat(url)
|
||||
with concurrent.futures.ThreadPoolExecutor(11) as executor:
|
||||
for url in outdated_urls:
|
||||
futures.append(executor.submit(renew_stat, url))
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
url_item = future.result()
|
||||
if url_item is not None:
|
||||
url, item = url_item
|
||||
github_stats[url] = item
|
||||
|
||||
with open('github-stats-cache.json', 'w', encoding='utf-8') as file:
|
||||
json.dump(github_stats, file, ensure_ascii=False, indent=4)
|
||||
|
||||
with open(GITHUB_STATS_FILENAME, 'w', encoding='utf-8') as file:
|
||||
for v in github_stats.values():
|
||||
|
||||
Reference in New Issue
Block a user