Refactor graphrag to remove redis lock (#5828)

### What problem does this PR solve?

Refactor graphrag to remove redis lock

### Type of change

- [x] Refactoring
This commit is contained in:
Zhichang Yu
2025-03-10 15:15:06 +08:00
committed by GitHub
parent 1163e9e409
commit 6ec6ca6971
9 changed files with 602 additions and 332 deletions

View File

@ -16,13 +16,12 @@
import logging
import json
import time
import uuid
import valkey as redis
from rag import settings
from rag.utils import singleton
from valkey.lock import Lock
class RedisMsg:
def __init__(self, consumer, queue_name, group_name, msg_id, message):
@ -281,29 +280,23 @@ REDIS_CONN = RedisDB()
class RedisDistributedLock:
def __init__(self, lock_key, timeout=10):
def __init__(self, lock_key, lock_value=None, timeout=10, blocking_timeout=1):
self.lock_key = lock_key
self.lock_value = str(uuid.uuid4())
if lock_value:
self.lock_value = lock_value
else:
self.lock_value = str(uuid.uuid4())
self.timeout = timeout
self.lock = Lock(REDIS_CONN.REDIS, lock_key, timeout=timeout, blocking_timeout=blocking_timeout)
@staticmethod
def clean_lock(lock_key):
REDIS_CONN.REDIS.delete(lock_key)
def acquire(self):
return self.lock.acquire()
def acquire_lock(self):
end_time = time.time() + self.timeout
while time.time() < end_time:
if REDIS_CONN.REDIS.setnx(self.lock_key, self.lock_value):
return True
time.sleep(1)
return False
def release_lock(self):
if REDIS_CONN.REDIS.get(self.lock_key) == self.lock_value:
REDIS_CONN.REDIS.delete(self.lock_key)
def release(self):
return self.lock.release()
def __enter__(self):
self.acquire_lock()
self.acquire()
def __exit__(self, exception_type, exception_value, exception_traceback):
self.release_lock()
self.release()