Skip to content

fix(lock): implement Redlock single-instance pattern in LockManagerService #537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
romanetar wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix(lock): implement Redlock single-instance pattern in LockManagerService #537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c6e6473
fix(lock): implement Redlock single-instance pattern in LockManagerSe…
romanetar May 4, 2026
fbe981c
fix: review feedback
romanetar Jun 15, 2026
4b0e311
fix: PR review feedback
romanetar Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/push.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ jobs:
- { name: "GuzzleTracingTest", filter: "--filter GuzzleTracingTest" }
- { name: "Repositories", filter: "--filter tests/Repositories/" }
- { name: "Services", filter: "--filter tests/Unit/Services/" }
- { name: "Integration", filter: "tests/Integration/" }
env:
OTEL_SERVICE_ENABLED: false
APP_ENV: testing
Expand Down
10 changes: 10 additions & 0 deletions Libs/Utils/ICacheService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,16 @@ public function setSingleValue($key, $value, $ttl = 0);
*/
public function addSingleValue($key, $value, $ttl = 0);

/**
* Atomically compare-and-delete: DEL the key only when its current value
* equals $expectedValue. Implementations MUST use an atomic operation
* (Lua EVAL or equivalent) — never a separate GET + conditional DEL.
* @param string $key
* @param string $expectedValue
* @return bool true iff the key existed, matched, and was deleted
*/
public function deleteIfValueMatches(string $key, string $expectedValue): bool;

/**
* Set time to live to a given key
* @param $key
Expand Down
39 changes: 24 additions & 15 deletions app/Services/Model/Imp/SummitOrderService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ private function buildPrePaidSaga(Member $owner, Summit $summit, array $payload)
$this->member_repository,
$this->attendee_repository,
$this->ticket_type_repository,
$this->promo_code_repository,
$this->tx_service,
$this->lock_service
));
Expand Down Expand Up @@ -827,7 +828,7 @@ public function run(array $formerState): array

$this->lock_service->lock('promocode.' . $promo_code->getId() . '.usage.lock', function () use ($promo_code, $qty, $owner_email) {
$promo_code->addUsage($owner_email, $qty);
});
}, 30);

});
// mark a done
Expand Down Expand Up @@ -865,7 +866,7 @@ public function undo()

$this->lock_service->lock('promocode.' . $promo_code->getId() . '.usage.lock', function () use ($promo_code, $info, $owner_email) {
$promo_code->removeUsage(intval($info['qty']), $owner_email);
});
}, 30);

});
}
Expand Down Expand Up @@ -950,7 +951,7 @@ public function run(array $formerState): array

$this->lock_service->lock('ticket_type.' . $ticket_type->getId() . '.sell.lock', function () use ($ticket_type, $reservations) {
$ticket_type->sell($reservations[$ticket_type->getId()]);
});
}, 30);

}
});
Expand All @@ -967,7 +968,7 @@ public function undo()
if (is_null($ticket_type)) return;
$this->lock_service->lock('ticket_type.' . $ticket_type->getId() . '.sell.lock', function () use ($ticket_type, $qty) {
$ticket_type->restore($qty);
});
}, 30);
});
}
}
Expand Down Expand Up @@ -1474,6 +1475,11 @@ final class AutoAssignPrePaidTicketTask extends AbstractTask
*/
private $ticket_type_repository;

/**
* @var ISummitRegistrationPromoCodeRepository
*/
private $promo_code_repository;

/**
* @var ILockManagerService
*/
Expand All @@ -1487,19 +1493,21 @@ final class AutoAssignPrePaidTicketTask extends AbstractTask
* @param IMemberRepository $member_repository
* @param ISummitAttendeeRepository $attendee_repository
* @param ISummitTicketTypeRepository $ticket_type_repository
* @param ISummitRegistrationPromoCodeRepository $promo_code_repository
* @param ITransactionService $tx_service
* @param ILockManagerService $lock_service
*/
public function __construct
(
?Member $owner,
Summit $summit,
array $payload,
IMemberRepository $member_repository,
ISummitAttendeeRepository $attendee_repository,
ISummitTicketTypeRepository $ticket_type_repository,
ITransactionService $tx_service,
ILockManagerService $lock_service
?Member $owner,
Summit $summit,
array $payload,
IMemberRepository $member_repository,
ISummitAttendeeRepository $attendee_repository,
ISummitTicketTypeRepository $ticket_type_repository,
ISummitRegistrationPromoCodeRepository $promo_code_repository,
ITransactionService $tx_service,
ILockManagerService $lock_service
)
{
$this->tx_service = $tx_service;
Expand All @@ -1510,6 +1518,7 @@ public function __construct
$this->member_repository = $member_repository;
$this->attendee_repository = $attendee_repository;
$this->ticket_type_repository = $ticket_type_repository;
$this->promo_code_repository = $promo_code_repository;
}

public function run(array $formerState): array
Expand All @@ -1536,7 +1545,7 @@ public function run(array $formerState): array
if (empty($promo_code_val)) throw new ValidationException("Promo code is required.");

$type_id = $ticket_dto['type_id'];
$order = $this->lock_service->lock('ticket_type.' . $type_id . 'promo_code.' . $promo_code_val . '.sell.lock',
$order = $this->lock_service->lock('ticket_type.' . $type_id . '.promo_code.' . $promo_code_val . '.sell.lock',

@smarcet smarcet Jul 13, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@romanetar The key fix is correct, but note it splits the lock namespace between fleets during a rolling deploy: old pods lock ticket_type.7promo_code.X.sell.lock while new pods lock ticket_type.7.promo_code.X.sell.lock, so the two fleets don't mutually exclude on this path for the duration of the rollout. Compounding it, old pods still run the unconditional-DEL-on-failed-acquire behavior this PR fixes, so they can delete new pods' token locks on any key. Since this call site is the one place where the Redis lock is the sole concurrency guard (no DB row lock — see my other comment), the deploy window is exactly where a race could materialize.

No code change needed — suggest a line in the release notes: deploy while prepaid-assignment traffic is idle, or drain old workers before starting the new fleet.

@romanetar romanetar Aug 3, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good flag, and confirmed against main's LockManagerService — releaseLock() there is an unconditional delete($name) and lock()'s finally calls it even after acquireLock throws UnacquiredLockException, so an old pod that loses contention on any key will delete it out from under whoever holds it, no ownership check. Combined with the key-namespace split you describe (missing dot on old pods vs. present on new), the two fleets neither mutually exclude on this path nor leave each other's locks alone during the rollout — exactly the window you're calling out.

No code change from this thread, but worth noting: the DB pessimistic lock added for your other comment on this same call site (getByValueExclusiveLock on the promo code, PESSIMISTIC_WRITE + HINT_REFRESH, held through commit) closes the concrete failure mode here — old and new pods share the same MySQL row lock regardless of what happens to the Redis key naming or unconditional deletes on either side. So the double-assignment scenario this thread worried about is now covered even mid-rollout; Redis staying inconsistent across fleets during the deploy window would at most cause spurious contention/retries, not an actual double-booked prepaid ticket.

Will add the release-note line about draining old workers / deploying during idle prepaid-assignment traffic anyway, since the underlying old-pod unconditional-release bug can still stomp on other locks in the system during the transition, not just this one.

function () use ($promo_code_val, $type_id) {

$attendee_email = $this->owner->getEmail();
Expand All @@ -1555,7 +1564,7 @@ function () use ($promo_code_val, $type_id) {
if (empty($attendee_last_name))
$attendee_last_name = $this->payload['owner_last_name'] ?? $this->owner->getLastName();

$promo_code = $this->summit->getPromoCodeByCode($promo_code_val);
$promo_code = $this->promo_code_repository->getByValueExclusiveLock($this->summit, $promo_code_val);
if (!PromoCodesUtils::isPrePaidPromoCode($promo_code))
throw new EntityNotFoundException("Promo code is not found.");

Expand Down Expand Up @@ -1658,7 +1667,7 @@ function () use ($promo_code_val, $type_id) {


return $order;
});
}, 30);

@smarcet smarcet Jul 13, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@romanetar This 30 s lifetime makes TTL expiry a silent loss of mutual exclusion, and this call site is the only one of the four where the Redis lock is the sole guard of the invariant — the fix should be a DB pessimistic lock, matching the sibling tasks.

The other three sites are already backed by row locks — ReserveTicketsTask::run/undo hold getByIdsExclusiveLock/getByIdExclusiveLock on the ticket types and RedeemPromoCodesTask holds getByValueExclusiveLock on the promo code — so the DB serializes competing writers regardless of what Redis does. This callback instead reads the promo code via $this->summit->getPromoCodeByCode() with no lock, and the check-then-act sequence (hasPrePaidTicketsAssignedBy → getNextAvailableTicketPerType → addTicket) relies entirely on this Redis lock. If the callback ever outlives the 30 s TTL, a concurrent request acquires the lock and can hand the same prepaid ticket to two attendees. This failure mode is new to this PR: on main the key effectively never expired (TTL was a unix timestamp, ≈55 years). There is also a second window the Redis lock cannot cover even when it works: it is released when the callback returns, but the enclosing tx_service->transaction commits afterwards, so a competitor can read pre-commit state.

Suggested fix — same pattern as RedeemPromoCodesTask in this file: inject ISummitRegistrationPromoCodeRepository into AutoAssignPrePaidTicketTask and fetch the promo code with getByValueExclusiveLock($this->summit, $promo_code_val) inside the transaction, instead of the unlocked getPromoCodeByCode(). That method takes PESSIMISTIC_WRITE with HINT_REFRESH (DoctrineSummitRegistrationPromoCodeRepository.php:617), so the entity is re-hydrated from committed DB truth after the row lock is granted, and the lock is held through commit — closing both the TTL-expiry hole and the release-before-commit window in one move. The Redis lock can stay as a fail-fast layer, consistent with the sibling sites.

Independently, releaseLock() discarding the deleteIfValueMatches() === false signal still deserves the log + metric suggested in the CodeRabbit thread on LockManagerService.php:79 — a mismatch at release is the only runtime evidence that a lock expired mid-callback anywhere in the system.

@romanetar romanetar Aug 3, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verified both points against the code and applied fixes for both (will push shortly).

DB pessimistic lock for AutoAssignPrePaidTicketTask: injected ISummitRegistrationPromoCodeRepository into the task (wired from SagaFactory::$promo_code_repository, which was already available there) and replaced the unlocked $this->summit->getPromoCodeByCode($promo_code_val) with $this->promo_code_repository->getByValueExclusiveLock($this->summit, $promo_code_val) — same pattern as ApplyPromoCodeTask. That row lock (PESSIMISTIC_WRITE + HINT_REFRESH) now covers the whole check-then-act sequence (hasPrePaidTicketsAssignedBy → getNextAvailableTicketPerType → addTicket) and is held through the enclosing transaction's commit, closing both the TTL-expiry hole and the release-before-commit window. The Redis lock stays in place as the fail-fast layer.

releaseLock() observability: it now checks the deleteIfValueMatches() result and, on a mismatch, logs a warning (lock name + token) and increments a lock_manager.release_mismatch counter via the existing ICacheService::incCounter() — no new metrics infra introduced. Added a regression test asserting the counter fires when release fails.

Ran tests/SummitOrderServiceTest.php against the real DB/Redis containers — no regressions (the 3 prepaid-ticket tests are pre-existing markTestSkipped('broken test.'), unrelated to this change, left as-is/out of scope for this PR).

return ['order' => $order];
});
}
Expand Down
13 changes: 7 additions & 6 deletions app/Services/Utils/ILockManagerService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,21 @@ interface ILockManagerService
* @param string $name
* @param int $lifetime
* @throws UnacquiredLockException
* @return mixed
* @return string ownership token — must be passed to releaseLock
*/
public function acquireLock(string $name,int $lifetime = self::DefaultLifetime);
public function acquireLock(string $name, int $lifetime = self::DefaultLifetime): string;

/**
* @param string $name
* @return mixed
* @param string $name
* @param string $token ownership token returned by acquireLock
*/
public function releaseLock(string $name);
public function releaseLock(string $name, string $token): void;

/**
* @param string $name
* @param Closure $callback
* @param int $lifetime
* @return mixed
*/
public function lock(string $name, Closure $callback, int $lifetime = self::DefaultLifetime);
public function lock(string $name, Closure $callback, int $lifetime = self::DefaultLifetime): mixed;
}
68 changes: 38 additions & 30 deletions app/Services/Utils/LockManagerService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@
*/
final class LockManagerService implements ILockManagerService {

const MaxRetries = 3;
const MaxRetries = 3;
const BackOffMultiplier = 2.0;
const BackOffBaseInterval = 100000; // 1 ms
const BackOffBaseInterval = 100000; // microseconds

/**
* @var ICacheService
*/
Expand All @@ -41,73 +42,80 @@ public function __construct(ICacheService $cache_service){
/**
* @param string $name
* @param int $lifetime
* @return LockManagerService
* @return string ownership token — pass to releaseLock
* @throws UnacquiredLockException
*/
public function acquireLock(string $name, int $lifetime = 3600):LockManagerService
public function acquireLock(string $name, int $lifetime = 3600): string
{
Log::debug(sprintf("LockManagerService::acquireLock name %s lifetime %s",$name, $lifetime));
$attempt = 0 ;
Log::debug(sprintf("LockManagerService::acquireLock name %s lifetime %s", $name, $lifetime));
if ($lifetime <= 0) {
throw new \InvalidArgumentException("Lock lifetime must be greater than zero seconds.");
}
$token = bin2hex(random_bytes(16));
$attempt = 0;
do {
$time = time() + $lifetime + 1;
$success = $this->cache_service->addSingleValue($name, $time, $time);
if($success) return $this;
$wait_interval = self::BackOffBaseInterval * ( self::BackOffMultiplier ^ $attempt );
Log::debug(sprintf("LockManagerService::acquireLock name %s retrying in %s microseconds (%s).", $name, $wait_interval, $attempt));
$success = $this->cache_service->addSingleValue($name, $token, $lifetime);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
if ($success) {
return $token;
}
$wait_interval = (int)(self::BackOffBaseInterval * (self::BackOffMultiplier ** $attempt));
Log::debug(sprintf("LockManagerService::acquireLock name %s retrying in %s µs (attempt %s)", $name, $wait_interval, $attempt));
usleep($wait_interval);
if($attempt >= (self::MaxRetries - 1 )) {
// only one time we could use this handle
if ($attempt >= (self::MaxRetries - 1)) {
Log::error(sprintf("LockManagerService::acquireLock name %s lifetime %s ERROR MAX RETRIES attempt %s", $name, $lifetime, $attempt));
throw new UnacquiredLockException(sprintf("lock name %s", $name));
}
++$attempt;
} while(1);
} while (1);
}

/**
* @param string $name
* @return $this
* @param string $token ownership token returned by acquireLock
*/
public function releaseLock(string $name):LockManagerService
public function releaseLock(string $name, string $token): void
{
Log::debug(sprintf("LockManagerService::releaseLock name %s",$name));
$this->cache_service->delete($name);
return $this;
Log::debug(sprintf("LockManagerService::releaseLock name %s", $name));
$released = $this->cache_service->deleteIfValueMatches($name, $token);
if (!$released) {
Log::warning(sprintf("LockManagerService::releaseLock name %s token %s lock was not held by this token at release time (expired or stolen).", $name, $token));
$this->cache_service->incCounter('lock_manager.release_mismatch');
}
}

/**
* @param string $name
* @param Closure $callback
* @param int $lifetime
* @return null
* @return mixed
* @throws UnacquiredLockException
* @throws Exception
*/
public function lock(string $name, Closure $callback, int $lifetime = 3600)
public function lock(string $name, Closure $callback, int $lifetime = 3600): mixed
{
$token = null;
$result = null;
Log::debug(sprintf("LockManagerService::lock name %s lifetime %s", $name, $lifetime));

try
{
$this->acquireLock($name, $lifetime);
try {
$token = $this->acquireLock($name, $lifetime);
Log::debug(sprintf("LockManagerService::lock name %s calling callback", $name));
$result = $callback($this);
}
catch(UnacquiredLockException $ex)
{
catch(UnacquiredLockException $ex) {
Log::warning($ex);
throw $ex;
}
catch(Exception $ex)
{
catch(Exception $ex) {
Log::error($ex);
throw $ex;
}
finally {
$this->releaseLock($name);
if ($token !== null) {
$this->releaseLock($name, $token);
}
}
return $result;
}

}
}
32 changes: 23 additions & 9 deletions app/Services/Utils/RedisCacheService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,9 +239,10 @@ public function storeHash($name, array $values, $ttl = 0)
public function incCounter($counter_name, $ttl = 0)
{
return $this->retryOnConnectionError(function ($conn) use ($counter_name, $ttl) {
if ($conn->setnx($counter_name, 1)) {
if ($ttl > 0) $conn->expire($counter_name, (int)$ttl);
return 1;
if ($ttl > 0) {
if ($conn->set($counter_name, 1, 'EX', (int)$ttl, 'NX') !== null) return 1;
} else {
if ($conn->set($counter_name, 1, 'NX') !== null) return 1;
}
return (int)$conn->incr($counter_name);
}, 0);
Expand Down Expand Up @@ -306,12 +307,11 @@ public function setSingleValue($key, $value, $ttl = 0)
public function addSingleValue($key, $value, $ttl = 0)
{
return $this->retryOnConnectionError(function ($conn) use ($key, $value, $ttl) {
$res = $conn->setnx($key, $value);
if ($res && $ttl > 0) {
$conn->expire($key, $ttl);
if ($ttl > 0) {
return $conn->set($key, $value, 'EX', (int)$ttl, 'NX') !== null;
}
return $res;
});
return $conn->set($key, $value, 'NX') !== null;
}, false);
}

Comment thread
romanetar marked this conversation as resolved.
public function setKeyExpiration($key, $ttl)
Expand All @@ -331,7 +331,21 @@ public function ttl($key)
return (int)$conn->ttl($key);
}, 0);
}


public function deleteIfValueMatches(string $key, string $expectedValue): bool
{
$lua = <<<'LUA'
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('del', KEYS[1])
else
return 0
end
LUA;
return $this->retryOnConnectionError(function ($conn) use ($lua, $key, $expectedValue) {
return (int)$conn->eval($lua, 1, $key, $expectedValue) === 1;
}, false);
}

/**
* @param string $cache_region_key
* @return void
Expand Down
Loading
Loading