fix(dispenser): recover stuck card preparation with reset
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -15,6 +16,7 @@ type fakeSequenceDevice struct {
|
||||
statusResponses []cmdResp
|
||||
commandErrors map[cmdType][]error
|
||||
commands []cmdType
|
||||
commandTimes []time.Time
|
||||
}
|
||||
|
||||
func newSequenceTestClient(t *testing.T, statusResponses ...cmdResp) (*Client, *fakeSequenceDevice) {
|
||||
@@ -51,6 +53,7 @@ func newSequenceTestClient(t *testing.T, statusResponses ...cmdResp) (*Client, *
|
||||
return
|
||||
case request := <-client.reqCh:
|
||||
device.commands = append(device.commands, request.typ)
|
||||
device.commandTimes = append(device.commandTimes, now)
|
||||
if request.typ == cmdStatus {
|
||||
if len(device.statusResponses) == 0 {
|
||||
request.respCh <- cmdResp{err: errors.New("unexpected status read")}
|
||||
@@ -90,6 +93,263 @@ func commandCount(commands []cmdType, target cmdType) int {
|
||||
return count
|
||||
}
|
||||
|
||||
func prepareFailureResponses(code byte, count int) []cmdResp {
|
||||
responses := make([]cmdResp, count)
|
||||
for i := range responses {
|
||||
responses[i] = cmdResp{status: []byte{code, 0x30, 0x30, 0x30}}
|
||||
}
|
||||
return responses
|
||||
}
|
||||
|
||||
func TestPreparationResetRecovery(t *testing.T) {
|
||||
for _, operation := range []string{"current", "next"} {
|
||||
for _, code := range []byte{0x32, 0x36} {
|
||||
t.Run(operation+string(rune(code)), func(t *testing.T) {
|
||||
responses := prepareFailureResponses(code, 9) // Initial read, seven polls, post-reset read.
|
||||
responses = append(responses, cmdResp{status: status(0x33)})
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
// An apparently fresh passive cache must never replace physical reads.
|
||||
client.lastStatus = status(0x33)
|
||||
client.lastStatusT = time.Now()
|
||||
client.statusTTL = time.Hour
|
||||
prepare := client.PrepareCurrentCard
|
||||
if operation == "next" {
|
||||
prepare = client.PrepareNextCard
|
||||
}
|
||||
if _, err := prepare(context.Background()); err != nil {
|
||||
t.Fatalf("prepare(%s, %X) = %v, want success", operation, code, err)
|
||||
}
|
||||
want := []cmdType{cmdStatus, cmdToEncoder}
|
||||
for i := 0; i < 7; i++ {
|
||||
want = append(want, cmdStatus)
|
||||
}
|
||||
want = append(want, cmdReset, cmdStatus, cmdToEncoder, cmdStatus)
|
||||
if !reflect.DeepEqual(device.commands, want) {
|
||||
t.Errorf("prepare commands = %v, want %v", device.commands, want)
|
||||
}
|
||||
if len(device.commandTimes) != len(want) {
|
||||
t.Fatalf("command times = %d, want %d", len(device.commandTimes), len(want))
|
||||
}
|
||||
if elapsed := device.commandTimes[9].Sub(device.commandTimes[1]); elapsed != sequenceRetryAfter {
|
||||
t.Errorf("RS elapsed = %s, want %s", elapsed, sequenceRetryAfter)
|
||||
}
|
||||
if elapsed := device.commandTimes[10].Sub(device.commandTimes[9]); elapsed != sequenceResetWait {
|
||||
t.Errorf("post-reset status elapsed = %s, want %s", elapsed, sequenceResetWait)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPreparationFailureAfterReset(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
code byte
|
||||
wantError string
|
||||
}{
|
||||
{"prepare failure times out", 0x32, "timed out"},
|
||||
{"combined failure is final", 0x36, "Command cannot execute"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client, device := newSequenceTestClient(t, prepareFailureResponses(test.code, 30)...)
|
||||
_, err := client.PrepareCurrentCard(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Errorf("PrepareCurrentCard(%X) error = %v, want %q", test.code, err, test.wantError)
|
||||
}
|
||||
if resets, commands := commandCount(device.commands, cmdReset), commandCount(device.commands, cmdToEncoder); resets != 1 || commands != 2 {
|
||||
t.Errorf("recovery commands = %v, want one RS and two FC7", device.commands)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationHardErrorsNeverReset(t *testing.T) {
|
||||
for _, failure := range [][]byte{
|
||||
{0x32, 0x30, 0x32, 0x30}, // Jam alongside prepare failure.
|
||||
{0x36, 0x30, 0x34, 0x30}, // Overlap alongside combined failure.
|
||||
{0x32, 0x30, 0x30, 0x38}, // Empty.
|
||||
{0x36, 0x32, 0x30, 0x30}, // Independent dispense error.
|
||||
{0x32, 0x31, 0x30, 0x30}, // Independent capture error.
|
||||
{0x34, 0x30, 0x30, 0x30}, // Command rejection alone.
|
||||
{0x30, 0x30, 0x30, 0x39}, // Unknown.
|
||||
{0x32}, // Malformed.
|
||||
} {
|
||||
responses := prepareFailureResponses(0x32, 7)
|
||||
responses = append(responses, cmdResp{status: failure})
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
if _, err := client.PrepareCurrentCard(context.Background()); err == nil {
|
||||
t.Errorf("PrepareCurrentCard(% X) error = nil, want hard error", failure)
|
||||
}
|
||||
if got := commandCount(device.commands, cmdReset); got != 0 {
|
||||
t.Errorf("PrepareCurrentCard(% X) resets = %d, want 0", failure, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationProgressAtRecoveryThreshold(t *testing.T) {
|
||||
for _, moving := range [][]byte{{0x31, 0x30, 0x30, 0x30}, {0x32, 0x38, 0x30, 0x30}} {
|
||||
responses := prepareFailureResponses(0x32, 7)
|
||||
responses = append(responses, cmdResp{status: moving}, cmdResp{status: moving}, cmdResp{status: status(0x33)})
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
if _, err := client.PrepareCurrentCard(context.Background()); err != nil {
|
||||
t.Fatalf("PrepareCurrentCard(moving % X) = %v, want success", moving, err)
|
||||
}
|
||||
if commandCount(device.commands, cmdReset) != 0 || commandCount(device.commands, cmdToEncoder) != 1 {
|
||||
t.Errorf("PrepareCurrentCard(moving % X) commands = %v, want initial FC7 only", moving, device.commands)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationRequiresConsecutiveFailures(t *testing.T) {
|
||||
responses := prepareFailureResponses(0x32, 7)
|
||||
responses = append(responses,
|
||||
cmdResp{status: []byte{0x31, 0x30, 0x30, 0x30}},
|
||||
cmdResp{status: []byte{0x32, 0x30, 0x30, 0x30}},
|
||||
cmdResp{status: status(0x33)},
|
||||
)
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
if _, err := client.PrepareCurrentCard(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := commandCount(device.commands, cmdReset); got != 0 {
|
||||
t.Errorf("PrepareCurrentCard(interrupted failure sequence) resets = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationPostResetStatus(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
responses []cmdResp
|
||||
wantFC7 int
|
||||
wantError string
|
||||
}{
|
||||
{"already at encoder", []cmdResp{{status: status(0x33)}}, 1, ""},
|
||||
{"moving then encoder", []cmdResp{{status: []byte{0x31, 0x30, 0x30, 0x30}}, {status: status(0x33)}}, 1, ""},
|
||||
{"moving then stationary", []cmdResp{{status: []byte{0x31, 0x30, 0x30, 0x30}}, {status: status(0x30)}, {status: status(0x33)}}, 2, ""},
|
||||
{"hard error", []cmdResp{{status: []byte{0x32, 0x30, 0x32, 0x30}}}, 1, "Card jammed"},
|
||||
{"read failure", []cmdResp{{err: errors.New("post-reset read failed")}}, 1, "post-reset read failed"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
responses := append(prepareFailureResponses(0x32, 8), test.responses...)
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
_, err := client.PrepareCurrentCard(context.Background())
|
||||
if test.wantError == "" && err != nil {
|
||||
t.Errorf("PrepareCurrentCard(%s) = %v, want success", test.name, err)
|
||||
}
|
||||
if test.wantError != "" && (err == nil || !strings.Contains(err.Error(), test.wantError)) {
|
||||
t.Errorf("PrepareCurrentCard(%s) = %v, want %q", test.name, err, test.wantError)
|
||||
}
|
||||
if commandCount(device.commands, cmdReset) != 1 || commandCount(device.commands, cmdToEncoder) != test.wantFC7 {
|
||||
t.Errorf("PrepareCurrentCard(%s) commands = %v, want one RS and %d FC7", test.name, device.commands, test.wantFC7)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationRecoveryCommandFailures(t *testing.T) {
|
||||
for _, command := range []cmdType{cmdReset, cmdToEncoder} {
|
||||
client, device := newSequenceTestClient(t, prepareFailureResponses(0x32, 9)...)
|
||||
failure := errors.New("command failed")
|
||||
device.commandErrors[command] = []error{failure}
|
||||
wantFC7 := 1
|
||||
if command == cmdToEncoder {
|
||||
device.commandErrors[command] = []error{nil, failure}
|
||||
wantFC7 = 2
|
||||
}
|
||||
_, err := client.PrepareCurrentCard(context.Background())
|
||||
if !errors.Is(err, failure) {
|
||||
t.Errorf("PrepareCurrentCard(failed command %v) = %v, want %v", command, err, failure)
|
||||
}
|
||||
if commandCount(device.commands, cmdReset) != 1 || commandCount(device.commands, cmdToEncoder) != wantFC7 {
|
||||
t.Errorf("PrepareCurrentCard(failed command %v) commands = %v, want one RS and %d FC7", command, device.commands, wantFC7)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationCancellationDuringResetSettle(t *testing.T) {
|
||||
client, device := newSequenceTestClient(t, prepareFailureResponses(0x32, 9)...)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
wait := client.sequenceTiming.wait
|
||||
client.sequenceTiming.wait = func(ctx context.Context, duration time.Duration) error {
|
||||
if duration == sequenceResetWait {
|
||||
cancel()
|
||||
}
|
||||
return wait(ctx, duration)
|
||||
}
|
||||
_, err := client.PrepareCurrentCard(ctx)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("PrepareCurrentCard(cancel during reset) = %v, want context.Canceled", err)
|
||||
}
|
||||
if commandCount(device.commands, cmdReset) != 1 || commandCount(device.commands, cmdToEncoder) != 1 || device.commands[len(device.commands)-1] != cmdReset {
|
||||
t.Errorf("PrepareCurrentCard(cancel during reset) commands = %v, want no commands after RS", device.commands)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetPacket(t *testing.T) {
|
||||
got := createPacket([]byte{0x30, 0x30}, commandRS)
|
||||
want := []byte{0x02, 0x30, 0x30, 0x00, 0x02, 0x52, 0x53, 0x03, 0x02}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("createPacket(RS) = % X, want % X", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationDeadlineDuringResetSettle(t *testing.T) {
|
||||
client, device := newSequenceTestClient(t, prepareFailureResponses(0x32, 4)...)
|
||||
wait := client.sequenceTiming.wait
|
||||
waits := 0
|
||||
client.sequenceTiming.wait = func(ctx context.Context, duration time.Duration) error {
|
||||
waits++
|
||||
if waits == 1 {
|
||||
// Simulate a delayed poll, leaving only one second for recovery.
|
||||
duration = 15 * time.Second
|
||||
} else if duration != time.Second {
|
||||
t.Errorf("reset settle duration = %s, want remaining 1s", duration)
|
||||
}
|
||||
return wait(ctx, duration)
|
||||
}
|
||||
_, err := client.PrepareCurrentCard(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "timed out") {
|
||||
t.Errorf("PrepareCurrentCard(late recovery) = %v, want timeout", err)
|
||||
}
|
||||
want := []cmdType{cmdStatus, cmdToEncoder, cmdStatus, cmdStatus, cmdReset}
|
||||
if !reflect.DeepEqual(device.commands, want) {
|
||||
t.Errorf("PrepareCurrentCard(late recovery) commands = %v, want %v", device.commands, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationPlainRetryConsumesRecoveryBudget(t *testing.T) {
|
||||
responses := make([]cmdResp, 8)
|
||||
for i := range responses {
|
||||
responses[i] = cmdResp{status: status(0x34)}
|
||||
}
|
||||
responses = append(responses, prepareFailureResponses(0x32, 20)...)
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
_, err := client.PrepareCurrentCard(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "timed out") {
|
||||
t.Errorf("PrepareCurrentCard(failure after plain retry) = %v, want timeout", err)
|
||||
}
|
||||
if commandCount(device.commands, cmdReset) != 0 || commandCount(device.commands, cmdToEncoder) != 2 {
|
||||
t.Errorf("PrepareCurrentCard(failure after plain retry) commands = %v, want two FC7 and no RS", device.commands)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationFirstAttemptDoesNotReset(t *testing.T) {
|
||||
for _, failures := range []int{0, 2} {
|
||||
responses := []cmdResp{{status: status(0x34)}}
|
||||
responses = append(responses, prepareFailureResponses(0x32, failures)...)
|
||||
responses = append(responses, cmdResp{status: status(0x33)})
|
||||
client, device := newSequenceTestClient(t, responses...)
|
||||
if _, err := client.PrepareCurrentCard(context.Background()); err != nil {
|
||||
t.Errorf("PrepareCurrentCard(%d transient failures) = %v, want success", failures, err)
|
||||
}
|
||||
if commandCount(device.commands, cmdReset) != 0 || commandCount(device.commands, cmdToEncoder) != 1 {
|
||||
t.Errorf("PrepareCurrentCard(%d transient failures) commands = %v, want one FC7 and no RS", failures, device.commands)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareCurrentCardAcceptsEncoderPositionWithStaleDiagnostics(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -326,7 +586,7 @@ func TestDeliverCurrentCardPreservesContextCancellation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPrepareCurrentCardRetriesOnceThenTimesOut(t *testing.T) {
|
||||
responses := make([]cmdResp, 13)
|
||||
responses := make([]cmdResp, 17)
|
||||
for index := range responses {
|
||||
responses[index] = cmdResp{status: status(0x34)}
|
||||
}
|
||||
@@ -339,8 +599,8 @@ func TestPrepareCurrentCardRetriesOnceThenTimesOut(t *testing.T) {
|
||||
if got := commandCount(device.commands, cmdToEncoder); got != 2 {
|
||||
t.Fatalf("to-encoder commands = %d, want initial command plus one halfway retry", got)
|
||||
}
|
||||
if got := commandCount(device.commands, cmdStatus); got != 13 {
|
||||
t.Fatalf("status reads = %d, want 13", got)
|
||||
if got := commandCount(device.commands, cmdStatus); got != 17 {
|
||||
t.Fatalf("status reads = %d, want 17", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user