me/neoq
1
0
Fork 0
mirror of https://github.com/acaloiaro/neoq synced 2026-07-21 18:29:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #24 from acaloiaro/add-with-logger-config

Browse source 
Add user-supplied logger support
This commit is contained in:
Adriano Caloiaro 2023-02-24 11:37:45 -07:00 committed by GitHub
commit 1911da5008
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 141 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
examples/add_future_postgres_job/main.go
View file

@ -13,8 +13,8 @@ func main() {
ctx := context.Background()
nq, err := neoq.New(ctx,
neoq.PgTransactionTimeout(1000),
neoq.BackendName("postgres"),
neoq.ConnectionString("postgres://postgres:postgres@127.0.0.1:5432/neoq"))
neoq.WithBackendName("postgres"),
neoq.WithConnectionString("postgres://postgres:postgres@127.0.0.1:5432/neoq"))
if err != nil {
log.Fatalf("error initializing neoq: %v", err)

4
examples/listen_for_jobs/main.go
View file

@ -12,8 +12,8 @@ func main() {
const queue = "foobar"
ctx := context.Background()
nq, err := neoq.New(ctx,
neoq.BackendName("postgres"),
neoq.ConnectionString("postgres://postgres:postgres@127.0.0.1:5432/neoq"))
neoq.WithBackendName("postgres"),
neoq.WithConnectionString("postgres://postgres:postgres@127.0.0.1:5432/neoq"))
if err != nil {
log.Fatalf("error initializing neoq: %v", err)
}

20
memory_backend.go
View file

@ -169,8 +169,13 @@ func (m *MemBackend) ListenCron(ctx context.Context, cronSpec string, h Handler)
return
}
// SetLogger sets this backend's logger
func (m *MemBackend) SetLogger(logger Logger) {
m.logger = logger
}
// Shutdown halts the worker
func (m *MemBackend) Shutdown(ctx context.Context) (err error) {
func (m *MemBackend) Shutdown(ctx context.Context) {
for _, f := range m.cancelFuncs {
f()
}
@ -180,11 +185,6 @@ func (m *MemBackend) Shutdown(ctx context.Context) (err error) {
return
}
// WithConfig configures neoq with with optional configuration
func (m *MemBackend) WithConfig(opt ConfigOption) (n Neoq) {
return
}
// start starts a queue listener, processes pending job, and fires up goroutines to process future jobs
func (m *MemBackend) start(ctx context.Context, queue string) (err error) {
var queueChan chan Job
@ -219,7 +219,7 @@ func (m *MemBackend) start(ctx context.Context, queue string) (err error) {
}
if err != nil {
m.logger.Error("error handling job", err, "job_id", job.ID)
m.logger.Error("job failed", err, "job_id", job.ID)
runAfter := calculateBackoff(job.Retries)
job.RunAfter = runAfter
m.queueFutureJob(job)
@ -279,9 +279,9 @@ func (m *MemBackend) handleJob(ctx context.Context, job Job, handler Handler) (e
}
// execute the queue handler of this job
handlerErr := execHandler(hctx, handler)
if handlerErr != nil {
job.Error = null.StringFrom(handlerErr.Error())
err = execHandler(hctx, handler)
if err != nil {
job.Error = null.StringFrom(err.Error())
}
return

6
memory_backend_test.go
View file

@ -25,7 +25,7 @@ func TestMemeoryBackendBasicJobProcessing(t *testing.T) {
t.Fatal(err)
}
nq, err := New(ctx, Backend(backend))
nq, err := New(ctx, WithBackend(backend))
if err != nil {
t.Fatal(err)
}
@ -103,7 +103,7 @@ func TestMemeoryBackendConfiguration(t *testing.T) {
t.Fatal(err)
}
nq, err := New(ctx, Backend(backend))
nq, err := New(ctx, WithBackend(backend))
if err != nil {
t.Fatal(err)
}
@ -154,7 +154,7 @@ func TestMemeoryBackendFutureJobScheduling(t *testing.T) {
t.Fatal(err)
}
nq, err := New(ctx, Backend(backend))
nq, err := New(ctx, WithBackend(backend))
if err != nil {
t.Fatal(err)
}

61
neoq.go
View file

@ -58,21 +58,17 @@ type Neoq interface {
// See: https://pkg.go.dev/github.com/robfig/cron?#hdr-CRON_Expression_Format for details on the cron spec format
ListenCron(ctx context.Context, cron string, h Handler) (err error)
// Shutdown halts the worker
Shutdown(ctx context.Context) (err error)
// SetLogger sets the logger for a backend
SetLogger(logger Logger)
// WithConfig configures neoq with with optional configuration
WithConfig(opt ConfigOption) (n Neoq)
// Shutdown halts job processing and releases resources
Shutdown(ctx context.Context)
}
// Logger interface is the interface that neoq's logger must implement
//
// This interface is a subset of [slog.Logger]. The slog interface was chosen under the assumption that its
// likely to be Golang's standard library logging interface.
//
// TODO Add WithLogger for user-supplied logger configuration
// currently loggers are initialized by default as slog, but users
// should be able to supply their own loggers.
type Logger interface {
Debug(msg string, args ...any)
Error(msg string, err error, args ...any)
@ -206,9 +202,9 @@ func JobFromContext(ctx context.Context) (j *Job, err error) {
return
}
// BackendName is a configuration option that instructs neoq to create a new backend with the given name upon
// WithBackendName configures neoq to create a new backend with the given name upon
// initialization
func BackendName(backendName string) ConfigOption {
func WithBackendName(backendName string) ConfigOption {
return func(n Neoq) {
switch c := n.(type) {
case *internalConfig:
@ -218,8 +214,8 @@ func BackendName(backendName string) ConfigOption {
}
}
// JobCheckInterval configures the duration of time between checking for future jobs
func JobCheckInterval(interval time.Duration) ConfigOption {
// WithJobCheckInterval configures the duration of time between checking for future jobs
func WithJobCheckInterval(interval time.Duration) ConfigOption {
return func(n Neoq) {
switch b := n.(type) {
case *internalConfig:
@ -233,9 +229,9 @@ func JobCheckInterval(interval time.Duration) ConfigOption {
}
}
// Backend is a configuration option that instructs neoq to use the specified backend rather than initializing a new one
// WithBackend configures neoq to use the specified backend rather than initializing a new one
// during initialization
func Backend(backend Neoq) ConfigOption {
func WithBackend(backend Neoq) ConfigOption {
return func(n Neoq) {
switch c := n.(type) {
case *internalConfig:
@ -245,8 +241,8 @@ func Backend(backend Neoq) ConfigOption {
}
}
// ConnectionString is a configuration option that sets a connection string for backend initialization
func ConnectionString(connectionString string) ConfigOption {
// WithConnectionString configures neoq to use connection string for backend initialization
func WithConnectionString(connectionString string) ConfigOption {
return func(n Neoq) {
switch c := n.(type) {
case *internalConfig:
@ -258,6 +254,19 @@ func ConnectionString(connectionString string) ConfigOption {
}
}
// WithLogger configures neoq to use the spcified logger
func WithLogger(logger Logger) ConfigOption {
return func(n Neoq) {
switch b := n.(type) {
case *MemBackend:
b.logger = logger
case *PgBackend:
b.logger = logger
default:
}
}
}
// handlerCtxVars are variables passed to every Handler context
type handlerCtxVars struct {
job *Job
@ -312,13 +321,11 @@ func (i internalConfig) ListenCron(ctx context.Context, cron string, h Handler)
return
}
func (i internalConfig) Shutdown(ctx context.Context) (err error) {
func (i internalConfig) SetLogger(logger Logger) {
return
}
func (i internalConfig) WithConfig(opt ConfigOption) (n Neoq) {
return
}
func (i internalConfig) Shutdown(ctx context.Context) {}
// exechandler executes handler functions with a concrete time deadline
func execHandler(ctx context.Context, handler Handler) (err error) {
@ -336,9 +343,19 @@ func execHandler(ctx context.Context, handler Handler) (err error) {
select {
case <-done:
err = <-errCh
return
if err != nil {
err = fmt.Errorf("job failed to process: %w", err)
}
case <-deadlineCtx.Done():
err = fmt.Errorf("job exceeded its %s deadline", handler.deadline)
ctxErr := deadlineCtx.Err()
if errors.Is(ctxErr, context.DeadlineExceeded) {
err = fmt.Errorf("job exceeded its %s deadline: %w", handler.deadline, ctxErr)
} else if errors.Is(ctxErr, context.Canceled) {
err = nil
} else {
err = fmt.Errorf("job failed to process: %w", ctxErr)
}
}
return

80
neoq_test.go
View file

@ -4,14 +4,36 @@ import (
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
)
var errTrigger = errors.New("triggerering a log error")
var errPeriodicTimeout = errors.New("timed out waiting for periodic job")
type testLogger struct {
l *log.Logger
done chan bool
}
func (h testLogger) Info(m string, args ...any) {
h.l.Println(m)
h.done <- true
}
func (h testLogger) Debug(m string, args ...any) {
h.l.Println(m)
h.done <- true
}
func (h testLogger) Error(m string, err error, args ...any) {
h.l.Println(m, err)
h.done <- true
}
func TestWorkerListenConn(t *testing.T) {
const queue = "testing"
timeout := false
complete := false
numJobs := 1
doneCnt := 0
var done = make(chan bool, numJobs)
@ -22,10 +44,11 @@ func TestWorkerListenConn(t *testing.T) {
t.Fatal(err)
}
nq, err := New(ctx, Backend(backend))
nq, err := New(ctx, WithBackend(backend))
if err != nil {
t.Fatal(err)
}
defer nq.Shutdown(ctx)
handler := NewHandler(func(ctx context.Context) (err error) {
done <- true
@ -68,7 +91,6 @@ func TestWorkerListenConn(t *testing.T) {
}
if doneCnt >= numJobs {
complete = true
break
}
@ -77,10 +99,9 @@ func TestWorkerListenConn(t *testing.T) {
}
}
if !complete {
if timeout {
t.Error(err)
}
}
func TestWorkerListenCron(t *testing.T) {
@ -90,8 +111,8 @@ func TestWorkerListenCron(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer nq.Shutdown(ctx)
complete := false
var done = make(chan bool)
handler := NewHandler(func(ctx context.Context) (err error) {
done <- true
@ -113,13 +134,52 @@ func TestWorkerListenCron(t *testing.T) {
time.Sleep(5 * time.Millisecond)
select {
case <-time.After(5 * time.Second):
err = errors.New("timed out waiting for periodic job")
case <-time.After(1 * time.Second):
err = errPeriodicTimeout
case <-done:
complete = true
}
if !complete {
if err != nil {
t.Error(err)
}
}
func TestNeoqAddLogger(t *testing.T) {
const queue = "testing"
var done = make(chan bool)
ctx := context.TODO()
buf := &strings.Builder{}
nq, err := New(ctx, WithLogger(testLogger{l: log.New(buf, "", 0), done: done}))
if err != nil {
t.Fatal(err)
}
defer nq.Shutdown(ctx)
handler := NewHandler(func(ctx context.Context) (err error) {
err = errTrigger
return
})
if err != nil {
t.Error(err)
}
// Listen for jobs on the queue
err = nq.Listen(ctx, queue, handler)
if err != nil {
t.Error(err)
}
_, err = nq.Enqueue(ctx, Job{Queue: queue})
if err != nil {
t.Error(err)
}
<-done
expectedLogMsg := "job failed job failed to process: triggerering a log error"
actualLogMsg := strings.Trim(buf.String(), "\n")
if actualLogMsg != expectedLogMsg {
t.Error(fmt.Errorf("%s != %s", actualLogMsg, expectedLogMsg)) //nolint:all
}
}

27
postgres_backend.go
View file

@ -391,7 +391,12 @@ func (w *PgBackend) ListenCron(ctx context.Context, cronSpec string, h Handler)
return
}
func (w *PgBackend) Shutdown(ctx context.Context) (err error) {
// SetLogger sets this backend's logger
func (w *PgBackend) SetLogger(logger Logger) {
w.logger = logger
}
func (w *PgBackend) Shutdown(ctx context.Context) {
w.pool.Close() // also closes the hijacked listenConn
w.cron.Stop()
@ -400,13 +405,6 @@ func (w *PgBackend) Shutdown(ctx context.Context) (err error) {
}
w.cancelFuncs = nil
return
}
func (w PgBackend) WithConfig(opt ConfigOption) Neoq {
opt(&w)
return &w
}
// enqueueJob adds jobs to the queue, returning the job ID
@ -559,10 +557,11 @@ func (w PgBackend) start(ctx context.Context, queue string) (err error) {
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
err = nil
} else {
w.logger.Error("error handling job", err, "job_id", jobID)
continue
}
w.logger.Error("job failed", err, "job_id", jobID)
continue
}
}
@ -726,13 +725,17 @@ func (w PgBackend) handleJob(ctx context.Context, jobID int64, handler Handler)
}
// execute the queue handler of this job
handlerErr := execHandler(ctx, handler)
err = w.updateJob(ctx, handlerErr)
jobErr := execHandler(ctx, handler)
err = w.updateJob(ctx, jobErr)
if err != nil {
err = fmt.Errorf("error updating job status: %w", err)
return
}
if jobErr != nil {
return jobErr
}
err = tx.Commit(ctx)
if err != nil {
w.logger.Error("unable to commit job transaction. retrying this job may dupliate work", err, "job_id", job.ID)