-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcoverage.html
More file actions
10455 lines (8955 loc) · 466 KB
/
coverage.html
File metadata and controls
10455 lines (8955 loc) · 466 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>app: Go Coverage Report</title>
<style>
body {
background: black;
color: rgb(80, 80, 80);
}
body, pre, #legend span {
font-family: Menlo, monospace;
font-weight: bold;
}
#topbar {
background: black;
position: fixed;
top: 0; left: 0; right: 0;
height: 42px;
border-bottom: 1px solid rgb(80, 80, 80);
}
#content {
margin-top: 50px;
}
#nav, #legend {
float: left;
margin-left: 10px;
}
#legend {
margin-top: 12px;
}
#nav {
margin-top: 10px;
}
#legend span {
margin: 0 5px;
}
.cov0 { color: rgb(192, 0, 0) }
.cov1 { color: rgb(128, 128, 128) }
.cov2 { color: rgb(116, 140, 131) }
.cov3 { color: rgb(104, 152, 134) }
.cov4 { color: rgb(92, 164, 137) }
.cov5 { color: rgb(80, 176, 140) }
.cov6 { color: rgb(68, 188, 143) }
.cov7 { color: rgb(56, 200, 146) }
.cov8 { color: rgb(44, 212, 149) }
.cov9 { color: rgb(32, 224, 152) }
.cov10 { color: rgb(20, 236, 155) }
</style>
</head>
<body>
<div id="topbar">
<div id="nav">
<select id="files">
<option value="file0">github.com/Harvey-AU/blue-banded-bee/cmd/app/main.go (12.3%)</option>
<option value="file1">github.com/Harvey-AU/blue-banded-bee/cmd/test_jobs/main.go (0.0%)</option>
<option value="file2">github.com/Harvey-AU/blue-banded-bee/internal/api/admin.go (40.0%)</option>
<option value="file3">github.com/Harvey-AU/blue-banded-bee/internal/api/auth.go (41.4%)</option>
<option value="file4">github.com/Harvey-AU/blue-banded-bee/internal/api/errors.go (100.0%)</option>
<option value="file5">github.com/Harvey-AU/blue-banded-bee/internal/api/handlers.go (86.0%)</option>
<option value="file6">github.com/Harvey-AU/blue-banded-bee/internal/api/jobs.go (82.2%)</option>
<option value="file7">github.com/Harvey-AU/blue-banded-bee/internal/api/middleware.go (100.0%)</option>
<option value="file8">github.com/Harvey-AU/blue-banded-bee/internal/api/response.go (94.1%)</option>
<option value="file9">github.com/Harvey-AU/blue-banded-bee/internal/api/test_mocks.go (58.9%)</option>
<option value="file10">github.com/Harvey-AU/blue-banded-bee/internal/auth/config.go (100.0%)</option>
<option value="file11">github.com/Harvey-AU/blue-banded-bee/internal/auth/middleware.go (48.1%)</option>
<option value="file12">github.com/Harvey-AU/blue-banded-bee/internal/cache/cache.go (100.0%)</option>
<option value="file13">github.com/Harvey-AU/blue-banded-bee/internal/crawler/config.go (100.0%)</option>
<option value="file14">github.com/Harvey-AU/blue-banded-bee/internal/crawler/crawler.go (51.9%)</option>
<option value="file15">github.com/Harvey-AU/blue-banded-bee/internal/crawler/robots.go (82.9%)</option>
<option value="file16">github.com/Harvey-AU/blue-banded-bee/internal/crawler/sitemap.go (80.3%)</option>
<option value="file17">github.com/Harvey-AU/blue-banded-bee/internal/db/dashboard.go (20.2%)</option>
<option value="file18">github.com/Harvey-AU/blue-banded-bee/internal/db/db.go (56.8%)</option>
<option value="file19">github.com/Harvey-AU/blue-banded-bee/internal/db/dsn_helper.go (100.0%)</option>
<option value="file20">github.com/Harvey-AU/blue-banded-bee/internal/db/health.go (0.0%)</option>
<option value="file21">github.com/Harvey-AU/blue-banded-bee/internal/db/pages.go (0.0%)</option>
<option value="file22">github.com/Harvey-AU/blue-banded-bee/internal/db/queue.go (89.5%)</option>
<option value="file23">github.com/Harvey-AU/blue-banded-bee/internal/db/users.go (0.0%)</option>
<option value="file24">github.com/Harvey-AU/blue-banded-bee/internal/jobs/manager.go (48.8%)</option>
<option value="file25">github.com/Harvey-AU/blue-banded-bee/internal/jobs/test_helpers.go (31.0%)</option>
<option value="file26">github.com/Harvey-AU/blue-banded-bee/internal/jobs/worker.go (26.9%)</option>
<option value="file27">github.com/Harvey-AU/blue-banded-bee/internal/mocks/auth_client.go (0.0%)</option>
<option value="file28">github.com/Harvey-AU/blue-banded-bee/internal/mocks/crawler.go (16.7%)</option>
<option value="file29">github.com/Harvey-AU/blue-banded-bee/internal/mocks/db.go (0.0%)</option>
<option value="file30">github.com/Harvey-AU/blue-banded-bee/internal/mocks/db_queue.go (16.7%)</option>
<option value="file31">github.com/Harvey-AU/blue-banded-bee/internal/mocks/http_client.go (0.0%)</option>
<option value="file32">github.com/Harvey-AU/blue-banded-bee/internal/mocks/worker_pool.go (0.0%)</option>
<option value="file33">github.com/Harvey-AU/blue-banded-bee/internal/testutil/testutil.go (0.0%)</option>
<option value="file34">github.com/Harvey-AU/blue-banded-bee/internal/util/url.go (86.5%)</option>
</select>
</div>
<div id="legend">
<span>not tracked</span>
<span class="cov0">not covered</span>
<span class="cov8">covered</span>
</div>
</div>
<div id="content">
<pre class="file" id="file0" style="display: none">package main
import (
"context"
"net"
"net/http"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"runtime/trace"
"github.com/Harvey-AU/blue-banded-bee/internal/api"
"github.com/Harvey-AU/blue-banded-bee/internal/crawler"
"github.com/Harvey-AU/blue-banded-bee/internal/db"
"github.com/Harvey-AU/blue-banded-bee/internal/jobs"
"github.com/getsentry/sentry-go"
"github.com/joho/godotenv"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"golang.org/x/time/rate"
)
// Config holds the application configuration loaded from environment variables
type Config struct {
Port string // HTTP port to listen on
Env string // Environment (development/production)
SentryDSN string // Sentry DSN for error tracking
LogLevel string // Log level (debug, info, warn, error)
FlightRecorderEnabled bool // Flight recorder for performance debugging
}
func main() <span class="cov0" title="0">{
// Load .env file if it exists
godotenv.Load()
// Load configuration
config := &Config{
Port: getEnvWithDefault("PORT", "8080"),
Env: getEnvWithDefault("APP_ENV", "development"),
SentryDSN: os.Getenv("SENTRY_DSN"),
LogLevel: getEnvWithDefault("LOG_LEVEL", "info"),
FlightRecorderEnabled: getEnvWithDefault("FLIGHT_RECORDER_ENABLED", "false") == "true",
}
// Start flight recorder if enabled
if config.FlightRecorderEnabled </span><span class="cov0" title="0">{
f, err := os.Create("trace.out")
if err != nil </span><span class="cov0" title="0">{
log.Fatal().Err(err).Msg("failed to create trace file")
}</span>
<span class="cov0" title="0">if err := trace.Start(f); err != nil </span><span class="cov0" title="0">{
log.Fatal().Err(err).Msg("failed to start flight recorder")
}</span>
<span class="cov0" title="0">log.Info().Msg("Flight recorder enabled, writing to trace.out")
// Defer closing the trace and the file to the shutdown sequence
defer func() </span><span class="cov0" title="0">{
trace.Stop()
f.Close()
log.Info().Msg("Flight recorder stopped and trace file closed.")
}</span>()
}
<span class="cov0" title="0">setupLogging(config)
// Initialise Sentry for error tracking and performance monitoring
if config.SentryDSN != "" </span><span class="cov0" title="0">{
err := sentry.Init(sentry.ClientOptions{
Dsn: config.SentryDSN,
Environment: config.Env,
TracesSampleRate: func() float64 </span><span class="cov0" title="0">{
if config.Env == "production" </span><span class="cov0" title="0">{
return 0.1 // 10% sampling in production
}</span>
<span class="cov0" title="0">return 1.0</span> // 100% sampling in development
}(),
AttachStacktrace: true,
Debug: config.Env == "development",
})
<span class="cov0" title="0">if err != nil </span><span class="cov0" title="0">{
log.Warn().Err(err).Msg("Failed to initialise Sentry")
}</span> else<span class="cov0" title="0"> {
log.Info().Str("environment", config.Env).Msg("Sentry initialised successfully")
// Ensure Sentry flushes before application exits
defer sentry.Flush(2 * time.Second)
}</span>
} else<span class="cov0" title="0"> {
log.Warn().Msg("Sentry DSN not configured, error tracking disabled")
}</span>
// Connect to PostgreSQL
<span class="cov0" title="0">pgDB, err := db.InitFromEnv()
if err != nil </span><span class="cov0" title="0">{
sentry.CaptureException(err)
log.Fatal().Err(err).Msg("Failed to connect to PostgreSQL database")
}</span>
<span class="cov0" title="0">defer pgDB.Close()
log.Info().Msg("Connected to PostgreSQL database")
// Initialise crawler
crawlerConfig := crawler.DefaultConfig()
cr := crawler.New(crawlerConfig) // QUESTION: Should we change cr to crawler for clarity, as others have clearer names.
// Create database queue for operations
dbQueue := db.NewDbQueue(pgDB)
// Create a worker pool for task processing
var jobWorkers int = 5 // QUESTION: Set in env or dynamically - consider impact throughout app where worker pool sizing is set.
workerPool := jobs.NewWorkerPool(pgDB.GetDB(), dbQueue, cr, jobWorkers, pgDB.GetConfig())
// Create job manager
jobsManager := jobs.NewJobManager(pgDB.GetDB(), dbQueue, cr, workerPool)
// Set the job manager in the worker pool for duplicate checking
workerPool.SetJobManager(jobsManager)
// Start the worker pool
workerPool.Start(context.Background())
defer workerPool.Stop()
// Start a goroutine to monitor job completion
go func() </span><span class="cov0" title="0">{
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
// Use for-range instead of for-select for better readability
for range ticker.C </span><span class="cov0" title="0">{
// Check for jobs that should be marked complete
rows, err := pgDB.GetDB().Query(`
UPDATE jobs
SET status = 'completed', completed_at = NOW()
WHERE (completed_tasks + failed_tasks) >= (total_tasks - COALESCE(skipped_tasks, 0))
AND status = 'running'
RETURNING id
`)
if err != nil </span><span class="cov0" title="0">{
sentry.CaptureException(err)
log.Error().Err(err).Msg("Failed to update completed jobs")
continue</span>
}
// Log completed jobs
<span class="cov0" title="0">for rows.Next() </span><span class="cov0" title="0">{
var jobID string
if err := rows.Scan(&jobID); err == nil </span><span class="cov0" title="0">{
log.Info().Str("job_id", jobID).Msg("Job marked as completed")
}</span>
}
<span class="cov0" title="0">rows.Close()</span>
}
}()
// Create a rate limiter
<span class="cov0" title="0">limiter := newRateLimiter()
// Create API handler with dependencies
apiHandler := api.NewHandler(pgDB, jobsManager)
// Create HTTP multiplexer
mux := http.NewServeMux()
// Setup API routes
apiHandler.SetupRoutes(mux)
// Create middleware stack
var handler http.Handler = mux
// Add rate limiting
handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) </span><span class="cov0" title="0">{
ip := getClientIP(r)
if !limiter.getLimiter(ip).Allow() </span><span class="cov0" title="0">{
api.WriteErrorMessage(w, r, "Too many requests", http.StatusTooManyRequests, api.ErrCodeRateLimit)
return
}</span>
<span class="cov0" title="0">mux.ServeHTTP(w, r)</span>
})
// Add middleware in reverse order (outermost first)
<span class="cov0" title="0">handler = api.LoggingMiddleware(handler)
handler = api.RequestIDMiddleware(handler)
handler = api.CrossOriginProtectionMiddleware(handler)
handler = api.CORSMiddleware(handler)
// Create a new HTTP server
server := &http.Server{
Addr: ":" + config.Port,
Handler: handler,
}
// Channel to listen for termination signals
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
// Channel to signal when the server has shut down
done := make(chan struct{})
go func() </span><span class="cov0" title="0">{
<-stop
log.Info().Msg("Shutting down server...")
// Create shutdown context with timeout
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Stop accepting new requests
if err := server.Shutdown(ctx); err != nil </span><span class="cov0" title="0">{
sentry.CaptureException(err)
log.Error().Err(err).Msg("Server forced to shutdown")
}</span>
<span class="cov0" title="0">close(done)</span>
}()
// Start the server
<span class="cov0" title="0">log.Info().Str("port", config.Port).Msg("Starting server")
if err := server.ListenAndServe(); err != http.ErrServerClosed </span><span class="cov0" title="0">{
sentry.CaptureException(err)
log.Fatal().Err(err).Msg("Server error")
}</span>
<span class="cov0" title="0"><-done // Wait for the shutdown process to complete
log.Info().Msg("Server stopped")</span>
}
// getEnvWithDefault retrieves an environment variable or returns a default value if not set
func getEnvWithDefault(key, defaultValue string) string <span class="cov0" title="0">{
value := os.Getenv(key)
if value == "" </span><span class="cov0" title="0">{
return defaultValue
}</span>
<span class="cov0" title="0">return value</span>
}
// setupLogging configures the logging system
func setupLogging(config *Config) <span class="cov0" title="0">{
// Configure log level
level, err := zerolog.ParseLevel(config.LogLevel)
if err != nil </span><span class="cov0" title="0">{
level = zerolog.WarnLevel
}</span>
<span class="cov0" title="0">zerolog.SetGlobalLevel(level)
// Use console writer in development
if config.Env == "development" </span><span class="cov0" title="0">{
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339})
}</span> else<span class="cov0" title="0"> {
// In production, use a more verbose JSON format that works well with Fly.io logs
log.Logger = zerolog.New(os.Stdout).
With().
Timestamp().
Str("service", "blue-banded-bee").
Logger()
}</span>
}
// RateLimiter represents a rate limiting system based on client IP addresses
type RateLimiter struct {
limits map[string]*IPRateLimiter
mu sync.Mutex
rate rate.Limit
capacity int
}
// IPRateLimiter wraps a token bucket rate limiter specific to an IP address
type IPRateLimiter struct {
limiter *rate.Limiter
}
// newRateLimiter creates a new rate limiter with default settings
func newRateLimiter() *RateLimiter <span class="cov8" title="1">{
return &RateLimiter{
limits: make(map[string]*IPRateLimiter),
rate: rate.Limit(5), // 5 requests per second
capacity: 5, // 5 burst capacity
}
}</span>
// getLimiter returns the rate limiter for a specific IP address
func (rl *RateLimiter) getLimiter(ip string) *IPRateLimiter <span class="cov8" title="1">{
rl.mu.Lock()
defer rl.mu.Unlock()
limiter, exists := rl.limits[ip]
if !exists </span><span class="cov8" title="1">{
limiter = &IPRateLimiter{
limiter: rate.NewLimiter(rl.rate, rl.capacity),
}
rl.limits[ip] = limiter
}</span>
<span class="cov8" title="1">return limiter</span>
}
// Allow checks if a request from this IP should be allowed
func (ipl *IPRateLimiter) Allow() bool <span class="cov8" title="1">{
return ipl.limiter.Allow()
}</span>
// getClientIP extracts the client's IP address from a request
func getClientIP(r *http.Request) string <span class="cov8" title="1">{
// Check for X-Forwarded-For header first (for clients behind proxies)
ip := r.Header.Get("X-Forwarded-For")
if ip != "" </span><span class="cov8" title="1">{
// X-Forwarded-For might contain multiple IPs, take the first one
ips := strings.Split(ip, ",")
ip = strings.TrimSpace(ips[0])
return ip
}</span>
// If no X-Forwarded-For, use RemoteAddr
<span class="cov0" title="0">ip, _, _ = net.SplitHostPort(r.RemoteAddr)
return ip</span>
}
</pre>
<pre class="file" id="file1" style="display: none">package main
import (
"context"
"os"
"strconv"
"time"
"github.com/Harvey-AU/blue-banded-bee/internal/crawler"
"github.com/Harvey-AU/blue-banded-bee/internal/db"
"github.com/Harvey-AU/blue-banded-bee/internal/jobs"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/joho/godotenv"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
/**
* Job Queue Test Utility
*
* This program tests the job queue system by:
* 1. Setting up a database connection
* 2. Initializing the job queue schema
* 3. Creating a worker pool with multiple workers
* 4. Creating and starting a test job
* 5. Monitoring job progress until completion
*
* Usage:
* go run cmd/test_jobs/main.go
*
* The program expects DATABASE_URL environment variable to be set in the .env file.
*/
func main() <span class="cov0" title="0">{
// Set up logging
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339})
zerolog.SetGlobalLevel(zerolog.InfoLevel)
// Load environment variables
if err := godotenv.Load(); err != nil </span><span class="cov0" title="0">{
log.Fatal().Err(err).Msg("Error loading .env file")
}</span>
// Get database details from environment
<span class="cov0" title="0">dbURL := os.Getenv("DATABASE_URL")
if dbURL == "" </span><span class="cov0" title="0">{
log.Fatal().Msg("DATABASE_URL must be set")
}</span>
// Connect to database
<span class="cov0" title="0">log.Info().Msg("Connecting to PostgreSQL database...")
database, err := db.InitFromEnv()
if err != nil </span><span class="cov0" title="0">{
log.Fatal().Err(err).Msg("Failed to connect to database")
}</span>
<span class="cov0" title="0">defer database.Close()
// Set up crawler
crawler := crawler.New(nil)
// Create database queue for operations
dbQueue := db.NewDbQueue(database)
// Create worker pool
var jobWorkers int = 3
dbConfig := &db.Config{
DatabaseURL: dbURL,
}
workerPool := jobs.NewWorkerPool(database.GetDB(), dbQueue, crawler, jobWorkers, dbConfig)
workerPool.Start(context.Background())
defer workerPool.Stop()
log.Info().Msg("Worker pool started with " + strconv.Itoa(jobWorkers) + " workers")
// Create a test job
jobManager := jobs.NewJobManager(database.GetDB(), dbQueue, crawler, workerPool)
// Set up job options
jobOptions := &jobs.JobOptions{
Domain: "example.com",
Concurrency: 2,
FindLinks: true,
MaxPages: 10,
UseSitemap: true,
}
// Submit the job to the queue
job, err := jobManager.CreateJob(context.Background(), jobOptions)
if err != nil </span><span class="cov0" title="0">{
log.Fatal().Err(err).Msg("Failed to create job")
}</span>
<span class="cov0" title="0">log.Info().Str("job_id", job.ID).Msg("Created test job")
// Add the job to the worker pool - it will automatically start processing pending tasks
workerPool.AddJob(job.ID, jobOptions)
log.Info().Str("job_id", job.ID).Msg("Added job to worker pool, monitoring progress...")
// Monitor job progress
for </span><span class="cov0" title="0">{
time.Sleep(1 * time.Second)
job, err := jobManager.GetJobStatus(context.Background(), job.ID)
if err != nil </span><span class="cov0" title="0">{
log.Error().Err(err).Msg("Failed to get job status")
continue</span>
}
<span class="cov0" title="0">log.Info().
Str("status", string(job.Status)).
Float64("progress", job.Progress).
Int("completed", job.CompletedTasks).
Int("failed", job.FailedTasks).
Int("total", job.TotalTasks).
Msg("Job progress")
if job.Status == jobs.JobStatusCompleted || job.Status == jobs.JobStatusFailed </span><span class="cov0" title="0">{
log.Info().Str("final_status", string(job.Status)).Msg("Job finished")
break</span>
}
<span class="cov0" title="0">if job.Status == jobs.JobStatusRunning && job.Progress >= 100.0 </span><span class="cov0" title="0">{
log.Info().Msg("Job complete")
break</span>
}
}
}
</pre>
<pre class="file" id="file2" style="display: none">package api
import (
"net/http"
"os"
"github.com/Harvey-AU/blue-banded-bee/internal/auth"
"github.com/getsentry/sentry-go"
"github.com/rs/zerolog/log"
)
// AdminResetDatabase handles the admin database reset endpoint
// Requires valid JWT with admin role and explicit environment enablement
func (h *Handler) AdminResetDatabase(w http.ResponseWriter, r *http.Request) <span class="cov8" title="1">{
if r.Method != http.MethodPost </span><span class="cov0" title="0">{
MethodNotAllowed(w, r)
return
}</span>
// Check if this is running in development environment
<span class="cov8" title="1">env := os.Getenv("APP_ENV")
if env == "production" </span><span class="cov8" title="1">{
NotFound(w, r, "Not found") // Return 404 in production to hide the endpoint
return
}</span>
// Require explicit enablement
<span class="cov8" title="1">if os.Getenv("ALLOW_DB_RESET") != "true" </span><span class="cov8" title="1">{
Forbidden(w, r, "Database reset not enabled. Set ALLOW_DB_RESET=true to enable")
return
}</span>
// Get user claims from context (set by AuthMiddleware)
<span class="cov8" title="1">claims, ok := auth.GetUserFromContext(r.Context())
if !ok </span><span class="cov8" title="1">{
Unauthorised(w, r, "Authentication required for admin endpoint")
return
}</span>
// Verify system admin role
<span class="cov0" title="0">if !hasSystemAdminRole(claims) </span><span class="cov0" title="0">{
log.Warn().
Str("user_id", claims.UserID).
Str("email", claims.Email).
Msg("Non-system-admin user attempted to access database reset endpoint")
Forbidden(w, r, "System administrator privileges required")
return
}</span>
// Verify user exists in database
<span class="cov0" title="0">user, err := h.DB.GetUser(claims.UserID)
if err != nil </span><span class="cov0" title="0">{
log.Error().Err(err).Str("user_id", claims.UserID).Msg("Failed to verify admin user")
Unauthorised(w, r, "User verification failed")
return
}</span>
// Log the admin action with full context
<span class="cov0" title="0">log.Warn().
Str("user_id", user.ID).
Str("email", user.Email).
Str("org_id", func() string </span><span class="cov0" title="0">{
if user.OrganisationID != nil </span><span class="cov0" title="0">{
return *user.OrganisationID
}</span>
<span class="cov0" title="0">return "none"</span>
}()).
Str("remote_addr", r.RemoteAddr).
Str("user_agent", r.Header.Get("User-Agent")).
Msg("Admin database reset requested")
// Capture in Sentry for audit trail
<span class="cov0" title="0">sentry.WithScope(func(scope *sentry.Scope) </span><span class="cov0" title="0">{
scope.SetTag("event_type", "admin_action")
scope.SetTag("action", "database_reset")
scope.SetUser(sentry.User{
ID: user.ID,
Email: user.Email,
})
scope.SetContext("admin_action", map[string]interface{}{
"endpoint": "/admin/reset-db",
"user_agent": r.Header.Get("User-Agent"),
"ip_address": r.RemoteAddr,
})
sentry.CaptureMessage("Admin database reset action")
}</span>)
// Perform the database reset
<span class="cov0" title="0">if err := h.DB.ResetSchema(); err != nil </span><span class="cov0" title="0">{
log.Error().Err(err).
Str("user_id", user.ID).
Msg("Failed to reset database schema")
InternalError(w, r, err)
return
}</span>
<span class="cov0" title="0">log.Info().
Str("user_id", user.ID).
Str("email", user.Email).
Msg("Database schema reset completed by admin")
WriteSuccess(w, r, nil, "Database schema reset successfully")</span>
}
// hasSystemAdminRole checks if the user has system administrator privileges via app_metadata
// This is distinct from organisation-level admin roles - system admins are Blue Banded Bee operators
// who have elevated privileges for system-level operations like database resets
func hasSystemAdminRole(claims *auth.UserClaims) bool <span class="cov8" title="1">{
if claims == nil || claims.AppMetadata == nil </span><span class="cov8" title="1">{
return false
}</span>
// Check for system_role = "system_admin" in app_metadata
<span class="cov8" title="1">if systemRole, exists := claims.AppMetadata["system_role"]; exists </span><span class="cov8" title="1">{
if roleStr, ok := systemRole.(string); ok && roleStr == "system_admin" </span><span class="cov8" title="1">{
return true
}</span>
}
<span class="cov8" title="1">return false</span>
}
</pre>
<pre class="file" id="file3" style="display: none">package api
import (
"encoding/json"
"net/http"
"strings"
emailverifier "github.com/AfterShip/email-verifier"
"github.com/Harvey-AU/blue-banded-bee/internal/auth"
"github.com/getsentry/sentry-go"
"github.com/rs/zerolog/log"
)
var (
verifier = emailverifier.NewVerifier()
)
// AuthRegisterRequest represents a user registration request
type AuthRegisterRequest struct {
UserID string `json:"user_id"`
Email string `json:"email"`
FullName *string `json:"full_name,omitempty"`
OrgName *string `json:"org_name,omitempty"`
}
// AuthSessionRequest represents a session validation request
type AuthSessionRequest struct {
Token string `json:"token"`
}
// UserResponse represents a user in API responses
type UserResponse struct {
ID string `json:"id"`
Email string `json:"email"`
FullName *string `json:"full_name"`
OrganisationID *string `json:"organisation_id"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
// OrganisationResponse represents an organisation in API responses
type OrganisationResponse struct {
ID string `json:"id"`
Name string `json:"name"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
// AuthRegister handles POST /v1/auth/register
func (h *Handler) AuthRegister(w http.ResponseWriter, r *http.Request) <span class="cov8" title="1">{
if r.Method != http.MethodPost </span><span class="cov8" title="1">{
MethodNotAllowed(w, r)
return
}</span>
<span class="cov8" title="1">var req AuthRegisterRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil </span><span class="cov8" title="1">{
BadRequest(w, r, "Invalid JSON request body")
return
}</span>
<span class="cov8" title="1">if req.UserID == "" || req.Email == "" </span><span class="cov8" title="1">{
BadRequest(w, r, "user_id and email are required")
return
}</span>
<span class="cov0" title="0">var orgName string
// 1. Org name if explicitly provided
if req.OrgName != nil && *req.OrgName != "" </span><span class="cov0" title="0">{
orgName = *req.OrgName
}</span>
// 2. Domain name if not generic (and org name not already set)
<span class="cov0" title="0">if orgName == "" </span><span class="cov0" title="0">{
result, err := verifier.Verify(req.Email)
if err != nil </span><span class="cov0" title="0">{
log.Warn().Err(err).Msg("Email verifier failed")
}</span> else<span class="cov0" title="0"> if !result.Free </span><span class="cov0" title="0">{
// Not a free provider, so use the domain name
if emailParts := strings.Split(req.Email, "@"); len(emailParts) == 2 </span><span class="cov0" title="0">{
domain := emailParts[1]
domainName := strings.Split(domain, ".")[0]
if len(domainName) > 0 </span><span class="cov0" title="0">{
// Capitalise first letter of domain name
orgName = strings.ToUpper(string(domainName[0])) + domainName[1:]
}</span>
}
}
}
// 3. Person's full name as fallback
<span class="cov0" title="0">if orgName == "" && req.FullName != nil && *req.FullName != "" </span><span class="cov0" title="0">{
orgName = *req.FullName
}</span>
// 4. Final default if nothing else worked
<span class="cov0" title="0">if orgName == "" </span><span class="cov0" title="0">{
orgName = "Personal Organisation"
}</span>
// Create user with organisation automatically
<span class="cov0" title="0">user, org, err := h.DB.CreateUser(req.UserID, req.Email, req.FullName, orgName)
if err != nil </span><span class="cov0" title="0">{
sentry.CaptureException(err)
log.Error().Err(err).Str("user_id", req.UserID).Msg("Failed to create user with organisation")
InternalError(w, r, err)
return
}</span>
<span class="cov0" title="0">userResp := UserResponse{
ID: user.ID,
Email: user.Email,
FullName: user.FullName,
OrganisationID: user.OrganisationID,
CreatedAt: user.CreatedAt.Format("2006-01-02T15:04:05Z"),
UpdatedAt: user.UpdatedAt.Format("2006-01-02T15:04:05Z"),
}
orgResp := OrganisationResponse{
ID: org.ID,
Name: org.Name,
CreatedAt: org.CreatedAt.Format("2006-01-02T15:04:05Z"),
UpdatedAt: org.UpdatedAt.Format("2006-01-02T15:04:05Z"),
}
WriteCreated(w, r, map[string]interface{}{
"user": userResp,
"organisation": orgResp,
}, "User registered successfully")</span>
}
// AuthSession handles POST /v1/auth/session
func (h *Handler) AuthSession(w http.ResponseWriter, r *http.Request) <span class="cov8" title="1">{
if r.Method != http.MethodPost </span><span class="cov8" title="1">{
MethodNotAllowed(w, r)
return
}</span>
<span class="cov8" title="1">var req AuthSessionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil </span><span class="cov8" title="1">{
BadRequest(w, r, "Invalid JSON request body")
return
}</span>
<span class="cov8" title="1">if req.Token == "" </span><span class="cov8" title="1">{
BadRequest(w, r, "token is required")
return
}</span>
<span class="cov8" title="1">sessionInfo := auth.ValidateSession(req.Token)
WriteSuccess(w, r, sessionInfo, "Session validated")</span>
}
// AuthProfile handles GET /v1/auth/profile
func (h *Handler) AuthProfile(w http.ResponseWriter, r *http.Request) <span class="cov8" title="1">{
if r.Method != http.MethodGet </span><span class="cov8" title="1">{
MethodNotAllowed(w, r)
return
}</span>
<span class="cov8" title="1">userClaims, ok := auth.GetUserFromContext(r.Context())
if !ok </span><span class="cov8" title="1">{
Unauthorised(w, r, "User information not found")
return
}</span>
// Auto-create user if they don't exist
<span class="cov0" title="0">user, err := h.DB.GetOrCreateUser(userClaims.UserID, userClaims.Email, nil)
if err != nil </span><span class="cov0" title="0">{
sentry.CaptureException(err)
log.Error().Err(err).Str("user_id", userClaims.UserID).Msg("Failed to get or create user")
InternalError(w, r, err)
return
}</span>
<span class="cov0" title="0">userResp := UserResponse{
ID: user.ID,
Email: user.Email,
FullName: user.FullName,
OrganisationID: user.OrganisationID,
CreatedAt: user.CreatedAt.Format("2006-01-02T15:04:05Z"),
UpdatedAt: user.UpdatedAt.Format("2006-01-02T15:04:05Z"),
}
response := map[string]interface{}{
"user": userResp,
}
// Get organisation if user has one
if user.OrganisationID != nil </span><span class="cov0" title="0">{
org, err := h.DB.GetOrganisation(*user.OrganisationID)
if err != nil </span><span class="cov0" title="0">{
log.Warn().Err(err).Str("organisation_id", *user.OrganisationID).Msg("Failed to get organisation")
}</span> else<span class="cov0" title="0"> {
orgResp := OrganisationResponse{
ID: org.ID,
Name: org.Name,
CreatedAt: org.CreatedAt.Format("2006-01-02T15:04:05Z"),
UpdatedAt: org.UpdatedAt.Format("2006-01-02T15:04:05Z"),
}
response["organisation"] = orgResp
}</span>
}
<span class="cov0" title="0">WriteSuccess(w, r, response, "Profile retrieved successfully")</span>
}
</pre>
<pre class="file" id="file4" style="display: none">package api
import (
"encoding/json"
"net/http"
"github.com/rs/zerolog/log"
)
// ErrorResponse represents a standardised error response
type ErrorResponse struct {
Status int `json:"status"`
Message string `json:"message"`
Code string `json:"code,omitempty"`
RequestID string `json:"request_id,omitempty"`
}
// ErrorCode represents standard error codes
type ErrorCode string
const (
// Client errors (4xx)
ErrCodeBadRequest ErrorCode = "BAD_REQUEST"
ErrCodeUnauthorised ErrorCode = "UNAUTHORISED"
ErrCodeForbidden ErrorCode = "FORBIDDEN"
ErrCodeNotFound ErrorCode = "NOT_FOUND"
ErrCodeMethodNotAllowed ErrorCode = "METHOD_NOT_ALLOWED"
ErrCodeConflict ErrorCode = "CONFLICT"
ErrCodeValidation ErrorCode = "VALIDATION_ERROR"
ErrCodeRateLimit ErrorCode = "RATE_LIMIT_EXCEEDED"
// Server errors (5xx)
ErrCodeInternal ErrorCode = "INTERNAL_ERROR"
ErrCodeServiceUnavailable ErrorCode = "SERVICE_UNAVAILABLE"
ErrCodeDatabaseError ErrorCode = "DATABASE_ERROR"
)
// WriteError writes a standardised error response
func WriteError(w http.ResponseWriter, r *http.Request, err error, status int, code ErrorCode) <span class="cov8" title="1">{
requestID := GetRequestID(r)
errResp := ErrorResponse{
Status: status,
Message: err.Error(),
Code: string(code),
RequestID: requestID,
}
// Log the error with context
log.Error().
Err(err).
Str("request_id", requestID).
Str("method", r.Method).
Str("path", r.URL.Path).
Int("status", status).
Str("code", string(code)).
Msg("API error response")
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(errResp)
}</span>
// WriteErrorMessage writes a standardised error response with a custom message
func WriteErrorMessage(w http.ResponseWriter, r *http.Request, message string, status int, code ErrorCode) <span class="cov8" title="1">{
requestID := GetRequestID(r)
errResp := ErrorResponse{
Status: status,
Message: message,
Code: string(code),
RequestID: requestID,
}
// Log the error with context
log.Error().
Str("request_id", requestID).