From d2d8add667a716826ef964101f3a41ba4a94c27b Mon Sep 17 00:00:00 2001 From: pranajaya161 Date: Fri, 7 Aug 2026 12:21:42 +0700 Subject: [PATCH] first commit --- .claude/SKILL.md | 1373 +++++++++++++++++ .dockerignore | 11 + .gitignore | 7 + .gitlab-ci.yml | 83 + CODEOWNERS | 17 + deployments/docker/omnix-broadcast/Dockerfile | 29 + .../docker/template-service/Dockerfile | 29 + docker-compose.yml | 22 + .../Collection_broadcast_worker_simplified.go | 452 ++++++ ...collection_broadcast_simplified_schema.sql | 319 ++++ docs/implementasi_guide.md | 453 ++++++ docs/quick_start.md | 319 ++++ docs/webhook_integration_request.md | 89 ++ framework/config/loader.go | 75 + framework/db/batch.go | 30 + framework/db/pool.go | 30 + framework/db/tx.go | 26 + framework/go.mod | 61 + framework/go.sum | 146 ++ framework/ingestion/batcher.go | 62 + framework/ingestion/deduplicator.go | 33 + framework/ingestion/retry.go | 41 + framework/logger/structured.go | 20 + framework/metrics/prometheus.go | 60 + framework/middleware/auth.go | 48 + framework/middleware/logging.go | 36 + framework/middleware/tracing.go | 24 + framework/query/analyzer.go | 30 + framework/query/cache.go | 37 + go.work | 7 + go.work.sum | 66 + services/omnix-broadcast/.env.example | 15 + services/omnix-broadcast/README.md | 91 ++ services/omnix-broadcast/client/sopiga.go | 107 ++ services/omnix-broadcast/config.go | 69 + services/omnix-broadcast/domain/broadcast.go | 63 + services/omnix-broadcast/dto/sopiga.go | 54 + services/omnix-broadcast/dto/webhook.go | 9 + services/omnix-broadcast/entity/broadcast.go | 24 + services/omnix-broadcast/go.mod | 20 + services/omnix-broadcast/go.sum | 11 + services/omnix-broadcast/handler/webhook.go | 82 + services/omnix-broadcast/main.go | 83 + .../001_create_collection_broadcasts.down.sql | 17 + .../001_create_collection_broadcasts.up.sql | 258 ++++ .../omnix-broadcast/repository/broadcast.go | 171 ++ services/omnix-broadcast/service/broadcast.go | 205 +++ .../omnix-broadcast/tests/transformer_test.go | 47 + .../omnix-broadcast/tests/validator_test.go | 33 + .../omnix-broadcast/transformer/broadcast.go | 76 + .../omnix-broadcast/validator/broadcast.go | 22 + .../worker/broadcast_worker.go | 85 + .../worker/delivery_sync_worker.go | 83 + services/template-service/.env.example | 11 + services/template-service/config.go | 49 + services/template-service/domain/order.go | 30 + services/template-service/dto/request.go | 7 + services/template-service/dto/response.go | 20 + services/template-service/entity/order.go | 10 + services/template-service/go.mod | 11 + services/template-service/handler/health.go | 28 + services/template-service/handler/ingest.go | 71 + services/template-service/main.go | 76 + .../migrations/001_create_orders.down.sql | 1 + .../migrations/001_create_orders.up.sql | 11 + .../template-service/repository/orders.go | 103 ++ services/template-service/service/orders.go | 83 + services/template-service/tests/fixtures.go | 16 + .../template-service/tests/service_test.go | 26 + .../template-service/transformer/orders.go | 40 + services/template-service/validator/orders.go | 22 + 71 files changed, 6275 insertions(+) create mode 100644 .claude/SKILL.md create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 CODEOWNERS create mode 100644 deployments/docker/omnix-broadcast/Dockerfile create mode 100644 deployments/docker/template-service/Dockerfile create mode 100644 docker-compose.yml create mode 100644 docs/Collection_broadcast_worker_simplified.go create mode 100644 docs/Gadai_collection_broadcast_simplified_schema.sql create mode 100644 docs/implementasi_guide.md create mode 100644 docs/quick_start.md create mode 100644 docs/webhook_integration_request.md create mode 100644 framework/config/loader.go create mode 100644 framework/db/batch.go create mode 100644 framework/db/pool.go create mode 100644 framework/db/tx.go create mode 100644 framework/go.mod create mode 100644 framework/go.sum create mode 100644 framework/ingestion/batcher.go create mode 100644 framework/ingestion/deduplicator.go create mode 100644 framework/ingestion/retry.go create mode 100644 framework/logger/structured.go create mode 100644 framework/metrics/prometheus.go create mode 100644 framework/middleware/auth.go create mode 100644 framework/middleware/logging.go create mode 100644 framework/middleware/tracing.go create mode 100644 framework/query/analyzer.go create mode 100644 framework/query/cache.go create mode 100644 go.work create mode 100644 go.work.sum create mode 100644 services/omnix-broadcast/.env.example create mode 100644 services/omnix-broadcast/README.md create mode 100644 services/omnix-broadcast/client/sopiga.go create mode 100644 services/omnix-broadcast/config.go create mode 100644 services/omnix-broadcast/domain/broadcast.go create mode 100644 services/omnix-broadcast/dto/sopiga.go create mode 100644 services/omnix-broadcast/dto/webhook.go create mode 100644 services/omnix-broadcast/entity/broadcast.go create mode 100644 services/omnix-broadcast/go.mod create mode 100644 services/omnix-broadcast/go.sum create mode 100644 services/omnix-broadcast/handler/webhook.go create mode 100644 services/omnix-broadcast/main.go create mode 100644 services/omnix-broadcast/migrations/001_create_collection_broadcasts.down.sql create mode 100644 services/omnix-broadcast/migrations/001_create_collection_broadcasts.up.sql create mode 100644 services/omnix-broadcast/repository/broadcast.go create mode 100644 services/omnix-broadcast/service/broadcast.go create mode 100644 services/omnix-broadcast/tests/transformer_test.go create mode 100644 services/omnix-broadcast/tests/validator_test.go create mode 100644 services/omnix-broadcast/transformer/broadcast.go create mode 100644 services/omnix-broadcast/validator/broadcast.go create mode 100644 services/omnix-broadcast/worker/broadcast_worker.go create mode 100644 services/omnix-broadcast/worker/delivery_sync_worker.go create mode 100644 services/template-service/.env.example create mode 100644 services/template-service/config.go create mode 100644 services/template-service/domain/order.go create mode 100644 services/template-service/dto/request.go create mode 100644 services/template-service/dto/response.go create mode 100644 services/template-service/entity/order.go create mode 100644 services/template-service/go.mod create mode 100644 services/template-service/handler/health.go create mode 100644 services/template-service/handler/ingest.go create mode 100644 services/template-service/main.go create mode 100644 services/template-service/migrations/001_create_orders.down.sql create mode 100644 services/template-service/migrations/001_create_orders.up.sql create mode 100644 services/template-service/repository/orders.go create mode 100644 services/template-service/service/orders.go create mode 100644 services/template-service/tests/fixtures.go create mode 100644 services/template-service/tests/service_test.go create mode 100644 services/template-service/transformer/orders.go create mode 100644 services/template-service/validator/orders.go diff --git a/.claude/SKILL.md b/.claude/SKILL.md new file mode 100644 index 0000000..0093e7c --- /dev/null +++ b/.claude/SKILL.md @@ -0,0 +1,1373 @@ +# Go Data Warehouse Framework - SKILL.md + +> **Engineering Guide - Go Microservices + PostgreSQL** +> Version: 1.0 +> Stack: Go 1.21+, PostgreSQL, Kafka, Redis, Prometheus, ELK +> Last Updated: 2026-07-24 + +--- + +## 1. Purpose + +Dokumen ini merupakan panduan resmi implementasi **Go Data Warehouse Framework** untuk standardisasi development di 30-person engineering team. + +Tujuan: +- Standardisasi architecture untuk semua data warehouse services +- Consistency di seluruh codebase +- Production-ready patterns dan best practices +- Reference untuk AI Coding Assistant +- Maintain performance dan reliability + +--- + +## 2. Stack Overview + +``` +Data Sources + ↓ +Services (Go + Gin) + ├── Ingestion Workers (batch via Kafka) + ├── Query API + └── Stream Processors + ↓ +PostgreSQL (Data Warehouse) + ↓ +Redis (Query Cache) + ↓ +Prometheus + Grafana (Metrics) +ELK Stack (Logs) +Jaeger (Tracing - optional) + ↓ +Clients +``` + +--- + +## 3. Business Capabilities + +Data Warehouse Platform provides: + +1. **Data Ingestion** - Accept data from multiple sources (batch, streaming) +2. **Data Storage** - PostgreSQL as primary warehouse +3. **Data Query** - API layer untuk access warehouse +4. **Data Quality** - Validation, deduplication, audit trail +5. **Observability** - Metrics, logs, traces + +--- + +## 4. Folder Structure + +### Project Level +``` +go-dw-platform/ +├── framework/ # Shared library (Go module) +│ ├── go.mod +│ ├── ingestion/ +│ │ ├── batcher.go +│ │ ├── deduplicator.go +│ │ └── retry.go +│ ├── query/ +│ │ ├── cache.go +│ │ └── analyzer.go +│ ├── db/ +│ │ ├── pool.go +│ │ ├── batch.go +│ │ └── tx.go +│ ├── metrics/ +│ │ └── prometheus.go +│ ├── logger/ +│ │ └── structured.go +│ ├── middleware/ +│ │ ├── auth.go +│ │ ├── tracing.go +│ │ └── logging.go +│ └── config/ +│ └── loader.go +│ +├── services/ +│ ├── ingestion-{domain}/ # Per-domain ingestion service +│ │ ├── main.go +│ │ ├── handler/ +│ │ ├── service/ +│ │ ├── repository/ +│ │ ├── transformer/ +│ │ ├── domain/ +│ │ ├── entity/ +│ │ ├── dto/ +│ │ ├── validator/ +│ │ ├── migrations/ +│ │ ├── tests/ +│ │ ├── go.mod +│ │ ├── config.go +│ │ └── README.md +│ │ +│ ├── query-api/ # Unified query API +│ │ ├── (same structure) +│ │ +│ └── template-service/ # Boilerplate untuk new service +│ +├── deployments/ +│ ├── docker/ +│ │ ├── Dockerfile.base +│ │ └── Dockerfile.service +│ ├── k8s/ # (if upgrade to K8s later) +│ └── docker-compose.yml +│ +├── docs/ +│ ├── ARCHITECTURE.md +│ ├── DATABASE.md +│ ├── PERFORMANCE.md +│ ├── API.md +│ └── DEPLOYMENT.md +│ +├── scripts/ +│ ├── new-service.sh +│ ├── migrate.sh +│ └── benchmark.sh +│ +├── go.work # Go workspace +├── Makefile +└── README.md +``` + +### Service Level (Ingestion Example) +``` +services/ingestion-orders/ +├── go.mod +├── main.go # Entry point +├── config.go # Configuration +│ +├── handler/ # HTTP + Kafka handlers +│ ├── ingest.go # POST /ingest endpoint +│ └── health.go # Health check +│ +├── service/ # Business logic +│ └── orders.go # Order ingestion logic +│ +├── repository/ # Database access +│ └── orders.go # CRUD + batch operations +│ +├── transformer/ # Data mapping +│ └── orders.go # Raw → Domain → Entity +│ +├── domain/ # Business objects +│ └── order.go # Order domain model +│ +├── entity/ # Database entities +│ └── order.go # DB table mapping +│ +├── dto/ # Request/Response +│ ├── request.go +│ └── response.go +│ +├── validator/ # Input validation +│ └── orders.go +│ +├── migrations/ # Database schema +│ ├── 001_create_orders.up.sql +│ └── 001_create_orders.down.sql +│ +├── tests/ # Unit & integration tests +│ ├── service_test.go +│ ├── repository_test.go +│ ├── handler_test.go +│ └── fixtures.go +│ +└── README.md +``` + +--- + +## 5. Architecture Layers + +``` +HTTP/Kafka Request + ↓ + ┌─────────────────┐ + │ HANDLER │ ← Receive request, auth, validation + └─────────────────┘ + ↓ + ┌─────────────────┐ + │ TRANSFORMER │ ← Map data, convert types + └─────────────────┘ + ↓ + ┌─────────────────┐ + │ SERVICE │ ← Business logic, batching, dedup + └─────────────────┘ + ↓ + ┌─────────────────┐ + │ REPOSITORY │ ← Database operations + └─────────────────┘ + ↓ + PostgreSQL +``` + +All requests MUST follow this flow. **No shortcuts allowed.** + +--- + +## 6. Layer Responsibilities + +### 6.1 Handler (HTTP/Kafka) + +**Responsibilities:** +- Receive request (HTTP POST atau Kafka message) +- Authentication & Authorization +- Request validation (format, required fields) +- Call Service layer +- Return HTTP Response atau handle errors +- Metrics: request count, latency + +**DO ✅** +```go +- Bind request to DTO +- Validate request format +- Extract auth token +- Call service.Process() +- Return standardized response +- Log request_id + action +``` + +**DON'T ❌** +```go +- Business logic +- SQL queries +- Database transactions +- Data transformation logic +- Direct repository access +``` + +**Example:** +```go +func (h *Handler) IngestOrders(c *gin.Context) { + ctx := c.Request.Context() + requestID := c.GetString("request_id") + + var req dto.IngestRequest + if err := c.ShouldBindJSON(&req); err != nil { + h.logger.Error("validation failed", + "request_id", requestID, + "error", err) + c.JSON(400, errorResponse(err)) + return + } + + result, err := h.service.Process(ctx, &req) + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + + c.JSON(200, successResponse(result)) +} +``` + +### 6.2 Transformer + +**Responsibilities:** +- Map data between layers +- Type conversion +- Normalization (trim, lowercase, etc) +- Timestamp handling + +**Transformations:** +``` +Request DTO → Domain Model +Domain Model → Entity (for DB) +Entity → Response DTO +``` + +**DO ✅** +```go +- Map Request → Domain +- Convert types +- Format dates/timestamps +- Validate data format +- Handle null values +``` + +**DON'T ❌** +```go +- Business rules/logic +- Database access +- HTTP operations +``` + +**Example:** +```go +func (t *Transformer) RequestToDomain(req *dto.IngestRequest) (*domain.Order, error) { + order := &domain.Order{ + OrderID: req.OrderID, + CustomerID: req.CustomerID, + Amount: req.Amount, + CreatedAt: time.Now(), + } + + // Validation + if err := order.Validate(); err != nil { + return nil, fmt.Errorf("transform: %w", err) + } + + return order, nil +} + +func (t *Transformer) DomainToEntity(domain *domain.Order) *entity.Order { + return &entity.Order{ + OrderID: domain.OrderID, + CustomerID: domain.CustomerID, + Amount: domain.Amount, + CreatedAt: domain.CreatedAt, + } +} +``` + +### 6.3 Service (Business Logic) + +**Responsibilities:** +- All business rules +- Batching strategy +- Deduplication logic +- Transaction control +- Retry logic +- Metrics calculation + +**DO ✅** +```go +- Implement business rules +- Control transactions +- Call repository methods +- Validate business constraints +- Handle retries +- Log business events +``` + +**DON'T ❌** +```go +- JSON parsing (use Handler) +- HTTP operations +- SQL queries (use Repository) +- Data mapping (use Transformer) +``` + +**Example:** +```go +func (s *Service) Process(ctx context.Context, req *dto.IngestRequest) (*dto.ProcessResult, error) { + // Transform + domain, err := s.transformer.RequestToDomain(req) + if err != nil { + return nil, fmt.Errorf("process: %w", err) + } + + // Business logic: deduplication + exists, err := s.repository.ExistsByKey(ctx, domain.OrderID) + if err != nil { + return nil, fmt.Errorf("check duplicate: %w", err) + } + if exists { + return nil, ErrDuplicate + } + + // Accumulate to batch + s.batcher.Add(domain) + + // Auto-flush on size or timeout + if s.batcher.IsFull() { + return s.FlushBatch(ctx) + } + + return &dto.ProcessResult{Status: "queued"}, nil +} + +func (s *Service) FlushBatch(ctx context.Context) (*dto.ProcessResult, error) { + batch := s.batcher.Get() + + // Transform to entities + entities := make([]*entity.Order, len(batch)) + for i, d := range batch { + entities[i] = s.transformer.DomainToEntity(d) + } + + // Insert with retry + rows, err := s.retrier.Do(ctx, func() (int64, error) { + return s.repository.InsertBatch(ctx, entities) + }) + if err != nil { + return nil, fmt.Errorf("flush batch: %w", err) + } + + // Metrics + s.metrics.RecordBatchInsert(len(batch), time.Since(start)) + + s.batcher.Clear() + return &dto.ProcessResult{ + Status: "success", + RowsInserted: rows, + }, nil +} +``` + +### 6.4 Repository (Data Access) + +**Responsibilities:** +- CRUD operations +- Query execution +- Batch operations +- Connection management +- Transaction handling (initiated by Service) + +**DO ✅** +```go +- CRUD operations +- Execute queries +- Manage batch inserts +- Use prepared statements +- Log slow queries +- Handle connection errors +``` + +**DON'T ❌** +```go +- Business logic +- Validation (beyond schema) +- Response mapping +- HTTP operations +``` + +**Example:** +```go +func (r *Repository) InsertBatch(ctx context.Context, entities []*entity.Order) (int64, error) { + const batchSize = 5000 + + tx, err := r.pool.Begin(ctx) + if err != nil { + return 0, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + stmt := `INSERT INTO orders (order_id, customer_id, amount, created_at) + VALUES ($1, $2, $3, $4)` + + batch := &pgx.Batch{} + for _, e := range entities { + batch.Queue(stmt, e.OrderID, e.CustomerID, e.Amount, e.CreatedAt) + } + + results := tx.SendBatch(ctx, batch) + defer results.Close() + + var rowsInserted int64 + for i := 0; i < len(entities); i++ { + tag, err := results.Exec() + if err != nil { + return 0, fmt.Errorf("exec batch: %w", err) + } + rowsInserted += tag.RowsAffected() + } + + if err := tx.Commit(ctx); err != nil { + return 0, fmt.Errorf("commit: %w", err) + } + + return rowsInserted, nil +} + +func (r *Repository) FindByFilters(ctx context.Context, filters *repository.Filter) ([]*entity.Order, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + query := `SELECT order_id, customer_id, amount, created_at + FROM orders + WHERE created_at >= $1 AND created_at <= $2 + LIMIT $3` + + rows, err := r.pool.Query(ctx, query, filters.StartDate, filters.EndDate, filters.Limit) + if err != nil { + return nil, fmt.Errorf("query: %w", err) + } + defer rows.Close() + + var orders []*entity.Order + for rows.Next() { + var o entity.Order + if err := rows.Scan(&o.OrderID, &o.CustomerID, &o.Amount, &o.CreatedAt); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + orders = append(orders, &o) + } + + return orders, rows.Err() +} +``` + +--- + +## 7. Dependency Rules + +**Allowed:** +``` +Handler + ↓ +Transformer + ↓ +Service + ↓ +Repository + ↓ +PostgreSQL +``` + +**FORBIDDEN (will be caught in code review):** +- Handler → Repository (bypass Service) +- Repository → Service (circular) +- Any layer → Handler (upward dependency) +- Handler → Database (direct SQL) + +--- + +## 8. Data Warehouse Specific Rules + +### 8.1 Batching + +```go +// Service layer manages batching +batcher := ingestion.NewBatcher(config.BatchSize) + +for _, record := range records { + batcher.Add(record) + + if batcher.IsFull() || batcher.IsExpired() { + err := s.flush(ctx) + } +} +``` + +**Configuration (per service in config.go):** +```go +type BatchConfig struct { + Size int // 5000 rows + TimeoutSec int // 30 seconds + MaxRetries int // 3 + RetryDelay time.Duration // exponential backoff +} +``` + +### 8.2 Deduplication + +**Natural Key:** +```go +// In domain/order.go +func (o *Order) NaturalKey() string { + return fmt.Sprintf("%s_%s_%d", o.OrderID, o.CustomerID, o.CreatedAt.Unix()) +} +``` + +**In Service:** +```go +exists, err := s.repository.ExistsByKey(ctx, order.NaturalKey()) +if exists { + // Skip atau return error based on business rule +} +``` + +### 8.3 Kafka Integration + +**Handler untuk Kafka:** +```go +type KafkaHandler struct { + service Service + logger Logger +} + +func (h *KafkaHandler) Handle(ctx context.Context, msg *kafka.Message) error { + var dto dto.IngestRequest + if err := json.Unmarshal(msg.Value, &dto); err != nil { + h.logger.Error("unmarshal failed", "error", err) + return err // will be retried + } + + _, err := h.service.Process(ctx, &dto) + return err +} +``` + +**Configuration (in config.go):** +```go +type KafkaConfig struct { + Brokers []string + Topic string + ConsumerGroup string + MaxConcurrency int +} +``` + +### 8.4 Redis Caching (Query API) + +**In Service:** +```go +func (s *QueryService) Execute(ctx context.Context, q *repository.Query) ([]Result, error) { + // Check cache + cacheKey := q.CacheKey() + if cached, err := s.cache.Get(ctx, cacheKey); err == nil { + return cached, nil + } + + // Query database + results, err := s.repository.Query(ctx, q) + if err != nil { + return nil, err + } + + // Store in cache (TTL: 5-60 minutes) + s.cache.Set(ctx, cacheKey, results, 5*time.Minute) + + return results, nil +} +``` + +**Cache Strategy:** +- Hot queries: 60 minutes +- Moderate queries: 15 minutes +- Cold queries: 5 minutes +- Invalidate on data update + +--- + +## 9. Naming Conventions + +### Package +```go +github.com/yourorg/go-dw-platform/framework +github.com/yourorg/go-dw-platform/services/ingestion-orders +``` + +### Files +- Lowercase, underscore separated +- `orders.go`, `orders_test.go`, `orders_integration_test.go` + +### Types +```go +type OrderService struct{} // PascalCase +type IngestRequest struct{} +type ErrOrderNotFound struct{} +``` + +### Functions/Methods +```go +func (s *Service) Process() error // PascalCase (exported) +func (s *Service) process() error // camelCase (private) +func (s *Service) FindByID() error +func (s *Service) CreateOrder() error +``` + +### Constants +```go +const ( + DefaultBatchSize = 5000 + MaxRetries = 3 +) +``` + +### Errors +```go +var ( + ErrOrderNotFound = errors.New("order not found") + ErrDuplicateOrder = errors.New("duplicate order") + ErrInvalidBatchSize = errors.New("invalid batch size") +) +``` + +**Error wrapping (always):** +```go +if err != nil { + return fmt.Errorf("process order: %w", err) +} +``` + +--- + +## 10. Error Handling Standard + +**Pattern:** +```go +// Always wrap with context +if err != nil { + return fmt.Errorf("insert batch: %w", err) +} + +// Custom errors for business logic +if !order.IsValid() { + return ErrInvalidOrder +} + +// Never panic in production code +// Only panic if application cannot continue +panic("database pool initialization failed") +``` + +**Error Response Format:** +```json +{ + "success": false, + "message": "Failed to process order", + "error_code": "DUPLICATE_ORDER", + "request_id": "req-12345" +} +``` + +--- + +## 11. Logging Standard + +**Structured Logging (JSON):** +```go +logger.Info("order processed", + "request_id", "req-12345", + "user_id", "user-456", + "order_id", "ord-789", + "duration_ms", 125, + "batch_size", 100, +) +``` + +**Minimal Fields (required):** +- `request_id` - Trace requests +- `module` - Service name +- `action` - Operation (insert, query, etc) +- `duration_ms` - Timing +- `error` - Error message (if applicable) + +**Log Levels:** +- `ERROR` - Errors that need attention +- `WARN` - Warnings (retries, slow queries) +- `INFO` - Business events (batch inserted, dedup) +- `DEBUG` - Detailed debugging (not in production) + +**NEVER log:** +- Passwords, tokens, API keys +- PII (passwords, emails in logs) +- Full database records (only IDs) +- Sensitive configuration + +```go +// ❌ Wrong +logger.Info("user login", "password", "secret123") + +// ✅ Correct +logger.Info("user login", "user_id", "user-123") +``` + +--- + +## 12. Validation Standard + +**Layer:** +- **Handler** - Format validation (required fields, JSON format) +- **Transformer** - Type validation (dates, numbers) +- **Service** - Business validation (business rules) + +**Example:** +```go +// Handler - format +if req.OrderID == "" { + return ErrMissingOrderID +} + +// Transformer - type +orderID, err := strconv.ParseInt(req.OrderID, 10, 64) +if err != nil { + return fmt.Errorf("invalid order ID: %w", err) +} + +// Service - business rule +if !s.IsValidCustomer(ctx, order.CustomerID) { + return ErrInvalidCustomer +} +``` + +--- + +## 13. Transaction Standard + +**Rule:** Transactions ONLY in Service layer. + +**Repository accepts context, Service creates tx:** +```go +// ❌ Wrong - Repository creates tx +func (r *Repository) Process(ctx context.Context) error { + tx, _ := r.pool.Begin(ctx) + // ... +} + +// ✅ Correct - Service creates tx +func (s *Service) Process(ctx context.Context) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + // Call repository with tx context + err = s.repository.Insert(ctx, entity) + if err != nil { + return err + } + + return tx.Commit(ctx) +} +``` + +--- + +## 14. Database Rules + +**Connection Pooling (in framework/db/pool.go):** +```go +type Config struct { + MaxConns int32 // 25 (tune per load) + MinConns int32 // 5 + MaxConnLifetime time.Duration // 15 minutes + MaxConnIdleTime time.Duration // 5 minutes +} +``` + +**All SQL in Repository:** +```go +// ❌ Wrong - SQL di Handler +func (h *Handler) GetOrder(c *gin.Context) { + rows, _ := db.Query("SELECT * FROM orders") +} + +// ✅ Correct - SQL di Repository +func (r *Repository) FindByID(ctx context.Context, id string) (*Order, error) { + row := r.pool.QueryRow(ctx, "SELECT * FROM orders WHERE id = $1", id) + // ... +} +``` + +**Prepared Statements (always for batch):** +```go +stmt := `INSERT INTO orders (id, customer_id, amount) VALUES ($1, $2, $3)` +batch := &pgx.Batch{} +for _, order := range orders { + batch.Queue(stmt, order.ID, order.CustomerID, order.Amount) +} +``` + +**Query Timeout:** +```go +ctx, cancel := context.WithTimeout(ctx, 30*time.Second) +defer cancel() + +rows, err := r.pool.Query(ctx, query, args...) +``` + +--- + +## 15. API Response Format + +**Success:** +```json +{ + "success": true, + "message": "Data retrieved successfully", + "data": { + "orders": [...], + "total": 100, + "page": 1 + }, + "request_id": "req-12345" +} +``` + +**Error:** +```json +{ + "success": false, + "message": "Validation failed", + "error_code": "VALIDATION_ERROR", + "details": { + "field": "order_id", + "error": "required" + }, + "request_id": "req-12345" +} +``` + +**Pagination:** +```json +{ + "data": [...], + "pagination": { + "page": 1, + "limit": 10, + "total": 150, + "pages": 15 + } +} +``` + +--- + +## 16. Security Rules + +**MANDATORY:** + +1. **Authentication** - JWT verification +```go +func AuthMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + token := c.GetHeader("Authorization") + if token == "" { + c.JSON(401, errorResponse("missing token")) + c.Abort() + return + } + // Verify token + } +} +``` + +2. **Authorization** - Role-based access +```go +func RoleMiddleware(required []string) gin.HandlerFunc { + return func(c *gin.Context) { + role := c.GetString("user_role") + // Check if role in required + } +} +``` + +3. **Input Validation** - Prevent injection +```go +// Always use parameterized queries +stmt := "SELECT * FROM users WHERE id = $1" // ✅ +stmt := "SELECT * FROM users WHERE id = " + id // ❌ +``` + +4. **Secret Management** - Use Vault +```go +type Config struct { + DBPassword string `vault:"db.password"` // Loaded from Vault +} +``` + +5. **Audit Trail** - Log sensitive operations +```go +logger.Info("data accessed", + "user_id", userID, + "action", "export", + "records", 1000, + "timestamp", time.Now(), +) +``` + +**NEVER:** +- Hardcode credentials +- Log passwords/tokens +- Store plaintext secrets +- Skip validation + +--- + +## 17. Performance Rules + +### Pagination (always) +```go +const DefaultLimit = 100 +const MaxLimit = 10000 + +func (h *Handler) List(c *gin.Context) { + limit := c.DefaultQuery("limit", "100") + page := c.DefaultQuery("page", "1") + + // Validate + if limit > MaxLimit { + limit = MaxLimit + } +} +``` + +### Context & Timeout +```go +// Always use context with timeout +ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +defer cancel() + +rows, err := r.pool.Query(ctx, query) +``` + +### Batch Processing +```go +// For large datasets, use batching +const BatchSize = 5000 + +for i := 0; i < len(records); i += BatchSize { + batch := records[i:min(i+BatchSize, len(records))] + r.InsertBatch(ctx, batch) +} +``` + +### Query Optimization +```go +// ✅ Good - With index +SELECT * FROM orders WHERE customer_id = $1 AND created_at > $2 LIMIT 1000 + +// ❌ Bad - Full table scan +SELECT * FROM orders WHERE amount * quantity > 1000 + +// Index needed on (customer_id, created_at) +``` + +### Avoid N+1 +```go +// ❌ Wrong +for _, order := range orders { + customer := r.GetCustomer(order.CustomerID) // N queries +} + +// ✅ Correct +customerIDs := extract(orders, "CustomerID") +customers := r.GetCustomersBatch(customerIDs) // 1 query +``` + +### Connection Pooling +```go +// Pool configured in framework +pool, _ := pgxpool.New(ctx, dsn) +defer pool.Close() + +// Connection automatically managed +row := pool.QueryRow(ctx, query) +``` + +--- + +## 18. Testing Standard + +**Minimum Coverage: 80%** + +**Layers to Test:** +- ✅ Service (business logic) +- ✅ Repository (database access) +- ✅ Transformer (data mapping) +- ⚠️ Handler (integration test, if needed) + +**Testing Framework: testify** + +### Service Test +```go +func TestOrderService_Process(t *testing.T) { + // Arrange + mockRepo := &MockRepository{} + service := NewService(mockRepo) + + req := &dto.IngestRequest{OrderID: "ord-123"} + + // Act + result, err := service.Process(context.Background(), req) + + // Assert + assert.NoError(t, err) + assert.Equal(t, "success", result.Status) + assert.Equal(t, 1, mockRepo.InsertCallCount) +} +``` + +### Repository Test +```go +func TestOrderRepository_InsertBatch(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + repo := NewRepository(db) + + entities := []*entity.Order{ + {OrderID: "1", CustomerID: "cust-1", Amount: 100}, + {OrderID: "2", CustomerID: "cust-2", Amount: 200}, + } + + rows, err := repo.InsertBatch(context.Background(), entities) + + assert.NoError(t, err) + assert.Equal(t, int64(2), rows) +} +``` + +### Integration Test +```go +func TestOrderHandler_Ingest(t *testing.T) { + router := setupTestRouter() + db := setupTestDB(t) + + payload := `{"order_id": "123", "customer_id": "cust-1", "amount": 100}` + + req, _ := http.NewRequest("POST", "/api/ingest", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + assert.Equal(t, 200, resp.Code) +} +``` + +**Test File Naming:** +``` +orders.go → orders_test.go +orders_handler.go → orders_handler_test.go +``` + +**Run Tests:** +```bash +go test ./... -cover # All tests +go test ./service -cover # Specific package +go test -run TestOrderService -v # Specific test +``` + +--- + +## 19. Observability + +### 19.1 Metrics (Prometheus) + +**Framework provides:** +```go +metrics.RecordHTTPRequest(method, endpoint, statusCode, duration) +metrics.RecordDatabaseQuery(operation, duration) +metrics.RecordBatchInsert(rowCount, duration) +metrics.RecordCacheHit(key, duration) +``` + +**In your code:** +```go +start := time.Now() +rows, err := r.InsertBatch(ctx, entities) +metrics.RecordBatchInsert(len(entities), time.Since(start)) +``` + +**Prometheus Queries:** +``` +rate(http_request_duration_seconds[5m]) +histogram_quantile(0.95, rate(db_query_duration_seconds[5m])) +``` + +### 19.2 Logging (ELK) + +**Framework provides structured logger:** +```go +logger.Info("batch inserted", + "module", "ingestion-orders", + "batch_size", 5000, + "duration_ms", 250, +) +``` + +**Kibana can parse JSON:** +```json +{ + "timestamp": "2026-07-24T10:30:45Z", + "level": "INFO", + "message": "batch inserted", + "module": "ingestion-orders", + "batch_size": 5000, + "duration_ms": 250, + "request_id": "req-12345" +} +``` + +### 19.3 Tracing (Optional - Jaeger) + +**Add trace context:** +```go +ctx, span := tracer.Start(ctx, "ProcessOrder") +defer span.End() + +span.SetAttributes( + attribute.String("order_id", order.ID), + attribute.Int64("customer_id", order.CustomerID), +) +``` + +--- + +## 20. CI/CD & Deployment + +### GitHub Actions +```yaml +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + - run: go test ./... -cover + - run: golangci-lint run +``` + +### Docker Build +```dockerfile +FROM golang:1.21-alpine AS builder +WORKDIR /app +COPY . . +RUN go build -o service . + +FROM alpine:latest +COPY --from=builder /app/service . +ENTRYPOINT ["./service"] +``` + +### Deployment Checklist +- [ ] All tests pass +- [ ] Code review approved +- [ ] Linting passes +- [ ] No vulnerabilities (gosec) +- [ ] Database migrations ready +- [ ] Configuration set +- [ ] Monitoring configured +- [ ] Rollback plan defined + +--- + +## 21. Common Patterns + +### Pattern: Graceful Shutdown +```go +func main() { + server := gin.Default() + + go func() { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + <-sigChan + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + server.Shutdown(ctx) + }() + + server.Run(":8080") +} +``` + +### Pattern: Health Check +```go +func (h *Handler) Health(c *gin.Context) { + health := map[string]interface{}{ + "status": "healthy", + "database": h.db.Ping(c.Request.Context()), + "cache": h.cache.Ping(c.Request.Context()), + } + c.JSON(200, health) +} +``` + +### Pattern: Middleware Chain +```go +router := gin.New() +router.Use(middleware.Logger()) +router.Use(middleware.Recovery()) +router.Use(middleware.Auth()) +router.Use(middleware.Tracing()) + +router.POST("/api/ingest", handler.Ingest) +``` + +--- + +## 22. Code Review Checklist + +Before submitting PR: + +- [ ] Follows layer architecture (Handler → Transformer → Service → Repository) +- [ ] No business logic in Handler +- [ ] Mapping in Transformer +- [ ] Business logic in Service +- [ ] SQL only in Repository +- [ ] All tests pass (≥80% coverage) +- [ ] Logging complete (request_id, action, duration) +- [ ] Error handling correct (wrapped) +- [ ] Context used in all I/O +- [ ] No hardcoded values +- [ ] Passes linting (`golangci-lint run`) +- [ ] No SQL injection vulnerabilities +- [ ] No secrets in code +- [ ] Performance acceptable (EXPLAIN ANALYZE for queries) +- [ ] Metrics added +- [ ] README updated (if new feature) + +--- + +## 23. Definition of Done + +Service is production-ready when: + +- ✅ All unit tests pass (≥80% coverage) +- ✅ Code review approved +- ✅ Linting passes (golangci-lint) +- ✅ No security vulnerabilities (gosec) +- ✅ Database migrations tested +- ✅ Configuration documented (.env.example) +- ✅ API documented (OpenAPI/Swagger) +- ✅ Logging configured (ELK integration) +- ✅ Metrics configured (Prometheus) +- ✅ Performance benchmarked +- ✅ Deployment tested (Docker build succeeds) +- ✅ Rollback plan documented +- ✅ README complete + +--- + +## 24. Project Structure Script + +**New Service Generation:** +```bash +./scripts/new-service.sh ingestion-payments + +# Generated: +# services/ingestion-payments/ +# ├── go.mod +# ├── main.go +# ├── config.go +# ├── handler/ +# ├── service/ +# ├── repository/ +# ├── transformer/ +# ├── domain/ +# ├── entity/ +# ├── dto/ +# ├── migrations/ +# ├── tests/ +# └── README.md +``` + +--- + +## 25. References + +- **Go Best Practices**: https://golang.org/doc/effective_go +- **PostgreSQL**: https://www.postgresql.org/docs/ +- **pgx Documentation**: https://github.com/jackc/pgx +- **Gin Web Framework**: https://gin-gonic.com/ +- **Prometheus**: https://prometheus.io/docs/ +- **Clean Architecture**: Robert C. Martin +- **Domain-Driven Design**: Eric Evans +- **Microservices Patterns**: Chris Richardson + +--- + +## 26. Change History + +| Version | Date | Changes | +|---------|------------|---------| +| 1.0 | 2026-07-24 | Initial Version - Go DW Framework | + +--- + +**Last Updated:** 2026-07-24 +**Owner:** Engineering Team +**Next Review:** 2026-10-24 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2fe7445 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +**/.env +!**/.env.example + +**/*.exe +**/*.test +**/*.out + +.git +.gitignore +.claude +docs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..483f82a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.env +!.env.example + +*.exe +*.test +*.out +vendor/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..f171b45 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,83 @@ +stages: + - test + - build + +variables: + GO_VERSION: "1.25" + +.go-cache: + image: golang:${GO_VERSION}-alpine + before_script: + - apk add --no-cache git + cache: + key: go-mod-cache + paths: + - .go-cache/ + variables: + GOPATH: "$CI_PROJECT_DIR/.go-cache" + +# ---------- omnix-broadcast ---------- +test:omnix-broadcast: + stage: test + extends: .go-cache + script: + - cd services/omnix-broadcast + - go vet ./... + - go test ./tests/... -v + rules: + - changes: + - framework/**/* + - services/omnix-broadcast/**/* + - go.work + - go.work.sum + +build:omnix-broadcast: + stage: build + image: docker:24 + services: + - docker:24-dind + script: + - docker build -f deployments/docker/omnix-broadcast/Dockerfile + -t "$CI_REGISTRY_IMAGE/omnix-broadcast:$CI_COMMIT_SHORT_SHA" . + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + - docker push "$CI_REGISTRY_IMAGE/omnix-broadcast:$CI_COMMIT_SHORT_SHA" + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + - framework/**/* + - services/omnix-broadcast/**/* + - go.work + - go.work.sum + +# ---------- template-service ---------- +test:template-service: + stage: test + extends: .go-cache + script: + - cd services/template-service + - go vet ./... + - go test ./tests/... -v + rules: + - changes: + - framework/**/* + - services/template-service/**/* + - go.work + - go.work.sum + +build:template-service: + stage: build + image: docker:24 + services: + - docker:24-dind + script: + - docker build -f deployments/docker/template-service/Dockerfile + -t "$CI_REGISTRY_IMAGE/template-service:$CI_COMMIT_SHORT_SHA" . + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + - docker push "$CI_REGISTRY_IMAGE/template-service:$CI_COMMIT_SHORT_SHA" + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + - framework/**/* + - services/template-service/**/* + - go.work + - go.work.sum diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..edac9bd --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,17 @@ +# GitLab Code Owners — https://docs.gitlab.com/ee/user/project/codeowners/ +# Update the group handles below to match your actual GitLab groups/usernames. + +[Platform / Framework] +framework/ @platform-team +go.work +go.work.sum + +[Omnix Broadcast Team] +services/omnix-broadcast/ @team-omnix-broadcast + +[Template Service Team] +services/template-service/ @team-template-service + +[Deployment] +deployments/ @platform-team +docker-compose.yml @platform-team diff --git a/deployments/docker/omnix-broadcast/Dockerfile b/deployments/docker/omnix-broadcast/Dockerfile new file mode 100644 index 0000000..ea59bfa --- /dev/null +++ b/deployments/docker/omnix-broadcast/Dockerfile @@ -0,0 +1,29 @@ +# Build context: repo root (../../..) +# docker build -f deployments/docker/omnix-broadcast/Dockerfile -t omnix-broadcast . + +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /src + +COPY go.work go.work.sum ./ +COPY framework ./framework +COPY services/omnix-broadcast ./services/omnix-broadcast +COPY services/template-service ./services/template-service + +WORKDIR /src/services/omnix-broadcast +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/omnix-broadcast . + +FROM alpine:3.20 + +RUN apk add --no-cache ca-certificates tzdata && \ + adduser -D -u 10001 app +USER app + +WORKDIR /app +COPY --from=builder /out/omnix-broadcast . +COPY services/omnix-broadcast/migrations ./migrations + +EXPOSE 8081 +ENTRYPOINT ["./omnix-broadcast"] diff --git a/deployments/docker/template-service/Dockerfile b/deployments/docker/template-service/Dockerfile new file mode 100644 index 0000000..e7aa25e --- /dev/null +++ b/deployments/docker/template-service/Dockerfile @@ -0,0 +1,29 @@ +# Build context: repo root (../../..) +# docker build -f deployments/docker/template-service/Dockerfile -t template-service . + +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /src + +COPY go.work go.work.sum ./ +COPY framework ./framework +COPY services/omnix-broadcast ./services/omnix-broadcast +COPY services/template-service ./services/template-service + +WORKDIR /src/services/template-service +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/template-service . + +FROM alpine:3.20 + +RUN apk add --no-cache ca-certificates tzdata && \ + adduser -D -u 10001 app +USER app + +WORKDIR /app +COPY --from=builder /out/template-service . +COPY services/template-service/migrations ./migrations + +EXPOSE 8080 +ENTRYPOINT ["./template-service"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..eb60745 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +services: + omnix-broadcast: + build: + context: . + dockerfile: deployments/docker/omnix-broadcast/Dockerfile + image: int-omnix/omnix-broadcast:latest + restart: unless-stopped + env_file: + - services/omnix-broadcast/.env + ports: + - "8081:8081" + + template-service: + build: + context: . + dockerfile: deployments/docker/template-service/Dockerfile + image: int-omnix/template-service:latest + restart: unless-stopped + env_file: + - services/template-service/.env + ports: + - "8080:8080" diff --git a/docs/Collection_broadcast_worker_simplified.go b/docs/Collection_broadcast_worker_simplified.go new file mode 100644 index 0000000..963ab71 --- /dev/null +++ b/docs/Collection_broadcast_worker_simplified.go @@ -0,0 +1,452 @@ +package collector + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + "time" +) + +// ============ DATABASE MODELS ============ + +type BroadcastRecord struct { + ID int64 `db:"id"` + SopigaCollarID int `db:"sopiga_collar_id"` + SopigaTemplateID int `db:"sopiga_template_id"` + MessagePayload map[string]interface{} `db:"message_payload"` // JSONB + Status string `db:"status"` + ErrorMessage *string `db:"error_message"` + ErrorCount int `db:"error_count"` + SopigaRecipientDetailID *int64 `db:"sopiga_recipient_detail_id"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +type TemplateVariable struct { + VariableOrder int `db:"variable_order"` + SopigaVarName string `db:"sopiga_variable_name"` + VariableType string `db:"variable_type"` + DBFieldSource string `db:"db_field_source"` + IsRequired bool `db:"is_required"` +} + +// ============ SOPIGA API MODELS ============ + +type CollarAddRecipientRequest struct { + BroadcastID int `json:"broadcast_id"` + TemplateID int `json:"template_id"` + Details CollarDetails `json:"details"` + Attachment *CollarAttachment `json:"attachment,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type CollarDetails struct { + Recipient string `json:"recipient"` + Message string `json:"message"` +} + +type CollarAttachment struct { + Type string `json:"type"` // document + Caption string `json:"caption"` + File string `json:"file"` // URL +} + +type CollarAPIResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + Data CollarResponseData `json:"data"` + Errors map[string][]string `json:"errors,omitempty"` +} + +type CollarResponseData struct { + BroadcastID int `json:"broadcast_id"` + RecipientDetailID int64 `json:"recipient_detail_id"` + Recipient string `json:"recipient"` + Status string `json:"status"` +} + +// ============ WORKER ============ + +type CollectionBroadcastWorker struct { + db *sql.DB + sopigaBaseURL string + sopigaToken string + httpClient *http.Client + checkInterval time.Duration + batchSize int + maxRetries int + logger *log.Logger +} + +func NewCollectionBroadcastWorker( + db *sql.DB, + sopigaBaseURL string, + sopigaToken string, + logger *log.Logger, +) *CollectionBroadcastWorker { + return &CollectionBroadcastWorker{ + db: db, + sopigaBaseURL: sopigaBaseURL, + sopigaToken: sopigaToken, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + checkInterval: 30 * time.Second, + batchSize: 100, + maxRetries: 3, + logger: logger, + } +} + +// Start — Main worker loop +func (w *CollectionBroadcastWorker) Start(ctx context.Context) { + w.logger.Println("Collection Broadcast Worker started") + + ticker := time.NewTicker(w.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + w.logger.Println("Collection Broadcast Worker stopped") + return + case <-ticker.C: + w.processPendingRecords(ctx) + } + } +} + +// processPendingRecords — Fetch & process pending records +func (w *CollectionBroadcastWorker) processPendingRecords(ctx context.Context) { + records, err := w.getPendingRecords(ctx) + if err != nil { + w.logger.Printf("ERROR fetching pending records: %v", err) + return + } + + if len(records) == 0 { + return + } + + w.logger.Printf("Found %d pending records to process", len(records)) + + // Process concurrently (max 5) + sem := make(chan struct{}, 5) + for _, rec := range records { + sem <- struct{}{} + go func(r BroadcastRecord) { + defer func() { <-sem }() + w.processSingleRecord(ctx, r) + }(rec) + } + + // Wait + for i := 0; i < 5; i++ { + sem <- struct{}{} + } +} + +// getPendingRecords — Query broadcast_staging untuk pending records +// SIMPLIFIED: Hanya query dari broadcast_staging, semua data ada di message_payload +func (w *CollectionBroadcastWorker) getPendingRecords(ctx context.Context) ([]BroadcastRecord, error) { + query := ` + SELECT + id, sopiga_collar_id, sopiga_template_id, message_payload, + status, error_message, error_count, sopiga_recipient_detail_id, + created_at, updated_at + FROM collection_broadcasts.broadcast_staging + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT $1 + ` + + rows, err := w.db.QueryContext(ctx, query, w.batchSize) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []BroadcastRecord + for rows.Next() { + var rec BroadcastRecord + var payloadJSON []byte + + err := rows.Scan( + &rec.ID, &rec.SopigaCollarID, &rec.SopigaTemplateID, &payloadJSON, + &rec.Status, &rec.ErrorMessage, &rec.ErrorCount, &rec.SopigaRecipientDetailID, + &rec.CreatedAt, &rec.UpdatedAt, + ) + if err != nil { + w.logger.Printf("ERROR scanning record: %v", err) + continue + } + + // Parse JSONB message_payload + err = json.Unmarshal(payloadJSON, &rec.MessagePayload) + if err != nil { + w.logger.Printf("ERROR parsing message_payload for record %d: %v", rec.ID, err) + continue + } + + records = append(records, rec) + } + + return records, rows.Err() +} + +// processSingleRecord — Call Sopiga API +func (w *CollectionBroadcastWorker) processSingleRecord(ctx context.Context, rec BroadcastRecord) { + // Build message dinamis dari template variable mapping + message, err := w.buildDynamicMessage(ctx, rec.SopigaTemplateID, rec.MessagePayload) + if err != nil { + w.logger.Printf("Record ID %d: ERROR building message: %v", rec.ID, err) + w.updateStatus(ctx, rec.ID, "failed", fmt.Sprintf("message build error: %v", err), nil) + return + } + + // Get recipient phone dari message_payload + recipientPhone, ok := rec.MessagePayload["nasabah_phone"].(string) + if !ok { + w.logger.Printf("Record ID %d: ERROR - nasabah_phone not in payload", rec.ID) + w.updateStatus(ctx, rec.ID, "failed", "nasabah_phone not in payload", nil) + return + } + + // Get invoice URL dari message_payload + invoiceURL, ok := rec.MessagePayload["invoice_pdf_url"].(string) + if !ok { + w.logger.Printf("Record ID %d: ERROR - invoice_pdf_url not in payload", rec.ID) + w.updateStatus(ctx, rec.ID, "failed", "invoice_pdf_url not in payload", nil) + return + } + + // Get nasabah_nama untuk caption + nasabahNama, _ := rec.MessagePayload["nasabah_nama"].(string) + + // Build attachment + attachment := &CollarAttachment{ + Type: "document", + Caption: fmt.Sprintf("Invoice - %s", nasabahNama), + File: invoiceURL, + } + + // Build labels + labels := map[string]string{ + "module": "gadai_collection", + } + + // Build request + collarReq := CollarAddRecipientRequest{ + BroadcastID: rec.SopigaCollarID, + TemplateID: rec.SopigaTemplateID, + Details: CollarDetails{ + Recipient: recipientPhone, + Message: message, + }, + Attachment: attachment, + Labels: labels, + } + + // Call API dengan retry + resp, err := w.callCollarAPIWithRetry(ctx, collarReq, 0) + if err != nil { + w.logger.Printf("Record ID %d: FAILED after %d retries: %v", rec.ID, w.maxRetries, err) + w.updateStatus(ctx, rec.ID, "failed", err.Error(), nil) + return + } + + // Success + w.logger.Printf("Record ID %d: DISPATCHED (recipient_detail_id: %d)", rec.ID, resp.Data.RecipientDetailID) + w.updateDispatchedStatus(ctx, rec.ID, resp.Data.RecipientDetailID) +} + +// buildDynamicMessage — Build message dari template variables mapping +// Query template_variable_mapping, interpolate sesuai urutan +func (w *CollectionBroadcastWorker) buildDynamicMessage( + ctx context.Context, + templateID int, + messagePayload map[string]interface{}, +) (string, error) { + // Query template variables + query := ` + SELECT variable_order, sopiga_variable_name, variable_type, db_field_source, is_required + FROM collection_broadcasts.template_variable_mapping + WHERE sopiga_template_id = $1 + ORDER BY variable_order ASC + ` + + rows, err := w.db.QueryContext(ctx, query, templateID) + if err != nil { + return "", fmt.Errorf("query template variables failed: %w", err) + } + defer rows.Close() + + var parts []string + for rows.Next() { + var tv TemplateVariable + if err := rows.Scan(&tv.VariableOrder, &tv.SopigaVarName, &tv.VariableType, &tv.DBFieldSource, &tv.IsRequired); err != nil { + return "", fmt.Errorf("scan template variable failed: %w", err) + } + + // Get value dari message_payload + value, exists := messagePayload[tv.DBFieldSource] + if !exists { + if tv.IsRequired { + return "", fmt.Errorf("missing required field: %s", tv.DBFieldSource) + } + value = "" + } + + // Format sesuai type + var formattedValue string + switch tv.VariableType { + case "string": + formattedValue = fmt.Sprintf("%v", value) + case "integer", "decimal": + formattedValue = fmt.Sprintf("%.0f", value) + case "date": + formattedValue = fmt.Sprintf("%v", value) + default: + formattedValue = fmt.Sprintf("%v", value) + } + + parts = append(parts, formattedValue) + } + + if err := rows.Err(); err != nil { + return "", err + } + + // Join dengan '#' + return strings.Join(parts, "#"), nil +} + +// callCollarAPIWithRetry — Call API dengan automatic retry +func (w *CollectionBroadcastWorker) callCollarAPIWithRetry( + ctx context.Context, + req CollarAddRecipientRequest, + attempt int, +) (*CollarAPIResponse, error) { + if attempt > w.maxRetries { + return nil, fmt.Errorf("max retries (%d) exceeded", w.maxRetries) + } + + resp, err := w.callCollarAPI(ctx, req) + if err != nil { + if attempt < w.maxRetries { + waitTime := time.Duration(1<= 0) +); + +-- Indexes untuk worker & monitoring queries +CREATE INDEX idx_broadcast_staging_status ON collection_broadcasts.broadcast_staging(status); +CREATE INDEX idx_broadcast_staging_created_at ON collection_broadcasts.broadcast_staging(created_at DESC); +CREATE INDEX idx_broadcast_staging_collar_id ON collection_broadcasts.broadcast_staging(sopiga_collar_id); +CREATE INDEX idx_broadcast_staging_template_id ON collection_broadcasts.broadcast_staging(sopiga_template_id); + +-- Composite index untuk worker (most frequent query) +CREATE INDEX idx_broadcast_staging_pending_query + ON collection_broadcasts.broadcast_staging(status, created_at ASC) + WHERE status = 'pending'; + +-- Composite index untuk retry queue +CREATE INDEX idx_broadcast_staging_retry_query + ON collection_broadcasts.broadcast_staging(status, error_count, created_at ASC) + WHERE status = 'retry_scheduled' AND error_count < 3; + +-- ============================================================================ +-- 3. AUDIT & LOGGING +-- ============================================================================ + +-- Status transition audit log +CREATE TABLE collection_broadcasts.broadcast_audit_log ( + id BIGSERIAL PRIMARY KEY, + broadcast_id BIGINT NOT NULL REFERENCES collection_broadcasts.broadcast_staging(id) ON DELETE CASCADE, + old_status VARCHAR(50), + new_status VARCHAR(50) NOT NULL, + reason VARCHAR(500), + sopiga_response JSONB, + changed_by VARCHAR(100) DEFAULT 'system', + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_broadcast_audit_log_broadcast_id ON collection_broadcasts.broadcast_audit_log(broadcast_id); +CREATE INDEX idx_broadcast_audit_log_created_at ON collection_broadcasts.broadcast_audit_log(created_at DESC); + +-- Error tracking +CREATE TABLE collection_broadcasts.broadcast_error_log ( + id BIGSERIAL PRIMARY KEY, + broadcast_id BIGINT NOT NULL REFERENCES collection_broadcasts.broadcast_staging(id) ON DELETE CASCADE, + error_type VARCHAR(100), -- network_error, api_error, validation_error + error_code VARCHAR(50), + error_message TEXT, + error_details JSONB, + attempt_number INT, + next_retry_at TIMESTAMP, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_broadcast_error_log_broadcast_id ON collection_broadcasts.broadcast_error_log(broadcast_id); +CREATE INDEX idx_broadcast_error_log_error_type ON collection_broadcasts.broadcast_error_log(error_type); + +-- Sopiga delivery status sync +CREATE TABLE collection_broadcasts.sopiga_sync_job ( + id BIGSERIAL PRIMARY KEY, + broadcast_id BIGINT NOT NULL REFERENCES collection_broadcasts.broadcast_staging(id) ON DELETE CASCADE, + sopiga_recipient_detail_id BIGINT, + last_synced_at TIMESTAMP, + last_status_from_sopiga VARCHAR(50), + sync_count INT DEFAULT 0, + next_sync_at TIMESTAMP, + completed_at TIMESTAMP, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_sopiga_sync_job_broadcast_id ON collection_broadcasts.sopiga_sync_job(broadcast_id); +CREATE INDEX idx_sopiga_sync_job_next_sync_at ON collection_broadcasts.sopiga_sync_job(next_sync_at); + +-- ============================================================================ +-- 4. VIEWS +-- ============================================================================ + +-- Template variables dengan urutan (untuk Worker reference) +CREATE VIEW collection_broadcasts.v_template_variables_ordered AS +SELECT + tvm.sopiga_template_id, + stc.template_name, + tvm.variable_order, + tvm.sopiga_variable_name, + tvm.variable_type, + tvm.db_field_source, + tvm.is_required, + tvm.example_value +FROM collection_broadcasts.template_variable_mapping tvm +JOIN collection_broadcasts.sopiga_template_config stc ON tvm.sopiga_template_id = stc.sopiga_template_id +WHERE stc.active = TRUE +ORDER BY tvm.sopiga_template_id, tvm.variable_order; + +-- Collar summary +CREATE VIEW collection_broadcasts.v_collar_summary AS +SELECT + sc.sopiga_collar_id, + sc.broadcast_name, + COUNT(*) as total_records, + COUNT(CASE WHEN bs.status = 'pending' THEN 1 END) as pending, + COUNT(CASE WHEN bs.status = 'dispatched' THEN 1 END) as dispatched, + COUNT(CASE WHEN bs.status = 'delivered' THEN 1 END) as delivered, + COUNT(CASE WHEN bs.status = 'failed' THEN 1 END) as failed, + ROUND(100.0 * COUNT(CASE WHEN bs.status = 'delivered' THEN 1 END) / NULLIF(COUNT(*), 0), 2) as delivery_rate_percent +FROM collection_broadcasts.sopiga_collar_config sc +LEFT JOIN collection_broadcasts.broadcast_staging bs ON sc.sopiga_collar_id = bs.sopiga_collar_id +GROUP BY sc.sopiga_collar_id, sc.broadcast_name; + +-- Failed records (last 24h) +CREATE VIEW collection_broadcasts.v_failed_records_24h AS +SELECT + id, + sopiga_collar_id, + sopiga_template_id, + message_payload->>'nasabah_nama' as nasabah_nama, + message_payload->>'nasabah_phone' as nasabah_phone, + error_message, + error_count, + failed_at, + created_at +FROM collection_broadcasts.broadcast_staging +WHERE status = 'failed' AND created_at > NOW() - INTERVAL 1 DAY +ORDER BY failed_at DESC; + +-- Delivery rate (last 24h) +CREATE VIEW collection_broadcasts.v_delivery_rate_24h AS +SELECT + COUNT(*) as total, + COUNT(CASE WHEN status = 'delivered' THEN 1 END) as delivered, + COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed, + COUNT(CASE WHEN status = 'dispatched' THEN 1 END) as in_progress, + ROUND(100.0 * COUNT(CASE WHEN status = 'delivered' THEN 1 END) / NULLIF(COUNT(*), 0), 2) as delivery_rate_percent +FROM collection_broadcasts.broadcast_staging +WHERE created_at > NOW() - INTERVAL 1 DAY; + +-- ============================================================================ +-- 5. HELPER FUNCTIONS +-- ============================================================================ + +-- Get template variables by template_id +CREATE OR REPLACE FUNCTION collection_broadcasts.get_template_variables( + p_sopiga_template_id INT +) +RETURNS TABLE( + variable_order INT, + sopiga_variable_name VARCHAR, + variable_type VARCHAR, + db_field_source VARCHAR, + is_required BOOLEAN +) AS $$ +BEGIN + RETURN QUERY + SELECT + tvm.variable_order, + tvm.sopiga_variable_name, + tvm.variable_type, + tvm.db_field_source, + tvm.is_required + FROM collection_broadcasts.template_variable_mapping tvm + WHERE tvm.sopiga_template_id = p_sopiga_template_id + ORDER BY tvm.variable_order ASC; +END; +$$ LANGUAGE plpgsql; + +-- Update status dengan audit log +CREATE OR REPLACE FUNCTION collection_broadcasts.update_broadcast_status( + p_broadcast_id BIGINT, + p_new_status VARCHAR, + p_error_message TEXT DEFAULT NULL, + p_sopiga_response JSONB DEFAULT NULL +) +RETURNS VOID AS $$ +DECLARE + v_old_status VARCHAR; +BEGIN + SELECT status INTO v_old_status + FROM collection_broadcasts.broadcast_staging + WHERE id = p_broadcast_id; + + UPDATE collection_broadcasts.broadcast_staging + SET + status = p_new_status, + error_message = p_error_message, + updated_at = NOW(), + dispatched_at = CASE WHEN p_new_status = 'dispatched' THEN NOW() ELSE dispatched_at END, + delivered_at = CASE WHEN p_new_status = 'delivered' THEN NOW() ELSE delivered_at END, + failed_at = CASE WHEN p_new_status = 'failed' THEN NOW() ELSE failed_at END + WHERE id = p_broadcast_id; + + INSERT INTO collection_broadcasts.broadcast_audit_log (broadcast_id, old_status, new_status, sopiga_response) + VALUES (p_broadcast_id, v_old_status, p_new_status, p_sopiga_response); +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================ +-- 6. INITIALIZATION DATA +-- ============================================================================ + +-- Insert sample template +INSERT INTO collection_broadcasts.sopiga_template_config (template_name, sopiga_template_id, channel, template_type, description) +VALUES ('Collection Invoice May 2026', 2, 'waba', 'utility', 'Invoice bulanan untuk collection Mei 2026'); + +-- Insert template variables +INSERT INTO collection_broadcasts.template_variable_mapping (sopiga_template_id, variable_order, sopiga_variable_name, variable_type, db_field_source, is_required, example_value) +VALUES + (2, 1, 'Nama', 'string', 'nasabah_nama', TRUE, 'Budi Santoso'), + (2, 2, 'TotalTagihan', 'integer', 'nominal_tagihan', TRUE, '1500000'), + (2, 3, 'TanggalJatuhTempo', 'date', 'tanggal_tempo', TRUE, '2026-06-30'); + +-- Insert collar config +INSERT INTO collection_broadcasts.sopiga_collar_config (broadcast_name, sopiga_collar_id, sopiga_template_id, status, description) +VALUES ('Collection Invoices May 2026', 70, 2, 'open', 'Broadcast collar untuk collection invoice bulanan Mei 2026'); \ No newline at end of file diff --git a/docs/implementasi_guide.md b/docs/implementasi_guide.md new file mode 100644 index 0000000..8a55a7b --- /dev/null +++ b/docs/implementasi_guide.md @@ -0,0 +1,453 @@ +# Gadai Mulia Collection Broadcast Integration +## Complete Implementation Guide + +--- + +## 📋 Overview + +**Purpose:** Send collection invoices via WhatsApp (Sopiga Collar API) to Gadai Mulia customers. + +**Architecture:** +``` +Gadai Collection Service + → INSERT broadcast_staging (JSON payload) + → Go Worker (poll every 30s) + → Query template_variable_mapping + → Build message dynamically + → POST Sopiga Collar API + → WhatsApp delivery + → Update status (delivered/failed) +``` + +**Key Innovation:** Flexible template system via `template_variable_mapping` table — NO code redeploy for new templates. + +--- + +## 🗄️ Database Schema (Simplified Denormalized) + +### Master Configuration (Read-only) + +**Table: sopiga_template_config** +- Stores Sopiga template references (pre-created in Sopiga) +- Fields: `id`, `sopiga_template_id`, `template_name`, `channel` (waba), `template_type`, `description`, `active` +- Pre-populate: One row per template in Sopiga + +**Table: template_variable_mapping** +- Maps Sopiga template variables → message_payload keys +- **CRITICAL:** `variable_order` determines message interpolation sequence +- Fields: `id`, `sopiga_template_id`, `variable_order`, `sopiga_variable_name`, `variable_type` (string/integer/date/decimal), `db_field_source`, `is_required`, `example_value` +- Example: + ``` + sopiga_template_id=2, order=1, name='Nama', field='nasabah_nama' + sopiga_template_id=2, order=2, name='TotalTagihan', field='nominal_tagihan' + sopiga_template_id=2, order=3, name='TanggalJatuhTempo', field='tanggal_tempo' + ``` + +**Table: sopiga_collar_config** +- Stores Sopiga collar references (pre-created in Sopiga) +- Fields: `id`, `sopiga_collar_id`, `broadcast_name`, `sopiga_template_id` (FK), `status` (open/closed), `description` +- Pre-populate: One row per collar in Sopiga + +### Processing (Core) + +**Table: broadcast_staging** ← Main table +- DENORMALIZED: All data in `message_payload` (JSONB) +- NO FK to nasabah or gadai_contract +- Fields: + - `id` (PK) + - `sopiga_collar_id` (FK) + - `sopiga_template_id` (FK) + - `message_payload` (JSONB) ← **Single source of truth** + - `status` (pending → dispatched → delivered/failed) + - `error_message`, `error_count` + - `sopiga_recipient_detail_id` (response from Sopiga) + - `created_at`, `updated_at`, `dispatched_at`, `delivered_at`, `failed_at` + +**Example message_payload:** +```json +{ + "nasabah_nama": "Budi Santoso", + "nasabah_phone": "6281234567890", + "nominal_tagihan": 1500000, + "tanggal_tempo": "2026-06-30", + "invoice_pdf_url": "https://storage.gadai.com/invoices/inv-001.pdf", + "contract_no": "GAD-2026-001" +} +``` + +### Audit & Tracking + +**Table: broadcast_audit_log** +- Tracks status transitions +- Fields: `id`, `broadcast_id` (FK), `old_status`, `new_status`, `reason`, `sopiga_response` (JSONB), `changed_by`, `created_at` + +**Table: broadcast_error_log** +- Detailed error tracking +- Fields: `id`, `broadcast_id` (FK), `error_type`, `error_code`, `error_message`, `error_details` (JSONB), `attempt_number`, `next_retry_at` + +**Table: sopiga_sync_job** +- Tracks delivery status sync from Sopiga +- Fields: `id`, `broadcast_id` (FK), `sopiga_recipient_detail_id`, `last_synced_at`, `last_status_from_sopiga`, `sync_count`, `next_sync_at` + +--- + +## 🔧 Setup Process + +### Step 1: Setup in Sopiga (Manual) + +1. **Create template** in Sopiga: + ``` + POST https://omnix.promas.site/api/client/template + { + "template_name": "Collection Invoice", + "channel": "waba", + "variables": [ + {"name": "Nama", "type": "string"}, + {"name": "TotalTagihan", "type": "integer"}, + {"name": "TanggalJatuhTempo", "type": "date"} + ] + } + → Response: template_id = 2 + ``` + +2. **Create collar** in Sopiga: + ``` + POST https://omnix.promas.site/api/client/collar + { + "judul_broadcast": "Invoice May 2026", + "template_id": 2 + } + → Response: collar_id = 70 + ``` + +### Step 2: Register in Database (5 minutes) + +**2.1 Insert template config:** +```sql +INSERT INTO collection_broadcasts.sopiga_template_config + (template_name, sopiga_template_id, channel, template_type, description) +VALUES + ('Collection Invoice May 2026', 2, 'waba', 'utility', 'Invoice bulanan'); +``` + +**2.2 Insert variable mappings (ORDER IS CRITICAL!):** +```sql +INSERT INTO collection_broadcasts.template_variable_mapping + (sopiga_template_id, variable_order, sopiga_variable_name, variable_type, db_field_source, is_required, example_value) +VALUES + (2, 1, 'Nama', 'string', 'nasabah_nama', TRUE, 'Budi Santoso'), + (2, 2, 'TotalTagihan', 'integer', 'nominal_tagihan', TRUE, '1500000'), + (2, 3, 'TanggalJatuhTempo', 'date', 'tanggal_tempo', TRUE, '2026-06-30'); +``` + +**2.3 Insert collar config:** +```sql +INSERT INTO collection_broadcasts.sopiga_collar_config + (broadcast_name, sopiga_collar_id, sopiga_template_id, status) +VALUES + ('Collection Invoices May 2026', 70, 2, 'open'); +``` + +### Step 3: Deploy Application + +Deploy Go worker with: +- Database connection string +- Sopiga base URL: `https://omnix.promas.site` +- Sopiga API token + +Worker runs continuously: +- **Broadcast dispatch:** Poll every 30s, process pending records +- **Status sync:** Poll every 5 min, sync delivery status from Sopiga + +--- + +## 🔄 How It Works + +### Runtime Flow + +**Gadai Collection Service (sends data):** +```go +// When collection reminder needed: +payload := map[string]interface{}{ + "nasabah_nama": "Budi Santoso", + "nasabah_phone": "6281234567890", + "nominal_tagihan": 1500000.0, + "tanggal_tempo": "2026-06-30", + "invoice_pdf_url": "https://storage.gadai.com/invoices/inv-001.pdf", + "contract_no": "GAD-2026-001", +} + +// Insert to broadcast_staging +db.Exec(` + INSERT INTO collection_broadcasts.broadcast_staging + (sopiga_collar_id, sopiga_template_id, message_payload, status) + VALUES ($1, $2, $3, 'pending') +`, 70, 2, payload) +``` + +**Go Worker (processes data - automatic):** +``` +1. Poll broadcast_staging WHERE status='pending' LIMIT 100 +2. For each record: + a. Get message_payload from database (already has all fields) + b. Query template_variable_mapping WHERE sopiga_template_id=2 ORDER BY variable_order + c. Build message: + - For each variable in order (1,2,3...): + - Get value from payload[db_field_source] + - Format by type (string/integer/date) + - Append to parts[] + - message = parts.join("#") + - Result: "Budi Santoso#1500000#2026-06-30" + + d. Call Sopiga API: + POST /api/client/collar/add-recipient + { + "broadcast_id": 70, + "template_id": 2, + "details": { + "recipient": "6281234567890", + "message": "Budi Santoso#1500000#2026-06-30" + }, + "attachment": { + "type": "document", + "file": "https://storage.gadai.com/invoices/inv-001.pdf" + } + } + + e. Response (HTTP 202 Accepted): + { + "success": true, + "data": { + "recipient_detail_id": 512, + "status": "pending" + } + } + + f. Update broadcast_staging: + UPDATE status='dispatched', sopiga_recipient_detail_id=512 + +3. Status sync worker (every 5 min): + - For dispatched records: + - GET /api/client/collar/add-recipient/{recipient_detail_id}/detail + - Update status: delivered OR failed +``` + +--- + +## 📊 Adding New Template (Zero Code Change) + +### Scenario: Template now has 4 variables instead of 3 + +**Old:** +``` +1. Nama +2. TotalTagihan +3. TanggalJatuhTempo +``` + +**New:** +``` +1. Nama +2. NoKontrak ← NEW! +3. TotalTagihan +4. TanggalJatuhTempo +``` + +### Setup (5 minutes, Database only) + +1. Create template in Sopiga → template_id = 3 +2. Create collar in Sopiga → collar_id = 71 + +3. Register in database: +```sql +-- Template +INSERT INTO sopiga_template_config (template_name, sopiga_template_id, ...) +VALUES ('Collection Invoice Extended', 3, ...); + +-- Variables (URUTAN PENTING!) +INSERT INTO template_variable_mapping + (sopiga_template_id, variable_order, sopiga_variable_name, variable_type, db_field_source, ...) +VALUES + (3, 1, 'Nama', 'string', 'nasabah_nama', ...), + (3, 2, 'NoKontrak', 'string', 'contract_no', ...), ← NEW! + (3, 3, 'TotalTagihan', 'integer', 'nominal_tagihan', ...), + (3, 4, 'TanggalJatuhTempo', 'date', 'tanggal_tempo', ...); + +-- Collar +INSERT INTO sopiga_collar_config (broadcast_name, sopiga_collar_id, sopiga_template_id, ...) +VALUES ('Collection Invoices June 2026', 71, 3, ...); +``` + +**Go code?** ✅ ZERO changes — `buildDynamicMessage()` queries mapping every time. + +--- + +## 🚀 Gadai Service Integration + +**How Gadai Collection Service calls this:** + +```go +package gadai + +type CollectionBroadcastService struct { + db *sql.DB +} + +func (s *CollectionBroadcastService) SendInvoiceReminder( + ctx context.Context, + nasabahID, contractNo string, + nominalTagihan float64, + dueDate time.Time, +) error { + // Get invoice URL (from Gadai storage or API) + invoiceURL := fmt.Sprintf("https://storage.gadai.com/invoices/%s.pdf", contractNo) + + // Build payload (flexible - can have extra fields) + payload := map[string]interface{}{ + "nasabah_nama": "Budi Santoso", + "nasabah_phone": "6281234567890", + "nominal_tagihan": nominalTagihan, + "tanggal_tempo": dueDate.Format("2006-01-02"), + "invoice_pdf_url": invoiceURL, + "contract_no": contractNo, + "cif": "12345", // Extra field - ignored by worker if not in mapping + } + + // Marshal to JSONB + payloadJSON, _ := json.Marshal(payload) + + // Insert to broadcast_staging + // Worker will automatically pick it up (every 30s poll) + query := ` + INSERT INTO collection_broadcasts.broadcast_staging + (sopiga_collar_id, sopiga_template_id, message_payload, status) + VALUES ($1, $2, $3, 'pending') + ` + + _, err := s.db.ExecContext(ctx, query, + 70, // sopiga_collar_id (hardcoded or from config) + 2, // sopiga_template_id (hardcoded or from config) + payloadJSON, + ) + + return err +} +``` + +--- + +## 📈 Monitoring & Reporting + +### Views Available + +**v_template_variables_ordered** +```sql +SELECT * FROM collection_broadcasts.v_template_variables_ordered; +-- Shows all template variables with proper ordering +``` + +**v_collar_summary** +```sql +SELECT * FROM collection_broadcasts.v_collar_summary; +-- Shows delivery rate per collar +``` + +**v_delivery_rate_24h** +```sql +SELECT * FROM collection_broadcasts.v_delivery_rate_24h; +-- Shows delivery rate for last 24 hours +``` + +**v_failed_records_24h** +```sql +SELECT * FROM collection_broadcasts.v_failed_records_24h; +-- Shows failed records with error details +``` + +--- + +## 🛠️ Troubleshooting + +### Issue: Message format wrong + +**Cause:** `variable_order` in mapping doesn't match Sopiga template order + +**Fix:** +```sql +-- Check current mapping +SELECT variable_order, sopiga_variable_name FROM template_variable_mapping +WHERE sopiga_template_id=2 ORDER BY variable_order; + +-- If order wrong, delete and re-insert correctly +DELETE FROM template_variable_mapping WHERE sopiga_template_id=2; +-- Re-insert with correct order +``` + +### Issue: Missing field in message_payload + +**Cause:** Gadai service didn't include field in payload, but it's marked required in mapping + +**Fix:** +```sql +-- Either make it optional +UPDATE template_variable_mapping SET is_required=FALSE +WHERE sopiga_variable_name='NoKontrak'; + +-- Or ensure Gadai service includes it +``` + +### Issue: Delivery status stuck at 'dispatched' + +**Cause:** Status sync worker not running or Sopiga API unreachable + +**Fix:** +```sql +-- Check sync jobs +SELECT * FROM sopiga_sync_job WHERE next_sync_at < NOW(); + +-- Manually trigger status check +SELECT collection_broadcasts.get_template_variables(2); +``` + +--- + +## 🔐 Security Considerations + +1. **Sopiga API Token:** Store in environment variable, never hardcode +2. **Database:** Restrict access to `collection_broadcasts` schema +3. **Phone Numbers:** Already PII, treat as sensitive +4. **PDFs:** Ensure URLs are time-limited or access-controlled + +--- + +## 📋 Checklist: New Template + +- [ ] Create template in Sopiga, note `template_id` +- [ ] Create collar in Sopiga, note `collar_id` +- [ ] Insert `sopiga_template_config` row +- [ ] Insert `template_variable_mapping` rows (check order!) +- [ ] Insert `sopiga_collar_config` row +- [ ] Verify mapping: `SELECT * FROM v_template_variables_ordered WHERE sopiga_template_id=X` +- [ ] Test with sample record in `broadcast_staging` +- [ ] Monitor first 10 deliveries in logs +- [ ] Confirm in `v_collar_summary` delivery rate + +--- + +## 📞 Support + +**Schema files:** +- `gadai_collection_broadcast_simplified_schema.sql` — Complete DDL + +**Application files:** +- `collection_broadcast_worker_simplified.go` — Go worker implementation + +**Deployment:** +- Requires: PostgreSQL 11+, Go 1.16+ +- Connects to: Sopiga API at `https://omnix.promas.site` + +--- + +**Version:** 1.0 +**Last Updated:** 2026-08-03 +**Status:** Ready for deployment ✅ \ No newline at end of file diff --git a/docs/quick_start.md b/docs/quick_start.md new file mode 100644 index 0000000..e2b83cd --- /dev/null +++ b/docs/quick_start.md @@ -0,0 +1,319 @@ +# Quick Start Guide +## Gadai Mulia Collection Broadcast Integration + +--- + +## ⚡ 5-Minute Setup + +### Prerequisites +- PostgreSQL database access +- Sopiga API credentials (token) +- Go 1.16+ (for worker deployment) + +### Step 1: Create Database Schema (2 minutes) + +```bash +# Run SQL schema +psql -U postgres -h localhost -d gadai_mulia < gadai_collection_broadcast_simplified_schema.sql +``` + +Verify schema created: +```sql +\dt collection_broadcasts.* +-- Should show: sopiga_template_config, template_variable_mapping, sopiga_collar_config, broadcast_staging, etc +``` + +### Step 2: Setup Sopiga Template (2 minutes) + +**In Sopiga UI or API:** + +```bash +# Create template +curl -X POST https://omnix.promas.site/api/client/template \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "template_name": "Collection Invoice", + "channel": "waba", + "variables": [ + {"name": "Nama", "type": "string"}, + {"name": "TotalTagihan", "type": "integer"}, + {"name": "TanggalJatuhTempo", "type": "date"} + ] + }' +# Response: {"data": {"id": 2}} + +# Create collar +curl -X POST https://omnix.promas.site/api/client/collar \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "judul_broadcast": "Invoice May 2026", + "template_id": 2 + }' +# Response: {"data": {"id": 70}} +``` + +Note down: +- `template_id = 2` +- `collar_id = 70` + +### Step 3: Register in Database (1 minute) + +**Connect to PostgreSQL:** +```bash +psql -U postgres -h localhost -d gadai_mulia +``` + +**Insert template:** +```sql +INSERT INTO collection_broadcasts.sopiga_template_config + (template_name, sopiga_template_id, channel, template_type, description) +VALUES + ('Collection Invoice May 2026', 2, 'waba', 'utility', 'Invoice bulanan untuk collection'); +``` + +**Insert variable mappings (URUTAN PENTING!):** +```sql +INSERT INTO collection_broadcasts.template_variable_mapping + (sopiga_template_id, variable_order, sopiga_variable_name, variable_type, db_field_source, is_required, example_value) +VALUES + (2, 1, 'Nama', 'string', 'nasabah_nama', TRUE, 'Budi Santoso'), + (2, 2, 'TotalTagihan', 'integer', 'nominal_tagihan', TRUE, '1500000'), + (2, 3, 'TanggalJatuhTempo', 'date', 'tanggal_tempo', TRUE, '2026-06-30'); +``` + +**Insert collar config:** +```sql +INSERT INTO collection_broadcasts.sopiga_collar_config + (broadcast_name, sopiga_collar_id, sopiga_template_id, status, description) +VALUES + ('Collection Invoices May 2026', 70, 2, 'open', 'Broadcast collar untuk collection invoice bulanan Mei 2026'); +``` + +**Verify setup:** +```sql +SELECT * FROM collection_broadcasts.v_template_variables_ordered +WHERE sopiga_template_id = 2; +-- Should show 3 rows with order 1, 2, 3 +``` + +### Step 4: Deploy Worker (1 minute) + +```bash +# Build +go build -o collection_broadcast_worker collection_broadcast_worker_simplified.go + +# Run with environment variables +export DB_HOST=localhost +export DB_PORT=5432 +export DB_NAME=gadai_mulia +export DB_USER=postgres +export DB_PASSWORD=xxx +export SOPIGA_BASE_URL=https://omnix.promas.site +export SOPIGA_TOKEN=your_api_token + +./collection_broadcast_worker +# Output: Collection Broadcast Worker started +``` + +--- + +## 🧪 Test It + +### Insert Test Record + +```sql +-- Insert to broadcast_staging +INSERT INTO collection_broadcasts.broadcast_staging + (sopiga_collar_id, sopiga_template_id, message_payload, status) +VALUES + ( + 70, + 2, + '{"nasabah_nama": "Budi Santoso", "nasabah_phone": "6281234567890", "nominal_tagihan": 1500000, "tanggal_tempo": "2026-06-30", "invoice_pdf_url": "https://storage.gadai.com/invoices/inv-001.pdf"}', + 'pending' + ); + +-- Check it's pending +SELECT id, status, created_at FROM collection_broadcasts.broadcast_staging WHERE status='pending'; +``` + +### Wait 30 seconds + +Worker polls every 30 seconds. Check status: + +```sql +SELECT id, status, sopiga_recipient_detail_id, dispatched_at, error_message +FROM collection_broadcasts.broadcast_staging +WHERE id = 1; + +-- Should show: status='dispatched', sopiga_recipient_detail_id=512 +``` + +### Check WhatsApp Delivery + +After 5 minutes, check sync job: + +```sql +SELECT id, status, last_status_from_sopiga, delivered_at +FROM collection_broadcasts.broadcast_staging +WHERE id = 1; + +-- Should show: status='delivered', delivered_at= +``` + +--- + +## 📊 Monitor + +### Check Delivery Rate +```sql +SELECT * FROM collection_broadcasts.v_delivery_rate_24h; +-- delivery_rate_percent should be > 95% +``` + +### Check Failed Records +```sql +SELECT * FROM collection_broadcasts.v_failed_records_24h; +-- Should be empty or < 5% +``` + +### Check Collar Summary +```sql +SELECT * FROM collection_broadcasts.v_collar_summary; +-- See totals per collar, delivery rates +``` + +--- + +## ➕ Add New Template (5 minutes) + +When Sopiga template changes (e.g., add `NoKontrak` field): + +### 1. Create in Sopiga +```bash +# template_id = 3, collar_id = 71 +``` + +### 2. Register in Database +```sql +-- Template +INSERT INTO sopiga_template_config (template_name, sopiga_template_id, ...) +VALUES ('Collection Invoice Extended', 3, ...); + +-- Variables (URUTAN PENTING!) +INSERT INTO template_variable_mapping + (sopiga_template_id, variable_order, sopiga_variable_name, variable_type, db_field_source, is_required) +VALUES + (3, 1, 'Nama', 'string', 'nasabah_nama', TRUE), + (3, 2, 'NoKontrak', 'string', 'contract_no', TRUE), ← NEW + (3, 3, 'TotalTagihan', 'integer', 'nominal_tagihan', TRUE), + (3, 4, 'TanggalJatuhTempo', 'date', 'tanggal_tempo', TRUE); + +-- Collar +INSERT INTO sopiga_collar_config (broadcast_name, sopiga_collar_id, sopiga_template_id, status) +VALUES ('Collection Invoices June 2026', 71, 3, 'open'); +``` + +### 3. NO code redeploy needed ✅ + +Worker automatically picks up new template on next poll. + +--- + +## 🚀 Gadai Service Integration + +**How Gadai calls this:** + +```go +import "database/sql" +import "encoding/json" + +db, _ := sql.Open("postgres", "postgres://user:pass@localhost/gadai_mulia") + +payload := map[string]interface{}{ + "nasabah_nama": "Budi Santoso", + "nasabah_phone": "6281234567890", + "nominal_tagihan": 1500000.0, + "tanggal_tempo": "2026-06-30", + "invoice_pdf_url": "https://storage.gadai.com/invoices/inv-001.pdf", + "contract_no": "GAD-2026-001", +} + +payloadJSON, _ := json.Marshal(payload) + +db.Exec(` + INSERT INTO collection_broadcasts.broadcast_staging + (sopiga_collar_id, sopiga_template_id, message_payload, status) + VALUES ($1, $2, $3, 'pending') +`, 70, 2, payloadJSON) + +// Done! Worker picks it up automatically +``` + +--- + +## 🐛 Troubleshooting + +### "Message format wrong" +→ Check template variable order matches Sopiga: +```sql +SELECT variable_order, sopiga_variable_name FROM template_variable_mapping +WHERE sopiga_template_id=2 ORDER BY variable_order; +``` + +### "Record stuck in pending" +→ Check worker logs: +```bash +# Look for errors in stdout/stderr +# Check DB connection +psql -U postgres -h localhost -d gadai_mulia -c "SELECT 1;" +``` + +### "Sopiga API errors" +→ Check token and URL: +```bash +curl -X GET https://omnix.promas.site/api/client/collar/list \ + -H "Authorization: Bearer YOUR_TOKEN" +``` + +--- + +## ✅ Checklist + +- [ ] Schema created +- [ ] Sopiga template created (note template_id) +- [ ] Sopiga collar created (note collar_id) +- [ ] Database populated with template_id, collar_id +- [ ] Variable mappings inserted (check order!) +- [ ] Worker deployed and running +- [ ] Test record inserted +- [ ] Record moved to 'dispatched' after 30s +- [ ] Record moved to 'delivered' after 5 min +- [ ] WhatsApp message received on test phone + +--- + +## 📞 Quick Reference + +**Database:** +- Schema: `collection_broadcasts` +- Main table: `broadcast_staging` +- Config tables: `sopiga_template_config`, `template_variable_mapping`, `sopiga_collar_config` + +**Sopiga URLs:** +- Base: `https://omnix.promas.site` +- Create template: POST `/api/client/template` +- Create collar: POST `/api/client/collar` +- Add recipient: POST `/api/client/collar/add-recipient` +- Get recipient detail: GET `/api/client/collar/add-recipient/{id}/detail` + +**Worker:** +- Poll interval: 30 seconds (pending records) +- Status sync: 5 minutes (delivery status) +- Max retries: 3 (exponential backoff: 1s, 2s, 4s) + +--- + +**Ready to go! 🚀** \ No newline at end of file diff --git a/docs/webhook_integration_request.md b/docs/webhook_integration_request.md new file mode 100644 index 0000000..e80486e --- /dev/null +++ b/docs/webhook_integration_request.md @@ -0,0 +1,89 @@ +# Permintaan Integrasi Webhook — Collection Broadcast + +**Ke:** Tim Omnix +**Dari:** Tim Collection Broadcast (Gadai Mulia) +**Tujuan:** Kami butuh notifikasi real-time saat status pengiriman WhatsApp +(sent/delivered/read/failed) berubah, supaya tidak perlu polling +`GET /api/client/collar/add-recipient/{id}/detail` berulang-ulang ke sistem +Omnix untuk setiap recipient. + +--- + +## 1. Yang kami minta dari tim Omnix + +1. **Konfirmasi ketersediaan fitur** — apakah Omnix/Sopiga sudah punya (atau bisa + dibuatkan) mekanisme outgoing webhook saat status recipient di collar berubah? + (Kami cek dokumentasi publik di `/docs?api-docs.yaml` dan tidak menemukan + endpoint registrasi webhook untuk collar/recipient — hanya ada `webhook_url` + di level `WhatsAppSession`, yang tampaknya untuk keperluan lain.) +2. **Endpoint/cara registrasi** URL callback kami ke sistem Omnix (dashboard, + API, atau config manual). +3. **Shared secret** untuk signing payload (lihat §3) — dikirim lewat kanal aman + (bukan email biasa), atau kalau Omnix punya skema signature sendiri (mis. HMAC + dengan public key, atau format berbeda), kasih tahu kami spesifikasinya — + kami sesuaikan. +4. **Kapan callback dikirim** — idealnya setiap kali status berubah + (`sent` → `delivered` → `read`, atau → `failed`), bukan cuma sekali di akhir. +5. **Retry policy di sisi Omnix** — kalau endpoint kami down/timeout, apakah + Omnix retry otomatis? Berapa kali, dengan interval berapa? + +## 2. Endpoint & secret yang kami berikan ke Omnix + +``` +URL : https:///webhooks/sopiga/delivery-status +Method: POST +Secret: (kirim terpisah lewat kanal aman — JANGAN taruh di email/chat biasa, + JANGAN commit ke dokumen/repo ini) +``` + +> ⚠️ URL di atas masih placeholder — isi dengan domain publik/staging service +> `omnix-broadcast` kami sebelum dikirim ke tim Omnix. Saat ini service jalan +> lokal di `localhost:8081`, belum bisa diakses dari luar. + +> 🔒 **Secret key** sudah kami generate (64 karakter hex, random 256-bit) dan +> tersimpan di `.env` service kami (`WEBHOOK_SECRET`). Kirim nilainya ke PIC +> Omnix lewat kanal aman (password manager, secret vault, atau chat terenkripsi +> — bukan email/Slack polos). Mereka pakai secret yang SAMA persis untuk +> menandatangani tiap request ke kami. + +## 3. Format payload — **dikonfirmasi tim Omnix** ✅ + +**Header:** +``` +X-Sopiga-Signature: +``` + +**Body:** +```json +{ + "recipient_detail_id": 1234, + "status": "sent", + "gateway": "628124878787" +} +``` + +| Field | Tipe | Keterangan | +|---|---|---| +| `recipient_detail_id` | integer | Sama dengan `recipient_detail_id` yang dikembalikan saat `add-recipient` | +| `status` | string | Salah satu: `pending`, `sent`, `delivered`, `read`, `failed` (atau `undelivered`) | +| `gateway` | string | Nomor WhatsApp pengirim (sender) | + +Sudah diuji end-to-end di sisi kami dengan payload persis seperti di atas — +`status: "sent"` diabaikan (belum final), `status: "delivered"`/`"read"` update +record jadi `delivered`, `status: "failed"`/`"undelivered"` update jadi `failed`. + +> Masih perlu dikonfirmasi: apakah header `X-Sopiga-Signature` (HMAC-SHA256) +> di atas juga dipakai Omnix, atau ada skema signature/auth lain di sisi mereka? + +## 4. Response yang kami kirim balik + +- `200 OK` — payload diterima & diproses. +- `400 Bad Request` — payload tidak valid/tidak bisa diparse. +- `401 Unauthorized` — signature tidak cocok. +- `500 Internal Server Error` — gagal proses di sisi kami (mohon di-retry). + +## 5. Fallback + +Selama callback belum aktif/terverifikasi, kami tetap jalankan polling manual +ke `GET /api/client/collar/add-recipient/{id}/detail` sebagai cadangan, jadi +tidak ada risiko data hilang selama masa transisi. diff --git a/framework/config/loader.go b/framework/config/loader.go new file mode 100644 index 0000000..8222615 --- /dev/null +++ b/framework/config/loader.go @@ -0,0 +1,75 @@ +package config + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// LoadDotEnv reads KEY=VALUE pairs from path (default ".env") and applies +// them via os.Setenv, without overriding variables already set in the +// environment. Missing file is not an error — real env vars always win. +func LoadDotEnv(path string) error { + if path == "" { + path = ".env" + } + + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.Trim(strings.TrimSpace(value), `"'`) + + if _, exists := os.LookupEnv(key); !exists { + os.Setenv(key, value) + } + } + + return scanner.Err() +} + +func GetString(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func GetInt(key string, fallback int) int { + v := os.Getenv(key) + if v == "" { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil { + return fallback + } + return n +} + +func MustGetString(key string) (string, error) { + v := os.Getenv(key) + if v == "" { + return "", fmt.Errorf("missing required environment variable: %s", key) + } + return v, nil +} diff --git a/framework/db/batch.go b/framework/db/batch.go new file mode 100644 index 0000000..08dede8 --- /dev/null +++ b/framework/db/batch.go @@ -0,0 +1,30 @@ +package db + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func ExecBatch(ctx context.Context, pool *pgxpool.Pool, stmt string, args [][]any) (int64, error) { + batch := &pgx.Batch{} + for _, a := range args { + batch.Queue(stmt, a...) + } + + results := pool.SendBatch(ctx, batch) + defer results.Close() + + var rowsAffected int64 + for i := 0; i < len(args); i++ { + tag, err := results.Exec() + if err != nil { + return rowsAffected, fmt.Errorf("exec batch item %d: %w", i, err) + } + rowsAffected += tag.RowsAffected() + } + + return rowsAffected, nil +} diff --git a/framework/db/pool.go b/framework/db/pool.go new file mode 100644 index 0000000..4e0e347 --- /dev/null +++ b/framework/db/pool.go @@ -0,0 +1,30 @@ +package db + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type Config struct { + DSN string + MaxConns int32 + MinConns int32 + MaxConnLifetime time.Duration + MaxConnIdleTime time.Duration +} + +func NewPool(ctx context.Context, cfg Config) (*pgxpool.Pool, error) { + poolCfg, err := pgxpool.ParseConfig(cfg.DSN) + if err != nil { + return nil, err + } + + poolCfg.MaxConns = cfg.MaxConns + poolCfg.MinConns = cfg.MinConns + poolCfg.MaxConnLifetime = cfg.MaxConnLifetime + poolCfg.MaxConnIdleTime = cfg.MaxConnIdleTime + + return pgxpool.NewWithConfig(ctx, poolCfg) +} diff --git a/framework/db/tx.go b/framework/db/tx.go new file mode 100644 index 0000000..16dcefd --- /dev/null +++ b/framework/db/tx.go @@ -0,0 +1,26 @@ +package db + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func WithTx(ctx context.Context, pool *pgxpool.Pool, fn func(ctx context.Context) error) error { + tx, err := pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + if err := fn(ctx); err != nil { + return err + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit tx: %w", err) + } + + return nil +} diff --git a/framework/go.mod b/framework/go.mod new file mode 100644 index 0000000..f589333 --- /dev/null +++ b/framework/go.mod @@ -0,0 +1,61 @@ +module github.com/yourorg/go-dw-platform/framework + +go 1.25.0 + +require ( + github.com/gin-gonic/gin v1.12.0 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.5.0 + github.com/prometheus/client_golang v1.17.0 + github.com/redis/go-redis/v9 v9.3.0 + go.opentelemetry.io/otel v1.44.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect +) diff --git a/framework/go.sum b/framework/go.sum new file mode 100644 index 0000000..b99962c --- /dev/null +++ b/framework/go.sum @@ -0,0 +1,146 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.0 h1:NxstgwndsTRy7eq9/kqYc/BZh5w2hHJV86wjvO+1xPw= +github.com/jackc/pgx/v5 v5.5.0/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= +github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/framework/ingestion/batcher.go b/framework/ingestion/batcher.go new file mode 100644 index 0000000..2b5776f --- /dev/null +++ b/framework/ingestion/batcher.go @@ -0,0 +1,62 @@ +package ingestion + +import ( + "sync" + "time" +) + +type Batcher[T any] struct { + mu sync.Mutex + items []T + size int + timeout time.Duration + createdAt time.Time +} + +func NewBatcher[T any](size int, timeout time.Duration) *Batcher[T] { + return &Batcher[T]{ + items: make([]T, 0, size), + size: size, + timeout: timeout, + createdAt: time.Now(), + } +} + +func (b *Batcher[T]) Add(item T) { + b.mu.Lock() + defer b.mu.Unlock() + + if len(b.items) == 0 { + b.createdAt = time.Now() + } + b.items = append(b.items, item) +} + +func (b *Batcher[T]) IsFull() bool { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.items) >= b.size +} + +func (b *Batcher[T]) IsExpired() bool { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.items) == 0 { + return false + } + return time.Since(b.createdAt) >= b.timeout +} + +func (b *Batcher[T]) Get() []T { + b.mu.Lock() + defer b.mu.Unlock() + items := make([]T, len(b.items)) + copy(items, b.items) + return items +} + +func (b *Batcher[T]) Clear() { + b.mu.Lock() + defer b.mu.Unlock() + b.items = b.items[:0] +} diff --git a/framework/ingestion/deduplicator.go b/framework/ingestion/deduplicator.go new file mode 100644 index 0000000..ce50e62 --- /dev/null +++ b/framework/ingestion/deduplicator.go @@ -0,0 +1,33 @@ +package ingestion + +import "sync" + +type Deduplicator struct { + mu sync.RWMutex + seen map[string]struct{} +} + +func NewDeduplicator() *Deduplicator { + return &Deduplicator{ + seen: make(map[string]struct{}), + } +} + +func (d *Deduplicator) Exists(key string) bool { + d.mu.RLock() + defer d.mu.RUnlock() + _, ok := d.seen[key] + return ok +} + +func (d *Deduplicator) Mark(key string) { + d.mu.Lock() + defer d.mu.Unlock() + d.seen[key] = struct{}{} +} + +func (d *Deduplicator) Reset() { + d.mu.Lock() + defer d.mu.Unlock() + d.seen = make(map[string]struct{}) +} diff --git a/framework/ingestion/retry.go b/framework/ingestion/retry.go new file mode 100644 index 0000000..cceb9fe --- /dev/null +++ b/framework/ingestion/retry.go @@ -0,0 +1,41 @@ +package ingestion + +import ( + "context" + "time" +) + +type Retrier struct { + MaxRetries int + BaseDelay time.Duration +} + +func NewRetrier(maxRetries int, baseDelay time.Duration) *Retrier { + return &Retrier{ + MaxRetries: maxRetries, + BaseDelay: baseDelay, + } +} + +func (r *Retrier) Do(ctx context.Context, fn func() (int64, error)) (int64, error) { + var lastErr error + + for attempt := 0; attempt <= r.MaxRetries; attempt++ { + if attempt > 0 { + delay := r.BaseDelay * time.Duration(1< MaxLimit { + return MaxLimit + } + return limit +} + +func Offset(page, limit int) int { + if page <= 1 { + return 0 + } + return (page - 1) * limit +} diff --git a/framework/query/cache.go b/framework/query/cache.go new file mode 100644 index 0000000..c292688 --- /dev/null +++ b/framework/query/cache.go @@ -0,0 +1,37 @@ +package query + +import ( + "context" + "encoding/json" + "time" + + "github.com/redis/go-redis/v9" +) + +type Cache struct { + client *redis.Client +} + +func NewCache(client *redis.Client) *Cache { + return &Cache{client: client} +} + +func (c *Cache) Get(ctx context.Context, key string, dest any) error { + val, err := c.client.Get(ctx, key).Bytes() + if err != nil { + return err + } + return json.Unmarshal(val, dest) +} + +func (c *Cache) Set(ctx context.Context, key string, value any, ttl time.Duration) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + return c.client.Set(ctx, key, data, ttl).Err() +} + +func (c *Cache) Invalidate(ctx context.Context, key string) error { + return c.client.Del(ctx, key).Err() +} diff --git a/go.work b/go.work new file mode 100644 index 0000000..d825664 --- /dev/null +++ b/go.work @@ -0,0 +1,7 @@ +go 1.25.0 + +use ( + ./framework + ./services/omnix-broadcast + ./services/template-service +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..ab61477 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,66 @@ +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/jordanlewis/gcassert v0.0.0-20250430164644-389ef753e22e/go.mod h1:ZybsQk6DWyN5t7An1MuPm1gtSZ1xDaTXS9ZjIOxvQrk= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/services/omnix-broadcast/.env.example b/services/omnix-broadcast/.env.example new file mode 100644 index 0000000..2006409 --- /dev/null +++ b/services/omnix-broadcast/.env.example @@ -0,0 +1,15 @@ +DATABASE_DSN=postgres://user:pass@localhost:5432/gadai_mulia +DB_MAX_CONNS=25 +DB_MIN_CONNS=5 + +SOPIGA_BASE_URL=https://omnix-dev.promas.site +SOPIGA_TOKEN= + +WORKER_CHECK_INTERVAL_SEC=30 +WORKER_BATCH_SIZE=100 +WORKER_MAX_RETRIES=3 +WORKER_SYNC_INTERVAL_SEC=300 +WORKER_SYNC_BATCH_SIZE=100 + +WEBHOOK_PORT=8081 +WEBHOOK_SECRET= diff --git a/services/omnix-broadcast/README.md b/services/omnix-broadcast/README.md new file mode 100644 index 0000000..f6c51e6 --- /dev/null +++ b/services/omnix-broadcast/README.md @@ -0,0 +1,91 @@ +# omnix-broadcast + +Collection broadcast worker: mengirim invoice via WhatsApp (Sopiga Collar API) ke nasabah Gadai Mulia. + +## Architecture + +``` +Gadai Collection Service + → INSERT broadcast_staging (message_payload JSONB, status=pending) + → BroadcastWorker (poll every 30s) + → Query template_variable_mapping + → Build message dynamically + → POST Sopiga Collar API + → status=dispatched (sopiga_recipient_detail_id disimpan) + → DeliverySyncWorker (poll every 5 min) + → GET recipient detail dari Sopiga + → status=delivered (delivered_at diisi) atau status=failed +``` + +Tiga komponen jalan bersamaan (lihat `main.go`): +- **BroadcastWorker** (`worker/broadcast_worker.go`) — poll `pending`, dispatch ke Sopiga. +- **Webhook receiver** (`handler/webhook.go`, `POST /webhooks/sopiga/delivery-status`) — + jalur utama update status delivery. Begitu Omnix push callback, status langsung + diupdate tanpa perlu polling balik ke Sopiga sama sekali. +- **DeliverySyncWorker** (`worker/delivery_sync_worker.go`) — polling **fallback/backstop** + untuk record `dispatched` yang tidak kunjung dapat webhook (mis. delivery gagal + terkirim/network hiccup di sisi Omnix). Karena webhook jadi jalur utama, interval ini + bisa diperlonggar jauh lebih besar dari 5 menit tanpa menambah beban signifikan ke Omnix. + +> ⚠️ **Catatan:** payload webhook di `dto/webhook.go` dan skema signature (`X-Sopiga-Signature`, +> HMAC-SHA256) masih **asumsi kita sendiri** — belum dikonfirmasi tim Omnix, karena endpoint +> registrasi callback tidak ditemukan di dokumentasi resmi (`/docs?api-docs.yaml`) saat +> implementasi ini dibuat. Sebelum pakai di production: (1) konfirmasi ke tim Omnix apakah +> mereka support outgoing webhook untuk status collar recipient, (2) minta format payload & +> skema signature asli mereka, (3) sesuaikan `dto/webhook.go` dan `handler/webhook.go`, +> (4) daftarkan URL `/webhooks/sopiga/delivery-status` ke mereka. + +Template baru cukup didaftarkan lewat database (`sopiga_template_config` + +`template_variable_mapping` + `sopiga_collar_config`) — tanpa redeploy kode. +Lihat [docs/implementasi_guide.md](../../docs/implementasi_guide.md) dan +[docs/quick_start.md](../../docs/quick_start.md) untuk panduan lengkap. + +## Layers + +| Layer | Responsibility | +|---|---| +| `worker/` | Poller loop untuk dispatch + delivery sync fallback, fan-out ke concurrent processing | +| `handler/` | `WebhookHandler` — HTTP receiver untuk callback status delivery dari Omnix | +| `transformer/` | Entity ↔ Domain mapping, `BuildDynamicMessage` interpolasi template | +| `service/` | Business logic: fetch pending/dispatched, validate, dispatch, sync delivery (polling & webhook), retry, mark failed | +| `repository/` | Query `broadcast_staging` & `template_variable_mapping`, update status, lookup by `recipient_detail_id` | +| `client/` | HTTP client ke Sopiga Collar API (add-recipient + get recipient detail) | +| `domain/` | `Broadcast`, `TemplateVariable` | +| `entity/` | DB row mapping | +| `dto/` | Sopiga API request/response | +| `validator/` | Validasi payload sebelum dispatch | +| `migrations/` | Schema `collection_broadcasts` (tables, views, functions) | + +## Configuration (env vars) + +| Var | Default | +|---|---| +| `DATABASE_DSN` | — | +| `DB_MAX_CONNS` | 25 | +| `DB_MIN_CONNS` | 5 | +| `SOPIGA_BASE_URL` | `https://omnix.promas.site` | +| `SOPIGA_TOKEN` | — | +| `WORKER_CHECK_INTERVAL_SEC` | 30 | +| `WORKER_BATCH_SIZE` | 100 | +| `WORKER_MAX_RETRIES` | 3 | +| `WORKER_SYNC_INTERVAL_SEC` | 300 (fallback saja — perlonggar kalau webhook sudah aktif) | +| `WORKER_SYNC_BATCH_SIZE` | 100 | +| `WEBHOOK_PORT` | 8081 | +| `WEBHOOK_SECRET` | — (HMAC-SHA256 shared secret; kosongkan untuk skip verifikasi saat dev) | + +## Run + +```bash +psql -d gadai_mulia -f migrations/001_create_collection_broadcasts.up.sql + +export DATABASE_DSN="postgres://user:pass@localhost:5432/gadai_mulia" +export SOPIGA_TOKEN="your_api_token" + +go run ./... +``` + +## Test + +```bash +go test ./tests/... -v +``` diff --git a/services/omnix-broadcast/client/sopiga.go b/services/omnix-broadcast/client/sopiga.go new file mode 100644 index 0000000..7735a23 --- /dev/null +++ b/services/omnix-broadcast/client/sopiga.go @@ -0,0 +1,107 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/dto" +) + +type SopigaClient struct { + baseURL string + token string + httpClient *http.Client +} + +func NewSopigaClient(baseURL, token string) *SopigaClient { + return &SopigaClient{ + baseURL: baseURL, + token: token, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +func (c *SopigaClient) AddRecipient(ctx context.Context, req dto.CollarAddRecipientRequest) (*dto.CollarAPIResponse, error) { + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + httpReq, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + fmt.Sprintf("%s/api/client/collar/add-recipient", c.baseURL), + bytes.NewReader(body), + ) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + httpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.token)) + httpReq.Header.Set("Content-Type", "application/json") + + httpResp, err := c.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http request: %w", err) + } + defer httpResp.Body.Close() + + bodyBytes, err := io.ReadAll(httpResp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + var resp dto.CollarAPIResponse + if err := json.Unmarshal(bodyBytes, &resp); err != nil { + return nil, fmt.Errorf("unmarshal response: %w", err) + } + + if httpResp.StatusCode != http.StatusAccepted && httpResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http status %d: %s", httpResp.StatusCode, resp.Message) + } + + return &resp, nil +} + +func (c *SopigaClient) GetRecipientDetail(ctx context.Context, recipientDetailID int64) (*dto.CollarRecipientDetailResponse, error) { + httpReq, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + fmt.Sprintf("%s/api/client/collar/add-recipient/%d/detail", c.baseURL, recipientDetailID), + nil, + ) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + httpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.token)) + + httpResp, err := c.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http request: %w", err) + } + defer httpResp.Body.Close() + + bodyBytes, err := io.ReadAll(httpResp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + var resp dto.CollarRecipientDetailResponse + if err := json.Unmarshal(bodyBytes, &resp); err != nil { + return nil, fmt.Errorf("unmarshal response: %w", err) + } + + if httpResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http status %d: %s", httpResp.StatusCode, resp.Message) + } + + return &resp, nil +} diff --git a/services/omnix-broadcast/config.go b/services/omnix-broadcast/config.go new file mode 100644 index 0000000..8337067 --- /dev/null +++ b/services/omnix-broadcast/config.go @@ -0,0 +1,69 @@ +package main + +import ( + "time" + + fwconfig "github.com/yourorg/go-dw-platform/framework/config" +) + +type Config struct { + DB DBConfig + Sopiga SopigaConfig + Worker WorkerConfig + Webhook WebhookConfig +} + +type DBConfig struct { + DSN string + MaxConns int32 + MinConns int32 + MaxConnLifetime time.Duration + MaxConnIdleTime time.Duration +} + +type SopigaConfig struct { + BaseURL string + Token string +} + +type WorkerConfig struct { + CheckInterval time.Duration + BatchSize int + MaxRetries int + RetryDelay time.Duration + SyncCheckInterval time.Duration + SyncBatchSize int +} + +type WebhookConfig struct { + Port string + Secret string +} + +func LoadConfig() *Config { + return &Config{ + DB: DBConfig{ + DSN: fwconfig.GetString("DATABASE_DSN", ""), + MaxConns: int32(fwconfig.GetInt("DB_MAX_CONNS", 25)), + MinConns: int32(fwconfig.GetInt("DB_MIN_CONNS", 5)), + MaxConnLifetime: 15 * time.Minute, + MaxConnIdleTime: 5 * time.Minute, + }, + Sopiga: SopigaConfig{ + BaseURL: fwconfig.GetString("SOPIGA_BASE_URL", "https://omnix.promas.site"), + Token: fwconfig.GetString("SOPIGA_TOKEN", ""), + }, + Worker: WorkerConfig{ + CheckInterval: time.Duration(fwconfig.GetInt("WORKER_CHECK_INTERVAL_SEC", 30)) * time.Second, + BatchSize: fwconfig.GetInt("WORKER_BATCH_SIZE", 100), + MaxRetries: fwconfig.GetInt("WORKER_MAX_RETRIES", 3), + RetryDelay: time.Second, + SyncCheckInterval: time.Duration(fwconfig.GetInt("WORKER_SYNC_INTERVAL_SEC", 300)) * time.Second, + SyncBatchSize: fwconfig.GetInt("WORKER_SYNC_BATCH_SIZE", 100), + }, + Webhook: WebhookConfig{ + Port: fwconfig.GetString("WEBHOOK_PORT", "8081"), + Secret: fwconfig.GetString("WEBHOOK_SECRET", ""), + }, + } +} diff --git a/services/omnix-broadcast/domain/broadcast.go b/services/omnix-broadcast/domain/broadcast.go new file mode 100644 index 0000000..5c44636 --- /dev/null +++ b/services/omnix-broadcast/domain/broadcast.go @@ -0,0 +1,63 @@ +package domain + +import "time" + +type BroadcastStatus string + +const ( + StatusPending BroadcastStatus = "pending" + StatusDispatched BroadcastStatus = "dispatched" + StatusDelivered BroadcastStatus = "delivered" + StatusFailed BroadcastStatus = "failed" + StatusRetryScheduled BroadcastStatus = "retry_scheduled" +) + +type Broadcast struct { + ID int64 + SopigaCollarID int + SopigaTemplateID int + MessagePayload map[string]any + Status BroadcastStatus + ErrorMessage *string + ErrorCount int + SopigaRecipientDetailID *int64 + CreatedAt time.Time + UpdatedAt time.Time +} + +func (b *Broadcast) RecipientPhone() (string, bool) { + v, ok := b.MessagePayload["nasabah_phone"].(string) + return v, ok +} + +func (b *Broadcast) InvoiceURL() (string, bool) { + v, ok := b.MessagePayload["invoice_pdf_url"].(string) + return v, ok +} + +func (b *Broadcast) CustomerName() string { + v, _ := b.MessagePayload["nasabah_nama"].(string) + return v +} + +// MapSopigaDeliveryStatus translates a Sopiga recipient status into our +// internal terminal status. ok is false while Sopiga still reports an +// in-flight state (e.g. "sent"), meaning there is nothing to update yet. +func MapSopigaDeliveryStatus(sopigaStatus string) (status BroadcastStatus, ok bool) { + switch sopigaStatus { + case "delivered", "read": + return StatusDelivered, true + case "failed", "undelivered": + return StatusFailed, true + default: // pending, sent, ... + return "", false + } +} + +type TemplateVariable struct { + VariableOrder int + VariableName string + VariableType string + FieldSource string + IsRequired bool +} diff --git a/services/omnix-broadcast/dto/sopiga.go b/services/omnix-broadcast/dto/sopiga.go new file mode 100644 index 0000000..de13a46 --- /dev/null +++ b/services/omnix-broadcast/dto/sopiga.go @@ -0,0 +1,54 @@ +package dto + +import "time" + +type CollarAddRecipientRequest struct { + BroadcastID int `json:"broadcast_id"` + TemplateID int `json:"template_id"` + Details CollarDetails `json:"details"` + Attachment *CollarAttachment `json:"attachment,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type CollarDetails struct { + Recipient string `json:"recipient"` + Message string `json:"message"` +} + +type CollarAttachment struct { + Type string `json:"type"` + Caption string `json:"caption"` + File string `json:"file"` +} + +type CollarAPIResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + Data CollarResponseData `json:"data"` + Errors map[string][]string `json:"errors,omitempty"` +} + +type CollarResponseData struct { + BroadcastID int `json:"broadcast_id"` + RecipientDetailID int64 `json:"recipient_detail_id"` + Recipient string `json:"recipient"` + Status string `json:"status"` +} + +type CollarRecipientDetailResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + Data CollarRecipientDetailData `json:"data"` +} + +type CollarRecipientDetailData struct { + ID int64 `json:"id"` + BroadcastID int `json:"broadcast_id"` + Recipient string `json:"recipient"` + TemplateID int `json:"template_id"` + Status string `json:"status"` // pending, sent, delivered, read, failed + ErrorMessage *string `json:"error_message"` + SentAt *time.Time `json:"sent_at"` + DeliveredAt *time.Time `json:"delivered_at"` + ReadAt *time.Time `json:"read_at"` +} diff --git a/services/omnix-broadcast/dto/webhook.go b/services/omnix-broadcast/dto/webhook.go new file mode 100644 index 0000000..3432436 --- /dev/null +++ b/services/omnix-broadcast/dto/webhook.go @@ -0,0 +1,9 @@ +package dto + +// DeliveryStatusWebhook is the confirmed contract for an inbound Omnix +// callback when a recipient's delivery status changes. +type DeliveryStatusWebhook struct { + RecipientDetailID int64 `json:"recipient_detail_id"` + Status string `json:"status"` // pending, sent, delivered, read, failed + Gateway string `json:"gateway"` // sender WhatsApp number +} diff --git a/services/omnix-broadcast/entity/broadcast.go b/services/omnix-broadcast/entity/broadcast.go new file mode 100644 index 0000000..49a8e89 --- /dev/null +++ b/services/omnix-broadcast/entity/broadcast.go @@ -0,0 +1,24 @@ +package entity + +import "time" + +type BroadcastStaging struct { + ID int64 `db:"id"` + SopigaCollarID int `db:"sopiga_collar_id"` + SopigaTemplateID int `db:"sopiga_template_id"` + MessagePayload []byte `db:"message_payload"` + Status string `db:"status"` + ErrorMessage *string `db:"error_message"` + ErrorCount int `db:"error_count"` + SopigaRecipientDetailID *int64 `db:"sopiga_recipient_detail_id"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +type TemplateVariableMapping struct { + VariableOrder int `db:"variable_order"` + SopigaVariableName string `db:"sopiga_variable_name"` + VariableType string `db:"variable_type"` + DBFieldSource string `db:"db_field_source"` + IsRequired bool `db:"is_required"` +} diff --git a/services/omnix-broadcast/go.mod b/services/omnix-broadcast/go.mod new file mode 100644 index 0000000..b30ef01 --- /dev/null +++ b/services/omnix-broadcast/go.mod @@ -0,0 +1,20 @@ +module github.com/yourorg/go-dw-platform/services/omnix-broadcast + +go 1.21 + +require ( + github.com/jackc/pgx/v5 v5.5.0 + github.com/yourorg/go-dw-platform/framework v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/stretchr/testify v1.8.3 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/text v0.9.0 // indirect +) + +replace github.com/yourorg/go-dw-platform/framework => ../../framework diff --git a/services/omnix-broadcast/go.sum b/services/omnix-broadcast/go.sum new file mode 100644 index 0000000..2101f14 --- /dev/null +++ b/services/omnix-broadcast/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgx/v5 v5.5.0 h1:NxstgwndsTRy7eq9/kqYc/BZh5w2hHJV86wjvO+1xPw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/services/omnix-broadcast/handler/webhook.go b/services/omnix-broadcast/handler/webhook.go new file mode 100644 index 0000000..d91da63 --- /dev/null +++ b/services/omnix-broadcast/handler/webhook.go @@ -0,0 +1,82 @@ +package handler + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "io" + "net/http" + + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/dto" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/service" +) + +const signatureHeader = "X-Sopiga-Signature" + +// WebhookHandler receives delivery-status callbacks pushed by Omnix, +// as an alternative to polling. Payload shape confirmed by the Omnix team +// (see dto.DeliveryStatusWebhook); the signature scheme is still our own +// proposal pending their confirmation. +type WebhookHandler struct { + service *service.Service + logger *fwlogger.Logger + secret string +} + +func NewWebhookHandler(svc *service.Service, log *fwlogger.Logger, secret string) *WebhookHandler { + return &WebhookHandler{service: svc, logger: log, secret: secret} +} + +func (h *WebhookHandler) HandleDeliveryStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + body, err := io.ReadAll(r.Body) + if err != nil { + h.logger.Error("webhook: read body failed", "error", err) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + if h.secret != "" && !h.verifySignature(body, r.Header.Get(signatureHeader)) { + h.logger.Error("webhook: signature verification failed") + w.WriteHeader(http.StatusUnauthorized) + return + } + + var payload dto.DeliveryStatusWebhook + if err := json.Unmarshal(body, &payload); err != nil { + h.logger.Error("webhook: invalid payload", "error", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + if err := h.service.SyncDeliveryFromWebhook(r.Context(), payload); err != nil { + h.logger.Error("webhook: sync delivery failed", + "recipient_detail_id", payload.RecipientDetailID, + "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + h.logger.Info("webhook: delivery status applied", + "recipient_detail_id", payload.RecipientDetailID, + "status", payload.Status, + "gateway", payload.Gateway) + + w.WriteHeader(http.StatusOK) +} + +func (h *WebhookHandler) verifySignature(body []byte, signature string) bool { + mac := hmac.New(sha256.New, []byte(h.secret)) + mac.Write(body) + expected := hex.EncodeToString(mac.Sum(nil)) + return subtle.ConstantTimeCompare([]byte(expected), []byte(signature)) == 1 +} diff --git a/services/omnix-broadcast/main.go b/services/omnix-broadcast/main.go new file mode 100644 index 0000000..159c206 --- /dev/null +++ b/services/omnix-broadcast/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + fwconfig "github.com/yourorg/go-dw-platform/framework/config" + fwdb "github.com/yourorg/go-dw-platform/framework/db" + fwingestion "github.com/yourorg/go-dw-platform/framework/ingestion" + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/client" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/handler" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/repository" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/service" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/transformer" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/worker" +) + +func main() { + _ = fwconfig.LoadDotEnv(".env") + cfg := LoadConfig() + log := fwlogger.New("omnix-broadcast") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pool, err := fwdb.NewPool(ctx, fwdb.Config{ + DSN: cfg.DB.DSN, + MaxConns: cfg.DB.MaxConns, + MinConns: cfg.DB.MinConns, + MaxConnLifetime: cfg.DB.MaxConnLifetime, + MaxConnIdleTime: cfg.DB.MaxConnIdleTime, + }) + if err != nil { + log.Error("failed to connect to database", "error", err) + os.Exit(1) + } + defer pool.Close() + + repo := repository.New(pool) + tf := transformer.New() + sopigaClient := client.NewSopigaClient(cfg.Sopiga.BaseURL, cfg.Sopiga.Token) + retrier := fwingestion.NewRetrier(cfg.Worker.MaxRetries, cfg.Worker.RetryDelay) + + svc := service.New(repo, tf, sopigaClient, retrier, log) + bcWorker := worker.New(svc, log, cfg.Worker.CheckInterval, cfg.Worker.BatchSize) + syncWorker := worker.NewDeliverySync(svc, log, cfg.Worker.SyncCheckInterval, cfg.Worker.SyncBatchSize) + + go bcWorker.Start(ctx) + go syncWorker.Start(ctx) + + webhookHandler := handler.NewWebhookHandler(svc, log, cfg.Webhook.Secret) + mux := http.NewServeMux() + mux.HandleFunc("/webhooks/sopiga/delivery-status", webhookHandler.HandleDeliveryStatus) + + httpServer := &http.Server{ + Addr: ":" + cfg.Webhook.Port, + Handler: mux, + } + + go func() { + log.Info("webhook server listening", "port", cfg.Webhook.Port) + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("webhook server stopped", "error", err) + } + }() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + <-sigChan + + log.Info("shutting down gracefully") + cancel() + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + _ = httpServer.Shutdown(shutdownCtx) +} diff --git a/services/omnix-broadcast/migrations/001_create_collection_broadcasts.down.sql b/services/omnix-broadcast/migrations/001_create_collection_broadcasts.down.sql new file mode 100644 index 0000000..9708a8b --- /dev/null +++ b/services/omnix-broadcast/migrations/001_create_collection_broadcasts.down.sql @@ -0,0 +1,17 @@ +DROP FUNCTION IF EXISTS collection_broadcasts.update_broadcast_status(BIGINT, VARCHAR, TEXT, JSONB); +DROP FUNCTION IF EXISTS collection_broadcasts.get_template_variables(INT); + +DROP VIEW IF EXISTS collection_broadcasts.v_delivery_rate_24h; +DROP VIEW IF EXISTS collection_broadcasts.v_failed_records_24h; +DROP VIEW IF EXISTS collection_broadcasts.v_collar_summary; +DROP VIEW IF EXISTS collection_broadcasts.v_template_variables_ordered; + +DROP TABLE IF EXISTS collection_broadcasts.sopiga_sync_job; +DROP TABLE IF EXISTS collection_broadcasts.broadcast_error_log; +DROP TABLE IF EXISTS collection_broadcasts.broadcast_audit_log; +DROP TABLE IF EXISTS collection_broadcasts.broadcast_staging; +DROP TABLE IF EXISTS collection_broadcasts.sopiga_collar_config; +DROP TABLE IF EXISTS collection_broadcasts.template_variable_mapping; +DROP TABLE IF EXISTS collection_broadcasts.sopiga_template_config; + +DROP SCHEMA IF EXISTS collection_broadcasts; diff --git a/services/omnix-broadcast/migrations/001_create_collection_broadcasts.up.sql b/services/omnix-broadcast/migrations/001_create_collection_broadcasts.up.sql new file mode 100644 index 0000000..3b08b3c --- /dev/null +++ b/services/omnix-broadcast/migrations/001_create_collection_broadcasts.up.sql @@ -0,0 +1,258 @@ +CREATE SCHEMA IF NOT EXISTS collection_broadcasts; + +-- ============================================================================ +-- 1. CONFIGURATION TABLES (Reference only) +-- ============================================================================ + +CREATE TABLE collection_broadcasts.sopiga_template_config ( + id BIGSERIAL PRIMARY KEY, + template_name VARCHAR(100) NOT NULL, + sopiga_template_id INT NOT NULL UNIQUE, + template_type VARCHAR(50), + channel VARCHAR(50), + description TEXT, + active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_sopiga_template_active ON collection_broadcasts.sopiga_template_config(active); + +CREATE TABLE collection_broadcasts.template_variable_mapping ( + id BIGSERIAL PRIMARY KEY, + sopiga_template_id INT NOT NULL REFERENCES collection_broadcasts.sopiga_template_config(sopiga_template_id) ON DELETE CASCADE, + variable_order INT NOT NULL, + sopiga_variable_name VARCHAR(100) NOT NULL, + variable_type VARCHAR(50), + db_field_source VARCHAR(100) NOT NULL, + is_required BOOLEAN DEFAULT TRUE, + example_value VARCHAR(500), + description TEXT, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + + CONSTRAINT unique_template_variable UNIQUE (sopiga_template_id, sopiga_variable_name), + CONSTRAINT unique_variable_order UNIQUE (sopiga_template_id, variable_order) +); + +CREATE INDEX idx_template_variable_mapping_template_id ON collection_broadcasts.template_variable_mapping(sopiga_template_id); +CREATE INDEX idx_template_variable_mapping_order ON collection_broadcasts.template_variable_mapping(sopiga_template_id, variable_order); + +CREATE TABLE collection_broadcasts.sopiga_collar_config ( + id BIGSERIAL PRIMARY KEY, + broadcast_name VARCHAR(100) NOT NULL, + sopiga_collar_id INT NOT NULL UNIQUE, + sopiga_template_id INT NOT NULL REFERENCES collection_broadcasts.sopiga_template_config(sopiga_template_id), + status VARCHAR(50), + description TEXT, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_sopiga_collar_status ON collection_broadcasts.sopiga_collar_config(status); + +-- ============================================================================ +-- 2. BROADCAST STAGING (CORE) - DENORMALIZED +-- ============================================================================ + +CREATE TABLE collection_broadcasts.broadcast_staging ( + id BIGSERIAL PRIMARY KEY, + + sopiga_collar_id INT NOT NULL REFERENCES collection_broadcasts.sopiga_collar_config(sopiga_collar_id), + sopiga_template_id INT NOT NULL REFERENCES collection_broadcasts.sopiga_template_config(sopiga_template_id), + + -- Single source of truth: all fields needed to build the message live here + message_payload JSONB NOT NULL, + + status VARCHAR(50) DEFAULT 'pending' NOT NULL, + error_message TEXT, + error_count INT DEFAULT 0, + + sopiga_recipient_detail_id BIGINT, + sopiga_external_id VARCHAR(100), + + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + dispatched_at TIMESTAMP, + delivered_at TIMESTAMP, + failed_at TIMESTAMP, + + CONSTRAINT status_valid CHECK (status IN ('pending', 'dispatched', 'delivered', 'failed', 'retry_scheduled')), + CONSTRAINT error_count_positive CHECK (error_count >= 0) +); + +CREATE INDEX idx_broadcast_staging_status ON collection_broadcasts.broadcast_staging(status); +CREATE INDEX idx_broadcast_staging_created_at ON collection_broadcasts.broadcast_staging(created_at DESC); +CREATE INDEX idx_broadcast_staging_collar_id ON collection_broadcasts.broadcast_staging(sopiga_collar_id); +CREATE INDEX idx_broadcast_staging_template_id ON collection_broadcasts.broadcast_staging(sopiga_template_id); + +CREATE INDEX idx_broadcast_staging_pending_query + ON collection_broadcasts.broadcast_staging(status, created_at ASC) + WHERE status = 'pending'; + +CREATE INDEX idx_broadcast_staging_retry_query + ON collection_broadcasts.broadcast_staging(status, error_count, created_at ASC) + WHERE status = 'retry_scheduled' AND error_count < 3; + +-- ============================================================================ +-- 3. AUDIT & LOGGING +-- ============================================================================ + +CREATE TABLE collection_broadcasts.broadcast_audit_log ( + id BIGSERIAL PRIMARY KEY, + broadcast_id BIGINT NOT NULL REFERENCES collection_broadcasts.broadcast_staging(id) ON DELETE CASCADE, + old_status VARCHAR(50), + new_status VARCHAR(50) NOT NULL, + reason VARCHAR(500), + sopiga_response JSONB, + changed_by VARCHAR(100) DEFAULT 'system', + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_broadcast_audit_log_broadcast_id ON collection_broadcasts.broadcast_audit_log(broadcast_id); +CREATE INDEX idx_broadcast_audit_log_created_at ON collection_broadcasts.broadcast_audit_log(created_at DESC); + +CREATE TABLE collection_broadcasts.broadcast_error_log ( + id BIGSERIAL PRIMARY KEY, + broadcast_id BIGINT NOT NULL REFERENCES collection_broadcasts.broadcast_staging(id) ON DELETE CASCADE, + error_type VARCHAR(100), + error_code VARCHAR(50), + error_message TEXT, + error_details JSONB, + attempt_number INT, + next_retry_at TIMESTAMP, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_broadcast_error_log_broadcast_id ON collection_broadcasts.broadcast_error_log(broadcast_id); +CREATE INDEX idx_broadcast_error_log_error_type ON collection_broadcasts.broadcast_error_log(error_type); + +CREATE TABLE collection_broadcasts.sopiga_sync_job ( + id BIGSERIAL PRIMARY KEY, + broadcast_id BIGINT NOT NULL REFERENCES collection_broadcasts.broadcast_staging(id) ON DELETE CASCADE, + sopiga_recipient_detail_id BIGINT, + last_synced_at TIMESTAMP, + last_status_from_sopiga VARCHAR(50), + sync_count INT DEFAULT 0, + next_sync_at TIMESTAMP, + completed_at TIMESTAMP, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_sopiga_sync_job_broadcast_id ON collection_broadcasts.sopiga_sync_job(broadcast_id); +CREATE INDEX idx_sopiga_sync_job_next_sync_at ON collection_broadcasts.sopiga_sync_job(next_sync_at); + +-- ============================================================================ +-- 4. VIEWS +-- ============================================================================ + +CREATE VIEW collection_broadcasts.v_template_variables_ordered AS +SELECT + tvm.sopiga_template_id, + stc.template_name, + tvm.variable_order, + tvm.sopiga_variable_name, + tvm.variable_type, + tvm.db_field_source, + tvm.is_required, + tvm.example_value +FROM collection_broadcasts.template_variable_mapping tvm +JOIN collection_broadcasts.sopiga_template_config stc ON tvm.sopiga_template_id = stc.sopiga_template_id +WHERE stc.active = TRUE +ORDER BY tvm.sopiga_template_id, tvm.variable_order; + +CREATE VIEW collection_broadcasts.v_collar_summary AS +SELECT + sc.sopiga_collar_id, + sc.broadcast_name, + COUNT(*) as total_records, + COUNT(CASE WHEN bs.status = 'pending' THEN 1 END) as pending, + COUNT(CASE WHEN bs.status = 'dispatched' THEN 1 END) as dispatched, + COUNT(CASE WHEN bs.status = 'delivered' THEN 1 END) as delivered, + COUNT(CASE WHEN bs.status = 'failed' THEN 1 END) as failed, + ROUND(100.0 * COUNT(CASE WHEN bs.status = 'delivered' THEN 1 END) / NULLIF(COUNT(*), 0), 2) as delivery_rate_percent +FROM collection_broadcasts.sopiga_collar_config sc +LEFT JOIN collection_broadcasts.broadcast_staging bs ON sc.sopiga_collar_id = bs.sopiga_collar_id +GROUP BY sc.sopiga_collar_id, sc.broadcast_name; + +CREATE VIEW collection_broadcasts.v_failed_records_24h AS +SELECT + id, + sopiga_collar_id, + sopiga_template_id, + message_payload->>'nasabah_nama' as nasabah_nama, + message_payload->>'nasabah_phone' as nasabah_phone, + error_message, + error_count, + failed_at, + created_at +FROM collection_broadcasts.broadcast_staging +WHERE status = 'failed' AND created_at > NOW() - INTERVAL '1 DAY' +ORDER BY failed_at DESC; + +CREATE VIEW collection_broadcasts.v_delivery_rate_24h AS +SELECT + COUNT(*) as total, + COUNT(CASE WHEN status = 'delivered' THEN 1 END) as delivered, + COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed, + COUNT(CASE WHEN status = 'dispatched' THEN 1 END) as in_progress, + ROUND(100.0 * COUNT(CASE WHEN status = 'delivered' THEN 1 END) / NULLIF(COUNT(*), 0), 2) as delivery_rate_percent +FROM collection_broadcasts.broadcast_staging +WHERE created_at > NOW() - INTERVAL '1 DAY'; + +-- ============================================================================ +-- 5. HELPER FUNCTIONS +-- ============================================================================ + +CREATE OR REPLACE FUNCTION collection_broadcasts.get_template_variables( + p_sopiga_template_id INT +) +RETURNS TABLE( + variable_order INT, + sopiga_variable_name VARCHAR, + variable_type VARCHAR, + db_field_source VARCHAR, + is_required BOOLEAN +) AS $$ +BEGIN + RETURN QUERY + SELECT + tvm.variable_order, + tvm.sopiga_variable_name, + tvm.variable_type, + tvm.db_field_source, + tvm.is_required + FROM collection_broadcasts.template_variable_mapping tvm + WHERE tvm.sopiga_template_id = p_sopiga_template_id + ORDER BY tvm.variable_order ASC; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION collection_broadcasts.update_broadcast_status( + p_broadcast_id BIGINT, + p_new_status VARCHAR, + p_error_message TEXT DEFAULT NULL, + p_sopiga_response JSONB DEFAULT NULL +) +RETURNS VOID AS $$ +DECLARE + v_old_status VARCHAR; +BEGIN + SELECT status INTO v_old_status + FROM collection_broadcasts.broadcast_staging + WHERE id = p_broadcast_id; + + UPDATE collection_broadcasts.broadcast_staging + SET + status = p_new_status, + error_message = p_error_message, + updated_at = NOW(), + dispatched_at = CASE WHEN p_new_status = 'dispatched' THEN NOW() ELSE dispatched_at END, + delivered_at = CASE WHEN p_new_status = 'delivered' THEN NOW() ELSE delivered_at END, + failed_at = CASE WHEN p_new_status = 'failed' THEN NOW() ELSE failed_at END + WHERE id = p_broadcast_id; + + INSERT INTO collection_broadcasts.broadcast_audit_log (broadcast_id, old_status, new_status, sopiga_response) + VALUES (p_broadcast_id, v_old_status, p_new_status, p_sopiga_response); +END; +$$ LANGUAGE plpgsql; diff --git a/services/omnix-broadcast/repository/broadcast.go b/services/omnix-broadcast/repository/broadcast.go new file mode 100644 index 0000000..b29c22f --- /dev/null +++ b/services/omnix-broadcast/repository/broadcast.go @@ -0,0 +1,171 @@ +package repository + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/entity" +) + +type Repository struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Repository { + return &Repository{pool: pool} +} + +func (r *Repository) FindPending(ctx context.Context, limit int) ([]*entity.BroadcastStaging, error) { + query := ` + SELECT + id, sopiga_collar_id, sopiga_template_id, message_payload, + status, error_message, error_count, sopiga_recipient_detail_id, + created_at, updated_at + FROM collection_broadcasts.broadcast_staging + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT $1 + ` + + rows, err := r.pool.Query(ctx, query, limit) + if err != nil { + return nil, fmt.Errorf("find pending: %w", err) + } + defer rows.Close() + + var records []*entity.BroadcastStaging + for rows.Next() { + var e entity.BroadcastStaging + if err := rows.Scan( + &e.ID, &e.SopigaCollarID, &e.SopigaTemplateID, &e.MessagePayload, + &e.Status, &e.ErrorMessage, &e.ErrorCount, &e.SopigaRecipientDetailID, + &e.CreatedAt, &e.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + records = append(records, &e) + } + + return records, rows.Err() +} + +func (r *Repository) FindTemplateVariables(ctx context.Context, templateID int) ([]*entity.TemplateVariableMapping, error) { + query := ` + SELECT variable_order, sopiga_variable_name, variable_type, db_field_source, is_required + FROM collection_broadcasts.template_variable_mapping + WHERE sopiga_template_id = $1 + ORDER BY variable_order ASC + ` + + rows, err := r.pool.Query(ctx, query, templateID) + if err != nil { + return nil, fmt.Errorf("find template variables: %w", err) + } + defer rows.Close() + + var variables []*entity.TemplateVariableMapping + for rows.Next() { + var v entity.TemplateVariableMapping + if err := rows.Scan(&v.VariableOrder, &v.SopigaVariableName, &v.VariableType, &v.DBFieldSource, &v.IsRequired); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + variables = append(variables, &v) + } + + return variables, rows.Err() +} + +func (r *Repository) FindByRecipientDetailID(ctx context.Context, recipientDetailID int64) (*entity.BroadcastStaging, error) { + query := ` + SELECT + id, sopiga_collar_id, sopiga_template_id, message_payload, + status, error_message, error_count, sopiga_recipient_detail_id, + created_at, updated_at + FROM collection_broadcasts.broadcast_staging + WHERE sopiga_recipient_detail_id = $1 + ` + + var e entity.BroadcastStaging + err := r.pool.QueryRow(ctx, query, recipientDetailID).Scan( + &e.ID, &e.SopigaCollarID, &e.SopigaTemplateID, &e.MessagePayload, + &e.Status, &e.ErrorMessage, &e.ErrorCount, &e.SopigaRecipientDetailID, + &e.CreatedAt, &e.UpdatedAt, + ) + if err != nil { + return nil, fmt.Errorf("find by recipient_detail_id %d: %w", recipientDetailID, err) + } + + return &e, nil +} + +func (r *Repository) UpdateStatus(ctx context.Context, id int64, status, errMsg string, sopigaResponse any) error { + var respJSON []byte + if sopigaResponse != nil { + var err error + respJSON, err = json.Marshal(sopigaResponse) + if err != nil { + return fmt.Errorf("marshal sopiga response: %w", err) + } + } + + query := `SELECT collection_broadcasts.update_broadcast_status($1, $2, $3, $4)` + _, err := r.pool.Exec(ctx, query, id, status, errMsg, respJSON) + if err != nil { + return fmt.Errorf("update status: %w", err) + } + return nil +} + +func (r *Repository) FindDispatched(ctx context.Context, limit int) ([]*entity.BroadcastStaging, error) { + query := ` + SELECT + id, sopiga_collar_id, sopiga_template_id, message_payload, + status, error_message, error_count, sopiga_recipient_detail_id, + created_at, updated_at + FROM collection_broadcasts.broadcast_staging + WHERE status = 'dispatched' AND sopiga_recipient_detail_id IS NOT NULL + ORDER BY dispatched_at ASC + LIMIT $1 + ` + + rows, err := r.pool.Query(ctx, query, limit) + if err != nil { + return nil, fmt.Errorf("find dispatched: %w", err) + } + defer rows.Close() + + var records []*entity.BroadcastStaging + for rows.Next() { + var e entity.BroadcastStaging + if err := rows.Scan( + &e.ID, &e.SopigaCollarID, &e.SopigaTemplateID, &e.MessagePayload, + &e.Status, &e.ErrorMessage, &e.ErrorCount, &e.SopigaRecipientDetailID, + &e.CreatedAt, &e.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + records = append(records, &e) + } + + return records, rows.Err() +} + +func (r *Repository) UpdateDispatched(ctx context.Context, id int64, recipientDetailID int64) error { + query := ` + UPDATE collection_broadcasts.broadcast_staging + SET + status = 'dispatched', + sopiga_recipient_detail_id = $1, + dispatched_at = NOW(), + updated_at = NOW() + WHERE id = $2 + ` + _, err := r.pool.Exec(ctx, query, recipientDetailID, id) + if err != nil { + return fmt.Errorf("update dispatched: %w", err) + } + return nil +} diff --git a/services/omnix-broadcast/service/broadcast.go b/services/omnix-broadcast/service/broadcast.go new file mode 100644 index 0000000..3396635 --- /dev/null +++ b/services/omnix-broadcast/service/broadcast.go @@ -0,0 +1,205 @@ +package service + +import ( + "context" + "fmt" + + fwingestion "github.com/yourorg/go-dw-platform/framework/ingestion" + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/client" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/dto" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/repository" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/transformer" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/validator" +) + +type Service struct { + repository *repository.Repository + transformer *transformer.Transformer + sopiga *client.SopigaClient + retrier *fwingestion.Retrier + logger *fwlogger.Logger +} + +func New( + repo *repository.Repository, + tf *transformer.Transformer, + sopigaClient *client.SopigaClient, + retrier *fwingestion.Retrier, + log *fwlogger.Logger, +) *Service { + return &Service{ + repository: repo, + transformer: tf, + sopiga: sopigaClient, + retrier: retrier, + logger: log, + } +} + +// FetchPending returns the next batch of broadcasts waiting to be dispatched. +func (s *Service) FetchPending(ctx context.Context, limit int) ([]*domain.Broadcast, error) { + entities, err := s.repository.FindPending(ctx, limit) + if err != nil { + return nil, fmt.Errorf("fetch pending: %w", err) + } + + broadcasts := make([]*domain.Broadcast, 0, len(entities)) + for _, e := range entities { + b, err := s.transformer.EntityToDomain(e) + if err != nil { + s.logger.Error("skip malformed record", "broadcast_id", e.ID, "error", err) + continue + } + broadcasts = append(broadcasts, b) + } + + return broadcasts, nil +} + +// Dispatch builds the templated message for a broadcast and sends it to Sopiga, +// updating status in the database based on the outcome. +func (s *Service) Dispatch(ctx context.Context, b *domain.Broadcast) error { + if err := validator.ValidateBroadcastPayload(b); err != nil { + s.markFailed(ctx, b.ID, err) + return err + } + + variableEntities, err := s.repository.FindTemplateVariables(ctx, b.SopigaTemplateID) + if err != nil { + s.markFailed(ctx, b.ID, err) + return fmt.Errorf("dispatch %d: %w", b.ID, err) + } + + variables := make([]*domain.TemplateVariable, len(variableEntities)) + for i, ve := range variableEntities { + variables[i] = s.transformer.EntityToDomainVariable(ve) + } + + message, err := s.transformer.BuildDynamicMessage(variables, b.MessagePayload) + if err != nil { + s.markFailed(ctx, b.ID, err) + return fmt.Errorf("dispatch %d: %w", b.ID, err) + } + + recipientPhone, _ := b.RecipientPhone() + invoiceURL, _ := b.InvoiceURL() + + req := dto.CollarAddRecipientRequest{ + BroadcastID: b.SopigaCollarID, + TemplateID: b.SopigaTemplateID, + Details: dto.CollarDetails{ + Recipient: recipientPhone, + Message: message, + }, + Attachment: &dto.CollarAttachment{ + Type: "document", + Caption: fmt.Sprintf("Invoice - %s", b.CustomerName()), + File: invoiceURL, + }, + Labels: map[string]string{"module": "gadai_collection"}, + } + + resp, err := s.retrier.Do(ctx, func() (int64, error) { + r, err := s.sopiga.AddRecipient(ctx, req) + if err != nil { + return 0, err + } + if !r.Success { + return 0, fmt.Errorf("sopiga error: %s", r.Message) + } + return r.Data.RecipientDetailID, nil + }) + if err != nil { + s.markFailed(ctx, b.ID, err) + return fmt.Errorf("dispatch %d: %w", b.ID, err) + } + + if err := s.repository.UpdateDispatched(ctx, b.ID, resp); err != nil { + return fmt.Errorf("dispatch %d: %w", b.ID, err) + } + + return nil +} + +// FetchDispatched returns broadcasts awaiting a delivery status update from Sopiga. +func (s *Service) FetchDispatched(ctx context.Context, limit int) ([]*domain.Broadcast, error) { + entities, err := s.repository.FindDispatched(ctx, limit) + if err != nil { + return nil, fmt.Errorf("fetch dispatched: %w", err) + } + + broadcasts := make([]*domain.Broadcast, 0, len(entities)) + for _, e := range entities { + b, err := s.transformer.EntityToDomain(e) + if err != nil { + s.logger.Error("skip malformed record", "broadcast_id", e.ID, "error", err) + continue + } + broadcasts = append(broadcasts, b) + } + + return broadcasts, nil +} + +// SyncDelivery checks the current delivery status of a dispatched broadcast +// with Sopiga and, if it has reached a terminal state, updates our record. +func (s *Service) SyncDelivery(ctx context.Context, b *domain.Broadcast) error { + if b.SopigaRecipientDetailID == nil { + return fmt.Errorf("sync delivery %d: missing recipient_detail_id", b.ID) + } + + resp, err := s.sopiga.GetRecipientDetail(ctx, *b.SopigaRecipientDetailID) + if err != nil { + return fmt.Errorf("sync delivery %d: %w", b.ID, err) + } + + status, ok := domain.MapSopigaDeliveryStatus(resp.Data.Status) + if !ok { + return nil // still in-flight (e.g. "sent"), nothing to update yet + } + + errMsg := "" + if resp.Data.ErrorMessage != nil { + errMsg = *resp.Data.ErrorMessage + } + + if err := s.repository.UpdateStatus(ctx, b.ID, string(status), errMsg, resp.Data); err != nil { + return fmt.Errorf("sync delivery %d: %w", b.ID, err) + } + + return nil +} + +// SyncDeliveryFromWebhook applies a delivery status pushed by an Omnix +// callback, avoiding the extra GET round-trip the polling sync worker needs. +func (s *Service) SyncDeliveryFromWebhook(ctx context.Context, payload dto.DeliveryStatusWebhook) error { + e, err := s.repository.FindByRecipientDetailID(ctx, payload.RecipientDetailID) + if err != nil { + return fmt.Errorf("sync delivery from webhook: %w", err) + } + + status, ok := domain.MapSopigaDeliveryStatus(payload.Status) + if !ok { + return nil // in-flight status (e.g. "sent"), nothing to update yet + } + + errMsg := "" + if status == domain.StatusFailed { + errMsg = fmt.Sprintf("delivery failed via gateway %s", payload.Gateway) + } + + if err := s.repository.UpdateStatus(ctx, e.ID, string(status), errMsg, payload); err != nil { + return fmt.Errorf("sync delivery from webhook %d: %w", e.ID, err) + } + + return nil +} + +func (s *Service) markFailed(ctx context.Context, id int64, cause error) { + if err := s.repository.UpdateStatus(ctx, id, string(domain.StatusFailed), cause.Error(), nil); err != nil { + s.logger.Error("failed to mark broadcast failed", "broadcast_id", id, "error", err) + } +} diff --git a/services/omnix-broadcast/tests/transformer_test.go b/services/omnix-broadcast/tests/transformer_test.go new file mode 100644 index 0000000..a0013a0 --- /dev/null +++ b/services/omnix-broadcast/tests/transformer_test.go @@ -0,0 +1,47 @@ +package tests + +import ( + "testing" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/transformer" +) + +func TestBuildDynamicMessage_Ordered(t *testing.T) { + tf := transformer.New() + + variables := []*domain.TemplateVariable{ + {VariableOrder: 1, VariableName: "Nama", VariableType: "string", FieldSource: "nasabah_nama", IsRequired: true}, + {VariableOrder: 2, VariableName: "TotalTagihan", VariableType: "integer", FieldSource: "nominal_tagihan", IsRequired: true}, + {VariableOrder: 3, VariableName: "TanggalJatuhTempo", VariableType: "date", FieldSource: "tanggal_tempo", IsRequired: true}, + } + + payload := map[string]any{ + "nasabah_nama": "Budi Santoso", + "nominal_tagihan": 1500000.0, + "tanggal_tempo": "2026-06-30", + } + + message, err := tf.BuildDynamicMessage(variables, payload) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + want := "Budi Santoso#1500000#2026-06-30" + if message != want { + t.Fatalf("expected %q, got %q", want, message) + } +} + +func TestBuildDynamicMessage_MissingRequiredField(t *testing.T) { + tf := transformer.New() + + variables := []*domain.TemplateVariable{ + {VariableOrder: 1, VariableName: "Nama", VariableType: "string", FieldSource: "nasabah_nama", IsRequired: true}, + } + + _, err := tf.BuildDynamicMessage(variables, map[string]any{}) + if err == nil { + t.Fatal("expected error for missing required field, got nil") + } +} diff --git a/services/omnix-broadcast/tests/validator_test.go b/services/omnix-broadcast/tests/validator_test.go new file mode 100644 index 0000000..b6ad7a7 --- /dev/null +++ b/services/omnix-broadcast/tests/validator_test.go @@ -0,0 +1,33 @@ +package tests + +import ( + "testing" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/validator" +) + +func TestValidateBroadcastPayload_MissingPhone(t *testing.T) { + b := &domain.Broadcast{ + MessagePayload: map[string]any{ + "invoice_pdf_url": "https://storage.gadai.com/invoices/inv-001.pdf", + }, + } + + if err := validator.ValidateBroadcastPayload(b); err != validator.ErrMissingRecipientPhone { + t.Fatalf("expected ErrMissingRecipientPhone, got %v", err) + } +} + +func TestValidateBroadcastPayload_Valid(t *testing.T) { + b := &domain.Broadcast{ + MessagePayload: map[string]any{ + "nasabah_phone": "6281234567890", + "invoice_pdf_url": "https://storage.gadai.com/invoices/inv-001.pdf", + }, + } + + if err := validator.ValidateBroadcastPayload(b); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} diff --git a/services/omnix-broadcast/transformer/broadcast.go b/services/omnix-broadcast/transformer/broadcast.go new file mode 100644 index 0000000..9f416e8 --- /dev/null +++ b/services/omnix-broadcast/transformer/broadcast.go @@ -0,0 +1,76 @@ +package transformer + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/entity" +) + +type Transformer struct{} + +func New() *Transformer { + return &Transformer{} +} + +func (t *Transformer) EntityToDomain(e *entity.BroadcastStaging) (*domain.Broadcast, error) { + var payload map[string]any + if err := json.Unmarshal(e.MessagePayload, &payload); err != nil { + return nil, fmt.Errorf("unmarshal message_payload: %w", err) + } + + return &domain.Broadcast{ + ID: e.ID, + SopigaCollarID: e.SopigaCollarID, + SopigaTemplateID: e.SopigaTemplateID, + MessagePayload: payload, + Status: domain.BroadcastStatus(e.Status), + ErrorMessage: e.ErrorMessage, + ErrorCount: e.ErrorCount, + SopigaRecipientDetailID: e.SopigaRecipientDetailID, + CreatedAt: e.CreatedAt, + UpdatedAt: e.UpdatedAt, + }, nil +} + +func (t *Transformer) EntityToDomainVariable(e *entity.TemplateVariableMapping) *domain.TemplateVariable { + return &domain.TemplateVariable{ + VariableOrder: e.VariableOrder, + VariableName: e.SopigaVariableName, + VariableType: e.VariableType, + FieldSource: e.DBFieldSource, + IsRequired: e.IsRequired, + } +} + +// BuildDynamicMessage interpolates the message payload against the ordered +// template variables, joining formatted values with "#" — the delimiter +// Sopiga expects between positional template placeholders. +func (t *Transformer) BuildDynamicMessage(variables []*domain.TemplateVariable, payload map[string]any) (string, error) { + parts := make([]string, 0, len(variables)) + + for _, v := range variables { + value, exists := payload[v.FieldSource] + if !exists { + if v.IsRequired { + return "", fmt.Errorf("missing required field: %s", v.FieldSource) + } + value = "" + } + + parts = append(parts, formatValue(v.VariableType, value)) + } + + return strings.Join(parts, "#"), nil +} + +func formatValue(variableType string, value any) string { + switch variableType { + case "integer", "decimal": + return fmt.Sprintf("%.0f", value) + default: + return fmt.Sprintf("%v", value) + } +} diff --git a/services/omnix-broadcast/validator/broadcast.go b/services/omnix-broadcast/validator/broadcast.go new file mode 100644 index 0000000..d70a630 --- /dev/null +++ b/services/omnix-broadcast/validator/broadcast.go @@ -0,0 +1,22 @@ +package validator + +import ( + "errors" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" +) + +var ( + ErrMissingRecipientPhone = errors.New("nasabah_phone not in payload") + ErrMissingInvoiceURL = errors.New("invoice_pdf_url not in payload") +) + +func ValidateBroadcastPayload(b *domain.Broadcast) error { + if _, ok := b.RecipientPhone(); !ok { + return ErrMissingRecipientPhone + } + if _, ok := b.InvoiceURL(); !ok { + return ErrMissingInvoiceURL + } + return nil +} diff --git a/services/omnix-broadcast/worker/broadcast_worker.go b/services/omnix-broadcast/worker/broadcast_worker.go new file mode 100644 index 0000000..091ce2b --- /dev/null +++ b/services/omnix-broadcast/worker/broadcast_worker.go @@ -0,0 +1,85 @@ +package worker + +import ( + "context" + "time" + + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/service" +) + +const maxConcurrency = 5 + +type BroadcastWorker struct { + service *service.Service + logger *fwlogger.Logger + checkInterval time.Duration + batchSize int +} + +func New(svc *service.Service, log *fwlogger.Logger, checkInterval time.Duration, batchSize int) *BroadcastWorker { + return &BroadcastWorker{ + service: svc, + logger: log, + checkInterval: checkInterval, + batchSize: batchSize, + } +} + +func (w *BroadcastWorker) Start(ctx context.Context) { + w.logger.Info("broadcast worker started") + + ticker := time.NewTicker(w.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + w.logger.Info("broadcast worker stopped") + return + case <-ticker.C: + w.processPendingRecords(ctx) + } + } +} + +func (w *BroadcastWorker) processPendingRecords(ctx context.Context) { + broadcasts, err := w.service.FetchPending(ctx, w.batchSize) + if err != nil { + w.logger.Error("fetch pending failed", "error", err) + return + } + if len(broadcasts) == 0 { + return + } + + w.logger.Info("processing pending broadcasts", "count", len(broadcasts)) + + sem := make(chan struct{}, maxConcurrency) + done := make(chan struct{}, len(broadcasts)) + + for _, b := range broadcasts { + sem <- struct{}{} + go func(b *domain.Broadcast) { + defer func() { + <-sem + done <- struct{}{} + }() + w.dispatchOne(ctx, b) + }(b) + } + + for i := 0; i < len(broadcasts); i++ { + <-done + } +} + +func (w *BroadcastWorker) dispatchOne(ctx context.Context, b *domain.Broadcast) { + if err := w.service.Dispatch(ctx, b); err != nil { + w.logger.Error("dispatch failed", "broadcast_id", b.ID, "error", err) + return + } + w.logger.Info("broadcast dispatched", "broadcast_id", b.ID) +} diff --git a/services/omnix-broadcast/worker/delivery_sync_worker.go b/services/omnix-broadcast/worker/delivery_sync_worker.go new file mode 100644 index 0000000..fa6441b --- /dev/null +++ b/services/omnix-broadcast/worker/delivery_sync_worker.go @@ -0,0 +1,83 @@ +package worker + +import ( + "context" + "time" + + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/domain" + "github.com/yourorg/go-dw-platform/services/omnix-broadcast/service" +) + +type DeliverySyncWorker struct { + service *service.Service + logger *fwlogger.Logger + checkInterval time.Duration + batchSize int +} + +func NewDeliverySync(svc *service.Service, log *fwlogger.Logger, checkInterval time.Duration, batchSize int) *DeliverySyncWorker { + return &DeliverySyncWorker{ + service: svc, + logger: log, + checkInterval: checkInterval, + batchSize: batchSize, + } +} + +func (w *DeliverySyncWorker) Start(ctx context.Context) { + w.logger.Info("delivery sync worker started") + + ticker := time.NewTicker(w.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + w.logger.Info("delivery sync worker stopped") + return + case <-ticker.C: + w.syncDispatchedRecords(ctx) + } + } +} + +func (w *DeliverySyncWorker) syncDispatchedRecords(ctx context.Context) { + broadcasts, err := w.service.FetchDispatched(ctx, w.batchSize) + if err != nil { + w.logger.Error("fetch dispatched failed", "error", err) + return + } + if len(broadcasts) == 0 { + return + } + + w.logger.Info("syncing delivery status", "count", len(broadcasts)) + + sem := make(chan struct{}, maxConcurrency) + done := make(chan struct{}, len(broadcasts)) + + for _, b := range broadcasts { + sem <- struct{}{} + go func(b *domain.Broadcast) { + defer func() { + <-sem + done <- struct{}{} + }() + w.syncOne(ctx, b) + }(b) + } + + for i := 0; i < len(broadcasts); i++ { + <-done + } +} + +func (w *DeliverySyncWorker) syncOne(ctx context.Context, b *domain.Broadcast) { + if err := w.service.SyncDelivery(ctx, b); err != nil { + w.logger.Error("sync delivery failed", "broadcast_id", b.ID, "error", err) + return + } + w.logger.Info("delivery status synced", "broadcast_id", b.ID) +} diff --git a/services/template-service/.env.example b/services/template-service/.env.example new file mode 100644 index 0000000..8e45e3e --- /dev/null +++ b/services/template-service/.env.example @@ -0,0 +1,11 @@ +PORT=8080 + +DATABASE_DSN=postgres://user:pass@localhost:5432/gadai_mulia +DB_MAX_CONNS=25 +DB_MIN_CONNS=5 + +BATCH_SIZE=5000 +BATCH_TIMEOUT_SEC=30 +BATCH_MAX_RETRIES=3 + +LOG_LEVEL=info diff --git a/services/template-service/config.go b/services/template-service/config.go new file mode 100644 index 0000000..0ad2db8 --- /dev/null +++ b/services/template-service/config.go @@ -0,0 +1,49 @@ +package main + +import ( + "time" + + fwconfig "github.com/yourorg/go-dw-platform/framework/config" +) + +type Config struct { + Port string + DB DBConfig + Batch BatchConfig + LogLevel string +} + +type DBConfig struct { + DSN string + MaxConns int32 + MinConns int32 + MaxConnLifetime time.Duration + MaxConnIdleTime time.Duration +} + +type BatchConfig struct { + Size int + TimeoutSec int + MaxRetries int + RetryDelay time.Duration +} + +func LoadConfig() *Config { + return &Config{ + Port: fwconfig.GetString("PORT", "8080"), + DB: DBConfig{ + DSN: fwconfig.GetString("DATABASE_DSN", ""), + MaxConns: int32(fwconfig.GetInt("DB_MAX_CONNS", 25)), + MinConns: int32(fwconfig.GetInt("DB_MIN_CONNS", 5)), + MaxConnLifetime: 15 * time.Minute, + MaxConnIdleTime: 5 * time.Minute, + }, + Batch: BatchConfig{ + Size: fwconfig.GetInt("BATCH_SIZE", 5000), + TimeoutSec: fwconfig.GetInt("BATCH_TIMEOUT_SEC", 30), + MaxRetries: fwconfig.GetInt("BATCH_MAX_RETRIES", 3), + RetryDelay: time.Second, + }, + LogLevel: fwconfig.GetString("LOG_LEVEL", "info"), + } +} diff --git a/services/template-service/domain/order.go b/services/template-service/domain/order.go new file mode 100644 index 0000000..ce36d90 --- /dev/null +++ b/services/template-service/domain/order.go @@ -0,0 +1,30 @@ +package domain + +import ( + "fmt" + "time" +) + +type Order struct { + OrderID string + CustomerID string + Amount float64 + CreatedAt time.Time +} + +func (o *Order) Validate() error { + if o.OrderID == "" { + return fmt.Errorf("order_id is required") + } + if o.CustomerID == "" { + return fmt.Errorf("customer_id is required") + } + if o.Amount <= 0 { + return fmt.Errorf("amount must be positive") + } + return nil +} + +func (o *Order) NaturalKey() string { + return fmt.Sprintf("%s_%s_%d", o.OrderID, o.CustomerID, o.CreatedAt.Unix()) +} diff --git a/services/template-service/dto/request.go b/services/template-service/dto/request.go new file mode 100644 index 0000000..9f5c56a --- /dev/null +++ b/services/template-service/dto/request.go @@ -0,0 +1,7 @@ +package dto + +type IngestRequest struct { + OrderID string `json:"order_id" binding:"required"` + CustomerID string `json:"customer_id" binding:"required"` + Amount float64 `json:"amount" binding:"required"` +} diff --git a/services/template-service/dto/response.go b/services/template-service/dto/response.go new file mode 100644 index 0000000..0104e39 --- /dev/null +++ b/services/template-service/dto/response.go @@ -0,0 +1,20 @@ +package dto + +type ProcessResult struct { + Status string `json:"status"` + RowsInserted int64 `json:"rows_inserted,omitempty"` +} + +type SuccessResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + Data any `json:"data,omitempty"` + RequestID string `json:"request_id"` +} + +type ErrorResponse struct { + Success bool `json:"success"` + Message string `json:"message"` + ErrorCode string `json:"error_code,omitempty"` + RequestID string `json:"request_id,omitempty"` +} diff --git a/services/template-service/entity/order.go b/services/template-service/entity/order.go new file mode 100644 index 0000000..03fca7b --- /dev/null +++ b/services/template-service/entity/order.go @@ -0,0 +1,10 @@ +package entity + +import "time" + +type Order struct { + OrderID string `db:"order_id"` + CustomerID string `db:"customer_id"` + Amount float64 `db:"amount"` + CreatedAt time.Time `db:"created_at"` +} diff --git a/services/template-service/go.mod b/services/template-service/go.mod new file mode 100644 index 0000000..9ce0bea --- /dev/null +++ b/services/template-service/go.mod @@ -0,0 +1,11 @@ +module github.com/yourorg/go-dw-platform/services/template-service + +go 1.21 + +require ( + github.com/gin-gonic/gin v1.9.1 + github.com/jackc/pgx/v5 v5.5.0 + github.com/yourorg/go-dw-platform/framework v0.0.0 +) + +replace github.com/yourorg/go-dw-platform/framework => ../../framework diff --git a/services/template-service/handler/health.go b/services/template-service/handler/health.go new file mode 100644 index 0000000..f11505f --- /dev/null +++ b/services/template-service/handler/health.go @@ -0,0 +1,28 @@ +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/jackc/pgx/v5/pgxpool" +) + +type HealthHandler struct { + pool *pgxpool.Pool +} + +func NewHealthHandler(pool *pgxpool.Pool) *HealthHandler { + return &HealthHandler{pool: pool} +} + +func (h *HealthHandler) Health(c *gin.Context) { + dbOK := h.pool.Ping(c.Request.Context()) == nil + + status := "healthy" + if !dbOK { + status = "degraded" + } + + c.JSON(200, gin.H{ + "status": status, + "database": dbOK, + }) +} diff --git a/services/template-service/handler/ingest.go b/services/template-service/handler/ingest.go new file mode 100644 index 0000000..3d03e56 --- /dev/null +++ b/services/template-service/handler/ingest.go @@ -0,0 +1,71 @@ +package handler + +import ( + "errors" + + "github.com/gin-gonic/gin" + + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + + "github.com/yourorg/go-dw-platform/services/template-service/dto" + "github.com/yourorg/go-dw-platform/services/template-service/service" +) + +type Handler struct { + service *service.Service + logger *fwlogger.Logger +} + +func New(svc *service.Service, log *fwlogger.Logger) *Handler { + return &Handler{service: svc, logger: log} +} + +func (h *Handler) IngestOrders(c *gin.Context) { + ctx := c.Request.Context() + requestID := c.GetString("request_id") + + var req dto.IngestRequest + if err := c.ShouldBindJSON(&req); err != nil { + h.logger.Error("validation failed", + "request_id", requestID, + "error", err) + c.JSON(400, dto.ErrorResponse{ + Success: false, + Message: "validation failed", + ErrorCode: "VALIDATION_ERROR", + RequestID: requestID, + }) + return + } + + result, err := h.service.Process(ctx, &req) + if err != nil { + if errors.Is(err, service.ErrDuplicate) { + c.JSON(409, dto.ErrorResponse{ + Success: false, + Message: "duplicate order", + ErrorCode: "DUPLICATE_ORDER", + RequestID: requestID, + }) + return + } + + h.logger.Error("process failed", + "request_id", requestID, + "error", err) + c.JSON(500, dto.ErrorResponse{ + Success: false, + Message: "failed to process order", + ErrorCode: "INTERNAL_ERROR", + RequestID: requestID, + }) + return + } + + c.JSON(200, dto.SuccessResponse{ + Success: true, + Message: "order processed successfully", + Data: result, + RequestID: requestID, + }) +} diff --git a/services/template-service/main.go b/services/template-service/main.go new file mode 100644 index 0000000..ae3c615 --- /dev/null +++ b/services/template-service/main.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gin-gonic/gin" + + fwdb "github.com/yourorg/go-dw-platform/framework/db" + fwlogger "github.com/yourorg/go-dw-platform/framework/logger" + fwmiddleware "github.com/yourorg/go-dw-platform/framework/middleware" + + "github.com/yourorg/go-dw-platform/services/template-service/handler" + "github.com/yourorg/go-dw-platform/services/template-service/repository" + "github.com/yourorg/go-dw-platform/services/template-service/service" + "github.com/yourorg/go-dw-platform/services/template-service/transformer" +) + +func main() { + cfg := LoadConfig() + log := fwlogger.New("template-service") + + ctx := context.Background() + + pool, err := fwdb.NewPool(ctx, fwdb.Config{ + DSN: cfg.DB.DSN, + MaxConns: cfg.DB.MaxConns, + MinConns: cfg.DB.MinConns, + MaxConnLifetime: cfg.DB.MaxConnLifetime, + MaxConnIdleTime: cfg.DB.MaxConnIdleTime, + }) + if err != nil { + log.Error("failed to connect to database", "error", err) + os.Exit(1) + } + defer pool.Close() + + repo := repository.New(pool) + tf := transformer.New() + svc := service.New( + repo, + tf, + cfg.Batch.Size, + time.Duration(cfg.Batch.TimeoutSec)*time.Second, + cfg.Batch.MaxRetries, + cfg.Batch.RetryDelay, + ) + + ingestHandler := handler.New(svc, log) + healthHandler := handler.NewHealthHandler(pool) + + router := gin.New() + router.Use(fwmiddleware.Logging(log)) + router.Use(gin.Recovery()) + + router.GET("/health", healthHandler.Health) + + api := router.Group("/api") + api.Use(fwmiddleware.AuthMiddleware()) + api.POST("/ingest", ingestHandler.IngestOrders) + + go func() { + if err := router.Run(":" + cfg.Port); err != nil { + log.Error("server stopped", "error", err) + } + }() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + <-sigChan + + log.Info("shutting down gracefully") +} diff --git a/services/template-service/migrations/001_create_orders.down.sql b/services/template-service/migrations/001_create_orders.down.sql new file mode 100644 index 0000000..d0f7099 --- /dev/null +++ b/services/template-service/migrations/001_create_orders.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS orders; diff --git a/services/template-service/migrations/001_create_orders.up.sql b/services/template-service/migrations/001_create_orders.up.sql new file mode 100644 index 0000000..4639b8b --- /dev/null +++ b/services/template-service/migrations/001_create_orders.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS orders ( + id BIGSERIAL PRIMARY KEY, + order_id VARCHAR(64) NOT NULL, + customer_id VARCHAR(64) NOT NULL, + amount NUMERIC(18, 2) NOT NULL, + natural_key VARCHAR(255) NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_orders_customer_created + ON orders (customer_id, created_at); diff --git a/services/template-service/repository/orders.go b/services/template-service/repository/orders.go new file mode 100644 index 0000000..eceff03 --- /dev/null +++ b/services/template-service/repository/orders.go @@ -0,0 +1,103 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/yourorg/go-dw-platform/services/template-service/entity" +) + +type Filter struct { + StartDate time.Time + EndDate time.Time + Limit int +} + +type Repository struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Repository { + return &Repository{pool: pool} +} + +func (r *Repository) ExistsByKey(ctx context.Context, naturalKey string) (bool, error) { + var exists bool + err := r.pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM orders WHERE natural_key = $1)`, + naturalKey, + ).Scan(&exists) + if err != nil { + return false, fmt.Errorf("exists by key: %w", err) + } + return exists, nil +} + +func (r *Repository) InsertBatch(ctx context.Context, entities []*entity.Order) (int64, error) { + tx, err := r.pool.Begin(ctx) + if err != nil { + return 0, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + stmt := `INSERT INTO orders (order_id, customer_id, amount, natural_key, created_at) + VALUES ($1, $2, $3, $4, $5)` + + batch := &pgx.Batch{} + for _, e := range entities { + naturalKey := fmt.Sprintf("%s_%s_%d", e.OrderID, e.CustomerID, e.CreatedAt.Unix()) + batch.Queue(stmt, e.OrderID, e.CustomerID, e.Amount, naturalKey, e.CreatedAt) + } + + results := tx.SendBatch(ctx, batch) + + var rowsInserted int64 + for i := 0; i < len(entities); i++ { + tag, err := results.Exec() + if err != nil { + results.Close() + return 0, fmt.Errorf("exec batch: %w", err) + } + rowsInserted += tag.RowsAffected() + } + if err := results.Close(); err != nil { + return 0, fmt.Errorf("close batch results: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return 0, fmt.Errorf("commit: %w", err) + } + + return rowsInserted, nil +} + +func (r *Repository) FindByFilters(ctx context.Context, filter *Filter) ([]*entity.Order, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + query := `SELECT order_id, customer_id, amount, created_at + FROM orders + WHERE created_at >= $1 AND created_at <= $2 + LIMIT $3` + + rows, err := r.pool.Query(ctx, query, filter.StartDate, filter.EndDate, filter.Limit) + if err != nil { + return nil, fmt.Errorf("query: %w", err) + } + defer rows.Close() + + var orders []*entity.Order + for rows.Next() { + var o entity.Order + if err := rows.Scan(&o.OrderID, &o.CustomerID, &o.Amount, &o.CreatedAt); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + orders = append(orders, &o) + } + + return orders, rows.Err() +} diff --git a/services/template-service/service/orders.go b/services/template-service/service/orders.go new file mode 100644 index 0000000..63400d0 --- /dev/null +++ b/services/template-service/service/orders.go @@ -0,0 +1,83 @@ +package service + +import ( + "context" + "errors" + "fmt" + "time" + + fwingestion "github.com/yourorg/go-dw-platform/framework/ingestion" + fwmetrics "github.com/yourorg/go-dw-platform/framework/metrics" + + "github.com/yourorg/go-dw-platform/services/template-service/domain" + "github.com/yourorg/go-dw-platform/services/template-service/dto" + "github.com/yourorg/go-dw-platform/services/template-service/entity" + "github.com/yourorg/go-dw-platform/services/template-service/repository" + "github.com/yourorg/go-dw-platform/services/template-service/transformer" +) + +var ErrDuplicate = errors.New("duplicate order") + +type Service struct { + repository *repository.Repository + transformer *transformer.Transformer + batcher *fwingestion.Batcher[*domain.Order] + retrier *fwingestion.Retrier +} + +func New(repo *repository.Repository, tf *transformer.Transformer, batchSize int, batchTimeout time.Duration, maxRetries int, retryDelay time.Duration) *Service { + return &Service{ + repository: repo, + transformer: tf, + batcher: fwingestion.NewBatcher[*domain.Order](batchSize, batchTimeout), + retrier: fwingestion.NewRetrier(maxRetries, retryDelay), + } +} + +func (s *Service) Process(ctx context.Context, req *dto.IngestRequest) (*dto.ProcessResult, error) { + order, err := s.transformer.RequestToDomain(req) + if err != nil { + return nil, fmt.Errorf("process: %w", err) + } + + exists, err := s.repository.ExistsByKey(ctx, order.NaturalKey()) + if err != nil { + return nil, fmt.Errorf("check duplicate: %w", err) + } + if exists { + return nil, ErrDuplicate + } + + s.batcher.Add(order) + + if s.batcher.IsFull() || s.batcher.IsExpired() { + return s.FlushBatch(ctx) + } + + return &dto.ProcessResult{Status: "queued"}, nil +} + +func (s *Service) FlushBatch(ctx context.Context) (*dto.ProcessResult, error) { + start := time.Now() + batch := s.batcher.Get() + + entities := make([]*entity.Order, len(batch)) + for i, d := range batch { + entities[i] = s.transformer.DomainToEntity(d) + } + + rows, err := s.retrier.Do(ctx, func() (int64, error) { + return s.repository.InsertBatch(ctx, entities) + }) + if err != nil { + return nil, fmt.Errorf("flush batch: %w", err) + } + + fwmetrics.RecordBatchInsert(len(batch), time.Since(start)) + + s.batcher.Clear() + return &dto.ProcessResult{ + Status: "success", + RowsInserted: rows, + }, nil +} diff --git a/services/template-service/tests/fixtures.go b/services/template-service/tests/fixtures.go new file mode 100644 index 0000000..24b9660 --- /dev/null +++ b/services/template-service/tests/fixtures.go @@ -0,0 +1,16 @@ +package tests + +import ( + "time" + + "github.com/yourorg/go-dw-platform/services/template-service/domain" +) + +func NewTestOrder(orderID, customerID string, amount float64) *domain.Order { + return &domain.Order{ + OrderID: orderID, + CustomerID: customerID, + Amount: amount, + CreatedAt: time.Now(), + } +} diff --git a/services/template-service/tests/service_test.go b/services/template-service/tests/service_test.go new file mode 100644 index 0000000..bb34753 --- /dev/null +++ b/services/template-service/tests/service_test.go @@ -0,0 +1,26 @@ +package tests + +import ( + "testing" + + "github.com/yourorg/go-dw-platform/services/template-service/dto" + "github.com/yourorg/go-dw-platform/services/template-service/validator" +) + +func TestValidateIngestRequest_MissingOrderID(t *testing.T) { + req := &dto.IngestRequest{CustomerID: "cust-1", Amount: 100} + + err := validator.ValidateIngestRequest(req) + + if err != validator.ErrMissingOrderID { + t.Fatalf("expected ErrMissingOrderID, got %v", err) + } +} + +func TestValidateIngestRequest_Valid(t *testing.T) { + req := &dto.IngestRequest{OrderID: "ord-1", CustomerID: "cust-1", Amount: 100} + + if err := validator.ValidateIngestRequest(req); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} diff --git a/services/template-service/transformer/orders.go b/services/template-service/transformer/orders.go new file mode 100644 index 0000000..9119e6e --- /dev/null +++ b/services/template-service/transformer/orders.go @@ -0,0 +1,40 @@ +package transformer + +import ( + "fmt" + "time" + + "github.com/yourorg/go-dw-platform/services/template-service/domain" + "github.com/yourorg/go-dw-platform/services/template-service/dto" + "github.com/yourorg/go-dw-platform/services/template-service/entity" +) + +type Transformer struct{} + +func New() *Transformer { + return &Transformer{} +} + +func (t *Transformer) RequestToDomain(req *dto.IngestRequest) (*domain.Order, error) { + order := &domain.Order{ + OrderID: req.OrderID, + CustomerID: req.CustomerID, + Amount: req.Amount, + CreatedAt: time.Now(), + } + + if err := order.Validate(); err != nil { + return nil, fmt.Errorf("transform: %w", err) + } + + return order, nil +} + +func (t *Transformer) DomainToEntity(d *domain.Order) *entity.Order { + return &entity.Order{ + OrderID: d.OrderID, + CustomerID: d.CustomerID, + Amount: d.Amount, + CreatedAt: d.CreatedAt, + } +} diff --git a/services/template-service/validator/orders.go b/services/template-service/validator/orders.go new file mode 100644 index 0000000..5d97d42 --- /dev/null +++ b/services/template-service/validator/orders.go @@ -0,0 +1,22 @@ +package validator + +import ( + "errors" + + "github.com/yourorg/go-dw-platform/services/template-service/dto" +) + +var ErrMissingOrderID = errors.New("order_id is required") + +func ValidateIngestRequest(req *dto.IngestRequest) error { + if req.OrderID == "" { + return ErrMissingOrderID + } + if req.CustomerID == "" { + return errors.New("customer_id is required") + } + if req.Amount <= 0 { + return errors.New("amount must be positive") + } + return nil +}