From e1c61289af9d6bfe680db2583c295408065a2fe0 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 9 Mar 2026 20:39:16 +0100 Subject: [PATCH] feat(config): Add MCP JSON bootstrap and unify docs Enable backend fallback to MCP JSON for database connection when DATABASE_URL is absent.\nAdd credentials bootstrap support from JSON for cloud/external services.\n\nConsolidate documentation with MCP integration guide and unified status.\nUpdate backlog to move video interviews endpoint out of current sprint. Co-Authored-By: Claude --- .gitignore | 1 + backend/internal/database/database.go | 62 +++++++++++++++ .../internal/services/credentials_service.go | 79 +++++++++++++++---- config/mcp.gohorsejobs.example.json | 44 +++++++++++ docs/MCP_INTEGRATION.md | 56 +++++++++++++ docs/README.md | 2 + docs/TASKS.md | 24 +++++- docs/UNIFIED_STATUS.md | 54 +++++++++++++ 8 files changed, 306 insertions(+), 16 deletions(-) create mode 100644 config/mcp.gohorsejobs.example.json create mode 100644 docs/MCP_INTEGRATION.md create mode 100644 docs/UNIFIED_STATUS.md diff --git a/.gitignore b/.gitignore index d7a0ac7..fb48e4e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ Thumbs.db .env.local .env.*.local !.env.example +config/mcp.gohorsejobs.json # ----------------------------------------------------------------------------- # Logs diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go index 0e0eb1b..1c76856 100644 --- a/backend/internal/database/database.go +++ b/backend/internal/database/database.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "encoding/json" "fmt" "log" "os" @@ -47,9 +48,70 @@ func BuildConnectionString() (string, error) { return dbURL, nil } + if mcpPath := strings.TrimSpace(os.Getenv("MCP_JSON_PATH")); mcpPath != "" { + dbURL, err := databaseURLFromMCPJSON(mcpPath) + if err != nil { + return "", fmt.Errorf("failed to load database url from MCP json (%s): %w", mcpPath, err) + } + if dbURL != "" { + log.Printf("Using DATABASE_URL from MCP_JSON_PATH (%s)", mcpPath) + return dbURL, nil + } + log.Printf("MCP_JSON_PATH is set but no database URL was found in %s", mcpPath) + } + return "", fmt.Errorf("DATABASE_URL environment variable not set") } +func databaseURLFromMCPJSON(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + + var payload map[string]interface{} + if err := json.Unmarshal(raw, &payload); err != nil { + return "", err + } + + candidates := [][]string{ + {"database_url"}, + {"databaseUrl"}, + {"database", "url"}, + {"infra", "database_url"}, + {"infra", "databaseUrl"}, + {"infra", "database", "url"}, + } + + for _, pathKeys := range candidates { + if val := nestedString(payload, pathKeys...); val != "" { + return val, nil + } + } + + return "", nil +} + +func nestedString(input map[string]interface{}, keys ...string) string { + var current interface{} = input + for _, key := range keys { + obj, ok := current.(map[string]interface{}) + if !ok { + return "" + } + current, ok = obj[key] + if !ok { + return "" + } + } + + str, ok := current.(string) + if !ok { + return "" + } + return strings.TrimSpace(str) +} + func RunMigrations() { migrationDir, err := resolveMigrationDir() if err != nil { diff --git a/backend/internal/services/credentials_service.go b/backend/internal/services/credentials_service.go index 7be8adc..6d1cff8 100644 --- a/backend/internal/services/credentials_service.go +++ b/backend/internal/services/credentials_service.go @@ -12,6 +12,7 @@ import ( "encoding/pem" "fmt" "os" + "strings" "sync" ) @@ -261,15 +262,15 @@ func (s *CredentialsService) EncryptPayload(payload string) (string, error) { // BootstrapCredentials checks if credentials are in DB, if not, migrates from Env func (s *CredentialsService) BootstrapCredentials(ctx context.Context) error { // List of services and their env mapping - services := map[string]func() interface{}{ - "stripe": func() interface{} { + services := map[string]func() map[string]string{ + "stripe": func() map[string]string { return map[string]string{ "secretKey": os.Getenv("STRIPE_SECRET_KEY"), "webhookSecret": os.Getenv("STRIPE_WEBHOOK_SECRET"), "publishableKey": os.Getenv("STRIPE_PUBLISHABLE_KEY"), } }, - "storage": func() interface{} { + "storage": func() map[string]string { return map[string]string{ "endpoint": os.Getenv("AWS_ENDPOINT"), "accessKey": os.Getenv("AWS_ACCESS_KEY_ID"), @@ -278,37 +279,37 @@ func (s *CredentialsService) BootstrapCredentials(ctx context.Context) error { "region": os.Getenv("AWS_REGION"), } }, - "lavinmq": func() interface{} { + "lavinmq": func() map[string]string { return map[string]string{ "amqpUrl": os.Getenv("AMQP_URL"), } }, - "cloudflare_config": func() interface{} { + "cloudflare_config": func() map[string]string { return map[string]string{ "apiToken": os.Getenv("CLOUDFLARE_API_TOKEN"), "zoneId": os.Getenv("CLOUDFLARE_ZONE_ID"), } }, - "cpanel": func() interface{} { + "cpanel": func() map[string]string { return map[string]string{ "host": os.Getenv("CPANEL_HOST"), "username": os.Getenv("CPANEL_USERNAME"), "apiToken": os.Getenv("CPANEL_API_TOKEN"), } }, - "appwrite": func() interface{} { + "appwrite": func() map[string]string { return map[string]string{ "endpoint": os.Getenv("APPWRITE_ENDPOINT"), "projectId": os.Getenv("APPWRITE_PROJECT_ID"), "apiKey": os.Getenv("APPWRITE_API_KEY"), } }, - "fcm_service_account": func() interface{} { + "fcm_service_account": func() map[string]string { return map[string]string{ "serviceAccountJson": os.Getenv("FCM_SERVICE_ACCOUNT_JSON"), } }, - "smtp": func() interface{} { + "smtp": func() map[string]string { return map[string]string{ "host": os.Getenv("SMTP_HOST"), "port": os.Getenv("SMTP_PORT"), @@ -321,6 +322,17 @@ func (s *CredentialsService) BootstrapCredentials(ctx context.Context) error { }, } + mcpServices := map[string]map[string]string{} + if mcpPath := strings.TrimSpace(os.Getenv("MCP_JSON_PATH")); mcpPath != "" { + loaded, err := loadMCPCredentialsFromFile(mcpPath) + if err != nil { + fmt.Printf("[CredentialsBootstrap] Warning: failed to read MCP JSON (%s): %v\n", mcpPath, err) + } else { + mcpServices = loaded + fmt.Printf("[CredentialsBootstrap] Loaded MCP JSON services (%d) from %s\n", len(mcpServices), mcpPath) + } + } + for service, getEnvData := range services { // Check if already configured configured, err := s.isServiceConfigured(ctx, service) @@ -331,19 +343,27 @@ func (s *CredentialsService) BootstrapCredentials(ctx context.Context) error { if !configured { data := getEnvData() - // Validate if env vars exist (naive check: at least one field not empty) + + // MCP JSON values take priority when present. + if fromMCP, ok := mcpServices[service]; ok { + for k, v := range fromMCP { + if strings.TrimSpace(v) != "" { + data[k] = v + } + } + } + + // Validate if at least one field is present. hasData := false - jsonBytes, _ := json.Marshal(data) - var stringMap map[string]string - json.Unmarshal(jsonBytes, &stringMap) - for _, v := range stringMap { - if v != "" { + for _, v := range data { + if strings.TrimSpace(v) != "" { hasData = true break } } if hasData { + jsonBytes, _ := json.Marshal(data) fmt.Printf("[CredentialsBootstrap] Migrating %s from Env to DB...\n", service) encrypted, err := s.EncryptPayload(string(jsonBytes)) if err != nil { @@ -361,6 +381,35 @@ func (s *CredentialsService) BootstrapCredentials(ctx context.Context) error { return nil } +func loadMCPCredentialsFromFile(path string) (map[string]map[string]string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var payload struct { + Services map[string]map[string]string `json:"services"` + Credentials map[string]map[string]string `json:"credentials"` + ExternalServices map[string]map[string]string `json:"external_services"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, err + } + + merged := map[string]map[string]string{} + for svc, values := range payload.Services { + merged[svc] = values + } + for svc, values := range payload.Credentials { + merged[svc] = values + } + for svc, values := range payload.ExternalServices { + merged[svc] = values + } + + return merged, nil +} + func (s *CredentialsService) isServiceConfigured(ctx context.Context, serviceName string) (bool, error) { var exists bool query := `SELECT EXISTS(SELECT 1 FROM external_services_credentials WHERE service_name = $1)` diff --git a/config/mcp.gohorsejobs.example.json b/config/mcp.gohorsejobs.example.json new file mode 100644 index 0000000..ee309f9 --- /dev/null +++ b/config/mcp.gohorsejobs.example.json @@ -0,0 +1,44 @@ +{ + "infra": { + "database": { + "url": "postgresql://user:password@host:5432/gohorsejobs_dev?sslmode=require" + }, + "cloud": { + "provider": "cloudflare", + "region": "sa-east-1" + } + }, + "external_services": { + "cloudflare_config": { + "apiToken": "CF_API_TOKEN", + "zoneId": "CF_ZONE_ID" + }, + "storage": { + "endpoint": "https://s3.example.com", + "accessKey": "ACCESS_KEY", + "secretKey": "SECRET_KEY", + "bucket": "gohorsejobs", + "region": "sa-east-1" + }, + "lavinmq": { + "amqpUrl": "amqps://user:pass@host/vhost" + }, + "appwrite": { + "endpoint": "https://cloud.appwrite.io/v1", + "projectId": "PROJECT_ID", + "apiKey": "APPWRITE_API_KEY" + }, + "fcm_service_account": { + "serviceAccountJson": "{\"type\":\"service_account\",\"project_id\":\"...\"}" + }, + "smtp": { + "host": "smtp.example.com", + "port": "587", + "username": "user@example.com", + "password": "password", + "from_email": "noreply@gohorsejobs.com", + "from_name": "GoHorse Jobs", + "secure": "false" + } + } +} diff --git a/docs/MCP_INTEGRATION.md b/docs/MCP_INTEGRATION.md new file mode 100644 index 0000000..4326a9f --- /dev/null +++ b/docs/MCP_INTEGRATION.md @@ -0,0 +1,56 @@ +# MCP Integration (JSON) - GoHorseJobs + +Este guia centraliza como conectar o GoHorseJobs a um arquivo JSON de configuração para MCP/infra cloud/banco. + +## Objetivo + +Permitir bootstrap de credenciais e URL de banco sem hardcode, usando um JSON local controlado por variável de ambiente. + +## Variável de ambiente + +Defina: + +```bash +MCP_JSON_PATH=/caminho/absoluto/para/mcp.gohorsejobs.json +``` + +## Estrutura esperada do JSON + +Use como base: + +- `config/mcp.gohorsejobs.example.json` + +Campos suportados pelo backend: + +- Banco: + - `infra.database.url` + - `database.url` + - `database_url` + - `databaseUrl` +- Credenciais de serviços: + - `external_services.` + - `credentials.` + - `services.` + +## Serviços reconhecidos no bootstrap + +- `stripe` +- `storage` +- `cloudflare_config` +- `cpanel` +- `lavinmq` +- `appwrite` +- `fcm_service_account` +- `smtp` + +## Comportamento de prioridade + +1. `DATABASE_URL` no ambiente continua tendo prioridade máxima. +2. Se `DATABASE_URL` não existir, o backend tenta `MCP_JSON_PATH`. +3. Para credenciais de serviços, o bootstrap usa env vars e sobrescreve com valores do JSON quando presentes. + +## Segurança operacional + +- Não commitar arquivo real com segredos. +- Commite apenas o template `config/mcp.gohorsejobs.example.json`. +- Mantenha `config/mcp.gohorsejobs.json` local e ignorado no git. diff --git a/docs/README.md b/docs/README.md index 50af649..e87ee48 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,8 @@ Choose a specific domain below to dive deep into our technical implementation an ### 🏗️ 2. High-Level Architecture * **[DevOps & Infrastructure (DEVOPS.md)](DEVOPS.md)**: Full mapping of Cloudflare DNS, Traefik, VPS (Redbull/Apolo), Docker/Coolify containers, and CI/CD pipelines (Forgejo/Drone). Includes rich Mermaid diagrams. * **[Database Schema (DATABASE.md)](DATABASE.md)**: PostgreSQL schemas, relationships, UUID v7 indexing strategies, and ERD visualizing the core data flow. +* **[MCP Integration (MCP_INTEGRATION.md)](MCP_INTEGRATION.md)**: JSON-based integration for infra/cloud/database bootstrap. +* **[Unified Status (UNIFIED_STATUS.md)](UNIFIED_STATUS.md)**: Consolidated view of docs + pending activities on `dev`. ### 🔌 3. Application Interfaces (APIs) * **[API Routes (API.md)](API.md)**: Endpoints mapped for the Go Backend (`/api/v1`), NestJS Backoffice services, and internal Node.js Seeder-API. diff --git a/docs/TASKS.md b/docs/TASKS.md index e52e171..c0f9cf4 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -49,7 +49,6 @@ Lista detalhada de tarefas para evitar retrabalho. ## 🔥 Sprint Atual ### Backend -- [ ] Video interviews endpoint - [ ] AI matching algorithm - [ ] Webhook sistema @@ -63,6 +62,9 @@ Lista detalhada de tarefas para evitar retrabalho. - [ ] User analytics - [ ] Export features +### Adiado (fora do escopo por enquanto) +- [ ] Video interviews endpoint + --- ## 🚧 Não Fazer (Evitar Retrabalho) @@ -115,3 +117,23 @@ NestJS → Consume → Fetch template → Render → Send - [ROADMAP.md](ROADMAP.md) - Roadmap geral - [API_SECURITY.md](API_SECURITY.md) - Segurança - [DEVOPS.md](DEVOPS.md) - Infraestrutura +- [MCP_INTEGRATION.md](MCP_INTEGRATION.md) - Integração MCP via JSON +- [UNIFIED_STATUS.md](UNIFIED_STATUS.md) - Consolidação de status e pendências + +--- + +## ✅ Verificação de Pendências (2026-03-09) + +Status conferido na branch `dev`: + +- Sprint atual permanece em aberto: + - AI matching algorithm + - Webhook sistema + - PWA manifest + - Service worker + - Offline support + - Revenue reports + - User analytics + - Export features +- Itens adiados: + - Video interviews endpoint diff --git a/docs/UNIFIED_STATUS.md b/docs/UNIFIED_STATUS.md new file mode 100644 index 0000000..af9ec80 --- /dev/null +++ b/docs/UNIFIED_STATUS.md @@ -0,0 +1,54 @@ +# GoHorseJobs - Documentacao Unificada e Status (dev) + +Atualizado em: 2026-03-09 + +## Escopo consolidado + +- Arquitetura e infraestrutura: `docs/DEVOPS.md`, `docs/DATABASE.md` +- APIs e seguranca: `docs/API.md`, `docs/API_SECURITY.md`, `docs/APPSEC_STRATEGY.md` +- Regras de agente: `docs/AGENTS.md`, `.agent/rules.md` +- Operacao MCP/JSON: `docs/MCP_INTEGRATION.md` + +## Pendencias verificadas + +Fonte principal: `docs/TASKS.md` e `docs/ROADMAP.md` + +Sprint atual em aberto: + +- Backend: + - [ ] AI matching algorithm + - [ ] Webhook sistema +- Frontend: + - [ ] PWA manifest + - [ ] Service worker + - [ ] Offline support +- Backoffice: + - [ ] Revenue reports + - [ ] User analytics + - [ ] Export features + +Itens adiados (fora de escopo no momento): + +- Backend: + - [ ] Video interviews endpoint + +Prioridades tecnicas em andamento (roadmap): + +- Confiabilidade do fluxo de vagas/candidaturas +- Seguranca e governanca (RBAC + auditoria) +- Operacao/deploy (padronizacao dev/hml/prd, rollback, scripts) + +## Decisao tecnica aplicada neste update + +- O backend passa a aceitar configuracao JSON via `MCP_JSON_PATH`: + - URL de banco como fallback quando `DATABASE_URL` nao estiver definido. + - Credenciais de servicos para bootstrap no banco. +- Template de referencia adicionado em `config/mcp.gohorsejobs.example.json`. + +## Proximos passos recomendados + +- Criar arquivo local real `config/mcp.gohorsejobs.json` (nao versionado). +- Setar `MCP_JSON_PATH` em dev/hml. +- Validar boot do backend com: + - banco via JSON + - bootstrap de credenciais sem regressao.