photum/backend/cmd/importer/main.go

package main

import (
	"context"
	"database/sql"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	_ "github.com/jackc/pgx/v5/stdlib"
	"github.com/joho/godotenv"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

func main() {
	// Load .env
	// Try loading from current directory first, typical when running 'go run cmd/importer/main.go' from root
	if err := godotenv.Load(".env"); err != nil {
		// Fallback for when running from cmd/importer
		if err := godotenv.Load("../../.env"); err != nil {
			log.Println("Warning: .env file not found in .env or ../../.env")
		}
	}

	dbURL := os.Getenv("DB_DSN")
	if dbURL == "" {
		log.Fatal("DB_DSN is not set")
	}

	db, err := sql.Open("pgx", dbURL)
	if err != nil {
		log.Fatalf("Failed to connect to database: %v", err)
	}
	defer db.Close()

	if err := db.Ping(); err != nil {
		log.Fatalf("Failed to ping database: %v", err)
	}

	// Open CSV File
	filename := "importacao_fots.csv"
	f, err := os.Open(filename)
	if err != nil {
		log.Fatalf("Failed to open CSV file: %v", err)
	}
	defer f.Close()

	// Use Windows-1252 Decoder
	r := csv.NewReader(transform.NewReader(f, charmap.Windows1252.NewDecoder()))
	r.Comma = ';'
	r.LazyQuotes = true // Allow messy quotes

	// Read Header
	header, err := r.Read()
	if err != nil {
		log.Fatalf("Failed to read header: %v", err)
	}
	fmt.Printf("Header: %v\n", header)

	ctx := context.Background()
	rowsProcessed := 0
	rowsInserted := 0
	rowsFailed := 0

	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Printf("Error reading row %d: %v. Skipping.", rowsProcessed+1, err)
			rowsFailed++
			continue
		}
		rowsProcessed++

		// Map Columns (Indices based on "FOT;Empresa;EF I / EF II;Observa‡äes;Institui‡Æo;Ano Formatura;Cidade;Estado;Gastos Capta‡Æo;Pr‚ Venda")
		// 0: FOT
		// 1: Empresa
		// 2: Cursos (was EF I / EF II)
		// 3: Observacoes
		// 4: Instituicao
		// 5: Ano Formatura
		// 6: Cidade
		// 7: Estado
		// 8: Gastos Captacao
		// 9: Pre Venda

		fotStr := strings.TrimSpace(record[0])
		empresaName := strings.TrimSpace(record[1])
		cursoName := strings.TrimSpace(record[2])
		observacoes := strings.TrimSpace(record[3])
		instituicao := strings.TrimSpace(record[4])
		anoFormatura := strings.TrimSpace(record[5])
		cidade := strings.TrimSpace(record[6])
		estado := strings.TrimSpace(record[7])
		gastosStr := strings.TrimSpace(record[8])
		preVendaStr := strings.TrimSpace(record[9])

		// Basic Validation
		if fotStr == "" || fotStr == "FOT" { // Skip header repetition or empty lines
			continue
		}

		fot, err := strconv.Atoi(fotStr)
		if err != nil {
			log.Printf("Row %d: Invalid FOT '%s'. Skipping.", rowsProcessed, fotStr)
			rowsFailed++
			continue
		}

		// 1. Upsert Empresa
		var empresaID string
		if empresaName == "" {
			empresaName = "Não Identificada"
		}
		err = db.QueryRowContext(ctx, `
			INSERT INTO empresas (nome) VALUES ($1)
			ON CONFLICT (nome) DO UPDATE SET nome = EXCLUDED.nome
			RETURNING id`, empresaName).Scan(&empresaID)
		if err != nil {
			log.Printf("Row %d: Failed to upsert empresa '%s': %v", rowsProcessed, empresaName, err)
			rowsFailed++
			continue
		}

		// 2. Upsert Curso
		var cursoID string
		if cursoName == "" || cursoName == "-" {
			cursoName = "Geral"
		}
		err = db.QueryRowContext(ctx, `
			INSERT INTO cursos (nome) VALUES ($1)
			ON CONFLICT (nome) DO UPDATE SET nome = EXCLUDED.nome
			RETURNING id`, cursoName).Scan(&cursoID)
		if err != nil {
			log.Printf("Row %d: Failed to upsert curso '%s': %v", rowsProcessed, cursoName, err)
			rowsFailed++
			continue
		}

		// 3. Upsert Ano Formatura
		var anoID string
		if anoFormatura == "" {
			anoFormatura = "Indefinido"
		}
		err = db.QueryRowContext(ctx, `
			INSERT INTO anos_formaturas (ano_semestre) VALUES ($1)
			ON CONFLICT (ano_semestre) DO UPDATE SET ano_semestre = EXCLUDED.ano_semestre
			RETURNING id`, anoFormatura).Scan(&anoID)
		if err != nil {
			log.Printf("Row %d: Failed to upsert ano '%s': %v", rowsProcessed, anoFormatura, err)
			rowsFailed++
			continue
		}

		// 4. Parse Currency (Gastos Captação)
		// Format: "R$ 2.176,60" -> 2176.60
		gastosVal := 0.0
		if gastosStr != "" && gastosStr != "-" {
			// Remove "R$", "." and trim
			clean := strings.ReplaceAll(gastosStr, "R$", "")
			clean = strings.ReplaceAll(clean, ".", "")  // Remove thousand separator
			clean = strings.ReplaceAll(clean, ",", ".") // Replace decimal separator
			clean = strings.TrimSpace(clean)

			// Handle trailing/leading spaces hidden chars if any
			// Just parse float
			val, err := strconv.ParseFloat(clean, 64)
			if err == nil {
				gastosVal = val
			} else {
				// log.Printf("Row %d: Warning parsing gastos '%s' -> '%s': %v", rowsProcessed, gastosStr, clean, err)
			}
		}

		// 5. Parse Boolean (Pre Venda)
		preVenda := false
		if strings.ToLower(preVendaStr) == "sim" {
			preVenda = true
		}

		// 6. Insert Cadastro FOT
		_, err = db.ExecContext(ctx, `
			INSERT INTO cadastro_fot (
				fot, empresa_id, curso_id, ano_formatura_id, 
				instituicao, cidade, estado, observacoes, 
				gastos_captacao, pre_venda
			) VALUES (
				$1, $2, $3, $4, 
				$5, $6, $7, $8, 
				$9, $10
			)
			ON CONFLICT (fot) DO UPDATE SET
				empresa_id = EXCLUDED.empresa_id,
				curso_id = EXCLUDED.curso_id,
				ano_formatura_id = EXCLUDED.ano_formatura_id,
				instituicao = EXCLUDED.instituicao,
				cidade = EXCLUDED.cidade,
				estado = EXCLUDED.estado,
				observacoes = EXCLUDED.observacoes,
				gastos_captacao = EXCLUDED.gastos_captacao,
				pre_venda = EXCLUDED.pre_venda,
				updated_at = NOW()
		`,
			fot, empresaID, cursoID, anoID,
			instituicao, cidade, estado, observacoes,
			gastosVal, preVenda,
		)

		if err != nil {
			log.Printf("Row %d: Failed to insert FOT %d: %v", rowsProcessed, fot, err)
			rowsFailed++
		} else {
			rowsInserted++
			if rowsInserted%50 == 0 {
				fmt.Printf("Progress: %d records upserted...\n", rowsInserted)
			}
		}
	}

	fmt.Printf("\n--- Import Summary ---\n")
	fmt.Printf("Total Rows Processed: %d\n", rowsProcessed)
	fmt.Printf("Successfully Upserted: %d\n", rowsInserted)
	fmt.Printf("Failed: %d\n", rowsFailed)
}