Konektory do Postgresu, pristupove udaje sifrovane
Pristupove udaje konektoru se ukladaji do databaze a prezijou restart. Popis v documentation/14-databaze.md. Databaze je volitelna a rezimy jsou oddelene: - postgres kdyz je DATABASE_URL i SECRETS_KEY - memory jinak, tedy pri nasazenem mockupu a lokalnim vyvoji bez DB Rozhodnuti je jen na jednom miste (src/data/connectorStore.ts). Nikde jinde se nezjistuje, jestli databaze je - kdyby se to rozlezlo po kodu, jedno misto by se zapomnelo a chovalo by se pak jinak nez zbytek. Chybejici databaze nesmi shodit start: container, ktery nenastartuje, je pro AppFactory nefunkcni sluzba. Misto toho se do logu napise proc a portal to ukaze na strance Konektory. Stejne tak kdyz migrace selzou - psat do rozbiteho schematu je horsi nez neukladat. Databaze potrebuje oboji. Bez SECRETS_KEY by se udaje ukladaly v plaintextu a to je horsi nez ztratit je pri restartu: tabulku vidi kazda zaloha a kazdy dump pri ladeni. Pridano: - pool v src/db/pool.ts vcetne transakci a dbFor(tenantId) jako sev pro budouci oddelenou databazi jednoho klienta - migrace ze src/db/migrations/*.sql pod pg_advisory_lock, jinak je pri rolling deployi pusti vsechny instance naraz. Jeden soubor je jedna transakce - sifrovani AES-256-GCM s nahodnym IV a verzi klice. Nerozsifrovatelna hodnota nepada, chova se jako nevyplnena a zaloguje se - jeden rozbity konektor nesmi shodit seznam ostatnich - /health/ready s pingem do DB. /health na databazi zamerne nezavisi, kratky vypadek by jinak vedl k restartovani containeru - GET /api/dashboard/storage a hlaska v portalu o tom, ze data jsou jen v pameti - jediny vychozi konektor na firmu a sluzbu hlida castecny unikatni index, ne jen kod. Dva soubezne zapisy by jinak udelaly dva vychozi Zmeneno: cteni i zapis konektoru je asynchronni, vcetne validace stromu. Overeno proti Postgresu 16 v kontejneru: migrace, sifrovani v tabulce, preziti restartu, rozsifrovani spravnym klicem, degradace pri spatnem klici, PATCH bez tajneho pole, prepnuti a smazani vychoziho konektoru, pametovy rezim bez DATABASE_URL. Kontejner po overeni smazan. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
3279dd7dac
commit
78e7f99d60
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Migrace schematu.
|
||||
*
|
||||
* Soubory `src/db/migrations/*.sql` se spousti v abecednim poradi, kazdy jednou.
|
||||
* Co uz proslo, je v tabulce `schema_migrations`.
|
||||
*
|
||||
* Dve veci, na kterych to stoji:
|
||||
*
|
||||
* 1. **Poradovy zamek.** Pri rolling deployi startuje vic instanci naraz
|
||||
* a bez zamku by migrace pustily vsechny. `pg_advisory_lock` zaridi, ze
|
||||
* projede jedna a ostatni pockaji.
|
||||
* 2. **Jeden soubor je jedna transakce.** Pri chybe se nic z nej neuplatni,
|
||||
* takze nevznikne rozdelane schema, o kterem nikdo nevi.
|
||||
*
|
||||
* Migrace se **nikdy neupravuji zpetne**. Uz projely u nekoho jineho, takze
|
||||
* zmena souboru znamena dve rozdilna schemata se stejnym cislem. Oprava je
|
||||
* vzdy novy soubor.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { db } from './pool.js';
|
||||
|
||||
/** Libovolne, ale stabilni cislo. Musi byt stejne ve vsech instancich. */
|
||||
const LOCK_ID = 918_273_645;
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/**
|
||||
* Slozka s migracemi.
|
||||
*
|
||||
* V nasazeni bezi zkompilovany kod z `dist/`, ale `.sql` soubory tsc nekopiruje.
|
||||
* Zkousi se proto obe cesty - vedle prelozeneho souboru i v `src`.
|
||||
*/
|
||||
async function migrationsDir(): Promise<string> {
|
||||
const candidates = [path.join(here, 'migrations'), path.resolve(here, '../../src/db/migrations')];
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await fs.access(candidate);
|
||||
return candidate;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
throw new Error(`Slozku s migracemi nelze najit. Zkouseno: ${candidates.join(', ')}`);
|
||||
}
|
||||
|
||||
export interface MigrationResult {
|
||||
applied: string[];
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export async function runMigrations(): Promise<MigrationResult> {
|
||||
const pool = db();
|
||||
const dir = await migrationsDir();
|
||||
|
||||
const files = (await fs.readdir(dir)).filter((name) => name.endsWith('.sql')).sort();
|
||||
if (files.length === 0) {
|
||||
console.warn(`[db] ve ${dir} nejsou zadne migrace`);
|
||||
return { applied: [], skipped: 0 };
|
||||
}
|
||||
|
||||
const client = await pool.connect();
|
||||
const applied: string[] = [];
|
||||
let skipped = 0;
|
||||
|
||||
try {
|
||||
// Zamek drzi jedna instance, ostatni tady pockaji. Uvolni se s odpojenim.
|
||||
await client.query('SELECT pg_advisory_lock($1)', [LOCK_ID]);
|
||||
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
name text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
`);
|
||||
|
||||
const done = new Set(
|
||||
(await client.query<{ name: string }>('SELECT name FROM schema_migrations')).rows.map(
|
||||
(row) => row.name,
|
||||
),
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
if (done.has(file)) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const sql = await fs.readFile(path.join(dir, file), 'utf8');
|
||||
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
await client.query(sql);
|
||||
await client.query('INSERT INTO schema_migrations (name) VALUES ($1)', [file]);
|
||||
await client.query('COMMIT');
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK').catch(() => undefined);
|
||||
// Rozbita migrace nesmi projit potichu. Bez schematu nema smysl bezet.
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(`Migrace ${file} selhala: ${message}`);
|
||||
}
|
||||
|
||||
applied.push(file);
|
||||
console.info(`[db] migrace ${file} proslá`);
|
||||
}
|
||||
} finally {
|
||||
await client.query('SELECT pg_advisory_unlock($1)', [LOCK_ID]).catch(() => undefined);
|
||||
client.release();
|
||||
}
|
||||
|
||||
if (applied.length === 0) console.info(`[db] schema je aktualni (${skipped} migraci)`);
|
||||
return { applied, skipped };
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
-- Konektory: napojeni jedne firmy na jednu sluzbu.
|
||||
--
|
||||
-- Sluzby zustavaji v katalogu v kodu (src/data/services.ts). Do databaze
|
||||
-- nepatri: jsou to definice, ktere delame my, a repo je u nich zdroj pravdy
|
||||
-- kvuli code review a historii v gitu. Viz documentation/12.
|
||||
--
|
||||
-- `tenant_id` je povinne u kazde business tabulky a indexy zacinaji jim.
|
||||
-- Hodnoty pristupovych udaju jsou sifrovane, tabulka nikdy nedrzi plaintext.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS connectors (
|
||||
id text PRIMARY KEY,
|
||||
tenant_id text NOT NULL,
|
||||
service_id text NOT NULL,
|
||||
name text NOT NULL,
|
||||
base_url text,
|
||||
-- Sifrovane hodnoty poli podle Service.credentials.
|
||||
-- Klic je id pole, hodnota je obalka se sifrou (viz src/db/secretBox.ts).
|
||||
-- JSONB, protoze se cte a zapisuje cele a nikdo se nad tim nedotazuje po polich.
|
||||
secrets jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
status text NOT NULL DEFAULT 'untested',
|
||||
last_check_at timestamptz,
|
||||
last_error text,
|
||||
is_default boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT connectors_status_check
|
||||
CHECK (status IN ('untested', 'ok', 'error'))
|
||||
);
|
||||
|
||||
-- Seznam konektoru firmy je nejcastejsi dotaz, proto index vede tenant_id.
|
||||
CREATE INDEX IF NOT EXISTS connectors_tenant_service_idx
|
||||
ON connectors (tenant_id, service_id);
|
||||
|
||||
-- Vychozi konektor smi byt na dvojici firma a sluzba jen jeden. Vynuceno
|
||||
-- databazi, ne jen kodem: bez toho by soubezne dva zapisy udelaly dva vychozi
|
||||
-- a krok bez vybraneho konektoru by si vybiral podle nahody.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS connectors_one_default_idx
|
||||
ON connectors (tenant_id, service_id)
|
||||
WHERE is_default;
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Pripojeni do Postgresu.
|
||||
*
|
||||
* Databaze je **volitelna**. Kdyz `DATABASE_URL` chybi, aplikace nastartuje
|
||||
* a jede v pameti procesu. Nemuze byt jinak: container, ktery nenastartuje,
|
||||
* je pro AppFactory nefunkcni sluzba (AGENTS.md).
|
||||
*
|
||||
* Rozdil mezi obema rezimy se resi na **jednom miste**, a to pri vyberu
|
||||
* implementace uloziste (`src/data/connectorStore.ts`). Nikde jinde se
|
||||
* nezjistuje, jestli databaze je - jinak by se to rozlezlo po celem kodu
|
||||
* a jedno misto by se zapomnelo.
|
||||
*/
|
||||
|
||||
import { Pool, type PoolClient, type QueryResultRow } from 'pg';
|
||||
import { config } from '../config.js';
|
||||
|
||||
let pool: Pool | null = null;
|
||||
|
||||
/** true = mame kam ukladat. Rozhoduje se podle toho vyber uloziste. */
|
||||
export function isDatabaseEnabled(): boolean {
|
||||
return config.databaseUrl !== '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Pool. Vytvori se az pri prvnim pouziti, aby si aplikace bez databaze
|
||||
* nezakladala spojeni, ktere nikdy nepouzije.
|
||||
*/
|
||||
export function db(): Pool {
|
||||
if (!isDatabaseEnabled()) {
|
||||
throw new Error('DATABASE_URL neni nastavena, databaze se nesmi pouzivat.');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
pool = new Pool({
|
||||
connectionString: config.databaseUrl,
|
||||
max: config.databasePoolMax,
|
||||
// Kratky timeout na ziskani spojeni. Radsi hlasnou chybu nez visici request.
|
||||
connectionTimeoutMillis: 5_000,
|
||||
idleTimeoutMillis: 30_000,
|
||||
...(config.databaseSsl ? { ssl: { rejectUnauthorized: false } } : {}),
|
||||
});
|
||||
|
||||
// Chyba na necinnem spojeni nesmi shodit proces. Pool si spojeni obnovi sam.
|
||||
pool.on('error', (err) => {
|
||||
console.error('[db] chyba na necinnem spojeni:', err.message);
|
||||
});
|
||||
|
||||
console.info(`[db] pool vytvoren, max ${config.databasePoolMax} spojeni`);
|
||||
}
|
||||
|
||||
return pool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dotaz.
|
||||
*
|
||||
* Zamerne tenka obalka, ne query builder. Cely projekt je psany tak, ze server
|
||||
* je autorita a filtr na firmu je povinny argument - to se hlida lip nad
|
||||
* viditelnym SQL nez pod nadstavbou.
|
||||
*/
|
||||
export async function query<T extends QueryResultRow>(
|
||||
sql: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<T[]> {
|
||||
const result = await db().query<T>(sql, params);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/** Prvni radek, nebo undefined. */
|
||||
export async function queryOne<T extends QueryResultRow>(
|
||||
sql: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<T | undefined> {
|
||||
const rows = await query<T>(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Transakce. Pri vyjimce se vraci zpatky.
|
||||
*
|
||||
* Az bude runtime automatizaci, bude tohle to podstatne: vysledek kroku
|
||||
* a zarazeni dalsiho ukolu musi byt jedna transakce, jinak vznikne beh
|
||||
* s hotovym krokem a bez pokracovani (viz documentation/10).
|
||||
*/
|
||||
export async function transaction<T>(work: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await db().connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const result = await work(client);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK').catch(() => undefined);
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Spojeni pro danou firmu.
|
||||
*
|
||||
* Dnes vraci vzdy tentyz pool. Je to zamerny sev: jednou prijde klient, ktery
|
||||
* bude chtit vlastni databazi nebo bude delat tricet procent provozu, a presun
|
||||
* ma byt konfigurace, ne prepisovani dotazu. Podminka je nikdy nespojovat
|
||||
* dotazem dva klienty, coz uz vynucuje povinny argument `tenantIds`
|
||||
* v ulozistich. Podrobnosti v documentation/10-runtime-a-kapacita.md.
|
||||
*/
|
||||
export function dbFor(_tenantId: string): Pool {
|
||||
return db();
|
||||
}
|
||||
|
||||
export interface DatabaseHealth {
|
||||
enabled: boolean;
|
||||
ok: boolean;
|
||||
latencyMs: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
let cachedHealth: { at: number; value: DatabaseHealth } | null = null;
|
||||
const HEALTH_CACHE_MS = 5_000;
|
||||
|
||||
/**
|
||||
* Stav databaze pro `/health/ready`.
|
||||
*
|
||||
* Vysledek se par sekund cachuje, aby monitoring nedelal dotaz pri kazdem
|
||||
* pingu. `/health` na databazi zamerne nezavisi - kratky vypadek DB by jinak
|
||||
* vedl k restartovani containeru, coz nic nespravi.
|
||||
*/
|
||||
export async function databaseHealth(): Promise<DatabaseHealth> {
|
||||
if (!isDatabaseEnabled()) {
|
||||
return { enabled: false, ok: true, latencyMs: null, error: null };
|
||||
}
|
||||
|
||||
if (cachedHealth && Date.now() - cachedHealth.at < HEALTH_CACHE_MS) {
|
||||
return cachedHealth.value;
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
let value: DatabaseHealth;
|
||||
try {
|
||||
await query('select 1');
|
||||
value = { enabled: true, ok: true, latencyMs: Date.now() - startedAt, error: null };
|
||||
} catch (err) {
|
||||
value = {
|
||||
enabled: true,
|
||||
ok: false,
|
||||
latencyMs: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
cachedHealth = { at: Date.now(), value };
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Zavre pool pri ukonceni procesu. */
|
||||
export async function closeDatabase(): Promise<void> {
|
||||
if (!pool) return;
|
||||
await pool.end().catch((err: unknown) => {
|
||||
console.warn('[db] pool se nepodarilo zavrit:', err);
|
||||
});
|
||||
pool = null;
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Sifrovani pristupovych udaju konektoru.
|
||||
*
|
||||
* Do databaze nikdy nesmi plaintext. Kdyby ano, tak by kazda zaloha, kazdy dump
|
||||
* pri ladeni a kazdy, kdo ma pristup ke cteni, mel klientske klice k iDokladu.
|
||||
*
|
||||
* AES-256-GCM: sifruje a zaroven overuje, ze se s daty nikdo nehral. Nahodne
|
||||
* IV pro kazdou hodnotu, aby dve stejne hodnoty nedaly stejnou sifru.
|
||||
*
|
||||
* `v` je verze klice. Vymena klice pak znamena precist starym, zapsat novym,
|
||||
* ne zahodit vsechna napojeni.
|
||||
*/
|
||||
|
||||
import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
|
||||
import { config } from '../config.js';
|
||||
|
||||
/** Obalka, ktera se uklada do JSONB. */
|
||||
export interface SealedValue {
|
||||
/** Verze klice. */
|
||||
v: number;
|
||||
/** Inicializacni vektor, base64url. */
|
||||
iv: string;
|
||||
/** Autentizacni tag GCM, base64url. */
|
||||
tag: string;
|
||||
/** Sifrovana hodnota, base64url. */
|
||||
data: string;
|
||||
}
|
||||
|
||||
const ALGORITHM = 'aes-256-gcm';
|
||||
const KEY_VERSION = 1;
|
||||
|
||||
/**
|
||||
* Klic z konfigurace, srovnany na 32 bajtu.
|
||||
*
|
||||
* SHA-256 z hodnoty promenne, aby fungoval jakkoliv dlouhy retezec. Neni to
|
||||
* derivace hesla (na to by patril scrypt), ale `SECRETS_KEY` ma byt nahodny
|
||||
* klic, ne heslo - a to je v dokumentaci napsane.
|
||||
*/
|
||||
function key(): Buffer {
|
||||
if (config.secretsKey === '') {
|
||||
throw new Error('SECRETS_KEY neni nastavena, pristupove udaje nelze sifrovat.');
|
||||
}
|
||||
return createHash('sha256').update(config.secretsKey).digest();
|
||||
}
|
||||
|
||||
export function canSealSecrets(): boolean {
|
||||
return config.secretsKey !== '';
|
||||
}
|
||||
|
||||
export function seal(value: string): SealedValue {
|
||||
const iv = randomBytes(12);
|
||||
const cipher = createCipheriv(ALGORITHM, key(), iv);
|
||||
const data = Buffer.concat([cipher.update(value, 'utf8'), cipher.final()]);
|
||||
|
||||
return {
|
||||
v: KEY_VERSION,
|
||||
iv: iv.toString('base64url'),
|
||||
tag: cipher.getAuthTag().toString('base64url'),
|
||||
data: data.toString('base64url'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rozsifruje hodnotu.
|
||||
*
|
||||
* Vraci `null`, kdyz to nejde - typicky po vymene klice bez prevodu dat.
|
||||
* Zamerne se **nepada**: jeden nerozsifrovatelny konektor nesmi shodit seznam
|
||||
* vsech ostatnich. Chybejici hodnota se pak chova jako nevyplnena, takze
|
||||
* uzivatel dostane "chybi udaje" a muze je zadat znovu.
|
||||
*/
|
||||
export function open(sealed: unknown): string | null {
|
||||
if (!isSealed(sealed)) return null;
|
||||
|
||||
try {
|
||||
const decipher = createDecipheriv(ALGORITHM, key(), Buffer.from(sealed.iv, 'base64url'));
|
||||
decipher.setAuthTag(Buffer.from(sealed.tag, 'base64url'));
|
||||
const plain = Buffer.concat([
|
||||
decipher.update(Buffer.from(sealed.data, 'base64url')),
|
||||
decipher.final(),
|
||||
]);
|
||||
return plain.toString('utf8');
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[secrets] hodnotu nelze rozsifrovat (verze klice ${sealed.v}): ` +
|
||||
(err instanceof Error ? err.message : String(err)),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isSealed(value: unknown): value is SealedValue {
|
||||
if (value === null || typeof value !== 'object') return false;
|
||||
const record = value as Record<string, unknown>;
|
||||
return (
|
||||
typeof record.iv === 'string' &&
|
||||
typeof record.tag === 'string' &&
|
||||
typeof record.data === 'string' &&
|
||||
typeof record.v === 'number'
|
||||
);
|
||||
}
|
||||
|
||||
/** Zasifruje celou sadu hodnot. */
|
||||
export function sealAll(values: Record<string, string>): Record<string, SealedValue> {
|
||||
const result: Record<string, SealedValue> = {};
|
||||
for (const [id, value] of Object.entries(values)) {
|
||||
if (value === '') continue;
|
||||
result[id] = seal(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Rozsifruje celou sadu. Co nejde rozsifrovat, se vynecha a zaloguje. */
|
||||
export function openAll(sealed: unknown): Record<string, string> {
|
||||
if (sealed === null || typeof sealed !== 'object') return {};
|
||||
|
||||
const result: Record<string, string> = {};
|
||||
for (const [id, value] of Object.entries(sealed as Record<string, unknown>)) {
|
||||
const plain = open(value);
|
||||
if (plain !== null) result[id] = plain;
|
||||
else console.warn(`[secrets] pole ${id} se nepodarilo rozsifrovat, chova se jako nevyplnene`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
Reference in New Issue
Block a user