From 80f42873a115bf7bc00acde4f0d0b100b24ba0cd Mon Sep 17 00:00:00 2001 From: jona159 Date: Sat, 13 Jun 2026 11:10:42 +0200 Subject: [PATCH 01/12] feat(wip): tiered rate limits --- .env.example | 3 + README.md | 14 +- app/lib/api-route-matching.ts | 55 ++++ app/lib/api-routes.ts | 24 +- app/middleware/rate-limit-api.server.ts | 325 ++++++++++++++++++++++++ app/middleware/tos-api.server.ts | 41 +-- app/routes/api.ts | 6 +- tests/lib/api-rate-limit.spec.ts | 171 +++++++++++++ 8 files changed, 601 insertions(+), 38 deletions(-) create mode 100644 app/lib/api-route-matching.ts create mode 100644 app/middleware/rate-limit-api.server.ts create mode 100644 tests/lib/api-rate-limit.spec.ts diff --git a/.env.example b/.env.example index df07d61a..0f113a37 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,9 @@ JWT_VALIDITY_MS=3600000 # 1 hour REFRESH_TOKEN_SECRET="I ALSO WANT TO BE CHANGED" REFRESH_TOKEN_ALGORITHM="sha256" REFRESH_TOKEN_VALIDITY_MS=604800000 # 1 week +# Optional JSON map for higher API rate limit tiers. Users match verified JWT +# subjects; credential hashes are SHA-256 hashes of API/service key values. +API_RATE_LIMIT_TIERS='{"education":{"multiplier":5,"emailDomains":["school.example"],"users":["teacher@school.example"],"credentialHashes":[]}}' NOMINATIM_SEARCH_API="https://nominatim.openstreetmap.org/search" diff --git a/README.md b/README.md index 59df44d9..bf235dec 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ running as a public beta at Screenshot OSeM - ## Project setup If you do need to set the project up locally yourself, feel free to follow these @@ -24,10 +23,15 @@ instructions: You can configure the API endpoint using the following environmental variables: -| ENV | Default value | -| ------------ | -------------------------------------------------------- | -| OSEM_API_URL | https://api.testing.opensensemap.org | -| DATABASE_URL | `postgresql://postgres:postgres@localhost:5432/postgres` | +| ENV | Default value | +| -------------------- | -------------------------------------------------------- | +| OSEM_API_URL | https://api.testing.opensensemap.org | +| DATABASE_URL | `postgresql://postgres:postgres@localhost:5432/postgres` | +| API_RATE_LIMIT_TIERS | Optional JSON map for higher API rate limit tiers | + +`API_RATE_LIMIT_TIERS` can grant higher limits by verified JWT user email, email +domain, or SHA-256 credential hash. Example: +`{"education":{"multiplier":5,"emailDomains":["school.example"],"users":["teacher@school.example"],"credentialHashes":[]}}`. You can create a copy of `.env.example`, rename it to `.env` and set the values. To run a local development version, you only need to adjust the `OSEM_API_URL` diff --git a/app/lib/api-route-matching.ts b/app/lib/api-route-matching.ts new file mode 100644 index 00000000..41239882 --- /dev/null +++ b/app/lib/api-route-matching.ts @@ -0,0 +1,55 @@ +import { type RouteInfo } from '~/lib/api-routes' + +type HttpMethod = RouteInfo['method'] | '*' + +export type CompiledApiRoute = { + route: T + kind: 'auth' | 'noauth' + method: HttpMethod + matcher: RegExp +} + +export function apiRoutePath(path: string) { + if (path === '/' || path === '') return '/api' + return `/api/${path.replace(/^\/+/, '')}` +} + +export function routeToRegex(apiPathPattern: string) { + const escaped = apiPathPattern + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + .replace(/\\:([A-Za-z0-9_]+)/g, '[^/]+') + const trailingSlash = apiPathPattern === '/api' ? '/?' : '' + return new RegExp(`^${escaped}${trailingSlash}$`) +} + +export function compileApiRoutes(routes: { + noauth: RouteInfo[] + auth: RouteInfo[] +}): CompiledApiRoute[] { + return [ + ...routes.noauth.map((route) => ({ + route, + kind: 'noauth' as const, + method: route.method, + matcher: routeToRegex(apiRoutePath(route.path)), + })), + ...routes.auth.map((route) => ({ + route, + kind: 'auth' as const, + method: route.method, + matcher: routeToRegex(apiRoutePath(route.path)), + })), + ] +} + +export function findApiRoute( + request: Request, + pathname: string, + compiledRoutes: CompiledApiRoute[], +) { + const method = request.method as HttpMethod + return compiledRoutes.find((rule) => { + if (rule.method !== '*' && rule.method !== method) return false + return rule.matcher.test(pathname) + }) +} diff --git a/app/lib/api-routes.ts b/app/lib/api-routes.ts index 87f61e7f..bddeb5d0 100644 --- a/app/lib/api-routes.ts +++ b/app/lib/api-routes.ts @@ -1,8 +1,20 @@ -type RouteInfo = { +export type RateLimitTierConfig = { + maxRequests?: number + multiplier?: number +} + +export type RateLimitConfig = { + windowMs: number + maxRequests: number + tiers?: Record +} + +export type RouteInfo = { path: string method: 'GET' | 'PUT' | 'POST' | 'DELETE' tosExempt: boolean deprecationNotice?: string + rateLimit?: RateLimitConfig | false } export const apiRoutes: { noauth: RouteInfo[]; auth: RouteInfo[] } = { @@ -36,11 +48,13 @@ export const apiRoutes: { noauth: RouteInfo[]; auth: RouteInfo[] } = { path: `boxes`, method: 'GET', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 120 }, }, { path: `boxes/data`, method: 'GET', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 60 }, }, // { // path: `boxes/:boxId`, @@ -72,36 +86,43 @@ export const apiRoutes: { noauth: RouteInfo[]; auth: RouteInfo[] } = { path: `boxes/:boxId/data`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 600 }, }, { path: `boxes/:boxId/:sensorId`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 600 }, }, { path: `users/register`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 5 }, }, { path: `users/request-password-reset`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 5 }, }, { path: `users/password-reset`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 5 }, }, { path: `users/confirm-email`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 10 }, }, { path: `users/sign-in`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 10 }, }, { path: `discourse/sso`, @@ -114,6 +135,7 @@ export const apiRoutes: { noauth: RouteInfo[]; auth: RouteInfo[] } = { path: `users/refresh-auth`, method: 'POST', tosExempt: true, + rateLimit: { windowMs: 60_000, maxRequests: 60 }, }, { path: `users/me`, diff --git a/app/middleware/rate-limit-api.server.ts b/app/middleware/rate-limit-api.server.ts new file mode 100644 index 00000000..660f1e97 --- /dev/null +++ b/app/middleware/rate-limit-api.server.ts @@ -0,0 +1,325 @@ +import { createHash } from 'node:crypto' +import jsonwebtoken, { type Algorithm } from 'jsonwebtoken' +import { compileApiRoutes, findApiRoute } from '~/lib/api-route-matching' +import { + apiRoutes, + type RateLimitConfig, + type RouteInfo, +} from '~/lib/api-routes' + +const { verify } = jsonwebtoken + +type HttpMethod = RouteInfo['method'] + +type RateLimitBucket = { + count: number + resetAt: number +} + +type RateLimitResult = { + allowed: boolean + limit: RateLimitConfig + tier: string + remaining: number + resetAt: number + retryAfterSeconds: number +} + +type RateLimitTierGrant = { + multiplier?: number + users?: string[] + emailDomains?: string[] + credentialHashes?: string[] +} + +type RateLimitTier = { + name: string + multiplier: number + users: Set + emailDomains: Set + credentialHashes: Set +} + +const DEFAULT_WINDOW_MS = 60_000 + +const DEFAULT_LIMITS: Record< + 'auth' | 'noauth', + Record +> = { + noauth: { + GET: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 300 }, + POST: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 60 }, + PUT: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 60 }, + DELETE: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 60 }, + }, + auth: { + GET: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 600 }, + POST: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 180 }, + PUT: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 120 }, + DELETE: { windowMs: DEFAULT_WINDOW_MS, maxRequests: 120 }, + }, +} + +const compiledApiRoutes = compileApiRoutes(apiRoutes) +const buckets = new Map() +let lastCleanupAt = 0 +let cachedTierConfigRaw: string | undefined +let cachedTierConfig: RateLimitTier[] = [] + +function json(body: unknown, status = 200, headers?: HeadersInit) { + return new Response(JSON.stringify(body), { + status, + headers: { + 'content-type': 'application/json; charset=utf-8', + ...headers, + }, + }) +} + +function getRateLimit(route: (typeof compiledApiRoutes)[number]) { + if (route.route.rateLimit === false) return null + return route.route.rateLimit ?? DEFAULT_LIMITS[route.kind][route.route.method] +} + +function normalizeEmail(value: string) { + return value.trim().toLowerCase() +} + +function normalizeDomain(value: string) { + return value.trim().replace(/^@/, '').toLowerCase() +} + +function parseTierConfig() { + const raw = process.env.API_RATE_LIMIT_TIERS + if (raw === cachedTierConfigRaw) return cachedTierConfig + + cachedTierConfigRaw = raw + cachedTierConfig = [] + + if (!raw) return cachedTierConfig + + try { + const parsed = JSON.parse(raw) as Record + cachedTierConfig = Object.entries(parsed).map(([name, grant]) => ({ + name, + multiplier: grant.multiplier ?? 1, + users: new Set((grant.users ?? []).map(normalizeEmail)), + emailDomains: new Set((grant.emailDomains ?? []).map(normalizeDomain)), + credentialHashes: new Set( + (grant.credentialHashes ?? []).map((hash) => hash.trim().toLowerCase()), + ), + })) + } catch (error) { + console.error('Invalid API_RATE_LIMIT_TIERS configuration', error) + } + + return cachedTierConfig +} + +function getClientAddress(request: Request) { + const forwardedFor = request.headers.get('x-forwarded-for') + if (forwardedFor) return forwardedFor.split(',')[0]?.trim() + + return ( + request.headers.get('cf-connecting-ip') ?? + request.headers.get('fly-client-ip') ?? + request.headers.get('x-real-ip') ?? + 'anonymous' + ) +} + +function getRequesterCredential(request: Request) { + const credential = + request.headers.get('authorization') ?? + request.headers.get('x-osem-device-api-key') ?? + request.headers.get('x-service-key') + + if (!credential) return null + + return { + raw: credential, + hash: createHash('sha256').update(credential).digest('hex'), + } +} + +function getRequesterKey(request: Request) { + const credential = getRequesterCredential(request) + if (credential) return `credential:${credential.hash}` + + return `ip:${getClientAddress(request)}` +} + +function getBearerToken(request: Request) { + const rawAuthorizationHeader = request.headers.get('authorization') + if (!rawAuthorizationHeader) return null + + const [bearer, jwtString] = rawAuthorizationHeader.split(' ') + if (bearer !== 'Bearer' || !jwtString) return null + + return jwtString +} + +function getVerifiedJwtPayload(request: Request) { + const jwtString = getBearerToken(request) + if (!jwtString) return null + + const { JWT_ALGORITHM, JWT_ISSUER, JWT_SECRET } = process.env + if (!JWT_ALGORITHM || !JWT_ISSUER || !JWT_SECRET) return null + + try { + const decoded = verify(jwtString, JWT_SECRET, { + algorithms: [JWT_ALGORITHM as Algorithm], + issuer: JWT_ISSUER, + }) + + return typeof decoded === 'string' ? null : decoded + } catch { + return null + } +} + +function emailMatchesTier(email: string, tier: RateLimitTier) { + const normalizedEmail = normalizeEmail(email) + const domain = normalizedEmail.split('@')[1] + + return ( + tier.users.has(normalizedEmail) || + (domain ? tier.emailDomains.has(domain) : false) + ) +} + +function resolveRateLimitTier(request: Request) { + const tiers = parseTierConfig() + if (tiers.length === 0) return { name: 'default', multiplier: 1 } + + const credential = getRequesterCredential(request) + if (credential) { + const matchedCredentialTier = tiers.find((tier) => + tier.credentialHashes.has(credential.hash), + ) + if (matchedCredentialTier) return matchedCredentialTier + } + + const jwtPayload = getVerifiedJwtPayload(request) + if (jwtPayload?.sub) { + const matchedUserTier = tiers.find((tier) => + emailMatchesTier(String(jwtPayload.sub), tier), + ) + if (matchedUserTier) return matchedUserTier + } + + return { name: 'default', multiplier: 1 } +} + +function resolveTierLimit( + baseLimit: RateLimitConfig, + tier: { name: string; multiplier: number }, +) { + const tierOverride = baseLimit.tiers?.[tier.name] + const multiplier = tierOverride?.multiplier ?? tier.multiplier + const maxRequests = + tierOverride?.maxRequests ?? + Math.max(1, Math.floor(baseLimit.maxRequests * multiplier)) + + return { + windowMs: baseLimit.windowMs, + maxRequests, + } satisfies RateLimitConfig +} + +function cleanupBuckets(now: number) { + if (now - lastCleanupAt < DEFAULT_WINDOW_MS) return + lastCleanupAt = now + + for (const [key, bucket] of buckets.entries()) { + if (bucket.resetAt <= now) buckets.delete(key) + } +} + +export function resetApiRateLimitForTests() { + buckets.clear() + lastCleanupAt = 0 + cachedTierConfigRaw = undefined + cachedTierConfig = [] +} + +export function checkApiRateLimit(request: Request, now = Date.now()) { + const url = new URL(request.url) + const route = findApiRoute(request, url.pathname, compiledApiRoutes) + if (!route) return null + + const baseLimit = getRateLimit(route) + if (!baseLimit) return null + + const tier = resolveRateLimitTier(request) + const limit = resolveTierLimit(baseLimit, tier) + + cleanupBuckets(now) + + const key = [ + route.kind, + route.route.method, + route.route.path, + tier.name, + getRequesterKey(request), + ].join(':') + + const existing = buckets.get(key) + const bucket = + existing && existing.resetAt > now + ? existing + : { count: 0, resetAt: now + limit.windowMs } + + bucket.count += 1 + buckets.set(key, bucket) + + const remaining = Math.max(limit.maxRequests - bucket.count, 0) + const retryAfterSeconds = Math.ceil((bucket.resetAt - now) / 1000) + + return { + allowed: bucket.count <= limit.maxRequests, + limit, + tier: tier.name, + remaining, + resetAt: bucket.resetAt, + retryAfterSeconds, + } satisfies RateLimitResult +} + +function rateLimitHeaders(result: RateLimitResult) { + return { + 'RateLimit-Limit': String(result.limit.maxRequests), + 'RateLimit-Remaining': String(result.remaining), + 'RateLimit-Reset': String(Math.ceil(result.resetAt / 1000)), + 'RateLimit-Policy': `${result.limit.maxRequests};w=${Math.ceil( + result.limit.windowMs / 1000, + )};tier="${result.tier}"`, + 'Retry-After': String(result.retryAfterSeconds), + } +} + +export async function apiRateLimitMiddleware( + { request }: { request: Request }, + next: () => Promise, +) { + const result = checkApiRateLimit(request) + if (!result) return next() + + if (!result.allowed) { + return json( + { + code: 'rate_limit_exceeded', + message: 'Too many requests. Please retry after the rate limit resets.', + }, + 429, + rateLimitHeaders(result), + ) + } + + const response = await next() + for (const [header, value] of Object.entries(rateLimitHeaders(result))) { + if (header === 'Retry-After') continue + response.headers.set(header, value) + } + return response +} diff --git a/app/middleware/tos-api.server.ts b/app/middleware/tos-api.server.ts index 0ecec8ea..cfd5c3cd 100644 --- a/app/middleware/tos-api.server.ts +++ b/app/middleware/tos-api.server.ts @@ -1,9 +1,12 @@ import { getTosRequirementForUser } from '~/db/models/tos.server' +import { + compileApiRoutes, + findApiRoute, + type CompiledApiRoute, +} from '~/lib/api-route-matching' import { apiRoutes } from '~/lib/api-routes' import { getUserFromJwt } from '~/lib/jwt' -type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' - function json(body: unknown, status = 200) { return new Response(JSON.stringify(body), { status, @@ -11,41 +14,17 @@ function json(body: unknown, status = 200) { }) } -type CompiledRule = { - method: HttpMethod | '*' - matcher: RegExp -} - -/** - * Convert a route pattern like "/api/users/me/boxes/:boxId" - * into a regex like ^/api/users/me/boxes/[^/]+$ - */ -function routeToRegex(apiPathPattern: string) { - const escaped = apiPathPattern - .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') // escape regex special chars - .replace(/\\:([A-Za-z0-9_]+)/g, '[^/]+') // replace ":param" segments - return new RegExp(`^${escaped}$`) -} - /** * Build allowlist from route metadata: * - `auth` routes with `tosExempt: true` bypass ToS checks */ -const API_TOS_ALLOWLIST: CompiledRule[] = [ - ...apiRoutes.auth - .filter((r: any) => r.tosExempt) - .map((r: any) => ({ - method: r.method as HttpMethod, - matcher: routeToRegex(`/api/${r.path}`), - })), -] +const API_TOS_ALLOWLIST: CompiledApiRoute[] = compileApiRoutes({ + noauth: [], + auth: apiRoutes.auth.filter((route) => route.tosExempt), +}) function isAllowedApi(request: Request, pathname: string) { - const method = request.method as HttpMethod - return API_TOS_ALLOWLIST.some((rule) => { - if (rule.method !== '*' && rule.method !== method) return false - return rule.matcher.test(pathname) - }) + return Boolean(findApiRoute(request, pathname, API_TOS_ALLOWLIST)) } export async function tosApiMiddleware( diff --git a/app/routes/api.ts b/app/routes/api.ts index 55bce50a..23f37273 100644 --- a/app/routes/api.ts +++ b/app/routes/api.ts @@ -1,9 +1,13 @@ import { type Route } from '../+types/root' import { apiRoutes as routes } from '~/lib/api-routes' +import { apiRateLimitMiddleware } from '~/middleware/rate-limit-api.server' import { tosApiMiddleware } from '~/middleware/tos-api.server' export { routes } -export const middleware: Route.MiddlewareFunction[] = [tosApiMiddleware] +export const middleware: Route.MiddlewareFunction[] = [ + apiRateLimitMiddleware, + tosApiMiddleware, +] export async function loader({}: Route.LoaderArgs) { const lines = [ diff --git a/tests/lib/api-rate-limit.spec.ts b/tests/lib/api-rate-limit.spec.ts new file mode 100644 index 00000000..41782f41 --- /dev/null +++ b/tests/lib/api-rate-limit.spec.ts @@ -0,0 +1,171 @@ +import { createHash } from 'node:crypto' +import jsonwebtoken from 'jsonwebtoken' +import { + checkApiRateLimit, + resetApiRateLimitForTests, +} from '~/middleware/rate-limit-api.server' + +const BASE_URL = 'http://localhost:4200' +const { sign } = jsonwebtoken + +function restoreEnv(name: string, value: string | undefined) { + if (value === undefined) delete process.env[name] + else process.env[name] = value +} + +describe('API rate limiting', () => { + beforeEach(() => { + delete process.env.API_RATE_LIMIT_TIERS + resetApiRateLimitForTests() + }) + + it('limits sensitive auth endpoints with route-specific limits', () => { + let result: ReturnType = null + + for (let i = 0; i < 10; i++) { + result = checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { 'x-forwarded-for': '203.0.113.1' }, + }), + 1_000, + ) + expect(result?.allowed).toBe(true) + } + + result = checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { 'x-forwarded-for': '203.0.113.1' }, + }), + 1_000, + ) + + expect(result?.allowed).toBe(false) + expect(result?.limit.maxRequests).toBe(10) + expect(result?.retryAfterSeconds).toBe(60) + }) + + it('separates buckets by credential before falling back to the client address', () => { + for (let i = 0; i < 10; i++) { + checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { + authorization: 'Bearer first-token', + 'x-forwarded-for': '203.0.113.2', + }, + }), + 1_000, + ) + } + + const sameAddressDifferentCredential = checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { + authorization: 'Bearer second-token', + 'x-forwarded-for': '203.0.113.2', + }, + }), + 1_000, + ) + + expect(sameAddressDifferentCredential?.allowed).toBe(true) + expect(sameAddressDifferentCredential?.remaining).toBe(9) + }) + + it('can grant a higher tier by credential hash', () => { + const credential = 'school-device-api-key' + const credentialHash = createHash('sha256').update(credential).digest('hex') + process.env.API_RATE_LIMIT_TIERS = JSON.stringify({ + education: { + multiplier: 2, + credentialHashes: [credentialHash], + }, + }) + + const result = checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { 'x-osem-device-api-key': credential }, + }), + 1_000, + ) + + expect(result?.tier).toBe('education') + expect(result?.limit.maxRequests).toBe(20) + expect(result?.remaining).toBe(19) + }) + + it('can grant a higher tier by verified JWT email domain', () => { + const originalAlgorithm = process.env.JWT_ALGORITHM + const originalIssuer = process.env.JWT_ISSUER + const originalSecret = process.env.JWT_SECRET + + try { + process.env.JWT_ALGORITHM = 'HS256' + process.env.JWT_ISSUER = 'opensensemap-test' + process.env.JWT_SECRET = 'test-secret' + process.env.API_RATE_LIMIT_TIERS = JSON.stringify({ + education: { + multiplier: 3, + emailDomains: ['school.example'], + }, + }) + + const token = sign({ role: 'user' }, process.env.JWT_SECRET, { + algorithm: 'HS256', + issuer: process.env.JWT_ISSUER, + subject: 'teacher@school.example', + }) + + const result = checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { authorization: `Bearer ${token}` }, + }), + 1_000, + ) + + expect(result?.tier).toBe('education') + expect(result?.limit.maxRequests).toBe(30) + } finally { + restoreEnv('JWT_ALGORITHM', originalAlgorithm) + restoreEnv('JWT_ISSUER', originalIssuer) + restoreEnv('JWT_SECRET', originalSecret) + } + }) + + it('opens a new bucket after the configured window resets', () => { + for (let i = 0; i < 10; i++) { + checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { 'x-forwarded-for': '203.0.113.3' }, + }), + 1_000, + ) + } + + const resetResult = checkApiRateLimit( + new Request(`${BASE_URL}/api/users/sign-in`, { + method: 'POST', + headers: { 'x-forwarded-for': '203.0.113.3' }, + }), + 61_001, + ) + + expect(resetResult?.allowed).toBe(true) + expect(resetResult?.remaining).toBe(9) + }) + + it('does not rate limit URLs that are not defined in apiRoutes', () => { + const result = checkApiRateLimit( + new Request(`${BASE_URL}/api/unknown`, { method: 'GET' }), + 1_000, + ) + + expect(result).toBeNull() + }) +}) From 50dd727048529cdca92883a6852fb7a52e71bce3 Mon Sep 17 00:00:00 2001 From: jona159 Date: Wed, 8 Jul 2026 12:25:34 +0200 Subject: [PATCH 02/12] feat: mv grant entries from environment variable to db table --- .env.example | 3 - README.md | 22 +- app/db/drizzle/0046_flowery_blacklash.sql | 11 + app/db/drizzle/meta/0046_snapshot.json | 1736 +++++++++++++++++++++ app/db/drizzle/meta/_journal.json | 9 +- app/db/schema/index.ts | 1 + app/db/schema/rate-limit-grant.ts | 30 + 7 files changed, 1794 insertions(+), 18 deletions(-) create mode 100644 app/db/drizzle/0046_flowery_blacklash.sql create mode 100644 app/db/drizzle/meta/0046_snapshot.json create mode 100644 app/db/schema/rate-limit-grant.ts diff --git a/.env.example b/.env.example index 850a091d..9cbea049 100644 --- a/.env.example +++ b/.env.example @@ -10,9 +10,6 @@ JWT_VALIDITY_MS=3600000 # 1 hour REFRESH_TOKEN_SECRET="I ALSO WANT TO BE CHANGED" REFRESH_TOKEN_ALGORITHM="sha256" REFRESH_TOKEN_VALIDITY_MS=604800000 # 1 week -# Optional JSON map for higher API rate limit tiers. Users match verified JWT -# subjects; credential hashes are SHA-256 hashes of API/service key values. -API_RATE_LIMIT_TIERS='{"education":{"multiplier":5,"emailDomains":["school.example"],"users":["teacher@school.example"],"credentialHashes":[]}}' NOMINATIM_SEARCH_API="https://nominatim.openstreetmap.org/search" diff --git a/README.md b/README.md index e71c6b39..39c8d60a 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,10 @@ instructions: You can configure the API endpoint using the following environmental variables: -| ENV | Default value | -| -------------------- | -------------------------------------------------------- | -| OSEM_API_URL | https://api.testing.opensensemap.org | -| DATABASE_URL | `postgresql://postgres:postgres@localhost:5432/postgres` | -| API_RATE_LIMIT_TIERS | Optional JSON map for higher API rate limit tiers | - -`API_RATE_LIMIT_TIERS` can grant higher limits by verified JWT user email, email -domain, or SHA-256 credential hash. Example: -`{"education":{"multiplier":5,"emailDomains":["school.example"],"users":["teacher@school.example"],"credentialHashes":[]}}`. +| ENV | Default value | +| ------------ | -------------------------------------------------------- | +| OSEM_API_URL | https://api.testing.opensensemap.org | +| DATABASE_URL | `postgresql://postgres:postgres@localhost:5432/postgres` | You can create a copy of `.env.example`, rename it to `.env` and set the values. To run a local development version, you only need to adjust the `OSEM_API_URL` @@ -167,11 +162,10 @@ flexibility to adjust the outputs to the needs of the respective use case. ##### Documenting an API Route -API route documentation is generated from route-local `zod-openapi` -definitions. Each API route can export an `openapi` object that describes the -route's OpenAPI path item. Request bodies, response bodies, path parameters, -query parameters, and headers should be described with Zod schemas wherever -possible. +API route documentation is generated from route-local `zod-openapi` definitions. +Each API route can export an `openapi` object that describes the route's OpenAPI +path item. Request bodies, response bodies, path parameters, query parameters, +and headers should be described with Zod schemas wherever possible. The main benefit of this approach is that schemas can be shared between validation and documentation. This keeps the OpenAPI documentation closer to the diff --git a/app/db/drizzle/0046_flowery_blacklash.sql b/app/db/drizzle/0046_flowery_blacklash.sql new file mode 100644 index 00000000..de8d137f --- /dev/null +++ b/app/db/drizzle/0046_flowery_blacklash.sql @@ -0,0 +1,11 @@ +CREATE TABLE "rate_limit_grant" ( + "id" text PRIMARY KEY NOT NULL, + "kind" text NOT NULL, + "value" text NOT NULL, + "tier" text NOT NULL, + "enabled" boolean DEFAULT true NOT NULL, + "note" text, + "expires_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); diff --git a/app/db/drizzle/meta/0046_snapshot.json b/app/db/drizzle/meta/0046_snapshot.json new file mode 100644 index 00000000..d27590f9 --- /dev/null +++ b/app/db/drizzle/meta/0046_snapshot.json @@ -0,0 +1,1736 @@ +{ + "id": "1ca42245-44ed-4870-b4b3-796b1c314efc", + "prevId": "ef9b819e-831a-4fac-8183-8eb681b40a71", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.device": { + "name": "device", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tags": { + "name": "tags", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "ARRAY[]::text[]" + }, + "link": { + "name": "link", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "use_auth": { + "name": "use_auth", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "apiKey": { + "name": "apiKey", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exposure": { + "name": "exposure", + "type": "exposure", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'inactive'" + }, + "model": { + "name": "model", + "type": "model", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'custom'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "orphaned_at": { + "name": "orphaned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "latitude": { + "name": "latitude", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "longitude": { + "name": "longitude", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "sensor_wiki_model": { + "name": "sensor_wiki_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "device_user_id_user_id_fk": { + "name": "device_user_id_user_id_fk", + "tableFrom": "device", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.device_to_location": { + "name": "device_to_location", + "schema": "", + "columns": { + "device_id": { + "name": "device_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location_id": { + "name": "location_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "device_to_location_device_id_device_id_fk": { + "name": "device_to_location_device_id_device_id_fk", + "tableFrom": "device_to_location", + "tableTo": "device", + "columnsFrom": [ + "device_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + }, + "device_to_location_location_id_location_id_fk": { + "name": "device_to_location_location_id_location_id_fk", + "tableFrom": "device_to_location", + "tableTo": "location", + "columnsFrom": [ + "location_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "device_to_location_device_id_location_id_time_pk": { + "name": "device_to_location_device_id_location_id_time_pk", + "columns": [ + "device_id", + "location_id", + "time" + ] + } + }, + "uniqueConstraints": { + "device_to_location_device_id_location_id_time_unique": { + "name": "device_to_location_device_id_location_id_time_unique", + "nullsNotDistinct": false, + "columns": [ + "device_id", + "location_id", + "time" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.measurement": { + "name": "measurement", + "schema": "", + "columns": { + "sensor_id": { + "name": "sensor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp (3) with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "value": { + "name": "value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "location_id": { + "name": "location_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "measurement_location_id_location_id_fk": { + "name": "measurement_location_id_location_id_fk", + "tableFrom": "measurement", + "tableTo": "location", + "columnsFrom": [ + "location_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "measurement_sensor_id_time_unique": { + "name": "measurement_sensor_id_time_unique", + "nullsNotDistinct": false, + "columns": [ + "sensor_id", + "time" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.password": { + "name": "password", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "password_user_id_user_id_fk": { + "name": "password_user_id_user_id_fk", + "tableFrom": "password", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.profile": { + "name": "profile", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "home_latitude": { + "name": "home_latitude", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "home_longitude": { + "name": "home_longitude", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "home_zoom": { + "name": "home_zoom", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "profile_user_id_user_id_fk": { + "name": "profile_user_id_user_id_fk", + "tableFrom": "profile", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "profile_user_id_unique": { + "name": "profile_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.profile_image": { + "name": "profile_image", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "alt_text": { + "name": "alt_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "blob": { + "name": "blob", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "profile_image_profile_id_profile_id_fk": { + "name": "profile_image_profile_id_profile_id_fk", + "tableFrom": "profile_image", + "tableTo": "profile", + "columnsFrom": [ + "profile_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sensor": { + "name": "sensor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "unit": { + "name": "unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sensor_type": { + "name": "sensor_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'inactive'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "device_id": { + "name": "device_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sensor_wiki_type": { + "name": "sensor_wiki_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sensor_wiki_phenomenon": { + "name": "sensor_wiki_phenomenon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sensor_wiki_unit": { + "name": "sensor_wiki_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lastMeasurement": { + "name": "lastMeasurement", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "sensor_device_id_device_id_fk": { + "name": "sensor_device_id_device_id_fk", + "tableFrom": "sensor", + "tableTo": "device", + "columnsFrom": [ + "device_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "unconfirmed_email": { + "name": "unconfirmed_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "theme_preference": { + "name": "theme_preference", + "type": "theme_preference", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'system'" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'en_US'" + }, + "email_is_confirmed": { + "name": "email_is_confirmed", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "newsletter_opt_in": { + "name": "newsletter_opt_in", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "accepted_tos_version_id": { + "name": "accepted_tos_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "accepted_tos_at": { + "name": "accepted_tos_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "user_accepted_tos_version_id_tos_version_id_fk": { + "name": "user_accepted_tos_version_id_tos_version_id_fk", + "tableFrom": "user", + "tableTo": "tos_version", + "columnsFrom": [ + "accepted_tos_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_name_unique": { + "name": "user_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + }, + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_unconfirmed_email_unique": { + "name": "user_unconfirmed_email_unique", + "nullsNotDistinct": false, + "columns": [ + "unconfirmed_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.location": { + "name": "location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "location": { + "name": "location", + "type": "geometry(point)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "location_index": { + "name": "location_index", + "columns": [ + { + "expression": "location", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gist", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "location_location_unique": { + "name": "location_location_unique", + "nullsNotDistinct": false, + "columns": [ + "location" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.log_entry": { + "name": "log_entry", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "device_id": { + "name": "device_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.refresh_token": { + "name": "refresh_token", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "refresh_token_user_id_user_id_fk": { + "name": "refresh_token_user_id_user_id_fk", + "tableFrom": "refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.token_revocation": { + "name": "token_revocation", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.claim": { + "name": "claim", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "box_id": { + "name": "box_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "claim_expires_at_idx": { + "name": "claim_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "claim_box_id_device_id_fk": { + "name": "claim_box_id_device_id_fk", + "tableFrom": "claim", + "tableTo": "device", + "columnsFrom": [ + "box_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "unique_box_id": { + "name": "unique_box_id", + "nullsNotDistinct": false, + "columns": [ + "box_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_url": { + "name": "service_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_key": { + "name": "service_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integration_slug_unique": { + "name": "integration_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tos_user_state": { + "name": "tos_user_state", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tos_version_id": { + "name": "tos_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accepted_at": { + "name": "accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "tos_user_state_user_idx": { + "name": "tos_user_state_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tos_user_state_user_id_user_id_fk": { + "name": "tos_user_state_user_id_user_id_fk", + "tableFrom": "tos_user_state", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tos_user_state_tos_version_id_tos_version_id_fk": { + "name": "tos_user_state_tos_version_id_tos_version_id_fk", + "tableFrom": "tos_user_state", + "tableTo": "tos_version", + "columnsFrom": [ + "tos_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tos_user_state_user_id_tos_version_id_pk": { + "name": "tos_user_state_user_id_tos_version_id_pk", + "columns": [ + "user_id", + "tos_version_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tos_version": { + "name": "tos_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "effective_from": { + "name": "effective_from", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "accept_by": { + "name": "accept_by", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tos_version_effective_from_idx": { + "name": "tos_version_effective_from_idx", + "columns": [ + { + "expression": "effective_from", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tos_version_accept_by_idx": { + "name": "tos_version_accept_by_idx", + "columns": [ + { + "expression": "accept_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "tos_version_version_unique": { + "name": "tos_version_version_unique", + "nullsNotDistinct": false, + "columns": [ + "version" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_token": { + "name": "action_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "action_token_user_purpose_uq": { + "name": "action_token_user_purpose_uq", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "purpose", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "action_token_expires_at_idx": { + "name": "action_token_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "action_token_user_id_user_id_fk": { + "name": "action_token_user_id_user_id_fk", + "tableFrom": "action_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "action_token_token_hash_unique": { + "name": "action_token_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_grant": { + "name": "rate_limit_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tier": { + "name": "tier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.exposure": { + "name": "exposure", + "schema": "public", + "values": [ + "indoor", + "outdoor", + "mobile", + "unknown" + ] + }, + "public.model": { + "name": "model", + "schema": "public", + "values": [ + "homeV2Lora", + "homeV2Ethernet", + "homeV2Wifi", + "homeEthernet", + "homeWifi", + "homeEthernetFeinstaub", + "homeWifiFeinstaub", + "luftdaten_sds011", + "luftdaten_sds011_dht11", + "luftdaten_sds011_dht22", + "luftdaten_sds011_bmp180", + "luftdaten_sds011_bme280", + "hackair_home_v2", + "senseBox:Edu", + "luftdaten.info", + "custom" + ] + }, + "public.status": { + "name": "status", + "schema": "public", + "values": [ + "active", + "inactive", + "old" + ] + }, + "public.theme_preference": { + "name": "theme_preference", + "schema": "public", + "values": [ + "light", + "dark", + "system" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": { + "public.measurement_10min": { + "columns": { + "sensor_id": { + "name": "sensor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "time": { + "name": "time", + "type": "timestamp (3) with time zone", + "primaryKey": false, + "notNull": false + }, + "avg_value": { + "name": "avg_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "total_values": { + "name": "total_values", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_value": { + "name": "min_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "max_value": { + "name": "max_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + } + }, + "name": "measurement_10min", + "schema": "public", + "isExisting": true, + "materialized": true + }, + "public.measurement_1day": { + "columns": { + "sensor_id": { + "name": "sensor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "time": { + "name": "time", + "type": "timestamp (3) with time zone", + "primaryKey": false, + "notNull": false + }, + "avg_value": { + "name": "avg_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "total_values": { + "name": "total_values", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_value": { + "name": "min_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "max_value": { + "name": "max_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + } + }, + "name": "measurement_1day", + "schema": "public", + "isExisting": true, + "materialized": true + }, + "public.measurement_1hour": { + "columns": { + "sensor_id": { + "name": "sensor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "time": { + "name": "time", + "type": "timestamp (3) with time zone", + "primaryKey": false, + "notNull": false + }, + "avg_value": { + "name": "avg_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "total_values": { + "name": "total_values", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_value": { + "name": "min_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "max_value": { + "name": "max_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + } + }, + "name": "measurement_1hour", + "schema": "public", + "isExisting": true, + "materialized": true + }, + "public.measurement_1month": { + "columns": { + "sensor_id": { + "name": "sensor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "time": { + "name": "time", + "type": "timestamp (3) with time zone", + "primaryKey": false, + "notNull": false + }, + "avg_value": { + "name": "avg_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "total_values": { + "name": "total_values", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_value": { + "name": "min_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "max_value": { + "name": "max_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + } + }, + "name": "measurement_1month", + "schema": "public", + "isExisting": true, + "materialized": true + }, + "public.measurement_1year": { + "columns": { + "sensor_id": { + "name": "sensor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "time": { + "name": "time", + "type": "timestamp (3) with time zone", + "primaryKey": false, + "notNull": false + }, + "avg_value": { + "name": "avg_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "total_values": { + "name": "total_values", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_value": { + "name": "min_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "max_value": { + "name": "max_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + } + }, + "name": "measurement_1year", + "schema": "public", + "isExisting": true, + "materialized": true + } + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/app/db/drizzle/meta/_journal.json b/app/db/drizzle/meta/_journal.json index 76c934ac..1545ed33 100644 --- a/app/db/drizzle/meta/_journal.json +++ b/app/db/drizzle/meta/_journal.json @@ -323,6 +323,13 @@ "when": 1782121660841, "tag": "0045_blushing_chameleon", "breakpoints": true + }, + { + "idx": 46, + "version": "7", + "when": 1783502216929, + "tag": "0046_flowery_blacklash", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/app/db/schema/index.ts b/app/db/schema/index.ts index 3f74b939..91b338b0 100644 --- a/app/db/schema/index.ts +++ b/app/db/schema/index.ts @@ -14,3 +14,4 @@ export * from './claim' export * from './integration' export * from './tos' export * from './action-token' +export * from './rate-limit-grant' diff --git a/app/db/schema/rate-limit-grant.ts b/app/db/schema/rate-limit-grant.ts new file mode 100644 index 00000000..5d11d034 --- /dev/null +++ b/app/db/schema/rate-limit-grant.ts @@ -0,0 +1,30 @@ +import { createId } from '@paralleldrive/cuid2' +import { pgTable, text, timestamp, boolean } from 'drizzle-orm/pg-core' + +export type RateLimitGrantKind = + | 'user_email' + | 'email_domain' + | 'credential_hash' +export type RateLimitTier = 'standard_plus' | 'trusted' | 'high_volume' + +export const rateLimitGrant = pgTable('rate_limit_grant', { + id: text('id') + .primaryKey() + .notNull() + .$defaultFn(() => createId()), + kind: text('kind').$type().notNull(), + value: text('value').notNull(), + tier: text('tier').$type().notNull(), + enabled: boolean('enabled').default(true).notNull(), + note: text('note'), + expiresAt: timestamp('expires_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }) + .defaultNow() + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .defaultNow() + .notNull(), +}) + +export type RateLimitGrant = typeof rateLimitGrant.$inferSelect +export type InsertRateLimitGrant = typeof rateLimitGrant.$inferInsert From 255341e18bcf958fcfdd2fef9a9cd7a4247de0d3 Mon Sep 17 00:00:00 2001 From: jona159 Date: Wed, 8 Jul 2026 12:27:35 +0200 Subject: [PATCH 03/12] feat: add db methods to manage rate limits --- app/db/models/rate-limit-grant.server.ts | 125 +++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 app/db/models/rate-limit-grant.server.ts diff --git a/app/db/models/rate-limit-grant.server.ts b/app/db/models/rate-limit-grant.server.ts new file mode 100644 index 00000000..fa755192 --- /dev/null +++ b/app/db/models/rate-limit-grant.server.ts @@ -0,0 +1,125 @@ +import { and, eq, gt, isNull, or, sql } from 'drizzle-orm' +import { + rateLimitGrant, + type RateLimitGrant, + type RateLimitGrantKind, + type RateLimitTier, +} from '~/db/schema' +import { drizzleClient } from '~/db.server' + +export type ActiveRateLimitGrant = Pick< + RateLimitGrant, + 'kind' | 'value' | 'tier' +> + +export function normalizeRateLimitGrantValue( + kind: RateLimitGrantKind, + value: string, +) { + const normalized = value.trim().toLowerCase() + return kind === 'email_domain' ? normalized.replace(/^@/, '') : normalized +} + +export async function getActiveRateLimitGrants(): Promise< + ActiveRateLimitGrant[] +> { + const now = new Date() + + const grants = await drizzleClient + .select({ + kind: rateLimitGrant.kind, + value: rateLimitGrant.value, + tier: rateLimitGrant.tier, + }) + .from(rateLimitGrant) + .where( + and( + eq(rateLimitGrant.enabled, true), + or(isNull(rateLimitGrant.expiresAt), gt(rateLimitGrant.expiresAt, now)), + ), + ) + + return grants.map((grant) => ({ + ...grant, + value: normalizeRateLimitGrantValue(grant.kind, grant.value), + })) +} + +export async function getRateLimitGrants() { + return drizzleClient + .select() + .from(rateLimitGrant) + .orderBy(rateLimitGrant.createdAt) +} + +export async function createRateLimitGrant({ + kind, + value, + tier, + note, + expiresAt, +}: { + kind: RateLimitGrantKind + value: string + tier: RateLimitTier + note?: string | null + expiresAt?: Date | null +}) { + const [grant] = await drizzleClient + .insert(rateLimitGrant) + .values({ + kind, + value: normalizeRateLimitGrantValue(kind, value), + tier, + note, + expiresAt, + }) + .returning() + + return grant +} + +export async function updateRateLimitGrant( + id: string, + { + kind, + value, + tier, + enabled, + note, + expiresAt, + }: { + kind: RateLimitGrantKind + value: string + tier: RateLimitTier + enabled: boolean + note?: string | null + expiresAt?: Date | null + }, +) { + const [grant] = await drizzleClient + .update(rateLimitGrant) + .set({ + kind, + value: normalizeRateLimitGrantValue(kind, value), + tier, + enabled, + note, + expiresAt, + updatedAt: sql`NOW()`, + }) + .where(eq(rateLimitGrant.id, id)) + .returning() + + return grant +} + +export async function disableRateLimitGrant(id: string) { + const [grant] = await drizzleClient + .update(rateLimitGrant) + .set({ enabled: false, updatedAt: sql`NOW()` }) + .where(eq(rateLimitGrant.id, id)) + .returning() + + return grant +} From 91c2c3f1083922913c34315465df6f3d14168f7e Mon Sep 17 00:00:00 2001 From: jona159 Date: Wed, 8 Jul 2026 12:34:58 +0200 Subject: [PATCH 04/12] feat: admin dashboard for rate limits --- app/routes/admin._index.tsx | 3 + app/routes/admin.rate-limits.tsx | 444 +++++++++++++++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 app/routes/admin.rate-limits.tsx diff --git a/app/routes/admin._index.tsx b/app/routes/admin._index.tsx index 20154bf2..96d9eef8 100644 --- a/app/routes/admin._index.tsx +++ b/app/routes/admin._index.tsx @@ -12,6 +12,9 @@ export default function AdminIndexRoute() {
Edit devices
+
+ Edit rate limits +
) diff --git a/app/routes/admin.rate-limits.tsx b/app/routes/admin.rate-limits.tsx new file mode 100644 index 00000000..80730f47 --- /dev/null +++ b/app/routes/admin.rate-limits.tsx @@ -0,0 +1,444 @@ +import { Form, Link, redirect, useActionData } from 'react-router' +import { type Route } from './+types/admin.rate-limits' +import { + createRateLimitGrant, + getRateLimitGrants, + updateRateLimitGrant, +} from '~/db/models/rate-limit-grant.server' +import { + type RateLimitGrantKind, + type RateLimitTier, +} from '~/db/schema/rate-limit-grant' + +const GRANT_KINDS = ['user_email', 'email_domain', 'credential_hash'] as const +const RATE_LIMIT_TIERS = ['standard_plus', 'trusted', 'high_volume'] as const + +type ActionData = { + error?: boolean + message?: string + fieldErrors?: { + kind?: string + value?: string + tier?: string + expiresAt?: string + } +} + +export async function loader() { + const grants = await getRateLimitGrants() + return { grants } +} + +export async function action({ + request, +}: Route.ActionArgs): Promise { + const formData = await request.formData() + const intent = getString(formData, '_action') + + const parsed = parseGrantForm(formData) + if (!parsed.ok) { + return { + error: true, + message: 'Please fix the highlighted fields.', + fieldErrors: parsed.fieldErrors, + } + } + + try { + switch (intent) { + case 'create': + await createRateLimitGrant(parsed.value) + return redirect('/admin/rate-limits') + + case 'update': { + const id = getString(formData, 'id') + if (!id) { + return { + error: true, + message: 'Missing grant id.', + } + } + + await updateRateLimitGrant(id, parsed.value) + return redirect('/admin/rate-limits') + } + + default: + return { + error: true, + message: 'Unknown action.', + } + } + } catch (error) { + return { + error: true, + message: + error instanceof Error + ? error.message + : 'Failed to save rate limit grant.', + } + } +} + +export default function AdminRateLimitsRoute({ + loaderData, +}: Route.ComponentProps) { + const { grants } = loaderData + const actionData = useActionData() + + return ( +
+
+ + ← Back to admin + +
+ +
+

Rate limit grants

+

+ Grant finite higher rate-limit tiers to users, domains, or credential + hashes. +

+
+ + {actionData?.message ? ( +

+ {actionData.message} +

+ ) : null} + +
+

Create grant

+
+ + +
+ +
+ +
+ +
+
+

Existing grants

+ Total: {grants.length} +
+ +
+ + + + + + + + + + + + + + {grants.map((grant) => ( + + + + + + +
KindValueTier + Expires + + Enabled + NoteActions
+
+ + + +
+ + + + + + + + +