From be7976b622fcf53deb93aa4c233d5b635f135feb Mon Sep 17 00:00:00 2001 From: Victor Petrovykh Date: Tue, 17 Dec 2024 19:50:14 -0500 Subject: [PATCH] Add the Prisma schema generator Add the `prisma` command to the list of valid @edgedb/generate commands. It requires a `--file` and some way of connecting the client (DSN or whatever other method). The generated Prisma schema will be output as specified by `--file`. --- packages/generate/src/cli.ts | 29 +- packages/generate/src/gel-prisma.ts | 732 ++++++++++++++++++++++++++++ 2 files changed, 757 insertions(+), 4 deletions(-) create mode 100644 packages/generate/src/gel-prisma.ts diff --git a/packages/generate/src/cli.ts b/packages/generate/src/cli.ts index c58a9f19c..e7f90c9e8 100644 --- a/packages/generate/src/cli.ts +++ b/packages/generate/src/cli.ts @@ -18,6 +18,7 @@ import { generateQueryBuilder } from "./edgeql-js"; import { runInterfacesGenerator } from "./interfaces"; import { type Target, exitWithError } from "./genutil"; import { generateQueryFiles } from "./queries"; +import { runGelPrismaGenerator } from "./gel-prisma"; const { path, readFileUtf8, exists } = adapter; @@ -25,6 +26,7 @@ enum Generator { QueryBuilder = "edgeql-js", Queries = "queries", Interfaces = "interfaces", + GelPrisma = "prisma", } const availableGeneratorsHelp = ` @@ -65,6 +67,8 @@ const run = async () => { case Generator.Interfaces: options.target = "ts"; break; + case Generator.GelPrisma: + break; } let projectRoot: string | null = null; @@ -190,7 +194,10 @@ const run = async () => { options.useHttpClient = true; break; case "--target": { - if (generator === Generator.Interfaces) { + if ( + generator === Generator.Interfaces || + generator === Generator.GelPrisma + ) { exitWithError( `--target is not supported for generator "${generator}"`, ); @@ -220,7 +227,10 @@ const run = async () => { options.out = getVal(); break; case "--file": - if (generator === Generator.Interfaces) { + if ( + generator === Generator.Interfaces || + generator === Generator.GelPrisma + ) { options.file = getVal(); } else if (generator === Generator.Queries) { if (args.length > 0 && args[0][0] !== "-") { @@ -290,9 +300,13 @@ const run = async () => { case Generator.Interfaces: console.log(`Generating TS interfaces from schema...`); break; + case Generator.GelPrisma: + console.log(`Generating Prisma schema from database...`); + break; } - if (!options.target) { + // don't need to do any of that for the prisma schema generator + if (!options.target && generator !== Generator.GelPrisma) { if (!projectRoot) { throw new Error( `Failed to detect project root. @@ -411,6 +425,12 @@ Run this command inside an EdgeDB project directory or specify the desired targe schemaDir, }); break; + case Generator.GelPrisma: + await runGelPrismaGenerator({ + options, + client, + }); + break; } } catch (e) { exitWithError((e as Error).message); @@ -432,6 +452,7 @@ COMMANDS: queries Generate typed functions from .edgeql files edgeql-js Generate query builder interfaces Generate TS interfaces for schema types + prisma Generate a Prisma schema for an existing database instance CONNECTION OPTIONS: @@ -460,7 +481,7 @@ OPTIONS: Change the output directory the querybuilder files are generated into (Only valid for 'edgeql-js' generator) --file - Change the output filepath of the 'queries' and 'interfaces' generators + Change the output filepath of the 'queries', 'interfaces', and 'prisma' generators When used with the 'queries' generator, also changes output to single-file mode --force-overwrite Overwrite contents without confirmation diff --git a/packages/generate/src/gel-prisma.ts b/packages/generate/src/gel-prisma.ts new file mode 100644 index 000000000..8117fbb80 --- /dev/null +++ b/packages/generate/src/gel-prisma.ts @@ -0,0 +1,732 @@ +import type { CommandOptions } from "./commandutil"; +import type {Client} from 'edgedb' +import * as fs from 'node:fs' + +const WARNINGS: string[] = [] + +// const GEL_DSN = 'edgedb://edgedb@localhost:5656/main?tls_security=insecure' +// const client = edgedb.createClient(GEL_DSN) + +const INTRO_QUERY = ` +with module schema +select ObjectType { + name, + links: { + name, + readonly, + required, + cardinality, + exclusive := exists ( + select .constraints + filter .name = 'std::exclusive' + ), + target: {name}, + + properties: { + name, + readonly, + required, + cardinality, + exclusive := exists ( + select .constraints + filter .name = 'std::exclusive' + ), + target: {name}, + }, + } filter .name != '__type__', + properties: { + name, + readonly, + required, + cardinality, + exclusive := exists ( + select .constraints + filter .name = 'std::exclusive' + ), + target: {name}, + }, + backlinks := >[], +} +filter + not .builtin + and + not .internal + and + not re_test('^(std|cfg|sys|schema)::', .name); +` + +const MODULE_QUERY = ` +with + module schema, + m := (select \`Module\` filter not .builtin) +select m.name; +` + +interface JSONField { + name: string + readonly?: boolean + required?: boolean + exclusive?: boolean + cardinality: string + target: {name: string} + has_link_target?: boolean + has_link_object?: boolean +} + +interface JSONLink extends JSONField { + fwname?: string + properties?: JSONField[] +} + +interface JSONType { + name: string + links: JSONLink[] + properties: JSONField[] + backlinks: JSONLink[] + backlink_renames?: {[key: string]: string} +} + +interface LinkTable { + module: string + name: string + table: string + source: string + target: string +} + +interface TableObj { + module: string + name: string + table: string + links: JSONField[] + properties: JSONField[] +} + +interface ProcessedSpec { + modules: string[] + object_types: JSONType[] + link_tables: LinkTable[] + link_objects: TableObj[] + prop_objects: TableObj[] +} + +interface MappedSpec { + link_objects: {[key: string]: TableObj} + object_types: {[key: string]: JSONType} +} + +const CLEAN_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/ + + +function warn(msg: string) { + // Keep record of warnigns during the generation process + WARNINGS.push(msg); +} + + +function getSQLName(name: string): string { + // Just remove the module name + return name.split('::').pop() +} + + +function getMod(fullname: string): string { + return fullname.split('::').slice(0, -1).join('::') +} + + +function getModAndName(fullname: string): string[] { + const mod = fullname.split('::') + const name = mod.pop() + return [mod.join('::'), name] +} + + +function validName(name: string): boolean { + // Just remove module separators and check the rest + name = name.replace('::', '') + if (!CLEAN_NAME.test(name)) { + warn( + `Non-alphanumeric names are not supported: ${ name }`) + return false + } + return true +} + + + +async function getSchemaJSON(client: Client) { + const types = JSON.parse(await client.queryJSON(INTRO_QUERY)) + const modules = JSON.parse(await client.queryJSON(MODULE_QUERY)) + + return processLinks(types, modules) +} + + +function _skipInvalidNames( + spec_list: any[], + recurse_into: string[] | null = null, +): any[] { + const valid = [] + for (const spec of spec_list) { + // skip invalid names + if (validName(spec.name)) { + if (recurse_into) { + for (const fname of recurse_into) { + if (spec[fname]) { + spec[fname] = _skipInvalidNames( + spec[fname], recurse_into) + } + } + } + valid.push(spec) + } + } + + return valid +} + + +function processLinks(types: JSONType[], modules: string[]): ProcessedSpec { + // Figure out all the backlinks, link tables, and links with link properties + // that require their own intermediate objects. + const type_map: {[key: string]: JSONType} = {} + const link_tables: LinkTable[] = [] + const link_objects: TableObj[] = [] + const prop_objects: TableObj[] = [] + + // All the names of types, props and links are valid beyond this point. + types = _skipInvalidNames(types, ['properties', 'links']) + for (const spec of types) { + type_map[spec.name] = spec + spec.backlink_renames = {} + } + + for (const spec of types) { + const mod = getMod(spec.name) + const sql_source = getSQLName(spec.name) + + for (const prop of spec.properties) { + const name = prop.name + const exclusive = prop.exclusive + const cardinality = prop.cardinality + const sql_name = getSQLName(name) + + if (cardinality === 'Many') { + // Multi property will make its own "link table". But since it + // doesn't link to any other object the link table itself must + // be reflected as an object. + const pobj: TableObj = { + 'module': mod, + 'name': `${ sql_source }_${ sql_name }_prop`, + 'table': `${ sql_source }.${ sql_name }`, + 'links': [{ + 'name': 'source', + 'required': true, + 'cardinality': exclusive ? 'One' : 'Many', + 'exclusive': false, + 'target': {'name': spec.name}, + 'has_link_object': false, + }], + 'properties': [{ + 'name': 'target', + 'required': true, + 'cardinality': 'One', + 'exclusive': false, + 'target': prop.target, + 'has_link_object': false, + }], + } + prop_objects.push(pobj) + } + } + + for (const link of spec.links) { + if (link.name != '__type__') { + const name = link.name + const target = link.target.name + const cardinality = link.cardinality + const exclusive = link.exclusive + const sql_name = getSQLName(name) + + const objtype = type_map[target] + objtype.backlinks.push({ + 'name': `backlink_via_${ sql_name }`, + // flip cardinality and exclusivity + 'cardinality': exclusive ? 'One' : 'Many', + 'exclusive': cardinality === 'One', + 'target': {'name': spec.name}, + 'has_link_object': false, + }) + + link.has_link_object = false + // Any link with properties should become its own intermediate + // object, since ORMs generally don't have a special convenient + // way of exposing this as just a link table. + if (link.properties!.length > 2) { + // more than just 'source' and 'target' properties + const lobj: TableObj = { + 'module': mod, + 'name': `${ sql_source }_${ sql_name }_link`, + 'table': `${ sql_source }.${ sql_name }`, + 'links': [], + 'properties': [], + } + for (const prop of link.properties!) { + if (prop.name === 'source' || prop.name === 'target') { + lobj.links.push(prop) + } else { + lobj.properties.push(prop) + } + } + + link_objects.push(lobj) + link['has_link_object'] = true + objtype['backlinks'][-1]['has_link_object'] = true + } else if (cardinality === 'Many') { + // Add a link table for One-to-Many and Many-to-Many + link_tables.push({ + 'module': mod, + 'name': `${ sql_source }_${ sql_name }_table`, + 'table': `${ sql_source }.${ sql_name }`, + 'source': spec.name, + 'target': target, + }) + } + } + } + + // Go over backlinks and resolve any name collisions using the type map. + for (const spec of types) { + // Find collisions in backlink names + const bk: { [key: string]: JSONLink[] } = {} + + for (const link of spec.backlinks) { + if (link.name.search(/^backlink_via_/) >= 0) { + if (!bk[link.name]) { + bk[link.name] = [] + } + bk[link.name].push(link) + } + } + + for (const bklinks of Object.values(bk)) { + if (bklinks.length > 1) { + // We have a collision, so each backlink in it must now be disambiguated + for (const link of bklinks) { + const origsrc = getSQLName(link.target.name) + const lname = link.name + link.name = `${ lname }_from_${ origsrc }` + + // Also update the original source of the link with the special backlink name + const source = type_map[link.target.name] + const fwname = lname.replace(/^backlink_via_/, '') + link.fwname = fwname + source.backlink_renames!.fwname = link.name + } + } + } + } + } + + return { + 'modules': modules, + 'object_types': types, + 'link_tables': link_tables, + 'link_objects': link_objects, + 'prop_objects': prop_objects, + } +} + + +const GEL_SCALAR_MAP: {[key: string]: string} = { + 'std::uuid': 'String @db.Uuid', + 'std::bigint': 'Decimal', + 'std::bool': 'Boolean', + 'std::bytes': 'Bytes', + 'std::decimal': 'Decimal', + 'std::float32': 'Float', + 'std::float64': 'Float', + 'std::int16': 'Int', + 'std::int32': 'Int', + 'std::int64': 'BigInt', + 'std::json': 'Json', + 'std::str': 'String', + 'std::datetime': 'DateTime', +} + +const BASE_STUB = `\ +// Automatically generated from Gel schema. + +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} +` + +class ModelClass { + name: string + table?: string + props: { [key: string]: string } = {} + links: { [key: string]: string } = {} + mlinks: { [key: string]: string } = {} + backlinks: { [key: string]: string } = {} + backlink_renames: { [key: string]: string } = {} + isLinkTable: boolean = false + + constructor(name: string) { + this.name = name + } + + getBacklinkName(name: string): string { + return this.backlink_renames[name] || `backlink_via_${ name }` + } +} + + +class ModelGenerator { + INDENT = ' ' + outfile: string + out: string[] = [] + _indentLevel: number = 0 + + constructor(outfile: string) { + this.outfile = outfile + this._indentLevel = 0 + } + + indent() { + this._indentLevel += 1 + } + + dedent() { + if (this._indentLevel > 0) { + this._indentLevel -= 1 + } + } + + resetIndent() { + this._indentLevel = 0 + } + + write(text: string = '') { + for (const line of text.split('\n')) { + this.out.push(`${ this.INDENT.repeat(this._indentLevel) }${line}`) + } + } + + renderOutput() { + fs.writeFileSync( + this.outfile, + BASE_STUB + this.out.join('\n') + ) + } + + specToModulesDict(spec: ProcessedSpec) { + const modules: {[key: string]: MappedSpec} = {} + + for (const mod of spec['modules'].sort()) { + modules[mod] = { link_objects: {}, object_types: {} } + } + + // convert link tables into full link objects + for (const rec of spec.link_tables) { + const mod = rec.module + modules[mod].link_objects[rec.table] = { + 'module': mod, + 'name': rec.name.replace(/_table$/, '_link'), + 'table': rec.table, + 'links': [{ + 'name': 'source', + 'required': true, + 'cardinality': 'One', + 'target': {'name': rec.source}, + 'has_link_object': false, + }, { + 'name': 'target', + 'required': true, + 'cardinality': 'One', + 'target': {'name': rec.target}, + 'has_link_object': false, + }], + 'properties': [], + } + } + + for (const rec of spec.link_objects) { + const mod = rec.module + modules[mod].link_objects[rec.table] = rec + } + + for (const rec of spec.object_types) { + const [mod, name] = getModAndName(rec.name) + modules[mod].object_types[name] = rec + } + + return modules['default'] + } + + buildModels(maps: MappedSpec) { + const modmap: { [key: string]: ModelClass } = {} + + for (const [name, rec] of Object.entries(maps.object_types)) { + const mod: ModelClass = new ModelClass(name) + mod.table = name + if (rec.backlink_renames !== undefined) { + mod.backlink_renames = rec.backlink_renames + } + + // copy backlink information + for (const link of rec.backlinks) { + const code = this.renderBacklinkLink(link) + if (code) { + mod.backlinks[link.name] = code + } + } + + // process properties as fields + for (const prop of rec.properties) { + const pname = prop.name + if (pname === 'id' || prop.cardinality === 'Many') { + continue + } + + const code = this.renderProp(prop) + if (code) { + mod.props[pname] = code + } + } + + // process single links as fields + for (const link of rec.links) { + if (link.cardinality === 'One') { + const lname = link.name + const bklink = mod.getBacklinkName(lname) + const code = this.renderSingleLink(link, bklink) + if (code) { + mod.links[lname] = code + // corresponding foreign key field + const opt = link['required'] ? '' : '?' + mod.props[lname + '_id'] = `String${ opt } @db.Uuid` + } + } + } + + modmap[mod.name] = mod + } + + for (const [table, rec] of Object.entries(maps.link_objects)) { + const [source, fwname] = table.split('.') + const mod = new ModelClass(`${ source }_${ fwname }`) + mod.table = table + mod.isLinkTable = true + + // Must have source and target + let target: string + for (const prop of rec.links) { + const [mtgt, tar] = getModAndName(prop.target.name) + if (mtgt !== 'default') { + // skip this whole link table + warn(`Skipping link ${fwname}: link target ${ mtgt + "::" + tar } ` + + `is not supported`) + continue + } + if (prop.name === 'target') { + target = tar + } + } + + mod.props['source_id'] = `String @map("source") @db.Uuid` + mod.props['target_id'] = `String @map("target") @db.Uuid` + + // Get source and target models and reconcile them with the link table + const src = modmap[source] + const tgt = modmap[target!] + const bkname = src.getBacklinkName(fwname) + + mod.links['target'] = ( + `${ target! } @relation("${ mod.name }", ` + + `fields: [target_id], references: [id], ` + + `onUpdate: NoAction, onDelete: NoAction)` + ) + mod.links['source'] = ( + `${ source } @relation("${ bkname }", ` + + `fields: [source_id], references: [id], ` + + `onUpdate: NoAction, onDelete: NoAction)` + ) + // Update the source and target models with the corresponding + // ManyToManyField. + src.mlinks[fwname] = ( + `${ mod.name }[] @relation("${ bkname }")` + ) + tgt.mlinks[bkname] = ( + `${ mod.name }[] @relation("${ mod.name }")` + ) + delete src.links[fwname] + delete tgt.backlinks[bkname] + + // process properties if any + for (const prop of rec['properties']) { + const pname = prop.name + const code = this.renderProp(prop) + if (code) { + mod.props[pname] = code + } + } + + modmap[mod.name] = mod + } + + return modmap + } + + renderProp(prop: JSONField): string { + const target = prop.target.name + const type: string | undefined = GEL_SCALAR_MAP[target] + + if (type === undefined) { + warn(`Scalar type ${ target } is not supported`) + return '' + } + // make props opional and let gel deal with the actual requirements + return type + '?' + } + + renderSingleLink(link: JSONLink, bklink: string): string { + const opt = link.required ? '' : '?' + const [mod, target] = getModAndName(link.target.name) + + if (mod !== 'default') { + warn( + `Skipping link ${ link.name }: link target ${ link.target.name } ` + + `is not supported` + ) + return '' + } + + return ( + `${ target }${ opt } @relation("${ bklink }", ` + + `fields: [${ link.name }_id], references: [id], ` + + `onUpdate: NoAction, onDelete: NoAction)` + ) + } + + renderBacklinkLink(link: JSONLink): string { + const multi = link.cardinality === 'One' ? '?' : '[]' + const [mod, target] = getModAndName(link.target.name) + + if (mod !== 'default') { + warn( + `Skipping link ${ link.name }: link target ${ link.target.name } ` + + `is not supported` + ) + return '' + } + + return ( + `${ target }${ multi } @relation("${ link.name }")` + ) + } + + renderModels(spec: ProcessedSpec) { + const mods = spec.modules + if (mods[0] != 'default' || mods.length > 1) { + const skipped = mods + .filter(m => m != 'default') + .join(', ') + warn( + `Skipping modules ${ skipped }: Prisma reflection doesn't support` + + 'multiple modules or non-default modules.' + ) + } + + if (spec.prop_objects.length > 0) { + warn( + "Skipping multi properties: Prisma reflection doesn't support multi" + + "properties as they produce models without 'id' field." + ) + } + + const maps = this.specToModulesDict(spec) + const modmap = this.buildModels(maps) + + for (const mod of Object.values(modmap)) { + this.write() + this.renderModelClass(mod) + } + + this.renderOutput() + } + + renderModelClass(mod: ModelClass) { + this.write(`model ${ mod.name } {`) + this.indent() + + if (!mod.isLinkTable) { + this.write( + 'id String @id @default(dbgenerated("uuid_generate_v4()"))' + + ' @db.Uuid' + ) + // not actually using this default, but this prevents Prisma from sending null + this.write( + 'gel_type_id String @default(dbgenerated("uuid_generate_v4()"))' + + ' @map("__type__") @db.Uuid' + ) + } + + if (Object.keys(mod.props).length > 0) { + this.write() + this.write('// properties') + for (const [name, val] of Object.entries(mod.props)) { + this.write(`${name} ${val}`) + } + } + + if (Object.keys(mod.links).length > 0) { + this.write() + this.write('// links') + for (const [name, val] of Object.entries(mod.links)) { + this.write(`${name} ${val}`) + } + } + + if (Object.keys(mod.mlinks).length > 0) { + this.write() + this.write('// multi-links') + for (const [name, val] of Object.entries(mod.mlinks)) { + this.write(`${name} ${val}`) + } + } + + if (Object.keys(mod.backlinks).length > 0) { + this.write() + this.write('// backlinks') + for (const [name, val] of Object.entries(mod.backlinks)) { + this.write(`${name} ${val}`) + } + } + + this.write() + if (mod.isLinkTable) { + this.write('@@id([source_id, target_id])') + } + this.write(`@@map("${ mod.table }")`) + + this.dedent() + this.write(`}`) + } +} + + +export async function runGelPrismaGenerator(params: { + options: CommandOptions; + client: Client; +}) { + const spec = await getSchemaJSON(params.client) + const gen = new ModelGenerator(params.options.file!) + gen.renderModels(spec) +}