open-source-mirrors
/
ass
mirror of https://github.com/tycrek/ass
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

impr: move sql config to "database", add config for different db variants

Browse Source
pull/245/head
xwashere 1 year ago
parent e113cb57ee
commit 9c9f2ac768
No known key found for this signature in database
GPG Key ID: 042F8BFA1B0EF93B
11 changed files with 213 additions and 90 deletions
Show Stats Download Patch File Download Diff File

17
backend/UserConfig.ts
Unescape View File

@ -102,12 +102,17 @@ export class UserConfig {
if (!Checkers.s3.credentials.secretKey(config.s3.credentials.secretKey)) throw new Error('Invalid S3 Secret key'); if (!Checkers.s3.credentials.secretKey(config.s3.credentials.secretKey)) throw new Error('Invalid S3 Secret key');
} }
// * Optional SQL config(s) (Currently only checks MySQL) // * Optional database config(s)
if (config.sql?.mySql != null) { if (config.database != null) {
if (!Checkers.sql.mySql.host(config.sql.mySql.host)) throw new Error('Invalid MySql Host'); // these both have the same schema so we can just check both
if (!Checkers.sql.mySql.user(config.sql.mySql.user)) throw new Error('Invalid MySql User'); if (config.database.kind == 'mysql' || config.database.kind == 'postgres') {
if (!Checkers.sql.mySql.password(config.sql.mySql.password)) throw new Error('Invalid MySql Password'); if (config.database.options != undefined) {
if (!Checkers.sql.mySql.database(config.sql.mySql.database)) throw new Error('Invalid MySql Database'); if (!Checkers.sql.mySql.host(config.database.options.host)) throw new Error('Invalid database host');
if (!Checkers.sql.mySql.user(config.database.options.user)) throw new Error('Invalid databse user');
if (!Checkers.sql.mySql.password(config.database.options.password)) throw new Error('Invalid database password');
if (!Checkers.sql.mySql.database(config.database.options.database)) throw new Error('Invalid database');
} else throw new Error('Database options missing');
}
} }
// * optional rate limit config // * optional rate limit config

20
backend/app.ts
Unescape View File

@ -15,6 +15,7 @@ import { UserConfig } from './UserConfig';
import { MySQLDatabase } from './sql/mysql'; import { MySQLDatabase } from './sql/mysql';
import { buildFrontendRouter } from './routers/_frontend'; import { buildFrontendRouter } from './routers/_frontend';
import { DBManager } from './sql/database'; import { DBManager } from './sql/database';
import { PostgreSQLDatabase } from './sql/postgres';
/** /**
* Top-level metadata exports * Top-level metadata exports
@ -114,10 +115,21 @@ async function main() {
.catch((err) => (err.code && err.code === 'ENOENT' ? {} : console.error(err), resolve(void 0)))); .catch((err) => (err.code && err.code === 'ENOENT' ? {} : console.error(err), resolve(void 0))));
// If user config is ready, try to configure SQL // If user config is ready, try to configure SQL
if (UserConfig.ready && UserConfig.config.sql?.mySql != null) { if (UserConfig.ready && UserConfig.config.database != null) {
try { await DBManager.use(new MySQLDatabase()); } try {
catch (err) { throw new Error(`Failed to configure SQL`); } switch (UserConfig.config.database?.kind) {
} else { case 'json':
await DBManager.use(new JSONDatabase());
break;
case 'mysql':
await DBManager.use(new MySQLDatabase());
break;
case 'postgres':
await DBManager.use(new PostgreSQLDatabase());
break;
}
} catch (err) { throw new Error(`Failed to configure SQL`); }
} else { // default to json database
await DBManager.use(new JSONDatabase()); await DBManager.use(new JSONDatabase());
} }

44
backend/data.ts
Unescape View File

@ -20,45 +20,17 @@ const PATHS = {
users: path.join('.ass-data/users.json') users: path.join('.ass-data/users.json')
}; };
const bothWriter = async (files: FilesSchema, users: UsersSchema) => { /**
await fs.writeJson(PATHS.files, files, { spaces: '\t' }); * database kind -> name mapping
await fs.writeJson(PATHS.users, users, { spaces: '\t' }); */
const DBNAMES = {
'mysql': 'MySQL',
'postgres': 'PostgreSQL',
'json': 'JSON'
}; };
export const setDataModeToSql = (): Promise<void> => new Promise(async (resolve, reject) => {
log.debug('Setting data mode to SQL');
// Main config check
if (!UserConfig.ready || !UserConfig.config.sql?.mySql) return reject(new Error('MySQL not configured'));
const mySqlConf = UserConfig.config.sql.mySql;
// Read data files
const [files, users]: [FilesSchema, UsersSchema] = await Promise.all([fs.readJson(PATHS.files), fs.readJson(PATHS.users)]);
// Check the MySQL configuration
const checker = (val: string) => val != null && val !== '';
const issue =
!checker(mySqlConf.host) ? 'Missing MySQL Host'
: !checker(mySqlConf.user) ? 'Missing MySQL User'
: !checker(mySqlConf.password) ? 'Missing MySQL Password'
: !checker(mySqlConf.database) ? 'Missing MySQL Database'
// ! Blame VS Code for this weird indentation
: undefined;
// Set the vars
files.useSql = issue == null;
users.useSql = issue == null;
// Write data & return
await bothWriter(files, users);
(issue) ? reject(new Error(issue)) : resolve(void 0);
});
export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Promise<void> => new Promise(async (resolve, reject) => { export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Promise<void> => new Promise(async (resolve, reject) => {
try { try {
const useSql = UserConfig.config.sql != undefined;
if (sector === 'files') { if (sector === 'files') {
// * 1: Save as files (image, video, etc) // * 1: Save as files (image, video, etc)
await DBManager.put('assfiles', key, data as AssFile); await DBManager.put('assfiles', key, data as AssFile);
@ -67,7 +39,7 @@ export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Prom
await DBManager.put('assusers', key, data as AssUser); await DBManager.put('assusers', key, data as AssUser);
} }
log.info(`PUT ${sector} data`, `using ${useSql ? 'SQL' : 'local JSON'}`, key); log.info(`PUT ${sector} data`, `using ${DBNAMES[UserConfig.config.database?.kind ?? 'json']}`, key);
resolve(void 0); resolve(void 0);
} catch (err) { } catch (err) {
reject(err); reject(err);

20
backend/routers/api.ts
Unescape View File

@ -9,6 +9,9 @@ import { nanoid } from '../generators';
import { UserConfig } from '../UserConfig'; import { UserConfig } from '../UserConfig';
import { rateLimiterMiddleware, setRateLimiter } from '../ratelimit'; import { rateLimiterMiddleware, setRateLimiter } from '../ratelimit';
import { DBManager } from '../sql/database'; import { DBManager } from '../sql/database';
import { JSONDatabase } from '../sql/json';
import { MySQLDatabase } from '../sql/mysql';
import { PostgreSQLDatabase } from '../sql/postgres';
const router = Router({ caseSensitive: true }); const router = Router({ caseSensitive: true });
@ -26,9 +29,20 @@ router.post('/setup', BodyParserJson(), async (req, res) => {
// Save config // Save config
await UserConfig.saveConfigFile(); await UserConfig.saveConfigFile();
// Set data storage (not files) to SQL if required // set up new databases
if (UserConfig.config.sql?.mySql != null) if (UserConfig.config.database) {
await Promise.all([DBManager.configure(), data.setDataModeToSql()]); switch (UserConfig.config.database.kind) {
case 'json':
await DBManager.use(new JSONDatabase());
break;
case 'mysql':
await DBManager.use(new MySQLDatabase());
break;
case 'postgres':
await DBManager.use(new PostgreSQLDatabase());
break;
}
}
// set rate limits // set rate limits
if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api); if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api);

12
backend/sql/json.ts
Unescape View File

@ -89,10 +89,10 @@ export const ensureFiles = (): Promise<void> => new Promise(async (resolve, reje
* JSON database. i know json isnt sql, shut up. * JSON database. i know json isnt sql, shut up.
*/ */
export class JSONDatabase implements Database { export class JSONDatabase implements Database {
open(): Promise<void> { return Promise.resolve() } public open(): Promise<void> { return Promise.resolve() }
close(): Promise<void> { return Promise.resolve() } public close(): Promise<void> { return Promise.resolve() }
configure(): Promise<void> { public configure(): Promise<void> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
ensureFiles(); ensureFiles();
@ -100,7 +100,7 @@ export class JSONDatabase implements Database {
}); });
} }
put(table: DatabaseTable, key: string, data: DatabaseValue): Promise<void> { public put(table: DatabaseTable, key: string, data: DatabaseValue): Promise<void> {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
if (table == 'assfiles') { if (table == 'assfiles') {
// ? Local JSON // ? Local JSON
@ -138,14 +138,14 @@ export class JSONDatabase implements Database {
}) })
} }
get(table: DatabaseTable, key: string): Promise<DatabaseValue | undefined> { public get(table: DatabaseTable, key: string): Promise<DatabaseValue | undefined> {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]][key]; const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]][key];
(!data) ? resolve(undefined) : resolve(data); (!data) ? resolve(undefined) : resolve(data);
}); });
} }
getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> { public getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]]; const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]];
(!data) ? resolve({}) : resolve(data); (!data) ? resolve({}) : resolve(data);

33
backend/sql/mysql.ts
Unescape View File

@ -25,6 +25,31 @@ export class MySQLDatabase implements Database {
.catch((err) => reject(err))); .catch((err) => reject(err)));
} }
/**
* validate the mysql config
*/
private _validateConfig(): string | undefined {
// make sure the configuration exists
if (!UserConfig.ready) return 'User configuration not ready';
if (typeof UserConfig.config.database != 'object') return 'MySQL configuration missing';
if (UserConfig.config.database.kind != "mysql") return 'Database not set to MySQL, but MySQL is in use, something has gone terribly wrong';
if (typeof UserConfig.config.database.options != 'object') return 'MySQL configuration missing';
let mySqlConf = UserConfig.config.database.options;
// Check the MySQL configuration
const checker = (val: string) => val != null && val !== '';
const issue =
!checker(mySqlConf.host) ? 'Missing MySQL Host'
: !checker(mySqlConf.user) ? 'Missing MySQL User'
: !checker(mySqlConf.password) ? 'Missing MySQL Password'
: !checker(mySqlConf.database) ? 'Missing MySQL Database'
// ! Blame VS Code for this weird indentation
: undefined;
return issue;
}
public open() { return Promise.resolve(); } public open() { return Promise.resolve(); }
public close() { return Promise.resolve(); } public close() { return Promise.resolve(); }
@ -35,11 +60,11 @@ export class MySQLDatabase implements Database {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
try { try {
// Config check // Config check
if (!UserConfig.ready) throw new Error('User configuration not ready'); let configError = this._validateConfig();
if (!UserConfig.config.sql?.mySql) throw new Error('MySQL configuration missing'); if (configError) throw new Error(configError);
// Create the pool // Create the pool
this._pool = mysql.createPool(UserConfig.config.sql.mySql); this._pool = mysql.createPool(UserConfig.config.database!.options!);
// Check if the pool is usable // Check if the pool is usable
const [rowz, _fields] = await this._pool.query(`SHOW FULL TABLES WHERE Table_Type LIKE 'BASE TABLE';`); const [rowz, _fields] = await this._pool.query(`SHOW FULL TABLES WHERE Table_Type LIKE 'BASE TABLE';`);
@ -61,7 +86,7 @@ export class MySQLDatabase implements Database {
// Check which tables ACTUALLY do exist // Check which tables ACTUALLY do exist
for (let row of rows_tableData) { for (let row of rows_tableData) {
const table = row[`Tables_in_${UserConfig.config.sql!.mySql!.database}` const table = row[`Tables_in_${UserConfig.config.database!.options!.database}`
] as DatabaseTable; ] as DatabaseTable;
if (table === 'assfiles') tablesExist.files = true; if (table === 'assfiles') tablesExist.files = true;
if (table === 'assusers') tablesExist.users = true; if (table === 'assusers') tablesExist.users = true;

35
backend/sql/postgres.ts
Unescape View File

@ -0,0 +1,35 @@
import { Client } from 'pg';
import { log } from '../log';
import { Database, DatabaseTable, DatabaseValue } from './database';
export class PostgreSQLDatabase implements Database {
private _client: Client;
public open(): Promise<void> { return Promise.resolve(); }
public close(): Promise<void> { return Promise.resolve(); }
public configure(): Promise<void> {
return new Promise((resolve, reject) => {
try {
} catch (err) {
log.error('PostgreSQL', 'failed to initialize');
console.error(err);
reject(err);
}
});
}
public put(table: DatabaseTable, key: string, data: DatabaseValue): Promise<void> {
throw new Error("Method not implemented.");
}
public get(table: DatabaseTable, key: string): Promise<DatabaseValue | undefined> {
throw new Error("Method not implemented.");
}
public getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue; }> {
throw new Error("Method not implemented.");
}
}

28
common/types.d.ts vendored
Unescape View File

@ -24,7 +24,7 @@ declare module 'ass' {
maximumFileSize: number; maximumFileSize: number;
s3?: S3Configuration; s3?: S3Configuration;
sql?: SqlConfiguration; database?: DatabaseConfiguration;
rateLimit?: RateLimitConfiguration; rateLimit?: RateLimitConfiguration;
} }
@ -51,13 +51,23 @@ declare module 'ass' {
} }
} }
interface SqlConfiguration { interface DatabaseConfiguration {
mySql?: { kind: 'mysql' | 'postgres' | 'json';
host: string; options?: MySQLConfiguration | PostgresConfiguration;
user: string; }
password: string;
database: string; interface MySQLConfiguration {
} host: string;
user: string;
password: string;
database: string;
}
interface PostgresConfiguration {
host: string;
user: string;
password: string;
database: string;
} }
/** /**
@ -219,7 +229,6 @@ declare module 'ass' {
files: { files: {
[key: NID]: AssFile; [key: NID]: AssFile;
} }
useSql: boolean;
meta: { [key: string]: any }; meta: { [key: string]: any };
} }
@ -232,7 +241,6 @@ declare module 'ass' {
[key: NID]: AssUser; [key: NID]: AssUser;
}; };
cliKey: string; cliKey: string;
useSql: boolean;
meta: { [key: string]: any }; meta: { [key: string]: any };
} }
} }

47
frontend/setup.mts
Unescape View File

@ -1,4 +1,4 @@
import { SlInput, SlButton } from '@shoelace-style/shoelace'; import { SlInput, SlButton, SlTab } from '@shoelace-style/shoelace';
import { IdType, UserConfiguration } from 'ass'; import { IdType, UserConfiguration } from 'ass';
const genericErrorAlert = () => alert('An error occured, please check the console for details'); const genericErrorAlert = () => alert('An error occured, please check the console for details');
@ -41,11 +41,20 @@ document.addEventListener('DOMContentLoaded', () => {
s3secretKey: document.querySelector('#s3-secretKey') as SlInput, s3secretKey: document.querySelector('#s3-secretKey') as SlInput,
s3region: document.querySelector('#s3-region') as SlInput, s3region: document.querySelector('#s3-region') as SlInput,
jsonTab: document.querySelector('#json-tab') as SlTab,
mySqlTab: document.querySelector('#mysql-tab') as SlTab,
mySqlHost: document.querySelector('#mysql-host') as SlInput, mySqlHost: document.querySelector('#mysql-host') as SlInput,
mySqlUser: document.querySelector('#mysql-user') as SlInput, mySqlUser: document.querySelector('#mysql-user') as SlInput,
mySqlPassword: document.querySelector('#mysql-password') as SlInput, mySqlPassword: document.querySelector('#mysql-password') as SlInput,
mySqlDatabase: document.querySelector('#mysql-database') as SlInput, mySqlDatabase: document.querySelector('#mysql-database') as SlInput,
pgsqlTab: document.querySelector('#pgsql-tab') as SlTab,
pgsqlHost: document.querySelector('#pgsql-host') as SlInput,
pgsqlUser: document.querySelector('#pgsql-user') as SlInput,
pgsqlPassword: document.querySelector('#pgsql-password') as SlInput,
pgsqlDatabase: document.querySelector('#pgsql-database') as SlInput,
userUsername: document.querySelector('#user-username') as SlInput, userUsername: document.querySelector('#user-username') as SlInput,
userPassword: document.querySelector('#user-password') as SlInput, userPassword: document.querySelector('#user-password') as SlInput,
@ -88,14 +97,34 @@ document.addEventListener('DOMContentLoaded', () => {
config.s3.region = Elements.s3region.value; config.s3.region = Elements.s3region.value;
} }
// Append MySQL to config, if specified // Append database to config, if specified
if (Elements.mySqlHost.value != null && Elements.mySqlHost.value !== '') { if (Elements.jsonTab.active) {
if (!config.sql) config.sql = {}; config.database = {
config.sql.mySql = { kind: 'json'
host: Elements.mySqlHost.value, };
user: Elements.mySqlUser.value, } else if (Elements.mySqlTab.active) {
password: Elements.mySqlPassword.value, if (Elements.mySqlHost.value != null && Elements.mySqlHost.value != '') {
database: Elements.mySqlDatabase.value config.database = {
kind: 'mysql',
options: {
host: Elements.mySqlHost.value,
user: Elements.mySqlUser.value,
password: Elements.mySqlPassword.value,
database: Elements.mySqlDatabase.value
}
}
};
} else if (Elements.pgsqlTab.active) {
if (Elements.pgsqlHost.value != null && Elements.pgsqlHost.value != '') {
config.database = {
kind: 'postgres',
options: {
host: Elements.pgsqlHost.value,
user: Elements.pgsqlUser.value,
password: Elements.pgsqlPassword.value,
database: Elements.pgsqlDatabase.value
}
}
}; };
} }

2
package.json
Unescape View File

@ -57,6 +57,7 @@
"mysql2": "^3.6.1", "mysql2": "^3.6.1",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",
"node-vibrant": "^3.1.6", "node-vibrant": "^3.1.6",
"pg": "^8.11.3",
"pug": "^3.0.2", "pug": "^3.0.2",
"sanitize-filename": "^1.6.3", "sanitize-filename": "^1.6.3",
"sharp": "^0.32.6", "sharp": "^0.32.6",
@ -76,6 +77,7 @@
"@types/luxon": "^3.3.2", "@types/luxon": "^3.3.2",
"@types/node": "^18.16.19", "@types/node": "^18.16.19",
"@types/node-fetch": "^2.6.6", "@types/node-fetch": "^2.6.6",
"@types/pg": "^8.10.7",
"@types/sharp": "^0.32.0", "@types/sharp": "^0.32.0",
"@types/tailwindcss": "^3.1.0", "@types/tailwindcss": "^3.1.0",
"@types/uuid": "^8.3.1" "@types/uuid": "^8.3.1"

45
views/setup.pug
Unescape View File

@ -30,6 +30,39 @@ block content
h3.setup-text-item-title Password h3.setup-text-item-title Password
sl-input#user-password(type='password' placeholder='the-most-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa') sl-input#user-password(type='password' placeholder='the-most-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa')
//- * Database
h2.setup-text-section-header.mt-4 Database
.setup-panel
sl-tab-group
//- * JSON
sl-tab#json-tab(slot='nav' panel='json') JSON
sl-tab-panel(name='json')
| you all good!
//- * MySQL
sl-tab#mysql-tab(slot='nav' panel='mysql') MySQL
sl-tab-panel(name='mysql')
h3.setup-text-item-title Host
sl-input#mysql-host(type='text' placeholder='mysql.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa')
h3.setup-text-item-title User
sl-input#mysql-user(type='text' placeholder='myassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa')
h3.setup-text-item-title Password
sl-input#mysql-password(type='password' placeholder='super-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa')
h3.setup-text-item-title Database
sl-input#mysql-database(type='text' placeholder='assdb' clearable): sl-icon(slot='prefix' name='fas-database' library='fa')
//- * PostgreSQL
sl-tab#pgsql-tab(slot='nav' panel='pgsql') PostgreSQL
sl-tab-panel(name='pgsql')
h3.setup-text-item-title Host
sl-input#pgsql-host(type='text' placeholder='postgres.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa')
h3.setup-text-item-title User
sl-input#pgsql-user(type='text' placeholder='posgrassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa')
h3.setup-text-item-title Password
sl-input#pgsql-password(type='password' placeholder='super-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa')
h3.setup-text-item-title Database
sl-input#pgsql-database(type='text' placeholder='assdb' clearable): sl-icon(slot='prefix' name='fas-database' library='fa')
//- * S3 //- * S3
h2.setup-text-section-header.mt-4 S3 #[span.setup-text-optional optional] h2.setup-text-section-header.mt-4 S3 #[span.setup-text-optional optional]
.setup-panel .setup-panel
@ -43,18 +76,6 @@ block content
sl-input#s3-secretKey(type='password' placeholder='EF56GH78IJ90KL12' clearable): sl-icon(slot='prefix' name='fas-user-secret' library='fa') sl-input#s3-secretKey(type='password' placeholder='EF56GH78IJ90KL12' clearable): sl-icon(slot='prefix' name='fas-user-secret' library='fa')
h3.setup-text-item-title Region #[span.setup-text-optional optional] h3.setup-text-item-title Region #[span.setup-text-optional optional]
sl-input#s3-region(type='text' placeholder='us-east' clearable): sl-icon(slot='prefix' name='fas-map-location-dot' library='fa') sl-input#s3-region(type='text' placeholder='us-east' clearable): sl-icon(slot='prefix' name='fas-map-location-dot' library='fa')
//- * MySQL
h2.setup-text-section-header.mt-4 MySQL #[span.setup-text-optional optional]
.setup-panel
h3.setup-text-item-title Host
sl-input#mysql-host(type='text' placeholder='mysql.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa')
h3.setup-text-item-title User
sl-input#mysql-user(type='text' placeholder='myassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa')
h3.setup-text-item-title Password
sl-input#mysql-password(type='password' placeholder='super-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa')
h3.setup-text-item-title Database
sl-input#mysql-database(type='text' placeholder='assdb' clearable): sl-icon(slot='prefix' name='fas-database' library='fa')
//- * Rate Limits //- * Rate Limits
h2.setup-text-section-header.mt-4 Rate Limits #[span.setup-text-optional optional] h2.setup-text-section-header.mt-4 Rate Limits #[span.setup-text-optional optional]

Write Preview
Loading…
Cancel
Save