From e87f2e447c72104666bf8e2c55124454c23d8f0f Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:13:24 +0100 Subject: [PATCH 1/7] Temporary commit while I move to a better machine --- .../aa07/paradise/taskdaemon/core/Core.java | 32 +- .../taskdaemon/core/database/DbCore.java | 16 +- .../modules/aclcleanup/AclCleanupJob.java | 3 +- .../bouncerrestart/BouncerRestartJob.java | 5 +- .../core/modules/devrank/DevRankJob.java | 5 +- .../IngameVerifiedSyncJob.java | 139 +++++++++ .../core/modules/ip2asn/Ip2AsnJob.java | 8 +- .../modules/patreonsync/PatreonSyncJob.java | 33 +- .../core/modules/prlabel/PrLabelJob.java | 3 +- .../core/modules/prlabel/StatusManager.java | 6 +- .../modules/prlabel/TestmergeManager.java | 7 +- .../profilercleanup/ProfilerCleanupJob.java | 3 +- .../profileringest/ProfilerWorker.java | 6 +- .../processors/ProfilerBaseProcessor.java | 4 - .../processors/ProfilerProcProcessor.java | 8 +- .../processors/ProfilerSendmapsProcessor.java | 8 +- .../taskdaemon/core/util/UtilConst.java | 12 + .../database/automation/Aa07Automation.java | 7 + .../database/automation/Indexes.java | 2 + .../taskdaemon/database/automation/Keys.java | 3 + .../database/automation/Tables.java | 6 + .../tables/IngameVerifiedDirectory.java | 168 ++++++++++ .../IngameVerifiedDirectoryRecord.java | 265 ++++++++++++++++ .../taskdaemon/database/gamedb/Indexes.java | 4 +- .../taskdaemon/database/gamedb/Keys.java | 6 +- .../database/gamedb/ParadiseGamedb.java | 14 +- .../taskdaemon/database/gamedb/Tables.java | 12 +- .../database/gamedb/tables/OauthTokens.java | 146 --------- .../database/gamedb/tables/TaskQueue.java | 166 ++++++++++ .../tables/records/OauthTokensRecord.java | 144 --------- .../tables/records/TaskQueueRecord.java | 295 ++++++++++++++++++ 31 files changed, 1155 insertions(+), 381 deletions(-) create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java create mode 100644 TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java create mode 100644 TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/records/IngameVerifiedDirectoryRecord.java delete mode 100644 TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/OauthTokens.java create mode 100644 TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/TaskQueue.java delete mode 100644 TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/OauthTokensRecord.java create mode 100644 TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/TaskQueueRecord.java diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java index 6c0ef6c..ad19693 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java @@ -9,6 +9,7 @@ import me.aa07.paradise.taskdaemon.core.modules.aclcleanup.AclCleanupJob; import me.aa07.paradise.taskdaemon.core.modules.bouncerrestart.BouncerRestartJob; import me.aa07.paradise.taskdaemon.core.modules.devrank.DevRankJob; +import me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync.IngameVerifiedSyncJob; import me.aa07.paradise.taskdaemon.core.modules.ip2asn.Ip2AsnJob; import me.aa07.paradise.taskdaemon.core.modules.patreonsync.PatreonSyncJob; import me.aa07.paradise.taskdaemon.core.modules.prlabel.PrLabelJob; @@ -185,6 +186,20 @@ private void setupJobs(Scheduler scheduler, DbCore dbCore, ConfigHolder config, .withSchedule(CronScheduleBuilder.cronSchedule("0 15 * * * ?")) // Every hour, offset 15 minutes .build(); + // Ingame verified sync + JobDataMap jdm_igvsync = new JobDataMap(); + jdm_igvsync.put("LOGGER", logger); + jdm_igvsync.put("DBCORE", dbCore); + JobDetail jd_igvsync = JobBuilder.newJob(IngameVerifiedSyncJob.class) + .withIdentity("igvsync", "igvsync") + .usingJobData(jdm_igvsync) + .build(); + CronTrigger ct_igvsync = TriggerBuilder.newTrigger() + .withIdentity("igvsync", "igvsync") + //.withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes + .withSchedule(CronScheduleBuilder.cronSchedule("0 * * * * ?")) // Every hour, offset 23 minutes + .build(); + // Lists that exist just to make sure these things show as recognised ArrayList bin1 = new ArrayList(); bin1.add(jd_aclcleanup); @@ -194,6 +209,7 @@ private void setupJobs(Scheduler scheduler, DbCore dbCore, ConfigHolder config, bin1.add(jd_profilercleanup); bin1.add(jd_pullrequests); bin1.add(jd_patreonsync); + bin1.add(jd_igvsync); ArrayList bin2 = new ArrayList(); bin2.add(ct_aclcleanup); bin2.add(ct_bouncerrestart); @@ -202,6 +218,7 @@ private void setupJobs(Scheduler scheduler, DbCore dbCore, ConfigHolder config, bin2.add(ct_profilercleanup); bin2.add(ct_pullrequests); bin2.add(ct_patreonsync); + bin2.add(ct_igvsync); // Is this necessary? Prolly not! bin1.clear(); @@ -209,13 +226,14 @@ private void setupJobs(Scheduler scheduler, DbCore dbCore, ConfigHolder config, // Schedule all - scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); - scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); - scheduler.scheduleJob(jd_devrank, ct_devrank); - scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); - scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); - scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); - scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); + //scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); + //scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); + //scheduler.scheduleJob(jd_devrank, ct_devrank); + //scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); + //scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); + //scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); + //scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); + scheduler.scheduleJob(jd_igvsync, ct_igvsync); } private void launchAll() { diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java index 8a65a0b..08414ea 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java @@ -9,6 +9,7 @@ import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; import me.aa07.paradise.taskdaemon.core.config.DatabaseConfig; import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; import org.jooq.SQLDialect; @@ -16,14 +17,14 @@ import org.jooq.impl.DSL; public class DbCore { - private HashMap connectionMap; + private HashMap> connectionMap; public DbCore(ConfigHolder config, Logger logger) { // Suppress JOOQ console spam System.getProperties().setProperty("org.jooq.no-logo", "true"); System.getProperties().setProperty("org.jooq.no-tips", "true"); - connectionMap = new HashMap(); + connectionMap = new HashMap>(); establishConnections(config); logger.info("Ready to handle DB requests"); @@ -79,21 +80,24 @@ private void establishConnections(ConfigHolder config) { for (DatabaseType dbtype : mysql_db_types.keySet()) { DatabaseConfig cfg = mysql_db_types.get(dbtype); DataSource ds = openMySqlDataSource(String.format("jdbc:mysql://%s/%s", cfg.host, cfg.database), cfg.username, cfg.password); - connectionMap.put(dbtype, ds); + connectionMap.put(dbtype, Pair.of(ds, SQLDialect.MYSQL)); } // And do special stuff for Postgres DatabaseConfig adb = config.authentikDatabase; - connectionMap.put(DatabaseType.Authentik, openPostgresDataSource(String.format("jdbc:postgresql://%s/%s", adb.host, adb.database), adb.username, adb.password)); + DataSource authentik_ds = openPostgresDataSource(String.format("jdbc:postgresql://%s/%s", adb.host, adb.database), adb.username, adb.password); + connectionMap.put(DatabaseType.Authentik, Pair.of(authentik_ds, SQLDialect.POSTGRES)); } // Get a DSL context - public DSLContext jooq(DatabaseType type, SQLDialect dialect) { + public DSLContext jooq(DatabaseType type) { if (!connectionMap.containsKey(type)) { throw new NoDataFoundException("Supplied type key not present in connection map"); } - return DSL.using(connectionMap.get(type), dialect); + Pair ds_pair = connectionMap.get(type); + + return DSL.using(ds_pair.getLeft(), ds_pair.getRight()); } // Easy way for NOW() in SQL diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java index 87a1d1b..e5b5c04 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java @@ -14,7 +14,6 @@ import me.aa07.paradise.taskdaemon.database.gamedb.Tables; import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; -import org.jooq.SQLDialect; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDataMap; @@ -113,7 +112,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException { public List checkIpsInDatabase(List ips, Logger logger, DbCore dbcore) { List to_remove = new ArrayList<>(); - DSLContext ctx = dbcore.jooq(DatabaseType.GameDb, SQLDialect.MYSQL); + DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); LocalDateTime ten_mins_ago = dbcore.now().minusMinutes(10); for (String ip : ips) { diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/bouncerrestart/BouncerRestartJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/bouncerrestart/BouncerRestartJob.java index 80f6177..f7b979e 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/bouncerrestart/BouncerRestartJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/bouncerrestart/BouncerRestartJob.java @@ -1,6 +1,5 @@ package me.aa07.paradise.taskdaemon.core.modules.bouncerrestart; -import com.google.gson.Gson; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -11,6 +10,7 @@ import java.util.Optional; import me.aa07.paradise.taskdaemon.core.config.TgsConfig; import me.aa07.paradise.taskdaemon.core.models.tgs.TokenResponseModel; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import org.apache.logging.log4j.Logger; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; @@ -76,8 +76,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { logger.info("[BouncerRestart] Sending auth token request"); HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); - Gson gson = new Gson(); - TokenResponseModel tokres = gson.fromJson(response.body(), TokenResponseModel.class); + TokenResponseModel tokres = UtilConst.GSON.fromJson(response.body(), TokenResponseModel.class); // And send our restart request HttpRequest httreq2 = HttpRequest.newBuilder() diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java index 6a72a31..776d3fc 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java @@ -16,7 +16,6 @@ import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; -import org.jooq.SQLDialect; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDataMap; @@ -49,8 +48,8 @@ public void execute(JobExecutionContext context) throws JobExecutionException { logger.info("[DevRank] Starting dev rank sync"); try { - DSLContext forums_db = dbcore.jooq(DatabaseType.Forums, SQLDialect.MYSQL); - DSLContext game_db = dbcore.jooq(DatabaseType.GameDb, SQLDialect.MYSQL); + DSLContext forums_db = dbcore.jooq(DatabaseType.Forums); + DSLContext game_db = dbcore.jooq(DatabaseType.GameDb); // Collect all dev team ckeys from forums database List dev_team_ckeys = new ArrayList<>(); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java new file mode 100644 index 0000000..7c1a984 --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java @@ -0,0 +1,139 @@ +package me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync; + +import java.util.ArrayList; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.core.util.UtilStr; +import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; +import me.aa07.paradise.taskdaemon.database.automation.Tables; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class IngameVerifiedSyncJob implements Job { + /* + + - pull raw data from authentik + - keep track of new ones + - get discord IDs and add into table + - get forum IDs and add into table + - remove entries from directory table that werent there before + - schedule task in ALICE to strip discord role + - remove forum group + - slap authentik ID onto rows in the player table + + */ + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[IngameVerifiedSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + logger.info("[IngameVerifiedSync] Starting up"); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[IngameVerifiedSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); + DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); + + // For now - we just want ckeys and authentik IDs - no more + Result authentik_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) + .fetch(); + + logger.info(String.format("[IngameVerifiedSync] Found %s accounts with linked byond accounts", authentik_records.size())); + + ArrayList only_authentik_ids = new ArrayList(); + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { + only_authentik_ids.add(acuscr.getUserId()); + } + + // Records to purge + Result records_to_purge = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.notIn(only_authentik_ids)) + .fetch(); + + int removed = 0; + for (IngameVerifiedDirectoryRecord ivdr : records_to_purge) { + // Step 1 - Strip their rank on the forums + + // Step 2 - Strip their rank on the discord + + // Step 3 - Clear their FUID ingame + + // Step 4 - Clear from directory + ivdr.delete(); + removed++; + } + logger.info(String.format("[IngameVerifiedSync] Removed %s records for members who unlinked byond", removed)); + + // Now begin the mass inserts + int raw_added = 0; + int raw_updated = 0; + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { + int authentik_user_ud = acuscr.getUserId(); + // First - see if they already exist in the directory + + if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)))) { + + // Update if they do + raw_updated++; + + IngameVerifiedDirectoryRecord ivdr = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)).fetchOne(); + + ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); + ivdr.store(); + } else { + // Insert if they dont + IngameVerifiedDirectoryRecord ivdr = automation_jooq.newRecord(Tables.INGAME_VERIFIED_DIRECTORY); + ivdr.setAuthentikId(acuscr.getUserId()); + ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); + ivdr.store(); + raw_added++; + } + } + + logger.info(String.format("[IngameVerifiedSync] Added %s records and updated %s for members who are linked", raw_added, raw_updated)); + + logger.info("[IngameVerifiedSync] Process complete"); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java index c20df52..143b6a0 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java @@ -1,6 +1,5 @@ package me.aa07.paradise.taskdaemon.core.modules.ip2asn; -import com.google.gson.Gson; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -14,12 +13,12 @@ import me.aa07.paradise.taskdaemon.core.database.DatabaseType; import me.aa07.paradise.taskdaemon.core.database.DbCore; import me.aa07.paradise.taskdaemon.core.models.ip2asn.Ip2AsnResponseModel; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import me.aa07.paradise.taskdaemon.database.gamedb.Tables; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.Ip2groupRecord; import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; import org.jooq.Record1; -import org.jooq.SQLDialect; import org.jooq.Select; import org.jooq.types.UInteger; import org.quartz.DisallowConcurrentExecution; @@ -80,7 +79,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { Ip2AsnSerivceConfig config = ip2asn_cfg_holder.get(); - DSLContext ctx = dbcore.jooq(DatabaseType.GameDb, SQLDialect.MYSQL); + DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); // So // JOOQ doesnt have INET_ATON, presumably because its a MySQL native @@ -134,7 +133,6 @@ public void execute(JobExecutionContext event) throws JobExecutionException { // Now do the big refresh HttpClient client = HttpClient.newHttpClient(); - Gson gson = new Gson(); for (String ip : ips_to_get) { try { // Get our AS - probably a more elegant way to do this but meh @@ -148,7 +146,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); // Decode the shenanigans - Ip2AsnResponseModel res_model = gson.fromJson(response.body(), Ip2AsnResponseModel.class); + Ip2AsnResponseModel res_model = UtilConst.GSON.fromJson(response.body(), Ip2AsnResponseModel.class); UInteger ip_uint = ip2uint(ip); UInteger asn = UInteger.valueOf(res_model.asn); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java index 08cac91..f610911 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java @@ -1,6 +1,5 @@ package me.aa07.paradise.taskdaemon.core.modules.patreonsync; -import com.google.gson.Gson; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -18,6 +17,7 @@ import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonRawResponseModel; import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonUser; import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import me.aa07.paradise.taskdaemon.core.util.UtilStr; import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; import me.aa07.paradise.taskdaemon.database.automation.Tables; @@ -28,7 +28,6 @@ import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; import org.jooq.Result; -import org.jooq.SQLDialect; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDataMap; @@ -48,10 +47,6 @@ public class PatreonSyncJob implements Job { */ - private static final UUID PATREON_OAUTH_SERVICE_ID = UUID.fromString("626f4935-d046-4957-9862-f26102e27a70"); - private static final UUID DISCORD_OAUTH_SERVICE_ID = UUID.fromString("bc9fb785-6a1f-434e-80da-26a37450a29a"); - private static final UUID BYOND_OAUTH_SERVICE_ID = UUID.fromString("d7a7b659-69bd-4435-b585-3b4ba97ca076"); - @Override public void execute(JobExecutionContext event) throws JobExecutionException { JobDataMap datamap = event.getMergedJobDataMap(); @@ -107,7 +102,6 @@ public void execute(JobExecutionContext event) throws JobExecutionException { // The patreon API is kinda bad // Everything goes into the same loose model and you have to manually parse out members and tiers // Boy thats fun - Gson gson = new Gson(); // we use this later ArrayList users = new ArrayList(); @@ -144,7 +138,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { fw.close(); */ - PatreonOuterResponseModel holder = gson.fromJson(response_body, PatreonOuterResponseModel.class); + PatreonOuterResponseModel holder = UtilConst.GSON.fromJson(response_body, PatreonOuterResponseModel.class); logger.info(String.format("[PatreonSync] Pulled %s raw models from Patreon", holder.data.size())); @@ -173,7 +167,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { } // Step 1 - Clear out old users - DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation, SQLDialect.MYSQL); + DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); Result records_to_purge = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) @@ -193,7 +187,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { // No I will not apologise for these variable names DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); drtam.discordId = database_discord_id; - String drtam_json = gson.toJson(drtam); + String drtam_json = UtilConst.GSON.toJson(drtam); tqr.setTaskArguments(drtam_json); tqr.setDateInserted(dbcore.now()); @@ -243,15 +237,15 @@ public void execute(JobExecutionContext event) throws JobExecutionException { logger.info(String.format("[PatreonSync] Inserted %s users, updated %s users", insert_count, update_count)); - DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb, SQLDialect.MYSQL); - DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik, SQLDialect.POSTGRES); + DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); + DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); // Find authentik users HashMap authentik_to_patreon = new HashMap(); findAuthentikUsers(only_member_ids, authentik_jooq, automation_jooq, logger, authentik_to_patreon); // Now find discord users - findDiscordUsers(authentik_jooq, automation_jooq, logger, authentik_to_patreon, gson, dbcore); + findDiscordUsers(authentik_jooq, automation_jooq, logger, authentik_to_patreon, dbcore); // Now get ckeys findCkeys(authentik_jooq, automation_jooq, logger, authentik_to_patreon); @@ -296,7 +290,7 @@ private static void findAuthentikUsers( AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.IDENTIFIER.eq(String.valueOf(member_id))) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(PATREON_OAUTH_SERVICE_ID)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.PATREON_OAUTH_SERVICE_ID)) .fetchOne(); if (authentik_record == null) { @@ -337,7 +331,6 @@ private static void findDiscordUsers( DSLContext automationJooq, Logger logger, HashMap authentik2patreon, - Gson gson, DbCore dbCore ) { HashMap authentik2discord = new HashMap(); @@ -349,7 +342,7 @@ private static void findDiscordUsers( AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(DISCORD_OAUTH_SERVICE_ID)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) .fetchOne(); if (authentik_record == null) { @@ -392,7 +385,7 @@ private static void findDiscordUsers( // No I will not apologise for these variable names DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); drtam.discordId = discord_id; - String drtam_json = gson.toJson(drtam); + String drtam_json = UtilConst.GSON.toJson(drtam); tqr.setTaskArguments(drtam_json); tqr.setDateInserted(dbCore.now()); @@ -412,7 +405,7 @@ private static void findDiscordUsers( // No I will not apologise for these variable names DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); drtam.discordId = discord_id; - String drtam_json = gson.toJson(drtam); + String drtam_json = UtilConst.GSON.toJson(drtam); tqr.setTaskArguments(drtam_json); tqr.setDateInserted(dbCore.now()); @@ -429,7 +422,7 @@ private static void findDiscordUsers( // No I will not apologise for these variable names DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); drtam.discordId = discord_id; - String drtam_json = gson.toJson(drtam); + String drtam_json = UtilConst.GSON.toJson(drtam); tqr.setTaskArguments(drtam_json); tqr.setDateInserted(dbCore.now()); @@ -461,7 +454,7 @@ private static void findCkeys( AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(BYOND_OAUTH_SERVICE_ID)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) .fetchOne(); if (authentik_record == null) { diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java index e9adf6d..ec51abf 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java @@ -20,7 +20,6 @@ import org.jooq.Record2; import org.jooq.Record4; import org.jooq.Result; -import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.kohsuke.github.GHIssueState; import org.kohsuke.github.GHLabel; @@ -104,7 +103,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { // Initial setup is done - lets now load the important bits // Get this now - we need it - DSLContext ctx = dbcore.jooq(DatabaseType.PullRequests, SQLDialect.MYSQL); + DSLContext ctx = dbcore.jooq(DatabaseType.PullRequests); // Dictionary that holds all this in a nice lookup HashMap prs = new HashMap(); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/StatusManager.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/StatusManager.java index 0c68a0b..cc14779 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/StatusManager.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/StatusManager.java @@ -9,6 +9,7 @@ import java.util.List; import me.aa07.paradise.taskdaemon.core.config.GithubConfig; import me.aa07.paradise.taskdaemon.core.models.prlabel.PullRequest; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVoteType; import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVotingGroup; import org.apache.logging.log4j.Logger; @@ -61,9 +62,8 @@ public void processPr(PullRequest pr) throws IOException { } // If we got here we can begin the rest - Type inner_string_type = new TypeToken>() { - }.getType(); - List pr_types = new Gson().fromJson(pr.prTypes, inner_string_type); + Type inner_string_type = new TypeToken>() {}.getType(); + List pr_types = UtilConst.GSON.fromJson(pr.prTypes, inner_string_type); boolean passes_approval = false; boolean passes_review = false; diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/TestmergeManager.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/TestmergeManager.java index 30fe2d7..4bcdfc6 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/TestmergeManager.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/TestmergeManager.java @@ -1,6 +1,5 @@ package me.aa07.paradise.taskdaemon.core.modules.prlabel; -import com.google.gson.Gson; import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -15,6 +14,7 @@ import me.aa07.paradise.taskdaemon.core.models.tgs.DreamDaemonResponse; import me.aa07.paradise.taskdaemon.core.models.tgs.TestMerge; import me.aa07.paradise.taskdaemon.core.models.tgs.TokenResponseModel; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import org.apache.logging.log4j.Logger; public class TestmergeManager implements IPullRequestProcessor { @@ -56,8 +56,7 @@ public void setup() { logger.info("[PrLabelJob] [TestmergeManager] Sending auth token request"); HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); - Gson gson = new Gson(); - TokenResponseModel tokres = gson.fromJson(response.body(), TokenResponseModel.class); + TokenResponseModel tokres = UtilConst.GSON.fromJson(response.body(), TokenResponseModel.class); HttpRequest httreq2 = HttpRequest.newBuilder() .uri(new URI(String.format("%s/api/DreamDaemon", tgsConfig.tgsHost))).GET() @@ -70,7 +69,7 @@ public void setup() { HttpClient client2 = HttpClient.newHttpClient(); HttpResponse response2 = client2.send(httreq2, BodyHandlers.ofString()); - DreamDaemonResponse ddr = gson.fromJson(response2.body(), DreamDaemonResponse.class); + DreamDaemonResponse ddr = UtilConst.GSON.fromJson(response2.body(), DreamDaemonResponse.class); if (ddr.activeCompileJob.revisionInformation.activeTestMerges != null) { for (TestMerge tm : ddr.activeCompileJob.revisionInformation.activeTestMerges) { diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java index e6bf235..2ac90d8 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java @@ -6,7 +6,6 @@ import me.aa07.paradise.taskdaemon.database.profiler.Tables; import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; -import org.jooq.SQLDialect; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDataMap; @@ -50,7 +49,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { DbCore dbcore = dbcore_holder.get(); - DSLContext profiler_db = dbcore.jooq(DatabaseType.ProfilerDb, SQLDialect.MYSQL); + DSLContext profiler_db = dbcore.jooq(DatabaseType.ProfilerDb); logger.info("[ProfilerCleanup] Cleaning out profiler DB - proc samples"); profiler_db.delete(Tables.SAMPLES).where(Tables.SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java index f5f2c99..158281b 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java @@ -1,18 +1,17 @@ package me.aa07.paradise.taskdaemon.core.modules.profileringest; -import com.google.gson.Gson; import java.util.ArrayList; import me.aa07.paradise.taskdaemon.core.database.DbCore; import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerProcProcessor; import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerSendmapsProcessor; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import org.apache.logging.log4j.Logger; public class ProfilerWorker { private ArrayList queue; private Logger logger; private Object listLock; - private Gson gson; private ProfilerProcProcessor procProcessor; private ProfilerSendmapsProcessor sendmapsProcessor; @@ -25,7 +24,6 @@ public ProfilerWorker(DbCore database, Logger logger) { queue = new ArrayList(); listLock = new Object(); - gson = new Gson(); } // TODO - Refactor into quartz job @@ -75,7 +73,7 @@ private void processEntry(String entry) { long start = System.currentTimeMillis(); logger.info(String.format("Processing entry with size %s bytes", entry.length())); - ProfilerHolder holder = gson.fromJson(entry, ProfilerHolder.class); + ProfilerHolder holder = UtilConst.GSON.fromJson(entry, ProfilerHolder.class); logger.info(String.format("Round ID: %s | Profile size: %s bytes", holder.roundId, holder.profilerData.length())); procProcessor.processData(holder); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java index d40cfb3..f009224 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java @@ -1,6 +1,5 @@ package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; -import com.google.gson.Gson; import me.aa07.paradise.taskdaemon.core.database.DbCore; import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; import org.apache.logging.log4j.Logger; @@ -8,15 +7,12 @@ public abstract class ProfilerBaseProcessor { protected DbCore database; protected Logger logger; - protected Gson gson; private String processorName; public ProfilerBaseProcessor(DbCore database, Logger logger, String processorName) { this.database = database; this.logger = logger; this.processorName = processorName; - - gson = new Gson(); } protected void log(String txt) { diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java index b7b55d3..22a7da8 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java @@ -6,12 +6,12 @@ import me.aa07.paradise.taskdaemon.core.database.DbCore; import me.aa07.paradise.taskdaemon.core.models.profiler.ProcData; import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import me.aa07.paradise.taskdaemon.database.profiler.Tables; import me.aa07.paradise.taskdaemon.database.profiler.tables.records.ProcsRecord; import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SamplesRecord; import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; -import org.jooq.SQLDialect; public class ProfilerProcProcessor extends ProfilerBaseProcessor { public ProfilerProcProcessor(DbCore database, Logger logger) { @@ -20,13 +20,13 @@ public ProfilerProcProcessor(DbCore database, Logger logger) { @Override protected void doProcessing(ProfilerHolder holder) { - List proc_list = gson.fromJson( + List proc_list = UtilConst.GSON.fromJson( holder.profilerData, new TypeToken>() {}.getType() ); log(String.format("Procs logged: %s", proc_list.size())); - DSLContext jooq = database.jooq(DatabaseType.ProfilerDb, SQLDialect.MYSQL); + DSLContext jooq = database.jooq(DatabaseType.ProfilerDb); for (ProcData procdata : proc_list) { long proc_id = getProcId(procdata.name); @@ -45,7 +45,7 @@ protected void doProcessing(ProfilerHolder holder) { @Override protected long getProcId(String procname) { - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb, SQLDialect.MYSQL); + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); if (!ctx.fetchExists(ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)))) { // We dont exist, make us log(String.format("%s did not exist in the DB. It does now.", procname)); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java index 5f3b449..9f7f867 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java @@ -6,12 +6,12 @@ import me.aa07.paradise.taskdaemon.core.database.DbCore; import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; import me.aa07.paradise.taskdaemon.core.models.profiler.SendmapsProcData; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; import me.aa07.paradise.taskdaemon.database.profiler.Tables; import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsProcsRecord; import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsSamplesRecord; import org.apache.logging.log4j.Logger; import org.jooq.DSLContext; -import org.jooq.SQLDialect; public class ProfilerSendmapsProcessor extends ProfilerBaseProcessor { public ProfilerSendmapsProcessor(DbCore database, Logger logger) { @@ -20,13 +20,13 @@ public ProfilerSendmapsProcessor(DbCore database, Logger logger) { @Override protected void doProcessing(ProfilerHolder holder) { - List proc_list = gson.fromJson( + List proc_list = UtilConst.GSON.fromJson( holder.sendmapsData, new TypeToken>() {}.getType() ); log(String.format("Procs logged: %s", proc_list.size())); - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb, SQLDialect.MYSQL); + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); for (SendmapsProcData procdata : proc_list) { long proc_id = getProcId(procdata.name); @@ -42,7 +42,7 @@ protected void doProcessing(ProfilerHolder holder) { @Override protected long getProcId(String procname) { - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb, SQLDialect.MYSQL); + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); if (!ctx.fetchExists(ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)))) { // We dont exist, make us log(String.format("%s did not exist in the DB. It does now.", procname)); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java new file mode 100644 index 0000000..e911cc6 --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java @@ -0,0 +1,12 @@ +package me.aa07.paradise.taskdaemon.core.util; + +import com.google.gson.Gson; +import java.util.UUID; + +public class UtilConst { + public static final UUID BYOND_OAUTH_SERVICE_ID = UUID.fromString("d7a7b659-69bd-4435-b585-3b4ba97ca076"); + public static final UUID DISCORD_OAUTH_SERVICE_ID = UUID.fromString("bc9fb785-6a1f-434e-80da-26a37450a29a"); + public static final UUID PATREON_OAUTH_SERVICE_ID = UUID.fromString("626f4935-d046-4957-9862-f26102e27a70"); + + public static final Gson GSON = new Gson(); +} diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Aa07Automation.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Aa07Automation.java index f732b91..e899594 100644 --- a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Aa07Automation.java +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Aa07Automation.java @@ -11,6 +11,7 @@ import me.aa07.paradise.taskdaemon.database.automation.tables.Availability; import me.aa07.paradise.taskdaemon.database.automation.tables.GithubLinks; import me.aa07.paradise.taskdaemon.database.automation.tables.Images; +import me.aa07.paradise.taskdaemon.database.automation.tables.IngameVerifiedDirectory; import me.aa07.paradise.taskdaemon.database.automation.tables.MemberCache; import me.aa07.paradise.taskdaemon.database.automation.tables.PatreonSupporters; import me.aa07.paradise.taskdaemon.database.automation.tables.ShortUrls; @@ -54,6 +55,11 @@ public class Aa07Automation extends SchemaImpl { */ public final Images IMAGES = Images.IMAGES; + /** + * The table aa07_automation.ingame_verified_directory. + */ + public final IngameVerifiedDirectory INGAME_VERIFIED_DIRECTORY = IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY; + /** * The table aa07_automation.member_cache. */ @@ -94,6 +100,7 @@ public final List> getTables() { Availability.AVAILABILITY, GithubLinks.GITHUB_LINKS, Images.IMAGES, + IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY, MemberCache.MEMBER_CACHE, PatreonSupporters.PATREON_SUPPORTERS, ShortUrls.SHORT_URLS, diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Indexes.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Indexes.java index 7c23d80..8a57fb0 100644 --- a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Indexes.java +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Indexes.java @@ -5,6 +5,7 @@ import me.aa07.paradise.taskdaemon.database.automation.tables.AdminPhpLog; +import me.aa07.paradise.taskdaemon.database.automation.tables.IngameVerifiedDirectory; import me.aa07.paradise.taskdaemon.database.automation.tables.PatreonSupporters; import me.aa07.paradise.taskdaemon.database.automation.tables.TaskQueue; @@ -25,6 +26,7 @@ public class Indexes { // ------------------------------------------------------------------------- public static final Index PATREON_SUPPORTERS_CKEY = Internal.createIndex(DSL.name("ckey"), PatreonSupporters.PATREON_SUPPORTERS, new OrderField[] { PatreonSupporters.PATREON_SUPPORTERS.CKEY }, false); + public static final Index INGAME_VERIFIED_DIRECTORY_DISCORD_ID = Internal.createIndex(DSL.name("discord_id"), IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY, new OrderField[] { IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.DISCORD_ID }, false); public static final Index ADMIN_PHP_LOG_FUID = Internal.createIndex(DSL.name("fuid"), AdminPhpLog.ADMIN_PHP_LOG, new OrderField[] { AdminPhpLog.ADMIN_PHP_LOG.FUID }, false); public static final Index TASK_QUEUE_PROCESSED = Internal.createIndex(DSL.name("processed"), TaskQueue.TASK_QUEUE, new OrderField[] { TaskQueue.TASK_QUEUE.PROCESSED }, false); public static final Index TASK_QUEUE_TASK_CONSUMER = Internal.createIndex(DSL.name("task_consumer"), TaskQueue.TASK_QUEUE, new OrderField[] { TaskQueue.TASK_QUEUE.TASK_CONSUMER }, false); diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java index 8cdfbf1..0fc02c7 100644 --- a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java @@ -8,6 +8,7 @@ import me.aa07.paradise.taskdaemon.database.automation.tables.Availability; import me.aa07.paradise.taskdaemon.database.automation.tables.GithubLinks; import me.aa07.paradise.taskdaemon.database.automation.tables.Images; +import me.aa07.paradise.taskdaemon.database.automation.tables.IngameVerifiedDirectory; import me.aa07.paradise.taskdaemon.database.automation.tables.MemberCache; import me.aa07.paradise.taskdaemon.database.automation.tables.PatreonSupporters; import me.aa07.paradise.taskdaemon.database.automation.tables.ShortUrls; @@ -16,6 +17,7 @@ import me.aa07.paradise.taskdaemon.database.automation.tables.records.AvailabilityRecord; import me.aa07.paradise.taskdaemon.database.automation.tables.records.GithubLinksRecord; import me.aa07.paradise.taskdaemon.database.automation.tables.records.ImagesRecord; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; import me.aa07.paradise.taskdaemon.database.automation.tables.records.MemberCacheRecord; import me.aa07.paradise.taskdaemon.database.automation.tables.records.PatreonSupportersRecord; import me.aa07.paradise.taskdaemon.database.automation.tables.records.ShortUrlsRecord; @@ -43,6 +45,7 @@ public class Keys { public static final UniqueKey KEY_AVAILABILITY_PRIMARY = Internal.createUniqueKey(Availability.AVAILABILITY, DSL.name("KEY_availability_PRIMARY"), new TableField[] { Availability.AVAILABILITY.ID }, true); public static final UniqueKey KEY_GITHUB_LINKS_PRIMARY = Internal.createUniqueKey(GithubLinks.GITHUB_LINKS, DSL.name("KEY_github_links_PRIMARY"), new TableField[] { GithubLinks.GITHUB_LINKS.GITHUB_ID }, true); public static final UniqueKey KEY_IMAGES_PRIMARY = Internal.createUniqueKey(Images.IMAGES, DSL.name("KEY_images_PRIMARY"), new TableField[] { Images.IMAGES.ID }, true); + public static final UniqueKey KEY_INGAME_VERIFIED_DIRECTORY_PRIMARY = Internal.createUniqueKey(IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY, DSL.name("KEY_ingame_verified_directory_PRIMARY"), new TableField[] { IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID }, true); public static final UniqueKey KEY_MEMBER_CACHE_PRIMARY = Internal.createUniqueKey(MemberCache.MEMBER_CACHE, DSL.name("KEY_member_cache_PRIMARY"), new TableField[] { MemberCache.MEMBER_CACHE.FUID }, true); public static final UniqueKey KEY_PATREON_SUPPORTERS_PRIMARY = Internal.createUniqueKey(PatreonSupporters.PATREON_SUPPORTERS, DSL.name("KEY_patreon_supporters_PRIMARY"), new TableField[] { PatreonSupporters.PATREON_SUPPORTERS.MEMBER_ID }, true); public static final UniqueKey KEY_SHORT_URLS_PRIMARY = Internal.createUniqueKey(ShortUrls.SHORT_URLS, DSL.name("KEY_short_urls_PRIMARY"), new TableField[] { ShortUrls.SHORT_URLS.URL_KEY }, true); diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Tables.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Tables.java index 9020f36..3f637ee 100644 --- a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Tables.java +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Tables.java @@ -8,6 +8,7 @@ import me.aa07.paradise.taskdaemon.database.automation.tables.Availability; import me.aa07.paradise.taskdaemon.database.automation.tables.GithubLinks; import me.aa07.paradise.taskdaemon.database.automation.tables.Images; +import me.aa07.paradise.taskdaemon.database.automation.tables.IngameVerifiedDirectory; import me.aa07.paradise.taskdaemon.database.automation.tables.MemberCache; import me.aa07.paradise.taskdaemon.database.automation.tables.PatreonSupporters; import me.aa07.paradise.taskdaemon.database.automation.tables.ShortUrls; @@ -40,6 +41,11 @@ public class Tables { */ public static final Images IMAGES = Images.IMAGES; + /** + * The table aa07_automation.ingame_verified_directory. + */ + public static final IngameVerifiedDirectory INGAME_VERIFIED_DIRECTORY = IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY; + /** * The table aa07_automation.member_cache. */ diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java new file mode 100644 index 0000000..ce8ed01 --- /dev/null +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java @@ -0,0 +1,168 @@ +/* + * This file is generated by jOOQ. + */ +package me.aa07.paradise.taskdaemon.database.automation.tables; + + +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; + +import me.aa07.paradise.taskdaemon.database.automation.Aa07Automation; +import me.aa07.paradise.taskdaemon.database.automation.Indexes; +import me.aa07.paradise.taskdaemon.database.automation.Keys; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; + +import org.jooq.Field; +import org.jooq.ForeignKey; +import org.jooq.Index; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.Row5; +import org.jooq.Schema; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.TableOptions; +import org.jooq.UniqueKey; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.TableImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes" }) +public class IngameVerifiedDirectory extends TableImpl { + + private static final long serialVersionUID = 1L; + + /** + * The reference instance of + * aa07_automation.ingame_verified_directory + */ + public static final IngameVerifiedDirectory INGAME_VERIFIED_DIRECTORY = new IngameVerifiedDirectory(); + + /** + * The class holding records for this type + */ + @Override + public Class getRecordType() { + return IngameVerifiedDirectoryRecord.class; + } + + /** + * The column + * aa07_automation.ingame_verified_directory.authentik_id. + */ + public final TableField AUTHENTIK_ID = createField(DSL.name("authentik_id"), SQLDataType.INTEGER.nullable(false), this, ""); + + /** + * The column aa07_automation.ingame_verified_directory.ckey. + */ + public final TableField CKEY = createField(DSL.name("ckey"), SQLDataType.VARCHAR(128).nullable(false), this, ""); + + /** + * The column + * aa07_automation.ingame_verified_directory.discord_id. + */ + public final TableField DISCORD_ID = createField(DSL.name("discord_id"), SQLDataType.BIGINT.defaultValue(DSL.inline("NULL", SQLDataType.BIGINT)), this, ""); + + /** + * The column + * aa07_automation.ingame_verified_directory.forum_id. + */ + public final TableField FORUM_ID = createField(DSL.name("forum_id"), SQLDataType.INTEGER.defaultValue(DSL.inline("NULL", SQLDataType.INTEGER)), this, ""); + + /** + * The column + * aa07_automation.ingame_verified_directory.date_inserted. + */ + public final TableField DATE_INSERTED = createField(DSL.name("date_inserted"), SQLDataType.LOCALDATETIME(0).nullable(false).defaultValue(DSL.field("current_timestamp()", SQLDataType.LOCALDATETIME)), this, ""); + + private IngameVerifiedDirectory(Name alias, Table aliased) { + this(alias, aliased, null); + } + + private IngameVerifiedDirectory(Name alias, Table aliased, Field[] parameters) { + super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.table()); + } + + /** + * Create an aliased aa07_automation.ingame_verified_directory + * table reference + */ + public IngameVerifiedDirectory(String alias) { + this(DSL.name(alias), INGAME_VERIFIED_DIRECTORY); + } + + /** + * Create an aliased aa07_automation.ingame_verified_directory + * table reference + */ + public IngameVerifiedDirectory(Name alias) { + this(alias, INGAME_VERIFIED_DIRECTORY); + } + + /** + * Create a aa07_automation.ingame_verified_directory table + * reference + */ + public IngameVerifiedDirectory() { + this(DSL.name("ingame_verified_directory"), null); + } + + public IngameVerifiedDirectory(Table child, ForeignKey key) { + super(child, key, INGAME_VERIFIED_DIRECTORY); + } + + @Override + public Schema getSchema() { + return aliased() ? null : Aa07Automation.AA07_AUTOMATION; + } + + @Override + public List getIndexes() { + return Arrays.asList(Indexes.INGAME_VERIFIED_DIRECTORY_DISCORD_ID); + } + + @Override + public UniqueKey getPrimaryKey() { + return Keys.KEY_INGAME_VERIFIED_DIRECTORY_PRIMARY; + } + + @Override + public IngameVerifiedDirectory as(String alias) { + return new IngameVerifiedDirectory(DSL.name(alias), this); + } + + @Override + public IngameVerifiedDirectory as(Name alias) { + return new IngameVerifiedDirectory(alias, this); + } + + /** + * Rename this table + */ + @Override + public IngameVerifiedDirectory rename(String name) { + return new IngameVerifiedDirectory(DSL.name(name), null); + } + + /** + * Rename this table + */ + @Override + public IngameVerifiedDirectory rename(Name name) { + return new IngameVerifiedDirectory(name, null); + } + + // ------------------------------------------------------------------------- + // Row5 type methods + // ------------------------------------------------------------------------- + + @Override + public Row5 fieldsRow() { + return (Row5) super.fieldsRow(); + } +} diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/records/IngameVerifiedDirectoryRecord.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/records/IngameVerifiedDirectoryRecord.java new file mode 100644 index 0000000..53e0221 --- /dev/null +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/records/IngameVerifiedDirectoryRecord.java @@ -0,0 +1,265 @@ +/* + * This file is generated by jOOQ. + */ +package me.aa07.paradise.taskdaemon.database.automation.tables.records; + + +import java.time.LocalDateTime; + +import me.aa07.paradise.taskdaemon.database.automation.tables.IngameVerifiedDirectory; + +import org.jooq.Field; +import org.jooq.Record1; +import org.jooq.Record5; +import org.jooq.Row5; +import org.jooq.impl.UpdatableRecordImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes" }) +public class IngameVerifiedDirectoryRecord extends UpdatableRecordImpl implements Record5 { + + private static final long serialVersionUID = 1L; + + /** + * Setter for + * aa07_automation.ingame_verified_directory.authentik_id. + */ + public void setAuthentikId(Integer value) { + set(0, value); + } + + /** + * Getter for + * aa07_automation.ingame_verified_directory.authentik_id. + */ + public Integer getAuthentikId() { + return (Integer) get(0); + } + + /** + * Setter for aa07_automation.ingame_verified_directory.ckey. + */ + public void setCkey(String value) { + set(1, value); + } + + /** + * Getter for aa07_automation.ingame_verified_directory.ckey. + */ + public String getCkey() { + return (String) get(1); + } + + /** + * Setter for + * aa07_automation.ingame_verified_directory.discord_id. + */ + public void setDiscordId(Long value) { + set(2, value); + } + + /** + * Getter for + * aa07_automation.ingame_verified_directory.discord_id. + */ + public Long getDiscordId() { + return (Long) get(2); + } + + /** + * Setter for + * aa07_automation.ingame_verified_directory.forum_id. + */ + public void setForumId(Integer value) { + set(3, value); + } + + /** + * Getter for + * aa07_automation.ingame_verified_directory.forum_id. + */ + public Integer getForumId() { + return (Integer) get(3); + } + + /** + * Setter for + * aa07_automation.ingame_verified_directory.date_inserted. + */ + public void setDateInserted(LocalDateTime value) { + set(4, value); + } + + /** + * Getter for + * aa07_automation.ingame_verified_directory.date_inserted. + */ + public LocalDateTime getDateInserted() { + return (LocalDateTime) get(4); + } + + // ------------------------------------------------------------------------- + // Primary key information + // ------------------------------------------------------------------------- + + @Override + public Record1 key() { + return (Record1) super.key(); + } + + // ------------------------------------------------------------------------- + // Record5 type implementation + // ------------------------------------------------------------------------- + + @Override + public Row5 fieldsRow() { + return (Row5) super.fieldsRow(); + } + + @Override + public Row5 valuesRow() { + return (Row5) super.valuesRow(); + } + + @Override + public Field field1() { + return IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID; + } + + @Override + public Field field2() { + return IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.CKEY; + } + + @Override + public Field field3() { + return IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.DISCORD_ID; + } + + @Override + public Field field4() { + return IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.FORUM_ID; + } + + @Override + public Field field5() { + return IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.DATE_INSERTED; + } + + @Override + public Integer component1() { + return getAuthentikId(); + } + + @Override + public String component2() { + return getCkey(); + } + + @Override + public Long component3() { + return getDiscordId(); + } + + @Override + public Integer component4() { + return getForumId(); + } + + @Override + public LocalDateTime component5() { + return getDateInserted(); + } + + @Override + public Integer value1() { + return getAuthentikId(); + } + + @Override + public String value2() { + return getCkey(); + } + + @Override + public Long value3() { + return getDiscordId(); + } + + @Override + public Integer value4() { + return getForumId(); + } + + @Override + public LocalDateTime value5() { + return getDateInserted(); + } + + @Override + public IngameVerifiedDirectoryRecord value1(Integer value) { + setAuthentikId(value); + return this; + } + + @Override + public IngameVerifiedDirectoryRecord value2(String value) { + setCkey(value); + return this; + } + + @Override + public IngameVerifiedDirectoryRecord value3(Long value) { + setDiscordId(value); + return this; + } + + @Override + public IngameVerifiedDirectoryRecord value4(Integer value) { + setForumId(value); + return this; + } + + @Override + public IngameVerifiedDirectoryRecord value5(LocalDateTime value) { + setDateInserted(value); + return this; + } + + @Override + public IngameVerifiedDirectoryRecord values(Integer value1, String value2, Long value3, Integer value4, LocalDateTime value5) { + value1(value1); + value2(value2); + value3(value3); + value4(value4); + value5(value5); + return this; + } + + // ------------------------------------------------------------------------- + // Constructors + // ------------------------------------------------------------------------- + + /** + * Create a detached IngameVerifiedDirectoryRecord + */ + public IngameVerifiedDirectoryRecord() { + super(IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY); + } + + /** + * Create a detached, initialised IngameVerifiedDirectoryRecord + */ + public IngameVerifiedDirectoryRecord(Integer authentikId, String ckey, Long discordId, Integer forumId, LocalDateTime dateInserted) { + super(IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY); + + setAuthentikId(authentikId); + setCkey(ckey); + setDiscordId(discordId); + setForumId(forumId); + setDateInserted(dateInserted); + } +} diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Indexes.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Indexes.java index 56b9fd5..f667a84 100644 --- a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Indexes.java +++ b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Indexes.java @@ -17,8 +17,8 @@ import me.aa07.paradise.taskdaemon.database.gamedb.tables.JsonDatumSaves; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Library; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Notes; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.OauthTokens; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Player; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.TaskQueue; import org.jooq.Index; import org.jooq.OrderField; @@ -45,7 +45,6 @@ public class Indexes { public static final Index JSON_DATUM_SAVES_CKEY = Internal.createIndex(DSL.name("ckey"), JsonDatumSaves.JSON_DATUM_SAVES, new OrderField[] { JsonDatumSaves.JSON_DATUM_SAVES.CKEY }, false); public static final Index LIBRARY_CKEY = Internal.createIndex(DSL.name("ckey"), Library.LIBRARY, new OrderField[] { Library.LIBRARY.CKEY }, false); public static final Index NOTES_CKEY = Internal.createIndex(DSL.name("ckey"), Notes.NOTES, new OrderField[] { Notes.NOTES.CKEY }, false); - public static final Index OAUTH_TOKENS_CKEY = Internal.createIndex(DSL.name("ckey"), OauthTokens.OAUTH_TOKENS, new OrderField[] { OauthTokens.OAUTH_TOKENS.CKEY }, false); public static final Index BAN_COMPUTERID = Internal.createIndex(DSL.name("computerid"), Ban.BAN, new OrderField[] { Ban.BAN.COMPUTERID }, false); public static final Index CONNECTION_LOG_COMPUTERID = Internal.createIndex(DSL.name("computerid"), ConnectionLog.CONNECTION_LOG, new OrderField[] { ConnectionLog.CONNECTION_LOG.COMPUTERID }, false); public static final Index PLAYER_COMPUTERID = Internal.createIndex(DSL.name("computerid"), Player.PLAYER, new OrderField[] { Player.PLAYER.COMPUTERID }, false); @@ -61,6 +60,7 @@ public class Indexes { public static final Index PLAYER_IP = Internal.createIndex(DSL.name("ip"), Player.PLAYER, new OrderField[] { Player.PLAYER.IP }, false); public static final Index PLAYER_LASTSEEN = Internal.createIndex(DSL.name("lastseen"), Player.PLAYER, new OrderField[] { Player.PLAYER.LASTSEEN }, false); public static final Index ADMIN_RANKS_NAME = Internal.createIndex(DSL.name("name"), AdminRanks.ADMIN_RANKS, new OrderField[] { AdminRanks.ADMIN_RANKS.NAME }, false); + public static final Index TASK_QUEUE_PROCESSED = Internal.createIndex(DSL.name("processed"), TaskQueue.TASK_QUEUE, new OrderField[] { TaskQueue.TASK_QUEUE.PROCESSED }, false); public static final Index NOTES_PUBLIC = Internal.createIndex(DSL.name("public"), Notes.NOTES, new OrderField[] { Notes.NOTES.PUBLIC }, false); public static final Index BUG_REPORTS_SUBMITTED = Internal.createIndex(DSL.name("submitted"), BugReports.BUG_REPORTS, new OrderField[] { BugReports.BUG_REPORTS.SUBMITTED }, false); } diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Keys.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Keys.java index 88ef9f7..796c0a2 100644 --- a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Keys.java +++ b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Keys.java @@ -24,12 +24,12 @@ import me.aa07.paradise.taskdaemon.database.gamedb.tables.Library; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Memo; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Notes; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.OauthTokens; import me.aa07.paradise.taskdaemon.database.gamedb.tables.PaiSaves; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Player; import me.aa07.paradise.taskdaemon.database.gamedb.tables.PlaytimeHistory; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Privacy; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Round; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.TaskQueue; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Tickets; import me.aa07.paradise.taskdaemon.database.gamedb.tables.VpnWhitelist; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Watch; @@ -54,12 +54,12 @@ import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.LibraryRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.MemoRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.NotesRecord; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.OauthTokensRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PaiSavesRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PlayerRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PlaytimeHistoryRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PrivacyRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.RoundRecord; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.TaskQueueRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.TicketsRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.VpnWhitelistRecord; import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.WatchRecord; @@ -104,7 +104,6 @@ public class Keys { public static final UniqueKey KEY_LIBRARY_PRIMARY = Internal.createUniqueKey(Library.LIBRARY, DSL.name("KEY_library_PRIMARY"), new TableField[] { Library.LIBRARY.ID }, true); public static final UniqueKey KEY_MEMO_PRIMARY = Internal.createUniqueKey(Memo.MEMO, DSL.name("KEY_memo_PRIMARY"), new TableField[] { Memo.MEMO.CKEY }, true); public static final UniqueKey KEY_NOTES_PRIMARY = Internal.createUniqueKey(Notes.NOTES, DSL.name("KEY_notes_PRIMARY"), new TableField[] { Notes.NOTES.ID }, true); - public static final UniqueKey KEY_OAUTH_TOKENS_PRIMARY = Internal.createUniqueKey(OauthTokens.OAUTH_TOKENS, DSL.name("KEY_oauth_tokens_PRIMARY"), new TableField[] { OauthTokens.OAUTH_TOKENS.TOKEN }, true); public static final UniqueKey KEY_PAI_SAVES_CKEY = Internal.createUniqueKey(PaiSaves.PAI_SAVES, DSL.name("KEY_pai_saves_ckey"), new TableField[] { PaiSaves.PAI_SAVES.CKEY }, true); public static final UniqueKey KEY_PAI_SAVES_PRIMARY = Internal.createUniqueKey(PaiSaves.PAI_SAVES, DSL.name("KEY_pai_saves_PRIMARY"), new TableField[] { PaiSaves.PAI_SAVES.ID }, true); public static final UniqueKey KEY_PLAYER_CKEY = Internal.createUniqueKey(Player.PLAYER, DSL.name("KEY_player_ckey"), new TableField[] { Player.PLAYER.CKEY }, true); @@ -112,6 +111,7 @@ public class Keys { public static final UniqueKey KEY_PLAYTIME_HISTORY_PRIMARY = Internal.createUniqueKey(PlaytimeHistory.PLAYTIME_HISTORY, DSL.name("KEY_playtime_history_PRIMARY"), new TableField[] { PlaytimeHistory.PLAYTIME_HISTORY.CKEY, PlaytimeHistory.PLAYTIME_HISTORY.DATE }, true); public static final UniqueKey KEY_PRIVACY_PRIMARY = Internal.createUniqueKey(Privacy.PRIVACY, DSL.name("KEY_privacy_PRIMARY"), new TableField[] { Privacy.PRIVACY.CKEY }, true); public static final UniqueKey KEY_ROUND_PRIMARY = Internal.createUniqueKey(Round.ROUND, DSL.name("KEY_round_PRIMARY"), new TableField[] { Round.ROUND.ID }, true); + public static final UniqueKey KEY_TASK_QUEUE_PRIMARY = Internal.createUniqueKey(TaskQueue.TASK_QUEUE, DSL.name("KEY_task_queue_PRIMARY"), new TableField[] { TaskQueue.TASK_QUEUE.TASK_ID }, true); public static final UniqueKey KEY_TICKETS_PRIMARY = Internal.createUniqueKey(Tickets.TICKETS, DSL.name("KEY_tickets_PRIMARY"), new TableField[] { Tickets.TICKETS.ID }, true); public static final UniqueKey KEY_VPN_WHITELIST_PRIMARY = Internal.createUniqueKey(VpnWhitelist.VPN_WHITELIST, DSL.name("KEY_vpn_whitelist_PRIMARY"), new TableField[] { VpnWhitelist.VPN_WHITELIST.CKEY }, true); public static final UniqueKey KEY_WATCH_PRIMARY = Internal.createUniqueKey(Watch.WATCH, DSL.name("KEY_watch_PRIMARY"), new TableField[] { Watch.WATCH.CKEY }, true); diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/ParadiseGamedb.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/ParadiseGamedb.java index 59bd0e3..d5d8547 100644 --- a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/ParadiseGamedb.java +++ b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/ParadiseGamedb.java @@ -27,12 +27,12 @@ import me.aa07.paradise.taskdaemon.database.gamedb.tables.Library; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Memo; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Notes; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.OauthTokens; import me.aa07.paradise.taskdaemon.database.gamedb.tables.PaiSaves; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Player; import me.aa07.paradise.taskdaemon.database.gamedb.tables.PlaytimeHistory; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Privacy; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Round; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.TaskQueue; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Tickets; import me.aa07.paradise.taskdaemon.database.gamedb.tables.VpnWhitelist; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Watch; @@ -161,11 +161,6 @@ public class ParadiseGamedb extends SchemaImpl { */ public final Notes NOTES = Notes.NOTES; - /** - * The table paradise_gamedb.oauth_tokens. - */ - public final OauthTokens OAUTH_TOKENS = OauthTokens.OAUTH_TOKENS; - /** * The table paradise_gamedb.pai_saves. */ @@ -191,6 +186,11 @@ public class ParadiseGamedb extends SchemaImpl { */ public final Round ROUND = Round.ROUND; + /** + * The table paradise_gamedb.task_queue. + */ + public final TaskQueue TASK_QUEUE = TaskQueue.TASK_QUEUE; + /** * The table paradise_gamedb.tickets. */ @@ -243,12 +243,12 @@ public final List> getTables() { Library.LIBRARY, Memo.MEMO, Notes.NOTES, - OauthTokens.OAUTH_TOKENS, PaiSaves.PAI_SAVES, Player.PLAYER, PlaytimeHistory.PLAYTIME_HISTORY, Privacy.PRIVACY, Round.ROUND, + TaskQueue.TASK_QUEUE, Tickets.TICKETS, VpnWhitelist.VPN_WHITELIST, Watch.WATCH diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Tables.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Tables.java index 119cfe0..e3ca981 100644 --- a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Tables.java +++ b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/Tables.java @@ -24,12 +24,12 @@ import me.aa07.paradise.taskdaemon.database.gamedb.tables.Library; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Memo; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Notes; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.OauthTokens; import me.aa07.paradise.taskdaemon.database.gamedb.tables.PaiSaves; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Player; import me.aa07.paradise.taskdaemon.database.gamedb.tables.PlaytimeHistory; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Privacy; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Round; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.TaskQueue; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Tickets; import me.aa07.paradise.taskdaemon.database.gamedb.tables.VpnWhitelist; import me.aa07.paradise.taskdaemon.database.gamedb.tables.Watch; @@ -147,11 +147,6 @@ public class Tables { */ public static final Notes NOTES = Notes.NOTES; - /** - * The table paradise_gamedb.oauth_tokens. - */ - public static final OauthTokens OAUTH_TOKENS = OauthTokens.OAUTH_TOKENS; - /** * The table paradise_gamedb.pai_saves. */ @@ -177,6 +172,11 @@ public class Tables { */ public static final Round ROUND = Round.ROUND; + /** + * The table paradise_gamedb.task_queue. + */ + public static final TaskQueue TASK_QUEUE = TaskQueue.TASK_QUEUE; + /** * The table paradise_gamedb.tickets. */ diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/OauthTokens.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/OauthTokens.java deleted file mode 100644 index 1856b28..0000000 --- a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/OauthTokens.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * This file is generated by jOOQ. - */ -package me.aa07.paradise.taskdaemon.database.gamedb.tables; - - -import java.util.Arrays; -import java.util.List; - -import me.aa07.paradise.taskdaemon.database.gamedb.Indexes; -import me.aa07.paradise.taskdaemon.database.gamedb.Keys; -import me.aa07.paradise.taskdaemon.database.gamedb.ParadiseGamedb; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.OauthTokensRecord; - -import org.jooq.Field; -import org.jooq.ForeignKey; -import org.jooq.Index; -import org.jooq.Name; -import org.jooq.Record; -import org.jooq.Row2; -import org.jooq.Schema; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.TableOptions; -import org.jooq.UniqueKey; -import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; -import org.jooq.impl.TableImpl; - - -/** - * This class is generated by jOOQ. - */ -@SuppressWarnings({ "all", "unchecked", "rawtypes" }) -public class OauthTokens extends TableImpl { - - private static final long serialVersionUID = 1L; - - /** - * The reference instance of paradise_gamedb.oauth_tokens - */ - public static final OauthTokens OAUTH_TOKENS = new OauthTokens(); - - /** - * The class holding records for this type - */ - @Override - public Class getRecordType() { - return OauthTokensRecord.class; - } - - /** - * The column paradise_gamedb.oauth_tokens.ckey. - */ - public final TableField CKEY = createField(DSL.name("ckey"), SQLDataType.VARCHAR(32).nullable(false), this, ""); - - /** - * The column paradise_gamedb.oauth_tokens.token. - */ - public final TableField TOKEN = createField(DSL.name("token"), SQLDataType.VARCHAR(32).nullable(false), this, ""); - - private OauthTokens(Name alias, Table aliased) { - this(alias, aliased, null); - } - - private OauthTokens(Name alias, Table aliased, Field[] parameters) { - super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.table()); - } - - /** - * Create an aliased paradise_gamedb.oauth_tokens table - * reference - */ - public OauthTokens(String alias) { - this(DSL.name(alias), OAUTH_TOKENS); - } - - /** - * Create an aliased paradise_gamedb.oauth_tokens table - * reference - */ - public OauthTokens(Name alias) { - this(alias, OAUTH_TOKENS); - } - - /** - * Create a paradise_gamedb.oauth_tokens table reference - */ - public OauthTokens() { - this(DSL.name("oauth_tokens"), null); - } - - public OauthTokens(Table child, ForeignKey key) { - super(child, key, OAUTH_TOKENS); - } - - @Override - public Schema getSchema() { - return aliased() ? null : ParadiseGamedb.PARADISE_GAMEDB; - } - - @Override - public List getIndexes() { - return Arrays.asList(Indexes.OAUTH_TOKENS_CKEY); - } - - @Override - public UniqueKey getPrimaryKey() { - return Keys.KEY_OAUTH_TOKENS_PRIMARY; - } - - @Override - public OauthTokens as(String alias) { - return new OauthTokens(DSL.name(alias), this); - } - - @Override - public OauthTokens as(Name alias) { - return new OauthTokens(alias, this); - } - - /** - * Rename this table - */ - @Override - public OauthTokens rename(String name) { - return new OauthTokens(DSL.name(name), null); - } - - /** - * Rename this table - */ - @Override - public OauthTokens rename(Name name) { - return new OauthTokens(name, null); - } - - // ------------------------------------------------------------------------- - // Row2 type methods - // ------------------------------------------------------------------------- - - @Override - public Row2 fieldsRow() { - return (Row2) super.fieldsRow(); - } -} diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/TaskQueue.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/TaskQueue.java new file mode 100644 index 0000000..38aad85 --- /dev/null +++ b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/TaskQueue.java @@ -0,0 +1,166 @@ +/* + * This file is generated by jOOQ. + */ +package me.aa07.paradise.taskdaemon.database.gamedb.tables; + + +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import me.aa07.paradise.taskdaemon.database.gamedb.Indexes; +import me.aa07.paradise.taskdaemon.database.gamedb.Keys; +import me.aa07.paradise.taskdaemon.database.gamedb.ParadiseGamedb; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.TaskQueueRecord; + +import org.jooq.Field; +import org.jooq.ForeignKey; +import org.jooq.Index; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.Row6; +import org.jooq.Schema; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.TableOptions; +import org.jooq.UniqueKey; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.TableImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes" }) +public class TaskQueue extends TableImpl { + + private static final long serialVersionUID = 1L; + + /** + * The reference instance of paradise_gamedb.task_queue + */ + public static final TaskQueue TASK_QUEUE = new TaskQueue(); + + /** + * The class holding records for this type + */ + @Override + public Class getRecordType() { + return TaskQueueRecord.class; + } + + /** + * The column paradise_gamedb.task_queue.task_id. + */ + public final TableField TASK_ID = createField(DSL.name("task_id"), SQLDataType.UUID.nullable(false), this, ""); + + /** + * The column paradise_gamedb.task_queue.task_type. + */ + public final TableField TASK_TYPE = createField(DSL.name("task_type"), SQLDataType.VARCHAR(128).nullable(false), this, ""); + + /** + * The column paradise_gamedb.task_queue.task_arguments. + */ + public final TableField TASK_ARGUMENTS = createField(DSL.name("task_arguments"), SQLDataType.VARCHAR(128).nullable(false), this, ""); + + /** + * The column paradise_gamedb.task_queue.date_inserted. + */ + public final TableField DATE_INSERTED = createField(DSL.name("date_inserted"), SQLDataType.LOCALDATETIME(0).nullable(false), this, ""); + + /** + * The column paradise_gamedb.task_queue.date_processed. + */ + public final TableField DATE_PROCESSED = createField(DSL.name("date_processed"), SQLDataType.LOCALDATETIME(0).defaultValue(DSL.inline("NULL", SQLDataType.LOCALDATETIME)), this, ""); + + /** + * The column paradise_gamedb.task_queue.processed. + */ + public final TableField PROCESSED = createField(DSL.name("processed"), SQLDataType.TINYINT.nullable(false).defaultValue(DSL.inline("0", SQLDataType.TINYINT)), this, ""); + + private TaskQueue(Name alias, Table aliased) { + this(alias, aliased, null); + } + + private TaskQueue(Name alias, Table aliased, Field[] parameters) { + super(alias, null, aliased, parameters, DSL.comment(""), TableOptions.table()); + } + + /** + * Create an aliased paradise_gamedb.task_queue table reference + */ + public TaskQueue(String alias) { + this(DSL.name(alias), TASK_QUEUE); + } + + /** + * Create an aliased paradise_gamedb.task_queue table reference + */ + public TaskQueue(Name alias) { + this(alias, TASK_QUEUE); + } + + /** + * Create a paradise_gamedb.task_queue table reference + */ + public TaskQueue() { + this(DSL.name("task_queue"), null); + } + + public TaskQueue(Table child, ForeignKey key) { + super(child, key, TASK_QUEUE); + } + + @Override + public Schema getSchema() { + return aliased() ? null : ParadiseGamedb.PARADISE_GAMEDB; + } + + @Override + public List getIndexes() { + return Arrays.asList(Indexes.TASK_QUEUE_PROCESSED); + } + + @Override + public UniqueKey getPrimaryKey() { + return Keys.KEY_TASK_QUEUE_PRIMARY; + } + + @Override + public TaskQueue as(String alias) { + return new TaskQueue(DSL.name(alias), this); + } + + @Override + public TaskQueue as(Name alias) { + return new TaskQueue(alias, this); + } + + /** + * Rename this table + */ + @Override + public TaskQueue rename(String name) { + return new TaskQueue(DSL.name(name), null); + } + + /** + * Rename this table + */ + @Override + public TaskQueue rename(Name name) { + return new TaskQueue(name, null); + } + + // ------------------------------------------------------------------------- + // Row6 type methods + // ------------------------------------------------------------------------- + + @Override + public Row6 fieldsRow() { + return (Row6) super.fieldsRow(); + } +} diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/OauthTokensRecord.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/OauthTokensRecord.java deleted file mode 100644 index 1550d99..0000000 --- a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/OauthTokensRecord.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * This file is generated by jOOQ. - */ -package me.aa07.paradise.taskdaemon.database.gamedb.tables.records; - - -import me.aa07.paradise.taskdaemon.database.gamedb.tables.OauthTokens; - -import org.jooq.Field; -import org.jooq.Record1; -import org.jooq.Record2; -import org.jooq.Row2; -import org.jooq.impl.UpdatableRecordImpl; - - -/** - * This class is generated by jOOQ. - */ -@SuppressWarnings({ "all", "unchecked", "rawtypes" }) -public class OauthTokensRecord extends UpdatableRecordImpl implements Record2 { - - private static final long serialVersionUID = 1L; - - /** - * Setter for paradise_gamedb.oauth_tokens.ckey. - */ - public void setCkey(String value) { - set(0, value); - } - - /** - * Getter for paradise_gamedb.oauth_tokens.ckey. - */ - public String getCkey() { - return (String) get(0); - } - - /** - * Setter for paradise_gamedb.oauth_tokens.token. - */ - public void setToken(String value) { - set(1, value); - } - - /** - * Getter for paradise_gamedb.oauth_tokens.token. - */ - public String getToken() { - return (String) get(1); - } - - // ------------------------------------------------------------------------- - // Primary key information - // ------------------------------------------------------------------------- - - @Override - public Record1 key() { - return (Record1) super.key(); - } - - // ------------------------------------------------------------------------- - // Record2 type implementation - // ------------------------------------------------------------------------- - - @Override - public Row2 fieldsRow() { - return (Row2) super.fieldsRow(); - } - - @Override - public Row2 valuesRow() { - return (Row2) super.valuesRow(); - } - - @Override - public Field field1() { - return OauthTokens.OAUTH_TOKENS.CKEY; - } - - @Override - public Field field2() { - return OauthTokens.OAUTH_TOKENS.TOKEN; - } - - @Override - public String component1() { - return getCkey(); - } - - @Override - public String component2() { - return getToken(); - } - - @Override - public String value1() { - return getCkey(); - } - - @Override - public String value2() { - return getToken(); - } - - @Override - public OauthTokensRecord value1(String value) { - setCkey(value); - return this; - } - - @Override - public OauthTokensRecord value2(String value) { - setToken(value); - return this; - } - - @Override - public OauthTokensRecord values(String value1, String value2) { - value1(value1); - value2(value2); - return this; - } - - // ------------------------------------------------------------------------- - // Constructors - // ------------------------------------------------------------------------- - - /** - * Create a detached OauthTokensRecord - */ - public OauthTokensRecord() { - super(OauthTokens.OAUTH_TOKENS); - } - - /** - * Create a detached, initialised OauthTokensRecord - */ - public OauthTokensRecord(String ckey, String token) { - super(OauthTokens.OAUTH_TOKENS); - - setCkey(ckey); - setToken(token); - } -} diff --git a/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/TaskQueueRecord.java b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/TaskQueueRecord.java new file mode 100644 index 0000000..2e6ad6e --- /dev/null +++ b/TaskDaemon.Database.GameDb/src/me/aa07/paradise/taskdaemon/database/gamedb/tables/records/TaskQueueRecord.java @@ -0,0 +1,295 @@ +/* + * This file is generated by jOOQ. + */ +package me.aa07.paradise.taskdaemon.database.gamedb.tables.records; + + +import java.time.LocalDateTime; +import java.util.UUID; + +import me.aa07.paradise.taskdaemon.database.gamedb.tables.TaskQueue; + +import org.jooq.Field; +import org.jooq.Record1; +import org.jooq.Record6; +import org.jooq.Row6; +import org.jooq.impl.UpdatableRecordImpl; + + +/** + * This class is generated by jOOQ. + */ +@SuppressWarnings({ "all", "unchecked", "rawtypes" }) +public class TaskQueueRecord extends UpdatableRecordImpl implements Record6 { + + private static final long serialVersionUID = 1L; + + /** + * Setter for paradise_gamedb.task_queue.task_id. + */ + public void setTaskId(UUID value) { + set(0, value); + } + + /** + * Getter for paradise_gamedb.task_queue.task_id. + */ + public UUID getTaskId() { + return (UUID) get(0); + } + + /** + * Setter for paradise_gamedb.task_queue.task_type. + */ + public void setTaskType(String value) { + set(1, value); + } + + /** + * Getter for paradise_gamedb.task_queue.task_type. + */ + public String getTaskType() { + return (String) get(1); + } + + /** + * Setter for paradise_gamedb.task_queue.task_arguments. + */ + public void setTaskArguments(String value) { + set(2, value); + } + + /** + * Getter for paradise_gamedb.task_queue.task_arguments. + */ + public String getTaskArguments() { + return (String) get(2); + } + + /** + * Setter for paradise_gamedb.task_queue.date_inserted. + */ + public void setDateInserted(LocalDateTime value) { + set(3, value); + } + + /** + * Getter for paradise_gamedb.task_queue.date_inserted. + */ + public LocalDateTime getDateInserted() { + return (LocalDateTime) get(3); + } + + /** + * Setter for paradise_gamedb.task_queue.date_processed. + */ + public void setDateProcessed(LocalDateTime value) { + set(4, value); + } + + /** + * Getter for paradise_gamedb.task_queue.date_processed. + */ + public LocalDateTime getDateProcessed() { + return (LocalDateTime) get(4); + } + + /** + * Setter for paradise_gamedb.task_queue.processed. + */ + public void setProcessed(Byte value) { + set(5, value); + } + + /** + * Getter for paradise_gamedb.task_queue.processed. + */ + public Byte getProcessed() { + return (Byte) get(5); + } + + // ------------------------------------------------------------------------- + // Primary key information + // ------------------------------------------------------------------------- + + @Override + public Record1 key() { + return (Record1) super.key(); + } + + // ------------------------------------------------------------------------- + // Record6 type implementation + // ------------------------------------------------------------------------- + + @Override + public Row6 fieldsRow() { + return (Row6) super.fieldsRow(); + } + + @Override + public Row6 valuesRow() { + return (Row6) super.valuesRow(); + } + + @Override + public Field field1() { + return TaskQueue.TASK_QUEUE.TASK_ID; + } + + @Override + public Field field2() { + return TaskQueue.TASK_QUEUE.TASK_TYPE; + } + + @Override + public Field field3() { + return TaskQueue.TASK_QUEUE.TASK_ARGUMENTS; + } + + @Override + public Field field4() { + return TaskQueue.TASK_QUEUE.DATE_INSERTED; + } + + @Override + public Field field5() { + return TaskQueue.TASK_QUEUE.DATE_PROCESSED; + } + + @Override + public Field field6() { + return TaskQueue.TASK_QUEUE.PROCESSED; + } + + @Override + public UUID component1() { + return getTaskId(); + } + + @Override + public String component2() { + return getTaskType(); + } + + @Override + public String component3() { + return getTaskArguments(); + } + + @Override + public LocalDateTime component4() { + return getDateInserted(); + } + + @Override + public LocalDateTime component5() { + return getDateProcessed(); + } + + @Override + public Byte component6() { + return getProcessed(); + } + + @Override + public UUID value1() { + return getTaskId(); + } + + @Override + public String value2() { + return getTaskType(); + } + + @Override + public String value3() { + return getTaskArguments(); + } + + @Override + public LocalDateTime value4() { + return getDateInserted(); + } + + @Override + public LocalDateTime value5() { + return getDateProcessed(); + } + + @Override + public Byte value6() { + return getProcessed(); + } + + @Override + public TaskQueueRecord value1(UUID value) { + setTaskId(value); + return this; + } + + @Override + public TaskQueueRecord value2(String value) { + setTaskType(value); + return this; + } + + @Override + public TaskQueueRecord value3(String value) { + setTaskArguments(value); + return this; + } + + @Override + public TaskQueueRecord value4(LocalDateTime value) { + setDateInserted(value); + return this; + } + + @Override + public TaskQueueRecord value5(LocalDateTime value) { + setDateProcessed(value); + return this; + } + + @Override + public TaskQueueRecord value6(Byte value) { + setProcessed(value); + return this; + } + + @Override + public TaskQueueRecord values(UUID value1, String value2, String value3, LocalDateTime value4, LocalDateTime value5, Byte value6) { + value1(value1); + value2(value2); + value3(value3); + value4(value4); + value5(value5); + value6(value6); + return this; + } + + // ------------------------------------------------------------------------- + // Constructors + // ------------------------------------------------------------------------- + + /** + * Create a detached TaskQueueRecord + */ + public TaskQueueRecord() { + super(TaskQueue.TASK_QUEUE); + } + + /** + * Create a detached, initialised TaskQueueRecord + */ + public TaskQueueRecord(UUID taskId, String taskType, String taskArguments, LocalDateTime dateInserted, LocalDateTime dateProcessed, Byte processed) { + super(TaskQueue.TASK_QUEUE); + + setTaskId(taskId); + setTaskType(taskType); + setTaskArguments(taskArguments); + setDateInserted(dateInserted); + setDateProcessed(dateProcessed); + setProcessed(processed); + } +} From c9e43a6f7fcbe361bd8ea9761c31c0f755a5bc31 Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:00:32 +0100 Subject: [PATCH 2/7] Much progress made --- .../aa07/paradise/taskdaemon/core/Core.java | 489 +++---- .../taskdaemon/core/config/ConfigHolder.java | 33 +- .../core/config/InvisionConfig.java | 6 + .../core/models/invision/GetUserModel.java | 24 + .../invision/UpdateSecondaryGroupsModel.java | 9 + .../modules/aclcleanup/AclCleanupJob.java | 386 +++--- .../core/modules/devrank/DevRankJob.java | 316 ++--- .../IngameVerifiedSyncJob.java | 461 +++++-- .../core/modules/ip2asn/Ip2AsnJob.java | 422 +++---- .../modules/patreonsync/PatreonSyncJob.java | 1124 ++++++++--------- .../core/modules/prlabel/PrLabelJob.java | 504 ++++---- .../profilercleanup/ProfilerCleanupJob.java | 122 +- .../profileringest/ProfilerWorker.java | 170 +-- .../processors/ProfilerBaseProcessor.java | 66 +- .../processors/ProfilerProcProcessor.java | 124 +- .../processors/ProfilerSendmapsProcessor.java | 118 +- .../{ => services}/database/DatabaseType.java | 10 +- .../core/{ => services}/database/DbCore.java | 216 ++-- .../services/invision/HttpRequestMethod.java | 5 + .../core/services/invision/InvisionUtil.java | 123 ++ .../taskdaemon/core/util/UtilConst.java | 28 +- .../taskdaemon/core/util/UtilStr.java | 59 +- config.toml.example | 132 +- 23 files changed, 2660 insertions(+), 2287 deletions(-) create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java rename TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/{ => services}/database/DatabaseType.java (60%) rename TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/{ => services}/database/DbCore.java (96%) create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java create mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java index ad19693..2419915 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java @@ -1,243 +1,246 @@ -package me.aa07.paradise.taskdaemon.core; - -import com.moandjiezana.toml.Toml; -import java.io.File; -import java.util.ArrayList; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.modules.aclcleanup.AclCleanupJob; -import me.aa07.paradise.taskdaemon.core.modules.bouncerrestart.BouncerRestartJob; -import me.aa07.paradise.taskdaemon.core.modules.devrank.DevRankJob; -import me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync.IngameVerifiedSyncJob; -import me.aa07.paradise.taskdaemon.core.modules.ip2asn.Ip2AsnJob; -import me.aa07.paradise.taskdaemon.core.modules.patreonsync.PatreonSyncJob; -import me.aa07.paradise.taskdaemon.core.modules.prlabel.PrLabelJob; -import me.aa07.paradise.taskdaemon.core.modules.profilercleanup.ProfilerCleanupJob; -import me.aa07.paradise.taskdaemon.core.modules.profileringest.ProfilerWorker; -import me.aa07.paradise.taskdaemon.core.redis.RedisManager; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.quartz.CronScheduleBuilder; -import org.quartz.CronTrigger; -import org.quartz.JobBuilder; -import org.quartz.JobDataMap; -import org.quartz.JobDetail; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.TriggerBuilder; -import org.quartz.impl.StdSchedulerFactory; - -public class Core { - // Threads - private Thread profilerWorkerThread; - private Thread redisthread; - - public void run() { - Logger logger = LogManager.getLogger(Core.class); - logger.info("Starting up"); - logger.info("Loading configuration..."); - - Optional config = Optional.empty(); - - try { - File configfile = new File("config.toml"); - config = Optional.of(new Toml().read(configfile).to(ConfigHolder.class)); - } catch (Exception exception) { - logger.fatal("Failed to load config.toml!"); - exception.printStackTrace(); - return; - } - - if (!config.isPresent()) { - logger.fatal("Failed to load config.toml - but we didnt throw an exception! What the heck?!"); - return; - } - - logger.info("Preparing threads..."); - setupThreads(config.get(), logger); - logger.info("Launching threads..."); - launchAll(); - logger.info("Core startup complete"); - } - - private void setupThreads(ConfigHolder config, Logger logger) { - // Initial setup - DbCore database = new DbCore(config, logger); - - // Profiler stuff - ProfilerWorker pw = new ProfilerWorker(database, logger); - profilerWorkerThread = new Thread(pw::run, "profiler-worker"); - - // Redis stuff - RedisManager rm = new RedisManager(config.redis, logger, pw); - redisthread = new Thread(rm::run, "redis"); - - // Launch Quartz - try { - Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); - setupJobs(scheduler, database, config, logger); - scheduler.start(); - } catch (SchedulerException ex) { - logger.error("Quartz had a hissy fit!", ex); - } - } - - private void setupJobs(Scheduler scheduler, DbCore dbCore, ConfigHolder config, Logger logger) throws SchedulerException { - // See below for CRON format - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html - - // ACL cleanup - JobDataMap jdm_aclcleanup = new JobDataMap(); - jdm_aclcleanup.put("LOGGER", logger); - jdm_aclcleanup.put("DBCORE", dbCore); - jdm_aclcleanup.put("PFSENSE_CFG", config.pfsense); - JobDetail jd_aclcleanup = JobBuilder.newJob(AclCleanupJob.class) - .withIdentity("aclcleanup", "aclcleanup") - .usingJobData(jdm_aclcleanup) - .build(); - CronTrigger ct_aclcleanup = TriggerBuilder.newTrigger() - .withIdentity("aclcleanup", "aclcleanup") - .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) - .build(); - - // Bouncer restart - JobDataMap jdm_bouncerrestart = new JobDataMap(); - jdm_bouncerrestart.put("LOGGER", logger); - jdm_bouncerrestart.put("TGS_CFG", config.tgs); - JobDetail jd_bouncerrestart = JobBuilder.newJob(BouncerRestartJob.class) - .withIdentity("bouncerrestart", "bouncerrestart") - .usingJobData(jdm_bouncerrestart) - .build(); - CronTrigger ct_bouncerrestart = TriggerBuilder.newTrigger() - .withIdentity("bouncerrestart", "bouncerrestart") - .withSchedule(CronScheduleBuilder.cronSchedule("0 0 0 * * ?")) // Every day - midnight - .build(); - - // DevRank update - JobDataMap jdm_devrank = new JobDataMap(); - jdm_devrank.put("LOGGER", logger); - jdm_devrank.put("DBCORE", dbCore); - - JobDetail jd_devrank = JobBuilder.newJob(DevRankJob.class) - .withIdentity("devrank", "devrank") - .usingJobData(jdm_devrank) - .build(); - - CronTrigger ct_devrank = TriggerBuilder.newTrigger() - .withIdentity("devrank", "devrank") - .withSchedule(CronScheduleBuilder.cronSchedule("0 0 * * * ?")) // Every hour on the hour - .build(); - - - // IP2ASN - JobDataMap jdm_ip2asn = new JobDataMap(); - jdm_ip2asn.put("LOGGER", logger); - jdm_ip2asn.put("DBCORE", dbCore); - jdm_ip2asn.put("IP2ASNCFG", config.ip2asn); - JobDetail jd_ip2asn = JobBuilder.newJob(Ip2AsnJob.class) - .withIdentity("ip2asn", "ip2asn") - .usingJobData(jdm_ip2asn) - .build(); - CronTrigger ct_ip2asn = TriggerBuilder.newTrigger() - .withIdentity("ip2asn", "ip2asn") - .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) // Every 10 minutes - .build(); - - // Profiler cleanup - JobDataMap jdm_profilercleanup = new JobDataMap(); - jdm_profilercleanup.put("LOGGER", logger); - jdm_profilercleanup.put("DBCORE", dbCore); - JobDetail jd_profilercleanup = JobBuilder.newJob(ProfilerCleanupJob.class) - .withIdentity("profilercleanup", "profilercleanup") - .usingJobData(jdm_profilercleanup) - .build(); - CronTrigger ct_profilercleanup = TriggerBuilder.newTrigger() - .withIdentity("profilercleanup", "profilercleanup") - .withSchedule(CronScheduleBuilder.cronSchedule("0 0 8 * * ?")) // Every day - 8AM - .build(); - - // PR Label bot - JobDataMap jdm_pullrequests = new JobDataMap(); - jdm_pullrequests.put("LOGGER", logger); - jdm_pullrequests.put("DBCORE", dbCore); - jdm_pullrequests.put("TGSCFG", config.tgs); - jdm_pullrequests.put("GITHUBCFG", config.github); - JobDetail jd_pullrequests = JobBuilder.newJob(PrLabelJob.class) - .withIdentity("pullrequests", "pullrequests") - .usingJobData(jdm_pullrequests) - .build(); - CronTrigger ct_pullrequests = TriggerBuilder.newTrigger() - .withIdentity("pullrequests", "pullrequests") - .withSchedule(CronScheduleBuilder.cronSchedule("0 10 * * * ?")) // Every hour, offset 10 minutes - .build(); - - // Patreon Sync - JobDataMap jdm_patreonsync = new JobDataMap(); - jdm_patreonsync.put("LOGGER", logger); - jdm_patreonsync.put("DBCORE", dbCore); - jdm_patreonsync.put("PTRCFG", config.patreon); - JobDetail jd_patreonsync = JobBuilder.newJob(PatreonSyncJob.class) - .withIdentity("patreonsync", "patreonsync") - .usingJobData(jdm_patreonsync) - .build(); - CronTrigger ct_patreonsync = TriggerBuilder.newTrigger() - .withIdentity("patreonsync", "patreonsync") - .withSchedule(CronScheduleBuilder.cronSchedule("0 15 * * * ?")) // Every hour, offset 15 minutes - .build(); - - // Ingame verified sync - JobDataMap jdm_igvsync = new JobDataMap(); - jdm_igvsync.put("LOGGER", logger); - jdm_igvsync.put("DBCORE", dbCore); - JobDetail jd_igvsync = JobBuilder.newJob(IngameVerifiedSyncJob.class) - .withIdentity("igvsync", "igvsync") - .usingJobData(jdm_igvsync) - .build(); - CronTrigger ct_igvsync = TriggerBuilder.newTrigger() - .withIdentity("igvsync", "igvsync") - //.withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes - .withSchedule(CronScheduleBuilder.cronSchedule("0 * * * * ?")) // Every hour, offset 23 minutes - .build(); - - // Lists that exist just to make sure these things show as recognised - ArrayList bin1 = new ArrayList(); - bin1.add(jd_aclcleanup); - bin1.add(jd_bouncerrestart); - bin1.add(jd_devrank); - bin1.add(jd_ip2asn); - bin1.add(jd_profilercleanup); - bin1.add(jd_pullrequests); - bin1.add(jd_patreonsync); - bin1.add(jd_igvsync); - ArrayList bin2 = new ArrayList(); - bin2.add(ct_aclcleanup); - bin2.add(ct_bouncerrestart); - bin2.add(ct_devrank); - bin2.add(ct_ip2asn); - bin2.add(ct_profilercleanup); - bin2.add(ct_pullrequests); - bin2.add(ct_patreonsync); - bin2.add(ct_igvsync); - - // Is this necessary? Prolly not! - bin1.clear(); - bin2.clear(); - - - // Schedule all - //scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); - //scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); - //scheduler.scheduleJob(jd_devrank, ct_devrank); - //scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); - //scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); - //scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); - //scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); - scheduler.scheduleJob(jd_igvsync, ct_igvsync); - } - - private void launchAll() { - profilerWorkerThread.start(); - redisthread.start(); - } -} +package me.aa07.paradise.taskdaemon.core; + +import com.moandjiezana.toml.Toml; +import java.io.File; +import java.util.ArrayList; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; +import me.aa07.paradise.taskdaemon.core.modules.aclcleanup.AclCleanupJob; +import me.aa07.paradise.taskdaemon.core.modules.bouncerrestart.BouncerRestartJob; +import me.aa07.paradise.taskdaemon.core.modules.devrank.DevRankJob; +import me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync.IngameVerifiedSyncJob; +import me.aa07.paradise.taskdaemon.core.modules.ip2asn.Ip2AsnJob; +import me.aa07.paradise.taskdaemon.core.modules.patreonsync.PatreonSyncJob; +import me.aa07.paradise.taskdaemon.core.modules.prlabel.PrLabelJob; +import me.aa07.paradise.taskdaemon.core.modules.profilercleanup.ProfilerCleanupJob; +import me.aa07.paradise.taskdaemon.core.modules.profileringest.ProfilerWorker; +import me.aa07.paradise.taskdaemon.core.redis.RedisManager; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.services.invision.InvisionUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.quartz.CronScheduleBuilder; +import org.quartz.CronTrigger; +import org.quartz.JobBuilder; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.TriggerBuilder; +import org.quartz.impl.StdSchedulerFactory; + +public class Core { + // Threads + private Thread profilerWorkerThread; + private Thread redisthread; + + public void run() { + Logger logger = LogManager.getLogger(Core.class); + logger.info("Starting up"); + logger.info("Loading configuration..."); + + Optional config = Optional.empty(); + + try { + File configfile = new File("config.toml"); + config = Optional.of(new Toml().read(configfile).to(ConfigHolder.class)); + } catch (Exception exception) { + logger.fatal("Failed to load config.toml!"); + exception.printStackTrace(); + return; + } + + if (!config.isPresent()) { + logger.fatal("Failed to load config.toml - but we didnt throw an exception! What the heck?!"); + return; + } + + logger.info("Preparing threads..."); + setupThreads(config.get(), logger); + logger.info("Launching threads..."); + launchAll(); + logger.info("Core startup complete"); + } + + private void setupThreads(ConfigHolder config, Logger logger) { + // Initial setup + DbCore database = new DbCore(config, logger); + InvisionUtil iu = new InvisionUtil(config.invisionConfig); + + // Profiler stuff + ProfilerWorker pw = new ProfilerWorker(database, logger); + profilerWorkerThread = new Thread(pw::run, "profiler-worker"); + + // Redis stuff + RedisManager rm = new RedisManager(config.redis, logger, pw); + redisthread = new Thread(rm::run, "redis"); + + // Launch Quartz + try { + Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); + setupJobs(scheduler, database, iu, config, logger); + scheduler.start(); + } catch (SchedulerException ex) { + logger.error("Quartz had a hissy fit!", ex); + } + } + + private void setupJobs(Scheduler scheduler, DbCore dbCore, InvisionUtil iu, ConfigHolder config, Logger logger) throws SchedulerException { + // See below for CRON format + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html + + // ACL cleanup + JobDataMap jdm_aclcleanup = new JobDataMap(); + jdm_aclcleanup.put("LOGGER", logger); + jdm_aclcleanup.put("DBCORE", dbCore); + jdm_aclcleanup.put("PFSENSE_CFG", config.pfsense); + JobDetail jd_aclcleanup = JobBuilder.newJob(AclCleanupJob.class) + .withIdentity("aclcleanup", "aclcleanup") + .usingJobData(jdm_aclcleanup) + .build(); + CronTrigger ct_aclcleanup = TriggerBuilder.newTrigger() + .withIdentity("aclcleanup", "aclcleanup") + .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) + .build(); + + // Bouncer restart + JobDataMap jdm_bouncerrestart = new JobDataMap(); + jdm_bouncerrestart.put("LOGGER", logger); + jdm_bouncerrestart.put("TGS_CFG", config.tgs); + JobDetail jd_bouncerrestart = JobBuilder.newJob(BouncerRestartJob.class) + .withIdentity("bouncerrestart", "bouncerrestart") + .usingJobData(jdm_bouncerrestart) + .build(); + CronTrigger ct_bouncerrestart = TriggerBuilder.newTrigger() + .withIdentity("bouncerrestart", "bouncerrestart") + .withSchedule(CronScheduleBuilder.cronSchedule("0 0 0 * * ?")) // Every day - midnight + .build(); + + // DevRank update + JobDataMap jdm_devrank = new JobDataMap(); + jdm_devrank.put("LOGGER", logger); + jdm_devrank.put("DBCORE", dbCore); + + JobDetail jd_devrank = JobBuilder.newJob(DevRankJob.class) + .withIdentity("devrank", "devrank") + .usingJobData(jdm_devrank) + .build(); + + CronTrigger ct_devrank = TriggerBuilder.newTrigger() + .withIdentity("devrank", "devrank") + .withSchedule(CronScheduleBuilder.cronSchedule("0 0 * * * ?")) // Every hour on the hour + .build(); + + + // IP2ASN + JobDataMap jdm_ip2asn = new JobDataMap(); + jdm_ip2asn.put("LOGGER", logger); + jdm_ip2asn.put("DBCORE", dbCore); + jdm_ip2asn.put("IP2ASNCFG", config.ip2asn); + JobDetail jd_ip2asn = JobBuilder.newJob(Ip2AsnJob.class) + .withIdentity("ip2asn", "ip2asn") + .usingJobData(jdm_ip2asn) + .build(); + CronTrigger ct_ip2asn = TriggerBuilder.newTrigger() + .withIdentity("ip2asn", "ip2asn") + .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) // Every 10 minutes + .build(); + + // Profiler cleanup + JobDataMap jdm_profilercleanup = new JobDataMap(); + jdm_profilercleanup.put("LOGGER", logger); + jdm_profilercleanup.put("DBCORE", dbCore); + JobDetail jd_profilercleanup = JobBuilder.newJob(ProfilerCleanupJob.class) + .withIdentity("profilercleanup", "profilercleanup") + .usingJobData(jdm_profilercleanup) + .build(); + CronTrigger ct_profilercleanup = TriggerBuilder.newTrigger() + .withIdentity("profilercleanup", "profilercleanup") + .withSchedule(CronScheduleBuilder.cronSchedule("0 0 8 * * ?")) // Every day - 8AM + .build(); + + // PR Label bot + JobDataMap jdm_pullrequests = new JobDataMap(); + jdm_pullrequests.put("LOGGER", logger); + jdm_pullrequests.put("DBCORE", dbCore); + jdm_pullrequests.put("TGSCFG", config.tgs); + jdm_pullrequests.put("GITHUBCFG", config.github); + JobDetail jd_pullrequests = JobBuilder.newJob(PrLabelJob.class) + .withIdentity("pullrequests", "pullrequests") + .usingJobData(jdm_pullrequests) + .build(); + CronTrigger ct_pullrequests = TriggerBuilder.newTrigger() + .withIdentity("pullrequests", "pullrequests") + .withSchedule(CronScheduleBuilder.cronSchedule("0 10 * * * ?")) // Every hour, offset 10 minutes + .build(); + + // Patreon Sync + JobDataMap jdm_patreonsync = new JobDataMap(); + jdm_patreonsync.put("LOGGER", logger); + jdm_patreonsync.put("DBCORE", dbCore); + jdm_patreonsync.put("PTRCFG", config.patreon); + JobDetail jd_patreonsync = JobBuilder.newJob(PatreonSyncJob.class) + .withIdentity("patreonsync", "patreonsync") + .usingJobData(jdm_patreonsync) + .build(); + CronTrigger ct_patreonsync = TriggerBuilder.newTrigger() + .withIdentity("patreonsync", "patreonsync") + .withSchedule(CronScheduleBuilder.cronSchedule("0 15 * * * ?")) // Every hour, offset 15 minutes + .build(); + + // Ingame verified sync + JobDataMap jdm_igvsync = new JobDataMap(); + jdm_igvsync.put("LOGGER", logger); + jdm_igvsync.put("DBCORE", dbCore); + jdm_igvsync.put("IU", iu); + JobDetail jd_igvsync = JobBuilder.newJob(IngameVerifiedSyncJob.class) + .withIdentity("igvsync", "igvsync") + .usingJobData(jdm_igvsync) + .build(); + CronTrigger ct_igvsync = TriggerBuilder.newTrigger() + .withIdentity("igvsync", "igvsync") + //.withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes + .withSchedule(CronScheduleBuilder.cronSchedule("0 * * * * ?")) // Every hour, offset 23 minutes + .build(); + + // Lists that exist just to make sure these things show as recognised + ArrayList bin1 = new ArrayList(); + bin1.add(jd_aclcleanup); + bin1.add(jd_bouncerrestart); + bin1.add(jd_devrank); + bin1.add(jd_ip2asn); + bin1.add(jd_profilercleanup); + bin1.add(jd_pullrequests); + bin1.add(jd_patreonsync); + bin1.add(jd_igvsync); + ArrayList bin2 = new ArrayList(); + bin2.add(ct_aclcleanup); + bin2.add(ct_bouncerrestart); + bin2.add(ct_devrank); + bin2.add(ct_ip2asn); + bin2.add(ct_profilercleanup); + bin2.add(ct_pullrequests); + bin2.add(ct_patreonsync); + bin2.add(ct_igvsync); + + // Is this necessary? Prolly not! + bin1.clear(); + bin2.clear(); + + + // Schedule all + //scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); + //scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); + //scheduler.scheduleJob(jd_devrank, ct_devrank); + //scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); + //scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); + //scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); + //scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); + scheduler.scheduleJob(jd_igvsync, ct_igvsync); + } + + private void launchAll() { + profilerWorkerThread.start(); + redisthread.start(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java index 9313dfe..69a44ff 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java @@ -1,16 +1,17 @@ -package me.aa07.paradise.taskdaemon.core.config; - -public class ConfigHolder { - public DatabaseConfig authentikDatabase; - public DatabaseConfig automationDatabase; - public DatabaseConfig forumsDatabase; - public DatabaseConfig gameDatabase; - public GithubConfig github; - public Ip2AsnSerivceConfig ip2asn; - public PatreonConfig patreon; - public PfsenseConfig pfsense; - public DatabaseConfig profilerDatabase; - public DatabaseConfig pullRequestsDatabase; - public RedisConfig redis; - public TgsConfig tgs; -} +package me.aa07.paradise.taskdaemon.core.config; + +public class ConfigHolder { + public DatabaseConfig authentikDatabase; + public DatabaseConfig automationDatabase; + public DatabaseConfig forumsDatabase; + public DatabaseConfig gameDatabase; + public GithubConfig github; + public InvisionConfig invisionConfig; + public Ip2AsnSerivceConfig ip2asn; + public PatreonConfig patreon; + public PfsenseConfig pfsense; + public DatabaseConfig profilerDatabase; + public DatabaseConfig pullRequestsDatabase; + public RedisConfig redis; + public TgsConfig tgs; +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java new file mode 100644 index 0000000..7ec464b --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java @@ -0,0 +1,6 @@ +package me.aa07.paradise.taskdaemon.core.config; + +public class InvisionConfig { + public String apiBaseUrl; + public String apiKey; +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java new file mode 100644 index 0000000..7fa8d6c --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java @@ -0,0 +1,24 @@ +package me.aa07.paradise.taskdaemon.core.models.invision; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +public class GetUserModel { + @SerializedName("id") // just incase GSON behaviour ever changes + public int id; + @SerializedName("name") + public String name; + @SerializedName("primaryGroup") + public GroupHolder primaryGroup; + @SerializedName("secondaryGroups") + public List secondaryGroups; + + public class GroupHolder { + @SerializedName("id") + public int id; + @SerializedName("name") + public String name; + @SerializedName("formattedName") + public String formattedName; + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java new file mode 100644 index 0000000..1ad21f0 --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java @@ -0,0 +1,9 @@ +package me.aa07.paradise.taskdaemon.core.models.invision; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +public class UpdateSecondaryGroupsModel { + @SerializedName("secondaryGroups") + public List secondaryGroups; +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java index e5b5c04..c385b40 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java @@ -1,193 +1,193 @@ -package me.aa07.paradise.taskdaemon.core.modules.aclcleanup; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.Socket; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.PfsenseConfig; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.database.gamedb.Tables; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -/** - * Scheduled job that cleans up ACL by removing inactive player IPs - */ -@DisallowConcurrentExecution // NO -public class AclCleanupJob implements Job { - - /** - * Main job execution - fetches current ACL, checks database for inactive IPs, and removes them - */ - @Override - public void execute(JobExecutionContext context) throws JobExecutionException { - JobDataMap datamap = context.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[AclCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[AclCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Get pfSense configuration - Object raw_pfs_cfg = datamap.get("PFSENSE_CFG"); - Optional pfs_cfg_holder = Optional.empty(); - - if (raw_pfs_cfg instanceof PfsenseConfig pfsCfg) { - pfs_cfg_holder = Optional.of(pfsCfg); - } - - if (pfs_cfg_holder.isEmpty()) { - logger.error("[AclCleanup] PFSENSE_CFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - PfsenseConfig pfs_cfg = pfs_cfg_holder.get(); - - try { - List current_acl_ips = fetchAcl(logger, pfs_cfg); - logger.info("IPs in ACL ({}): {}", current_acl_ips.size(), String.join(" ", current_acl_ips)); - - List ips_to_remove = checkIpsInDatabase(current_acl_ips, logger, dbcore); - - if (ips_to_remove.isEmpty()) { - logger.info("No IPs to remove from ACL"); - } else { - logger.info("IPs to remove from ACL ({}): {}", ips_to_remove.size(), String.join(" ", ips_to_remove)); - - for (String ip : ips_to_remove) { - removeIpFromAcl(ip, logger, pfs_cfg); - } - - logger.info("Removed {} IPs from ACL", ips_to_remove.size()); - } - - List updated_acl_ips = fetchAcl(logger, pfs_cfg); - logger.info("IPs now in ACL ({}): {}", updated_acl_ips.size(), String.join(" ", updated_acl_ips)); - - } catch (IOException e) { - logger.error("Error in AclCleanupJob!"); - logger.error(e); - } - } - - /** - * Checks which IPs should be removed from ACL based on database activity - * Removes IPs that don't exist in DB or haven't been seen in last 10 minutes - */ - public List checkIpsInDatabase(List ips, Logger logger, DbCore dbcore) { - List to_remove = new ArrayList<>(); - - DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); - LocalDateTime ten_mins_ago = dbcore.now().minusMinutes(10); - - for (String ip : ips) { - if (!ctx.fetchExists(ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)))) { - logger.info("IP {} no longer in database.", ip); - to_remove.add(ip); - continue; - } - - boolean seen_in_last_10m = ctx.fetchExists( - ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)) - .and(Tables.PLAYER.LASTSEEN.gt(ten_mins_ago)) - ); - - if (!seen_in_last_10m) { - logger.info("IP {} not active within last 10 minutes.", ip); - to_remove.add(ip); - } - } - - return to_remove; - } - - /** - * Retrieves current IP list from ACL via socket connection - */ - public List fetchAcl(Logger logger, PfsenseConfig cfg) throws IOException { - List ip_list = new ArrayList<>(); - - Socket socket = new Socket(cfg.host, cfg.port); - socket.setSoTimeout(10000); - - OutputStream out = socket.getOutputStream(); - InputStream in = socket.getInputStream(); - - String command = "show acl #1\n"; - out.write(command.getBytes("ascii")); - - StringBuilder retval = new StringBuilder(); - byte[] buffer = new byte[16]; - int bytes_read; - - while ((bytes_read = in.read(buffer)) != -1) { - retval.append(new String(buffer, 0, bytes_read)); - } - - socket.close(); - - String[] raw_ips = retval.toString().split("\n"); - for (String ip : raw_ips) { - if (ip.trim().isEmpty()) { - continue; - } - - String[] parts = ip.split(" "); - if (parts.length >= 2) { - ip_list.add(parts[1]); - } - } - - return ip_list; - } - - /** - * Removes a specific IP from ACL #1 via socket command - */ - public void removeIpFromAcl(String ip, Logger logger, PfsenseConfig cfg) throws IOException { - logger.info("Removing {} from ACL", ip); - - Socket socket = new Socket(cfg.host, cfg.port); - OutputStream out = socket.getOutputStream(); - - String command = "del acl #1 " + ip + "\n"; - out.write(command.getBytes("ascii")); - - socket.close(); - } -} +package me.aa07.paradise.taskdaemon.core.modules.aclcleanup; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.PfsenseConfig; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.gamedb.Tables; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +/** + * Scheduled job that cleans up ACL by removing inactive player IPs + */ +@DisallowConcurrentExecution // NO +public class AclCleanupJob implements Job { + + /** + * Main job execution - fetches current ACL, checks database for inactive IPs, and removes them + */ + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + JobDataMap datamap = context.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[AclCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[AclCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Get pfSense configuration + Object raw_pfs_cfg = datamap.get("PFSENSE_CFG"); + Optional pfs_cfg_holder = Optional.empty(); + + if (raw_pfs_cfg instanceof PfsenseConfig pfsCfg) { + pfs_cfg_holder = Optional.of(pfsCfg); + } + + if (pfs_cfg_holder.isEmpty()) { + logger.error("[AclCleanup] PFSENSE_CFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + PfsenseConfig pfs_cfg = pfs_cfg_holder.get(); + + try { + List current_acl_ips = fetchAcl(logger, pfs_cfg); + logger.info("IPs in ACL ({}): {}", current_acl_ips.size(), String.join(" ", current_acl_ips)); + + List ips_to_remove = checkIpsInDatabase(current_acl_ips, logger, dbcore); + + if (ips_to_remove.isEmpty()) { + logger.info("No IPs to remove from ACL"); + } else { + logger.info("IPs to remove from ACL ({}): {}", ips_to_remove.size(), String.join(" ", ips_to_remove)); + + for (String ip : ips_to_remove) { + removeIpFromAcl(ip, logger, pfs_cfg); + } + + logger.info("Removed {} IPs from ACL", ips_to_remove.size()); + } + + List updated_acl_ips = fetchAcl(logger, pfs_cfg); + logger.info("IPs now in ACL ({}): {}", updated_acl_ips.size(), String.join(" ", updated_acl_ips)); + + } catch (IOException e) { + logger.error("Error in AclCleanupJob!"); + logger.error(e); + } + } + + /** + * Checks which IPs should be removed from ACL based on database activity + * Removes IPs that don't exist in DB or haven't been seen in last 10 minutes + */ + public List checkIpsInDatabase(List ips, Logger logger, DbCore dbcore) { + List to_remove = new ArrayList<>(); + + DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); + LocalDateTime ten_mins_ago = dbcore.now().minusMinutes(10); + + for (String ip : ips) { + if (!ctx.fetchExists(ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)))) { + logger.info("IP {} no longer in database.", ip); + to_remove.add(ip); + continue; + } + + boolean seen_in_last_10m = ctx.fetchExists( + ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)) + .and(Tables.PLAYER.LASTSEEN.gt(ten_mins_ago)) + ); + + if (!seen_in_last_10m) { + logger.info("IP {} not active within last 10 minutes.", ip); + to_remove.add(ip); + } + } + + return to_remove; + } + + /** + * Retrieves current IP list from ACL via socket connection + */ + public List fetchAcl(Logger logger, PfsenseConfig cfg) throws IOException { + List ip_list = new ArrayList<>(); + + Socket socket = new Socket(cfg.host, cfg.port); + socket.setSoTimeout(10000); + + OutputStream out = socket.getOutputStream(); + InputStream in = socket.getInputStream(); + + String command = "show acl #1\n"; + out.write(command.getBytes("ascii")); + + StringBuilder retval = new StringBuilder(); + byte[] buffer = new byte[16]; + int bytes_read; + + while ((bytes_read = in.read(buffer)) != -1) { + retval.append(new String(buffer, 0, bytes_read)); + } + + socket.close(); + + String[] raw_ips = retval.toString().split("\n"); + for (String ip : raw_ips) { + if (ip.trim().isEmpty()) { + continue; + } + + String[] parts = ip.split(" "); + if (parts.length >= 2) { + ip_list.add(parts[1]); + } + } + + return ip_list; + } + + /** + * Removes a specific IP from ACL #1 via socket command + */ + public void removeIpFromAcl(String ip, Logger logger, PfsenseConfig cfg) throws IOException { + logger.info("Removing {} from ACL", ip); + + Socket socket = new Socket(cfg.host, cfg.port); + OutputStream out = socket.getOutputStream(); + + String command = "del acl #1 " + ip + "\n"; + out.write(command.getBytes("ascii")); + + socket.close(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java index 776d3fc..520ea91 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java @@ -1,158 +1,158 @@ -package me.aa07.paradise.taskdaemon.core.modules.devrank; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.database.forums.Tables; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.Admin; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.AdminRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.Result; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -/** - * Scheduled job that syncs dev team permissions between forums and game - * databases. - */ -@DisallowConcurrentExecution // NO -public class DevRankJob implements Job { - private static final int DEV_TEAM_GROUP = 39; - private static final int DEV_RANK = 100; - private static final int REMOVED_RANK = 105; - - @Override - public void execute(JobExecutionContext context) throws JobExecutionException { - JobDataMap data_map = context.getMergedJobDataMap(); - - // Get logger and db - Logger logger = (Logger) data_map.get("LOGGER"); - DbCore dbcore = (DbCore) data_map.get("DBCORE"); - - if (logger == null || dbcore == null) { - System.err.println("[DevRank] LOGGER or DBCORE was not set in JobDataMap."); - return; - } - - logger.info("[DevRank] Starting dev rank sync"); - - try { - DSLContext forums_db = dbcore.jooq(DatabaseType.Forums); - DSLContext game_db = dbcore.jooq(DatabaseType.GameDb); - - // Collect all dev team ckeys from forums database - List dev_team_ckeys = new ArrayList<>(); - - Result forum_records = forums_db.select( - Tables.CORE_MEMBERS.MEMBER_ID, - Tables.CORE_MEMBERS.NAME, - Tables.CORE_MEMBERS.MEMBER_GROUP_ID, - Tables.CORE_MEMBERS.MGROUP_OTHERS, - Tables.CORE_PFIELDS_CONTENT.FIELD_10).from(Tables.CORE_MEMBERS) - .leftJoin(Tables.CORE_PFIELDS_CONTENT) - .on(Tables.CORE_MEMBERS.MEMBER_ID.eq(Tables.CORE_PFIELDS_CONTENT.MEMBER_ID)) - .fetch(); - - for (Record rec : forum_records) { - int primary_group = rec.get(Tables.CORE_MEMBERS.MEMBER_GROUP_ID); - String other_groups = rec.get(Tables.CORE_MEMBERS.MGROUP_OTHERS); - String ckey = rec.get(Tables.CORE_PFIELDS_CONTENT.FIELD_10); - - Set all_groups = new HashSet(); - all_groups.add(primary_group); - - if (other_groups != null && !other_groups.isBlank()) { - Arrays.stream(other_groups.split(",")) - .filter(g -> !g.isBlank()) - .map(Integer::parseInt) - .forEach(all_groups::add); - } - - if (all_groups.contains(DEV_TEAM_GROUP)) { - if (ckey == null || ckey.isBlank()) { - logger.warn("[DevRank] Forums user {} (ID {}) has no linked ckey", - rec.get(Tables.CORE_MEMBERS.NAME), rec.get(Tables.CORE_MEMBERS.MEMBER_ID)); - continue; - } - - String cleaned = ckey.toLowerCase().replaceAll("[\\s_\\-]", ""); - dev_team_ckeys.add(cleaned); - } - } - - // Load all ingame admins - Map ingame_admins = new HashMap<>(); - - Result admin_records = game_db.selectFrom(Admin.ADMIN).fetch(); - - for (AdminRecord rec : admin_records) { - String ckey = rec.get(Admin.ADMIN.CKEY); - int perm_rank = REMOVED_RANK; - if (rec.getPermissionsRank() != null) { - perm_rank = rec.getPermissionsRank(); - } - - ingame_admins.put(ckey, new AdminEntry(rec.get(Admin.ADMIN.ID), perm_rank)); - } - - // Apply permissions to those who need them - for (String ckey : dev_team_ckeys) { - AdminEntry entry = ingame_admins.get(ckey); - - if (entry == null) { - logger.info("[DevRank] Ckey {} not in admin table, adding new dev", ckey); - game_db.insertInto(Admin.ADMIN) - .set(Admin.ADMIN.CKEY, ckey) - .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) - .execute(); - } else { - if (entry.rank() == REMOVED_RANK) { - logger.info("[DevRank] Resetting {} to dev team with new flag", ckey); - game_db.update(Admin.ADMIN) - .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) - .where(Admin.ADMIN.ID.eq(entry.id())) - .execute(); - } else { - logger.debug("[DevRank] {} already has dev flag", ckey); - } - } - } - - // Remove devteam flag from those who no longer qualify - for (Map.Entry entry : ingame_admins.entrySet()) { - String ckey = entry.getKey(); - AdminEntry admin = entry.getValue(); - - if (!dev_team_ckeys.contains(ckey)) { - if (admin.rank() == DEV_RANK) { - logger.info("[DevRank] {} only had dev flag, removing rank and flags", ckey); - game_db.update(Admin.ADMIN) - .set(Admin.ADMIN.PERMISSIONS_RANK, REMOVED_RANK) - .where(Admin.ADMIN.ID.eq(admin.id())) - .execute(); - } - } - } - - logger.info("[DevRank] Finished sync, {} devteam users processed", dev_team_ckeys.size()); - - } catch (Exception e) { - logger.error("[DevRank] Error during sync", e); - throw new JobExecutionException(e); - } - } - - private record AdminEntry(int id, int rank) { - } -} +package me.aa07.paradise.taskdaemon.core.modules.devrank; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.forums.Tables; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.Admin; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.AdminRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +/** + * Scheduled job that syncs dev team permissions between forums and game + * databases. + */ +@DisallowConcurrentExecution // NO +public class DevRankJob implements Job { + private static final int DEV_TEAM_GROUP = 39; + private static final int DEV_RANK = 100; + private static final int REMOVED_RANK = 105; + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + JobDataMap data_map = context.getMergedJobDataMap(); + + // Get logger and db + Logger logger = (Logger) data_map.get("LOGGER"); + DbCore dbcore = (DbCore) data_map.get("DBCORE"); + + if (logger == null || dbcore == null) { + System.err.println("[DevRank] LOGGER or DBCORE was not set in JobDataMap."); + return; + } + + logger.info("[DevRank] Starting dev rank sync"); + + try { + DSLContext forums_db = dbcore.jooq(DatabaseType.Forums); + DSLContext game_db = dbcore.jooq(DatabaseType.GameDb); + + // Collect all dev team ckeys from forums database + List dev_team_ckeys = new ArrayList<>(); + + Result forum_records = forums_db.select( + Tables.CORE_MEMBERS.MEMBER_ID, + Tables.CORE_MEMBERS.NAME, + Tables.CORE_MEMBERS.MEMBER_GROUP_ID, + Tables.CORE_MEMBERS.MGROUP_OTHERS, + Tables.CORE_PFIELDS_CONTENT.FIELD_10).from(Tables.CORE_MEMBERS) + .leftJoin(Tables.CORE_PFIELDS_CONTENT) + .on(Tables.CORE_MEMBERS.MEMBER_ID.eq(Tables.CORE_PFIELDS_CONTENT.MEMBER_ID)) + .fetch(); + + for (Record rec : forum_records) { + int primary_group = rec.get(Tables.CORE_MEMBERS.MEMBER_GROUP_ID); + String other_groups = rec.get(Tables.CORE_MEMBERS.MGROUP_OTHERS); + String ckey = rec.get(Tables.CORE_PFIELDS_CONTENT.FIELD_10); + + Set all_groups = new HashSet(); + all_groups.add(primary_group); + + if (other_groups != null && !other_groups.isBlank()) { + Arrays.stream(other_groups.split(",")) + .filter(g -> !g.isBlank()) + .map(Integer::parseInt) + .forEach(all_groups::add); + } + + if (all_groups.contains(DEV_TEAM_GROUP)) { + if (ckey == null || ckey.isBlank()) { + logger.warn("[DevRank] Forums user {} (ID {}) has no linked ckey", + rec.get(Tables.CORE_MEMBERS.NAME), rec.get(Tables.CORE_MEMBERS.MEMBER_ID)); + continue; + } + + String cleaned = ckey.toLowerCase().replaceAll("[\\s_\\-]", ""); + dev_team_ckeys.add(cleaned); + } + } + + // Load all ingame admins + Map ingame_admins = new HashMap<>(); + + Result admin_records = game_db.selectFrom(Admin.ADMIN).fetch(); + + for (AdminRecord rec : admin_records) { + String ckey = rec.get(Admin.ADMIN.CKEY); + int perm_rank = REMOVED_RANK; + if (rec.getPermissionsRank() != null) { + perm_rank = rec.getPermissionsRank(); + } + + ingame_admins.put(ckey, new AdminEntry(rec.get(Admin.ADMIN.ID), perm_rank)); + } + + // Apply permissions to those who need them + for (String ckey : dev_team_ckeys) { + AdminEntry entry = ingame_admins.get(ckey); + + if (entry == null) { + logger.info("[DevRank] Ckey {} not in admin table, adding new dev", ckey); + game_db.insertInto(Admin.ADMIN) + .set(Admin.ADMIN.CKEY, ckey) + .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) + .execute(); + } else { + if (entry.rank() == REMOVED_RANK) { + logger.info("[DevRank] Resetting {} to dev team with new flag", ckey); + game_db.update(Admin.ADMIN) + .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) + .where(Admin.ADMIN.ID.eq(entry.id())) + .execute(); + } else { + logger.debug("[DevRank] {} already has dev flag", ckey); + } + } + } + + // Remove devteam flag from those who no longer qualify + for (Map.Entry entry : ingame_admins.entrySet()) { + String ckey = entry.getKey(); + AdminEntry admin = entry.getValue(); + + if (!dev_team_ckeys.contains(ckey)) { + if (admin.rank() == DEV_RANK) { + logger.info("[DevRank] {} only had dev flag, removing rank and flags", ckey); + game_db.update(Admin.ADMIN) + .set(Admin.ADMIN.PERMISSIONS_RANK, REMOVED_RANK) + .where(Admin.ADMIN.ID.eq(admin.id())) + .execute(); + } + } + } + + logger.info("[DevRank] Finished sync, {} devteam users processed", dev_team_ckeys.size()); + + } catch (Exception e) { + logger.error("[DevRank] Error during sync", e); + throw new JobExecutionException(e); + } + } + + private record AdminEntry(int id, int rank) { + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java index 7c1a984..7565c5e 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java @@ -1,139 +1,322 @@ -package me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync; - -import java.util.ArrayList; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.core.util.UtilStr; -import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; -import me.aa07.paradise.taskdaemon.database.automation.Tables; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Result; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class IngameVerifiedSyncJob implements Job { - /* - - - pull raw data from authentik - - keep track of new ones - - get discord IDs and add into table - - get forum IDs and add into table - - remove entries from directory table that werent there before - - schedule task in ALICE to strip discord role - - remove forum group - - slap authentik ID onto rows in the player table - - */ - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[IngameVerifiedSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - logger.info("[IngameVerifiedSync] Starting up"); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[IngameVerifiedSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); - DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); - - // For now - we just want ckeys and authentik IDs - no more - Result authentik_records = authentik_jooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) - .fetch(); - - logger.info(String.format("[IngameVerifiedSync] Found %s accounts with linked byond accounts", authentik_records.size())); - - ArrayList only_authentik_ids = new ArrayList(); - - for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { - only_authentik_ids.add(acuscr.getUserId()); - } - - // Records to purge - Result records_to_purge = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.notIn(only_authentik_ids)) - .fetch(); - - int removed = 0; - for (IngameVerifiedDirectoryRecord ivdr : records_to_purge) { - // Step 1 - Strip their rank on the forums - - // Step 2 - Strip their rank on the discord - - // Step 3 - Clear their FUID ingame - - // Step 4 - Clear from directory - ivdr.delete(); - removed++; - } - logger.info(String.format("[IngameVerifiedSync] Removed %s records for members who unlinked byond", removed)); - - // Now begin the mass inserts - int raw_added = 0; - int raw_updated = 0; - for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { - int authentik_user_ud = acuscr.getUserId(); - // First - see if they already exist in the directory - - if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)))) { - - // Update if they do - raw_updated++; - - IngameVerifiedDirectoryRecord ivdr = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)).fetchOne(); - - ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); - ivdr.store(); - } else { - // Insert if they dont - IngameVerifiedDirectoryRecord ivdr = automation_jooq.newRecord(Tables.INGAME_VERIFIED_DIRECTORY); - ivdr.setAuthentikId(acuscr.getUserId()); - ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); - ivdr.store(); - raw_added++; - } - } - - logger.info(String.format("[IngameVerifiedSync] Added %s records and updated %s for members who are linked", raw_added, raw_updated)); - - logger.info("[IngameVerifiedSync] Process complete"); - } -} +package me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.services.invision.InvisionUtil; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.core.util.UtilStr; +import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; +import me.aa07.paradise.taskdaemon.database.automation.Tables; +import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PlayerRecord; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class IngameVerifiedSyncJob implements Job { + /* + + - pull raw data from authentik + - keep track of new ones + - get discord IDs and add into table + - get forum IDs and add into table + - do forum syncage based on if their IGV or IGB + - remove entries from directory table that werent there before + - schedule task in ALICE to strip discord role + - remove forum group + - slap authentik ID onto rows in the player table + + */ + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[IngameVerifiedSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + logger.info("[IngameVerifiedSync] Starting up"); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[IngameVerifiedSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Now get the invision util + Object raw_iu = datamap.get("IU"); + Optional iu_holder = Optional.empty(); + + if (raw_iu instanceof InvisionUtil iu2) { + iu_holder = Optional.of(iu2); + } + + if (!iu_holder.isPresent()) { + logger.error("[IngameVerifiedSync] IU WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + InvisionUtil iu = iu_holder.get(); + + DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); + DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); + DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); + + // For now - we just want ckeys and authentik IDs - no more + Result authentik_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) + .fetch(); + + logger.info(String.format("[IngameVerifiedSync] Found %s accounts with linked byond accounts", authentik_records.size())); + + ArrayList only_authentik_ids = new ArrayList(); + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { + only_authentik_ids.add(acuscr.getUserId()); + } + + // Records to purge + Result records_to_purge = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.notIn(only_authentik_ids)) + .fetch(); + + int removed = 0; + for (IngameVerifiedDirectoryRecord ivdr : records_to_purge) { + // Step 1 - Strip their rank on the forums + if (ivdr.getForumId() != null) { + int forum_id = ivdr.getForumId(); + Pair> gsg_response = iu.getUserSecondaryGroups(forum_id); + + if (gsg_response.getLeft()) { + List forum_secondaries = gsg_response.getRight(); + if (forum_secondaries.contains(Integer.valueOf(UtilConst.INVISION_INGAMEVERIFIED_GID))) { + // This cast looks silly - but without it, java treats it as a numerical index not an object + forum_secondaries.remove(Integer.valueOf(UtilConst.INVISION_INGAMEVERIFIED_GID)); + + boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); + + if (!ssg_response) { + logger.warn(String.format("[IngameVerifiedSync] Could not update secondaries for %s", forum_id)); + } + } + } else { + logger.warn(String.format("[IngameVerifiedSync] Could not get secondaries for %s", forum_id)); + } + } + + // Step 2 - Strip their rank on the discord + if (ivdr.getDiscordId() != null) { + TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("REMOVE_IGV_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = ivdr.getDiscordId(); + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbcore.now()); + tqr.store(); + } + + // Step 3 - Clear their FUID ingame + PlayerRecord ingame_record = game_jooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER.CKEY.eq(ivdr.getCkey())) + .fetchOne(); + + if (ingame_record != null) { + ingame_record.setFuid(null); + ingame_record.store(); + } + + // Step 4 - Clear from directory + ivdr.delete(); + removed++; + } + logger.info(String.format("[IngameVerifiedSync] Removed %s records for members who unlinked byond", removed)); + + // Now begin the mass inserts + int raw_added = 0; + int raw_updated = 0; + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { + int authentik_user_ud = acuscr.getUserId(); + // First - see if they already exist in the directory + + if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)))) { + + // Update if they do + raw_updated++; + + IngameVerifiedDirectoryRecord ivdr = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)).fetchOne(); + + ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); + ivdr.store(); + } else { + // Insert if they dont + IngameVerifiedDirectoryRecord ivdr = automation_jooq.newRecord(Tables.INGAME_VERIFIED_DIRECTORY); + ivdr.setAuthentikId(acuscr.getUserId()); + ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); + ivdr.store(); + raw_added++; + } + } + + logger.info(String.format("[IngameVerifiedSync] Added %s records and updated %s for members who are linked", raw_added, raw_updated)); + + // Now slap forum IDs and discord onto them + Result records_to_update = automation_jooq + .selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .fetch(); + + Result authentik_forums_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.INVISION_OAUTH_SERVICE_ID)) + .fetch(); + + Result authentik_discord_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) + .fetch(); + + Result gamedb_records = game_jooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER).fetch(); + + HashMap authentik2forum = new HashMap(); + HashMap authentik2discord = new HashMap(); + HashMap ckey2gamerecord = new HashMap(); + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_forums_records) { + String forum_id = UtilStr.onlyNumbers(acuscr.getIdentifier()); // Some of these have UTF8-BOMs on them. I wish I was kidding. + authentik2forum.put(acuscr.getUserId(), Integer.parseInt(forum_id)); + } + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_discord_records) { + authentik2discord.put(acuscr.getUserId(), Long.parseLong(acuscr.getIdentifier())); + } + + for (PlayerRecord pr : gamedb_records) { + ckey2gamerecord.put(pr.getCkey(), pr); + } + + int forum_updated = 0; + int discord_updated = 0; + int game_updated = 0; + for (IngameVerifiedDirectoryRecord ivdr : records_to_update) { + boolean update_made = false; + int authentik_id = ivdr.getAuthentikId(); + int forum_id = 0; + long discord_id = 0L; + + if (authentik2forum.containsKey(authentik_id)) { + forum_id = authentik2forum.get(authentik_id); + } + + if (authentik2discord.containsKey(authentik_id)) { + discord_id = authentik2discord.get(authentik_id); + } + + // Add forums + if (forum_id > 0 && (ivdr.getForumId() == null || !ivdr.getForumId().equals(forum_id))) { + // See if we have them as an ID + ivdr.setForumId(forum_id); + update_made = true; + forum_updated++; + + // ADD INVISION SECONDARY GROUPS HERE + Pair> gsg_response = iu.getUserSecondaryGroups(forum_id); + + if (gsg_response.getLeft()) { + List forum_secondaries = gsg_response.getRight(); + if (!forum_secondaries.contains(UtilConst.INVISION_INGAMEVERIFIED_GID)) { + forum_secondaries.add(UtilConst.INVISION_INGAMEVERIFIED_GID); + + boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); + + if (!ssg_response) { + logger.warn(String.format("[IngameVerifiedSync] Could not update secondaries for %s", forum_id)); + } + } + } else { + logger.warn(String.format("[IngameVerifiedSync] Could not get secondaries for %s", forum_id)); + } + } + + // Add discord + if (discord_id > 0 && (ivdr.getDiscordId() == null || !ivdr.getDiscordId().equals(discord_id))) { + ivdr.setDiscordId(discord_id); + update_made = true; + discord_updated++; + + // And make the ALICE task + TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("ADD_IGV_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbcore.now()); + tqr.store(); + } + + if (ckey2gamerecord.containsKey(ivdr.getCkey())) { + PlayerRecord game_row = ckey2gamerecord.get(ivdr.getCkey()); + if (game_row.getFuid() == null || game_row.getFuid() != authentik_id) { + game_row.setFuid((long) authentik_id); + game_row.store(); + game_updated++; + } + } + + if (update_made) { + ivdr.store(); + } + } + + logger.info(String.format("[IngameVerifiedSync] Update stats discord/forum/game - %s/%s/%s", discord_updated, forum_updated, game_updated)); + logger.info("[IngameVerifiedSync] Process complete"); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java index 143b6a0..542437d 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java @@ -1,211 +1,211 @@ -package me.aa07.paradise.taskdaemon.core.modules.ip2asn; - -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.Ip2AsnSerivceConfig; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.ip2asn.Ip2AsnResponseModel; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.database.gamedb.Tables; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.Ip2groupRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Select; -import org.jooq.types.UInteger; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class Ip2AsnJob implements Job { - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[Ip2Asn] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[Ip2Asn] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Now get our config - Object raw_ip2asn_cfg = datamap.get("IP2ASNCFG"); - Optional ip2asn_cfg_holder = Optional.empty(); - - if (raw_ip2asn_cfg instanceof Ip2AsnSerivceConfig ip2asnCfg2) { - ip2asn_cfg_holder = Optional.of(ip2asnCfg2); - } - - if (!ip2asn_cfg_holder.isPresent()) { - logger.error("[Ip2Asn] IP2ASNCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Ip2AsnSerivceConfig config = ip2asn_cfg_holder.get(); - - DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); - - // So - // JOOQ doesnt have INET_ATON, presumably because its a MySQL native - // So we have to do the manual selects then work out the differences - // To do this we need: - // 1. All IPs from player seen in the last 30 days - // 2. Then remove the IPs in ip2group that have been updated in the last 7 days - // Then we need to update IP2Group or insert where required - LocalDateTime last_30_days = LocalDateTime.now().minusDays(30); - LocalDateTime last_7_days = LocalDateTime.now().minusDays(7); - - logger.info("[Ip2Asn] Pulling player IPs..."); - - // Get the player IPs - List player_ips = ctx.select(Tables.PLAYER.IP).from(Tables.PLAYER) - .where(Tables.PLAYER.LASTSEEN.gt(last_30_days)).fetch() - .getValues(Tables.PLAYER.IP, String.class); - - logger.info(String.format("[Ip2Asn] Pulled %s player IPs, now pulling ip2group IPs...", player_ips.size())); - - List ip2group_ips = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) - .where(Tables.IP2GROUP.DATE.gt(last_7_days)).fetch() - .getValues(Tables.IP2GROUP.IP, UInteger.class); - - logger.info(String.format("[Ip2Asn] Pulled %s ip2group IPs, now parsing...", ip2group_ips.size())); - - // Now turn that to regular IPs - ArrayList parsed_ip2group_ips = new ArrayList(); - - for (UInteger ip_uint : ip2group_ips) { - String ip_str = uint2ip(ip_uint); - parsed_ip2group_ips.add(ip_str); - } - - ArrayList ips_to_get = new ArrayList(); - - // Start with the IPs in player - then see if they exist in the last 7 days in - // ip2group - for (String ip : player_ips) { - if (!parsed_ip2group_ips.contains(ip)) { - ips_to_get.add(ip); - } - } - - logger.info(String.format("[Ip2Asn] Found %s IPs to refresh groups for", ips_to_get.size())); - - if (ips_to_get.size() == 0) { - logger.info("[Ip2Asn] No work to do"); - return; - } - - // Now do the big refresh - HttpClient client = HttpClient.newHttpClient(); - for (String ip : ips_to_get) { - try { - // Get our AS - probably a more elegant way to do this but meh - HttpRequest httpreq = HttpRequest.newBuilder() - .uri(new URI(String.format("%s%s", config.host, ip))) - .GET() - .header("Accept", "application/json") - .build(); - - // Send the request off - HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); - - // Decode the shenanigans - Ip2AsnResponseModel res_model = UtilConst.GSON.fromJson(response.body(), Ip2AsnResponseModel.class); - UInteger ip_uint = ip2uint(ip); - UInteger asn = UInteger.valueOf(res_model.asn); - - // See if our row exists, we update if it does - Select> fetch_test = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) - .where(Tables.IP2GROUP.IP.eq(ip_uint)); - - if (ctx.fetchExists(fetch_test)) { - // It exists, update - logger.info(String.format("[Ip2Asn] Updating row for %s", ip)); - ctx.update(Tables.IP2GROUP) - .set(Tables.IP2GROUP.GROUPSTR, asn) - .set(Tables.IP2GROUP.DATE, dbcore.now()) - .where(Tables.IP2GROUP.IP.eq(ip_uint)) - .execute(); - } else { - // It dont exist, insert - logger.info(String.format("[Ip2Asn] Inserting row for %s", ip)); - Ip2groupRecord record = ctx.newRecord(Tables.IP2GROUP); - record.setIp(ip_uint); - record.setDate(dbcore.now()); - record.setGroupstr(asn); - record.store(); - } - - } catch (Exception e) { - logger.warn(String.format("[Ip2Asn] Failed to get AS info for IP %s - skipping", ip)); - logger.warn(e); - } - } - - logger.info("[Ip2Asn] Processing complete"); - } - - // Turns a uint IP into its string form - private String uint2ip(UInteger ip) { - long ip_long = ip.longValue(); - return String.format("%d.%d.%d.%d", - (ip_long >> 24 & 0xff), - (ip_long >> 16 & 0xff), - (ip_long >> 8 & 0xff), - (ip_long & 0xff)); - } - - // Turns a string IP into its uint form - private UInteger ip2uint(String ip) { - String[] octets = ip.split("\\."); - - long ip_long = 0; - for (int i = 0; i < 4; i++) { - int octet = Integer.parseInt(octets[i]); - if (octet < 0 || octet > 255) { - throw new IllegalArgumentException("Invalid octet in IP address: " + octets[i]); - } - // This does the shift magic with the for loop, - // with each iteration shifting by 8 - ip_long |= ((long) octet << (24 - 8 * i)); - } - - return UInteger.valueOf(ip_long); - } -} +package me.aa07.paradise.taskdaemon.core.modules.ip2asn; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.Ip2AsnSerivceConfig; +import me.aa07.paradise.taskdaemon.core.models.ip2asn.Ip2AsnResponseModel; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.database.gamedb.Tables; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.Ip2groupRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Select; +import org.jooq.types.UInteger; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class Ip2AsnJob implements Job { + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[Ip2Asn] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[Ip2Asn] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Now get our config + Object raw_ip2asn_cfg = datamap.get("IP2ASNCFG"); + Optional ip2asn_cfg_holder = Optional.empty(); + + if (raw_ip2asn_cfg instanceof Ip2AsnSerivceConfig ip2asnCfg2) { + ip2asn_cfg_holder = Optional.of(ip2asnCfg2); + } + + if (!ip2asn_cfg_holder.isPresent()) { + logger.error("[Ip2Asn] IP2ASNCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Ip2AsnSerivceConfig config = ip2asn_cfg_holder.get(); + + DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); + + // So + // JOOQ doesnt have INET_ATON, presumably because its a MySQL native + // So we have to do the manual selects then work out the differences + // To do this we need: + // 1. All IPs from player seen in the last 30 days + // 2. Then remove the IPs in ip2group that have been updated in the last 7 days + // Then we need to update IP2Group or insert where required + LocalDateTime last_30_days = LocalDateTime.now().minusDays(30); + LocalDateTime last_7_days = LocalDateTime.now().minusDays(7); + + logger.info("[Ip2Asn] Pulling player IPs..."); + + // Get the player IPs + List player_ips = ctx.select(Tables.PLAYER.IP).from(Tables.PLAYER) + .where(Tables.PLAYER.LASTSEEN.gt(last_30_days)).fetch() + .getValues(Tables.PLAYER.IP, String.class); + + logger.info(String.format("[Ip2Asn] Pulled %s player IPs, now pulling ip2group IPs...", player_ips.size())); + + List ip2group_ips = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) + .where(Tables.IP2GROUP.DATE.gt(last_7_days)).fetch() + .getValues(Tables.IP2GROUP.IP, UInteger.class); + + logger.info(String.format("[Ip2Asn] Pulled %s ip2group IPs, now parsing...", ip2group_ips.size())); + + // Now turn that to regular IPs + ArrayList parsed_ip2group_ips = new ArrayList(); + + for (UInteger ip_uint : ip2group_ips) { + String ip_str = uint2ip(ip_uint); + parsed_ip2group_ips.add(ip_str); + } + + ArrayList ips_to_get = new ArrayList(); + + // Start with the IPs in player - then see if they exist in the last 7 days in + // ip2group + for (String ip : player_ips) { + if (!parsed_ip2group_ips.contains(ip)) { + ips_to_get.add(ip); + } + } + + logger.info(String.format("[Ip2Asn] Found %s IPs to refresh groups for", ips_to_get.size())); + + if (ips_to_get.size() == 0) { + logger.info("[Ip2Asn] No work to do"); + return; + } + + // Now do the big refresh + HttpClient client = HttpClient.newHttpClient(); + for (String ip : ips_to_get) { + try { + // Get our AS - probably a more elegant way to do this but meh + HttpRequest httpreq = HttpRequest.newBuilder() + .uri(new URI(String.format("%s%s", config.host, ip))) + .GET() + .header("Accept", "application/json") + .build(); + + // Send the request off + HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); + + // Decode the shenanigans + Ip2AsnResponseModel res_model = UtilConst.GSON.fromJson(response.body(), Ip2AsnResponseModel.class); + UInteger ip_uint = ip2uint(ip); + UInteger asn = UInteger.valueOf(res_model.asn); + + // See if our row exists, we update if it does + Select> fetch_test = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) + .where(Tables.IP2GROUP.IP.eq(ip_uint)); + + if (ctx.fetchExists(fetch_test)) { + // It exists, update + logger.info(String.format("[Ip2Asn] Updating row for %s", ip)); + ctx.update(Tables.IP2GROUP) + .set(Tables.IP2GROUP.GROUPSTR, asn) + .set(Tables.IP2GROUP.DATE, dbcore.now()) + .where(Tables.IP2GROUP.IP.eq(ip_uint)) + .execute(); + } else { + // It dont exist, insert + logger.info(String.format("[Ip2Asn] Inserting row for %s", ip)); + Ip2groupRecord record = ctx.newRecord(Tables.IP2GROUP); + record.setIp(ip_uint); + record.setDate(dbcore.now()); + record.setGroupstr(asn); + record.store(); + } + + } catch (Exception e) { + logger.warn(String.format("[Ip2Asn] Failed to get AS info for IP %s - skipping", ip)); + logger.warn(e); + } + } + + logger.info("[Ip2Asn] Processing complete"); + } + + // Turns a uint IP into its string form + private String uint2ip(UInteger ip) { + long ip_long = ip.longValue(); + return String.format("%d.%d.%d.%d", + (ip_long >> 24 & 0xff), + (ip_long >> 16 & 0xff), + (ip_long >> 8 & 0xff), + (ip_long & 0xff)); + } + + // Turns a string IP into its uint form + private UInteger ip2uint(String ip) { + String[] octets = ip.split("\\."); + + long ip_long = 0; + for (int i = 0; i < 4; i++) { + int octet = Integer.parseInt(octets[i]); + if (octet < 0 || octet > 255) { + throw new IllegalArgumentException("Invalid octet in IP address: " + octets[i]); + } + // This does the shift magic with the for loop, + // with each iteration shifting by 8 + ip_long |= ((long) octet << (24 - 8 * i)); + } + + return UInteger.valueOf(ip_long); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java index f610911..5d67511 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java @@ -1,562 +1,562 @@ -package me.aa07.paradise.taskdaemon.core.modules.patreonsync; - -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import me.aa07.paradise.taskdaemon.core.config.PatreonConfig; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonOuterResponseModel; -import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonRawResponseModel; -import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonUser; -import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.core.util.UtilStr; -import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; -import me.aa07.paradise.taskdaemon.database.automation.Tables; -import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.PatreonSupportersRecord; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.DonatorsRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Result; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class PatreonSyncJob implements Job { - /* - - - pull raw data from patreon - - empty the holding table and fill with the patreon raw data - - get authentik IDs and add into table - - get discord IDs and add into table - - get ckeys and add into table - - mass sync ingame stuff - - */ - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[PatreonSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - logger.info("[PatreonSync] Starting up"); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[PatreonSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - dbcore.getClass(); - - // Now get our config - Object raw_ptrcfg = datamap.get("PTRCFG"); - Optional ptrcfg_holder = Optional.empty(); - - if (raw_ptrcfg instanceof PatreonConfig ptrcfg2) { - ptrcfg_holder = Optional.of(ptrcfg2); - } - - if (!ptrcfg_holder.isPresent()) { - logger.error("[PatreonSync] PTRCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - PatreonConfig ptrcfg = ptrcfg_holder.get(); - - // So - // The patreon API is kinda bad - // Everything goes into the same loose model and you have to manually parse out members and tiers - // Boy thats fun - - ArrayList users = new ArrayList(); - - try { - HttpRequest httpreq = HttpRequest.newBuilder() - .uri(URI.create("https://www.patreon.com/api/oauth2/v2/campaigns/" - + String.valueOf(ptrcfg.campaignId) - + "/members" - + "?include=currently_entitled_tiers,user" - + "&fields[member]=patron_status,currently_entitled_amount_cents" - + "&fields[tier]=amount_cents" - + "&page[count]=1000")) // This will break if we get >1000 patrons - // TODO Add pagination - .GET() - .header("Accept", "application/json") - .header("Authorization", String.format("Bearer %s", ptrcfg.creatorToken)) - .header("User-Agent", "ParadiseSS13-TaskDaemon/1.0") - .build(); - - HttpClient client = HttpClient.newHttpClient(); - logger.info("[PatreonSync] Pulling data"); - HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); - - String response_body = response.body(); - - /* - - // Uncomment if you need horrendous debugging - - File file = new File("patreon_debug_response.json"); - file.createNewFile(); - FileWriter fw = new FileWriter(file); - fw.write(response_body); - fw.close(); - */ - - PatreonOuterResponseModel holder = UtilConst.GSON.fromJson(response_body, PatreonOuterResponseModel.class); - - logger.info(String.format("[PatreonSync] Pulled %s raw models from Patreon", holder.data.size())); - - for (PatreonRawResponseModel raw_model : holder.data) { - if (raw_model.type.equalsIgnoreCase("member")) { - // Make sure theyre valid first - if (raw_model.attributes != null && raw_model.attributes.patronStatus != null) { - if (raw_model.attributes.patronStatus.equalsIgnoreCase("active_patron")) { - PatreonUser user = raw_model.asUser(); - users.add(user); - continue; - } - } - } - } - } catch (Exception ex) { - logger.error(ex); - } - - logger.info(String.format("[PatreonSync] Loaded %s active users", users.size())); - - ArrayList only_member_ids = new ArrayList(); - - for (PatreonUser pu : users) { - only_member_ids.add(pu.userId); - } - - // Step 1 - Clear out old users - DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); - - - Result records_to_purge = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.notIn(only_member_ids)) - .fetch(); - - for (PatreonSupportersRecord psr : records_to_purge) { - Long database_discord_id = psr.getDiscordId(); - - // Store our removal task if necessary - if (database_discord_id != null && (psr.getDonationTier() >= 2)) { - TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("REMOVE_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = database_discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbcore.now()); - tqr.store(); - } - - // Clear out the supporter record - psr.delete(); - } - - logger.info("[PatreonSync] Cleaned out old users"); - - int insert_count = 0; - int update_count = 0; - - // Step 2 - Add or insert new - for (PatreonUser pu : users) { - if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)))) { - - update_count++; - - // Fetch and update our record - PatreonSupportersRecord psr = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)) - .fetchSingle(); - - psr.setAmountCents(pu.amountInCents); - psr.setDonationTier(pu.maxTier().tierLevel); - - // And store - psr.store(); - } else { - insert_count++; - - // Make a new record - PatreonSupportersRecord psr = automation_jooq.newRecord(Tables.PATREON_SUPPORTERS); - psr.setMemberId(pu.userId); - psr.setAmountCents(pu.amountInCents); - psr.setDonationTier(pu.maxTier().tierLevel); - - // And store - psr.store(); - } - } - - logger.info(String.format("[PatreonSync] Inserted %s users, updated %s users", insert_count, update_count)); - - - DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); - DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); - - // Find authentik users - HashMap authentik_to_patreon = new HashMap(); - findAuthentikUsers(only_member_ids, authentik_jooq, automation_jooq, logger, authentik_to_patreon); - - // Now find discord users - findDiscordUsers(authentik_jooq, automation_jooq, logger, authentik_to_patreon, dbcore); - - // Now get ckeys - findCkeys(authentik_jooq, automation_jooq, logger, authentik_to_patreon); - - // And commit to game DB - updateGameDb(automation_jooq, game_jooq, logger); - - System.gc(); // This is important after this one - the response model is like multiple megabytes - logger.info("[PatreonSync] Process complete"); - } - - private static int patreonTier2ingameTier(int patreonTier) { - switch (patreonTier) { - case 0: // No tier - return 0; - case 1: // Supporter - return 0; - case 2: // Icon - return 1; - case 3: // Loadout - return 2; - case 4: // Silver - return 3; - case 5: // Gold - return 4; - default: - return 0; - } - } - - private static void findAuthentikUsers( - List memberIds, - DSLContext authentikJooq, - DSLContext automationJooq, - Logger logger, - HashMap authentik2patreon - ) { - for (int member_id : memberIds) { - // This loop should only have active member IDs for the DB given we just flushed everything else out - - // Get ready for a TON of absolute pathing - AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.IDENTIFIER.eq(String.valueOf(member_id))) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.PATREON_OAUTH_SERVICE_ID)) - .fetchOne(); - - if (authentik_record == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in Authentik!", member_id)); - continue; - } - - authentik2patreon.put(authentik_record.getUserId(), member_id); - } - - logger.info(String.format("[PatreonSync] Loaded %s Authentik users", authentik2patreon.size())); - - int supporters_authentik_updated = 0; - for (int authentik_id : authentik2patreon.keySet()) { - int patreon_id = authentik2patreon.get(authentik_id); - - // Get our DB record - - PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) - .fetchOne(); - - if (psr == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); - continue; - } - - supporters_authentik_updated++; - psr.setAuthentikId(authentik_id); - psr.store(); // Store it back in - } - - logger.info(String.format("[PatreonSync] Updated %s rows with Authentik users", supporters_authentik_updated)); - } - - private static void findDiscordUsers( - DSLContext authentikJooq, - DSLContext automationJooq, - Logger logger, - HashMap authentik2patreon, - DbCore dbCore - ) { - HashMap authentik2discord = new HashMap(); - - for (int authentik_id : authentik2patreon.keySet()) { - // This loop should only have active member IDs for the DB given we just flushed everything else out - - // Get ready for a TON of absolute pathing - AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) - .fetchOne(); - - if (authentik_record == null) { - logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); - continue; - } - - authentik2discord.put(authentik_id, Long.valueOf(authentik_record.getIdentifier())); - } - - logger.info(String.format("[PatreonSync] Loaded %s Discord users", authentik2discord.size())); - - int supporters_updated = 0; - for (int authentik_id : authentik2discord.keySet()) { - if (!authentik2patreon.containsKey(authentik_id)) { - logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); - } - - int patreon_id = authentik2patreon.get(authentik_id); - long discord_id = authentik2discord.get(authentik_id); - - // Get our DB record - PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) - .fetchOne(); - - if (psr == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); - continue; - } - - // See if we have a role to add - if (psr.getDiscordId() == null && psr.getDonationTier() >= 2) { - // We do - lets add the role - TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("ADD_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbCore.now()); - tqr.store(); - } - - // See if their tier changed between runs - if (psr.getDiscordId() != null && (psr.getLastrunDonationTier().intValue() != psr.getDonationTier().intValue())) { - // Tier changed - see if they went below the threshold - if (psr.getDonationTier() <= 1) { - // They went below the threshold - strip the role - TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("REMOVE_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbCore.now()); - tqr.store(); - } - - // Add if we just upgraded - if (psr.getDonationTier() >= 2 && psr.getLastrunDonationTier() == 1) { - TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("ADD_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbCore.now()); - tqr.store(); - } - } - - supporters_updated++; - psr.setLastrunDonationTier(psr.getDonationTier()); - psr.setDiscordId(discord_id); - psr.store(); // Store it back in - } - - logger.info(String.format("[PatreonSync] Updated %s rows with Discord users", supporters_updated)); - } - - private static void findCkeys( - DSLContext authentikJooq, - DSLContext automationJooq, - Logger logger, - HashMap authentik2patreon - ) { - HashMap authentik2ckey = new HashMap(); - - for (int authentik_id : authentik2patreon.keySet()) { - // This loop should only have active member IDs for the DB given we just flushed everything else out - - // Get ready for a TON of absolute pathing - AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) - .fetchOne(); - - if (authentik_record == null) { - logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); - continue; - } - - // Normalise the ckey - String ckey = UtilStr.cleanCkey(authentik_record.getIdentifier()); - - authentik2ckey.put(authentik_id, ckey); - } - - logger.info(String.format("[PatreonSync] Loaded %s ckeys", authentik2ckey.size())); - - int supporters_updated = 0; - for (int authentik_id : authentik2ckey.keySet()) { - if (!authentik2patreon.containsKey(authentik_id)) { - logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); - } - - int patreon_id = authentik2patreon.get(authentik_id); - String ckey = authentik2ckey.get(authentik_id); - - // Get our DB record - PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) - .fetchOne(); - - if (psr == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); - continue; - } - - supporters_updated++; - psr.setCkey(ckey); - psr.store(); // Store it back in - } - - logger.info(String.format("[PatreonSync] Updated %s rows with ckeys", supporters_updated)); - } - - private static void updateGameDb(DSLContext automationJooq, DSLContext gameJooq, Logger logger) { - - Result valid_records = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.CKEY.isNotNull()) - .and(Tables.PATREON_SUPPORTERS.DONATION_TIER.ge(2)) // Tier 2 or higher to appear ingame - .fetch(); - - logger.info(String.format("[PatreonSync] Loaded %s records to update in the Game DB", valid_records.size())); - - ArrayList only_ckeys = new ArrayList(); - - for (PatreonSupportersRecord psr : valid_records) { - only_ckeys.add(psr.getCkey()); - } - - // Remove from BYOND where not in this list - gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.notIn(only_ckeys)) - .execute(); - - // And where not active to keep the table short - gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.ACTIVE.ne((byte) 0x01)) // tinyint is a byte - .execute(); - - int insert_count = 0; - int update_count = 0; - // Now do our record stuff - for (PatreonSupportersRecord psr : valid_records) { - DonatorsRecord dr = gameJooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.eq(psr.getCkey())) - .fetchOne(); - - if (dr == null) { - // Empty record - make a new one - DonatorsRecord dr2 = gameJooq.newRecord(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS); - dr2.setPatreonName(psr.getMemberId().toString()); - dr2.setTier(patreonTier2ingameTier(psr.getDonationTier())); - dr2.setCkey(psr.getCkey()); - dr2.setStartDate(null); - dr2.setEndDate(null); - dr2.setActive((byte) 0x01); - - dr2.store(); - - insert_count++; - } else { - // Existing record - update - dr.setTier(patreonTier2ingameTier(psr.getDonationTier())); - dr.setCkey(psr.getCkey()); - dr.setStartDate(null); - dr.setEndDate(null); - dr.setActive((byte) 0x01); - - dr.store(); - - update_count++; - } - } - - logger.info(String.format("[PatreonSync] Inserted %s records, updated %s records in the game DB.", insert_count, update_count)); - } -} +package me.aa07.paradise.taskdaemon.core.modules.patreonsync; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import me.aa07.paradise.taskdaemon.core.config.PatreonConfig; +import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonOuterResponseModel; +import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonRawResponseModel; +import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonUser; +import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.core.util.UtilStr; +import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; +import me.aa07.paradise.taskdaemon.database.automation.Tables; +import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.PatreonSupportersRecord; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.DonatorsRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class PatreonSyncJob implements Job { + /* + + - pull raw data from patreon + - empty the holding table and fill with the patreon raw data + - get authentik IDs and add into table + - get discord IDs and add into table + - get ckeys and add into table + - mass sync ingame stuff + + */ + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[PatreonSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + logger.info("[PatreonSync] Starting up"); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[PatreonSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + dbcore.getClass(); + + // Now get our config + Object raw_ptrcfg = datamap.get("PTRCFG"); + Optional ptrcfg_holder = Optional.empty(); + + if (raw_ptrcfg instanceof PatreonConfig ptrcfg2) { + ptrcfg_holder = Optional.of(ptrcfg2); + } + + if (!ptrcfg_holder.isPresent()) { + logger.error("[PatreonSync] PTRCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + PatreonConfig ptrcfg = ptrcfg_holder.get(); + + // So + // The patreon API is kinda bad + // Everything goes into the same loose model and you have to manually parse out members and tiers + // Boy thats fun + + ArrayList users = new ArrayList(); + + try { + HttpRequest httpreq = HttpRequest.newBuilder() + .uri(URI.create("https://www.patreon.com/api/oauth2/v2/campaigns/" + + String.valueOf(ptrcfg.campaignId) + + "/members" + + "?include=currently_entitled_tiers,user" + + "&fields[member]=patron_status,currently_entitled_amount_cents" + + "&fields[tier]=amount_cents" + + "&page[count]=1000")) // This will break if we get >1000 patrons + // TODO Add pagination + .GET() + .header("Accept", "application/json") + .header("Authorization", String.format("Bearer %s", ptrcfg.creatorToken)) + .header("User-Agent", "ParadiseSS13-TaskDaemon/1.0") + .build(); + + HttpClient client = HttpClient.newHttpClient(); + logger.info("[PatreonSync] Pulling data"); + HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); + + String response_body = response.body(); + + /* + + // Uncomment if you need horrendous debugging + + File file = new File("patreon_debug_response.json"); + file.createNewFile(); + FileWriter fw = new FileWriter(file); + fw.write(response_body); + fw.close(); + */ + + PatreonOuterResponseModel holder = UtilConst.GSON.fromJson(response_body, PatreonOuterResponseModel.class); + + logger.info(String.format("[PatreonSync] Pulled %s raw models from Patreon", holder.data.size())); + + for (PatreonRawResponseModel raw_model : holder.data) { + if (raw_model.type.equalsIgnoreCase("member")) { + // Make sure theyre valid first + if (raw_model.attributes != null && raw_model.attributes.patronStatus != null) { + if (raw_model.attributes.patronStatus.equalsIgnoreCase("active_patron")) { + PatreonUser user = raw_model.asUser(); + users.add(user); + continue; + } + } + } + } + } catch (Exception ex) { + logger.error(ex); + } + + logger.info(String.format("[PatreonSync] Loaded %s active users", users.size())); + + ArrayList only_member_ids = new ArrayList(); + + for (PatreonUser pu : users) { + only_member_ids.add(pu.userId); + } + + // Step 1 - Clear out old users + DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); + + + Result records_to_purge = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.notIn(only_member_ids)) + .fetch(); + + for (PatreonSupportersRecord psr : records_to_purge) { + Long database_discord_id = psr.getDiscordId(); + + // Store our removal task if necessary + if (database_discord_id != null && (psr.getDonationTier() >= 2)) { + TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("REMOVE_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = database_discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbcore.now()); + tqr.store(); + } + + // Clear out the supporter record + psr.delete(); + } + + logger.info("[PatreonSync] Cleaned out old users"); + + int insert_count = 0; + int update_count = 0; + + // Step 2 - Add or insert new + for (PatreonUser pu : users) { + if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)))) { + + update_count++; + + // Fetch and update our record + PatreonSupportersRecord psr = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)) + .fetchSingle(); + + psr.setAmountCents(pu.amountInCents); + psr.setDonationTier(pu.maxTier().tierLevel); + + // And store + psr.store(); + } else { + insert_count++; + + // Make a new record + PatreonSupportersRecord psr = automation_jooq.newRecord(Tables.PATREON_SUPPORTERS); + psr.setMemberId(pu.userId); + psr.setAmountCents(pu.amountInCents); + psr.setDonationTier(pu.maxTier().tierLevel); + + // And store + psr.store(); + } + } + + logger.info(String.format("[PatreonSync] Inserted %s users, updated %s users", insert_count, update_count)); + + + DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); + DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); + + // Find authentik users + HashMap authentik_to_patreon = new HashMap(); + findAuthentikUsers(only_member_ids, authentik_jooq, automation_jooq, logger, authentik_to_patreon); + + // Now find discord users + findDiscordUsers(authentik_jooq, automation_jooq, logger, authentik_to_patreon, dbcore); + + // Now get ckeys + findCkeys(authentik_jooq, automation_jooq, logger, authentik_to_patreon); + + // And commit to game DB + updateGameDb(automation_jooq, game_jooq, logger); + + System.gc(); // This is important after this one - the response model is like multiple megabytes + logger.info("[PatreonSync] Process complete"); + } + + private static int patreonTier2ingameTier(int patreonTier) { + switch (patreonTier) { + case 0: // No tier + return 0; + case 1: // Supporter + return 0; + case 2: // Icon + return 1; + case 3: // Loadout + return 2; + case 4: // Silver + return 3; + case 5: // Gold + return 4; + default: + return 0; + } + } + + private static void findAuthentikUsers( + List memberIds, + DSLContext authentikJooq, + DSLContext automationJooq, + Logger logger, + HashMap authentik2patreon + ) { + for (int member_id : memberIds) { + // This loop should only have active member IDs for the DB given we just flushed everything else out + + // Get ready for a TON of absolute pathing + AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.IDENTIFIER.eq(String.valueOf(member_id))) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.PATREON_OAUTH_SERVICE_ID)) + .fetchOne(); + + if (authentik_record == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in Authentik!", member_id)); + continue; + } + + authentik2patreon.put(authentik_record.getUserId(), member_id); + } + + logger.info(String.format("[PatreonSync] Loaded %s Authentik users", authentik2patreon.size())); + + int supporters_authentik_updated = 0; + for (int authentik_id : authentik2patreon.keySet()) { + int patreon_id = authentik2patreon.get(authentik_id); + + // Get our DB record + + PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) + .fetchOne(); + + if (psr == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); + continue; + } + + supporters_authentik_updated++; + psr.setAuthentikId(authentik_id); + psr.store(); // Store it back in + } + + logger.info(String.format("[PatreonSync] Updated %s rows with Authentik users", supporters_authentik_updated)); + } + + private static void findDiscordUsers( + DSLContext authentikJooq, + DSLContext automationJooq, + Logger logger, + HashMap authentik2patreon, + DbCore dbCore + ) { + HashMap authentik2discord = new HashMap(); + + for (int authentik_id : authentik2patreon.keySet()) { + // This loop should only have active member IDs for the DB given we just flushed everything else out + + // Get ready for a TON of absolute pathing + AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) + .fetchOne(); + + if (authentik_record == null) { + logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); + continue; + } + + authentik2discord.put(authentik_id, Long.valueOf(authentik_record.getIdentifier())); + } + + logger.info(String.format("[PatreonSync] Loaded %s Discord users", authentik2discord.size())); + + int supporters_updated = 0; + for (int authentik_id : authentik2discord.keySet()) { + if (!authentik2patreon.containsKey(authentik_id)) { + logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); + } + + int patreon_id = authentik2patreon.get(authentik_id); + long discord_id = authentik2discord.get(authentik_id); + + // Get our DB record + PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) + .fetchOne(); + + if (psr == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); + continue; + } + + // See if we have a role to add + if (psr.getDiscordId() == null && psr.getDonationTier() >= 2) { + // We do - lets add the role + TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("ADD_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbCore.now()); + tqr.store(); + } + + // See if their tier changed between runs + if (psr.getDiscordId() != null && (psr.getLastrunDonationTier().intValue() != psr.getDonationTier().intValue())) { + // Tier changed - see if they went below the threshold + if (psr.getDonationTier() <= 1) { + // They went below the threshold - strip the role + TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("REMOVE_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbCore.now()); + tqr.store(); + } + + // Add if we just upgraded + if (psr.getDonationTier() >= 2 && psr.getLastrunDonationTier() == 1) { + TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("ADD_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbCore.now()); + tqr.store(); + } + } + + supporters_updated++; + psr.setLastrunDonationTier(psr.getDonationTier()); + psr.setDiscordId(discord_id); + psr.store(); // Store it back in + } + + logger.info(String.format("[PatreonSync] Updated %s rows with Discord users", supporters_updated)); + } + + private static void findCkeys( + DSLContext authentikJooq, + DSLContext automationJooq, + Logger logger, + HashMap authentik2patreon + ) { + HashMap authentik2ckey = new HashMap(); + + for (int authentik_id : authentik2patreon.keySet()) { + // This loop should only have active member IDs for the DB given we just flushed everything else out + + // Get ready for a TON of absolute pathing + AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) + .fetchOne(); + + if (authentik_record == null) { + logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); + continue; + } + + // Normalise the ckey + String ckey = UtilStr.cleanCkey(authentik_record.getIdentifier()); + + authentik2ckey.put(authentik_id, ckey); + } + + logger.info(String.format("[PatreonSync] Loaded %s ckeys", authentik2ckey.size())); + + int supporters_updated = 0; + for (int authentik_id : authentik2ckey.keySet()) { + if (!authentik2patreon.containsKey(authentik_id)) { + logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); + } + + int patreon_id = authentik2patreon.get(authentik_id); + String ckey = authentik2ckey.get(authentik_id); + + // Get our DB record + PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) + .fetchOne(); + + if (psr == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); + continue; + } + + supporters_updated++; + psr.setCkey(ckey); + psr.store(); // Store it back in + } + + logger.info(String.format("[PatreonSync] Updated %s rows with ckeys", supporters_updated)); + } + + private static void updateGameDb(DSLContext automationJooq, DSLContext gameJooq, Logger logger) { + + Result valid_records = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.CKEY.isNotNull()) + .and(Tables.PATREON_SUPPORTERS.DONATION_TIER.ge(2)) // Tier 2 or higher to appear ingame + .fetch(); + + logger.info(String.format("[PatreonSync] Loaded %s records to update in the Game DB", valid_records.size())); + + ArrayList only_ckeys = new ArrayList(); + + for (PatreonSupportersRecord psr : valid_records) { + only_ckeys.add(psr.getCkey()); + } + + // Remove from BYOND where not in this list + gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.notIn(only_ckeys)) + .execute(); + + // And where not active to keep the table short + gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.ACTIVE.ne((byte) 0x01)) // tinyint is a byte + .execute(); + + int insert_count = 0; + int update_count = 0; + // Now do our record stuff + for (PatreonSupportersRecord psr : valid_records) { + DonatorsRecord dr = gameJooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.eq(psr.getCkey())) + .fetchOne(); + + if (dr == null) { + // Empty record - make a new one + DonatorsRecord dr2 = gameJooq.newRecord(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS); + dr2.setPatreonName(psr.getMemberId().toString()); + dr2.setTier(patreonTier2ingameTier(psr.getDonationTier())); + dr2.setCkey(psr.getCkey()); + dr2.setStartDate(null); + dr2.setEndDate(null); + dr2.setActive((byte) 0x01); + + dr2.store(); + + insert_count++; + } else { + // Existing record - update + dr.setTier(patreonTier2ingameTier(psr.getDonationTier())); + dr.setCkey(psr.getCkey()); + dr.setStartDate(null); + dr.setEndDate(null); + dr.setActive((byte) 0x01); + + dr.store(); + + update_count++; + } + } + + logger.info(String.format("[PatreonSync] Inserted %s records, updated %s records in the game DB.", insert_count, update_count)); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java index ec51abf..1c8c988 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java @@ -1,252 +1,252 @@ -package me.aa07.paradise.taskdaemon.core.modules.prlabel; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.GithubConfig; -import me.aa07.paradise.taskdaemon.core.config.TgsConfig; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.prlabel.PullRequest; -import me.aa07.paradise.taskdaemon.database.pullrequests.Tables; -import me.aa07.paradise.taskdaemon.database.pullrequests.enums.PrsPrStatus; -import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVoteType; -import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVotingGroup; -import me.aa07.paradise.taskdaemon.database.pullrequests.tables.records.PrsRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Record2; -import org.jooq.Record4; -import org.jooq.Result; -import org.jooq.impl.DSL; -import org.kohsuke.github.GHIssueState; -import org.kohsuke.github.GHLabel; -import org.kohsuke.github.GHPullRequest; -import org.kohsuke.github.GHRepository; -import org.kohsuke.github.GitHub; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class PrLabelJob implements Job { - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - // INITIAL SETUP - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[PrLabelJob] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[PrLabelJob] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Now get our config - Object raw_github_config = datamap.get("GITHUBCFG"); - Optional github_cfg_holder = Optional.empty(); - - if (raw_github_config instanceof GithubConfig githubCfg2) { - github_cfg_holder = Optional.of(githubCfg2); - } - - if (!github_cfg_holder.isPresent()) { - logger.error("[PrLabelJob] GITHUBCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - GithubConfig github_config = github_cfg_holder.get(); - - // Now get our other config - Object raw_tgs_config = datamap.get("TGSCFG"); - Optional tgs_cfg_holder = Optional.empty(); - - if (raw_tgs_config instanceof TgsConfig tgsCfg2) { - tgs_cfg_holder = Optional.of(tgsCfg2); - } - - if (!tgs_cfg_holder.isPresent()) { - logger.error("[PrLabelJob] TGSCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - TgsConfig tgs_cfg = tgs_cfg_holder.get(); - - // Initial setup is done - lets now load the important bits - - // Get this now - we need it - DSLContext ctx = dbcore.jooq(DatabaseType.PullRequests); - - // Dictionary that holds all this in a nice lookup - HashMap prs = new HashMap(); - - // And init these - // TM manager - TestmergeManager tmm = new TestmergeManager(tgs_cfg, logger); - tmm.setup(); - - // Files changed manager - FilesChangedManager fcm = new FilesChangedManager(logger); - - // Vote status manager - StatusManager sm = new StatusManager(logger, github_config); - // We set this up later to contain the exception - - // Put everything in a list to aid iteration - ArrayList processors = new ArrayList(); - processors.add(tmm); - processors.add(fcm); - processors.add(sm); - - logger.info(String.format("[PrLabelJob] Loaded %s active TMs from TGS - now loading PRs from GitHub", - tmm.loadedActiveTms())); - - // Evertything in the GH client throws IOException so we need to baby proof that - try { - // Set up the review team - sm.setup(); - - // Now set up the rest - GitHub gh = GitHub.connectUsingOAuth(github_config.githubToken); - GHRepository repo = gh.getRepositoryById(github_config.repoId); - - // Get our open PRs - List open_prs = repo.queryPullRequests().state(GHIssueState.OPEN).list().toList(); - - // And parse them into that array - for (GHPullRequest pr : open_prs) { - PullRequest internal_pr = new PullRequest(); - internal_pr.prObject = pr; - internal_pr.pullNumber = pr.getNumber(); - - for (GHLabel label : pr.getLabels()) { - internal_pr.labels.add(label.getName()); - } - - prs.put(internal_pr.pullNumber, internal_pr); - } - - logger.info(String.format("[PrLabelJob] Loaded %s PRs from GitHub - now loading from DB", prs.size())); - - // Now get the data from the DB, just the PR metadata - Result db_prs = ctx.selectFrom(Tables.PRS).where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) - .fetch(); - - for (PrsRecord db_pr : db_prs) { - int pr_num = db_pr.getPrNumber().intValue(); - if (prs.containsKey(pr_num)) { - // Set values up - PullRequest pr = prs.get(pr_num); - pr.prTypes = db_pr.getPrTypes(); - } else { - logger.warn(String.format("[PrLabelJob] PR #%s in DB but not open!", pr_num)); - } - } - - // Now the testmerge requests - logger.info(String.format("[PrLabelJob] Processed %s DB PRs - now loading TMRs", prs.size())); - - Result> db_tmrs = ctx.select(Tables.TM_REQUESTS.PR_NUMBER, DSL.count()) - .from(Tables.TM_REQUESTS) - .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.TM_REQUESTS.PR_NUMBER)) - .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) - .groupBy(Tables.TM_REQUESTS.PR_NUMBER) - .fetch(); - - for (Record2 db_tmr_rec : db_tmrs) { - int pr_num = db_tmr_rec.value1().intValue(); - int tmr_count = db_tmr_rec.value2().intValue(); - - if (prs.containsKey(pr_num)) { - // Set values up - PullRequest pr = prs.get(pr_num); - pr.testmergeRequests = tmr_count; - } - } - - // Now the PR votes - logger.info(String.format("[PrLabelJob] Processed %s DB TMRs - now loading votes", db_tmrs.size())); - - Result> db_votes = ctx.select( - Tables.VOTES_NEW.PR_NUMBER, - Tables.VOTES_NEW.VOTING_GROUP, - Tables.VOTES_NEW.VOTE_TYPE, - DSL.count()).from(Tables.VOTES_NEW) - .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.VOTES_NEW.PR_NUMBER)) - .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) - .groupBy(Tables.VOTES_NEW.PR_NUMBER, Tables.VOTES_NEW.VOTING_GROUP, Tables.VOTES_NEW.VOTE_TYPE) - .fetch(); - - for (Record4 db_vote_rec : db_votes) { - int pr_num = db_vote_rec.value1().intValue(); - VotesNewVotingGroup vg = db_vote_rec.value2(); - VotesNewVoteType vt = db_vote_rec.value3(); - int vote_count = db_vote_rec.value4().intValue(); - - if (prs.containsKey(pr_num)) { - // Set values up - PullRequest pr = prs.get(pr_num); - - if (!pr.votes.containsKey(vg)) { - pr.votes.put(vg, new HashMap()); - } - - // No check needed - this is fine to add or replace - HashMap pr_votes_map = pr.votes.get(vg); - pr_votes_map.put(vt, vote_count); - } - } - - // Now we have the votes we can actually parse the PRs - logger.info( - String.format("[PrLabelJob] Processed %s DB votes sets - now doing the main thing", - db_votes.size())); - - for (int pr_number : prs.keySet()) { - logger.info(String.format("[PrLabelJob] Processing #%s", pr_number)); - PullRequest pr_object = prs.get(pr_number); - - // Handle on each processor - for (IPullRequestProcessor processor : processors) { - processor.processPr(pr_object); - } - } - - logger.info(String.format("[PrLabelJob] Done", db_votes.size())); - - } catch (IOException e) { - logger.error("[PrLabelJob] Failed due to an IOException"); - logger.error(e); - return; - } - } - -} +package me.aa07.paradise.taskdaemon.core.modules.prlabel; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.GithubConfig; +import me.aa07.paradise.taskdaemon.core.config.TgsConfig; +import me.aa07.paradise.taskdaemon.core.models.prlabel.PullRequest; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.pullrequests.Tables; +import me.aa07.paradise.taskdaemon.database.pullrequests.enums.PrsPrStatus; +import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVoteType; +import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVotingGroup; +import me.aa07.paradise.taskdaemon.database.pullrequests.tables.records.PrsRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Record2; +import org.jooq.Record4; +import org.jooq.Result; +import org.jooq.impl.DSL; +import org.kohsuke.github.GHIssueState; +import org.kohsuke.github.GHLabel; +import org.kohsuke.github.GHPullRequest; +import org.kohsuke.github.GHRepository; +import org.kohsuke.github.GitHub; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class PrLabelJob implements Job { + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + // INITIAL SETUP + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[PrLabelJob] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[PrLabelJob] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Now get our config + Object raw_github_config = datamap.get("GITHUBCFG"); + Optional github_cfg_holder = Optional.empty(); + + if (raw_github_config instanceof GithubConfig githubCfg2) { + github_cfg_holder = Optional.of(githubCfg2); + } + + if (!github_cfg_holder.isPresent()) { + logger.error("[PrLabelJob] GITHUBCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + GithubConfig github_config = github_cfg_holder.get(); + + // Now get our other config + Object raw_tgs_config = datamap.get("TGSCFG"); + Optional tgs_cfg_holder = Optional.empty(); + + if (raw_tgs_config instanceof TgsConfig tgsCfg2) { + tgs_cfg_holder = Optional.of(tgsCfg2); + } + + if (!tgs_cfg_holder.isPresent()) { + logger.error("[PrLabelJob] TGSCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + TgsConfig tgs_cfg = tgs_cfg_holder.get(); + + // Initial setup is done - lets now load the important bits + + // Get this now - we need it + DSLContext ctx = dbcore.jooq(DatabaseType.PullRequests); + + // Dictionary that holds all this in a nice lookup + HashMap prs = new HashMap(); + + // And init these + // TM manager + TestmergeManager tmm = new TestmergeManager(tgs_cfg, logger); + tmm.setup(); + + // Files changed manager + FilesChangedManager fcm = new FilesChangedManager(logger); + + // Vote status manager + StatusManager sm = new StatusManager(logger, github_config); + // We set this up later to contain the exception + + // Put everything in a list to aid iteration + ArrayList processors = new ArrayList(); + processors.add(tmm); + processors.add(fcm); + processors.add(sm); + + logger.info(String.format("[PrLabelJob] Loaded %s active TMs from TGS - now loading PRs from GitHub", + tmm.loadedActiveTms())); + + // Evertything in the GH client throws IOException so we need to baby proof that + try { + // Set up the review team + sm.setup(); + + // Now set up the rest + GitHub gh = GitHub.connectUsingOAuth(github_config.githubToken); + GHRepository repo = gh.getRepositoryById(github_config.repoId); + + // Get our open PRs + List open_prs = repo.queryPullRequests().state(GHIssueState.OPEN).list().toList(); + + // And parse them into that array + for (GHPullRequest pr : open_prs) { + PullRequest internal_pr = new PullRequest(); + internal_pr.prObject = pr; + internal_pr.pullNumber = pr.getNumber(); + + for (GHLabel label : pr.getLabels()) { + internal_pr.labels.add(label.getName()); + } + + prs.put(internal_pr.pullNumber, internal_pr); + } + + logger.info(String.format("[PrLabelJob] Loaded %s PRs from GitHub - now loading from DB", prs.size())); + + // Now get the data from the DB, just the PR metadata + Result db_prs = ctx.selectFrom(Tables.PRS).where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) + .fetch(); + + for (PrsRecord db_pr : db_prs) { + int pr_num = db_pr.getPrNumber().intValue(); + if (prs.containsKey(pr_num)) { + // Set values up + PullRequest pr = prs.get(pr_num); + pr.prTypes = db_pr.getPrTypes(); + } else { + logger.warn(String.format("[PrLabelJob] PR #%s in DB but not open!", pr_num)); + } + } + + // Now the testmerge requests + logger.info(String.format("[PrLabelJob] Processed %s DB PRs - now loading TMRs", prs.size())); + + Result> db_tmrs = ctx.select(Tables.TM_REQUESTS.PR_NUMBER, DSL.count()) + .from(Tables.TM_REQUESTS) + .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.TM_REQUESTS.PR_NUMBER)) + .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) + .groupBy(Tables.TM_REQUESTS.PR_NUMBER) + .fetch(); + + for (Record2 db_tmr_rec : db_tmrs) { + int pr_num = db_tmr_rec.value1().intValue(); + int tmr_count = db_tmr_rec.value2().intValue(); + + if (prs.containsKey(pr_num)) { + // Set values up + PullRequest pr = prs.get(pr_num); + pr.testmergeRequests = tmr_count; + } + } + + // Now the PR votes + logger.info(String.format("[PrLabelJob] Processed %s DB TMRs - now loading votes", db_tmrs.size())); + + Result> db_votes = ctx.select( + Tables.VOTES_NEW.PR_NUMBER, + Tables.VOTES_NEW.VOTING_GROUP, + Tables.VOTES_NEW.VOTE_TYPE, + DSL.count()).from(Tables.VOTES_NEW) + .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.VOTES_NEW.PR_NUMBER)) + .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) + .groupBy(Tables.VOTES_NEW.PR_NUMBER, Tables.VOTES_NEW.VOTING_GROUP, Tables.VOTES_NEW.VOTE_TYPE) + .fetch(); + + for (Record4 db_vote_rec : db_votes) { + int pr_num = db_vote_rec.value1().intValue(); + VotesNewVotingGroup vg = db_vote_rec.value2(); + VotesNewVoteType vt = db_vote_rec.value3(); + int vote_count = db_vote_rec.value4().intValue(); + + if (prs.containsKey(pr_num)) { + // Set values up + PullRequest pr = prs.get(pr_num); + + if (!pr.votes.containsKey(vg)) { + pr.votes.put(vg, new HashMap()); + } + + // No check needed - this is fine to add or replace + HashMap pr_votes_map = pr.votes.get(vg); + pr_votes_map.put(vt, vote_count); + } + } + + // Now we have the votes we can actually parse the PRs + logger.info( + String.format("[PrLabelJob] Processed %s DB votes sets - now doing the main thing", + db_votes.size())); + + for (int pr_number : prs.keySet()) { + logger.info(String.format("[PrLabelJob] Processing #%s", pr_number)); + PullRequest pr_object = prs.get(pr_number); + + // Handle on each processor + for (IPullRequestProcessor processor : processors) { + processor.processPr(pr_object); + } + } + + logger.info(String.format("[PrLabelJob] Done", db_votes.size())); + + } catch (IOException e) { + logger.error("[PrLabelJob] Failed due to an IOException"); + logger.error(e); + return; + } + } + +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java index 2ac90d8..e862281 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java @@ -1,61 +1,61 @@ -package me.aa07.paradise.taskdaemon.core.modules.profilercleanup; - -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.database.profiler.Tables; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class ProfilerCleanupJob implements Job { - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[ProfilerCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[ProfilerCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - DSLContext profiler_db = dbcore.jooq(DatabaseType.ProfilerDb); - - logger.info("[ProfilerCleanup] Cleaning out profiler DB - proc samples"); - profiler_db.delete(Tables.SAMPLES).where(Tables.SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); - logger.info("[ProfilerCleanup] Cleaning out profiler DB - sendmaps samples"); - profiler_db.delete(Tables.SENDMAPS_SAMPLES).where(Tables.SENDMAPS_SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); - logger.info("[ProfilerCleanup] Cleaned"); - } - -} +package me.aa07.paradise.taskdaemon.core.modules.profilercleanup; + +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.profiler.Tables; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class ProfilerCleanupJob implements Job { + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[ProfilerCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[ProfilerCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + DSLContext profiler_db = dbcore.jooq(DatabaseType.ProfilerDb); + + logger.info("[ProfilerCleanup] Cleaning out profiler DB - proc samples"); + profiler_db.delete(Tables.SAMPLES).where(Tables.SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); + logger.info("[ProfilerCleanup] Cleaning out profiler DB - sendmaps samples"); + profiler_db.delete(Tables.SENDMAPS_SAMPLES).where(Tables.SENDMAPS_SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); + logger.info("[ProfilerCleanup] Cleaned"); + } + +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java index 158281b..6b12b7b 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java @@ -1,85 +1,85 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest; - -import java.util.ArrayList; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerProcProcessor; -import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerSendmapsProcessor; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import org.apache.logging.log4j.Logger; - -public class ProfilerWorker { - private ArrayList queue; - private Logger logger; - private Object listLock; - - private ProfilerProcProcessor procProcessor; - private ProfilerSendmapsProcessor sendmapsProcessor; - - public ProfilerWorker(DbCore database, Logger logger) { - this.logger = logger; - - procProcessor = new ProfilerProcProcessor(database, logger); - sendmapsProcessor = new ProfilerSendmapsProcessor(database, logger); - - queue = new ArrayList(); - listLock = new Object(); - } - - // TODO - Refactor into quartz job - public void run() { - logger.info("Starting..."); - - try { - while (true) { - String current_run = safePop(); - if (current_run == null) { - Thread.sleep(10000); - continue; - } - processEntry(current_run); - - } - } catch (InterruptedException exception) { - logger.fatal("Worker thread was interrupted!"); - exception.printStackTrace(); - System.exit(1); - } - } - - // Add something to the list in a thread-safe matter - public void safeAdd(String data) { - synchronized (listLock) { - queue.add(data); - } - } - - // Extract one item from the list in a thread-safe matter - private String safePop() { - String output = ""; - synchronized (listLock) { - if (queue.size() > 0) { - output = queue.get(0); - queue.remove(0); - } else { - return null; - } - } - - return output; - } - - private void processEntry(String entry) { - long start = System.currentTimeMillis(); - logger.info(String.format("Processing entry with size %s bytes", entry.length())); - - ProfilerHolder holder = UtilConst.GSON.fromJson(entry, ProfilerHolder.class); - logger.info(String.format("Round ID: %s | Profile size: %s bytes", holder.roundId, holder.profilerData.length())); - - procProcessor.processData(holder); - sendmapsProcessor.processData(holder); - - long duration = System.currentTimeMillis() - start; - logger.info(String.format("All processing complete within %sms", duration)); - } -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest; + +import java.util.ArrayList; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerProcProcessor; +import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerSendmapsProcessor; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import org.apache.logging.log4j.Logger; + +public class ProfilerWorker { + private ArrayList queue; + private Logger logger; + private Object listLock; + + private ProfilerProcProcessor procProcessor; + private ProfilerSendmapsProcessor sendmapsProcessor; + + public ProfilerWorker(DbCore database, Logger logger) { + this.logger = logger; + + procProcessor = new ProfilerProcProcessor(database, logger); + sendmapsProcessor = new ProfilerSendmapsProcessor(database, logger); + + queue = new ArrayList(); + listLock = new Object(); + } + + // TODO - Refactor into quartz job + public void run() { + logger.info("Starting..."); + + try { + while (true) { + String current_run = safePop(); + if (current_run == null) { + Thread.sleep(10000); + continue; + } + processEntry(current_run); + + } + } catch (InterruptedException exception) { + logger.fatal("Worker thread was interrupted!"); + exception.printStackTrace(); + System.exit(1); + } + } + + // Add something to the list in a thread-safe matter + public void safeAdd(String data) { + synchronized (listLock) { + queue.add(data); + } + } + + // Extract one item from the list in a thread-safe matter + private String safePop() { + String output = ""; + synchronized (listLock) { + if (queue.size() > 0) { + output = queue.get(0); + queue.remove(0); + } else { + return null; + } + } + + return output; + } + + private void processEntry(String entry) { + long start = System.currentTimeMillis(); + logger.info(String.format("Processing entry with size %s bytes", entry.length())); + + ProfilerHolder holder = UtilConst.GSON.fromJson(entry, ProfilerHolder.class); + logger.info(String.format("Round ID: %s | Profile size: %s bytes", holder.roundId, holder.profilerData.length())); + + procProcessor.processData(holder); + sendmapsProcessor.processData(holder); + + long duration = System.currentTimeMillis() - start; + logger.info(String.format("All processing complete within %sms", duration)); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java index f009224..237b5fc 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java @@ -1,33 +1,33 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; - -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import org.apache.logging.log4j.Logger; - -public abstract class ProfilerBaseProcessor { - protected DbCore database; - protected Logger logger; - private String processorName; - - public ProfilerBaseProcessor(DbCore database, Logger logger, String processorName) { - this.database = database; - this.logger = logger; - this.processorName = processorName; - } - - protected void log(String txt) { - logger.info(String.format("[%s] %s", processorName, txt)); - } - - public void processData(ProfilerHolder holder) { - long start = System.currentTimeMillis(); - log("Starting processing"); - doProcessing(holder); - long duration = System.currentTimeMillis() - start; - log(String.format("Processing complete within %sms", duration)); - } - - protected abstract void doProcessing(ProfilerHolder holder); - - protected abstract long getProcId(String procname); -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; + +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import org.apache.logging.log4j.Logger; + +public abstract class ProfilerBaseProcessor { + protected DbCore database; + protected Logger logger; + private String processorName; + + public ProfilerBaseProcessor(DbCore database, Logger logger, String processorName) { + this.database = database; + this.logger = logger; + this.processorName = processorName; + } + + protected void log(String txt) { + logger.info(String.format("[%s] %s", processorName, txt)); + } + + public void processData(ProfilerHolder holder) { + long start = System.currentTimeMillis(); + log("Starting processing"); + doProcessing(holder); + long duration = System.currentTimeMillis() - start; + log(String.format("Processing complete within %sms", duration)); + } + + protected abstract void doProcessing(ProfilerHolder holder); + + protected abstract long getProcId(String procname); +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java index 22a7da8..5dfabfd 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java @@ -1,62 +1,62 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; - -import com.google.gson.reflect.TypeToken; -import java.util.List; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProcData; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.database.profiler.Tables; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.ProcsRecord; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SamplesRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; - -public class ProfilerProcProcessor extends ProfilerBaseProcessor { - public ProfilerProcProcessor(DbCore database, Logger logger) { - super(database, logger, "ProcProcessor"); - } - - @Override - protected void doProcessing(ProfilerHolder holder) { - List proc_list = UtilConst.GSON.fromJson( - holder.profilerData, new TypeToken>() {}.getType() - ); - - log(String.format("Procs logged: %s", proc_list.size())); - - DSLContext jooq = database.jooq(DatabaseType.ProfilerDb); - - for (ProcData procdata : proc_list) { - long proc_id = getProcId(procdata.name); - SamplesRecord sr = jooq.newRecord(Tables.SAMPLES); - sr.setRoundid(holder.roundId); - sr.setSampletime(database.now()); - sr.setProcid(proc_id); - sr.setSelf(procdata.self); - sr.setTotal(procdata.total); - sr.setReal(procdata.real); - sr.setOver(procdata.over); - sr.setCalls(procdata.calls); - sr.store(); - } - } - - @Override - protected long getProcId(String procname) { - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); - if (!ctx.fetchExists(ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)))) { - // We dont exist, make us - log(String.format("%s did not exist in the DB. It does now.", procname)); - - ProcsRecord record = ctx.newRecord(Tables.PROCS); - record.setProcpath(procname); - record.store(); - return record.getId(); - } - - // We do exist, just grab - return ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)).fetchOne().value1(); - } -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; + +import com.google.gson.reflect.TypeToken; +import java.util.List; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProcData; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.database.profiler.Tables; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.ProcsRecord; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SamplesRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; + +public class ProfilerProcProcessor extends ProfilerBaseProcessor { + public ProfilerProcProcessor(DbCore database, Logger logger) { + super(database, logger, "ProcProcessor"); + } + + @Override + protected void doProcessing(ProfilerHolder holder) { + List proc_list = UtilConst.GSON.fromJson( + holder.profilerData, new TypeToken>() {}.getType() + ); + + log(String.format("Procs logged: %s", proc_list.size())); + + DSLContext jooq = database.jooq(DatabaseType.ProfilerDb); + + for (ProcData procdata : proc_list) { + long proc_id = getProcId(procdata.name); + SamplesRecord sr = jooq.newRecord(Tables.SAMPLES); + sr.setRoundid(holder.roundId); + sr.setSampletime(database.now()); + sr.setProcid(proc_id); + sr.setSelf(procdata.self); + sr.setTotal(procdata.total); + sr.setReal(procdata.real); + sr.setOver(procdata.over); + sr.setCalls(procdata.calls); + sr.store(); + } + } + + @Override + protected long getProcId(String procname) { + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); + if (!ctx.fetchExists(ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)))) { + // We dont exist, make us + log(String.format("%s did not exist in the DB. It does now.", procname)); + + ProcsRecord record = ctx.newRecord(Tables.PROCS); + record.setProcpath(procname); + record.store(); + return record.getId(); + } + + // We do exist, just grab + return ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)).fetchOne().value1(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java index 9f7f867..0103491 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java @@ -1,59 +1,59 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; - -import com.google.gson.reflect.TypeToken; -import java.util.List; -import me.aa07.paradise.taskdaemon.core.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.database.DbCore; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.models.profiler.SendmapsProcData; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.database.profiler.Tables; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsProcsRecord; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsSamplesRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; - -public class ProfilerSendmapsProcessor extends ProfilerBaseProcessor { - public ProfilerSendmapsProcessor(DbCore database, Logger logger) { - super(database, logger, "SendmapsProcessor"); - } - - @Override - protected void doProcessing(ProfilerHolder holder) { - List proc_list = UtilConst.GSON.fromJson( - holder.sendmapsData, new TypeToken>() {}.getType() - ); - - log(String.format("Procs logged: %s", proc_list.size())); - - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); - - for (SendmapsProcData procdata : proc_list) { - long proc_id = getProcId(procdata.name); - SendmapsSamplesRecord sr = ctx.newRecord(Tables.SENDMAPS_SAMPLES); - sr.setRoundid(holder.roundId); - sr.setSampletime(database.now()); - sr.setProcid(proc_id); - sr.setValue(procdata.value); - sr.setCalls(procdata.calls); - sr.store(); - } - } - - @Override - protected long getProcId(String procname) { - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); - if (!ctx.fetchExists(ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)))) { - // We dont exist, make us - log(String.format("%s did not exist in the DB. It does now.", procname)); - - SendmapsProcsRecord record = ctx.newRecord(Tables.SENDMAPS_PROCS); - record.setProcpath(procname); - record.store(); - return record.getId(); - } - - // We do exist, just grab - return ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)).fetchOne().value1(); - } -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; + +import com.google.gson.reflect.TypeToken; +import java.util.List; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.models.profiler.SendmapsProcData; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.database.profiler.Tables; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsProcsRecord; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsSamplesRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; + +public class ProfilerSendmapsProcessor extends ProfilerBaseProcessor { + public ProfilerSendmapsProcessor(DbCore database, Logger logger) { + super(database, logger, "SendmapsProcessor"); + } + + @Override + protected void doProcessing(ProfilerHolder holder) { + List proc_list = UtilConst.GSON.fromJson( + holder.sendmapsData, new TypeToken>() {}.getType() + ); + + log(String.format("Procs logged: %s", proc_list.size())); + + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); + + for (SendmapsProcData procdata : proc_list) { + long proc_id = getProcId(procdata.name); + SendmapsSamplesRecord sr = ctx.newRecord(Tables.SENDMAPS_SAMPLES); + sr.setRoundid(holder.roundId); + sr.setSampletime(database.now()); + sr.setProcid(proc_id); + sr.setValue(procdata.value); + sr.setCalls(procdata.calls); + sr.store(); + } + } + + @Override + protected long getProcId(String procname) { + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); + if (!ctx.fetchExists(ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)))) { + // We dont exist, make us + log(String.format("%s did not exist in the DB. It does now.", procname)); + + SendmapsProcsRecord record = ctx.newRecord(Tables.SENDMAPS_PROCS); + record.setProcpath(procname); + record.store(); + return record.getId(); + } + + // We do exist, just grab + return ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)).fetchOne().value1(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DatabaseType.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java similarity index 60% rename from TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DatabaseType.java rename to TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java index f58a88e..8e038d8 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DatabaseType.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java @@ -1,5 +1,5 @@ -package me.aa07.paradise.taskdaemon.core.database; - -public enum DatabaseType { - Authentik, Automation, GameDb, Forums, ProfilerDb, PullRequests -} +package me.aa07.paradise.taskdaemon.core.services.database; + +public enum DatabaseType { + Authentik, Automation, GameDb, Forums, ProfilerDb, PullRequests +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java similarity index 96% rename from TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java rename to TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java index 08414ea..90d0ac2 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/database/DbCore.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java @@ -1,108 +1,108 @@ -package me.aa07.paradise.taskdaemon.core.database; - -import java.sql.Connection; -import java.sql.Timestamp; -import java.time.LocalDateTime; -import java.util.Date; -import java.util.HashMap; -import javax.sql.DataSource; -import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; -import me.aa07.paradise.taskdaemon.core.config.DatabaseConfig; -import org.apache.commons.dbcp2.BasicDataSource; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.exception.NoDataFoundException; -import org.jooq.impl.DSL; - -public class DbCore { - private HashMap> connectionMap; - - public DbCore(ConfigHolder config, Logger logger) { - // Suppress JOOQ console spam - System.getProperties().setProperty("org.jooq.no-logo", "true"); - System.getProperties().setProperty("org.jooq.no-tips", "true"); - - connectionMap = new HashMap>(); - - establishConnections(config); - logger.info("Ready to handle DB requests"); - } - - private DataSource openMySqlDataSource(String url, String username, String password) { - BasicDataSource source = new BasicDataSource(); - source.addConnectionProperty("autoReconnect", "true"); - source.addConnectionProperty("allowMultiQueries", "true"); - source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); - source.addConnectionProperty("connectionTimeZone", "UTC"); - source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); - source.setDriverClassName("com.mysql.cj.jdbc.Driver"); - source.setUrl(url); - source.setUsername(username); - source.setPassword(password); - source.setMaxTotal(8); - source.setMaxIdle(8); - source.setTimeBetweenEvictionRunsMillis(180 * 1000); - source.setSoftMinEvictableIdleTimeMillis(180 * 1000); - - return source; - } - - private DataSource openPostgresDataSource(String url, String username, String password) { - BasicDataSource source = new BasicDataSource(); - source.addConnectionProperty("autoReconnect", "true"); - source.addConnectionProperty("allowMultiQueries", "true"); - source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); - source.addConnectionProperty("connectionTimeZone", "UTC"); - source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); - source.setDriverClassName("org.postgresql.Driver"); - source.setUrl(url); - source.setUsername(username); - source.setPassword(password); - source.setMaxTotal(8); - source.setMaxIdle(8); - source.setTimeBetweenEvictionRunsMillis(180 * 1000); - source.setSoftMinEvictableIdleTimeMillis(180 * 1000); - - return source; - } - - private void establishConnections(ConfigHolder config) { - HashMap mysql_db_types = new HashMap(); - - mysql_db_types.put(DatabaseType.Automation, config.automationDatabase); - mysql_db_types.put(DatabaseType.Forums, config.forumsDatabase); - mysql_db_types.put(DatabaseType.GameDb, config.gameDatabase); - mysql_db_types.put(DatabaseType.ProfilerDb, config.profilerDatabase); - mysql_db_types.put(DatabaseType.PullRequests, config.pullRequestsDatabase); - - for (DatabaseType dbtype : mysql_db_types.keySet()) { - DatabaseConfig cfg = mysql_db_types.get(dbtype); - DataSource ds = openMySqlDataSource(String.format("jdbc:mysql://%s/%s", cfg.host, cfg.database), cfg.username, cfg.password); - connectionMap.put(dbtype, Pair.of(ds, SQLDialect.MYSQL)); - } - - // And do special stuff for Postgres - DatabaseConfig adb = config.authentikDatabase; - DataSource authentik_ds = openPostgresDataSource(String.format("jdbc:postgresql://%s/%s", adb.host, adb.database), adb.username, adb.password); - connectionMap.put(DatabaseType.Authentik, Pair.of(authentik_ds, SQLDialect.POSTGRES)); - } - - // Get a DSL context - public DSLContext jooq(DatabaseType type) { - if (!connectionMap.containsKey(type)) { - throw new NoDataFoundException("Supplied type key not present in connection map"); - } - - Pair ds_pair = connectionMap.get(type); - - return DSL.using(ds_pair.getLeft(), ds_pair.getRight()); - } - - // Easy way for NOW() in SQL - public LocalDateTime now() { - return new Timestamp(new Date().getTime()).toLocalDateTime(); - } - -} +package me.aa07.paradise.taskdaemon.core.services.database; + +import java.sql.Connection; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.Date; +import java.util.HashMap; +import javax.sql.DataSource; +import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; +import me.aa07.paradise.taskdaemon.core.config.DatabaseConfig; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.exception.NoDataFoundException; +import org.jooq.impl.DSL; + +public class DbCore { + private HashMap> connectionMap; + + public DbCore(ConfigHolder config, Logger logger) { + // Suppress JOOQ console spam + System.getProperties().setProperty("org.jooq.no-logo", "true"); + System.getProperties().setProperty("org.jooq.no-tips", "true"); + + connectionMap = new HashMap>(); + + establishConnections(config); + logger.info("Ready to handle DB requests"); + } + + private DataSource openMySqlDataSource(String url, String username, String password) { + BasicDataSource source = new BasicDataSource(); + source.addConnectionProperty("autoReconnect", "true"); + source.addConnectionProperty("allowMultiQueries", "true"); + source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); + source.addConnectionProperty("connectionTimeZone", "UTC"); + source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + source.setDriverClassName("com.mysql.cj.jdbc.Driver"); + source.setUrl(url); + source.setUsername(username); + source.setPassword(password); + source.setMaxTotal(8); + source.setMaxIdle(8); + source.setTimeBetweenEvictionRunsMillis(180 * 1000); + source.setSoftMinEvictableIdleTimeMillis(180 * 1000); + + return source; + } + + private DataSource openPostgresDataSource(String url, String username, String password) { + BasicDataSource source = new BasicDataSource(); + source.addConnectionProperty("autoReconnect", "true"); + source.addConnectionProperty("allowMultiQueries", "true"); + source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); + source.addConnectionProperty("connectionTimeZone", "UTC"); + source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + source.setDriverClassName("org.postgresql.Driver"); + source.setUrl(url); + source.setUsername(username); + source.setPassword(password); + source.setMaxTotal(8); + source.setMaxIdle(8); + source.setTimeBetweenEvictionRunsMillis(180 * 1000); + source.setSoftMinEvictableIdleTimeMillis(180 * 1000); + + return source; + } + + private void establishConnections(ConfigHolder config) { + HashMap mysql_db_types = new HashMap(); + + mysql_db_types.put(DatabaseType.Automation, config.automationDatabase); + mysql_db_types.put(DatabaseType.Forums, config.forumsDatabase); + mysql_db_types.put(DatabaseType.GameDb, config.gameDatabase); + mysql_db_types.put(DatabaseType.ProfilerDb, config.profilerDatabase); + mysql_db_types.put(DatabaseType.PullRequests, config.pullRequestsDatabase); + + for (DatabaseType dbtype : mysql_db_types.keySet()) { + DatabaseConfig cfg = mysql_db_types.get(dbtype); + DataSource ds = openMySqlDataSource(String.format("jdbc:mysql://%s/%s", cfg.host, cfg.database), cfg.username, cfg.password); + connectionMap.put(dbtype, Pair.of(ds, SQLDialect.MYSQL)); + } + + // And do special stuff for Postgres + DatabaseConfig adb = config.authentikDatabase; + DataSource authentik_ds = openPostgresDataSource(String.format("jdbc:postgresql://%s/%s", adb.host, adb.database), adb.username, adb.password); + connectionMap.put(DatabaseType.Authentik, Pair.of(authentik_ds, SQLDialect.POSTGRES)); + } + + // Get a DSL context + public DSLContext jooq(DatabaseType type) { + if (!connectionMap.containsKey(type)) { + throw new NoDataFoundException("Supplied type key not present in connection map"); + } + + Pair ds_pair = connectionMap.get(type); + + return DSL.using(ds_pair.getLeft(), ds_pair.getRight()); + } + + // Easy way for NOW() in SQL + public LocalDateTime now() { + return new Timestamp(new Date().getTime()).toLocalDateTime(); + } + +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java new file mode 100644 index 0000000..4a199ca --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java @@ -0,0 +1,5 @@ +package me.aa07.paradise.taskdaemon.core.services.invision; + +public enum HttpRequestMethod { + Get, Post +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java new file mode 100644 index 0000000..3718e5e --- /dev/null +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java @@ -0,0 +1,123 @@ +package me.aa07.paradise.taskdaemon.core.services.invision; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.Builder; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import me.aa07.paradise.taskdaemon.core.config.InvisionConfig; +import me.aa07.paradise.taskdaemon.core.models.invision.GetUserModel; +import me.aa07.paradise.taskdaemon.core.models.invision.UpdateSecondaryGroupsModel; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import org.apache.commons.lang3.tuple.Pair; + +public class InvisionUtil { + private HttpClient httpClient; + private InvisionConfig invisionConfig; + + public InvisionUtil(InvisionConfig invisionConfig) { + this.invisionConfig = invisionConfig; + httpClient = HttpClient.newHttpClient(); + } + + private Pair makeInvisionRequest(String uri, HttpRequestMethod reqType, String bodyContent) throws Exception { + // Step 1 - Make our base + Builder request_builder = HttpRequest.newBuilder().uri(new URI(String.format("%s/%s", invisionConfig.apiBaseUrl, uri))); + + switch (reqType) { + case Get: + request_builder = request_builder.GET(); + break; + case Post: + if (bodyContent == null) { + request_builder = request_builder.POST(HttpRequest.BodyPublishers.noBody()); + } else { + request_builder = request_builder.POST(HttpRequest.BodyPublishers.ofString(bodyContent)); + } + break; + default: + throw new Exception(String.format("Request type %s not supported", reqType.toString())); + } + + // Generate auth header + String creds = String.format("%s:", invisionConfig.apiKey); + byte[] encoded_auth = Base64.getEncoder().encode(creds.getBytes(StandardCharsets.UTF_8)); + String auth_header = String.format("Basic %s", new String(encoded_auth)); + + HttpRequest final_request = request_builder.setHeader("Accept", "application/json") + .setHeader("Content-Type", "application/json") + .setHeader("Authorization", auth_header).build(); + + HttpResponse response = httpClient.send(final_request, BodyHandlers.ofString()); + + return Pair.of(response.body(), response.statusCode()); + } + + public Pair> getUserSecondaryGroups(int userId) { + ArrayList out_list = new ArrayList(); + String request_uri = String.format("core/members/%s", userId); + + Pair invision_response = null; + + // Send it off + try { + invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Get, null); + } catch (Exception e) { + e.printStackTrace(); + return Pair.of(false, out_list); + } + + // Shouldnt ever happen + if (invision_response == null || invision_response.getRight() != 200) { + return Pair.of(false, out_list); + } + + // Try decode + GetUserModel gum = UtilConst.GSON.fromJson(invision_response.getLeft(), GetUserModel.class); + + if (gum == null) { + return Pair.of(false, out_list); + } + + // This one does return true - they could have 0 secondary groups + if (gum.secondaryGroups == null) { + return Pair.of(true, out_list); + } + + // If we are here we can decode + for (GetUserModel.GroupHolder gh : gum.secondaryGroups) { + out_list.add(gh.id); + } + + return Pair.of(true, out_list); + } + + public boolean updateUserSecondaryGroups(int userId, List secondaryGroups) { + String request_uri = String.format("core/members/%s", userId); + UpdateSecondaryGroupsModel usgm = new UpdateSecondaryGroupsModel(); + usgm.secondaryGroups = secondaryGroups; + + String request_body = UtilConst.GSON.toJson(usgm); + Pair invision_response = null; + + // Send it off + try { + invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Post, request_body); + } catch (Exception e) { + e.printStackTrace(); + return false; + } + + // Shouldnt ever happen + if (invision_response == null || invision_response.getRight() != 200) { + return false; + } + + return true; + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java index e911cc6..e762d90 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java @@ -1,12 +1,16 @@ -package me.aa07.paradise.taskdaemon.core.util; - -import com.google.gson.Gson; -import java.util.UUID; - -public class UtilConst { - public static final UUID BYOND_OAUTH_SERVICE_ID = UUID.fromString("d7a7b659-69bd-4435-b585-3b4ba97ca076"); - public static final UUID DISCORD_OAUTH_SERVICE_ID = UUID.fromString("bc9fb785-6a1f-434e-80da-26a37450a29a"); - public static final UUID PATREON_OAUTH_SERVICE_ID = UUID.fromString("626f4935-d046-4957-9862-f26102e27a70"); - - public static final Gson GSON = new Gson(); -} +package me.aa07.paradise.taskdaemon.core.util; + +import com.google.gson.Gson; +import java.util.UUID; + +public class UtilConst { + public static final UUID BYOND_OAUTH_SERVICE_ID = UUID.fromString("d7a7b659-69bd-4435-b585-3b4ba97ca076"); + public static final UUID DISCORD_OAUTH_SERVICE_ID = UUID.fromString("bc9fb785-6a1f-434e-80da-26a37450a29a"); + public static final UUID INVISION_OAUTH_SERVICE_ID = UUID.fromString("4634c7d7-2bad-4139-a05f-13f296b20af8"); + public static final UUID PATREON_OAUTH_SERVICE_ID = UUID.fromString("626f4935-d046-4957-9862-f26102e27a70"); + + public static final int INVISION_INGAMEVERIFIED_GID = 30; + public static final int INVISION_INGAMEBANNED_GID = 31; + + public static final Gson GSON = new Gson(); +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java index 478615a..740a49c 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java @@ -1,24 +1,35 @@ -package me.aa07.paradise.taskdaemon.core.util; - -import java.util.regex.Pattern; - -public class UtilStr { - private static final Pattern CKEY_PATTERN = Pattern.compile("[^a-z0-9]"); - - public static String cleanCkey(String ckey) { - // If blank got passed, dont do it - if (ckey == null) { - return ""; - } - - // Lowercase it - ckey = ckey.toLowerCase(); - - // Clean these out from Authentik - ckey = ckey.replace("user:", ""); - - - // Split anything else out - return CKEY_PATTERN.matcher(ckey).replaceAll(""); - } -} +package me.aa07.paradise.taskdaemon.core.util; + +import java.util.regex.Pattern; + +public class UtilStr { + private static final Pattern CKEY_PATTERN = Pattern.compile("[^a-z0-9]"); + private static final Pattern ONLY_NUMBERS = Pattern.compile("[^0-9]"); + + public static String cleanCkey(String ckey) { + // If blank got passed, dont do it + if (ckey == null) { + return ""; + } + + // Lowercase it + ckey = ckey.toLowerCase(); + + // Clean these out from Authentik + ckey = ckey.replace("user:", ""); + + + // Split anything else out + return CKEY_PATTERN.matcher(ckey).replaceAll(""); + } + + public static String onlyNumbers(String input) { + // If blank got passed, dont do it + if (input == null) { + return ""; + } + + // Annihilate everything else + return ONLY_NUMBERS.matcher(input).replaceAll(""); + } +} diff --git a/config.toml.example b/config.toml.example index cb006f1..3353fee 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,64 +1,68 @@ -# This is an example configuration. Please see README.MD - -[authentikDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "authentik" - -[automationDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "aa07_automation" - -[forumsDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "paradise_forums" - -[gameDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "paradise_profilerdaemon" - -[github] -githubToken = "your token here" -repoId = 123456 -orgName = "ParadiseSS13" -reviewTeamSlug = "review-team" # Team name but with the dashes like "review-team" not "Review Team" -reviewTeamToken = "your token here" # Review team gets a separate token because it needs org admin to get team members - -[ip2asn] -host = "http://127.0.0.1:53661/v1/as/ip/" # Fill out URL - IP will be appended at the end - -[patreon] -campaignId = 123 -creatorToken = "token goes here" - -[pfsense] -host = "127.0.0.1" -port = 1234 - -[profilerDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "paradise_profilerdaemon" - -[pullRequestsDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "aa07_pr_voting" - -[redis] -host = "172.16.0.200" - -[tgs] -tgsHost = "http://10.0.0.30:5000" # dont put a trailing slash on the end of this -username = "username_here" -password = "password_here" -instanceId = 8 # instance ID to restart every day +# This is an example configuration. Please see README.MD + +[authentikDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "authentik" + +[automationDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "aa07_automation" + +[forumsDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "paradise_forums" + +[gameDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "paradise_profilerdaemon" + +[github] +githubToken = "your token here" +repoId = 123456 +orgName = "ParadiseSS13" +reviewTeamSlug = "review-team" # Team name but with the dashes like "review-team" not "Review Team" +reviewTeamToken = "your token here" # Review team gets a separate token because it needs org admin to get team members + +[invisionConfig] +apiBaseUrl = "https://www.paradisestation.org/forum/api" +apiKey = "key goes here" + +[ip2asn] +host = "http://127.0.0.1:53661/v1/as/ip/" # Fill out URL - IP will be appended at the end + +[patreon] +campaignId = 123 +creatorToken = "token goes here" + +[pfsense] +host = "127.0.0.1" +port = 1234 + +[profilerDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "paradise_profilerdaemon" + +[pullRequestsDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "aa07_pr_voting" + +[redis] +host = "172.16.0.200" + +[tgs] +tgsHost = "http://10.0.0.30:5000" # dont put a trailing slash on the end of this +username = "username_here" +password = "password_here" +instanceId = 8 # instance ID to restart every day From 182424d15e721e41d5bfb31d7f8de137ba45097c Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:18:55 +0100 Subject: [PATCH 3/7] DB update --- .../aa07/paradise/taskdaemon/database/automation/Keys.java | 1 + .../database/automation/tables/IngameVerifiedDirectory.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java index 0fc02c7..93e6150 100644 --- a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/Keys.java @@ -45,6 +45,7 @@ public class Keys { public static final UniqueKey KEY_AVAILABILITY_PRIMARY = Internal.createUniqueKey(Availability.AVAILABILITY, DSL.name("KEY_availability_PRIMARY"), new TableField[] { Availability.AVAILABILITY.ID }, true); public static final UniqueKey KEY_GITHUB_LINKS_PRIMARY = Internal.createUniqueKey(GithubLinks.GITHUB_LINKS, DSL.name("KEY_github_links_PRIMARY"), new TableField[] { GithubLinks.GITHUB_LINKS.GITHUB_ID }, true); public static final UniqueKey KEY_IMAGES_PRIMARY = Internal.createUniqueKey(Images.IMAGES, DSL.name("KEY_images_PRIMARY"), new TableField[] { Images.IMAGES.ID }, true); + public static final UniqueKey KEY_INGAME_VERIFIED_DIRECTORY_CKEY = Internal.createUniqueKey(IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY, DSL.name("KEY_ingame_verified_directory_ckey"), new TableField[] { IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.CKEY }, true); public static final UniqueKey KEY_INGAME_VERIFIED_DIRECTORY_PRIMARY = Internal.createUniqueKey(IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY, DSL.name("KEY_ingame_verified_directory_PRIMARY"), new TableField[] { IngameVerifiedDirectory.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID }, true); public static final UniqueKey KEY_MEMBER_CACHE_PRIMARY = Internal.createUniqueKey(MemberCache.MEMBER_CACHE, DSL.name("KEY_member_cache_PRIMARY"), new TableField[] { MemberCache.MEMBER_CACHE.FUID }, true); public static final UniqueKey KEY_PATREON_SUPPORTERS_PRIMARY = Internal.createUniqueKey(PatreonSupporters.PATREON_SUPPORTERS, DSL.name("KEY_patreon_supporters_PRIMARY"), new TableField[] { PatreonSupporters.PATREON_SUPPORTERS.MEMBER_ID }, true); diff --git a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java index ce8ed01..4ac2d94 100644 --- a/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java +++ b/TaskDaemon.Database.Automation/src/me/aa07/paradise/taskdaemon/database/automation/tables/IngameVerifiedDirectory.java @@ -131,6 +131,11 @@ public UniqueKey getPrimaryKey() { return Keys.KEY_INGAME_VERIFIED_DIRECTORY_PRIMARY; } + @Override + public List> getUniqueKeys() { + return Arrays.asList(Keys.KEY_INGAME_VERIFIED_DIRECTORY_CKEY); + } + @Override public IngameVerifiedDirectory as(String alias) { return new IngameVerifiedDirectory(DSL.name(alias), this); From 9126545dabcbd8c80731bf8c2f7667ca9478b074 Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:11:51 +0100 Subject: [PATCH 4/7] Final commit --- README.MD | 179 +++++++++--------- .../aa07/paradise/taskdaemon/core/Core.java | 17 +- .../invision/UpdateSecondaryGroupsModel.java | 9 - .../IngameVerifiedSyncJob.java | 42 +++- .../core/services/invision/InvisionUtil.java | 14 +- 5 files changed, 147 insertions(+), 114 deletions(-) delete mode 100644 TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java diff --git a/README.MD b/README.MD index a6ea85f..7c9a128 100644 --- a/README.MD +++ b/README.MD @@ -1,87 +1,92 @@ -# Paradise Task Daemon - -## What is it? - -Java application designed to automate various backend tasks we have within ParadiseSS13. - -It handles: - -- ACL Cleanup -- Bouncer Restart -- Dev Rank Handling -- IP2ASN -- Patreon Sync -- Profiler Daemon Ingest -- Profiler DB Cleanup -- PR Label Bot - -### ACL Cleanup - -Removes entries from a HAproxy ACL via its socket interface in order for inactivate players t o be put through the bouncer server again. - -### Bouncer Restart - -Restarts our "bouncer" server every day. This is the thing that does 2FA, server queue, and region redirection. - -### Dev Rank - -Adds or removes members from development rank when changes are made. Performed hourly. - -### IP2ASN - -Caches ASNs for IP addresses of recent players. - -### Profiler Daemon Ingest - -This takes BYOND Profiler results and split them apart to be indexed at a later date, allowing multiple procs from multiple rounds to easily be visualised alongside eachother over time. An example SS13 codebase implementation can be found at . - -This project itself does not provide a data extraction method, but that can be found at . - -### Patreon Sync - -Pulls data from Patreon to store in a DB. This is then used for Discord role sync, and ingame perk granting. - -### Profiler DB Cleanup - -Just wipes rows older than 7 days out the profiler table. - -### PR Label Bot - -Handles labelling PRs on the GitHub based on TM status, assigned type, votes and other stuff. - -## Provided Projects - -- `TaskDaemon.Core` - Core application which handles running tasks. -- `TaskDaemon.Database.Authentik` - Database project automatically generated with JOOQ for the `authentik` database. -- `TaskDaemon.Database.Automation` - Database project automatically generated with JOOQ for the `aa07_automation` database. - - No I will not apologise for that DB name. -- `TaskDaemon.Database.Forums` - Database project automatically generated with JOOQ for the `paradise_forums` database. -- `TaskDaemon.Database.GameDb` - Database project automatically generated with JOOQ for the `paradise_gamedb` database. -- `TaskDaemon.Database.Profiler` - Database project automatically generated with JOOQ for the `paradise_profiler` database. -- `TaskDaemon.Database.PullRequests` - Database project automatically generated with JOOQ for the `aa07_pr_voting` database. - - No I will not apologise for that DB name. - -Use `mvn package -P update-db` to update DB projects. - -## Building - -To build, run `mvn package` to generate the JAR file. This will then appear in the `out` directory. - -## Running - -1. Compile the codebase (See `Building`). -2. Copy the jar and the example `config.toml` in the repository root to a running location. -3. Import the relevant schemas from the `schemas` dir to your database server. -4. Setup the configuration as per section labels. -5. Start the application up, it will now run tasks in the background. - -## Code Guidelines - -- Code must be tested. - - Good luck. -- Code must adhere to the style guidelines set in `checkstyle.xml`. - - These are google's guidelines but with minor tweaks for my own needs. - -## License - -This project is licensed under GPL-3. You can find a copy of this license in the file `LICENSE.MD`. +# Paradise Task Daemon + +## What is it? + +Java application designed to automate various backend tasks we have within ParadiseSS13. + +It handles: + +- ACL Cleanup +- Bouncer Restart +- Dev Rank Handling +- Ingame Verified Syncc +- IP2ASN +- Patreon Sync +- Profiler Daemon Ingest +- Profiler DB Cleanup +- PR Label Bot + +### ACL Cleanup + +Removes entries from a HAproxy ACL via its socket interface in order for inactivate players t o be put through the bouncer server again. + +### Bouncer Restart + +Restarts our "bouncer" server every day. This is the thing that does 2FA, server queue, and region redirection. + +### Dev Rank + +Adds or removes members from development rank when changes are made. Performed hourly. + +### Ingame Verified Sync + +Tacks roles onto the relevant people for their ingame verified state + +### IP2ASN + +Caches ASNs for IP addresses of recent players. + +### Profiler Daemon Ingest + +This takes BYOND Profiler results and split them apart to be indexed at a later date, allowing multiple procs from multiple rounds to easily be visualised alongside eachother over time. An example SS13 codebase implementation can be found at . + +This project itself does not provide a data extraction method, but that can be found at . + +### Patreon Sync + +Pulls data from Patreon to store in a DB. This is then used for Discord role sync, and ingame perk granting. + +### Profiler DB Cleanup + +Just wipes rows older than 7 days out the profiler table. + +### PR Label Bot + +Handles labelling PRs on the GitHub based on TM status, assigned type, votes and other stuff. + +## Provided Projects + +- `TaskDaemon.Core` - Core application which handles running tasks. +- `TaskDaemon.Database.Authentik` - Database project automatically generated with JOOQ for the `authentik` database. +- `TaskDaemon.Database.Automation` - Database project automatically generated with JOOQ for the `aa07_automation` database. + - No I will not apologise for that DB name. +- `TaskDaemon.Database.Forums` - Database project automatically generated with JOOQ for the `paradise_forums` database. +- `TaskDaemon.Database.GameDb` - Database project automatically generated with JOOQ for the `paradise_gamedb` database. +- `TaskDaemon.Database.Profiler` - Database project automatically generated with JOOQ for the `paradise_profiler` database. +- `TaskDaemon.Database.PullRequests` - Database project automatically generated with JOOQ for the `aa07_pr_voting` database. + - No I will not apologise for that DB name. + +Use `mvn package -P update-db` to update DB projects. + +## Building + +To build, run `mvn package` to generate the JAR file. This will then appear in the `out` directory. + +## Running + +1. Compile the codebase (See `Building`). +2. Copy the jar and the example `config.toml` in the repository root to a running location. +3. Import the relevant schemas from the `schemas` dir to your database server. +4. Setup the configuration as per section labels. +5. Start the application up, it will now run tasks in the background. + +## Code Guidelines + +- Code must be tested. + - Good luck. +- Code must adhere to the style guidelines set in `checkstyle.xml`. + - These are google's guidelines but with minor tweaks for my own needs. + +## License + +This project is licensed under GPL-3. You can find a copy of this license in the file `LICENSE.MD`. diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java index 2419915..f7565e1 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java @@ -199,8 +199,7 @@ private void setupJobs(Scheduler scheduler, DbCore dbCore, InvisionUtil iu, Conf .build(); CronTrigger ct_igvsync = TriggerBuilder.newTrigger() .withIdentity("igvsync", "igvsync") - //.withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes - .withSchedule(CronScheduleBuilder.cronSchedule("0 * * * * ?")) // Every hour, offset 23 minutes + .withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes .build(); // Lists that exist just to make sure these things show as recognised @@ -229,13 +228,13 @@ private void setupJobs(Scheduler scheduler, DbCore dbCore, InvisionUtil iu, Conf // Schedule all - //scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); - //scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); - //scheduler.scheduleJob(jd_devrank, ct_devrank); - //scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); - //scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); - //scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); - //scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); + scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); + scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); + scheduler.scheduleJob(jd_devrank, ct_devrank); + scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); + scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); + scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); + scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); scheduler.scheduleJob(jd_igvsync, ct_igvsync); } diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java deleted file mode 100644 index 1ad21f0..0000000 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/UpdateSecondaryGroupsModel.java +++ /dev/null @@ -1,9 +0,0 @@ -package me.aa07.paradise.taskdaemon.core.models.invision; - -import com.google.gson.annotations.SerializedName; -import java.util.List; - -public class UpdateSecondaryGroupsModel { - @SerializedName("secondaryGroups") - public List secondaryGroups; -} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java index 7565c5e..6088ff5 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -19,7 +20,9 @@ import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PlayerRecord; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.Logger; +import org.jooq.Condition; import org.jooq.DSLContext; +import org.jooq.Record1; import org.jooq.Result; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; @@ -173,6 +176,37 @@ public void execute(JobExecutionContext event) throws JobExecutionException { } logger.info(String.format("[IngameVerifiedSync] Removed %s records for members who unlinked byond", removed)); + // Now get a list of banned people + HashSet banned_ckeys = new HashSet(); + + // Shared + Condition unbanned_null = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.UNBANNED.isNull(); + Condition unbanned_0 = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.UNBANNED.eq((byte) 0); + Condition unban_combined_or = unbanned_null.or(unbanned_0); + + // Permaban + Condition bantype_perma = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.BANTYPE.eq("PERMABAN"); + + // Tempban + Condition bantype_temp = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.BANTYPE.eq("TEMPBAN"); + Condition tempban_expired = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.EXPIRATION_TIME.gt(dbcore.now()); + Condition tempban_combined_final = bantype_temp.and(tempban_expired); + + // Final condition + Condition ban_check_merged = bantype_perma.or(tempban_combined_final); + Condition is_banned = unban_combined_or.and(ban_check_merged); + + Result> raw_bans_result = game_jooq.selectDistinct(me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.CKEY) + .from(me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN) + .where(is_banned) + .fetch(); + + for (Record1 rbr : raw_bans_result) { + banned_ckeys.add(rbr.value1()); + } + + logger.info(String.format("[IngameVerifiedSync] Loaded %s banned ckeys", banned_ckeys.size())); + // Now begin the mass inserts int raw_added = 0; int raw_updated = 0; @@ -243,6 +277,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { for (IngameVerifiedDirectoryRecord ivdr : records_to_update) { boolean update_made = false; int authentik_id = ivdr.getAuthentikId(); + String ckey = ivdr.getCkey(); int forum_id = 0; long discord_id = 0L; @@ -266,8 +301,9 @@ public void execute(JobExecutionContext event) throws JobExecutionException { if (gsg_response.getLeft()) { List forum_secondaries = gsg_response.getRight(); - if (!forum_secondaries.contains(UtilConst.INVISION_INGAMEVERIFIED_GID)) { - forum_secondaries.add(UtilConst.INVISION_INGAMEVERIFIED_GID); + int target_group = banned_ckeys.contains(ckey) ? UtilConst.INVISION_INGAMEBANNED_GID : UtilConst.INVISION_INGAMEVERIFIED_GID; + if (!forum_secondaries.contains(target_group)) { + forum_secondaries.add(target_group); boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); @@ -290,7 +326,7 @@ public void execute(JobExecutionContext event) throws JobExecutionException { TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); tqr.setTaskId(UUID.randomUUID()); tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("ADD_IGV_ROLE"); + tqr.setTaskType(banned_ckeys.contains(ckey) ? "ADD_IGB_ROLE" : "ADD_IGV_ROLE"); // No I will not apologise for these variable names DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java index 3718e5e..a129e38 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java @@ -12,7 +12,6 @@ import java.util.List; import me.aa07.paradise.taskdaemon.core.config.InvisionConfig; import me.aa07.paradise.taskdaemon.core.models.invision.GetUserModel; -import me.aa07.paradise.taskdaemon.core.models.invision.UpdateSecondaryGroupsModel; import me.aa07.paradise.taskdaemon.core.util.UtilConst; import org.apache.commons.lang3.tuple.Pair; @@ -98,16 +97,19 @@ public Pair> getUserSecondaryGroups(int userId) { } public boolean updateUserSecondaryGroups(int userId, List secondaryGroups) { - String request_uri = String.format("core/members/%s", userId); - UpdateSecondaryGroupsModel usgm = new UpdateSecondaryGroupsModel(); - usgm.secondaryGroups = secondaryGroups; + ArrayList group_ids_list_str = new ArrayList(); + for (Integer i : secondaryGroups) { + // I hate PHP array syntax + group_ids_list_str.add(String.format("secondaryGroups[]=%s", i.toString())); + } + String secondary_groups_str = String.join("&", group_ids_list_str); - String request_body = UtilConst.GSON.toJson(usgm); + String request_uri = String.format("core/members/%s?%s", userId, secondary_groups_str); Pair invision_response = null; // Send it off try { - invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Post, request_body); + invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Post, null); } catch (Exception e) { e.printStackTrace(); return false; From 3192bf6382b58de0d6af28db120d525e7ca3a1d8 Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:17:10 +0100 Subject: [PATCH 5/7] Line endings ough --- .../aa07/paradise/taskdaemon/core/Core.java | 490 +++---- .../taskdaemon/core/config/ConfigHolder.java | 34 +- .../core/config/InvisionConfig.java | 12 +- .../core/models/invision/GetUserModel.java | 48 +- .../modules/aclcleanup/AclCleanupJob.java | 386 +++--- .../core/modules/devrank/DevRankJob.java | 316 ++--- .../IngameVerifiedSyncJob.java | 716 +++++------ .../core/modules/ip2asn/Ip2AsnJob.java | 422 +++---- .../modules/patreonsync/PatreonSyncJob.java | 1124 ++++++++--------- .../core/modules/prlabel/PrLabelJob.java | 504 ++++---- .../profilercleanup/ProfilerCleanupJob.java | 122 +- .../profileringest/ProfilerWorker.java | 170 +-- .../processors/ProfilerBaseProcessor.java | 66 +- .../processors/ProfilerProcProcessor.java | 124 +- .../processors/ProfilerSendmapsProcessor.java | 118 +- .../core/services/database/DatabaseType.java | 10 +- .../core/services/database/DbCore.java | 216 ++-- .../services/invision/HttpRequestMethod.java | 10 +- .../core/services/invision/InvisionUtil.java | 250 ++-- .../taskdaemon/core/util/UtilConst.java | 32 +- .../taskdaemon/core/util/UtilStr.java | 70 +- 21 files changed, 2620 insertions(+), 2620 deletions(-) diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java index f7565e1..65e477c 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/Core.java @@ -1,245 +1,245 @@ -package me.aa07.paradise.taskdaemon.core; - -import com.moandjiezana.toml.Toml; -import java.io.File; -import java.util.ArrayList; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; -import me.aa07.paradise.taskdaemon.core.modules.aclcleanup.AclCleanupJob; -import me.aa07.paradise.taskdaemon.core.modules.bouncerrestart.BouncerRestartJob; -import me.aa07.paradise.taskdaemon.core.modules.devrank.DevRankJob; -import me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync.IngameVerifiedSyncJob; -import me.aa07.paradise.taskdaemon.core.modules.ip2asn.Ip2AsnJob; -import me.aa07.paradise.taskdaemon.core.modules.patreonsync.PatreonSyncJob; -import me.aa07.paradise.taskdaemon.core.modules.prlabel.PrLabelJob; -import me.aa07.paradise.taskdaemon.core.modules.profilercleanup.ProfilerCleanupJob; -import me.aa07.paradise.taskdaemon.core.modules.profileringest.ProfilerWorker; -import me.aa07.paradise.taskdaemon.core.redis.RedisManager; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.services.invision.InvisionUtil; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.quartz.CronScheduleBuilder; -import org.quartz.CronTrigger; -import org.quartz.JobBuilder; -import org.quartz.JobDataMap; -import org.quartz.JobDetail; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.TriggerBuilder; -import org.quartz.impl.StdSchedulerFactory; - -public class Core { - // Threads - private Thread profilerWorkerThread; - private Thread redisthread; - - public void run() { - Logger logger = LogManager.getLogger(Core.class); - logger.info("Starting up"); - logger.info("Loading configuration..."); - - Optional config = Optional.empty(); - - try { - File configfile = new File("config.toml"); - config = Optional.of(new Toml().read(configfile).to(ConfigHolder.class)); - } catch (Exception exception) { - logger.fatal("Failed to load config.toml!"); - exception.printStackTrace(); - return; - } - - if (!config.isPresent()) { - logger.fatal("Failed to load config.toml - but we didnt throw an exception! What the heck?!"); - return; - } - - logger.info("Preparing threads..."); - setupThreads(config.get(), logger); - logger.info("Launching threads..."); - launchAll(); - logger.info("Core startup complete"); - } - - private void setupThreads(ConfigHolder config, Logger logger) { - // Initial setup - DbCore database = new DbCore(config, logger); - InvisionUtil iu = new InvisionUtil(config.invisionConfig); - - // Profiler stuff - ProfilerWorker pw = new ProfilerWorker(database, logger); - profilerWorkerThread = new Thread(pw::run, "profiler-worker"); - - // Redis stuff - RedisManager rm = new RedisManager(config.redis, logger, pw); - redisthread = new Thread(rm::run, "redis"); - - // Launch Quartz - try { - Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); - setupJobs(scheduler, database, iu, config, logger); - scheduler.start(); - } catch (SchedulerException ex) { - logger.error("Quartz had a hissy fit!", ex); - } - } - - private void setupJobs(Scheduler scheduler, DbCore dbCore, InvisionUtil iu, ConfigHolder config, Logger logger) throws SchedulerException { - // See below for CRON format - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html - - // ACL cleanup - JobDataMap jdm_aclcleanup = new JobDataMap(); - jdm_aclcleanup.put("LOGGER", logger); - jdm_aclcleanup.put("DBCORE", dbCore); - jdm_aclcleanup.put("PFSENSE_CFG", config.pfsense); - JobDetail jd_aclcleanup = JobBuilder.newJob(AclCleanupJob.class) - .withIdentity("aclcleanup", "aclcleanup") - .usingJobData(jdm_aclcleanup) - .build(); - CronTrigger ct_aclcleanup = TriggerBuilder.newTrigger() - .withIdentity("aclcleanup", "aclcleanup") - .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) - .build(); - - // Bouncer restart - JobDataMap jdm_bouncerrestart = new JobDataMap(); - jdm_bouncerrestart.put("LOGGER", logger); - jdm_bouncerrestart.put("TGS_CFG", config.tgs); - JobDetail jd_bouncerrestart = JobBuilder.newJob(BouncerRestartJob.class) - .withIdentity("bouncerrestart", "bouncerrestart") - .usingJobData(jdm_bouncerrestart) - .build(); - CronTrigger ct_bouncerrestart = TriggerBuilder.newTrigger() - .withIdentity("bouncerrestart", "bouncerrestart") - .withSchedule(CronScheduleBuilder.cronSchedule("0 0 0 * * ?")) // Every day - midnight - .build(); - - // DevRank update - JobDataMap jdm_devrank = new JobDataMap(); - jdm_devrank.put("LOGGER", logger); - jdm_devrank.put("DBCORE", dbCore); - - JobDetail jd_devrank = JobBuilder.newJob(DevRankJob.class) - .withIdentity("devrank", "devrank") - .usingJobData(jdm_devrank) - .build(); - - CronTrigger ct_devrank = TriggerBuilder.newTrigger() - .withIdentity("devrank", "devrank") - .withSchedule(CronScheduleBuilder.cronSchedule("0 0 * * * ?")) // Every hour on the hour - .build(); - - - // IP2ASN - JobDataMap jdm_ip2asn = new JobDataMap(); - jdm_ip2asn.put("LOGGER", logger); - jdm_ip2asn.put("DBCORE", dbCore); - jdm_ip2asn.put("IP2ASNCFG", config.ip2asn); - JobDetail jd_ip2asn = JobBuilder.newJob(Ip2AsnJob.class) - .withIdentity("ip2asn", "ip2asn") - .usingJobData(jdm_ip2asn) - .build(); - CronTrigger ct_ip2asn = TriggerBuilder.newTrigger() - .withIdentity("ip2asn", "ip2asn") - .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) // Every 10 minutes - .build(); - - // Profiler cleanup - JobDataMap jdm_profilercleanup = new JobDataMap(); - jdm_profilercleanup.put("LOGGER", logger); - jdm_profilercleanup.put("DBCORE", dbCore); - JobDetail jd_profilercleanup = JobBuilder.newJob(ProfilerCleanupJob.class) - .withIdentity("profilercleanup", "profilercleanup") - .usingJobData(jdm_profilercleanup) - .build(); - CronTrigger ct_profilercleanup = TriggerBuilder.newTrigger() - .withIdentity("profilercleanup", "profilercleanup") - .withSchedule(CronScheduleBuilder.cronSchedule("0 0 8 * * ?")) // Every day - 8AM - .build(); - - // PR Label bot - JobDataMap jdm_pullrequests = new JobDataMap(); - jdm_pullrequests.put("LOGGER", logger); - jdm_pullrequests.put("DBCORE", dbCore); - jdm_pullrequests.put("TGSCFG", config.tgs); - jdm_pullrequests.put("GITHUBCFG", config.github); - JobDetail jd_pullrequests = JobBuilder.newJob(PrLabelJob.class) - .withIdentity("pullrequests", "pullrequests") - .usingJobData(jdm_pullrequests) - .build(); - CronTrigger ct_pullrequests = TriggerBuilder.newTrigger() - .withIdentity("pullrequests", "pullrequests") - .withSchedule(CronScheduleBuilder.cronSchedule("0 10 * * * ?")) // Every hour, offset 10 minutes - .build(); - - // Patreon Sync - JobDataMap jdm_patreonsync = new JobDataMap(); - jdm_patreonsync.put("LOGGER", logger); - jdm_patreonsync.put("DBCORE", dbCore); - jdm_patreonsync.put("PTRCFG", config.patreon); - JobDetail jd_patreonsync = JobBuilder.newJob(PatreonSyncJob.class) - .withIdentity("patreonsync", "patreonsync") - .usingJobData(jdm_patreonsync) - .build(); - CronTrigger ct_patreonsync = TriggerBuilder.newTrigger() - .withIdentity("patreonsync", "patreonsync") - .withSchedule(CronScheduleBuilder.cronSchedule("0 15 * * * ?")) // Every hour, offset 15 minutes - .build(); - - // Ingame verified sync - JobDataMap jdm_igvsync = new JobDataMap(); - jdm_igvsync.put("LOGGER", logger); - jdm_igvsync.put("DBCORE", dbCore); - jdm_igvsync.put("IU", iu); - JobDetail jd_igvsync = JobBuilder.newJob(IngameVerifiedSyncJob.class) - .withIdentity("igvsync", "igvsync") - .usingJobData(jdm_igvsync) - .build(); - CronTrigger ct_igvsync = TriggerBuilder.newTrigger() - .withIdentity("igvsync", "igvsync") - .withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes - .build(); - - // Lists that exist just to make sure these things show as recognised - ArrayList bin1 = new ArrayList(); - bin1.add(jd_aclcleanup); - bin1.add(jd_bouncerrestart); - bin1.add(jd_devrank); - bin1.add(jd_ip2asn); - bin1.add(jd_profilercleanup); - bin1.add(jd_pullrequests); - bin1.add(jd_patreonsync); - bin1.add(jd_igvsync); - ArrayList bin2 = new ArrayList(); - bin2.add(ct_aclcleanup); - bin2.add(ct_bouncerrestart); - bin2.add(ct_devrank); - bin2.add(ct_ip2asn); - bin2.add(ct_profilercleanup); - bin2.add(ct_pullrequests); - bin2.add(ct_patreonsync); - bin2.add(ct_igvsync); - - // Is this necessary? Prolly not! - bin1.clear(); - bin2.clear(); - - - // Schedule all - scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); - scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); - scheduler.scheduleJob(jd_devrank, ct_devrank); - scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); - scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); - scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); - scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); - scheduler.scheduleJob(jd_igvsync, ct_igvsync); - } - - private void launchAll() { - profilerWorkerThread.start(); - redisthread.start(); - } -} +package me.aa07.paradise.taskdaemon.core; + +import com.moandjiezana.toml.Toml; +import java.io.File; +import java.util.ArrayList; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; +import me.aa07.paradise.taskdaemon.core.modules.aclcleanup.AclCleanupJob; +import me.aa07.paradise.taskdaemon.core.modules.bouncerrestart.BouncerRestartJob; +import me.aa07.paradise.taskdaemon.core.modules.devrank.DevRankJob; +import me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync.IngameVerifiedSyncJob; +import me.aa07.paradise.taskdaemon.core.modules.ip2asn.Ip2AsnJob; +import me.aa07.paradise.taskdaemon.core.modules.patreonsync.PatreonSyncJob; +import me.aa07.paradise.taskdaemon.core.modules.prlabel.PrLabelJob; +import me.aa07.paradise.taskdaemon.core.modules.profilercleanup.ProfilerCleanupJob; +import me.aa07.paradise.taskdaemon.core.modules.profileringest.ProfilerWorker; +import me.aa07.paradise.taskdaemon.core.redis.RedisManager; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.services.invision.InvisionUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.quartz.CronScheduleBuilder; +import org.quartz.CronTrigger; +import org.quartz.JobBuilder; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.TriggerBuilder; +import org.quartz.impl.StdSchedulerFactory; + +public class Core { + // Threads + private Thread profilerWorkerThread; + private Thread redisthread; + + public void run() { + Logger logger = LogManager.getLogger(Core.class); + logger.info("Starting up"); + logger.info("Loading configuration..."); + + Optional config = Optional.empty(); + + try { + File configfile = new File("config.toml"); + config = Optional.of(new Toml().read(configfile).to(ConfigHolder.class)); + } catch (Exception exception) { + logger.fatal("Failed to load config.toml!"); + exception.printStackTrace(); + return; + } + + if (!config.isPresent()) { + logger.fatal("Failed to load config.toml - but we didnt throw an exception! What the heck?!"); + return; + } + + logger.info("Preparing threads..."); + setupThreads(config.get(), logger); + logger.info("Launching threads..."); + launchAll(); + logger.info("Core startup complete"); + } + + private void setupThreads(ConfigHolder config, Logger logger) { + // Initial setup + DbCore database = new DbCore(config, logger); + InvisionUtil iu = new InvisionUtil(config.invisionConfig); + + // Profiler stuff + ProfilerWorker pw = new ProfilerWorker(database, logger); + profilerWorkerThread = new Thread(pw::run, "profiler-worker"); + + // Redis stuff + RedisManager rm = new RedisManager(config.redis, logger, pw); + redisthread = new Thread(rm::run, "redis"); + + // Launch Quartz + try { + Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); + setupJobs(scheduler, database, iu, config, logger); + scheduler.start(); + } catch (SchedulerException ex) { + logger.error("Quartz had a hissy fit!", ex); + } + } + + private void setupJobs(Scheduler scheduler, DbCore dbCore, InvisionUtil iu, ConfigHolder config, Logger logger) throws SchedulerException { + // See below for CRON format + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html + + // ACL cleanup + JobDataMap jdm_aclcleanup = new JobDataMap(); + jdm_aclcleanup.put("LOGGER", logger); + jdm_aclcleanup.put("DBCORE", dbCore); + jdm_aclcleanup.put("PFSENSE_CFG", config.pfsense); + JobDetail jd_aclcleanup = JobBuilder.newJob(AclCleanupJob.class) + .withIdentity("aclcleanup", "aclcleanup") + .usingJobData(jdm_aclcleanup) + .build(); + CronTrigger ct_aclcleanup = TriggerBuilder.newTrigger() + .withIdentity("aclcleanup", "aclcleanup") + .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) + .build(); + + // Bouncer restart + JobDataMap jdm_bouncerrestart = new JobDataMap(); + jdm_bouncerrestart.put("LOGGER", logger); + jdm_bouncerrestart.put("TGS_CFG", config.tgs); + JobDetail jd_bouncerrestart = JobBuilder.newJob(BouncerRestartJob.class) + .withIdentity("bouncerrestart", "bouncerrestart") + .usingJobData(jdm_bouncerrestart) + .build(); + CronTrigger ct_bouncerrestart = TriggerBuilder.newTrigger() + .withIdentity("bouncerrestart", "bouncerrestart") + .withSchedule(CronScheduleBuilder.cronSchedule("0 0 0 * * ?")) // Every day - midnight + .build(); + + // DevRank update + JobDataMap jdm_devrank = new JobDataMap(); + jdm_devrank.put("LOGGER", logger); + jdm_devrank.put("DBCORE", dbCore); + + JobDetail jd_devrank = JobBuilder.newJob(DevRankJob.class) + .withIdentity("devrank", "devrank") + .usingJobData(jdm_devrank) + .build(); + + CronTrigger ct_devrank = TriggerBuilder.newTrigger() + .withIdentity("devrank", "devrank") + .withSchedule(CronScheduleBuilder.cronSchedule("0 0 * * * ?")) // Every hour on the hour + .build(); + + + // IP2ASN + JobDataMap jdm_ip2asn = new JobDataMap(); + jdm_ip2asn.put("LOGGER", logger); + jdm_ip2asn.put("DBCORE", dbCore); + jdm_ip2asn.put("IP2ASNCFG", config.ip2asn); + JobDetail jd_ip2asn = JobBuilder.newJob(Ip2AsnJob.class) + .withIdentity("ip2asn", "ip2asn") + .usingJobData(jdm_ip2asn) + .build(); + CronTrigger ct_ip2asn = TriggerBuilder.newTrigger() + .withIdentity("ip2asn", "ip2asn") + .withSchedule(CronScheduleBuilder.cronSchedule("0 */10 * * * ?")) // Every 10 minutes + .build(); + + // Profiler cleanup + JobDataMap jdm_profilercleanup = new JobDataMap(); + jdm_profilercleanup.put("LOGGER", logger); + jdm_profilercleanup.put("DBCORE", dbCore); + JobDetail jd_profilercleanup = JobBuilder.newJob(ProfilerCleanupJob.class) + .withIdentity("profilercleanup", "profilercleanup") + .usingJobData(jdm_profilercleanup) + .build(); + CronTrigger ct_profilercleanup = TriggerBuilder.newTrigger() + .withIdentity("profilercleanup", "profilercleanup") + .withSchedule(CronScheduleBuilder.cronSchedule("0 0 8 * * ?")) // Every day - 8AM + .build(); + + // PR Label bot + JobDataMap jdm_pullrequests = new JobDataMap(); + jdm_pullrequests.put("LOGGER", logger); + jdm_pullrequests.put("DBCORE", dbCore); + jdm_pullrequests.put("TGSCFG", config.tgs); + jdm_pullrequests.put("GITHUBCFG", config.github); + JobDetail jd_pullrequests = JobBuilder.newJob(PrLabelJob.class) + .withIdentity("pullrequests", "pullrequests") + .usingJobData(jdm_pullrequests) + .build(); + CronTrigger ct_pullrequests = TriggerBuilder.newTrigger() + .withIdentity("pullrequests", "pullrequests") + .withSchedule(CronScheduleBuilder.cronSchedule("0 10 * * * ?")) // Every hour, offset 10 minutes + .build(); + + // Patreon Sync + JobDataMap jdm_patreonsync = new JobDataMap(); + jdm_patreonsync.put("LOGGER", logger); + jdm_patreonsync.put("DBCORE", dbCore); + jdm_patreonsync.put("PTRCFG", config.patreon); + JobDetail jd_patreonsync = JobBuilder.newJob(PatreonSyncJob.class) + .withIdentity("patreonsync", "patreonsync") + .usingJobData(jdm_patreonsync) + .build(); + CronTrigger ct_patreonsync = TriggerBuilder.newTrigger() + .withIdentity("patreonsync", "patreonsync") + .withSchedule(CronScheduleBuilder.cronSchedule("0 15 * * * ?")) // Every hour, offset 15 minutes + .build(); + + // Ingame verified sync + JobDataMap jdm_igvsync = new JobDataMap(); + jdm_igvsync.put("LOGGER", logger); + jdm_igvsync.put("DBCORE", dbCore); + jdm_igvsync.put("IU", iu); + JobDetail jd_igvsync = JobBuilder.newJob(IngameVerifiedSyncJob.class) + .withIdentity("igvsync", "igvsync") + .usingJobData(jdm_igvsync) + .build(); + CronTrigger ct_igvsync = TriggerBuilder.newTrigger() + .withIdentity("igvsync", "igvsync") + .withSchedule(CronScheduleBuilder.cronSchedule("0 23 * * * ?")) // Every hour, offset 23 minutes + .build(); + + // Lists that exist just to make sure these things show as recognised + ArrayList bin1 = new ArrayList(); + bin1.add(jd_aclcleanup); + bin1.add(jd_bouncerrestart); + bin1.add(jd_devrank); + bin1.add(jd_ip2asn); + bin1.add(jd_profilercleanup); + bin1.add(jd_pullrequests); + bin1.add(jd_patreonsync); + bin1.add(jd_igvsync); + ArrayList bin2 = new ArrayList(); + bin2.add(ct_aclcleanup); + bin2.add(ct_bouncerrestart); + bin2.add(ct_devrank); + bin2.add(ct_ip2asn); + bin2.add(ct_profilercleanup); + bin2.add(ct_pullrequests); + bin2.add(ct_patreonsync); + bin2.add(ct_igvsync); + + // Is this necessary? Prolly not! + bin1.clear(); + bin2.clear(); + + + // Schedule all + scheduler.scheduleJob(jd_aclcleanup, ct_aclcleanup); + scheduler.scheduleJob(jd_bouncerrestart, ct_bouncerrestart); + scheduler.scheduleJob(jd_devrank, ct_devrank); + scheduler.scheduleJob(jd_ip2asn, ct_ip2asn); + scheduler.scheduleJob(jd_profilercleanup, ct_profilercleanup); + scheduler.scheduleJob(jd_pullrequests, ct_pullrequests); + scheduler.scheduleJob(jd_patreonsync, ct_patreonsync); + scheduler.scheduleJob(jd_igvsync, ct_igvsync); + } + + private void launchAll() { + profilerWorkerThread.start(); + redisthread.start(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java index 69a44ff..efd52e1 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/ConfigHolder.java @@ -1,17 +1,17 @@ -package me.aa07.paradise.taskdaemon.core.config; - -public class ConfigHolder { - public DatabaseConfig authentikDatabase; - public DatabaseConfig automationDatabase; - public DatabaseConfig forumsDatabase; - public DatabaseConfig gameDatabase; - public GithubConfig github; - public InvisionConfig invisionConfig; - public Ip2AsnSerivceConfig ip2asn; - public PatreonConfig patreon; - public PfsenseConfig pfsense; - public DatabaseConfig profilerDatabase; - public DatabaseConfig pullRequestsDatabase; - public RedisConfig redis; - public TgsConfig tgs; -} +package me.aa07.paradise.taskdaemon.core.config; + +public class ConfigHolder { + public DatabaseConfig authentikDatabase; + public DatabaseConfig automationDatabase; + public DatabaseConfig forumsDatabase; + public DatabaseConfig gameDatabase; + public GithubConfig github; + public InvisionConfig invisionConfig; + public Ip2AsnSerivceConfig ip2asn; + public PatreonConfig patreon; + public PfsenseConfig pfsense; + public DatabaseConfig profilerDatabase; + public DatabaseConfig pullRequestsDatabase; + public RedisConfig redis; + public TgsConfig tgs; +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java index 7ec464b..41484f2 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/config/InvisionConfig.java @@ -1,6 +1,6 @@ -package me.aa07.paradise.taskdaemon.core.config; - -public class InvisionConfig { - public String apiBaseUrl; - public String apiKey; -} +package me.aa07.paradise.taskdaemon.core.config; + +public class InvisionConfig { + public String apiBaseUrl; + public String apiKey; +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java index 7fa8d6c..9a18859 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/models/invision/GetUserModel.java @@ -1,24 +1,24 @@ -package me.aa07.paradise.taskdaemon.core.models.invision; - -import com.google.gson.annotations.SerializedName; -import java.util.List; - -public class GetUserModel { - @SerializedName("id") // just incase GSON behaviour ever changes - public int id; - @SerializedName("name") - public String name; - @SerializedName("primaryGroup") - public GroupHolder primaryGroup; - @SerializedName("secondaryGroups") - public List secondaryGroups; - - public class GroupHolder { - @SerializedName("id") - public int id; - @SerializedName("name") - public String name; - @SerializedName("formattedName") - public String formattedName; - } -} +package me.aa07.paradise.taskdaemon.core.models.invision; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +public class GetUserModel { + @SerializedName("id") // just incase GSON behaviour ever changes + public int id; + @SerializedName("name") + public String name; + @SerializedName("primaryGroup") + public GroupHolder primaryGroup; + @SerializedName("secondaryGroups") + public List secondaryGroups; + + public class GroupHolder { + @SerializedName("id") + public int id; + @SerializedName("name") + public String name; + @SerializedName("formattedName") + public String formattedName; + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java index c385b40..9499775 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/aclcleanup/AclCleanupJob.java @@ -1,193 +1,193 @@ -package me.aa07.paradise.taskdaemon.core.modules.aclcleanup; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.Socket; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.PfsenseConfig; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.database.gamedb.Tables; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -/** - * Scheduled job that cleans up ACL by removing inactive player IPs - */ -@DisallowConcurrentExecution // NO -public class AclCleanupJob implements Job { - - /** - * Main job execution - fetches current ACL, checks database for inactive IPs, and removes them - */ - @Override - public void execute(JobExecutionContext context) throws JobExecutionException { - JobDataMap datamap = context.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[AclCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[AclCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Get pfSense configuration - Object raw_pfs_cfg = datamap.get("PFSENSE_CFG"); - Optional pfs_cfg_holder = Optional.empty(); - - if (raw_pfs_cfg instanceof PfsenseConfig pfsCfg) { - pfs_cfg_holder = Optional.of(pfsCfg); - } - - if (pfs_cfg_holder.isEmpty()) { - logger.error("[AclCleanup] PFSENSE_CFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - PfsenseConfig pfs_cfg = pfs_cfg_holder.get(); - - try { - List current_acl_ips = fetchAcl(logger, pfs_cfg); - logger.info("IPs in ACL ({}): {}", current_acl_ips.size(), String.join(" ", current_acl_ips)); - - List ips_to_remove = checkIpsInDatabase(current_acl_ips, logger, dbcore); - - if (ips_to_remove.isEmpty()) { - logger.info("No IPs to remove from ACL"); - } else { - logger.info("IPs to remove from ACL ({}): {}", ips_to_remove.size(), String.join(" ", ips_to_remove)); - - for (String ip : ips_to_remove) { - removeIpFromAcl(ip, logger, pfs_cfg); - } - - logger.info("Removed {} IPs from ACL", ips_to_remove.size()); - } - - List updated_acl_ips = fetchAcl(logger, pfs_cfg); - logger.info("IPs now in ACL ({}): {}", updated_acl_ips.size(), String.join(" ", updated_acl_ips)); - - } catch (IOException e) { - logger.error("Error in AclCleanupJob!"); - logger.error(e); - } - } - - /** - * Checks which IPs should be removed from ACL based on database activity - * Removes IPs that don't exist in DB or haven't been seen in last 10 minutes - */ - public List checkIpsInDatabase(List ips, Logger logger, DbCore dbcore) { - List to_remove = new ArrayList<>(); - - DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); - LocalDateTime ten_mins_ago = dbcore.now().minusMinutes(10); - - for (String ip : ips) { - if (!ctx.fetchExists(ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)))) { - logger.info("IP {} no longer in database.", ip); - to_remove.add(ip); - continue; - } - - boolean seen_in_last_10m = ctx.fetchExists( - ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)) - .and(Tables.PLAYER.LASTSEEN.gt(ten_mins_ago)) - ); - - if (!seen_in_last_10m) { - logger.info("IP {} not active within last 10 minutes.", ip); - to_remove.add(ip); - } - } - - return to_remove; - } - - /** - * Retrieves current IP list from ACL via socket connection - */ - public List fetchAcl(Logger logger, PfsenseConfig cfg) throws IOException { - List ip_list = new ArrayList<>(); - - Socket socket = new Socket(cfg.host, cfg.port); - socket.setSoTimeout(10000); - - OutputStream out = socket.getOutputStream(); - InputStream in = socket.getInputStream(); - - String command = "show acl #1\n"; - out.write(command.getBytes("ascii")); - - StringBuilder retval = new StringBuilder(); - byte[] buffer = new byte[16]; - int bytes_read; - - while ((bytes_read = in.read(buffer)) != -1) { - retval.append(new String(buffer, 0, bytes_read)); - } - - socket.close(); - - String[] raw_ips = retval.toString().split("\n"); - for (String ip : raw_ips) { - if (ip.trim().isEmpty()) { - continue; - } - - String[] parts = ip.split(" "); - if (parts.length >= 2) { - ip_list.add(parts[1]); - } - } - - return ip_list; - } - - /** - * Removes a specific IP from ACL #1 via socket command - */ - public void removeIpFromAcl(String ip, Logger logger, PfsenseConfig cfg) throws IOException { - logger.info("Removing {} from ACL", ip); - - Socket socket = new Socket(cfg.host, cfg.port); - OutputStream out = socket.getOutputStream(); - - String command = "del acl #1 " + ip + "\n"; - out.write(command.getBytes("ascii")); - - socket.close(); - } -} +package me.aa07.paradise.taskdaemon.core.modules.aclcleanup; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.PfsenseConfig; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.gamedb.Tables; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +/** + * Scheduled job that cleans up ACL by removing inactive player IPs + */ +@DisallowConcurrentExecution // NO +public class AclCleanupJob implements Job { + + /** + * Main job execution - fetches current ACL, checks database for inactive IPs, and removes them + */ + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + JobDataMap datamap = context.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[AclCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[AclCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Get pfSense configuration + Object raw_pfs_cfg = datamap.get("PFSENSE_CFG"); + Optional pfs_cfg_holder = Optional.empty(); + + if (raw_pfs_cfg instanceof PfsenseConfig pfsCfg) { + pfs_cfg_holder = Optional.of(pfsCfg); + } + + if (pfs_cfg_holder.isEmpty()) { + logger.error("[AclCleanup] PFSENSE_CFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + PfsenseConfig pfs_cfg = pfs_cfg_holder.get(); + + try { + List current_acl_ips = fetchAcl(logger, pfs_cfg); + logger.info("IPs in ACL ({}): {}", current_acl_ips.size(), String.join(" ", current_acl_ips)); + + List ips_to_remove = checkIpsInDatabase(current_acl_ips, logger, dbcore); + + if (ips_to_remove.isEmpty()) { + logger.info("No IPs to remove from ACL"); + } else { + logger.info("IPs to remove from ACL ({}): {}", ips_to_remove.size(), String.join(" ", ips_to_remove)); + + for (String ip : ips_to_remove) { + removeIpFromAcl(ip, logger, pfs_cfg); + } + + logger.info("Removed {} IPs from ACL", ips_to_remove.size()); + } + + List updated_acl_ips = fetchAcl(logger, pfs_cfg); + logger.info("IPs now in ACL ({}): {}", updated_acl_ips.size(), String.join(" ", updated_acl_ips)); + + } catch (IOException e) { + logger.error("Error in AclCleanupJob!"); + logger.error(e); + } + } + + /** + * Checks which IPs should be removed from ACL based on database activity + * Removes IPs that don't exist in DB or haven't been seen in last 10 minutes + */ + public List checkIpsInDatabase(List ips, Logger logger, DbCore dbcore) { + List to_remove = new ArrayList<>(); + + DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); + LocalDateTime ten_mins_ago = dbcore.now().minusMinutes(10); + + for (String ip : ips) { + if (!ctx.fetchExists(ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)))) { + logger.info("IP {} no longer in database.", ip); + to_remove.add(ip); + continue; + } + + boolean seen_in_last_10m = ctx.fetchExists( + ctx.select(Tables.PLAYER.CKEY).from(Tables.PLAYER).where(Tables.PLAYER.IP.eq(ip)) + .and(Tables.PLAYER.LASTSEEN.gt(ten_mins_ago)) + ); + + if (!seen_in_last_10m) { + logger.info("IP {} not active within last 10 minutes.", ip); + to_remove.add(ip); + } + } + + return to_remove; + } + + /** + * Retrieves current IP list from ACL via socket connection + */ + public List fetchAcl(Logger logger, PfsenseConfig cfg) throws IOException { + List ip_list = new ArrayList<>(); + + Socket socket = new Socket(cfg.host, cfg.port); + socket.setSoTimeout(10000); + + OutputStream out = socket.getOutputStream(); + InputStream in = socket.getInputStream(); + + String command = "show acl #1\n"; + out.write(command.getBytes("ascii")); + + StringBuilder retval = new StringBuilder(); + byte[] buffer = new byte[16]; + int bytes_read; + + while ((bytes_read = in.read(buffer)) != -1) { + retval.append(new String(buffer, 0, bytes_read)); + } + + socket.close(); + + String[] raw_ips = retval.toString().split("\n"); + for (String ip : raw_ips) { + if (ip.trim().isEmpty()) { + continue; + } + + String[] parts = ip.split(" "); + if (parts.length >= 2) { + ip_list.add(parts[1]); + } + } + + return ip_list; + } + + /** + * Removes a specific IP from ACL #1 via socket command + */ + public void removeIpFromAcl(String ip, Logger logger, PfsenseConfig cfg) throws IOException { + logger.info("Removing {} from ACL", ip); + + Socket socket = new Socket(cfg.host, cfg.port); + OutputStream out = socket.getOutputStream(); + + String command = "del acl #1 " + ip + "\n"; + out.write(command.getBytes("ascii")); + + socket.close(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java index 520ea91..811ac88 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/devrank/DevRankJob.java @@ -1,158 +1,158 @@ -package me.aa07.paradise.taskdaemon.core.modules.devrank; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.database.forums.Tables; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.Admin; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.AdminRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.Result; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -/** - * Scheduled job that syncs dev team permissions between forums and game - * databases. - */ -@DisallowConcurrentExecution // NO -public class DevRankJob implements Job { - private static final int DEV_TEAM_GROUP = 39; - private static final int DEV_RANK = 100; - private static final int REMOVED_RANK = 105; - - @Override - public void execute(JobExecutionContext context) throws JobExecutionException { - JobDataMap data_map = context.getMergedJobDataMap(); - - // Get logger and db - Logger logger = (Logger) data_map.get("LOGGER"); - DbCore dbcore = (DbCore) data_map.get("DBCORE"); - - if (logger == null || dbcore == null) { - System.err.println("[DevRank] LOGGER or DBCORE was not set in JobDataMap."); - return; - } - - logger.info("[DevRank] Starting dev rank sync"); - - try { - DSLContext forums_db = dbcore.jooq(DatabaseType.Forums); - DSLContext game_db = dbcore.jooq(DatabaseType.GameDb); - - // Collect all dev team ckeys from forums database - List dev_team_ckeys = new ArrayList<>(); - - Result forum_records = forums_db.select( - Tables.CORE_MEMBERS.MEMBER_ID, - Tables.CORE_MEMBERS.NAME, - Tables.CORE_MEMBERS.MEMBER_GROUP_ID, - Tables.CORE_MEMBERS.MGROUP_OTHERS, - Tables.CORE_PFIELDS_CONTENT.FIELD_10).from(Tables.CORE_MEMBERS) - .leftJoin(Tables.CORE_PFIELDS_CONTENT) - .on(Tables.CORE_MEMBERS.MEMBER_ID.eq(Tables.CORE_PFIELDS_CONTENT.MEMBER_ID)) - .fetch(); - - for (Record rec : forum_records) { - int primary_group = rec.get(Tables.CORE_MEMBERS.MEMBER_GROUP_ID); - String other_groups = rec.get(Tables.CORE_MEMBERS.MGROUP_OTHERS); - String ckey = rec.get(Tables.CORE_PFIELDS_CONTENT.FIELD_10); - - Set all_groups = new HashSet(); - all_groups.add(primary_group); - - if (other_groups != null && !other_groups.isBlank()) { - Arrays.stream(other_groups.split(",")) - .filter(g -> !g.isBlank()) - .map(Integer::parseInt) - .forEach(all_groups::add); - } - - if (all_groups.contains(DEV_TEAM_GROUP)) { - if (ckey == null || ckey.isBlank()) { - logger.warn("[DevRank] Forums user {} (ID {}) has no linked ckey", - rec.get(Tables.CORE_MEMBERS.NAME), rec.get(Tables.CORE_MEMBERS.MEMBER_ID)); - continue; - } - - String cleaned = ckey.toLowerCase().replaceAll("[\\s_\\-]", ""); - dev_team_ckeys.add(cleaned); - } - } - - // Load all ingame admins - Map ingame_admins = new HashMap<>(); - - Result admin_records = game_db.selectFrom(Admin.ADMIN).fetch(); - - for (AdminRecord rec : admin_records) { - String ckey = rec.get(Admin.ADMIN.CKEY); - int perm_rank = REMOVED_RANK; - if (rec.getPermissionsRank() != null) { - perm_rank = rec.getPermissionsRank(); - } - - ingame_admins.put(ckey, new AdminEntry(rec.get(Admin.ADMIN.ID), perm_rank)); - } - - // Apply permissions to those who need them - for (String ckey : dev_team_ckeys) { - AdminEntry entry = ingame_admins.get(ckey); - - if (entry == null) { - logger.info("[DevRank] Ckey {} not in admin table, adding new dev", ckey); - game_db.insertInto(Admin.ADMIN) - .set(Admin.ADMIN.CKEY, ckey) - .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) - .execute(); - } else { - if (entry.rank() == REMOVED_RANK) { - logger.info("[DevRank] Resetting {} to dev team with new flag", ckey); - game_db.update(Admin.ADMIN) - .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) - .where(Admin.ADMIN.ID.eq(entry.id())) - .execute(); - } else { - logger.debug("[DevRank] {} already has dev flag", ckey); - } - } - } - - // Remove devteam flag from those who no longer qualify - for (Map.Entry entry : ingame_admins.entrySet()) { - String ckey = entry.getKey(); - AdminEntry admin = entry.getValue(); - - if (!dev_team_ckeys.contains(ckey)) { - if (admin.rank() == DEV_RANK) { - logger.info("[DevRank] {} only had dev flag, removing rank and flags", ckey); - game_db.update(Admin.ADMIN) - .set(Admin.ADMIN.PERMISSIONS_RANK, REMOVED_RANK) - .where(Admin.ADMIN.ID.eq(admin.id())) - .execute(); - } - } - } - - logger.info("[DevRank] Finished sync, {} devteam users processed", dev_team_ckeys.size()); - - } catch (Exception e) { - logger.error("[DevRank] Error during sync", e); - throw new JobExecutionException(e); - } - } - - private record AdminEntry(int id, int rank) { - } -} +package me.aa07.paradise.taskdaemon.core.modules.devrank; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.forums.Tables; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.Admin; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.AdminRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +/** + * Scheduled job that syncs dev team permissions between forums and game + * databases. + */ +@DisallowConcurrentExecution // NO +public class DevRankJob implements Job { + private static final int DEV_TEAM_GROUP = 39; + private static final int DEV_RANK = 100; + private static final int REMOVED_RANK = 105; + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + JobDataMap data_map = context.getMergedJobDataMap(); + + // Get logger and db + Logger logger = (Logger) data_map.get("LOGGER"); + DbCore dbcore = (DbCore) data_map.get("DBCORE"); + + if (logger == null || dbcore == null) { + System.err.println("[DevRank] LOGGER or DBCORE was not set in JobDataMap."); + return; + } + + logger.info("[DevRank] Starting dev rank sync"); + + try { + DSLContext forums_db = dbcore.jooq(DatabaseType.Forums); + DSLContext game_db = dbcore.jooq(DatabaseType.GameDb); + + // Collect all dev team ckeys from forums database + List dev_team_ckeys = new ArrayList<>(); + + Result forum_records = forums_db.select( + Tables.CORE_MEMBERS.MEMBER_ID, + Tables.CORE_MEMBERS.NAME, + Tables.CORE_MEMBERS.MEMBER_GROUP_ID, + Tables.CORE_MEMBERS.MGROUP_OTHERS, + Tables.CORE_PFIELDS_CONTENT.FIELD_10).from(Tables.CORE_MEMBERS) + .leftJoin(Tables.CORE_PFIELDS_CONTENT) + .on(Tables.CORE_MEMBERS.MEMBER_ID.eq(Tables.CORE_PFIELDS_CONTENT.MEMBER_ID)) + .fetch(); + + for (Record rec : forum_records) { + int primary_group = rec.get(Tables.CORE_MEMBERS.MEMBER_GROUP_ID); + String other_groups = rec.get(Tables.CORE_MEMBERS.MGROUP_OTHERS); + String ckey = rec.get(Tables.CORE_PFIELDS_CONTENT.FIELD_10); + + Set all_groups = new HashSet(); + all_groups.add(primary_group); + + if (other_groups != null && !other_groups.isBlank()) { + Arrays.stream(other_groups.split(",")) + .filter(g -> !g.isBlank()) + .map(Integer::parseInt) + .forEach(all_groups::add); + } + + if (all_groups.contains(DEV_TEAM_GROUP)) { + if (ckey == null || ckey.isBlank()) { + logger.warn("[DevRank] Forums user {} (ID {}) has no linked ckey", + rec.get(Tables.CORE_MEMBERS.NAME), rec.get(Tables.CORE_MEMBERS.MEMBER_ID)); + continue; + } + + String cleaned = ckey.toLowerCase().replaceAll("[\\s_\\-]", ""); + dev_team_ckeys.add(cleaned); + } + } + + // Load all ingame admins + Map ingame_admins = new HashMap<>(); + + Result admin_records = game_db.selectFrom(Admin.ADMIN).fetch(); + + for (AdminRecord rec : admin_records) { + String ckey = rec.get(Admin.ADMIN.CKEY); + int perm_rank = REMOVED_RANK; + if (rec.getPermissionsRank() != null) { + perm_rank = rec.getPermissionsRank(); + } + + ingame_admins.put(ckey, new AdminEntry(rec.get(Admin.ADMIN.ID), perm_rank)); + } + + // Apply permissions to those who need them + for (String ckey : dev_team_ckeys) { + AdminEntry entry = ingame_admins.get(ckey); + + if (entry == null) { + logger.info("[DevRank] Ckey {} not in admin table, adding new dev", ckey); + game_db.insertInto(Admin.ADMIN) + .set(Admin.ADMIN.CKEY, ckey) + .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) + .execute(); + } else { + if (entry.rank() == REMOVED_RANK) { + logger.info("[DevRank] Resetting {} to dev team with new flag", ckey); + game_db.update(Admin.ADMIN) + .set(Admin.ADMIN.PERMISSIONS_RANK, DEV_RANK) + .where(Admin.ADMIN.ID.eq(entry.id())) + .execute(); + } else { + logger.debug("[DevRank] {} already has dev flag", ckey); + } + } + } + + // Remove devteam flag from those who no longer qualify + for (Map.Entry entry : ingame_admins.entrySet()) { + String ckey = entry.getKey(); + AdminEntry admin = entry.getValue(); + + if (!dev_team_ckeys.contains(ckey)) { + if (admin.rank() == DEV_RANK) { + logger.info("[DevRank] {} only had dev flag, removing rank and flags", ckey); + game_db.update(Admin.ADMIN) + .set(Admin.ADMIN.PERMISSIONS_RANK, REMOVED_RANK) + .where(Admin.ADMIN.ID.eq(admin.id())) + .execute(); + } + } + } + + logger.info("[DevRank] Finished sync, {} devteam users processed", dev_team_ckeys.size()); + + } catch (Exception e) { + logger.error("[DevRank] Error during sync", e); + throw new JobExecutionException(e); + } + } + + private record AdminEntry(int id, int rank) { + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java index 6088ff5..4c5cd29 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ingameverifiedsync/IngameVerifiedSyncJob.java @@ -1,358 +1,358 @@ -package me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.services.invision.InvisionUtil; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.core.util.UtilStr; -import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; -import me.aa07.paradise.taskdaemon.database.automation.Tables; -import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PlayerRecord; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.Logger; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Result; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class IngameVerifiedSyncJob implements Job { - /* - - - pull raw data from authentik - - keep track of new ones - - get discord IDs and add into table - - get forum IDs and add into table - - do forum syncage based on if their IGV or IGB - - remove entries from directory table that werent there before - - schedule task in ALICE to strip discord role - - remove forum group - - slap authentik ID onto rows in the player table - - */ - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[IngameVerifiedSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - logger.info("[IngameVerifiedSync] Starting up"); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[IngameVerifiedSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Now get the invision util - Object raw_iu = datamap.get("IU"); - Optional iu_holder = Optional.empty(); - - if (raw_iu instanceof InvisionUtil iu2) { - iu_holder = Optional.of(iu2); - } - - if (!iu_holder.isPresent()) { - logger.error("[IngameVerifiedSync] IU WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - InvisionUtil iu = iu_holder.get(); - - DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); - DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); - DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); - - // For now - we just want ckeys and authentik IDs - no more - Result authentik_records = authentik_jooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) - .fetch(); - - logger.info(String.format("[IngameVerifiedSync] Found %s accounts with linked byond accounts", authentik_records.size())); - - ArrayList only_authentik_ids = new ArrayList(); - - for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { - only_authentik_ids.add(acuscr.getUserId()); - } - - // Records to purge - Result records_to_purge = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.notIn(only_authentik_ids)) - .fetch(); - - int removed = 0; - for (IngameVerifiedDirectoryRecord ivdr : records_to_purge) { - // Step 1 - Strip their rank on the forums - if (ivdr.getForumId() != null) { - int forum_id = ivdr.getForumId(); - Pair> gsg_response = iu.getUserSecondaryGroups(forum_id); - - if (gsg_response.getLeft()) { - List forum_secondaries = gsg_response.getRight(); - if (forum_secondaries.contains(Integer.valueOf(UtilConst.INVISION_INGAMEVERIFIED_GID))) { - // This cast looks silly - but without it, java treats it as a numerical index not an object - forum_secondaries.remove(Integer.valueOf(UtilConst.INVISION_INGAMEVERIFIED_GID)); - - boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); - - if (!ssg_response) { - logger.warn(String.format("[IngameVerifiedSync] Could not update secondaries for %s", forum_id)); - } - } - } else { - logger.warn(String.format("[IngameVerifiedSync] Could not get secondaries for %s", forum_id)); - } - } - - // Step 2 - Strip their rank on the discord - if (ivdr.getDiscordId() != null) { - TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("REMOVE_IGV_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = ivdr.getDiscordId(); - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbcore.now()); - tqr.store(); - } - - // Step 3 - Clear their FUID ingame - PlayerRecord ingame_record = game_jooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER.CKEY.eq(ivdr.getCkey())) - .fetchOne(); - - if (ingame_record != null) { - ingame_record.setFuid(null); - ingame_record.store(); - } - - // Step 4 - Clear from directory - ivdr.delete(); - removed++; - } - logger.info(String.format("[IngameVerifiedSync] Removed %s records for members who unlinked byond", removed)); - - // Now get a list of banned people - HashSet banned_ckeys = new HashSet(); - - // Shared - Condition unbanned_null = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.UNBANNED.isNull(); - Condition unbanned_0 = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.UNBANNED.eq((byte) 0); - Condition unban_combined_or = unbanned_null.or(unbanned_0); - - // Permaban - Condition bantype_perma = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.BANTYPE.eq("PERMABAN"); - - // Tempban - Condition bantype_temp = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.BANTYPE.eq("TEMPBAN"); - Condition tempban_expired = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.EXPIRATION_TIME.gt(dbcore.now()); - Condition tempban_combined_final = bantype_temp.and(tempban_expired); - - // Final condition - Condition ban_check_merged = bantype_perma.or(tempban_combined_final); - Condition is_banned = unban_combined_or.and(ban_check_merged); - - Result> raw_bans_result = game_jooq.selectDistinct(me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.CKEY) - .from(me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN) - .where(is_banned) - .fetch(); - - for (Record1 rbr : raw_bans_result) { - banned_ckeys.add(rbr.value1()); - } - - logger.info(String.format("[IngameVerifiedSync] Loaded %s banned ckeys", banned_ckeys.size())); - - // Now begin the mass inserts - int raw_added = 0; - int raw_updated = 0; - for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { - int authentik_user_ud = acuscr.getUserId(); - // First - see if they already exist in the directory - - if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)))) { - - // Update if they do - raw_updated++; - - IngameVerifiedDirectoryRecord ivdr = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)).fetchOne(); - - ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); - ivdr.store(); - } else { - // Insert if they dont - IngameVerifiedDirectoryRecord ivdr = automation_jooq.newRecord(Tables.INGAME_VERIFIED_DIRECTORY); - ivdr.setAuthentikId(acuscr.getUserId()); - ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); - ivdr.store(); - raw_added++; - } - } - - logger.info(String.format("[IngameVerifiedSync] Added %s records and updated %s for members who are linked", raw_added, raw_updated)); - - // Now slap forum IDs and discord onto them - Result records_to_update = automation_jooq - .selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) - .fetch(); - - Result authentik_forums_records = authentik_jooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.INVISION_OAUTH_SERVICE_ID)) - .fetch(); - - Result authentik_discord_records = authentik_jooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) - .fetch(); - - Result gamedb_records = game_jooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER).fetch(); - - HashMap authentik2forum = new HashMap(); - HashMap authentik2discord = new HashMap(); - HashMap ckey2gamerecord = new HashMap(); - - for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_forums_records) { - String forum_id = UtilStr.onlyNumbers(acuscr.getIdentifier()); // Some of these have UTF8-BOMs on them. I wish I was kidding. - authentik2forum.put(acuscr.getUserId(), Integer.parseInt(forum_id)); - } - - for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_discord_records) { - authentik2discord.put(acuscr.getUserId(), Long.parseLong(acuscr.getIdentifier())); - } - - for (PlayerRecord pr : gamedb_records) { - ckey2gamerecord.put(pr.getCkey(), pr); - } - - int forum_updated = 0; - int discord_updated = 0; - int game_updated = 0; - for (IngameVerifiedDirectoryRecord ivdr : records_to_update) { - boolean update_made = false; - int authentik_id = ivdr.getAuthentikId(); - String ckey = ivdr.getCkey(); - int forum_id = 0; - long discord_id = 0L; - - if (authentik2forum.containsKey(authentik_id)) { - forum_id = authentik2forum.get(authentik_id); - } - - if (authentik2discord.containsKey(authentik_id)) { - discord_id = authentik2discord.get(authentik_id); - } - - // Add forums - if (forum_id > 0 && (ivdr.getForumId() == null || !ivdr.getForumId().equals(forum_id))) { - // See if we have them as an ID - ivdr.setForumId(forum_id); - update_made = true; - forum_updated++; - - // ADD INVISION SECONDARY GROUPS HERE - Pair> gsg_response = iu.getUserSecondaryGroups(forum_id); - - if (gsg_response.getLeft()) { - List forum_secondaries = gsg_response.getRight(); - int target_group = banned_ckeys.contains(ckey) ? UtilConst.INVISION_INGAMEBANNED_GID : UtilConst.INVISION_INGAMEVERIFIED_GID; - if (!forum_secondaries.contains(target_group)) { - forum_secondaries.add(target_group); - - boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); - - if (!ssg_response) { - logger.warn(String.format("[IngameVerifiedSync] Could not update secondaries for %s", forum_id)); - } - } - } else { - logger.warn(String.format("[IngameVerifiedSync] Could not get secondaries for %s", forum_id)); - } - } - - // Add discord - if (discord_id > 0 && (ivdr.getDiscordId() == null || !ivdr.getDiscordId().equals(discord_id))) { - ivdr.setDiscordId(discord_id); - update_made = true; - discord_updated++; - - // And make the ALICE task - TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType(banned_ckeys.contains(ckey) ? "ADD_IGB_ROLE" : "ADD_IGV_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbcore.now()); - tqr.store(); - } - - if (ckey2gamerecord.containsKey(ivdr.getCkey())) { - PlayerRecord game_row = ckey2gamerecord.get(ivdr.getCkey()); - if (game_row.getFuid() == null || game_row.getFuid() != authentik_id) { - game_row.setFuid((long) authentik_id); - game_row.store(); - game_updated++; - } - } - - if (update_made) { - ivdr.store(); - } - } - - logger.info(String.format("[IngameVerifiedSync] Update stats discord/forum/game - %s/%s/%s", discord_updated, forum_updated, game_updated)); - logger.info("[IngameVerifiedSync] Process complete"); - } -} +package me.aa07.paradise.taskdaemon.core.modules.ingameverifiedsync; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.services.invision.InvisionUtil; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.core.util.UtilStr; +import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; +import me.aa07.paradise.taskdaemon.database.automation.Tables; +import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.IngameVerifiedDirectoryRecord; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.PlayerRecord; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.Logger; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class IngameVerifiedSyncJob implements Job { + /* + + - pull raw data from authentik + - keep track of new ones + - get discord IDs and add into table + - get forum IDs and add into table + - do forum syncage based on if their IGV or IGB + - remove entries from directory table that werent there before + - schedule task in ALICE to strip discord role + - remove forum group + - slap authentik ID onto rows in the player table + + */ + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[IngameVerifiedSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + logger.info("[IngameVerifiedSync] Starting up"); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[IngameVerifiedSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Now get the invision util + Object raw_iu = datamap.get("IU"); + Optional iu_holder = Optional.empty(); + + if (raw_iu instanceof InvisionUtil iu2) { + iu_holder = Optional.of(iu2); + } + + if (!iu_holder.isPresent()) { + logger.error("[IngameVerifiedSync] IU WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + InvisionUtil iu = iu_holder.get(); + + DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); + DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); + DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); + + // For now - we just want ckeys and authentik IDs - no more + Result authentik_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) + .fetch(); + + logger.info(String.format("[IngameVerifiedSync] Found %s accounts with linked byond accounts", authentik_records.size())); + + ArrayList only_authentik_ids = new ArrayList(); + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { + only_authentik_ids.add(acuscr.getUserId()); + } + + // Records to purge + Result records_to_purge = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.notIn(only_authentik_ids)) + .fetch(); + + int removed = 0; + for (IngameVerifiedDirectoryRecord ivdr : records_to_purge) { + // Step 1 - Strip their rank on the forums + if (ivdr.getForumId() != null) { + int forum_id = ivdr.getForumId(); + Pair> gsg_response = iu.getUserSecondaryGroups(forum_id); + + if (gsg_response.getLeft()) { + List forum_secondaries = gsg_response.getRight(); + if (forum_secondaries.contains(Integer.valueOf(UtilConst.INVISION_INGAMEVERIFIED_GID))) { + // This cast looks silly - but without it, java treats it as a numerical index not an object + forum_secondaries.remove(Integer.valueOf(UtilConst.INVISION_INGAMEVERIFIED_GID)); + + boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); + + if (!ssg_response) { + logger.warn(String.format("[IngameVerifiedSync] Could not update secondaries for %s", forum_id)); + } + } + } else { + logger.warn(String.format("[IngameVerifiedSync] Could not get secondaries for %s", forum_id)); + } + } + + // Step 2 - Strip their rank on the discord + if (ivdr.getDiscordId() != null) { + TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("REMOVE_IGV_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = ivdr.getDiscordId(); + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbcore.now()); + tqr.store(); + } + + // Step 3 - Clear their FUID ingame + PlayerRecord ingame_record = game_jooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER.CKEY.eq(ivdr.getCkey())) + .fetchOne(); + + if (ingame_record != null) { + ingame_record.setFuid(null); + ingame_record.store(); + } + + // Step 4 - Clear from directory + ivdr.delete(); + removed++; + } + logger.info(String.format("[IngameVerifiedSync] Removed %s records for members who unlinked byond", removed)); + + // Now get a list of banned people + HashSet banned_ckeys = new HashSet(); + + // Shared + Condition unbanned_null = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.UNBANNED.isNull(); + Condition unbanned_0 = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.UNBANNED.eq((byte) 0); + Condition unban_combined_or = unbanned_null.or(unbanned_0); + + // Permaban + Condition bantype_perma = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.BANTYPE.eq("PERMABAN"); + + // Tempban + Condition bantype_temp = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.BANTYPE.eq("TEMPBAN"); + Condition tempban_expired = me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.EXPIRATION_TIME.gt(dbcore.now()); + Condition tempban_combined_final = bantype_temp.and(tempban_expired); + + // Final condition + Condition ban_check_merged = bantype_perma.or(tempban_combined_final); + Condition is_banned = unban_combined_or.and(ban_check_merged); + + Result> raw_bans_result = game_jooq.selectDistinct(me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN.CKEY) + .from(me.aa07.paradise.taskdaemon.database.gamedb.Tables.BAN) + .where(is_banned) + .fetch(); + + for (Record1 rbr : raw_bans_result) { + banned_ckeys.add(rbr.value1()); + } + + logger.info(String.format("[IngameVerifiedSync] Loaded %s banned ckeys", banned_ckeys.size())); + + // Now begin the mass inserts + int raw_added = 0; + int raw_updated = 0; + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_records) { + int authentik_user_ud = acuscr.getUserId(); + // First - see if they already exist in the directory + + if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)))) { + + // Update if they do + raw_updated++; + + IngameVerifiedDirectoryRecord ivdr = automation_jooq.selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .where(Tables.INGAME_VERIFIED_DIRECTORY.AUTHENTIK_ID.eq(authentik_user_ud)).fetchOne(); + + ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); + ivdr.store(); + } else { + // Insert if they dont + IngameVerifiedDirectoryRecord ivdr = automation_jooq.newRecord(Tables.INGAME_VERIFIED_DIRECTORY); + ivdr.setAuthentikId(acuscr.getUserId()); + ivdr.setCkey(UtilStr.cleanCkey(acuscr.getIdentifier())); + ivdr.store(); + raw_added++; + } + } + + logger.info(String.format("[IngameVerifiedSync] Added %s records and updated %s for members who are linked", raw_added, raw_updated)); + + // Now slap forum IDs and discord onto them + Result records_to_update = automation_jooq + .selectFrom(Tables.INGAME_VERIFIED_DIRECTORY) + .fetch(); + + Result authentik_forums_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.INVISION_OAUTH_SERVICE_ID)) + .fetch(); + + Result authentik_discord_records = authentik_jooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) + .fetch(); + + Result gamedb_records = game_jooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.PLAYER).fetch(); + + HashMap authentik2forum = new HashMap(); + HashMap authentik2discord = new HashMap(); + HashMap ckey2gamerecord = new HashMap(); + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_forums_records) { + String forum_id = UtilStr.onlyNumbers(acuscr.getIdentifier()); // Some of these have UTF8-BOMs on them. I wish I was kidding. + authentik2forum.put(acuscr.getUserId(), Integer.parseInt(forum_id)); + } + + for (AuthentikCoreUsersourceconnectionRecord acuscr : authentik_discord_records) { + authentik2discord.put(acuscr.getUserId(), Long.parseLong(acuscr.getIdentifier())); + } + + for (PlayerRecord pr : gamedb_records) { + ckey2gamerecord.put(pr.getCkey(), pr); + } + + int forum_updated = 0; + int discord_updated = 0; + int game_updated = 0; + for (IngameVerifiedDirectoryRecord ivdr : records_to_update) { + boolean update_made = false; + int authentik_id = ivdr.getAuthentikId(); + String ckey = ivdr.getCkey(); + int forum_id = 0; + long discord_id = 0L; + + if (authentik2forum.containsKey(authentik_id)) { + forum_id = authentik2forum.get(authentik_id); + } + + if (authentik2discord.containsKey(authentik_id)) { + discord_id = authentik2discord.get(authentik_id); + } + + // Add forums + if (forum_id > 0 && (ivdr.getForumId() == null || !ivdr.getForumId().equals(forum_id))) { + // See if we have them as an ID + ivdr.setForumId(forum_id); + update_made = true; + forum_updated++; + + // ADD INVISION SECONDARY GROUPS HERE + Pair> gsg_response = iu.getUserSecondaryGroups(forum_id); + + if (gsg_response.getLeft()) { + List forum_secondaries = gsg_response.getRight(); + int target_group = banned_ckeys.contains(ckey) ? UtilConst.INVISION_INGAMEBANNED_GID : UtilConst.INVISION_INGAMEVERIFIED_GID; + if (!forum_secondaries.contains(target_group)) { + forum_secondaries.add(target_group); + + boolean ssg_response = iu.updateUserSecondaryGroups(forum_id, forum_secondaries); + + if (!ssg_response) { + logger.warn(String.format("[IngameVerifiedSync] Could not update secondaries for %s", forum_id)); + } + } + } else { + logger.warn(String.format("[IngameVerifiedSync] Could not get secondaries for %s", forum_id)); + } + } + + // Add discord + if (discord_id > 0 && (ivdr.getDiscordId() == null || !ivdr.getDiscordId().equals(discord_id))) { + ivdr.setDiscordId(discord_id); + update_made = true; + discord_updated++; + + // And make the ALICE task + TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType(banned_ckeys.contains(ckey) ? "ADD_IGB_ROLE" : "ADD_IGV_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbcore.now()); + tqr.store(); + } + + if (ckey2gamerecord.containsKey(ivdr.getCkey())) { + PlayerRecord game_row = ckey2gamerecord.get(ivdr.getCkey()); + if (game_row.getFuid() == null || game_row.getFuid() != authentik_id) { + game_row.setFuid((long) authentik_id); + game_row.store(); + game_updated++; + } + } + + if (update_made) { + ivdr.store(); + } + } + + logger.info(String.format("[IngameVerifiedSync] Update stats discord/forum/game - %s/%s/%s", discord_updated, forum_updated, game_updated)); + logger.info("[IngameVerifiedSync] Process complete"); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java index 542437d..a48e0d1 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/ip2asn/Ip2AsnJob.java @@ -1,211 +1,211 @@ -package me.aa07.paradise.taskdaemon.core.modules.ip2asn; - -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.Ip2AsnSerivceConfig; -import me.aa07.paradise.taskdaemon.core.models.ip2asn.Ip2AsnResponseModel; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.database.gamedb.Tables; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.Ip2groupRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Select; -import org.jooq.types.UInteger; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class Ip2AsnJob implements Job { - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[Ip2Asn] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[Ip2Asn] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Now get our config - Object raw_ip2asn_cfg = datamap.get("IP2ASNCFG"); - Optional ip2asn_cfg_holder = Optional.empty(); - - if (raw_ip2asn_cfg instanceof Ip2AsnSerivceConfig ip2asnCfg2) { - ip2asn_cfg_holder = Optional.of(ip2asnCfg2); - } - - if (!ip2asn_cfg_holder.isPresent()) { - logger.error("[Ip2Asn] IP2ASNCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Ip2AsnSerivceConfig config = ip2asn_cfg_holder.get(); - - DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); - - // So - // JOOQ doesnt have INET_ATON, presumably because its a MySQL native - // So we have to do the manual selects then work out the differences - // To do this we need: - // 1. All IPs from player seen in the last 30 days - // 2. Then remove the IPs in ip2group that have been updated in the last 7 days - // Then we need to update IP2Group or insert where required - LocalDateTime last_30_days = LocalDateTime.now().minusDays(30); - LocalDateTime last_7_days = LocalDateTime.now().minusDays(7); - - logger.info("[Ip2Asn] Pulling player IPs..."); - - // Get the player IPs - List player_ips = ctx.select(Tables.PLAYER.IP).from(Tables.PLAYER) - .where(Tables.PLAYER.LASTSEEN.gt(last_30_days)).fetch() - .getValues(Tables.PLAYER.IP, String.class); - - logger.info(String.format("[Ip2Asn] Pulled %s player IPs, now pulling ip2group IPs...", player_ips.size())); - - List ip2group_ips = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) - .where(Tables.IP2GROUP.DATE.gt(last_7_days)).fetch() - .getValues(Tables.IP2GROUP.IP, UInteger.class); - - logger.info(String.format("[Ip2Asn] Pulled %s ip2group IPs, now parsing...", ip2group_ips.size())); - - // Now turn that to regular IPs - ArrayList parsed_ip2group_ips = new ArrayList(); - - for (UInteger ip_uint : ip2group_ips) { - String ip_str = uint2ip(ip_uint); - parsed_ip2group_ips.add(ip_str); - } - - ArrayList ips_to_get = new ArrayList(); - - // Start with the IPs in player - then see if they exist in the last 7 days in - // ip2group - for (String ip : player_ips) { - if (!parsed_ip2group_ips.contains(ip)) { - ips_to_get.add(ip); - } - } - - logger.info(String.format("[Ip2Asn] Found %s IPs to refresh groups for", ips_to_get.size())); - - if (ips_to_get.size() == 0) { - logger.info("[Ip2Asn] No work to do"); - return; - } - - // Now do the big refresh - HttpClient client = HttpClient.newHttpClient(); - for (String ip : ips_to_get) { - try { - // Get our AS - probably a more elegant way to do this but meh - HttpRequest httpreq = HttpRequest.newBuilder() - .uri(new URI(String.format("%s%s", config.host, ip))) - .GET() - .header("Accept", "application/json") - .build(); - - // Send the request off - HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); - - // Decode the shenanigans - Ip2AsnResponseModel res_model = UtilConst.GSON.fromJson(response.body(), Ip2AsnResponseModel.class); - UInteger ip_uint = ip2uint(ip); - UInteger asn = UInteger.valueOf(res_model.asn); - - // See if our row exists, we update if it does - Select> fetch_test = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) - .where(Tables.IP2GROUP.IP.eq(ip_uint)); - - if (ctx.fetchExists(fetch_test)) { - // It exists, update - logger.info(String.format("[Ip2Asn] Updating row for %s", ip)); - ctx.update(Tables.IP2GROUP) - .set(Tables.IP2GROUP.GROUPSTR, asn) - .set(Tables.IP2GROUP.DATE, dbcore.now()) - .where(Tables.IP2GROUP.IP.eq(ip_uint)) - .execute(); - } else { - // It dont exist, insert - logger.info(String.format("[Ip2Asn] Inserting row for %s", ip)); - Ip2groupRecord record = ctx.newRecord(Tables.IP2GROUP); - record.setIp(ip_uint); - record.setDate(dbcore.now()); - record.setGroupstr(asn); - record.store(); - } - - } catch (Exception e) { - logger.warn(String.format("[Ip2Asn] Failed to get AS info for IP %s - skipping", ip)); - logger.warn(e); - } - } - - logger.info("[Ip2Asn] Processing complete"); - } - - // Turns a uint IP into its string form - private String uint2ip(UInteger ip) { - long ip_long = ip.longValue(); - return String.format("%d.%d.%d.%d", - (ip_long >> 24 & 0xff), - (ip_long >> 16 & 0xff), - (ip_long >> 8 & 0xff), - (ip_long & 0xff)); - } - - // Turns a string IP into its uint form - private UInteger ip2uint(String ip) { - String[] octets = ip.split("\\."); - - long ip_long = 0; - for (int i = 0; i < 4; i++) { - int octet = Integer.parseInt(octets[i]); - if (octet < 0 || octet > 255) { - throw new IllegalArgumentException("Invalid octet in IP address: " + octets[i]); - } - // This does the shift magic with the for loop, - // with each iteration shifting by 8 - ip_long |= ((long) octet << (24 - 8 * i)); - } - - return UInteger.valueOf(ip_long); - } -} +package me.aa07.paradise.taskdaemon.core.modules.ip2asn; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.Ip2AsnSerivceConfig; +import me.aa07.paradise.taskdaemon.core.models.ip2asn.Ip2AsnResponseModel; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.database.gamedb.Tables; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.Ip2groupRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Select; +import org.jooq.types.UInteger; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class Ip2AsnJob implements Job { + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[Ip2Asn] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[Ip2Asn] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Now get our config + Object raw_ip2asn_cfg = datamap.get("IP2ASNCFG"); + Optional ip2asn_cfg_holder = Optional.empty(); + + if (raw_ip2asn_cfg instanceof Ip2AsnSerivceConfig ip2asnCfg2) { + ip2asn_cfg_holder = Optional.of(ip2asnCfg2); + } + + if (!ip2asn_cfg_holder.isPresent()) { + logger.error("[Ip2Asn] IP2ASNCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Ip2AsnSerivceConfig config = ip2asn_cfg_holder.get(); + + DSLContext ctx = dbcore.jooq(DatabaseType.GameDb); + + // So + // JOOQ doesnt have INET_ATON, presumably because its a MySQL native + // So we have to do the manual selects then work out the differences + // To do this we need: + // 1. All IPs from player seen in the last 30 days + // 2. Then remove the IPs in ip2group that have been updated in the last 7 days + // Then we need to update IP2Group or insert where required + LocalDateTime last_30_days = LocalDateTime.now().minusDays(30); + LocalDateTime last_7_days = LocalDateTime.now().minusDays(7); + + logger.info("[Ip2Asn] Pulling player IPs..."); + + // Get the player IPs + List player_ips = ctx.select(Tables.PLAYER.IP).from(Tables.PLAYER) + .where(Tables.PLAYER.LASTSEEN.gt(last_30_days)).fetch() + .getValues(Tables.PLAYER.IP, String.class); + + logger.info(String.format("[Ip2Asn] Pulled %s player IPs, now pulling ip2group IPs...", player_ips.size())); + + List ip2group_ips = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) + .where(Tables.IP2GROUP.DATE.gt(last_7_days)).fetch() + .getValues(Tables.IP2GROUP.IP, UInteger.class); + + logger.info(String.format("[Ip2Asn] Pulled %s ip2group IPs, now parsing...", ip2group_ips.size())); + + // Now turn that to regular IPs + ArrayList parsed_ip2group_ips = new ArrayList(); + + for (UInteger ip_uint : ip2group_ips) { + String ip_str = uint2ip(ip_uint); + parsed_ip2group_ips.add(ip_str); + } + + ArrayList ips_to_get = new ArrayList(); + + // Start with the IPs in player - then see if they exist in the last 7 days in + // ip2group + for (String ip : player_ips) { + if (!parsed_ip2group_ips.contains(ip)) { + ips_to_get.add(ip); + } + } + + logger.info(String.format("[Ip2Asn] Found %s IPs to refresh groups for", ips_to_get.size())); + + if (ips_to_get.size() == 0) { + logger.info("[Ip2Asn] No work to do"); + return; + } + + // Now do the big refresh + HttpClient client = HttpClient.newHttpClient(); + for (String ip : ips_to_get) { + try { + // Get our AS - probably a more elegant way to do this but meh + HttpRequest httpreq = HttpRequest.newBuilder() + .uri(new URI(String.format("%s%s", config.host, ip))) + .GET() + .header("Accept", "application/json") + .build(); + + // Send the request off + HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); + + // Decode the shenanigans + Ip2AsnResponseModel res_model = UtilConst.GSON.fromJson(response.body(), Ip2AsnResponseModel.class); + UInteger ip_uint = ip2uint(ip); + UInteger asn = UInteger.valueOf(res_model.asn); + + // See if our row exists, we update if it does + Select> fetch_test = ctx.select(Tables.IP2GROUP.IP).from(Tables.IP2GROUP) + .where(Tables.IP2GROUP.IP.eq(ip_uint)); + + if (ctx.fetchExists(fetch_test)) { + // It exists, update + logger.info(String.format("[Ip2Asn] Updating row for %s", ip)); + ctx.update(Tables.IP2GROUP) + .set(Tables.IP2GROUP.GROUPSTR, asn) + .set(Tables.IP2GROUP.DATE, dbcore.now()) + .where(Tables.IP2GROUP.IP.eq(ip_uint)) + .execute(); + } else { + // It dont exist, insert + logger.info(String.format("[Ip2Asn] Inserting row for %s", ip)); + Ip2groupRecord record = ctx.newRecord(Tables.IP2GROUP); + record.setIp(ip_uint); + record.setDate(dbcore.now()); + record.setGroupstr(asn); + record.store(); + } + + } catch (Exception e) { + logger.warn(String.format("[Ip2Asn] Failed to get AS info for IP %s - skipping", ip)); + logger.warn(e); + } + } + + logger.info("[Ip2Asn] Processing complete"); + } + + // Turns a uint IP into its string form + private String uint2ip(UInteger ip) { + long ip_long = ip.longValue(); + return String.format("%d.%d.%d.%d", + (ip_long >> 24 & 0xff), + (ip_long >> 16 & 0xff), + (ip_long >> 8 & 0xff), + (ip_long & 0xff)); + } + + // Turns a string IP into its uint form + private UInteger ip2uint(String ip) { + String[] octets = ip.split("\\."); + + long ip_long = 0; + for (int i = 0; i < 4; i++) { + int octet = Integer.parseInt(octets[i]); + if (octet < 0 || octet > 255) { + throw new IllegalArgumentException("Invalid octet in IP address: " + octets[i]); + } + // This does the shift magic with the for loop, + // with each iteration shifting by 8 + ip_long |= ((long) octet << (24 - 8 * i)); + } + + return UInteger.valueOf(ip_long); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java index 5d67511..cc3dc58 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/patreonsync/PatreonSyncJob.java @@ -1,562 +1,562 @@ -package me.aa07.paradise.taskdaemon.core.modules.patreonsync; - -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import me.aa07.paradise.taskdaemon.core.config.PatreonConfig; -import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonOuterResponseModel; -import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonRawResponseModel; -import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonUser; -import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.core.util.UtilStr; -import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; -import me.aa07.paradise.taskdaemon.database.automation.Tables; -import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.PatreonSupportersRecord; -import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; -import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.DonatorsRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Result; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class PatreonSyncJob implements Job { - /* - - - pull raw data from patreon - - empty the holding table and fill with the patreon raw data - - get authentik IDs and add into table - - get discord IDs and add into table - - get ckeys and add into table - - mass sync ingame stuff - - */ - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[PatreonSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - logger.info("[PatreonSync] Starting up"); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[PatreonSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - dbcore.getClass(); - - // Now get our config - Object raw_ptrcfg = datamap.get("PTRCFG"); - Optional ptrcfg_holder = Optional.empty(); - - if (raw_ptrcfg instanceof PatreonConfig ptrcfg2) { - ptrcfg_holder = Optional.of(ptrcfg2); - } - - if (!ptrcfg_holder.isPresent()) { - logger.error("[PatreonSync] PTRCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - PatreonConfig ptrcfg = ptrcfg_holder.get(); - - // So - // The patreon API is kinda bad - // Everything goes into the same loose model and you have to manually parse out members and tiers - // Boy thats fun - - ArrayList users = new ArrayList(); - - try { - HttpRequest httpreq = HttpRequest.newBuilder() - .uri(URI.create("https://www.patreon.com/api/oauth2/v2/campaigns/" - + String.valueOf(ptrcfg.campaignId) - + "/members" - + "?include=currently_entitled_tiers,user" - + "&fields[member]=patron_status,currently_entitled_amount_cents" - + "&fields[tier]=amount_cents" - + "&page[count]=1000")) // This will break if we get >1000 patrons - // TODO Add pagination - .GET() - .header("Accept", "application/json") - .header("Authorization", String.format("Bearer %s", ptrcfg.creatorToken)) - .header("User-Agent", "ParadiseSS13-TaskDaemon/1.0") - .build(); - - HttpClient client = HttpClient.newHttpClient(); - logger.info("[PatreonSync] Pulling data"); - HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); - - String response_body = response.body(); - - /* - - // Uncomment if you need horrendous debugging - - File file = new File("patreon_debug_response.json"); - file.createNewFile(); - FileWriter fw = new FileWriter(file); - fw.write(response_body); - fw.close(); - */ - - PatreonOuterResponseModel holder = UtilConst.GSON.fromJson(response_body, PatreonOuterResponseModel.class); - - logger.info(String.format("[PatreonSync] Pulled %s raw models from Patreon", holder.data.size())); - - for (PatreonRawResponseModel raw_model : holder.data) { - if (raw_model.type.equalsIgnoreCase("member")) { - // Make sure theyre valid first - if (raw_model.attributes != null && raw_model.attributes.patronStatus != null) { - if (raw_model.attributes.patronStatus.equalsIgnoreCase("active_patron")) { - PatreonUser user = raw_model.asUser(); - users.add(user); - continue; - } - } - } - } - } catch (Exception ex) { - logger.error(ex); - } - - logger.info(String.format("[PatreonSync] Loaded %s active users", users.size())); - - ArrayList only_member_ids = new ArrayList(); - - for (PatreonUser pu : users) { - only_member_ids.add(pu.userId); - } - - // Step 1 - Clear out old users - DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); - - - Result records_to_purge = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.notIn(only_member_ids)) - .fetch(); - - for (PatreonSupportersRecord psr : records_to_purge) { - Long database_discord_id = psr.getDiscordId(); - - // Store our removal task if necessary - if (database_discord_id != null && (psr.getDonationTier() >= 2)) { - TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("REMOVE_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = database_discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbcore.now()); - tqr.store(); - } - - // Clear out the supporter record - psr.delete(); - } - - logger.info("[PatreonSync] Cleaned out old users"); - - int insert_count = 0; - int update_count = 0; - - // Step 2 - Add or insert new - for (PatreonUser pu : users) { - if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)))) { - - update_count++; - - // Fetch and update our record - PatreonSupportersRecord psr = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)) - .fetchSingle(); - - psr.setAmountCents(pu.amountInCents); - psr.setDonationTier(pu.maxTier().tierLevel); - - // And store - psr.store(); - } else { - insert_count++; - - // Make a new record - PatreonSupportersRecord psr = automation_jooq.newRecord(Tables.PATREON_SUPPORTERS); - psr.setMemberId(pu.userId); - psr.setAmountCents(pu.amountInCents); - psr.setDonationTier(pu.maxTier().tierLevel); - - // And store - psr.store(); - } - } - - logger.info(String.format("[PatreonSync] Inserted %s users, updated %s users", insert_count, update_count)); - - - DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); - DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); - - // Find authentik users - HashMap authentik_to_patreon = new HashMap(); - findAuthentikUsers(only_member_ids, authentik_jooq, automation_jooq, logger, authentik_to_patreon); - - // Now find discord users - findDiscordUsers(authentik_jooq, automation_jooq, logger, authentik_to_patreon, dbcore); - - // Now get ckeys - findCkeys(authentik_jooq, automation_jooq, logger, authentik_to_patreon); - - // And commit to game DB - updateGameDb(automation_jooq, game_jooq, logger); - - System.gc(); // This is important after this one - the response model is like multiple megabytes - logger.info("[PatreonSync] Process complete"); - } - - private static int patreonTier2ingameTier(int patreonTier) { - switch (patreonTier) { - case 0: // No tier - return 0; - case 1: // Supporter - return 0; - case 2: // Icon - return 1; - case 3: // Loadout - return 2; - case 4: // Silver - return 3; - case 5: // Gold - return 4; - default: - return 0; - } - } - - private static void findAuthentikUsers( - List memberIds, - DSLContext authentikJooq, - DSLContext automationJooq, - Logger logger, - HashMap authentik2patreon - ) { - for (int member_id : memberIds) { - // This loop should only have active member IDs for the DB given we just flushed everything else out - - // Get ready for a TON of absolute pathing - AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.IDENTIFIER.eq(String.valueOf(member_id))) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.PATREON_OAUTH_SERVICE_ID)) - .fetchOne(); - - if (authentik_record == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in Authentik!", member_id)); - continue; - } - - authentik2patreon.put(authentik_record.getUserId(), member_id); - } - - logger.info(String.format("[PatreonSync] Loaded %s Authentik users", authentik2patreon.size())); - - int supporters_authentik_updated = 0; - for (int authentik_id : authentik2patreon.keySet()) { - int patreon_id = authentik2patreon.get(authentik_id); - - // Get our DB record - - PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) - .fetchOne(); - - if (psr == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); - continue; - } - - supporters_authentik_updated++; - psr.setAuthentikId(authentik_id); - psr.store(); // Store it back in - } - - logger.info(String.format("[PatreonSync] Updated %s rows with Authentik users", supporters_authentik_updated)); - } - - private static void findDiscordUsers( - DSLContext authentikJooq, - DSLContext automationJooq, - Logger logger, - HashMap authentik2patreon, - DbCore dbCore - ) { - HashMap authentik2discord = new HashMap(); - - for (int authentik_id : authentik2patreon.keySet()) { - // This loop should only have active member IDs for the DB given we just flushed everything else out - - // Get ready for a TON of absolute pathing - AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) - .fetchOne(); - - if (authentik_record == null) { - logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); - continue; - } - - authentik2discord.put(authentik_id, Long.valueOf(authentik_record.getIdentifier())); - } - - logger.info(String.format("[PatreonSync] Loaded %s Discord users", authentik2discord.size())); - - int supporters_updated = 0; - for (int authentik_id : authentik2discord.keySet()) { - if (!authentik2patreon.containsKey(authentik_id)) { - logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); - } - - int patreon_id = authentik2patreon.get(authentik_id); - long discord_id = authentik2discord.get(authentik_id); - - // Get our DB record - PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) - .fetchOne(); - - if (psr == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); - continue; - } - - // See if we have a role to add - if (psr.getDiscordId() == null && psr.getDonationTier() >= 2) { - // We do - lets add the role - TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("ADD_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbCore.now()); - tqr.store(); - } - - // See if their tier changed between runs - if (psr.getDiscordId() != null && (psr.getLastrunDonationTier().intValue() != psr.getDonationTier().intValue())) { - // Tier changed - see if they went below the threshold - if (psr.getDonationTier() <= 1) { - // They went below the threshold - strip the role - TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("REMOVE_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbCore.now()); - tqr.store(); - } - - // Add if we just upgraded - if (psr.getDonationTier() >= 2 && psr.getLastrunDonationTier() == 1) { - TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); - tqr.setTaskId(UUID.randomUUID()); - tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); - tqr.setTaskType("ADD_PATREON_ROLE"); - - // No I will not apologise for these variable names - DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); - drtam.discordId = discord_id; - String drtam_json = UtilConst.GSON.toJson(drtam); - - tqr.setTaskArguments(drtam_json); - tqr.setDateInserted(dbCore.now()); - tqr.store(); - } - } - - supporters_updated++; - psr.setLastrunDonationTier(psr.getDonationTier()); - psr.setDiscordId(discord_id); - psr.store(); // Store it back in - } - - logger.info(String.format("[PatreonSync] Updated %s rows with Discord users", supporters_updated)); - } - - private static void findCkeys( - DSLContext authentikJooq, - DSLContext automationJooq, - Logger logger, - HashMap authentik2patreon - ) { - HashMap authentik2ckey = new HashMap(); - - for (int authentik_id : authentik2patreon.keySet()) { - // This loop should only have active member IDs for the DB given we just flushed everything else out - - // Get ready for a TON of absolute pathing - AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq - .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) - .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) - .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) - .fetchOne(); - - if (authentik_record == null) { - logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); - continue; - } - - // Normalise the ckey - String ckey = UtilStr.cleanCkey(authentik_record.getIdentifier()); - - authentik2ckey.put(authentik_id, ckey); - } - - logger.info(String.format("[PatreonSync] Loaded %s ckeys", authentik2ckey.size())); - - int supporters_updated = 0; - for (int authentik_id : authentik2ckey.keySet()) { - if (!authentik2patreon.containsKey(authentik_id)) { - logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); - } - - int patreon_id = authentik2patreon.get(authentik_id); - String ckey = authentik2ckey.get(authentik_id); - - // Get our DB record - PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) - .fetchOne(); - - if (psr == null) { - logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); - continue; - } - - supporters_updated++; - psr.setCkey(ckey); - psr.store(); // Store it back in - } - - logger.info(String.format("[PatreonSync] Updated %s rows with ckeys", supporters_updated)); - } - - private static void updateGameDb(DSLContext automationJooq, DSLContext gameJooq, Logger logger) { - - Result valid_records = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) - .where(Tables.PATREON_SUPPORTERS.CKEY.isNotNull()) - .and(Tables.PATREON_SUPPORTERS.DONATION_TIER.ge(2)) // Tier 2 or higher to appear ingame - .fetch(); - - logger.info(String.format("[PatreonSync] Loaded %s records to update in the Game DB", valid_records.size())); - - ArrayList only_ckeys = new ArrayList(); - - for (PatreonSupportersRecord psr : valid_records) { - only_ckeys.add(psr.getCkey()); - } - - // Remove from BYOND where not in this list - gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.notIn(only_ckeys)) - .execute(); - - // And where not active to keep the table short - gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.ACTIVE.ne((byte) 0x01)) // tinyint is a byte - .execute(); - - int insert_count = 0; - int update_count = 0; - // Now do our record stuff - for (PatreonSupportersRecord psr : valid_records) { - DonatorsRecord dr = gameJooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) - .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.eq(psr.getCkey())) - .fetchOne(); - - if (dr == null) { - // Empty record - make a new one - DonatorsRecord dr2 = gameJooq.newRecord(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS); - dr2.setPatreonName(psr.getMemberId().toString()); - dr2.setTier(patreonTier2ingameTier(psr.getDonationTier())); - dr2.setCkey(psr.getCkey()); - dr2.setStartDate(null); - dr2.setEndDate(null); - dr2.setActive((byte) 0x01); - - dr2.store(); - - insert_count++; - } else { - // Existing record - update - dr.setTier(patreonTier2ingameTier(psr.getDonationTier())); - dr.setCkey(psr.getCkey()); - dr.setStartDate(null); - dr.setEndDate(null); - dr.setActive((byte) 0x01); - - dr.store(); - - update_count++; - } - } - - logger.info(String.format("[PatreonSync] Inserted %s records, updated %s records in the game DB.", insert_count, update_count)); - } -} +package me.aa07.paradise.taskdaemon.core.modules.patreonsync; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import me.aa07.paradise.taskdaemon.core.config.PatreonConfig; +import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonOuterResponseModel; +import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonRawResponseModel; +import me.aa07.paradise.taskdaemon.core.models.patreon.PatreonUser; +import me.aa07.paradise.taskdaemon.core.models.tasks.DiscordRoleTaskArgsModel; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.core.util.UtilStr; +import me.aa07.paradise.taskdaemon.database.authentik.tables.records.AuthentikCoreUsersourceconnectionRecord; +import me.aa07.paradise.taskdaemon.database.automation.Tables; +import me.aa07.paradise.taskdaemon.database.automation.enums.TaskQueueTaskConsumer; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.PatreonSupportersRecord; +import me.aa07.paradise.taskdaemon.database.automation.tables.records.TaskQueueRecord; +import me.aa07.paradise.taskdaemon.database.gamedb.tables.records.DonatorsRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Result; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class PatreonSyncJob implements Job { + /* + + - pull raw data from patreon + - empty the holding table and fill with the patreon raw data + - get authentik IDs and add into table + - get discord IDs and add into table + - get ckeys and add into table + - mass sync ingame stuff + + */ + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[PatreonSync] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + logger.info("[PatreonSync] Starting up"); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[PatreonSync] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + dbcore.getClass(); + + // Now get our config + Object raw_ptrcfg = datamap.get("PTRCFG"); + Optional ptrcfg_holder = Optional.empty(); + + if (raw_ptrcfg instanceof PatreonConfig ptrcfg2) { + ptrcfg_holder = Optional.of(ptrcfg2); + } + + if (!ptrcfg_holder.isPresent()) { + logger.error("[PatreonSync] PTRCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + PatreonConfig ptrcfg = ptrcfg_holder.get(); + + // So + // The patreon API is kinda bad + // Everything goes into the same loose model and you have to manually parse out members and tiers + // Boy thats fun + + ArrayList users = new ArrayList(); + + try { + HttpRequest httpreq = HttpRequest.newBuilder() + .uri(URI.create("https://www.patreon.com/api/oauth2/v2/campaigns/" + + String.valueOf(ptrcfg.campaignId) + + "/members" + + "?include=currently_entitled_tiers,user" + + "&fields[member]=patron_status,currently_entitled_amount_cents" + + "&fields[tier]=amount_cents" + + "&page[count]=1000")) // This will break if we get >1000 patrons + // TODO Add pagination + .GET() + .header("Accept", "application/json") + .header("Authorization", String.format("Bearer %s", ptrcfg.creatorToken)) + .header("User-Agent", "ParadiseSS13-TaskDaemon/1.0") + .build(); + + HttpClient client = HttpClient.newHttpClient(); + logger.info("[PatreonSync] Pulling data"); + HttpResponse response = client.send(httpreq, BodyHandlers.ofString()); + + String response_body = response.body(); + + /* + + // Uncomment if you need horrendous debugging + + File file = new File("patreon_debug_response.json"); + file.createNewFile(); + FileWriter fw = new FileWriter(file); + fw.write(response_body); + fw.close(); + */ + + PatreonOuterResponseModel holder = UtilConst.GSON.fromJson(response_body, PatreonOuterResponseModel.class); + + logger.info(String.format("[PatreonSync] Pulled %s raw models from Patreon", holder.data.size())); + + for (PatreonRawResponseModel raw_model : holder.data) { + if (raw_model.type.equalsIgnoreCase("member")) { + // Make sure theyre valid first + if (raw_model.attributes != null && raw_model.attributes.patronStatus != null) { + if (raw_model.attributes.patronStatus.equalsIgnoreCase("active_patron")) { + PatreonUser user = raw_model.asUser(); + users.add(user); + continue; + } + } + } + } + } catch (Exception ex) { + logger.error(ex); + } + + logger.info(String.format("[PatreonSync] Loaded %s active users", users.size())); + + ArrayList only_member_ids = new ArrayList(); + + for (PatreonUser pu : users) { + only_member_ids.add(pu.userId); + } + + // Step 1 - Clear out old users + DSLContext automation_jooq = dbcore.jooq(DatabaseType.Automation); + + + Result records_to_purge = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.notIn(only_member_ids)) + .fetch(); + + for (PatreonSupportersRecord psr : records_to_purge) { + Long database_discord_id = psr.getDiscordId(); + + // Store our removal task if necessary + if (database_discord_id != null && (psr.getDonationTier() >= 2)) { + TaskQueueRecord tqr = automation_jooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("REMOVE_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = database_discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbcore.now()); + tqr.store(); + } + + // Clear out the supporter record + psr.delete(); + } + + logger.info("[PatreonSync] Cleaned out old users"); + + int insert_count = 0; + int update_count = 0; + + // Step 2 - Add or insert new + for (PatreonUser pu : users) { + if (automation_jooq.fetchExists(automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)))) { + + update_count++; + + // Fetch and update our record + PatreonSupportersRecord psr = automation_jooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(pu.userId)) + .fetchSingle(); + + psr.setAmountCents(pu.amountInCents); + psr.setDonationTier(pu.maxTier().tierLevel); + + // And store + psr.store(); + } else { + insert_count++; + + // Make a new record + PatreonSupportersRecord psr = automation_jooq.newRecord(Tables.PATREON_SUPPORTERS); + psr.setMemberId(pu.userId); + psr.setAmountCents(pu.amountInCents); + psr.setDonationTier(pu.maxTier().tierLevel); + + // And store + psr.store(); + } + } + + logger.info(String.format("[PatreonSync] Inserted %s users, updated %s users", insert_count, update_count)); + + + DSLContext game_jooq = dbcore.jooq(DatabaseType.GameDb); + DSLContext authentik_jooq = dbcore.jooq(DatabaseType.Authentik); + + // Find authentik users + HashMap authentik_to_patreon = new HashMap(); + findAuthentikUsers(only_member_ids, authentik_jooq, automation_jooq, logger, authentik_to_patreon); + + // Now find discord users + findDiscordUsers(authentik_jooq, automation_jooq, logger, authentik_to_patreon, dbcore); + + // Now get ckeys + findCkeys(authentik_jooq, automation_jooq, logger, authentik_to_patreon); + + // And commit to game DB + updateGameDb(automation_jooq, game_jooq, logger); + + System.gc(); // This is important after this one - the response model is like multiple megabytes + logger.info("[PatreonSync] Process complete"); + } + + private static int patreonTier2ingameTier(int patreonTier) { + switch (patreonTier) { + case 0: // No tier + return 0; + case 1: // Supporter + return 0; + case 2: // Icon + return 1; + case 3: // Loadout + return 2; + case 4: // Silver + return 3; + case 5: // Gold + return 4; + default: + return 0; + } + } + + private static void findAuthentikUsers( + List memberIds, + DSLContext authentikJooq, + DSLContext automationJooq, + Logger logger, + HashMap authentik2patreon + ) { + for (int member_id : memberIds) { + // This loop should only have active member IDs for the DB given we just flushed everything else out + + // Get ready for a TON of absolute pathing + AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.IDENTIFIER.eq(String.valueOf(member_id))) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.PATREON_OAUTH_SERVICE_ID)) + .fetchOne(); + + if (authentik_record == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in Authentik!", member_id)); + continue; + } + + authentik2patreon.put(authentik_record.getUserId(), member_id); + } + + logger.info(String.format("[PatreonSync] Loaded %s Authentik users", authentik2patreon.size())); + + int supporters_authentik_updated = 0; + for (int authentik_id : authentik2patreon.keySet()) { + int patreon_id = authentik2patreon.get(authentik_id); + + // Get our DB record + + PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) + .fetchOne(); + + if (psr == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); + continue; + } + + supporters_authentik_updated++; + psr.setAuthentikId(authentik_id); + psr.store(); // Store it back in + } + + logger.info(String.format("[PatreonSync] Updated %s rows with Authentik users", supporters_authentik_updated)); + } + + private static void findDiscordUsers( + DSLContext authentikJooq, + DSLContext automationJooq, + Logger logger, + HashMap authentik2patreon, + DbCore dbCore + ) { + HashMap authentik2discord = new HashMap(); + + for (int authentik_id : authentik2patreon.keySet()) { + // This loop should only have active member IDs for the DB given we just flushed everything else out + + // Get ready for a TON of absolute pathing + AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.DISCORD_OAUTH_SERVICE_ID)) + .fetchOne(); + + if (authentik_record == null) { + logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); + continue; + } + + authentik2discord.put(authentik_id, Long.valueOf(authentik_record.getIdentifier())); + } + + logger.info(String.format("[PatreonSync] Loaded %s Discord users", authentik2discord.size())); + + int supporters_updated = 0; + for (int authentik_id : authentik2discord.keySet()) { + if (!authentik2patreon.containsKey(authentik_id)) { + logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); + } + + int patreon_id = authentik2patreon.get(authentik_id); + long discord_id = authentik2discord.get(authentik_id); + + // Get our DB record + PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) + .fetchOne(); + + if (psr == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); + continue; + } + + // See if we have a role to add + if (psr.getDiscordId() == null && psr.getDonationTier() >= 2) { + // We do - lets add the role + TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("ADD_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbCore.now()); + tqr.store(); + } + + // See if their tier changed between runs + if (psr.getDiscordId() != null && (psr.getLastrunDonationTier().intValue() != psr.getDonationTier().intValue())) { + // Tier changed - see if they went below the threshold + if (psr.getDonationTier() <= 1) { + // They went below the threshold - strip the role + TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("REMOVE_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbCore.now()); + tqr.store(); + } + + // Add if we just upgraded + if (psr.getDonationTier() >= 2 && psr.getLastrunDonationTier() == 1) { + TaskQueueRecord tqr = automationJooq.newRecord(Tables.TASK_QUEUE); + tqr.setTaskId(UUID.randomUUID()); + tqr.setTaskConsumer(TaskQueueTaskConsumer.ALICE); + tqr.setTaskType("ADD_PATREON_ROLE"); + + // No I will not apologise for these variable names + DiscordRoleTaskArgsModel drtam = new DiscordRoleTaskArgsModel(); + drtam.discordId = discord_id; + String drtam_json = UtilConst.GSON.toJson(drtam); + + tqr.setTaskArguments(drtam_json); + tqr.setDateInserted(dbCore.now()); + tqr.store(); + } + } + + supporters_updated++; + psr.setLastrunDonationTier(psr.getDonationTier()); + psr.setDiscordId(discord_id); + psr.store(); // Store it back in + } + + logger.info(String.format("[PatreonSync] Updated %s rows with Discord users", supporters_updated)); + } + + private static void findCkeys( + DSLContext authentikJooq, + DSLContext automationJooq, + Logger logger, + HashMap authentik2patreon + ) { + HashMap authentik2ckey = new HashMap(); + + for (int authentik_id : authentik2patreon.keySet()) { + // This loop should only have active member IDs for the DB given we just flushed everything else out + + // Get ready for a TON of absolute pathing + AuthentikCoreUsersourceconnectionRecord authentik_record = authentikJooq + .selectFrom(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION) + .where(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.USER_ID.eq(authentik_id)) + .and(me.aa07.paradise.taskdaemon.database.authentik.Tables.AUTHENTIK_CORE_USERSOURCECONNECTION.SOURCE_ID.eq(UtilConst.BYOND_OAUTH_SERVICE_ID)) + .fetchOne(); + + if (authentik_record == null) { + logger.warn(String.format("[PatreonSync] Couldnt find a discord link for Authentik user %s!", authentik_id)); + continue; + } + + // Normalise the ckey + String ckey = UtilStr.cleanCkey(authentik_record.getIdentifier()); + + authentik2ckey.put(authentik_id, ckey); + } + + logger.info(String.format("[PatreonSync] Loaded %s ckeys", authentik2ckey.size())); + + int supporters_updated = 0; + for (int authentik_id : authentik2ckey.keySet()) { + if (!authentik2patreon.containsKey(authentik_id)) { + logger.warn(String.format("[PatreonSync] Couldnt find a patreon user for Authentik user %s - this shouldnt be possible!", authentik_id)); + } + + int patreon_id = authentik2patreon.get(authentik_id); + String ckey = authentik2ckey.get(authentik_id); + + // Get our DB record + PatreonSupportersRecord psr = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.MEMBER_ID.eq(patreon_id)) + .fetchOne(); + + if (psr == null) { + logger.warn(String.format("[PatreonSync] Patreon user ID %s not found in supporters table. This shouldnt be possible!", patreon_id)); + continue; + } + + supporters_updated++; + psr.setCkey(ckey); + psr.store(); // Store it back in + } + + logger.info(String.format("[PatreonSync] Updated %s rows with ckeys", supporters_updated)); + } + + private static void updateGameDb(DSLContext automationJooq, DSLContext gameJooq, Logger logger) { + + Result valid_records = automationJooq.selectFrom(Tables.PATREON_SUPPORTERS) + .where(Tables.PATREON_SUPPORTERS.CKEY.isNotNull()) + .and(Tables.PATREON_SUPPORTERS.DONATION_TIER.ge(2)) // Tier 2 or higher to appear ingame + .fetch(); + + logger.info(String.format("[PatreonSync] Loaded %s records to update in the Game DB", valid_records.size())); + + ArrayList only_ckeys = new ArrayList(); + + for (PatreonSupportersRecord psr : valid_records) { + only_ckeys.add(psr.getCkey()); + } + + // Remove from BYOND where not in this list + gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.notIn(only_ckeys)) + .execute(); + + // And where not active to keep the table short + gameJooq.deleteFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.ACTIVE.ne((byte) 0x01)) // tinyint is a byte + .execute(); + + int insert_count = 0; + int update_count = 0; + // Now do our record stuff + for (PatreonSupportersRecord psr : valid_records) { + DonatorsRecord dr = gameJooq.selectFrom(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS) + .where(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS.CKEY.eq(psr.getCkey())) + .fetchOne(); + + if (dr == null) { + // Empty record - make a new one + DonatorsRecord dr2 = gameJooq.newRecord(me.aa07.paradise.taskdaemon.database.gamedb.Tables.DONATORS); + dr2.setPatreonName(psr.getMemberId().toString()); + dr2.setTier(patreonTier2ingameTier(psr.getDonationTier())); + dr2.setCkey(psr.getCkey()); + dr2.setStartDate(null); + dr2.setEndDate(null); + dr2.setActive((byte) 0x01); + + dr2.store(); + + insert_count++; + } else { + // Existing record - update + dr.setTier(patreonTier2ingameTier(psr.getDonationTier())); + dr.setCkey(psr.getCkey()); + dr.setStartDate(null); + dr.setEndDate(null); + dr.setActive((byte) 0x01); + + dr.store(); + + update_count++; + } + } + + logger.info(String.format("[PatreonSync] Inserted %s records, updated %s records in the game DB.", insert_count, update_count)); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java index 1c8c988..7ddd913 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/prlabel/PrLabelJob.java @@ -1,252 +1,252 @@ -package me.aa07.paradise.taskdaemon.core.modules.prlabel; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.config.GithubConfig; -import me.aa07.paradise.taskdaemon.core.config.TgsConfig; -import me.aa07.paradise.taskdaemon.core.models.prlabel.PullRequest; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.database.pullrequests.Tables; -import me.aa07.paradise.taskdaemon.database.pullrequests.enums.PrsPrStatus; -import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVoteType; -import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVotingGroup; -import me.aa07.paradise.taskdaemon.database.pullrequests.tables.records.PrsRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.Record2; -import org.jooq.Record4; -import org.jooq.Result; -import org.jooq.impl.DSL; -import org.kohsuke.github.GHIssueState; -import org.kohsuke.github.GHLabel; -import org.kohsuke.github.GHPullRequest; -import org.kohsuke.github.GHRepository; -import org.kohsuke.github.GitHub; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class PrLabelJob implements Job { - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - // INITIAL SETUP - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[PrLabelJob] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[PrLabelJob] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - // Now get our config - Object raw_github_config = datamap.get("GITHUBCFG"); - Optional github_cfg_holder = Optional.empty(); - - if (raw_github_config instanceof GithubConfig githubCfg2) { - github_cfg_holder = Optional.of(githubCfg2); - } - - if (!github_cfg_holder.isPresent()) { - logger.error("[PrLabelJob] GITHUBCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - GithubConfig github_config = github_cfg_holder.get(); - - // Now get our other config - Object raw_tgs_config = datamap.get("TGSCFG"); - Optional tgs_cfg_holder = Optional.empty(); - - if (raw_tgs_config instanceof TgsConfig tgsCfg2) { - tgs_cfg_holder = Optional.of(tgsCfg2); - } - - if (!tgs_cfg_holder.isPresent()) { - logger.error("[PrLabelJob] TGSCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - TgsConfig tgs_cfg = tgs_cfg_holder.get(); - - // Initial setup is done - lets now load the important bits - - // Get this now - we need it - DSLContext ctx = dbcore.jooq(DatabaseType.PullRequests); - - // Dictionary that holds all this in a nice lookup - HashMap prs = new HashMap(); - - // And init these - // TM manager - TestmergeManager tmm = new TestmergeManager(tgs_cfg, logger); - tmm.setup(); - - // Files changed manager - FilesChangedManager fcm = new FilesChangedManager(logger); - - // Vote status manager - StatusManager sm = new StatusManager(logger, github_config); - // We set this up later to contain the exception - - // Put everything in a list to aid iteration - ArrayList processors = new ArrayList(); - processors.add(tmm); - processors.add(fcm); - processors.add(sm); - - logger.info(String.format("[PrLabelJob] Loaded %s active TMs from TGS - now loading PRs from GitHub", - tmm.loadedActiveTms())); - - // Evertything in the GH client throws IOException so we need to baby proof that - try { - // Set up the review team - sm.setup(); - - // Now set up the rest - GitHub gh = GitHub.connectUsingOAuth(github_config.githubToken); - GHRepository repo = gh.getRepositoryById(github_config.repoId); - - // Get our open PRs - List open_prs = repo.queryPullRequests().state(GHIssueState.OPEN).list().toList(); - - // And parse them into that array - for (GHPullRequest pr : open_prs) { - PullRequest internal_pr = new PullRequest(); - internal_pr.prObject = pr; - internal_pr.pullNumber = pr.getNumber(); - - for (GHLabel label : pr.getLabels()) { - internal_pr.labels.add(label.getName()); - } - - prs.put(internal_pr.pullNumber, internal_pr); - } - - logger.info(String.format("[PrLabelJob] Loaded %s PRs from GitHub - now loading from DB", prs.size())); - - // Now get the data from the DB, just the PR metadata - Result db_prs = ctx.selectFrom(Tables.PRS).where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) - .fetch(); - - for (PrsRecord db_pr : db_prs) { - int pr_num = db_pr.getPrNumber().intValue(); - if (prs.containsKey(pr_num)) { - // Set values up - PullRequest pr = prs.get(pr_num); - pr.prTypes = db_pr.getPrTypes(); - } else { - logger.warn(String.format("[PrLabelJob] PR #%s in DB but not open!", pr_num)); - } - } - - // Now the testmerge requests - logger.info(String.format("[PrLabelJob] Processed %s DB PRs - now loading TMRs", prs.size())); - - Result> db_tmrs = ctx.select(Tables.TM_REQUESTS.PR_NUMBER, DSL.count()) - .from(Tables.TM_REQUESTS) - .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.TM_REQUESTS.PR_NUMBER)) - .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) - .groupBy(Tables.TM_REQUESTS.PR_NUMBER) - .fetch(); - - for (Record2 db_tmr_rec : db_tmrs) { - int pr_num = db_tmr_rec.value1().intValue(); - int tmr_count = db_tmr_rec.value2().intValue(); - - if (prs.containsKey(pr_num)) { - // Set values up - PullRequest pr = prs.get(pr_num); - pr.testmergeRequests = tmr_count; - } - } - - // Now the PR votes - logger.info(String.format("[PrLabelJob] Processed %s DB TMRs - now loading votes", db_tmrs.size())); - - Result> db_votes = ctx.select( - Tables.VOTES_NEW.PR_NUMBER, - Tables.VOTES_NEW.VOTING_GROUP, - Tables.VOTES_NEW.VOTE_TYPE, - DSL.count()).from(Tables.VOTES_NEW) - .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.VOTES_NEW.PR_NUMBER)) - .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) - .groupBy(Tables.VOTES_NEW.PR_NUMBER, Tables.VOTES_NEW.VOTING_GROUP, Tables.VOTES_NEW.VOTE_TYPE) - .fetch(); - - for (Record4 db_vote_rec : db_votes) { - int pr_num = db_vote_rec.value1().intValue(); - VotesNewVotingGroup vg = db_vote_rec.value2(); - VotesNewVoteType vt = db_vote_rec.value3(); - int vote_count = db_vote_rec.value4().intValue(); - - if (prs.containsKey(pr_num)) { - // Set values up - PullRequest pr = prs.get(pr_num); - - if (!pr.votes.containsKey(vg)) { - pr.votes.put(vg, new HashMap()); - } - - // No check needed - this is fine to add or replace - HashMap pr_votes_map = pr.votes.get(vg); - pr_votes_map.put(vt, vote_count); - } - } - - // Now we have the votes we can actually parse the PRs - logger.info( - String.format("[PrLabelJob] Processed %s DB votes sets - now doing the main thing", - db_votes.size())); - - for (int pr_number : prs.keySet()) { - logger.info(String.format("[PrLabelJob] Processing #%s", pr_number)); - PullRequest pr_object = prs.get(pr_number); - - // Handle on each processor - for (IPullRequestProcessor processor : processors) { - processor.processPr(pr_object); - } - } - - logger.info(String.format("[PrLabelJob] Done", db_votes.size())); - - } catch (IOException e) { - logger.error("[PrLabelJob] Failed due to an IOException"); - logger.error(e); - return; - } - } - -} +package me.aa07.paradise.taskdaemon.core.modules.prlabel; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.config.GithubConfig; +import me.aa07.paradise.taskdaemon.core.config.TgsConfig; +import me.aa07.paradise.taskdaemon.core.models.prlabel.PullRequest; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.pullrequests.Tables; +import me.aa07.paradise.taskdaemon.database.pullrequests.enums.PrsPrStatus; +import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVoteType; +import me.aa07.paradise.taskdaemon.database.pullrequests.enums.VotesNewVotingGroup; +import me.aa07.paradise.taskdaemon.database.pullrequests.tables.records.PrsRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.Record2; +import org.jooq.Record4; +import org.jooq.Result; +import org.jooq.impl.DSL; +import org.kohsuke.github.GHIssueState; +import org.kohsuke.github.GHLabel; +import org.kohsuke.github.GHPullRequest; +import org.kohsuke.github.GHRepository; +import org.kohsuke.github.GitHub; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class PrLabelJob implements Job { + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + // INITIAL SETUP + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[PrLabelJob] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[PrLabelJob] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + // Now get our config + Object raw_github_config = datamap.get("GITHUBCFG"); + Optional github_cfg_holder = Optional.empty(); + + if (raw_github_config instanceof GithubConfig githubCfg2) { + github_cfg_holder = Optional.of(githubCfg2); + } + + if (!github_cfg_holder.isPresent()) { + logger.error("[PrLabelJob] GITHUBCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + GithubConfig github_config = github_cfg_holder.get(); + + // Now get our other config + Object raw_tgs_config = datamap.get("TGSCFG"); + Optional tgs_cfg_holder = Optional.empty(); + + if (raw_tgs_config instanceof TgsConfig tgsCfg2) { + tgs_cfg_holder = Optional.of(tgsCfg2); + } + + if (!tgs_cfg_holder.isPresent()) { + logger.error("[PrLabelJob] TGSCFG WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + TgsConfig tgs_cfg = tgs_cfg_holder.get(); + + // Initial setup is done - lets now load the important bits + + // Get this now - we need it + DSLContext ctx = dbcore.jooq(DatabaseType.PullRequests); + + // Dictionary that holds all this in a nice lookup + HashMap prs = new HashMap(); + + // And init these + // TM manager + TestmergeManager tmm = new TestmergeManager(tgs_cfg, logger); + tmm.setup(); + + // Files changed manager + FilesChangedManager fcm = new FilesChangedManager(logger); + + // Vote status manager + StatusManager sm = new StatusManager(logger, github_config); + // We set this up later to contain the exception + + // Put everything in a list to aid iteration + ArrayList processors = new ArrayList(); + processors.add(tmm); + processors.add(fcm); + processors.add(sm); + + logger.info(String.format("[PrLabelJob] Loaded %s active TMs from TGS - now loading PRs from GitHub", + tmm.loadedActiveTms())); + + // Evertything in the GH client throws IOException so we need to baby proof that + try { + // Set up the review team + sm.setup(); + + // Now set up the rest + GitHub gh = GitHub.connectUsingOAuth(github_config.githubToken); + GHRepository repo = gh.getRepositoryById(github_config.repoId); + + // Get our open PRs + List open_prs = repo.queryPullRequests().state(GHIssueState.OPEN).list().toList(); + + // And parse them into that array + for (GHPullRequest pr : open_prs) { + PullRequest internal_pr = new PullRequest(); + internal_pr.prObject = pr; + internal_pr.pullNumber = pr.getNumber(); + + for (GHLabel label : pr.getLabels()) { + internal_pr.labels.add(label.getName()); + } + + prs.put(internal_pr.pullNumber, internal_pr); + } + + logger.info(String.format("[PrLabelJob] Loaded %s PRs from GitHub - now loading from DB", prs.size())); + + // Now get the data from the DB, just the PR metadata + Result db_prs = ctx.selectFrom(Tables.PRS).where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) + .fetch(); + + for (PrsRecord db_pr : db_prs) { + int pr_num = db_pr.getPrNumber().intValue(); + if (prs.containsKey(pr_num)) { + // Set values up + PullRequest pr = prs.get(pr_num); + pr.prTypes = db_pr.getPrTypes(); + } else { + logger.warn(String.format("[PrLabelJob] PR #%s in DB but not open!", pr_num)); + } + } + + // Now the testmerge requests + logger.info(String.format("[PrLabelJob] Processed %s DB PRs - now loading TMRs", prs.size())); + + Result> db_tmrs = ctx.select(Tables.TM_REQUESTS.PR_NUMBER, DSL.count()) + .from(Tables.TM_REQUESTS) + .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.TM_REQUESTS.PR_NUMBER)) + .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) + .groupBy(Tables.TM_REQUESTS.PR_NUMBER) + .fetch(); + + for (Record2 db_tmr_rec : db_tmrs) { + int pr_num = db_tmr_rec.value1().intValue(); + int tmr_count = db_tmr_rec.value2().intValue(); + + if (prs.containsKey(pr_num)) { + // Set values up + PullRequest pr = prs.get(pr_num); + pr.testmergeRequests = tmr_count; + } + } + + // Now the PR votes + logger.info(String.format("[PrLabelJob] Processed %s DB TMRs - now loading votes", db_tmrs.size())); + + Result> db_votes = ctx.select( + Tables.VOTES_NEW.PR_NUMBER, + Tables.VOTES_NEW.VOTING_GROUP, + Tables.VOTES_NEW.VOTE_TYPE, + DSL.count()).from(Tables.VOTES_NEW) + .leftJoin(Tables.PRS).on(Tables.PRS.PR_NUMBER.eq(Tables.VOTES_NEW.PR_NUMBER)) + .where(Tables.PRS.PR_STATUS.eq(PrsPrStatus.OPEN)) + .groupBy(Tables.VOTES_NEW.PR_NUMBER, Tables.VOTES_NEW.VOTING_GROUP, Tables.VOTES_NEW.VOTE_TYPE) + .fetch(); + + for (Record4 db_vote_rec : db_votes) { + int pr_num = db_vote_rec.value1().intValue(); + VotesNewVotingGroup vg = db_vote_rec.value2(); + VotesNewVoteType vt = db_vote_rec.value3(); + int vote_count = db_vote_rec.value4().intValue(); + + if (prs.containsKey(pr_num)) { + // Set values up + PullRequest pr = prs.get(pr_num); + + if (!pr.votes.containsKey(vg)) { + pr.votes.put(vg, new HashMap()); + } + + // No check needed - this is fine to add or replace + HashMap pr_votes_map = pr.votes.get(vg); + pr_votes_map.put(vt, vote_count); + } + } + + // Now we have the votes we can actually parse the PRs + logger.info( + String.format("[PrLabelJob] Processed %s DB votes sets - now doing the main thing", + db_votes.size())); + + for (int pr_number : prs.keySet()) { + logger.info(String.format("[PrLabelJob] Processing #%s", pr_number)); + PullRequest pr_object = prs.get(pr_number); + + // Handle on each processor + for (IPullRequestProcessor processor : processors) { + processor.processPr(pr_object); + } + } + + logger.info(String.format("[PrLabelJob] Done", db_votes.size())); + + } catch (IOException e) { + logger.error("[PrLabelJob] Failed due to an IOException"); + logger.error(e); + return; + } + } + +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java index e862281..ddb198a 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profilercleanup/ProfilerCleanupJob.java @@ -1,61 +1,61 @@ -package me.aa07.paradise.taskdaemon.core.modules.profilercleanup; - -import java.util.Optional; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.database.profiler.Tables; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@DisallowConcurrentExecution // NO -public class ProfilerCleanupJob implements Job { - - @Override - public void execute(JobExecutionContext event) throws JobExecutionException { - JobDataMap datamap = event.getMergedJobDataMap(); - - // Get our logger - important - Object raw_logger = datamap.get("LOGGER"); - Optional logger_holder = Optional.empty(); - - if (raw_logger instanceof Logger l2) { - logger_holder = Optional.of(l2); - } - - if (!logger_holder.isPresent()) { - System.out.println("[ProfilerCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - Logger logger = logger_holder.get(); - - // Now get our DB - Object raw_db = datamap.get("DBCORE"); - Optional dbcore_holder = Optional.empty(); - - if (raw_db instanceof DbCore db2) { - dbcore_holder = Optional.of(db2); - } - - if (!dbcore_holder.isPresent()) { - logger.error("[ProfilerCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); - return; - } - - DbCore dbcore = dbcore_holder.get(); - - DSLContext profiler_db = dbcore.jooq(DatabaseType.ProfilerDb); - - logger.info("[ProfilerCleanup] Cleaning out profiler DB - proc samples"); - profiler_db.delete(Tables.SAMPLES).where(Tables.SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); - logger.info("[ProfilerCleanup] Cleaning out profiler DB - sendmaps samples"); - profiler_db.delete(Tables.SENDMAPS_SAMPLES).where(Tables.SENDMAPS_SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); - logger.info("[ProfilerCleanup] Cleaned"); - } - -} +package me.aa07.paradise.taskdaemon.core.modules.profilercleanup; + +import java.util.Optional; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.database.profiler.Tables; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +@DisallowConcurrentExecution // NO +public class ProfilerCleanupJob implements Job { + + @Override + public void execute(JobExecutionContext event) throws JobExecutionException { + JobDataMap datamap = event.getMergedJobDataMap(); + + // Get our logger - important + Object raw_logger = datamap.get("LOGGER"); + Optional logger_holder = Optional.empty(); + + if (raw_logger instanceof Logger l2) { + logger_holder = Optional.of(l2); + } + + if (!logger_holder.isPresent()) { + System.out.println("[ProfilerCleanup] LOGGER WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + Logger logger = logger_holder.get(); + + // Now get our DB + Object raw_db = datamap.get("DBCORE"); + Optional dbcore_holder = Optional.empty(); + + if (raw_db instanceof DbCore db2) { + dbcore_holder = Optional.of(db2); + } + + if (!dbcore_holder.isPresent()) { + logger.error("[ProfilerCleanup] DBCORE WAS SOMEHOW NULL - THIS IS VERY BAD"); + return; + } + + DbCore dbcore = dbcore_holder.get(); + + DSLContext profiler_db = dbcore.jooq(DatabaseType.ProfilerDb); + + logger.info("[ProfilerCleanup] Cleaning out profiler DB - proc samples"); + profiler_db.delete(Tables.SAMPLES).where(Tables.SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); + logger.info("[ProfilerCleanup] Cleaning out profiler DB - sendmaps samples"); + profiler_db.delete(Tables.SENDMAPS_SAMPLES).where(Tables.SENDMAPS_SAMPLES.SAMPLETIME.lt(dbcore.now().minusDays(7))).execute(); + logger.info("[ProfilerCleanup] Cleaned"); + } + +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java index 6b12b7b..af34140 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/ProfilerWorker.java @@ -1,85 +1,85 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest; - -import java.util.ArrayList; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerProcProcessor; -import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerSendmapsProcessor; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import org.apache.logging.log4j.Logger; - -public class ProfilerWorker { - private ArrayList queue; - private Logger logger; - private Object listLock; - - private ProfilerProcProcessor procProcessor; - private ProfilerSendmapsProcessor sendmapsProcessor; - - public ProfilerWorker(DbCore database, Logger logger) { - this.logger = logger; - - procProcessor = new ProfilerProcProcessor(database, logger); - sendmapsProcessor = new ProfilerSendmapsProcessor(database, logger); - - queue = new ArrayList(); - listLock = new Object(); - } - - // TODO - Refactor into quartz job - public void run() { - logger.info("Starting..."); - - try { - while (true) { - String current_run = safePop(); - if (current_run == null) { - Thread.sleep(10000); - continue; - } - processEntry(current_run); - - } - } catch (InterruptedException exception) { - logger.fatal("Worker thread was interrupted!"); - exception.printStackTrace(); - System.exit(1); - } - } - - // Add something to the list in a thread-safe matter - public void safeAdd(String data) { - synchronized (listLock) { - queue.add(data); - } - } - - // Extract one item from the list in a thread-safe matter - private String safePop() { - String output = ""; - synchronized (listLock) { - if (queue.size() > 0) { - output = queue.get(0); - queue.remove(0); - } else { - return null; - } - } - - return output; - } - - private void processEntry(String entry) { - long start = System.currentTimeMillis(); - logger.info(String.format("Processing entry with size %s bytes", entry.length())); - - ProfilerHolder holder = UtilConst.GSON.fromJson(entry, ProfilerHolder.class); - logger.info(String.format("Round ID: %s | Profile size: %s bytes", holder.roundId, holder.profilerData.length())); - - procProcessor.processData(holder); - sendmapsProcessor.processData(holder); - - long duration = System.currentTimeMillis() - start; - logger.info(String.format("All processing complete within %sms", duration)); - } -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest; + +import java.util.ArrayList; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerProcProcessor; +import me.aa07.paradise.taskdaemon.core.modules.profileringest.processors.ProfilerSendmapsProcessor; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import org.apache.logging.log4j.Logger; + +public class ProfilerWorker { + private ArrayList queue; + private Logger logger; + private Object listLock; + + private ProfilerProcProcessor procProcessor; + private ProfilerSendmapsProcessor sendmapsProcessor; + + public ProfilerWorker(DbCore database, Logger logger) { + this.logger = logger; + + procProcessor = new ProfilerProcProcessor(database, logger); + sendmapsProcessor = new ProfilerSendmapsProcessor(database, logger); + + queue = new ArrayList(); + listLock = new Object(); + } + + // TODO - Refactor into quartz job + public void run() { + logger.info("Starting..."); + + try { + while (true) { + String current_run = safePop(); + if (current_run == null) { + Thread.sleep(10000); + continue; + } + processEntry(current_run); + + } + } catch (InterruptedException exception) { + logger.fatal("Worker thread was interrupted!"); + exception.printStackTrace(); + System.exit(1); + } + } + + // Add something to the list in a thread-safe matter + public void safeAdd(String data) { + synchronized (listLock) { + queue.add(data); + } + } + + // Extract one item from the list in a thread-safe matter + private String safePop() { + String output = ""; + synchronized (listLock) { + if (queue.size() > 0) { + output = queue.get(0); + queue.remove(0); + } else { + return null; + } + } + + return output; + } + + private void processEntry(String entry) { + long start = System.currentTimeMillis(); + logger.info(String.format("Processing entry with size %s bytes", entry.length())); + + ProfilerHolder holder = UtilConst.GSON.fromJson(entry, ProfilerHolder.class); + logger.info(String.format("Round ID: %s | Profile size: %s bytes", holder.roundId, holder.profilerData.length())); + + procProcessor.processData(holder); + sendmapsProcessor.processData(holder); + + long duration = System.currentTimeMillis() - start; + logger.info(String.format("All processing complete within %sms", duration)); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java index 237b5fc..d5427f7 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerBaseProcessor.java @@ -1,33 +1,33 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; - -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import org.apache.logging.log4j.Logger; - -public abstract class ProfilerBaseProcessor { - protected DbCore database; - protected Logger logger; - private String processorName; - - public ProfilerBaseProcessor(DbCore database, Logger logger, String processorName) { - this.database = database; - this.logger = logger; - this.processorName = processorName; - } - - protected void log(String txt) { - logger.info(String.format("[%s] %s", processorName, txt)); - } - - public void processData(ProfilerHolder holder) { - long start = System.currentTimeMillis(); - log("Starting processing"); - doProcessing(holder); - long duration = System.currentTimeMillis() - start; - log(String.format("Processing complete within %sms", duration)); - } - - protected abstract void doProcessing(ProfilerHolder holder); - - protected abstract long getProcId(String procname); -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; + +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import org.apache.logging.log4j.Logger; + +public abstract class ProfilerBaseProcessor { + protected DbCore database; + protected Logger logger; + private String processorName; + + public ProfilerBaseProcessor(DbCore database, Logger logger, String processorName) { + this.database = database; + this.logger = logger; + this.processorName = processorName; + } + + protected void log(String txt) { + logger.info(String.format("[%s] %s", processorName, txt)); + } + + public void processData(ProfilerHolder holder) { + long start = System.currentTimeMillis(); + log("Starting processing"); + doProcessing(holder); + long duration = System.currentTimeMillis() - start; + log(String.format("Processing complete within %sms", duration)); + } + + protected abstract void doProcessing(ProfilerHolder holder); + + protected abstract long getProcId(String procname); +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java index 5dfabfd..00719fc 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerProcProcessor.java @@ -1,62 +1,62 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; - -import com.google.gson.reflect.TypeToken; -import java.util.List; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProcData; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.database.profiler.Tables; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.ProcsRecord; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SamplesRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; - -public class ProfilerProcProcessor extends ProfilerBaseProcessor { - public ProfilerProcProcessor(DbCore database, Logger logger) { - super(database, logger, "ProcProcessor"); - } - - @Override - protected void doProcessing(ProfilerHolder holder) { - List proc_list = UtilConst.GSON.fromJson( - holder.profilerData, new TypeToken>() {}.getType() - ); - - log(String.format("Procs logged: %s", proc_list.size())); - - DSLContext jooq = database.jooq(DatabaseType.ProfilerDb); - - for (ProcData procdata : proc_list) { - long proc_id = getProcId(procdata.name); - SamplesRecord sr = jooq.newRecord(Tables.SAMPLES); - sr.setRoundid(holder.roundId); - sr.setSampletime(database.now()); - sr.setProcid(proc_id); - sr.setSelf(procdata.self); - sr.setTotal(procdata.total); - sr.setReal(procdata.real); - sr.setOver(procdata.over); - sr.setCalls(procdata.calls); - sr.store(); - } - } - - @Override - protected long getProcId(String procname) { - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); - if (!ctx.fetchExists(ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)))) { - // We dont exist, make us - log(String.format("%s did not exist in the DB. It does now.", procname)); - - ProcsRecord record = ctx.newRecord(Tables.PROCS); - record.setProcpath(procname); - record.store(); - return record.getId(); - } - - // We do exist, just grab - return ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)).fetchOne().value1(); - } -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; + +import com.google.gson.reflect.TypeToken; +import java.util.List; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProcData; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.database.profiler.Tables; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.ProcsRecord; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SamplesRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; + +public class ProfilerProcProcessor extends ProfilerBaseProcessor { + public ProfilerProcProcessor(DbCore database, Logger logger) { + super(database, logger, "ProcProcessor"); + } + + @Override + protected void doProcessing(ProfilerHolder holder) { + List proc_list = UtilConst.GSON.fromJson( + holder.profilerData, new TypeToken>() {}.getType() + ); + + log(String.format("Procs logged: %s", proc_list.size())); + + DSLContext jooq = database.jooq(DatabaseType.ProfilerDb); + + for (ProcData procdata : proc_list) { + long proc_id = getProcId(procdata.name); + SamplesRecord sr = jooq.newRecord(Tables.SAMPLES); + sr.setRoundid(holder.roundId); + sr.setSampletime(database.now()); + sr.setProcid(proc_id); + sr.setSelf(procdata.self); + sr.setTotal(procdata.total); + sr.setReal(procdata.real); + sr.setOver(procdata.over); + sr.setCalls(procdata.calls); + sr.store(); + } + } + + @Override + protected long getProcId(String procname) { + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); + if (!ctx.fetchExists(ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)))) { + // We dont exist, make us + log(String.format("%s did not exist in the DB. It does now.", procname)); + + ProcsRecord record = ctx.newRecord(Tables.PROCS); + record.setProcpath(procname); + record.store(); + return record.getId(); + } + + // We do exist, just grab + return ctx.select(Tables.PROCS.ID).from(Tables.PROCS).where(Tables.PROCS.PROCPATH.eq(procname)).fetchOne().value1(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java index 0103491..a7eab6d 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/modules/profileringest/processors/ProfilerSendmapsProcessor.java @@ -1,59 +1,59 @@ -package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; - -import com.google.gson.reflect.TypeToken; -import java.util.List; -import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; -import me.aa07.paradise.taskdaemon.core.models.profiler.SendmapsProcData; -import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; -import me.aa07.paradise.taskdaemon.core.services.database.DbCore; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import me.aa07.paradise.taskdaemon.database.profiler.Tables; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsProcsRecord; -import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsSamplesRecord; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; - -public class ProfilerSendmapsProcessor extends ProfilerBaseProcessor { - public ProfilerSendmapsProcessor(DbCore database, Logger logger) { - super(database, logger, "SendmapsProcessor"); - } - - @Override - protected void doProcessing(ProfilerHolder holder) { - List proc_list = UtilConst.GSON.fromJson( - holder.sendmapsData, new TypeToken>() {}.getType() - ); - - log(String.format("Procs logged: %s", proc_list.size())); - - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); - - for (SendmapsProcData procdata : proc_list) { - long proc_id = getProcId(procdata.name); - SendmapsSamplesRecord sr = ctx.newRecord(Tables.SENDMAPS_SAMPLES); - sr.setRoundid(holder.roundId); - sr.setSampletime(database.now()); - sr.setProcid(proc_id); - sr.setValue(procdata.value); - sr.setCalls(procdata.calls); - sr.store(); - } - } - - @Override - protected long getProcId(String procname) { - DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); - if (!ctx.fetchExists(ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)))) { - // We dont exist, make us - log(String.format("%s did not exist in the DB. It does now.", procname)); - - SendmapsProcsRecord record = ctx.newRecord(Tables.SENDMAPS_PROCS); - record.setProcpath(procname); - record.store(); - return record.getId(); - } - - // We do exist, just grab - return ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)).fetchOne().value1(); - } -} +package me.aa07.paradise.taskdaemon.core.modules.profileringest.processors; + +import com.google.gson.reflect.TypeToken; +import java.util.List; +import me.aa07.paradise.taskdaemon.core.models.profiler.ProfilerHolder; +import me.aa07.paradise.taskdaemon.core.models.profiler.SendmapsProcData; +import me.aa07.paradise.taskdaemon.core.services.database.DatabaseType; +import me.aa07.paradise.taskdaemon.core.services.database.DbCore; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import me.aa07.paradise.taskdaemon.database.profiler.Tables; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsProcsRecord; +import me.aa07.paradise.taskdaemon.database.profiler.tables.records.SendmapsSamplesRecord; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; + +public class ProfilerSendmapsProcessor extends ProfilerBaseProcessor { + public ProfilerSendmapsProcessor(DbCore database, Logger logger) { + super(database, logger, "SendmapsProcessor"); + } + + @Override + protected void doProcessing(ProfilerHolder holder) { + List proc_list = UtilConst.GSON.fromJson( + holder.sendmapsData, new TypeToken>() {}.getType() + ); + + log(String.format("Procs logged: %s", proc_list.size())); + + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); + + for (SendmapsProcData procdata : proc_list) { + long proc_id = getProcId(procdata.name); + SendmapsSamplesRecord sr = ctx.newRecord(Tables.SENDMAPS_SAMPLES); + sr.setRoundid(holder.roundId); + sr.setSampletime(database.now()); + sr.setProcid(proc_id); + sr.setValue(procdata.value); + sr.setCalls(procdata.calls); + sr.store(); + } + } + + @Override + protected long getProcId(String procname) { + DSLContext ctx = database.jooq(DatabaseType.ProfilerDb); + if (!ctx.fetchExists(ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)))) { + // We dont exist, make us + log(String.format("%s did not exist in the DB. It does now.", procname)); + + SendmapsProcsRecord record = ctx.newRecord(Tables.SENDMAPS_PROCS); + record.setProcpath(procname); + record.store(); + return record.getId(); + } + + // We do exist, just grab + return ctx.select(Tables.SENDMAPS_PROCS.ID).from(Tables.SENDMAPS_PROCS).where(Tables.SENDMAPS_PROCS.PROCPATH.eq(procname)).fetchOne().value1(); + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java index 8e038d8..6a8e47f 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DatabaseType.java @@ -1,5 +1,5 @@ -package me.aa07.paradise.taskdaemon.core.services.database; - -public enum DatabaseType { - Authentik, Automation, GameDb, Forums, ProfilerDb, PullRequests -} +package me.aa07.paradise.taskdaemon.core.services.database; + +public enum DatabaseType { + Authentik, Automation, GameDb, Forums, ProfilerDb, PullRequests +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java index 90d0ac2..b67e59a 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/database/DbCore.java @@ -1,108 +1,108 @@ -package me.aa07.paradise.taskdaemon.core.services.database; - -import java.sql.Connection; -import java.sql.Timestamp; -import java.time.LocalDateTime; -import java.util.Date; -import java.util.HashMap; -import javax.sql.DataSource; -import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; -import me.aa07.paradise.taskdaemon.core.config.DatabaseConfig; -import org.apache.commons.dbcp2.BasicDataSource; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.Logger; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.exception.NoDataFoundException; -import org.jooq.impl.DSL; - -public class DbCore { - private HashMap> connectionMap; - - public DbCore(ConfigHolder config, Logger logger) { - // Suppress JOOQ console spam - System.getProperties().setProperty("org.jooq.no-logo", "true"); - System.getProperties().setProperty("org.jooq.no-tips", "true"); - - connectionMap = new HashMap>(); - - establishConnections(config); - logger.info("Ready to handle DB requests"); - } - - private DataSource openMySqlDataSource(String url, String username, String password) { - BasicDataSource source = new BasicDataSource(); - source.addConnectionProperty("autoReconnect", "true"); - source.addConnectionProperty("allowMultiQueries", "true"); - source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); - source.addConnectionProperty("connectionTimeZone", "UTC"); - source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); - source.setDriverClassName("com.mysql.cj.jdbc.Driver"); - source.setUrl(url); - source.setUsername(username); - source.setPassword(password); - source.setMaxTotal(8); - source.setMaxIdle(8); - source.setTimeBetweenEvictionRunsMillis(180 * 1000); - source.setSoftMinEvictableIdleTimeMillis(180 * 1000); - - return source; - } - - private DataSource openPostgresDataSource(String url, String username, String password) { - BasicDataSource source = new BasicDataSource(); - source.addConnectionProperty("autoReconnect", "true"); - source.addConnectionProperty("allowMultiQueries", "true"); - source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); - source.addConnectionProperty("connectionTimeZone", "UTC"); - source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); - source.setDriverClassName("org.postgresql.Driver"); - source.setUrl(url); - source.setUsername(username); - source.setPassword(password); - source.setMaxTotal(8); - source.setMaxIdle(8); - source.setTimeBetweenEvictionRunsMillis(180 * 1000); - source.setSoftMinEvictableIdleTimeMillis(180 * 1000); - - return source; - } - - private void establishConnections(ConfigHolder config) { - HashMap mysql_db_types = new HashMap(); - - mysql_db_types.put(DatabaseType.Automation, config.automationDatabase); - mysql_db_types.put(DatabaseType.Forums, config.forumsDatabase); - mysql_db_types.put(DatabaseType.GameDb, config.gameDatabase); - mysql_db_types.put(DatabaseType.ProfilerDb, config.profilerDatabase); - mysql_db_types.put(DatabaseType.PullRequests, config.pullRequestsDatabase); - - for (DatabaseType dbtype : mysql_db_types.keySet()) { - DatabaseConfig cfg = mysql_db_types.get(dbtype); - DataSource ds = openMySqlDataSource(String.format("jdbc:mysql://%s/%s", cfg.host, cfg.database), cfg.username, cfg.password); - connectionMap.put(dbtype, Pair.of(ds, SQLDialect.MYSQL)); - } - - // And do special stuff for Postgres - DatabaseConfig adb = config.authentikDatabase; - DataSource authentik_ds = openPostgresDataSource(String.format("jdbc:postgresql://%s/%s", adb.host, adb.database), adb.username, adb.password); - connectionMap.put(DatabaseType.Authentik, Pair.of(authentik_ds, SQLDialect.POSTGRES)); - } - - // Get a DSL context - public DSLContext jooq(DatabaseType type) { - if (!connectionMap.containsKey(type)) { - throw new NoDataFoundException("Supplied type key not present in connection map"); - } - - Pair ds_pair = connectionMap.get(type); - - return DSL.using(ds_pair.getLeft(), ds_pair.getRight()); - } - - // Easy way for NOW() in SQL - public LocalDateTime now() { - return new Timestamp(new Date().getTime()).toLocalDateTime(); - } - -} +package me.aa07.paradise.taskdaemon.core.services.database; + +import java.sql.Connection; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.Date; +import java.util.HashMap; +import javax.sql.DataSource; +import me.aa07.paradise.taskdaemon.core.config.ConfigHolder; +import me.aa07.paradise.taskdaemon.core.config.DatabaseConfig; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.Logger; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.exception.NoDataFoundException; +import org.jooq.impl.DSL; + +public class DbCore { + private HashMap> connectionMap; + + public DbCore(ConfigHolder config, Logger logger) { + // Suppress JOOQ console spam + System.getProperties().setProperty("org.jooq.no-logo", "true"); + System.getProperties().setProperty("org.jooq.no-tips", "true"); + + connectionMap = new HashMap>(); + + establishConnections(config); + logger.info("Ready to handle DB requests"); + } + + private DataSource openMySqlDataSource(String url, String username, String password) { + BasicDataSource source = new BasicDataSource(); + source.addConnectionProperty("autoReconnect", "true"); + source.addConnectionProperty("allowMultiQueries", "true"); + source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); + source.addConnectionProperty("connectionTimeZone", "UTC"); + source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + source.setDriverClassName("com.mysql.cj.jdbc.Driver"); + source.setUrl(url); + source.setUsername(username); + source.setPassword(password); + source.setMaxTotal(8); + source.setMaxIdle(8); + source.setTimeBetweenEvictionRunsMillis(180 * 1000); + source.setSoftMinEvictableIdleTimeMillis(180 * 1000); + + return source; + } + + private DataSource openPostgresDataSource(String url, String username, String password) { + BasicDataSource source = new BasicDataSource(); + source.addConnectionProperty("autoReconnect", "true"); + source.addConnectionProperty("allowMultiQueries", "true"); + source.addConnectionProperty("zeroDateTimeBehavior", "convertToNull"); + source.addConnectionProperty("connectionTimeZone", "UTC"); + source.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + source.setDriverClassName("org.postgresql.Driver"); + source.setUrl(url); + source.setUsername(username); + source.setPassword(password); + source.setMaxTotal(8); + source.setMaxIdle(8); + source.setTimeBetweenEvictionRunsMillis(180 * 1000); + source.setSoftMinEvictableIdleTimeMillis(180 * 1000); + + return source; + } + + private void establishConnections(ConfigHolder config) { + HashMap mysql_db_types = new HashMap(); + + mysql_db_types.put(DatabaseType.Automation, config.automationDatabase); + mysql_db_types.put(DatabaseType.Forums, config.forumsDatabase); + mysql_db_types.put(DatabaseType.GameDb, config.gameDatabase); + mysql_db_types.put(DatabaseType.ProfilerDb, config.profilerDatabase); + mysql_db_types.put(DatabaseType.PullRequests, config.pullRequestsDatabase); + + for (DatabaseType dbtype : mysql_db_types.keySet()) { + DatabaseConfig cfg = mysql_db_types.get(dbtype); + DataSource ds = openMySqlDataSource(String.format("jdbc:mysql://%s/%s", cfg.host, cfg.database), cfg.username, cfg.password); + connectionMap.put(dbtype, Pair.of(ds, SQLDialect.MYSQL)); + } + + // And do special stuff for Postgres + DatabaseConfig adb = config.authentikDatabase; + DataSource authentik_ds = openPostgresDataSource(String.format("jdbc:postgresql://%s/%s", adb.host, adb.database), adb.username, adb.password); + connectionMap.put(DatabaseType.Authentik, Pair.of(authentik_ds, SQLDialect.POSTGRES)); + } + + // Get a DSL context + public DSLContext jooq(DatabaseType type) { + if (!connectionMap.containsKey(type)) { + throw new NoDataFoundException("Supplied type key not present in connection map"); + } + + Pair ds_pair = connectionMap.get(type); + + return DSL.using(ds_pair.getLeft(), ds_pair.getRight()); + } + + // Easy way for NOW() in SQL + public LocalDateTime now() { + return new Timestamp(new Date().getTime()).toLocalDateTime(); + } + +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java index 4a199ca..92828ee 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/HttpRequestMethod.java @@ -1,5 +1,5 @@ -package me.aa07.paradise.taskdaemon.core.services.invision; - -public enum HttpRequestMethod { - Get, Post -} +package me.aa07.paradise.taskdaemon.core.services.invision; + +public enum HttpRequestMethod { + Get, Post +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java index a129e38..ee4a813 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/services/invision/InvisionUtil.java @@ -1,125 +1,125 @@ -package me.aa07.paradise.taskdaemon.core.services.invision; - -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpRequest.Builder; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import me.aa07.paradise.taskdaemon.core.config.InvisionConfig; -import me.aa07.paradise.taskdaemon.core.models.invision.GetUserModel; -import me.aa07.paradise.taskdaemon.core.util.UtilConst; -import org.apache.commons.lang3.tuple.Pair; - -public class InvisionUtil { - private HttpClient httpClient; - private InvisionConfig invisionConfig; - - public InvisionUtil(InvisionConfig invisionConfig) { - this.invisionConfig = invisionConfig; - httpClient = HttpClient.newHttpClient(); - } - - private Pair makeInvisionRequest(String uri, HttpRequestMethod reqType, String bodyContent) throws Exception { - // Step 1 - Make our base - Builder request_builder = HttpRequest.newBuilder().uri(new URI(String.format("%s/%s", invisionConfig.apiBaseUrl, uri))); - - switch (reqType) { - case Get: - request_builder = request_builder.GET(); - break; - case Post: - if (bodyContent == null) { - request_builder = request_builder.POST(HttpRequest.BodyPublishers.noBody()); - } else { - request_builder = request_builder.POST(HttpRequest.BodyPublishers.ofString(bodyContent)); - } - break; - default: - throw new Exception(String.format("Request type %s not supported", reqType.toString())); - } - - // Generate auth header - String creds = String.format("%s:", invisionConfig.apiKey); - byte[] encoded_auth = Base64.getEncoder().encode(creds.getBytes(StandardCharsets.UTF_8)); - String auth_header = String.format("Basic %s", new String(encoded_auth)); - - HttpRequest final_request = request_builder.setHeader("Accept", "application/json") - .setHeader("Content-Type", "application/json") - .setHeader("Authorization", auth_header).build(); - - HttpResponse response = httpClient.send(final_request, BodyHandlers.ofString()); - - return Pair.of(response.body(), response.statusCode()); - } - - public Pair> getUserSecondaryGroups(int userId) { - ArrayList out_list = new ArrayList(); - String request_uri = String.format("core/members/%s", userId); - - Pair invision_response = null; - - // Send it off - try { - invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Get, null); - } catch (Exception e) { - e.printStackTrace(); - return Pair.of(false, out_list); - } - - // Shouldnt ever happen - if (invision_response == null || invision_response.getRight() != 200) { - return Pair.of(false, out_list); - } - - // Try decode - GetUserModel gum = UtilConst.GSON.fromJson(invision_response.getLeft(), GetUserModel.class); - - if (gum == null) { - return Pair.of(false, out_list); - } - - // This one does return true - they could have 0 secondary groups - if (gum.secondaryGroups == null) { - return Pair.of(true, out_list); - } - - // If we are here we can decode - for (GetUserModel.GroupHolder gh : gum.secondaryGroups) { - out_list.add(gh.id); - } - - return Pair.of(true, out_list); - } - - public boolean updateUserSecondaryGroups(int userId, List secondaryGroups) { - ArrayList group_ids_list_str = new ArrayList(); - for (Integer i : secondaryGroups) { - // I hate PHP array syntax - group_ids_list_str.add(String.format("secondaryGroups[]=%s", i.toString())); - } - String secondary_groups_str = String.join("&", group_ids_list_str); - - String request_uri = String.format("core/members/%s?%s", userId, secondary_groups_str); - Pair invision_response = null; - - // Send it off - try { - invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Post, null); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - // Shouldnt ever happen - if (invision_response == null || invision_response.getRight() != 200) { - return false; - } - - return true; - } -} +package me.aa07.paradise.taskdaemon.core.services.invision; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.Builder; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import me.aa07.paradise.taskdaemon.core.config.InvisionConfig; +import me.aa07.paradise.taskdaemon.core.models.invision.GetUserModel; +import me.aa07.paradise.taskdaemon.core.util.UtilConst; +import org.apache.commons.lang3.tuple.Pair; + +public class InvisionUtil { + private HttpClient httpClient; + private InvisionConfig invisionConfig; + + public InvisionUtil(InvisionConfig invisionConfig) { + this.invisionConfig = invisionConfig; + httpClient = HttpClient.newHttpClient(); + } + + private Pair makeInvisionRequest(String uri, HttpRequestMethod reqType, String bodyContent) throws Exception { + // Step 1 - Make our base + Builder request_builder = HttpRequest.newBuilder().uri(new URI(String.format("%s/%s", invisionConfig.apiBaseUrl, uri))); + + switch (reqType) { + case Get: + request_builder = request_builder.GET(); + break; + case Post: + if (bodyContent == null) { + request_builder = request_builder.POST(HttpRequest.BodyPublishers.noBody()); + } else { + request_builder = request_builder.POST(HttpRequest.BodyPublishers.ofString(bodyContent)); + } + break; + default: + throw new Exception(String.format("Request type %s not supported", reqType.toString())); + } + + // Generate auth header + String creds = String.format("%s:", invisionConfig.apiKey); + byte[] encoded_auth = Base64.getEncoder().encode(creds.getBytes(StandardCharsets.UTF_8)); + String auth_header = String.format("Basic %s", new String(encoded_auth)); + + HttpRequest final_request = request_builder.setHeader("Accept", "application/json") + .setHeader("Content-Type", "application/json") + .setHeader("Authorization", auth_header).build(); + + HttpResponse response = httpClient.send(final_request, BodyHandlers.ofString()); + + return Pair.of(response.body(), response.statusCode()); + } + + public Pair> getUserSecondaryGroups(int userId) { + ArrayList out_list = new ArrayList(); + String request_uri = String.format("core/members/%s", userId); + + Pair invision_response = null; + + // Send it off + try { + invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Get, null); + } catch (Exception e) { + e.printStackTrace(); + return Pair.of(false, out_list); + } + + // Shouldnt ever happen + if (invision_response == null || invision_response.getRight() != 200) { + return Pair.of(false, out_list); + } + + // Try decode + GetUserModel gum = UtilConst.GSON.fromJson(invision_response.getLeft(), GetUserModel.class); + + if (gum == null) { + return Pair.of(false, out_list); + } + + // This one does return true - they could have 0 secondary groups + if (gum.secondaryGroups == null) { + return Pair.of(true, out_list); + } + + // If we are here we can decode + for (GetUserModel.GroupHolder gh : gum.secondaryGroups) { + out_list.add(gh.id); + } + + return Pair.of(true, out_list); + } + + public boolean updateUserSecondaryGroups(int userId, List secondaryGroups) { + ArrayList group_ids_list_str = new ArrayList(); + for (Integer i : secondaryGroups) { + // I hate PHP array syntax + group_ids_list_str.add(String.format("secondaryGroups[]=%s", i.toString())); + } + String secondary_groups_str = String.join("&", group_ids_list_str); + + String request_uri = String.format("core/members/%s?%s", userId, secondary_groups_str); + Pair invision_response = null; + + // Send it off + try { + invision_response = makeInvisionRequest(request_uri, HttpRequestMethod.Post, null); + } catch (Exception e) { + e.printStackTrace(); + return false; + } + + // Shouldnt ever happen + if (invision_response == null || invision_response.getRight() != 200) { + return false; + } + + return true; + } +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java index e762d90..fb448f9 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilConst.java @@ -1,16 +1,16 @@ -package me.aa07.paradise.taskdaemon.core.util; - -import com.google.gson.Gson; -import java.util.UUID; - -public class UtilConst { - public static final UUID BYOND_OAUTH_SERVICE_ID = UUID.fromString("d7a7b659-69bd-4435-b585-3b4ba97ca076"); - public static final UUID DISCORD_OAUTH_SERVICE_ID = UUID.fromString("bc9fb785-6a1f-434e-80da-26a37450a29a"); - public static final UUID INVISION_OAUTH_SERVICE_ID = UUID.fromString("4634c7d7-2bad-4139-a05f-13f296b20af8"); - public static final UUID PATREON_OAUTH_SERVICE_ID = UUID.fromString("626f4935-d046-4957-9862-f26102e27a70"); - - public static final int INVISION_INGAMEVERIFIED_GID = 30; - public static final int INVISION_INGAMEBANNED_GID = 31; - - public static final Gson GSON = new Gson(); -} +package me.aa07.paradise.taskdaemon.core.util; + +import com.google.gson.Gson; +import java.util.UUID; + +public class UtilConst { + public static final UUID BYOND_OAUTH_SERVICE_ID = UUID.fromString("d7a7b659-69bd-4435-b585-3b4ba97ca076"); + public static final UUID DISCORD_OAUTH_SERVICE_ID = UUID.fromString("bc9fb785-6a1f-434e-80da-26a37450a29a"); + public static final UUID INVISION_OAUTH_SERVICE_ID = UUID.fromString("4634c7d7-2bad-4139-a05f-13f296b20af8"); + public static final UUID PATREON_OAUTH_SERVICE_ID = UUID.fromString("626f4935-d046-4957-9862-f26102e27a70"); + + public static final int INVISION_INGAMEVERIFIED_GID = 30; + public static final int INVISION_INGAMEBANNED_GID = 31; + + public static final Gson GSON = new Gson(); +} diff --git a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java index 740a49c..8f7afd1 100644 --- a/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java +++ b/TaskDaemon.Core/src/me/aa07/paradise/taskdaemon/core/util/UtilStr.java @@ -1,35 +1,35 @@ -package me.aa07.paradise.taskdaemon.core.util; - -import java.util.regex.Pattern; - -public class UtilStr { - private static final Pattern CKEY_PATTERN = Pattern.compile("[^a-z0-9]"); - private static final Pattern ONLY_NUMBERS = Pattern.compile("[^0-9]"); - - public static String cleanCkey(String ckey) { - // If blank got passed, dont do it - if (ckey == null) { - return ""; - } - - // Lowercase it - ckey = ckey.toLowerCase(); - - // Clean these out from Authentik - ckey = ckey.replace("user:", ""); - - - // Split anything else out - return CKEY_PATTERN.matcher(ckey).replaceAll(""); - } - - public static String onlyNumbers(String input) { - // If blank got passed, dont do it - if (input == null) { - return ""; - } - - // Annihilate everything else - return ONLY_NUMBERS.matcher(input).replaceAll(""); - } -} +package me.aa07.paradise.taskdaemon.core.util; + +import java.util.regex.Pattern; + +public class UtilStr { + private static final Pattern CKEY_PATTERN = Pattern.compile("[^a-z0-9]"); + private static final Pattern ONLY_NUMBERS = Pattern.compile("[^0-9]"); + + public static String cleanCkey(String ckey) { + // If blank got passed, dont do it + if (ckey == null) { + return ""; + } + + // Lowercase it + ckey = ckey.toLowerCase(); + + // Clean these out from Authentik + ckey = ckey.replace("user:", ""); + + + // Split anything else out + return CKEY_PATTERN.matcher(ckey).replaceAll(""); + } + + public static String onlyNumbers(String input) { + // If blank got passed, dont do it + if (input == null) { + return ""; + } + + // Annihilate everything else + return ONLY_NUMBERS.matcher(input).replaceAll(""); + } +} From b86975314b31dba863569b7fbfb3ad26364c4718 Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:18:03 +0100 Subject: [PATCH 6/7] This too --- README.MD | 184 +++++++++++++++++++++++++++--------------------------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/README.MD b/README.MD index 7c9a128..d3b3d96 100644 --- a/README.MD +++ b/README.MD @@ -1,92 +1,92 @@ -# Paradise Task Daemon - -## What is it? - -Java application designed to automate various backend tasks we have within ParadiseSS13. - -It handles: - -- ACL Cleanup -- Bouncer Restart -- Dev Rank Handling -- Ingame Verified Syncc -- IP2ASN -- Patreon Sync -- Profiler Daemon Ingest -- Profiler DB Cleanup -- PR Label Bot - -### ACL Cleanup - -Removes entries from a HAproxy ACL via its socket interface in order for inactivate players t o be put through the bouncer server again. - -### Bouncer Restart - -Restarts our "bouncer" server every day. This is the thing that does 2FA, server queue, and region redirection. - -### Dev Rank - -Adds or removes members from development rank when changes are made. Performed hourly. - -### Ingame Verified Sync - -Tacks roles onto the relevant people for their ingame verified state - -### IP2ASN - -Caches ASNs for IP addresses of recent players. - -### Profiler Daemon Ingest - -This takes BYOND Profiler results and split them apart to be indexed at a later date, allowing multiple procs from multiple rounds to easily be visualised alongside eachother over time. An example SS13 codebase implementation can be found at . - -This project itself does not provide a data extraction method, but that can be found at . - -### Patreon Sync - -Pulls data from Patreon to store in a DB. This is then used for Discord role sync, and ingame perk granting. - -### Profiler DB Cleanup - -Just wipes rows older than 7 days out the profiler table. - -### PR Label Bot - -Handles labelling PRs on the GitHub based on TM status, assigned type, votes and other stuff. - -## Provided Projects - -- `TaskDaemon.Core` - Core application which handles running tasks. -- `TaskDaemon.Database.Authentik` - Database project automatically generated with JOOQ for the `authentik` database. -- `TaskDaemon.Database.Automation` - Database project automatically generated with JOOQ for the `aa07_automation` database. - - No I will not apologise for that DB name. -- `TaskDaemon.Database.Forums` - Database project automatically generated with JOOQ for the `paradise_forums` database. -- `TaskDaemon.Database.GameDb` - Database project automatically generated with JOOQ for the `paradise_gamedb` database. -- `TaskDaemon.Database.Profiler` - Database project automatically generated with JOOQ for the `paradise_profiler` database. -- `TaskDaemon.Database.PullRequests` - Database project automatically generated with JOOQ for the `aa07_pr_voting` database. - - No I will not apologise for that DB name. - -Use `mvn package -P update-db` to update DB projects. - -## Building - -To build, run `mvn package` to generate the JAR file. This will then appear in the `out` directory. - -## Running - -1. Compile the codebase (See `Building`). -2. Copy the jar and the example `config.toml` in the repository root to a running location. -3. Import the relevant schemas from the `schemas` dir to your database server. -4. Setup the configuration as per section labels. -5. Start the application up, it will now run tasks in the background. - -## Code Guidelines - -- Code must be tested. - - Good luck. -- Code must adhere to the style guidelines set in `checkstyle.xml`. - - These are google's guidelines but with minor tweaks for my own needs. - -## License - -This project is licensed under GPL-3. You can find a copy of this license in the file `LICENSE.MD`. +# Paradise Task Daemon + +## What is it? + +Java application designed to automate various backend tasks we have within ParadiseSS13. + +It handles: + +- ACL Cleanup +- Bouncer Restart +- Dev Rank Handling +- Ingame Verified Syncc +- IP2ASN +- Patreon Sync +- Profiler Daemon Ingest +- Profiler DB Cleanup +- PR Label Bot + +### ACL Cleanup + +Removes entries from a HAproxy ACL via its socket interface in order for inactivate players t o be put through the bouncer server again. + +### Bouncer Restart + +Restarts our "bouncer" server every day. This is the thing that does 2FA, server queue, and region redirection. + +### Dev Rank + +Adds or removes members from development rank when changes are made. Performed hourly. + +### Ingame Verified Sync + +Tacks roles onto the relevant people for their ingame verified state + +### IP2ASN + +Caches ASNs for IP addresses of recent players. + +### Profiler Daemon Ingest + +This takes BYOND Profiler results and split them apart to be indexed at a later date, allowing multiple procs from multiple rounds to easily be visualised alongside eachother over time. An example SS13 codebase implementation can be found at . + +This project itself does not provide a data extraction method, but that can be found at . + +### Patreon Sync + +Pulls data from Patreon to store in a DB. This is then used for Discord role sync, and ingame perk granting. + +### Profiler DB Cleanup + +Just wipes rows older than 7 days out the profiler table. + +### PR Label Bot + +Handles labelling PRs on the GitHub based on TM status, assigned type, votes and other stuff. + +## Provided Projects + +- `TaskDaemon.Core` - Core application which handles running tasks. +- `TaskDaemon.Database.Authentik` - Database project automatically generated with JOOQ for the `authentik` database. +- `TaskDaemon.Database.Automation` - Database project automatically generated with JOOQ for the `aa07_automation` database. + - No I will not apologise for that DB name. +- `TaskDaemon.Database.Forums` - Database project automatically generated with JOOQ for the `paradise_forums` database. +- `TaskDaemon.Database.GameDb` - Database project automatically generated with JOOQ for the `paradise_gamedb` database. +- `TaskDaemon.Database.Profiler` - Database project automatically generated with JOOQ for the `paradise_profiler` database. +- `TaskDaemon.Database.PullRequests` - Database project automatically generated with JOOQ for the `aa07_pr_voting` database. + - No I will not apologise for that DB name. + +Use `mvn package -P update-db` to update DB projects. + +## Building + +To build, run `mvn package` to generate the JAR file. This will then appear in the `out` directory. + +## Running + +1. Compile the codebase (See `Building`). +2. Copy the jar and the example `config.toml` in the repository root to a running location. +3. Import the relevant schemas from the `schemas` dir to your database server. +4. Setup the configuration as per section labels. +5. Start the application up, it will now run tasks in the background. + +## Code Guidelines + +- Code must be tested. + - Good luck. +- Code must adhere to the style guidelines set in `checkstyle.xml`. + - These are google's guidelines but with minor tweaks for my own needs. + +## License + +This project is licensed under GPL-3. You can find a copy of this license in the file `LICENSE.MD`. From ff11515bfa731c208dc9d19c734f3330e32f30ca Mon Sep 17 00:00:00 2001 From: AffectedArc07 <25063394+AffectedArc07@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:18:24 +0100 Subject: [PATCH 7/7] Linux dev was a mistake --- config.toml.example | 136 ++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/config.toml.example b/config.toml.example index 3353fee..a97b7f0 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,68 +1,68 @@ -# This is an example configuration. Please see README.MD - -[authentikDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "authentik" - -[automationDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "aa07_automation" - -[forumsDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "paradise_forums" - -[gameDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "paradise_profilerdaemon" - -[github] -githubToken = "your token here" -repoId = 123456 -orgName = "ParadiseSS13" -reviewTeamSlug = "review-team" # Team name but with the dashes like "review-team" not "Review Team" -reviewTeamToken = "your token here" # Review team gets a separate token because it needs org admin to get team members - -[invisionConfig] -apiBaseUrl = "https://www.paradisestation.org/forum/api" -apiKey = "key goes here" - -[ip2asn] -host = "http://127.0.0.1:53661/v1/as/ip/" # Fill out URL - IP will be appended at the end - -[patreon] -campaignId = 123 -creatorToken = "token goes here" - -[pfsense] -host = "127.0.0.1" -port = 1234 - -[profilerDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "paradise_profilerdaemon" - -[pullRequestsDatabase] -host = "172.16.0.200" -username = "myuser" -password = "mypassword" -database = "aa07_pr_voting" - -[redis] -host = "172.16.0.200" - -[tgs] -tgsHost = "http://10.0.0.30:5000" # dont put a trailing slash on the end of this -username = "username_here" -password = "password_here" -instanceId = 8 # instance ID to restart every day +# This is an example configuration. Please see README.MD + +[authentikDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "authentik" + +[automationDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "aa07_automation" + +[forumsDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "paradise_forums" + +[gameDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "paradise_profilerdaemon" + +[github] +githubToken = "your token here" +repoId = 123456 +orgName = "ParadiseSS13" +reviewTeamSlug = "review-team" # Team name but with the dashes like "review-team" not "Review Team" +reviewTeamToken = "your token here" # Review team gets a separate token because it needs org admin to get team members + +[invisionConfig] +apiBaseUrl = "https://www.paradisestation.org/forum/api" +apiKey = "key goes here" + +[ip2asn] +host = "http://127.0.0.1:53661/v1/as/ip/" # Fill out URL - IP will be appended at the end + +[patreon] +campaignId = 123 +creatorToken = "token goes here" + +[pfsense] +host = "127.0.0.1" +port = 1234 + +[profilerDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "paradise_profilerdaemon" + +[pullRequestsDatabase] +host = "172.16.0.200" +username = "myuser" +password = "mypassword" +database = "aa07_pr_voting" + +[redis] +host = "172.16.0.200" + +[tgs] +tgsHost = "http://10.0.0.30:5000" # dont put a trailing slash on the end of this +username = "username_here" +password = "password_here" +instanceId = 8 # instance ID to restart every day