AppType=StandardJava Build1=Default,b4j.JRDCMultiDB Build2=New_1,b4j.JRDCMulti File1=config.DB2.properties File2=config.DB3.properties File3=config.DB4.properties File4=config.properties File5=reiniciaProcesoBow.bat File6=reiniciaProcesoPM2.bat File7=start.bat File8=start2.bat File9=stop.bat FileGroup1=Default Group FileGroup2=Default Group FileGroup3=Default Group FileGroup4=Default Group FileGroup5=Default Group FileGroup6=Default Group FileGroup7=Default Group FileGroup8=Default Group FileGroup9=Default Group Group=Default Group Library1=bcrypt Library2=byteconverter Library3=javaobject Library4=jcore Library5=jrandomaccessfile Library6=jserver Library7=jshell Library8=json Library9=jsql Module1=Cambios Module10=LoginHandler Module11=LogoutHandler Module12=Manager Module13=ParameterValidationUtils Module14=ping Module15=RDCConnector Module16=SSE Module17=SSEHandler Module18=TestHandler Module2=ChangePassHandler Module3=ConnectionPoolManager Module4=DBHandlerB4X Module5=DBHandlerJSON Module6=DoLoginHandler Module7=faviconHandler Module8=GlobalParameters Module9=HikariConnectionPool NumberOfFiles=9 NumberOfLibraries=9 NumberOfModules=18 Version=10.3 @EndOfDesignText@ 'Non-UI application (console / server application) #Region Project Attributes ' Specify command line arguments if any #CommandLineArgs: ' Merge all referenced libraries into the final JAR #MergeLibraries: True ' VERSION 5.10.25 '########################################################################################################### '###################### PULL ############################################################# ' IDE helper link to perform a 'git pull' 'Ctrl + click ide://run?file=%WINDIR%\System32\cmd.exe&Args=/c&Args=git&Args=pull '########################################################################################################### '###################### PUSH ############################################################# ' IDE helper link to perform a 'git push' (using a custom script/alias 'github') 'Ctrl + click ide://run?file=%WINDIR%\System32\WindowsPowerShell\v1.0\powershell.exe&Args=github&Args=..\..\ '########################################################################################################### '###################### PUSH TORTOISE GIT ######################################################### ' IDE helper link to open the TortoiseGit commit dialog 'Ctrl + click ide://run?file=%WINDIR%\System32\WindowsPowerShell\v1.0\powershell.exe&Args=TortoiseGitProc&Args=/command:commit&Args=/path:"../"&Args=/closeonend:2 '########################################################################################################### #End Region ' --- JDBC Driver Selection --- ' Change based on the jdbc jar file '#AdditionalJar: mysql-connector-java-5.1.27-bin '#AdditionalJar: postgresql-42.7.0 ' Using Oracle JDBC driver #AdditionalJar: ojdbc11 ' Using SQLite JDBC driver (for user/log database) #AdditionalJar: sqlite-jdbc-3.7.2 ' --- Critical Dependencies for HikariCP (Connection Pooling) and SLF4J (Logging) --- #AdditionalJar: HikariCP-4.0.3 #AdditionalJar: slf4j-api-1.7.25 #AdditionalJar: slf4j-simple-1.7.25 ' --- Global variables for the entire application --- Sub Process_Globals ' The main B4J HTTP server object Public srvr As Server ' The current version of this modified jRDC server Public const VERSION As Float = 2.23 ' Custom types for serializing/deserializing data Type DBCommand (Name As String, Parameters() As Object) Type DBResult (Tag As Object, Columns As Map, Rows As List) ' Holds a list of configured database identifiers (e.g., "DB1", "DB2") Public listaDeCP As List ' A temporary list to store found configuration file names during startup Private cpFiles As List ' Global maps to manage database connectors and loaded SQL commands Public Connectors, commandsMap As Map ' SQL object for interacting with the local users and logs database (SQLite) Public SQL1 As SQL ' Object for securely hashing and verifying passwords Private bc As BCrypt ' A Java ReentrantLock object to protect Main.Connectors during Hot-Swapping (thread-safety) Public MainConnectorsLock As JavaObject ' A Java ReentrantLock object to protect the log caches (QueryLogCache and ErrorLogCache) Public LogCacheLock As JavaObject ' Timer for executing periodic tasks, such as log cleanup Public timerLogs As Timer ' Map to store the SQLite logging status (True/False) for each DBKey (DB1, DB2, etc.) Public SQLiteLoggingStatusByDB As Map ' Global flag indicating if AT LEAST one database has SQLite logging enabled Public IsAnySQLiteLoggingEnabled As Boolean ' Type to encapsulate the result of parameter validation Type ParameterValidationResult ( _ Success As Boolean, _ ErrorMessage As String, _ ParamsToExecute As List _ ' The final list of parameters to use in the SQL execution ) ' In-memory cache for performance logs (query_logs) Public QueryLogCache As List ' In-memory cache for error and warning logs Public ErrorLogCache As List ' Threshold of records to force a batch write to the DB Public LOG_CACHE_THRESHOLD As Int = 400 ' Flag to enable/disable verbose logging, set in AppStart Dim logger As Boolean ' Thread-Safe Map to store the latest metrics for each connection pool Public LatestPoolStats As Map ' Counter for timer ticks to control VACUUM frequency Private TimerTickCount As Int = 0 ' Run VACUUM every 48 cycles (48 * 30 minutes = 24 hours) Private const VACUUM_CYCLES As Int = 48 ' Granular control for TEXT file logging (CSV) Public TextLoggingStatusByDB As Map ' Main object for managing all connection pools (RDCConnector instances) Public ConnectionPoolManager1 As ConnectionPoolManager End Sub ' --- Main application entry point --- Sub AppStart (Args() As String) ' Initialize Server-Sent Events handler SSE.Initialize ' Set logger flag based on build mode (DEBUG or RELEASE) #if DEBUG logger = True ' Use a small threshold in DEBUG mode for easier testing LOG_CACHE_THRESHOLD = 10 #else logger = False #End If Log("LOG_CACHE_THRESHOLD: " & LOG_CACHE_THRESHOLD) ' Copy web admin panel files if they don't exist CopiarRecursoSiNoExiste("manager.html", "www") CopiarRecursoSiNoExiste("login.html", "www") ' Copy root files (configs, start/stop scripts) if they don't exist CopiarRecursoSiNoExiste("config.properties", "") ' CopiarRecursoSiNoExiste("config.DB2.properties", "") ' CopiarRecursoSiNoExiste("config.DB3.properties", "") CopiarRecursoSiNoExiste("start.bat", "") CopiarRecursoSiNoExiste("start2.bat", "") CopiarRecursoSiNoExiste("stop.bat", "") CopiarRecursoSiNoExiste("reiniciaProcesoBow.bat", "") CopiarRecursoSiNoExiste("reiniciaProcesoPM2.bat", "") ' Initialize the BCrypt password hashing library bc.Initialize("BC") ' Initialize in-memory log caches QueryLogCache.Initialize ErrorLogCache.Initialize ' === 1. Initialize the local user database (SQLite) and log tables === InitializeSQLiteDatabase ' === 2. Initialize global maps defined in GlobalParameters.bas === GlobalParameters.mpLogs.Initialize GlobalParameters.mpTotalRequests.Initialize GlobalParameters.mpTotalConnections.Initialize GlobalParameters.mpBlockConnection.Initialize ' Ensure the active request counter map is thread-safe GlobalParameters.ActiveRequestsCountByDB = srvr.CreateThreadSafeMap ' === 3. Initialize the main HTTP server structures === listaDeCP.Initialize srvr.Initialize("") ' Connectors map must be thread-safe Connectors = srvr.CreateThreadSafeMap commandsMap.Initialize ' Initialize the stats map as Thread-Safe LatestPoolStats = srvr.CreateThreadSafeMap ' Initialize the map for text logging status TextLoggingStatusByDB.Initialize ' Initialize the granular SQLite logging status map SQLiteLoggingStatusByDB.Initialize ' Create a ReentrantLock instance to protect Main.Connectors MainConnectorsLock.InitializeNewInstance("java.util.concurrent.locks.ReentrantLock", Null) ' Initialize the lock for log caches LogCacheLock.InitializeNewInstance("java.util.concurrent.locks.ReentrantLock", Null) ' Initialize the Manager, which in turn initializes all pool wrappers. ConnectionPoolManager1.Initialize ' === 4. INITIALIZATION OF THE MAIN DATABASE CONNECTOR (DB1) === Try ' Initialize the main 'DB1' connector Dim con1 As RDCConnector con1.Initialize("DB1") ' Add it to the thread-safe map Connectors.Put("DB1", con1) ' Set the server port from the DB1 config file srvr.Port = con1.serverPort ' Add 'DB1' to the list of active database keys listaDeCP.Add("DB1") Log($"Main.AppStart: Connector 'DB1' initialized successfully on port: ${srvr.Port}"$) ' Read the 'enableSQLiteLogs' setting from config.properties (default to 0) Dim enableLogsSetting As Int = con1.config.GetDefault("enableSQLiteLogs", 0).As(Int) Dim isEnabled As Boolean = (enableLogsSetting = 1) ' Store the status in the granular map SQLiteLoggingStatusByDB.Put("DB1", isEnabled) ' Read the 'enableTextLogging' setting Dim enableTextLogsSetting As Int = con1.config.GetDefault("enableTextLogging", 0).As(Int) Dim isTextEnabled As Boolean = (enableTextLogsSetting = 1) ' Store the text log status TextLoggingStatusByDB.Put("DB1", isTextEnabled) Catch ' This is a critical failure; the server cannot start without DB1 Dim ErrorMsg As String = $"Main.AppStart: CRITICAL ERROR initializing connector 'DB1': ${LastException.Message}"$ Log(ErrorMsg) ' Log the error to the SQLite DB (if it's already initialized) LogServerError("ERROR", "Main.AppStart", ErrorMsg, "DB1", Null, Null) ' Stop the application ExitApplication End Try ' === 5. DETECTION AND INITIALIZATION OF ADDITIONAL DATABASES (DB2, DB3, DB4) === ' Scan the application's root directory for configuration files cpFiles = File.ListFiles("./") If cpFiles.Size > 0 Then For Each fileName As String In cpFiles Dim keyPrefix As String = "config." Dim keySuffix As String = ".properties" ' 1. Filter and exclude DB1 (which is already loaded) ' Find files matching "config.xxx.properties" but not "config.properties" If fileName.StartsWith(keyPrefix) And fileName.EndsWith(keySuffix) And fileName <> "config.properties" Then Try ' 2. Extract the key ("xxx" from config.xxx.properties) Dim keyLength As Int = fileName.Length - keySuffix.Length Dim dbKey As String = fileName.SubString2(keyPrefix.Length, keyLength) ' ROBUSTNESS: Ensure the key is UPPERCASE for consistency. ' Handlers normalize the key to uppercase, so we must match that. dbKey = dbKey.ToUpperCase.Trim Log($"Main.AppStart: Configuration file detected: '${fileName}'. Initializing connector '${dbKey}'."$) Dim newCon As RDCConnector ' 3. Initialize the RDC Connector (which reads its own config.dbKey.properties file) newCon.Initialize(dbKey) ' 4. Update global structures (Thread-Safe Maps) Connectors.Put(dbKey, newCon) listaDeCP.Add(dbKey) ' 5. Granular Logging Logic ' Capture the logging status for this new DB Dim enableLogsSetting As Int = newCon.config.GetDefault("enableSQLiteLogs", 0).As(Int) Dim isEnabled As Boolean = (enableLogsSetting = 1) SQLiteLoggingStatusByDB.Put(dbKey, isEnabled) ' Capture text logging status for this new DB Dim enableTextLogsSetting As Int = newCon.config.GetDefault("enableTextLogging", 0).As(Int) Dim isTextEnabled As Boolean = (enableTextLogsSetting = 1) TextLoggingStatusByDB.Put(dbKey, isTextEnabled) Log("TEXT LOGGING STATUS BY DB: " & TextLoggingStatusByDB) ' Note: Global re-evaluation of IsAnySQLiteLoggingEnabled is done at the end of AppStart. Catch ' 6. Error Handling: If a file is invalid (e.g., bad credentials, malformed URL), ' the server should log the error but continue trying with the next file. Dim ErrorMsg As String = $"Main.AppStart: CRITICAL ERROR initializing connector '${dbKey}' from '${fileName}': ${LastException.Message}"$ Log(ErrorMsg) LogServerError("ERROR", "Main.AppStart", ErrorMsg, dbKey, Null, Null) End Try End If Next End If ' Final log of all databases the server is managing. Dim sbListaDeCP_Log As StringBuilder sbListaDeCP_Log.Initialize For Each item As String In listaDeCP sbListaDeCP_Log.Append(item).Append(", ") Next If sbListaDeCP_Log.Length > 0 Then sbListaDeCP_Log.Remove(sbListaDeCP_Log.Length - 2, sbListaDeCP_Log.Length) End If Log($"Main.AppStart: Configured and ready databases: [${sbListaDeCP_Log.ToString}]"$) ' <<<< Initialization block for the log cleanup Timer >>>> ' UNCONDITIONAL Initialization of the Timer (Ensures the object exists and prevents IllegalStateException) timerLogs.Initialize("TimerLogs", 1800000) ' 30 minutes = 1800 * 1000 = 1800000 ms ' CONDITIONAL CONTROL BASED ON GRANULAR STATUS IsAnySQLiteLoggingEnabled = False For Each dbStatus As Boolean In SQLiteLoggingStatusByDB.Values If dbStatus Then IsAnySQLiteLoggingEnabled = True Exit ' If one is active, it's enough to turn on the Timer End If Next If IsAnySQLiteLoggingEnabled Then timerLogs.Enabled = True If logger Then Log("Main.AppStart: Log cleanup timer ACTIVATED (at least one DB requires logs).") Else timerLogs.Enabled = False If logger Then Log("Main.AppStart: Log cleanup timer DISABLED (no DB requires logs).") End If ' <<<< End of Timer block >>>> ' === 6. REGISTERING HTTP HANDLERS FOR THE SERVER === srvr.AddHandler("/ping", "ping", False) srvr.AddHandler("/test", "TestHandler", False) srvr.AddHandler("/login", "LoginHandler", False) srvr.AddHandler("/dologin", "DoLoginHandler", False) srvr.AddHandler("/logout", "LogoutHandler", False) srvr.AddHandler("/changepass", "ChangePassHandler", False) srvr.AddHandler("/manager", "Manager", False) srvr.AddHandler("/DBJ", "DBHandlerJSON", False) srvr.AddHandler("/dbrquery", "DBHandlerJSON", False) srvr.AddHandler("/favicon.ico", "faviconHandler", False) srvr.AddHandler("/stats-stream", "SSEHandler", False) srvr.AddHandler("/*", "DBHandlerB4X", False) ' 7. Start the HTTP server. srvr.Start Log("===========================================================") Log($"-=== jRDC is running on port: ${srvr.Port} (version = $1.2{VERSION}) ===-"$) Log("===========================================================") ' 8. Start the B4J message loop. StartMessageLoop End Sub ' --- Subroutine to initialize the local user database (SQLite) --- Sub InitializeSQLiteDatabase Dim dbFileName As String = "users.db" ' Check if the database file already exists If File.Exists(File.DirApp, dbFileName) = False Then ' --- Create a new database --- Log("Creating new user database: " & dbFileName) SQL1.InitializeSQLite(File.DirApp, dbFileName, True) ' Create 'users' table Dim createUserTable As String = "CREATE TABLE users (username TEXT PRIMARY KEY, password_hash TEXT NOT NULL)" SQL1.ExecNonQuery(createUserTable) ' Create 'query_logs' table If logger Then Log("Creating 'query_logs' table with performance columns.") Dim createQueryLogsTable As String = "CREATE TABLE query_logs (id INTEGER PRIMARY KEY AUTOINCREMENT, query_name TEXT, duration_ms INTEGER, timestamp INTEGER, db_key TEXT, client_ip TEXT, busy_connections INTEGER, handler_active_requests INTEGER)" SQL1.ExecNonQuery(createQueryLogsTable) ' Set PRAGMA for better performance (Write-Ahead Logging) SQL1.ExecNonQuery("PRAGMA journal_mode=WAL;") SQL1.ExecNonQuery("PRAGMA synchronous=NORMAL;") ' Insert default user Dim defaultUser As String = "admin" Dim defaultPass As String = "admin" Dim hashedPass As String = bc.hashpw(defaultPass, bc.gensalt) SQL1.ExecNonQuery2("INSERT INTO users (username, password_hash) VALUES (?, ?)", Array As Object(defaultUser, hashedPass)) Log($"Default user created -> user: ${defaultUser}, pass: ${defaultPass}"$) ' Create 'errores' (errors) table Log("Creating 'errores' table for event logging.") Dim createErrorsTable As String = "CREATE TABLE errores (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER, type TEXT, source TEXT, message TEXT, db_key TEXT, command_name TEXT, client_ip TEXT)" SQL1.ExecNonQuery(createErrorsTable) If logger Then Log("Creating performance indexes on log tables.") ' Index on timestamp for fast cleanup (DELETE/ORDER BY) in query_logs SQL1.ExecNonQuery("CREATE INDEX idx_query_timestamp ON query_logs(timestamp)") ' Index on duration_ms for the 'slowqueries' query (ORDER BY) SQL1.ExecNonQuery("CREATE INDEX idx_query_duration ON query_logs(duration_ms)") ' Index on timestamp for fast cleanup of the errors table SQL1.ExecNonQuery("CREATE INDEX idx_error_timestamp ON errores(timestamp)") Else ' --- Load existing database --- SQL1.InitializeSQLite(File.DirApp, dbFileName, True) Log("User database loaded.") ' Ensure WAL mode is set on existing DBs SQL1.ExecNonQuery("PRAGMA journal_mode=WAL;") SQL1.ExecNonQuery("PRAGMA synchronous=NORMAL;") ' >>> START: Migration logic (ALTER TABLE) if the DB already existed <<< If logger Then Log("Verifying and migrating 'query_logs' table if necessary.") ' Check if 'query_logs' table exists If SQL1.ExecQuerySingleResult("SELECT name FROM sqlite_master WHERE type='table' AND name='query_logs'") = Null Then If logger Then Log("'query_logs' table not found, creating it with performance columns.") Dim createQueryLogsTable As String = "CREATE TABLE query_logs (id INTEGER PRIMARY KEY AUTOINCREMENT, query_name TEXT, duration_ms INTEGER, timestamp INTEGER, db_key TEXT, client_ip TEXT, busy_connections INTEGER, handler_active_requests INTEGER)" SQL1.ExecNonQuery(createQueryLogsTable) Else ' If the query_logs table already exists, check and add missing columns Dim columnExists As Boolean Dim rs As ResultSet ' --- VERIFY AND ADD busy_connections --- columnExists = False rs = SQL1.ExecQuery("PRAGMA table_info(query_logs)") Do While rs.NextRow If rs.GetString("name").EqualsIgnoreCase("busy_connections") Then columnExists = True Exit ' Column already exists, exit loop End If Loop rs.Close If columnExists = False Then If logger Then Log("Adding column 'busy_connections' to query_logs.") SQL1.ExecNonQuery("ALTER TABLE query_logs ADD COLUMN busy_connections INTEGER DEFAULT 0") End If ' --- VERIFY AND ADD handler_active_requests --- columnExists = False rs = SQL1.ExecQuery("PRAGMA table_info(query_logs)") Do While rs.NextRow If rs.GetString("name").EqualsIgnoreCase("handler_active_requests") Then columnExists = True Exit ' Column already exists, exit loop End If Loop rs.Close If columnExists = False Then If logger Then Log("Adding column 'handler_active_requests' to query_logs.") SQL1.ExecNonQuery("ALTER TABLE query_logs ADD COLUMN handler_active_requests INTEGER DEFAULT 0") End If ' --- VERIFY AND ADD timestamp_text_local --- columnExists = False rs = SQL1.ExecQuery("PRAGMA table_info(query_logs)") Do While rs.NextRow If rs.GetString("name").EqualsIgnoreCase("timestamp_text_local") Then columnExists = True Exit ' Column already exists, exit loop End If Loop rs.Close If columnExists = False Then If logger Then Log("Adding column 'timestamp_text_local' to query_logs.") ' Use 'TEXT' to store the formatted date/time string. SQL1.ExecNonQuery("ALTER TABLE query_logs ADD COLUMN timestamp_text_local TEXT") End If ' >>> START: Migration logic for 'errores' if DB already existed <<< If logger Then Log("Verifying and migrating 'errores' table if necessary.") If SQL1.ExecQuerySingleResult("SELECT name FROM sqlite_master WHERE type='table' AND name='errores'") = Null Then If logger Then Log("'errores' table not found, creating it.") Dim createErrorsTable As String = "CREATE TABLE errores (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER, type TEXT, source TEXT, message TEXT, db_key TEXT, command_name TEXT, client_ip TEXT)" SQL1.ExecNonQuery(createErrorsTable) Else If logger Then Log("'errores' table already exists.") End If ' >>> END: Migration logic for 'errores' <<< End If ' >>> END: Migration logic (ALTER TABLE) <<< End If End Sub ' Public subroutine to log query performance. Public Sub LogQueryPerformance(QueryName As String, DurationMs As Long, DbKey As String, ClientIp As String, HandlerActiveRequests As Int, PoolBusyConnections As Int) ' Check if logging is enabled for this specific DBKey Dim isEnabled As Boolean = SQLiteLoggingStatusByDB.GetDefault(DbKey, False) If isEnabled Then ' Set date format for the new text timestamp column DateTime.DateFormat = "yyyy-MM-dd HH:mm:ss.SSS" Dim formattedTimestamp As String = DateTime.Date(DateTime.Now) ' 1. Create the data map (log entry) Dim logEntry As Map = CreateMap("query_name": QueryName, "duration_ms": DurationMs, "timestamp": DateTime.Now, _ "db_key": DbKey, "client_ip": ClientIp, "busy_connections": PoolBusyConnections, _ "handler_active_requests": HandlerActiveRequests, "timestamp_text_local": formattedTimestamp) ' 2. Critical Zone: Add to cache and check threshold Dim shouldWriteBatch As Boolean = False ' Use the *cache lock* to ensure adding and size-checking are atomic LogCacheLock.RunMethod("lock", Null) QueryLogCache.Add(logEntry) If QueryLogCache.Size >= LOG_CACHE_THRESHOLD Then shouldWriteBatch = True End If LogCacheLock.RunMethod("unlock", Null) ' 3. If the threshold was reached, trigger the write. ' This MUST be done OUTSIDE the lock. If shouldWriteBatch Then CallSub(Me, "WriteQueryLogsBatch") End If End If End Sub ' --- Subroutine to log errors and warnings in the 'errores' table. --- Public Sub LogServerError(Type0 As String, Source As String, Message As String, DBKey As String, CommandName As String, ClientIp As String) If logger Then Log($">>>> LogServerError <<<<<${CRLF}tipo:${Type0}, source:${Source}, message:${Message}, dbkey:${DBKey}, commandanme:${CommandName}"$) ' Check if logging is enabled for this specific DBKey (or fallback if DBKey is null) Dim isEnabled As Boolean = SQLiteLoggingStatusByDB.GetDefault(DBKey, False) If isEnabled Then ' 1. Create the log entry map Dim logEntry As Map = CreateMap("timestamp": DateTime.Now, "type": Type0, "source": Source, "message": Message, _ "db_key": DBKey, "command_name": CommandName, "client_ip": ClientIp) Dim shouldWriteBatch As Boolean = False ' 2. Critical Zone: Add to cache and check threshold ' Use the *cache lock* for Thread Safety LogCacheLock.RunMethod("lock", Null) ' Log(">>>>> Agregamos a errorLog") ErrorLogCache.Add(logEntry) If ErrorLogCache.Size >= LOG_CACHE_THRESHOLD Then shouldWriteBatch = True End If LogCacheLock.RunMethod("unlock", Null) ' 3. If threshold was reached, trigger the write If shouldWriteBatch Then CallSub(Me, "WriteErrorLogsBatch") End If Else ' Logging is disabled for this DBKey, so the log is skipped. End If End Sub ' Writes the cached performance logs to the SQLite DB in a single transaction Public Sub WriteQueryLogsBatch Dim logsToWrite As List logsToWrite.Initialize ' 1. Initialize the local list (CRITICAL) ' === STEP 1: Atomic Cache Swap (Protected by ReentrantLock) === LogCacheLock.RunMethod("lock", Null) If QueryLogCache.Size = 0 Then ' Cache is empty, release lock and return LogCacheLock.RunMethod("unlock", Null) Return End If ' *** CRITICAL FIX: Copy content (AddAll) instead of reference. *** logsToWrite.AddAll(QueryLogCache) Dim batchSize As Int = logsToWrite.Size ' Clear the global cache. logsToWrite now holds the copy of the items. QueryLogCache.Initialize LogCacheLock.RunMethod("unlock", Null) ' Check if text logging is enabled for any of these logs If logsToWrite.Size > 0 Then ' Call the text archiving sub on a separate worker thread. ' This is NON-BLOCKING for the current thread, which will proceed to the SQLite transaction. CallSubDelayed2(Me, "ArchiveQueryLogsToDailyFile", logsToWrite) End If ' === STEP 2: Transactional Write to SQLite === Try ' 1. Begin the transaction: Everything that follows is a single disk operation. SQL1.BeginTransaction For Each logEntry As Map In logsToWrite ' Insert the log entry SQL1.ExecNonQuery2("INSERT INTO query_logs (query_name, duration_ms, timestamp, db_key, client_ip, busy_connections, handler_active_requests, timestamp_text_local) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", _ Array As Object(logEntry.Get("query_name"), logEntry.Get("duration_ms"), logEntry.Get("timestamp"), logEntry.Get("db_key"), _ logEntry.Get("client_ip"), logEntry.Get("busy_connections"), logEntry.Get("handler_active_requests"), _ logEntry.Get("timestamp_text_local"))) Next ' 2. Finalize the transaction: Efficient write to disk. SQL1.TransactionSuccessful If logger Then Log($"[LOG BATCH] Batch of ${batchSize} performance logs written successfully."$) Catch ' If it fails, undo all logs in this batch and log the failure. SQL1.Rollback Dim ErrorMsg As String = "CRITICAL ERROR: Failed to write performance log batch to SQLite: " & LastException.Message Log(ErrorMsg) ' Use LogServerError so the failure is recorded in the 'errores' table (if logging is enabled) LogServerError("ERROR", "Main.WriteQueryLogsBatch", ErrorMsg, Null, "log_batch_write_performance", Null) End Try End Sub ' --- Event subroutine for the 'timerLogs' Timer. --- ' The 'Enabled' state of the Timer is already controlled by IsAnySQLiteLoggingEnabled in AppStart and Manager. Sub TimerLogs_Tick Try ' 1. Flush performance logs WriteQueryLogsBatch ' 2. Flush error logs WriteErrorLogsBatch ' 3. Clean up and VACUUM (this sub also checks IsAnySQLiteLoggingEnabled) borraArribaDe30000Logs Catch Dim ErrorMsg As String = "ERROR in TimerLogs_Tick while trying to clear logs: " & LastException.Message Log(ErrorMsg) LogServerError("ERROR", "Main.TimerLogs_Tick", ErrorMsg, Null, "log_cleanup", Null) End Try End Sub ' Writes the cached error logs to the SQLite DB in a single transaction Public Sub WriteErrorLogsBatch Dim logsToWrite As List logsToWrite.Initialize ' === STEP 1: Atomic Cache Swap (Protected by ReentrantLock) === ' Bloqueamos el LogCacheLock para garantizar la atomicidad de la copia y limpieza. LogCacheLock.RunMethod("lock", Null) If ErrorLogCache.Size = 0 Then ' La caché está vacía, liberamos el lock inmediatamente y salimos. LogCacheLock.RunMethod("unlock", Null) Return End If ' *** Copiar el contenido de la caché global de forma atómica. *** logsToWrite.AddAll(ErrorLogCache) ' Usar el tamaño de la lista copiada para el procesamiento. Dim batchSize As Int = logsToWrite.Size ' Log(logsToWrite) ' Limpiar la caché global. logsToWrite es ahora una lista independiente y poblada. ErrorLogCache.Initialize LogCacheLock.RunMethod("unlock", Null) ' Liberar el lock. If logger Then Log($"[LOG BATCH] Starting transactional write of ${batchSize} ERROR logs to SQLite. Logs copied: ${batchSize}"$) ' === La corrección de Lógica ocurre aquí: La llamada a ArchiveErrorLogsToDailyFile ' y el proceso transaccional ocurren AHORA, después de asegurar que logsToWrite ' tiene contenido y que el lock fue liberado. === ' 1. (Opcional, si el logging de texto CSV está habilitado) If batchSize > 0 Then ' Delegar a una nueva subrutina para manejar la I/O de disco CSV (CallSubDelayed2) CallSubDelayed2(Me, "ArchiveErrorLogsToDailyFile", logsToWrite) End If ' === STEP 2: Escritura Transaccional a SQLite (Usa logsToWrite) === If batchSize = 0 Then ' Este caso no debería ocurrir con la lógica anterior, pero es un chequeo de seguridad. Log("WARNING: Failed to copy list. logsToWrite is empty. Aborting write.") Return End If Try ' 1. Iniciar la transacción. SQL1.BeginTransaction For Each logEntry As Map In logsToWrite ' Insertar la entrada de log SQL1.ExecNonQuery2("INSERT INTO errores (timestamp, type, source, message, db_key, command_name, client_ip) VALUES (?, ?, ?, ?, ?, ?, ?)", _ Array As Object(logEntry.Get("timestamp"), logEntry.Get("type"), logEntry.Get("source"), logEntry.Get("message"), _ logEntry.Get("db_key"), logEntry.Get("command_name"), logEntry.Get("client_ip"))) Next ' 2. Confirmar la transacción. SQL1.TransactionSuccessful If logger Then Log($"[LOG BATCH] Batch of ${logsToWrite.Size} ERROR logs written successfully."$) Catch ' 3. Rollback si falla. SQL1.Rollback Dim ErrorMsg As String = "CRITICAL ERROR: Failed to write ERROR log batch to SQLite: " & LastException.Message Log(ErrorMsg) End Try End Sub ' Deletes the oldest records from 'query_logs' table and runs VACUUM. Sub borraArribaDe30000Logs 'ignore If IsAnySQLiteLoggingEnabled Then ' Only run if at least one DB requires logs. ' 1. Cleanup of Performance Logs (query_logs) If logger Then Log("Trimming 'query_logs' table, limit of 30,000 records.") Dim fechaCorte As Long ' (cutoff date/timestamp) ' First, try to find the timestamp of the 30,001st record. Try ' OFFSET 30000 skips the 30,000 most recent. fechaCorte = SQL1.ExecQuerySingleResult($"SELECT timestamp FROM query_logs ORDER BY timestamp DESC LIMIT 1 OFFSET 30000"$) Catch ' If the table has fewer than 30,000 records, the result is NULL or throws an exception. fechaCorte = 0 ' Force to 0 so it doesn't delete anything. End Try ' If a cutoff time was found (i.e., there are more than 30,000 records)... If fechaCorte > 0 Then ' Execute the simple DELETE, which is very fast using the idx_query_timestamp index. SQL1.ExecNonQuery2("DELETE FROM query_logs WHERE timestamp < ?", Array As Object(fechaCorte)) End If ' 2. Cleanup of Error Logs (errores) Dim fechaCorteError As Long Try ' OFFSET 15000 skips the 15,000 most recent. fechaCorteError = SQL1.ExecQuerySingleResult($"SELECT timestamp FROM errores ORDER BY timestamp DESC LIMIT 1 OFFSET 15000"$) Catch ' If the table has fewer than 15,000 records, result is NULL. fechaCorteError = 0 End Try ' If a cutoff time was found... If fechaCorteError > 0 Then SQL1.ExecNonQuery2("DELETE FROM errores WHERE timestamp < ?", Array As Object(fechaCorteError)) End If ' 3. Control and Conditional Execution of VACUUM TimerTickCount = TimerTickCount + 1 If TimerTickCount >= VACUUM_CYCLES Then If logger Then Log("EXECUTING VACUUM (24-hour cycle completed).") SQL1.ExecNonQuery("vacuum;") ' Execute VACUUM. TimerTickCount = 0 ' Reset the counter. Else ' Show how many cycles are left, only if logger is active. If logger Then Log($"VACUUM skipped. ${VACUUM_CYCLES - TimerTickCount} cycles remaining until daily execution."$) End If Else ' If IsAnySQLiteLoggingEnabled is False, the Timer should not be active. If logger Then Log("NOTICE: Log cleanup task skipped. Global SQLite logging is disabled.") End If End Sub 'Copies resources from the jar to the app directory Sub CopiarRecursoSiNoExiste(NombreArchivo As String, SubCarpeta As String) Dim DirDestino As String = File.Combine(File.DirApp, SubCarpeta) ' If the subfolder is not empty and doesn't exist, create it If SubCarpeta <> "" And File.Exists(DirDestino, "") = False Then File.MakeDir(DirDestino, "") End If Dim ArchivoDestino As String = File.Combine(DirDestino, NombreArchivo) ' If the target file doesn't exist, copy it from resources If File.Exists(DirDestino, NombreArchivo) = False Then Dim RutaRecurso As String If SubCarpeta <> "" Then ' Path inside the JAR (e.g., "Files/www/manager.html") RutaRecurso = "Files/" & SubCarpeta & "/" & NombreArchivo Else ' Path inside the JAR (e.g., "Files/config.properties") RutaRecurso = "Files/" & NombreArchivo End If ' Get the correct class loader to access JAR resources Dim classLoader As JavaObject = GetThreadContextClassLoader Dim InStream As InputStream = classLoader.RunMethod("getResourceAsStream", Array(RutaRecurso)) If InStream.IsInitialized Then Log($"Copiando recurso: '${RutaRecurso}'..."$) ' Call our own manual copy function Dim OutStream As OutputStream = File.OpenOutput(DirDestino, NombreArchivo, False) CopiarStreamManualmente(InStream, OutStream) Log($"'${ArchivoDestino}' copiado correctamente."$) Else Log($"ERROR: Could not find the resource with the internal path: '${RutaRecurso}'"$) End If End If End Sub ' Does not depend on any external libraries (like File.Copy) Sub CopiarStreamManualmente (InStream As InputStream, OutStream As OutputStream) Try Dim buffer(1024) As Byte Dim len As Int ' Read bytes from input stream len = InStream.ReadBytes(buffer, 0, buffer.Length) Do While len > 0 ' Write bytes to output stream OutStream.WriteBytes(buffer, 0, len) ' Read next chunk len = InStream.ReadBytes(buffer, 0, buffer.Length) Loop Catch LogError(LastException) End Try ' Always close streams InStream.Close OutStream.Close End Sub ' Helper function to get the correct Class Loader Sub GetThreadContextClassLoader As JavaObject Dim thread As JavaObject thread = thread.InitializeStatic("java.lang.Thread").RunMethod("currentThread", Null) Return thread.RunMethod("getContextClassLoader", Null) End Sub ' This runs on a separate worker thread (via CallSubDelayed2) Private Sub ArchiveQueryLogsToDailyFile(logs As List) ' Set date format for the filename DateTime.DateFormat = "yyyy-MM-dd" Dim dateStr As String = DateTime.Date(DateTime.Now) Dim fileBaseName As String = $"query_logs_${dateStr}.csv"$ ' Fields based on the log structure: Dim HEADER_LINE As String = $""timestamp_text_local","query_name","duration_ms","db_key","client_ip","busy_connections","handler_active_requests","timestamp_millis""$ Dim sbContent As StringBuilder sbContent.Initialize ' === 1. CHECK AND WRITE HEADER (CRITICAL) === ' Only write the header if the file does NOT exist OR if it exists but is empty. Dim writeHeader As Boolean = False If Not(File.Exists(File.DirApp, fileBaseName)) Or File.Size(File.DirApp, fileBaseName) = 0 Then writeHeader = True End If If writeHeader Then sbContent.Append(HEADER_LINE).Append(CRLF) End If ' === 2. GENERATE CONTENT IN MEMORY === For Each logEntry As Map In logs ' Check if text logging is enabled for this specific DBKey Dim dbKey As String = logEntry.Get("db_key").As(String) If TextLoggingStatusByDB.GetDefault(dbKey, False) Then ' Format the log line (using double quotes for CSV) Dim line As String = $""${logEntry.Get("timestamp_text_local")}","${logEntry.Get("query_name")}",${logEntry.Get("duration_ms")},"${dbKey}","${logEntry.Get("client_ip")}",${logEntry.Get("busy_connections")},${logEntry.Get("handler_active_requests")},${logEntry.Get("timestamp")}"$ sbContent.Append(line).Append(CRLF) End If Next ' === 3. CONSOLIDATED WRITE TO DISK === If sbContent.Length > 0 Then Dim outStream As OutputStream Try ' APPEND mode (True) ensures that both the header (if written) and the logs are added. outStream = File.OpenOutput(File.DirApp, fileBaseName, True) Dim bytes() As Byte = sbContent.ToString.GetBytes("UTF8") outStream.WriteBytes(bytes, 0, bytes.Length) Catch ' Log the error to the SQLite database LogServerError("ADVERTENCIA", "ArchiveQueryLogsToDailyFile", $"Fallo al escribir lote + HEADER en ${fileBaseName}: ${LastException.Message}"$, "SYSTEM", "Log Batch Write", "N/A") End Try If outStream.IsInitialized Then outStream.Close End If End If End Sub ' This runs on a separate worker thread (via CallSubDelayed2) Private Sub ArchiveErrorLogsToDailyFile(logs As List) Log(">>>>>>> ArchiveErrorLogsToDailyFile <<<<<<<<<<<<< ") ' Log(CRLF & logs) ' Set date format for the filename DateTime.DateFormat = "yyyy-MM-dd" Dim dateStr As String = DateTime.Date(DateTime.Now) Dim fileBaseName As String = $"error_logs_${dateStr}.csv"$ ' Different filename ' AVAILABLE ERROR FIELDS (See LogServerError): ' timestamp, type, source, message, db_key, command_name, client_ip Dim HEADER_LINE As String = $""timestamp","type","source","message","db_key","command_name","client_ip""$ Dim sbContent As StringBuilder sbContent.Initialize ' === 1. CONDITIONAL WRITE OF HEADER === ' Check if file exists or is empty. Only write header if it's the first time today. Dim writeHeader As Boolean = False Try If Not(File.Exists(File.DirApp, fileBaseName)) Or File.Size(File.DirApp, fileBaseName) = 0 Then writeHeader = True End If Catch ' In case of an I/O error checking the file, assume we should try to write the header. writeHeader = True End Try If writeHeader Then sbContent.Append(HEADER_LINE).Append(CRLF) End If ' === 2. GENERATE CONTENT IN MEMORY === For Each logEntry As Map In logs ' Log($"--- agregamos ${logEntry.Get("db_key").As(String).ToUpperCase}"$) Dim dbKey As String = logEntry.Get("db_key").As(String).ToUpperCase ' Log($"==== ${dbKey} -> ${TextLoggingStatusByDB.GetDefault(dbKey, False)}"$) ' Log(TextLoggingStatusByDB) If TextLoggingStatusByDB.GetDefault(dbKey, False) Then ' Format the log line for CSV. ' The 'message' field is CRITICAL, as it can contain multiline exceptions or special characters. Dim line As String = $""${logEntry.Get("timestamp")}","${logEntry.Get("type")}","${logEntry.Get("source")}","${logEntry.Get("message")}","${dbKey}","${logEntry.Get("command_name")}","${logEntry.Get("client_ip")}""$ sbContent.Append(line).Append(CRLF) End If Next ' === 3. CONSOLIDATED WRITE TO DISK === If sbContent.Length > 0 Then Dim outStream As OutputStream Try ' Open the stream in APPEND mode (True) to add the batch outStream = File.OpenOutput(File.DirApp, fileBaseName, True) Dim bytes() As Byte = sbContent.ToString.GetBytes("UTF8") outStream.WriteBytes(bytes, 0, bytes.Length) Catch ' I/O failure in secondary thread. Use LogServerError so it's logged to SQLite (if active) LogServerError("ERROR", "ArchiveErrorLogsToDailyFile", $"Fallo E/S al escribir logs de ERRORES en ${fileBaseName}: ${LastException.Message}"$, "SYSTEM", "Error Log Batch Write", "N/A") End Try If outStream.IsInitialized Then outStream.Close End If End If End Sub