diff --git a/cos-to-sql/app.mjs b/cos-to-sql/app.mjs index ea9605d1f..123569fcf 100644 --- a/cos-to-sql/app.mjs +++ b/cos-to-sql/app.mjs @@ -1,4 +1,5 @@ import { stat } from "fs/promises"; +import crypto from "crypto"; import express from "express"; import favicon from "serve-favicon"; import path from "path"; @@ -15,8 +16,11 @@ import { ContainerAuthenticator } from "ibm-cloud-sdk-core"; import { addUser, closeDBConnection, deleteUsers, getPgClient, listUsers } from "./utils/db.mjs"; import { getObjectContent } from "./utils/cos.mjs"; import { convertCsvToDataStruct } from "./utils/utils.mjs"; +import { getCodeEngineLogger } from "./utils/logging.mjs"; -console.info("Starting the app ..."); +const appLogger = getCodeEngineLogger("app"); + +appLogger.info("Starting the app ..."); const requiredEnvVars = [ "COS_REGION", @@ -28,7 +32,7 @@ const requiredEnvVars = [ requiredEnvVars.forEach((envVarName) => { if (!process.env[envVarName]) { - console.log(`Failed to start app. Missing '${envVarName}' environment variable`); + appLogger.warn(`Failed to start app. Missing '${envVarName}' environment variable`); process.exit(1); } }); @@ -60,18 +64,42 @@ const app = express(); app.use(express.json()); app.use(favicon(path.join(__dirname, "public", "favicon.ico"))); +// helper function to send JSON responses +function sendJSONResponse(request, response, returnCode, jsonObject) { + response.status(returnCode); + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify(jsonObject)); + const duration = Date.now() - request.startTime; + request.logger.info(`handled ${request.method} request for '${request.path}' in ${duration}ms; rc: '${returnCode}'`, { + returnCode, + duration_ms: duration, + }); +} + // use router to bundle all routes to / const router = express.Router(); + +// adding a middleware that extracts the correlation id and initializes the logger +router.use((req, res, next) => { + const correlationId = req.header("x-request-id") || crypto.randomBytes(8).toString("hex"); + req.startTime = Date.now(); + req.correlationId = correlationId; + const operationId = `${req.method}:${req.path}`; + req.operationId = operationId; + req.logger = getCodeEngineLogger("handle-request").child({ correlationId, operationId }); + req.logger.info(`handling incoming ${req.method} request for '${req.path}'`); + next(); +}); + app.use("/", router); // // Default http endpoint, which prints the list of all users in the database router.get("/", async (req, res) => { - console.log(`handling / for '${req.url}'`); const pgClient = await getPgClient(secretsManager, smPgSecretId); const allUsers = await listUsers(pgClient); - res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ users: allUsers.rows })); + + return sendJSONResponse(req, res, 200, { users: allUsers.rows }); }); // @@ -80,111 +108,102 @@ router.get("/readiness", async (req, res) => { console.log(`handling /readiness`); if (!(await stat("/var/run/secrets/codeengine.cloud.ibm.com/compute-resource-token/token"))) { console.error("Mounting the trusted profile compute resource token is not enabled"); - res.writeHead(500, { "Content-Type": "application/json" }); - res.end('{"error": "Mounting the trusted profile compute resource token is not enabled"}'); - return; + + return sendJSONResponse(req, res, 500, { + error: "Mounting the trusted profile compute resource token is not enabled", + }); } - res.writeHead(200, { "Content-Type": "application/json" }); - res.end('{"status": "ok"}'); - return; + return sendJSONResponse(req, res, 200, { status: "ok" }); }); // // Ingestion endpoint router.post("/cos-to-sql", async (req, res) => { - console.log(`handling /cos-to-sql for '${req.url}'`); - console.log(`request headers: '${JSON.stringify(req.headers)}`); + req.logger.debug(`request headers: '${JSON.stringify(req.headers)}`); const event = req.body; - console.log(`request body: '${event}'`); + req.logger.debug(`request body: '${event}'`); // // assess whether the request payload contains information about the COS file that got updated if (!event) { - console.log("Request does not contain any event data"); - res.writeHead(400, { "Content-Type": "application/json" }); - res.end('{"error": "request does not contain any event data"}'); - return; + req.logger.info("Request does not contain any event data"); + return sendJSONResponse(req, res, 400, { error: "request does not contain any event data" }); } // // make sure that the event relates to a COS write operation if (event.notification.event_type !== "Object:Write") { - console.log(`COS operation '${event.notification.event_type}' does not match expectations 'Object:Write'`); - res.writeHead(400, { "Content-Type": "application/json" }); - res.end(`{"error": "COS operation '${event.notification.event_type}' does not match expectations 'Object:Write'"}`); - return; + req.logger.info(`COS operation '${event.notification.event_type}' does not match expectations 'Object:Write'`); + + return sendJSONResponse(req, res, 400, { + error: `COS operation '${event.notification.event_type}' does not match expectations 'Object:Write'`, + }); } if (event.notification.content_type !== "text/csv") { - console.log( + req.logger.info( `COS update did happen on file '${event.key}' which is of type '${event.notification.content_type}' (expected type 'text/csv')` ); - res.writeHead(400, { "Content-Type": "application/json" }); - res.end( - `{"error": "COS update did happen on file '${event.key}' which is of type '${event.notification.content_type}' (expected type 'text/csv')"}` - ); - return; + + return sendJSONResponse(req, res, 400, { + error: `COS update did happen on file '${event.key}' which is of type '${event.notification.content_type}' (expected type 'text/csv')`, + }); } - console.log(`Received a COS update event on the CSV file '${event.key}' in bucket '${event.bucket}'`); + req.logger.info(`Received a COS update event on the CSV file '${event.key}' in bucket '${event.bucket}'`); // // Retrieve the COS object that got updated - console.log(`Retrieving file content of '${event.key}' from bucket ${event.bucket} ...`); + req.logger.info(`Retrieving file content of '${event.key}' from bucket ${event.bucket} ...`); const fileContent = await getObjectContent(cosAuthenticator, cosRegion, event.bucket, event.key); // // Convert CSV to a object structure representing an array of users - console.log(`Converting CSV data to a data struct ...`); + req.logger.info(`Converting CSV data to a data struct ...`); const users = await convertCsvToDataStruct(fileContent); - console.log(`users: ${JSON.stringify(users)}`); + req.logger.info(`users: ${JSON.stringify(users)}`); const pgClient = await getPgClient(secretsManager, smPgSecretId); // // Iterate through the list of users - console.log(`Writing converted CSV data to the PostgreSQL database ...`); + req.logger.info(`Writing converted CSV data to the PostgreSQL database ...`); let numberOfProcessedUsers = 0; for (const userToAdd of users) { try { // Perform a single SQL insert statement per user const result = await addUser(pgClient, userToAdd.Firstname, userToAdd.Lastname); - console.log(`Added ${JSON.stringify(userToAdd)} -> ${JSON.stringify(result)}`); + req.logger.info(`Added ${JSON.stringify(userToAdd)} -> ${JSON.stringify(result)}`); numberOfProcessedUsers++; } catch (err) { - console.error(`Failed to add user '${JSON.stringify(userToAdd)}' to the database`, err); + req.logger.error(`Failed to add user '${JSON.stringify(userToAdd)}' to the database`, err); } } - console.log(`Processed ${numberOfProcessedUsers} user records!`); - res.writeHead(200, { "Content-Type": "application/json" }); - res.end(`{"status": "done"}`); - return; + req.logger.info(`Processed ${numberOfProcessedUsers} user records!`); + return sendJSONResponse(req, res, 200, { status: "done" }); }); // // Endpoint that drops the users table router.get("/clear", async (req, res) => { - console.log(`handling /clear for '${req.url}'`); const pgClient = await getPgClient(secretsManager, smPgSecretId); await deleteUsers(pgClient); - console.log(`Deletions done!`); - res.writeHead(200, { "Content-Type": "application/json" }); - res.end(`{"status": "done"}`); - return; + req.logger.info(`Deletions done!`); + return sendJSONResponse(req, res, 200, { status: "done" }); }); // start server const port = 8080; const server = app.listen(port, () => { - console.log(`Server is up and running on port ${port}!`); + appLogger.info(`Server is up and running on port ${port}!`); }); process.on("SIGTERM", async () => { - console.info("SIGTERM signal received."); + appLogger.info("SIGTERM signal received."); await closeDBConnection(); server.close(() => { - console.log("Http server closed."); + appLogger.info("Http server closed."); }); }); diff --git a/cos-to-sql/package-lock.json b/cos-to-sql/package-lock.json index fbb34d22a..f2777f4de 100644 --- a/cos-to-sql/package-lock.json +++ b/cos-to-sql/package-lock.json @@ -15,7 +15,28 @@ "ibm-cloud-sdk-core": "^5.3.0", "pg": "^8.14.0", "pg-connection-string": "^2.7.0", - "serve-favicon": "^2.5.0" + "serve-favicon": "^2.5.0", + "winston": "^3.17.0" + } + }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "license": "MIT", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" } }, "node_modules/@ibm-cloud/secrets-manager": { @@ -61,6 +82,12 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "license": "MIT" + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -89,6 +116,12 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -210,6 +243,51 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "license": "MIT" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "license": "MIT", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -342,6 +420,12 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "license": "MIT" + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -469,6 +553,12 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "license": "MIT" + }, "node_modules/file-type": { "version": "16.5.4", "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", @@ -515,6 +605,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", + "license": "MIT" + }, "node_modules/follow-redirects": { "version": "1.15.9", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", @@ -722,6 +818,24 @@ "node": ">= 0.10" } }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -767,6 +881,12 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -802,6 +922,23 @@ "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "license": "MIT", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -899,6 +1036,15 @@ "node": ">= 0.8" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "license": "MIT", + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1213,6 +1359,15 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1385,6 +1540,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -1393,6 +1557,15 @@ "node": ">= 10.x" } }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1425,6 +1598,12 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "license": "MIT" + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -1463,6 +1642,15 @@ "node": ">=6" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -1505,6 +1693,12 @@ "requires-port": "^1.0.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1521,6 +1715,70 @@ "node": ">= 0.8" } }, + "node_modules/winston": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", + "license": "MIT", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "license": "MIT", + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/cos-to-sql/package.json b/cos-to-sql/package.json index 7b2692a2b..cf9686c73 100644 --- a/cos-to-sql/package.json +++ b/cos-to-sql/package.json @@ -5,7 +5,6 @@ "main": "app.mjs", "scripts": { "start": "node .", - "local": "node ./src/job.mjs", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [ @@ -23,6 +22,7 @@ "ibm-cloud-sdk-core": "^5.3.0", "pg": "^8.14.0", "pg-connection-string": "^2.7.0", - "serve-favicon": "^2.5.0" + "serve-favicon": "^2.5.0", + "winston": "^3.17.0" } } diff --git a/cos-to-sql/utils/logging.mjs b/cos-to-sql/utils/logging.mjs new file mode 100644 index 000000000..ce450dd46 --- /dev/null +++ b/cos-to-sql/utils/logging.mjs @@ -0,0 +1,17 @@ +import winston from "winston"; +const { combine, timestamp, json } = winston.format; + +export function getCodeEngineLogger(componentName) { + if (!winston.loggers.get(componentName)) { + const ceTransport = new winston.transports.Console({ + level: "debug", + format: combine(timestamp(), json()), + defaultMeta: { logger: componentName }, + }); + + winston.loggers.add(componentName, { + transports: [ceTransport], + }); + } + return winston.loggers.get(componentName); +} diff --git a/logging/.ceignore b/logging/.ceignore new file mode 100644 index 000000000..ef43a4df7 --- /dev/null +++ b/logging/.ceignore @@ -0,0 +1,3 @@ +node_modules/ +target/ +vendor/ \ No newline at end of file diff --git a/logging/README.md b/logging/README.md new file mode 100644 index 000000000..7c994b868 --- /dev/null +++ b/logging/README.md @@ -0,0 +1,16 @@ +# Code Engine logging examples + +The following sample provides recommended example to demonstrate **unstructured** and **structured** logging. + +Use the following command to build and deploy all examples to the Code Engine project of your choice. +``` +ibmcloud ce project select --name + +./run all +``` + +For structured logs, following libraries have been used: +* Node.js: [winston](https://www.npmjs.com/package/winston) +* Python: [Loguru](https://github.com/Delgan/loguru) +* Java: [SLF4J](https://www.slf4j.org/), [Logback](https://logback.qos.ch/) +* Golang: [built-in slog](https://go.dev/blog/slog) diff --git a/logging/go-structured/.ceignore b/logging/go-structured/.ceignore new file mode 100644 index 000000000..5657f6ea7 --- /dev/null +++ b/logging/go-structured/.ceignore @@ -0,0 +1 @@ +vendor \ No newline at end of file diff --git a/logging/go-structured/.dockerignore b/logging/go-structured/.dockerignore new file mode 100644 index 000000000..210eadbce --- /dev/null +++ b/logging/go-structured/.dockerignore @@ -0,0 +1,4 @@ +.dockerignore +build +Dockerfile +vendor \ No newline at end of file diff --git a/logging/go-structured/Dockerfile b/logging/go-structured/Dockerfile new file mode 100644 index 000000000..98dff608a --- /dev/null +++ b/logging/go-structured/Dockerfile @@ -0,0 +1,10 @@ +FROM quay.io/projectquay/golang:1.25 AS build-env +WORKDIR /go/src/app +COPY . . + +RUN CGO_ENABLED=0 go build -o /go/bin/app main.go + +# Copy the executable into a smaller base image +FROM gcr.io/distroless/static-debian12 +COPY --from=build-env /go/bin/app / +ENTRYPOINT ["/app"] diff --git a/logging/go-structured/go.mod b/logging/go-structured/go.mod new file mode 100644 index 000000000..4eaf5ac81 --- /dev/null +++ b/logging/go-structured/go.mod @@ -0,0 +1,3 @@ +module github.com/IBM/CodeEngine/logging/go + +go 1.25.0 diff --git a/logging/go-structured/go.sum b/logging/go-structured/go.sum new file mode 100644 index 000000000..e69de29bb diff --git a/logging/go-structured/main.go b/logging/go-structured/main.go new file mode 100644 index 000000000..3fa30de50 --- /dev/null +++ b/logging/go-structured/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "errors" + "log/slog" + "os" + "runtime/debug" +) + +func main() { + handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + // Remove time and rename msg->message + ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + // Drop the time attribute + if attr.Key == slog.TimeKey { + return slog.Attr{} // empty => removed + } + // Rename msg to message + if attr.Key == slog.MessageKey { + return slog.String("message", attr.Value.String()) + } + return attr + }, + }) + logger := slog.New(handler) + + // Expect to be rendered as INFO level log message + logger.Info("This is a structured log message") + + // Expect to be rendered as DEBUG level log message + logger.Debug("This is a structured log message") + + // Expect to be rendered as WARN level log message + logger.Warn("This is a structured log message") + + // Expect to be rendered as ERROR level log message + logger.Error("This is a structured log message") + + // Expect to be rendered as DEBUG level log message. The extra key is available as a searchable, filterable field + logger.Debug("A structured log entry that contains an extra key", + slog.String("extra_key", "extra_value"), + ) + + // Expect to be rendered as INFO level log message. The additional JSON struct is available as a searchable, filterable fields + logger.Info("A structured log entry that carries a ton of additional fields", + slog.String("requestId", "some-request-id"), + slog.String("userId", "user-123456"), + slog.String("action", "test"), + slog.Group("metadata", + slog.String("foo", "bar"), + ), + ) + + // Multi-line example. Expect to be rendered in a single log message + logger.Info(`Multi-line log sample: + Line 1: initialization started + Line 2: loading modules + Line 3: modules loaded + Line 4: entering main loop\nEnd of sample`) + + // Error logging. The error stack trace is rendered in a single log message (see field stack) + err := errors.New("boom!") + logger.Error("An error occurred", + slog.Any("err", err), + slog.String("stack", string(debug.Stack())), + ) +} diff --git a/logging/go-unstructured/.ceignore b/logging/go-unstructured/.ceignore new file mode 100644 index 000000000..5657f6ea7 --- /dev/null +++ b/logging/go-unstructured/.ceignore @@ -0,0 +1 @@ +vendor \ No newline at end of file diff --git a/logging/go-unstructured/.dockerignore b/logging/go-unstructured/.dockerignore new file mode 100644 index 000000000..210eadbce --- /dev/null +++ b/logging/go-unstructured/.dockerignore @@ -0,0 +1,4 @@ +.dockerignore +build +Dockerfile +vendor \ No newline at end of file diff --git a/logging/go-unstructured/Dockerfile b/logging/go-unstructured/Dockerfile new file mode 100644 index 000000000..98dff608a --- /dev/null +++ b/logging/go-unstructured/Dockerfile @@ -0,0 +1,10 @@ +FROM quay.io/projectquay/golang:1.25 AS build-env +WORKDIR /go/src/app +COPY . . + +RUN CGO_ENABLED=0 go build -o /go/bin/app main.go + +# Copy the executable into a smaller base image +FROM gcr.io/distroless/static-debian12 +COPY --from=build-env /go/bin/app / +ENTRYPOINT ["/app"] diff --git a/logging/go-unstructured/go.mod b/logging/go-unstructured/go.mod new file mode 100644 index 000000000..4eaf5ac81 --- /dev/null +++ b/logging/go-unstructured/go.mod @@ -0,0 +1,3 @@ +module github.com/IBM/CodeEngine/logging/go + +go 1.25.0 diff --git a/logging/go-unstructured/go.sum b/logging/go-unstructured/go.sum new file mode 100644 index 000000000..e69de29bb diff --git a/logging/go-unstructured/main.go b/logging/go-unstructured/main.go new file mode 100644 index 000000000..44ee7a4b1 --- /dev/null +++ b/logging/go-unstructured/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "runtime/debug" + "time" +) + +// This simple Code Engine job demonstrates how to write unstructured log lines +// using the built-in capabilities of Golang + +func main() { + + // expect to be rendered as INFO level log message + fmt.Println("This is a unstructured log message without a severity identifier") + + // expect to be rendered as WARN level log message + fmt.Println("This is a unstructured log message with a severity identifier WARN") + + // expect to be rendered as ERROR level log message, without the keyword ERROR being part of the message + fmt.Println("ERROR This is a unstructured log message prefixed with the level") + + // expect to be rendered as INFO level log message, without the timestamp being part of the message + fmt.Println(time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + " This is a unstructured log message prefixed with the timestamp") + + // expect to be rendered as DEBUG level log message, without the timestamp and keyword DEBUG being part of the message + fmt.Println(time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + " DEBUG This is a unstructured log message prefixed with the timestamp and level") + + // Multi-line example. Expect to be rendered in a single log message + fmt.Println("Multi-line log sample...\\nStep 1: Validating input...\\nStep 2: Processing payment...") + + // Error logging. Expect that the stacktrace is rendered in multiple log statements + // Note: Use structure logs to support multi-line error stack traces + func() { + defer func() { + if r := recover(); r != nil { + fmt.Println("Stacktrace example", r) + fmt.Print(string(debug.Stack())) + } + }() + panic("boom!") + }() +} diff --git a/logging/java-structured/.ceignore b/logging/java-structured/.ceignore new file mode 100644 index 000000000..9f970225a --- /dev/null +++ b/logging/java-structured/.ceignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/logging/java-structured/.dockerignore b/logging/java-structured/.dockerignore new file mode 100644 index 000000000..ba4a1025a --- /dev/null +++ b/logging/java-structured/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +.gitignore +build +Dockerfile +target diff --git a/logging/java-structured/.gitignore b/logging/java-structured/.gitignore new file mode 100644 index 000000000..1de565933 --- /dev/null +++ b/logging/java-structured/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/logging/java-structured/Dockerfile b/logging/java-structured/Dockerfile new file mode 100644 index 000000000..fb4348e55 --- /dev/null +++ b/logging/java-structured/Dockerfile @@ -0,0 +1,16 @@ +# Download dependencies and compile in builder stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder + +COPY --chown=${UID} . /src +WORKDIR /src +RUN mvn package -Dmaven.test.skip=true + +# Build final stage +FROM gcr.io/distroless/java21 + +COPY --chown=1001:0 --from=builder /src/target/logging-1.0.0-SNAPSHOT.jar /app/logging-1.0.0-SNAPSHOT.jar + +USER 1001:0 +WORKDIR /app + +CMD ["logging-1.0.0-SNAPSHOT.jar"] diff --git a/logging/java-structured/pom.xml b/logging/java-structured/pom.xml new file mode 100644 index 000000000..d7e1f9bed --- /dev/null +++ b/logging/java-structured/pom.xml @@ -0,0 +1,86 @@ + + + 4.0.0 + + com.ibm.cloud.codeengine.sample + logging + 1.0.0-SNAPSHOT + + logging + https://github.com/IBM/CodeEngine + + + UTF-8 + 21 + + + + + org.slf4j + slf4j-api + 2.0.17 + + + ch.qos.logback + logback-classic + 1.5.32 + + + ch.qos.logback + logback-core + 1.5.32 + + + net.logstash.logback + logstash-logback-encoder + 9.0 + + + + + + + + + maven-clean-plugin + 3.4.0 + + + + maven-compiler-plugin + 3.13.0 + + + maven-jar-plugin + 3.4.2 + + + + + + + maven-shade-plugin + 3.6.0 + + false + + + com.ibm.cloud.codeengine.sample.App + + + + + + package + + shade + + + + + + + \ No newline at end of file diff --git a/logging/java-structured/src/main/java/com/ibm/cloud/codeengine/sample/App.java b/logging/java-structured/src/main/java/com/ibm/cloud/codeengine/sample/App.java new file mode 100644 index 000000000..280d21a0c --- /dev/null +++ b/logging/java-structured/src/main/java/com/ibm/cloud/codeengine/sample/App.java @@ -0,0 +1,56 @@ +package com.ibm.cloud.codeengine.sample; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class App { + + public static void main(String[] args) { + + Logger logger = LoggerFactory.getLogger("my_logger"); + // Expect to be rendered as INFO level log message + logger.info("This is a structured log message"); + + // Expect to be rendered as DEBUG level log message + logger.debug("This is a structured log message"); + + // Expect to be rendered as WARN level log message + logger.warn("This is a structured log message"); + + // Expect to be rendered as ERROR level log message + logger.error("This is a structured log message"); + + logger.atDebug().addKeyValue("extra_key", "extra_value") + .log("A structured log entry that contains an extra key"); + + // Expect to be rendered as INFO level log message. The additional JSON struct + // is available as a searchable, filterable fields + logger.atInfo() + .addKeyValue("requestId", "some-request-id") + .addKeyValue("userId", "user-123456") + .addKeyValue("action", "test") + .addKeyValue("metadata", Map.of("foo", "bar")) + .log("A structured log entry that carries a ton of additional fields"); + + // Multi-line example. Expect to be rendered in a single log message + logger.atInfo().log(""" + Multi-line log sample: + Line 1: initialization started + Line 2: loading modules + Line 3: modules loaded + Line 4: entering main loop + End of sample"""); + + // Error logging. + // The error stack trace is rendered in a single log message (see field stack_trace) + try { + throw new RuntimeException("boom!"); + } catch (Exception e) { + logger.atError() + .setCause(e) // also sets throwable on the event + .log("An error occurred"); + } + } +} diff --git a/logging/java-structured/src/main/resources/logback.xml b/logging/java-structured/src/main/resources/logback.xml new file mode 100644 index 000000000..7bc191421 --- /dev/null +++ b/logging/java-structured/src/main/resources/logback.xml @@ -0,0 +1,17 @@ + + + + UTC + + [ignore] + [ignore] + [ignore] + [ignore] + + + + + + + + \ No newline at end of file diff --git a/logging/java-unstructured/.ceignore b/logging/java-unstructured/.ceignore new file mode 100644 index 000000000..9f970225a --- /dev/null +++ b/logging/java-unstructured/.ceignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/logging/java-unstructured/.dockerignore b/logging/java-unstructured/.dockerignore new file mode 100644 index 000000000..ba4a1025a --- /dev/null +++ b/logging/java-unstructured/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +.gitignore +build +Dockerfile +target diff --git a/logging/java-unstructured/.gitignore b/logging/java-unstructured/.gitignore new file mode 100644 index 000000000..1de565933 --- /dev/null +++ b/logging/java-unstructured/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/logging/java-unstructured/Dockerfile b/logging/java-unstructured/Dockerfile new file mode 100644 index 000000000..fb4348e55 --- /dev/null +++ b/logging/java-unstructured/Dockerfile @@ -0,0 +1,16 @@ +# Download dependencies and compile in builder stage +FROM registry.access.redhat.com/ubi9/openjdk-21 AS builder + +COPY --chown=${UID} . /src +WORKDIR /src +RUN mvn package -Dmaven.test.skip=true + +# Build final stage +FROM gcr.io/distroless/java21 + +COPY --chown=1001:0 --from=builder /src/target/logging-1.0.0-SNAPSHOT.jar /app/logging-1.0.0-SNAPSHOT.jar + +USER 1001:0 +WORKDIR /app + +CMD ["logging-1.0.0-SNAPSHOT.jar"] diff --git a/logging/java-unstructured/pom.xml b/logging/java-unstructured/pom.xml new file mode 100644 index 000000000..c5a65a733 --- /dev/null +++ b/logging/java-unstructured/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + com.ibm.cloud.codeengine.sample + logging + 1.0.0-SNAPSHOT + + logging + https://github.com/IBM/CodeEngine + + + UTF-8 + 21 + + + + + + + + + + + maven-clean-plugin + 3.4.0 + + + + maven-compiler-plugin + 3.13.0 + + + maven-jar-plugin + 3.4.2 + + + + + + + maven-shade-plugin + 3.6.0 + + false + + + com.ibm.cloud.codeengine.sample.App + + + + + + package + + shade + + + + + + + \ No newline at end of file diff --git a/logging/java-unstructured/src/main/java/com/ibm/cloud/codeengine/sample/App.java b/logging/java-unstructured/src/main/java/com/ibm/cloud/codeengine/sample/App.java new file mode 100644 index 000000000..0d168744a --- /dev/null +++ b/logging/java-unstructured/src/main/java/com/ibm/cloud/codeengine/sample/App.java @@ -0,0 +1,46 @@ +package com.ibm.cloud.codeengine.sample; + +import java.time.Instant; + +// This simple Code Engine job demonstrates how to write unstructured log lines +// using the built-in capabilities of Java + +public class App { + + public static void main(String[] args) { + + // expect to be rendered as INFO level log message + System.out.println("This is a unstructured log message without a severity identifier"); + + // expect to be rendered as WARN level log message + System.out.println("This is a unstructured log message with a severity identifier WARN"); + + // expect to be rendered as ERROR level log message, without the keyword ERROR + // being part of the message + System.out.println("ERROR This is a unstructured log message prefixed with the level"); + + // expect to be rendered as INFO level log message, without the timestamp being + // part of the message + System.out + .println(Instant.now().toString() + " This is a unstructured log message prefixed with the timestamp"); + + // expect to be rendered as DEBUG level log message, without the timestamp and + // keyword DEBUG being part of the message + System.out.println(Instant.now().toString() + + " DEBUG This is a unstructured log message prefixed with the timestamp and level"); + + // Multi-line example. Expect to be rendered in a single log message + System.out.println("Multi-line log sample...\\nStep 1: Validating input...\\nStep 2: Processing payment..."); + + // Error logging. + // Expect that the stacktrace is rendered in multiple log statements + // Note: Use structure logs to support multi-line error stack traces + try { + throw new RuntimeException("boom!"); + } catch (Exception e) { + System.err.println("Stacktrace example " + e); // error message + e.printStackTrace(); // full stack trace to stderr + } + + } +} diff --git a/logging/node-structured/.ceignore b/logging/node-structured/.ceignore new file mode 100644 index 000000000..b512c09d4 --- /dev/null +++ b/logging/node-structured/.ceignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/logging/node-structured/.dockerignore b/logging/node-structured/.dockerignore new file mode 100644 index 000000000..0474a63e8 --- /dev/null +++ b/logging/node-structured/.dockerignore @@ -0,0 +1,3 @@ +.dockerignore +Dockerfile +node_modules/ \ No newline at end of file diff --git a/logging/node-structured/Dockerfile b/logging/node-structured/Dockerfile new file mode 100644 index 000000000..f3f043303 --- /dev/null +++ b/logging/node-structured/Dockerfile @@ -0,0 +1,19 @@ +# Download dependencies in builder stage +FROM registry.access.redhat.com/ubi9/nodejs-24:latest AS builder + +COPY --chown=${CNB_USER_ID}:${CNB_GROUP_ID} package.json /app/ +WORKDIR /app +RUN npm i --omit=dev + + +# Use a small distroless image for as runtime image +FROM gcr.io/distroless/nodejs24 + +COPY --chown=1001:0 --from=builder /app/node_modules /app/node_modules +COPY --chown=1001:0 . /app/ + +USER 1001:0 +WORKDIR /app +EXPOSE 8080 + +CMD ["index.mjs"] \ No newline at end of file diff --git a/logging/node-structured/index.mjs b/logging/node-structured/index.mjs new file mode 100644 index 000000000..e26cf5c9c --- /dev/null +++ b/logging/node-structured/index.mjs @@ -0,0 +1,53 @@ +import winston from "winston"; +const { combine, json } = winston.format; + +// This simple Code Engine job demonstrates how to write structured log lines +// using the the external package winston. Other frameworks like bunyan, log4js, pino work similar + +const logger = winston.createLogger({ + level: "debug", + transports: [new winston.transports.Console()], + format: combine(json()), +}); + +// Expect to be rendered as INFO level log message +logger.info("This is a structured log message"); + +// Expect to be rendered as DEBUG level log message +logger.debug("This is a structured log message"); + +// Expect to be rendered as WARN level log message +logger.warn("This is a structured log message"); + +// Expect to be rendered as ERROR level log message +logger.error("This is a structured log message"); + +// Expect to be rendered as DEBUG level log message. The extra key is available as a searchable, filterable field +logger.debug("A structured log entry that contains an extra key", { + extra_key: "extra_value", +}); + +// Expect to be rendered as INFO level log message. The additional JSON struct is available as a searchable, filterable fields +logger.info("A structured log entry that carries a ton of additional fields", { + requestId: "some-request-id", + userId: "user-123456", + action: "test", + metadata: { + foo: "bar", + }, +}); + +// Multi-line example. Expect to be rendered in a single log message +logger.info(`Multi-line log sample: +Line 1: initialization started +Line 2: loading modules +Line 3: modules loaded +Line 4: entering main loop +End of sample`); + +// Error logging. The error stack trace is rendered in a single log message (see field stack) +try { + throw new Error("boom!"); +} catch (err) { + logger.error("An error occurred", err); +} diff --git a/logging/node-structured/package.json b/logging/node-structured/package.json new file mode 100644 index 000000000..0171ef832 --- /dev/null +++ b/logging/node-structured/package.json @@ -0,0 +1,16 @@ +{ + "name": "ce-logging", + "version": "1.0.0", + "description": "", + "main": "index.mjs", + "scripts": { + "start": "node .", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "license": "MIT", + "homepage": "https://cloud.ibm.com/containers/serverless", + "dependencies": { + "winston": "^3.19.0" + } + } \ No newline at end of file diff --git a/logging/node-unstructured/.ceignore b/logging/node-unstructured/.ceignore new file mode 100644 index 000000000..b512c09d4 --- /dev/null +++ b/logging/node-unstructured/.ceignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/logging/node-unstructured/.dockerignore b/logging/node-unstructured/.dockerignore new file mode 100644 index 000000000..0474a63e8 --- /dev/null +++ b/logging/node-unstructured/.dockerignore @@ -0,0 +1,3 @@ +.dockerignore +Dockerfile +node_modules/ \ No newline at end of file diff --git a/logging/node-unstructured/Dockerfile b/logging/node-unstructured/Dockerfile new file mode 100644 index 000000000..d1f9ae7f8 --- /dev/null +++ b/logging/node-unstructured/Dockerfile @@ -0,0 +1,18 @@ +# Download dependencies in builder stage +FROM registry.access.redhat.com/ubi9/nodejs-24:latest AS builder + +COPY --chown=${CNB_USER_ID}:${CNB_GROUP_ID} package.json /app/ +WORKDIR /app +RUN npm i --omit=dev + + +# Use a small distroless image for as runtime image +FROM gcr.io/distroless/nodejs24 + +COPY --chown=1001:0 . /app/ + +USER 1001:0 +WORKDIR /app +EXPOSE 8080 + +CMD ["index.mjs"] \ No newline at end of file diff --git a/logging/node-unstructured/index.mjs b/logging/node-unstructured/index.mjs new file mode 100644 index 000000000..5ff7a5572 --- /dev/null +++ b/logging/node-unstructured/index.mjs @@ -0,0 +1,31 @@ +// This simple Code Engine job demonstrates how to write unstructured log lines +// using the built-in capabilities of Node.js + +// Expect to be rendered as INFO level log message +console.log("This is a unstructured log message without a severity identifier"); + +// Expect to be rendered as WARN level log message +console.log("This is a unstructured log message with a severity identifier WARN"); + +// Expect to be rendered as INFO level log message +console.warn("This is a unstructured log message using a specific level"); + +// Expect to be rendered as ERROR level log message, without the keyword ERROR being part of the message +console.log("ERROR This is a unstructured log message prefixed with the level"); + +// Expect to be rendered as INFO level log message, without the timestamp being part of the message +console.log(`${new Date().toISOString()} This is a unstructured log message prefixed with the timestamp`); + +// Expect to be rendered as DEBUG level log message, without the timestamp and keyword DEBUG being part of the message +console.log(`${new Date().toISOString()} DEBUG This is a unstructured log message prefixed with the timestamp and level`); + +// Multi-line example. Expect to be rendered in a single log message +console.log("Multi-line log sample...\\nStep 1: Validating input...\\nStep 2: Processing payment..."); + +// Error logging. Expect that the stacktrace is rendered in multiple log statements +// Note: Use structure logs to support multi-line error stack traces +try { + throw new Error("boom!"); +} catch (err) { + console.error("Stacktrace example", err); +} diff --git a/logging/node-unstructured/package.json b/logging/node-unstructured/package.json new file mode 100644 index 000000000..3b4f247d4 --- /dev/null +++ b/logging/node-unstructured/package.json @@ -0,0 +1,15 @@ +{ + "name": "ce-logging", + "version": "1.0.0", + "description": "", + "main": "index.mjs", + "scripts": { + "start": "node .", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "license": "MIT", + "homepage": "https://cloud.ibm.com/containers/serverless", + "dependencies": { + } + } \ No newline at end of file diff --git a/logging/python-structured/.dockerignore b/logging/python-structured/.dockerignore new file mode 100644 index 000000000..9a715f53b --- /dev/null +++ b/logging/python-structured/.dockerignore @@ -0,0 +1,2 @@ +.dockerignore +Dockerfile \ No newline at end of file diff --git a/logging/python-structured/Dockerfile b/logging/python-structured/Dockerfile new file mode 100644 index 000000000..de687ace9 --- /dev/null +++ b/logging/python-structured/Dockerfile @@ -0,0 +1,18 @@ +# Download dependencies in builder stage +FROM registry.access.redhat.com/ubi9/python-312 AS builder + +COPY requirements.txt . +RUN python -m pip install -r requirements.txt + +# Build final stage +FROM gcr.io/distroless/python3 + +ENV PYTHONPATH=/app/site-packages + +COPY --chown=1001:0 --from=builder /opt/app-root/lib/python3.12/site-packages ${PYTHONPATH} +COPY --chown=1001:0 main.py /app/ + +USER 1001:0 +WORKDIR /app + +CMD ["main.py"] diff --git a/logging/python-structured/main.py b/logging/python-structured/main.py new file mode 100644 index 000000000..894d13245 --- /dev/null +++ b/logging/python-structured/main.py @@ -0,0 +1,87 @@ + +from loguru import logger +import sys +import json +import traceback + +# This simple Code Engine job demonstrates how to write structured log lines +# using the logging library loguru (https://github.com/Delgan/loguru) + +# Define a custom JSON sink +def json_sink(message): + record = message.record + + # Base fields: level + message, no timestamp + payload = { + "level": record["level"].name, # e.g., "INFO" + "message": record["message"], # rendered message + } + + # Merge in any bound extra fields as top-level keys + # (skip reserved keys to avoid accidental overwrite) + for k, v in record["extra"].items(): + if k not in ("level", "message", "stack"): + payload[k] = v + + # If an exception is attached, render full stack trace into "stack" + exc = record["exception"] + if exc: + # exc.type, exc.value, exc.traceback are available from Loguru + stack_text = "".join(traceback.format_exception(exc.type, exc.value, exc.traceback)) + payload["stack"] = stack_text + + # Emit a single JSON line + sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n") + sys.stdout.flush() + + +# Remove default handler (which includes timestamp, etc.) and add our custom sink +logger.remove() +logger.add(json_sink, level="DEBUG") # lowest level you want to capture + + +if __name__ == '__main__': + + # expect to be rendered as INFO level log message + logger.info("This is a structured log message"); + + # expect to be rendered as DEBUG level log message + logger.debug("This is a structured log message"); + + # expect to be rendered as WARN level log message + logger.warning("This is a structured log message"); + + # expect to be rendered as ERROR level log message + logger.error("This is a structured log message"); + + # Expect to be rendered as DEBUG level log message. The extra key is available as a searchable, filterable field + logger.bind(extra_key="extra_value").debug("A structured log entry that contains an extra key") + + # Expect to be rendered as INFO level log message. The additional JSON struct is available as a searchable, filterable fields + logger.bind( + requestId="some-request-id", + userId="user-123456", + action="test", + metadata={"foo": "bar"}, + ).info("A structured log entry that carries a ton of additional fields") + + + # Multi-line example. Expect to be rendered in a single log message + logger.info( + "Multi-line log sample:\n" + "Line 1: initialization started\n" + "Line 2: loading modules\n" + "Line 3: modules loaded\n" + "Line 4: entering main loop\n" + "End of sample" + ) + + # Error logging. The error stack trace is rendered in a single log message (see field stack) + try: + raise RuntimeError("boom!") + except Exception: + # logger.exception() automatically attaches the current exception info + logger.exception("An error occurred") + + + diff --git a/logging/python-structured/requirements.txt b/logging/python-structured/requirements.txt new file mode 100644 index 000000000..91405a63b --- /dev/null +++ b/logging/python-structured/requirements.txt @@ -0,0 +1 @@ +loguru~=0.7.3 \ No newline at end of file diff --git a/logging/python-unstructured/.dockerignore b/logging/python-unstructured/.dockerignore new file mode 100644 index 000000000..9a715f53b --- /dev/null +++ b/logging/python-unstructured/.dockerignore @@ -0,0 +1,2 @@ +.dockerignore +Dockerfile \ No newline at end of file diff --git a/logging/python-unstructured/Dockerfile b/logging/python-unstructured/Dockerfile new file mode 100644 index 000000000..de687ace9 --- /dev/null +++ b/logging/python-unstructured/Dockerfile @@ -0,0 +1,18 @@ +# Download dependencies in builder stage +FROM registry.access.redhat.com/ubi9/python-312 AS builder + +COPY requirements.txt . +RUN python -m pip install -r requirements.txt + +# Build final stage +FROM gcr.io/distroless/python3 + +ENV PYTHONPATH=/app/site-packages + +COPY --chown=1001:0 --from=builder /opt/app-root/lib/python3.12/site-packages ${PYTHONPATH} +COPY --chown=1001:0 main.py /app/ + +USER 1001:0 +WORKDIR /app + +CMD ["main.py"] diff --git a/logging/python-unstructured/main.py b/logging/python-unstructured/main.py new file mode 100644 index 000000000..b96de0fb3 --- /dev/null +++ b/logging/python-unstructured/main.py @@ -0,0 +1,31 @@ +# This simple Code Engine job demonstrates how to write unstructured log lines +# using the built-in capabilities of Python + +# Expect to be rendered as INFO level log message +print("This is a unstructured log message without a severity identifier"); + +# Expect to be rendered as WARN level log message +print("This is a unstructured log message with a severity identifier WARN"); + +# Expect to be rendered as ERROR level log message, without the keyword ERROR being part of the message +print("ERROR This is a unstructured log message prefixed with the level"); + +# Expect to be rendered as INFO level log message, without the timestamp being part of the message +from datetime import datetime, timezone +ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" +print(f"{ts} This is an unstructured log message prefixed with the timestamp") + +# Expect to be rendered as DEBUG level log message, without the timestamp and keyword DEBUG being part of the message +print(f"{datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")} DEBUG This is a unstructured log message prefixed with the timestamp and level") + +# Multi-line example. Expect to be rendered in a single log message +print("Multi-line log sample...\\nStep 1: Validating input...\\nStep 2: Processing payment...") + +# Error logging. Expect that the stacktrace is rendered in multiple log statements +# Note: Use structure logs to support multi-line error stack traces +try: + raise Exception("boom!") +except Exception as err: + print("Stacktrace example", err) + import traceback + print(traceback.format_exc(), end="") diff --git a/logging/python-unstructured/requirements.txt b/logging/python-unstructured/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/logging/run b/logging/run new file mode 100755 index 000000000..12d3d6468 --- /dev/null +++ b/logging/run @@ -0,0 +1,46 @@ +#!/bin/bash +set -eo pipefail + +PREFIX="${PREFIX:=logging-sample}" +LANGUAGE="${1:=all}" + +languages=( + node-unstructured + node-structured + python-unstructured + python-structured + go-unstructured + go-structured + java-unstructured + java-structured +) + +for i in "${languages[@]}"; do + + if [[ "$LANGUAGE" == "all" || "$LANGUAGE" == "$i" ]];then + echo "Deploying Code Engine job for $i ..." + job_name="$i-${PREFIX}" + + create_or_update=update + if ! ibmcloud ce job get --name $job_name >/dev/null 2>&1; then + echo -e "\nCreating the job '$job_name' ..." + create_or_update=create + else + echo -e "\nUpdating the job '$job_name' ..." + fi + + ibmcloud ce job $create_or_update --n "$job_name" --src . --context-dir "$i/" --memory 0.5G --cpu 0.25 --wait --retrylimit 0 + + echo "Submit a job run for $i ..." + jobrun_name="$job_name-$(openssl rand -hex 6)" + ibmcloud ce jobrun submit --job "$job_name" --name "$jobrun_name" --wait + + echo "Printing the logs for '$jobrun_name' ..." + ibmcloud ce jobrun logs --jobrun "$jobrun_name" + + else + continue; + fi +done + +echo "Done"