fix start.sh

Reviewed-on: #1

.gitignore

@@ -3,3 +3,5 @@ private/
 */private/
 package-lock.json
 .env
+ytmusic_cache
+.vscode

README.md

@@ -0,0 +1,2 @@
+# oddbyte.dev
+This is the source code repository for my website, https://oddbyte.dev.

index.js

@@ -29,12 +29,12 @@ const playlistId = 'PLnlTMS4cxfx3-et_L8APzpEgy_eCf18-U';
 /**
  * Port that nginx proxies to public
  */
-const port = process.env.PORT || 48915;
+const port = process.env.PORT || 48916;
 
 /**
  * Are we prod or dev
  */
-const dev = process.env.DEV || false;
+const dev = process.env.DEV || "true";
 
 // Helper functions I've made to do things and stuff :P
 
@@ -60,36 +60,6 @@ const loopHourly = (something, skipFirst = false) => {
 	}, 3600000 - new Date().getTime() % 3600000);
 }
 
-function sendSource(req, res, filePath) {
-	try {
-		// This will throw an error, which I catch, if the file isnt found
-		// Use lstatSync to have a sync read while also not going through symlinks (to prevent arbritrary reads via symlinks)
-		// Further restricted by my use of the --permission commandline flag
-		// I also added a manual check to block any file reads outside the base dir (/prod/portfolio as of the time of writing this)
-		const stat = fs.lstatSync(filePath);
-
-		if (!stat.isFile()) {
-			// If it's not a file, error
-			res.status(403).send('<pre>Not a file</pre>');
-		} else {
-			// If it's a file, send the content
-			const fileContent = fs.readFileSync(filePath, 'utf8');
-			res.render('sourceviewer', { faviconb64, filePath, fileContent });
-		}
-	} catch (err) {
-		// Legit why are you doing this
-		// If it's whitelisted, it's listed under the index page
-		// If it's not whitelisted, you cant access it anyway :P
-		res.status(404).send('File not found');
-	}
-}
-
-// --------------------------------------------------------------------------------------
-// -                 Below is black magic. I have no idea how it works.                 -
-// -    I have either stolen it from somewhere or completely forgotten how it works.    -
-// - If you know how it works please explain it to me PLEASE I BEG OF YOU HALP :AAAAAA: -
-// --------------------------------------------------------------------------------------
-
 /**
  * Pulls all the pictures for the songs and stores them in a base64 key:value array where the key is the song id and the value is the thumbnail
  * @param {*} song 
@@ -118,6 +88,8 @@ async function updateSongs() {
 	if (typeof playlistSongs == 'object') {
 		// For each song that youtube moosic has yelled back at us, give them individually to populateThumbnails()
 		await Promise.all(playlistSongs.map(async song => await populateThumbnails(song)));
+		// Write all of this to cache
+		fs.writeFileSync(path.join(baseDir + '/ytmusic_cache'), JSON.stringify({playlistSongs, thumbnails}));
 	}
 	else {
 		// like legit this shouldnt even be possible
@@ -128,246 +100,6 @@ async function updateSongs() {
 	}
 }
 
-/**
- * @param {Array} pathPattern This thingy is the indexPaths below this function 
- * @returns {Array} an array called `results`
- */
-function getDirectoryContents(pathPattern) {
-	const results = [];
-
-	// Handle wildcards (including multiple wildcards like blog/*/*)
-	if (pathPattern.includes('*')) {
-		const segments = pathPattern.split('/').filter(s => s);
-
-		/**
-		 * Recursively resolves the pattern segments against the file system.
-		 * @param {Array} remainingSegments - The parts of the path left to match (e.g., ['*']).
-		 * @param {String} currentAbsPath - The absolute path on disk we are currently checking.
-		 * @returns {Array} - Array of matched file/directory objects.
-		 */
-		function traverse(remainingSegments, currentAbsPath) {
-			// Base case: We have matched all segments. Return the node at this location.
-			if (remainingSegments.length === 0) {
-				return [createNode(currentAbsPath)];
-			}
-
-			const segment = remainingSegments[0];
-			const rest = remainingSegments.slice(1);
-
-			if (segment === '*') {
-				// Wildcard: Read current directory and recurse for all children
-				try {
-					const files = fs.readdirSync(currentAbsPath, { withFileTypes: true });
-					const matches = [];
-
-					files.forEach(file => {
-						const fullPath = path.join(currentAbsPath, file.name);
-						// Recurse with the remaining segments
-						const childMatches = traverse(rest, fullPath);
-						matches.push(...childMatches);
-					});
-
-					return matches;
-				} catch (err) {
-					console.error(`Error reading directory ${currentAbsPath}:`, err);
-					return [];
-				}
-			} else {
-				// Exact segment: Join path and recurse
-				const fullPath = path.join(currentAbsPath, segment);
-				return traverse(rest, fullPath);
-			}
-		}
-
-		/**
-		 * Creates the result object for a specific path.
-		 * If it's a directory, it recursively fetches children.
-		 */
-		function createNode(fullPath) {
-			try {
-				const stat = fs.statSync(fullPath);
-				const relativePath = path.relative(baseDir, fullPath);
-
-				if (stat.isDirectory()) {
-					// Recursively get children using the main function
-					// We append '/*' to get the immediate contents of this directory
-					const children = getDirectoryContents(`${relativePath}/*`);
-					return {
-						name: path.basename(fullPath),
-						path: relativePath,
-						type: 'directory',
-						children: children
-					};
-				} else {
-					return {
-						name: path.basename(fullPath),
-						path: relativePath,
-						type: 'file'
-					};
-				}
-			} catch (err) {
-				console.error(`Error reading path ${fullPath}:`, err);
-				return null;
-			}
-		}
-
-		// Start traversal from the base directory
-		const nodes = traverse(segments, baseDir);
-		// Filter out any null results from errors
-		nodes.forEach(node => {
-			if (node) results.push(node);
-		});
-
-	} else {
-		// Handle exact paths
-
-		// Thanks past me, for the extremely helpful comment above
-		// I now understand exactly what this does /s
-
-		const fullPath = path.join(baseDir, pathPattern);
-		try {
-			const stat = fs.statSync(fullPath);
-			const relativePath = path.relative(baseDir, fullPath);
-
-			if (stat.isDirectory()) {
-				const files = fs.readdirSync(fullPath, { withFileTypes: true });
-				files.forEach(file => {
-					const childPath = path.join(fullPath, file.name);
-					const childRelativePath = path.relative(baseDir, childPath);
-
-					if (file.isDirectory()) {
-						const children = getDirectoryContents(`${childRelativePath}/*`);
-						results.push({
-							name: file.name,
-							path: childRelativePath,
-							type: 'directory',
-							children: children
-						});
-					} else {
-						results.push({
-							name: file.name,
-							path: childRelativePath,
-							type: 'file'
-						});
-					}
-				});
-			} else {
-				results.push({
-					name: path.basename(fullPath),
-					path: relativePath,
-					type: 'file'
-				});
-			}
-		} catch (err) {
-			console.error(`Error reading path ${fullPath}:`, err);
-		}
-	}
-
-	return results;
-}
-
-/**
- * An array of the whitelisted directories and their contents
- */
-let whitelistedFiles = [];
-
-/**
- * Whitelisted paths for the source code (to prevent arbritrary reads and stuffs),
- * Also referred to as `pathPattern` in whatever the hell `getDirectoryContents()` is
- */
-const indexPaths = [
-	'blog/*',
-	'blog/*/*',
-	'public/*',
-	'public/piskelfile/*',
-	'views/*',
-	'views/partials/*',
-	'index.js'
-];
-
-/**
- * directory tree object (built in the func buildDirectoryTree below)
- */
-let directoryTree;
-
-/**
- * sorted result
- */
-let sortedResult;
-
-// Build directory tree
-const buildDirectoryTree = (items) => {
-	const tree = {};
-	items.forEach(item => {
-		const parts = item.path.split(path.sep).filter(part => part !== '');
-		let currentLevel = tree;
-		for (let i = 0; i < parts.length; i++) {
-			const part = parts[i];
-			if (!currentLevel[part]) {
-				currentLevel[part] = {
-					name: part,
-					path: parts.slice(0, i + 1).join(path.sep),
-					type: i === parts.length - 1 ? item.type : 'directory',
-					children: {}
-				};
-			}
-			currentLevel = currentLevel[part].children;
-		}
-		if (item.type === 'file') {
-			currentLevel[parts[parts.length - 1]] = {
-				...currentLevel[parts[parts.length - 1]],
-				type: 'file'
-			};
-		}
-	});
-	return tree;
-};
-
-// Convert the tree to an array for easier rendering
-const treeToArray = (node) => {
-	const result = [];
-	Object.values(node).forEach(item => {
-		if (item.type === 'directory') {
-			const directoryItem = {
-				name: item.name,
-				path: item.path,
-				type: 'directory',
-				children: treeToArray(item.children)
-			};
-			result.push(directoryItem);
-		} else {
-			result.push({
-				name: item.name,
-				path: item.path,
-				type: 'file'
-			});
-		}
-	});
-	return result.sort((a, b) => {
-		if (a.type === 'directory' && b.type === 'file') return -1;
-		if (a.type === 'file' && b.type === 'directory') return 1;
-		return a.name.localeCompare(b.name);
-	});
-};
-
-async function updateFileIndex() {
-	// Clear whitelisted files to prevent memory leak
-	whitelistedFiles = [];
-	
-	indexPaths.forEach(path => {
-		let contents = getDirectoryContents(path);
-		whitelistedFiles.push(...contents);
-	});
-	
-	directoryTree = buildDirectoryTree(whitelistedFiles);
-	sortedResult = treeToArray(directoryTree);
-}
-
-// ----------------------------------------------------------
-// -                    End black magic                     -
-// - I am pretty sure I know what is going on below here :p -
-// ----------------------------------------------------------
-
 var blog; // Setup the blog variable
 
 // Blog stuffs
@@ -394,6 +126,14 @@ async function send404(req, res) {
 
 // Begin the server-ing things
 const app = express();
+
+app.use((req, res, next) => {
+	if (dev == "true") {
+		console.log(`got a req to ${req.path}?${req.query} with a body of ${req.body}`);
+	}
+	next();
+});
+
 var server; // Set at app.listen (bottom), used to kill server on ctrl-c
 
 // YT Music stuff
@@ -445,45 +185,6 @@ app.get('/art', (req, res) => {
 	res.render('art', { faviconb64 });
 });
 
-// Source code page
-app.get('/source', (req, res) => {
-	res.render('source', { faviconb64, paths: sortedResult });
-});
-
-// Route to serve indexed files
-app.get(/^\/source\/.*/, (req, res) => {
-	const requestedPath = req.path.split('/source/')[1];
-
-	// Check if the requested file is whitelisted
-	const isIndexed = indexPaths.some(pattern => {
-		if (pattern.toString().includes("*")) { 
-			return whitelistedFiles.some(pattern => { return requestedPath.startsWith(pattern.path) && !(requestedPath.split(pattern.path).some(pattern => {return pattern.includes("/");}));});
-		}
-		return requestedPath === pattern;
-	});
-
-	if (!isIndexed) {
-		return res.status(403).send('<pre>File not whitelisted</pre>');
-	}
-
-	const filePath = path.join(baseDir, requestedPath);
-
-	// Completely prevent ../ attacks (hopefully)
-	// Well I mean it wont prevent bind mounts but if you can bind mount I'm fucked anyways cause you're root
-	// Symlinks might (?) bypass this??
-	// Hardlinks defo bypass this afaik.
-	// because i pass the --permission flag and only pass --allow-fs-read=/prod/portfolio/*
-	// you can't write via NJS at all
-	// nor read outside /prod/portfolio
-	// you'd need to break outta nodejs and start a native process
-	// and my filesystem permissions wont letchu write stuff anyway
-	if (!filePath.startsWith(baseDir)) {
-		return res.status(403).send('<pre>File not whitelisted</pre>');
-	}
-
-	sendSource(req, res, filePath);
-});
-
 // Moosic page
 app.get('/music', (req, res) => {
 	res.render('moosic', { faviconb64, playlistId, playlistSongs, thumbnails });
@@ -551,7 +252,6 @@ function printHelp() {
 	rl.write("\n\n-------------------------------HELP-----------------------------------------\n");
 	rl.write("> help [?] - prints this message\n");
 	rl.write("> updateMusic [um] - runs updateSongs()\n");
-	rl.write("> updateFileIndex [ufi] - runs updateFileIndex()\n");
 	rl.write("> eval [exec] - runs raw JS code, returns output (if any)\n");
 	rl.write("> quit [exit] - stops the server, and quits\n");
 	rl.write("-------------------------------HELP-----------------------------------------\n\n")
@@ -571,6 +271,7 @@ async function customConsole() {
 		const cmd_raw_args = cmd_raw.split(" ");
 		const cmd_args = cmd_raw_args.slice(1);
 		const cmd = cmd_raw_args[0];
+		const cmd_body = cmd_raw.substring(cmd.length).trim();
 
 		switch(cmd) {
 			case "?":
@@ -581,17 +282,13 @@ async function customConsole() {
 			case "updateMusic":
 				updateSongs();
 				break;
-			case "ufi":
-			case "updateFileIndex":
-				updateFileIndex();
-				break;
 			case "exec":
 			case "eval":
 				// TODO: Fix this not actually using the correct context for whatever reason it just evals in an empty context
 				// ...... i dont know what or how or why but magically it is working now so i am just not gonna touch it and hope
 				// whatever black magic made it work keeps working .-.
 				try {
-					rl.write(eval(cmd_args.toString()) + "\n");
+					rl.write(eval(cmd_body) + "\n");
 				} catch (err) {
 					console.error(err);
 				}
@@ -610,38 +307,52 @@ async function customConsole() {
 	}
 }
 
-// Wrap in a async function to wait for youtube music response before starting http server
-// (to prevent a race condition where people can view the moosic page be4 it is ready)
 async function main() {
 	
-	// Init the moosics stuff (black magic)
+	if (dev == "true") console.log("Starting custom console");
+	// Start console
+	customConsole(); // DO NOT CALL `await` ON THIS!! THIS WILL CAUSE IT TO NEVER RETURN!!
+	if (dev == "true") console.log("Console started");
+
+	if (dev == "true") console.log("Loading ytmusic stuffs from cache file");
+	// Load from cache
+	await (async () => {
+		let cachefile;
+		cachefile = fs.readFileSync(path.join(baseDir, '/ytmusic_cache'));
+		let parsed_cachefile = JSON.parse(cachefile);
+		playlistSongs = parsed_cachefile.playlistSongs;
+		thumbnails = parsed_cachefile.thumbnails;
+		if (dev == "true") console.log("Finished loading from ytmusic stuffs from cache");
+	})();
+	
+	// Wrap in async but don't await as to not delay server boot, we can load from cache faster then we can load from yt music
+	(async () => {
+		if (dev == "true") console.log("ytmusic init begining");
+		// Init the moosics stuff
 		await ytmusic.initialize();
+		if (dev == "true") console.log("ytmusic initialized");
 
 		// Populate playlistSongs and thumbnails
 		await updateSongs();
+		if (dev == "true") console.log("Loaded new data from YTMusic!\n");
+	})();
 	
+	if (dev == "true") console.log("populating blog index")
 	// Populate the blog index
 	await generateBlogIndex();
-
+	if (dev == "true") console.log("populated blog index");
-	// Populate file index
-	await updateFileIndex();
 
 	server = http.createServer(app);
 
+	if (dev == "true") console.log("Starting server");
 	await (async () => {
 		server.listen(port, () => {
-			console.log(`Listening to ${port}`);
+			console.log(`Listening to ${port}\n`);
 
 			// Start hourly loop to update playlist
 			loopHourly(async () => await updateSongs());
-
-			// Start hourly loop to update file index
-			loopHourly(async () => await updateFileIndex());
 		});
 	})();
-	
-	// Start console
-	customConsole();
 }
 
 // Handle a few signals

start.sh

@@ -8,5 +8,5 @@ sudo -iu prod fuser $(sudo -iu prod which node) -k
 echo '==== Restarting Nginx ===='
 rc-service nginx restart
 echo '===== Starting server ===='
-screen -dmS portfolio bash -c "sudo -iu prod bash -c 'cd /prod/portfolio && node --permission --allow-fs-read=/prod/portfolio/* .'"
+screen -dmS portfolio bash -c "sudo -iu prod bash -c 'cd /prod/portfolio && node --permission --allow-fs-read=/prod/portfolio/* --allow-fs-write=/prod/portfolio/ytmusic_cache .'"
 echo 'boosh'

start_dev.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+echo '==== Cleaning up dead screens ===='
+screen -wipe portfolio_devel
+echo '==== Restarting Nginx ===='
+rc-service nginx restart
+echo '===== Starting server ===='
+sudo -iu prod bash -c 'cd /development/website && node --permission --allow-fs-read=/development/website/* --allow-fs-write=/development/website/ytmusic_cache .'

views/home.ejs

@@ -33,7 +33,8 @@
                         Hihi :D<br />
                         I am a nerd that happens to like hacking stuff.<br />
                         I mainly nerd about privacy, android, web development, cybersecurity, and sysadmin.<br />
-                        My goal is to become a CISO.
+                        My goal is to become a CISO.<br />
+                        I host my own git server at <a href="/git">/git</a> to gain sysadmin experience and to move off of microslop's github.
                     </p>
                     <div id="v-break"></div>
                     <h2 id="stats" class="selectDisable">Stats</h2>
@@ -80,22 +81,20 @@
                     <p style="text-align: left; max-width: 500px; align-self:center; display:inline-block;">You can contact me by messaging me on:<br />
                         - Signal (preferred), <a rel="me" target="_blank" href="https://signal.me/#eu/s6M3E2BSkJVb9E0fwib4PTcbDLH36jPlQTP9basoZ37rl-XoyD5L4Yvao-jZQ0xn">@oddbyte.01</a>,<br />
 			            - Mastodon, <a rel="me" target="_blank" href="https://mastodon.social/@oddbyte">@oddbyte@mastodon.social</a>,<br />
-                        - Github, <a rel="me" target="_blank" href="https://github.com/oddbyte">@oddbyte</a>, or my<br />
 			            - Email, <a rel="me" target="_blank" href="mailto:contact@oddbyte.dev">contact@oddbyte.dev</a><br />
                     </p>
                     <div id="v-break"></div>
-                    <h2 id="hoomans" class="selectDisable">Other hooman</h2>
+                    <h2 id="hoomans" class="selectDisable">Other hoomans</h2>
                     <p>
                         Check out this random internet person I happen to be aware of: <a target="_blank" href="https://voxel.top">voxel.top</a><br />
                         He is also a cybersecurity / privacy nerd<br />
                         His website <s>sucks ass</s> <i>needs improvement</i><br />
                         <sup>(yeah, lets go with "needs improvement")</sup>
                     </p>
-<%/*                    <p>
+                    <p>
                         Check out my buddy's site, <a target="_blank" href="https://catocat.uk">catocat.uk</a><br />
                         She <i>may</i> have <s>stolen things</s> <i>taken inspiration</i> from my website.
                     </p>
-*/%>
                 </main>
             </div>
         </div>

views/partials/funfact.ejs

@@ -3,12 +3,12 @@
     'Android was originally an operating system for cameras',
     '"password" is still one of the most commonly used passwords',
     'People call me funny sometimes I guess',
-    'Haiii <3',
     'e^(i*pi) = -1',
     'According to the leading fan theory, 1 + 1 might actually not be 11',
     'xkcd.com is a thing, and it is funny most of the time',
     'Apparently I am a hooman',
     'This message changes every time you refresh the page',
     'I wish my friends would talk to me more often',
-    'I am a functional introvert'
+    'I am a functional introvert',
+    'Haiii <3'
   ]; %>Fun Fact: <%= facts[Math.floor(Math.random() * facts.length)] %></p>

views/partials/navbar.ejs

@@ -3,6 +3,6 @@
             <a id="button" href="/">Home</a>
 	        <a id="button" href="/art">Art</a>
             <a id="button" href="/music">Playlist</a>
-            <a id="button" href="/source">Source code</a>
+            <a id="button" href="/git/oddbyte/website">Source</a>
         </div>
     </div>

views/source.ejs

@@ -1,79 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Directory Tree</title>
-    <meta name="fediverse:creator" content="@oddbyte@mastodon.social">
-    <style>
-        <%- include('partials/style') %>
-        .directory {
-            font-weight: bold;
-            color: #0066cc;
-            cursor: pointer;
-        }
-        .file {
-            color: #00af00;
-        }
-        .directory-tree {
-            text-align-last: left;
-        }
-    </style>
-</head>
-<body>
-<%- include('partials/navbar') %>
-    <div id="container_top">
-        <div id="container_main">
-            <div id="container_thing">
-                <main>
-                <h1>Directory Tree</h1>
-                <p>(here is where all the raw files are)</p>
-
-                <ul class="directory-tree">
-                    <% function renderTree(items, indentLevel = 0) { %>
-                        <% items.forEach(item => { %>
-                            <li>
-                                <% if (item.type === 'directory') { %>
-                                    <span class="directory"><%= item.name %></span>
-                                    <% if (item.children && item.children.length > 0) { %>
-                                        <ul>
-                                            <%= renderTree(item.children, indentLevel + 1) %>
-                                        </ul>
-                                    <% } %>
-                                <% } else { %>
-                                    <a href="/source/<%= encodeURIComponent(item.path).replaceAll('%2F', '/') %>" class="file"><%= item.name %></a>
-                                <% } %>
-                            </li>
-                        <% }); %>
-                    <% } %>
-                        
-                    <%= renderTree(paths) %>
-                </ul>
-            
-                <h1>Rendered Pages</h1>
-                <div style="display: flex; align-items: center; justify-content: center; flex-wrap: nowrap; flex-direction: column; align-self: center;">
-                    <div style="max-width: 500px; display: flex; align-items: center; justify-content: center; flex-wrap: nowrap; flex-direction: column; align-self: center;">
-                        <p>these are the thingies that you see when you use my site like a normal person (in a browser, hopefully)</p>
-                        <p style="text-align-last: left;">
-                            <a href="/">/</a> &lt;-- Main Page <br />
-                            <a href="/music">/music</a> &lt;-- My Youtube Music playlist<br />
-                            <a href="/source">/source</a> &lt;-- You are here :P <br />
-                            <a href="/art">/art</a> &lt;-- My art ^.^<br />
-                            <a href="/blog">/blog</a> &lt;-- My crappy blog<br />
-                        </p>
-                    </div>
-                </div>
-                <h1>Files</h1>
-                <div style="display: flex; align-items: center; justify-content: center; flex-wrap: nowrap; flex-direction: column; align-self: center;">
-                    <div style="max-width: 500px; display: flex; align-items: center; justify-content: center; flex-wrap: nowrap; flex-direction: column; align-self: center;">
-	                    <p style="text-align-last: left;">     
-	                        <a href="/favicon.ico">/favicon.ico</a> &lt;-- icon</br>
-	                        <a href="/robots.txt">/robots.txt</a> &lt;-- tells some robots to go away
-	                    </p>
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-</body>
-</html>

views/sourceviewer.ejs

@@ -1,25 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>Oddbyte</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <link id="favicon" rel="shortcut icon" type="image/png" href="data:image/png;base64,<%- faviconb64 %>">
-    <link rel="canonical" href="https://oddbyte.dev/"/>
-    <meta name="fediverse:creator" content="@oddbyte@mastodon.social">
-    <style>
-        body {
-            background-color: black;
-            color: white;
-        }
-    </style>
-</head>
-
-<body>
-    <h1 class="selectDisable">Source code for <code><%-filePath %></code></h1>
-    <p><a href="/source" style="color: white;">Back</a></p>
-    <hr />
-    <pre><code><%= fileContent %></code></pre>
-    <hr />
-    <p><a href="/source" style="color: white;">Back</a></p>
-</body>
-</html>