deblan-mirror/web-check
1
0
Fork 0
mirror of https://codeberg.org/alicia/web-check.git synced 2024-06-06 10:32:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #31 from Lissy93/FEAT/node-server

Browse source 
[FEAT] New Server Arch
This commit is contained in:
Alicia Sykes 2023-08-11 23:05:40 +01:00 committed by GitHub
parent 361c65348d 18faeb631b
commit 6a31927562
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 3830 additions and 792 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
.gitignore vendored
View file

@ -1,5 +1,5 @@
# Keys
# keys
.env
# dependencies
@ -7,22 +7,21 @@
/.pnp
.pnp.js
# logs
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# testing
/coverage
# production
/build
# misc
.DS_Store
.env.local
.env.development.local
.env.test.local
.env.production.local
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Local Netlify folder
# Random AWS and Netlify crap
.netlify
.serverless
.webpack
# OS generated files
.DS_Store

5
Dockerfile
View file

@ -7,6 +7,7 @@ RUN apt-get update && \
chmod 755 /usr/bin/chromium && \
rm -rf /var/lib/apt/lists/*
RUN npm install --force
EXPOSE 8888
RUN npm run build
EXPOSE ${PORT:-3000}
ENV CHROME_PATH='/usr/bin/chromium'
CMD ["npm", "run", "serve"]
CMD ["npm", "start"]

51
api/_common/aws-webpack.config.js Normal file
View file

@ -0,0 +1,51 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
module.exports = {
target: 'node',
mode: 'production',
entry: {
'carbon': './api/carbon.js',
'cookies': './api/cookies.js',
'dns-server': './api/dns-server.js',
'dns': './api/dns.js',
'dnssec': './api/dnssec.js',
'features': './api/features.js',
'get-ip': './api/get-ip.js',
'headers': './api/headers.js',
'hsts': './api/hsts.js',
'linked-pages': './api/linked-pages.js',
'mail-config': './api/mail-config.js',
'ports': './api/ports.js',
'quality': './api/quality.js',
'redirects': './api/redirects.js',
'robots-txt': './api/robots-txt.js',
'screenshot': './api/screenshot.js',
'security-txt': './api/security-txt.js',
'sitemap': './api/sitemap.js',
'social-tags': './api/social-tags.js',
'ssl': './api/ssl.js',
'status': './api/status.js',
'tech-stack': './api/tech-stack.js',
'trace-route': './api/trace-route.js',
'txt-records': './api/txt-records.js',
'whois': './api/whois.js',
},
externals: [nodeExternals()],
output: {
filename: '[name].js',
path: path.resolve(__dirname, '.webpack'),
libraryTarget: 'commonjs2'
},
module: {
rules: [
{
test: /\.js$/,
use: {
loader: 'babel-loader'
},
exclude: /node_modules/,
}
]
}
};

38
api/_common/middleware.js Normal file
View file

@ -0,0 +1,38 @@
const normalizeUrl = (url) => {
return url.startsWith('http') ? url : `https://${url}`;
};
const commonMiddleware = (handler) => {
return async (event, context, callback) => {
const queryParams = event.queryStringParameters || event.query || {};
const rawUrl = queryParams.url;
if (!rawUrl) {
callback(null, {
statusCode: 500,
body: JSON.stringify({ error: 'No URL specified' }),
});
}
const url = normalizeUrl(rawUrl);
try {
const response = await handler(url, event, context);
if (response.body && response.statusCode) {
callback(null, response);
} else {
callback(null, {
statusCode: 200,
body: typeof response === 'object' ? JSON.stringify(response) : response,
});
}
} catch (error) {
callback(null, {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
});
}
};
};
module.exports = commonMiddleware;

23
api/get-carbon.js → api/carbon.js
View file

@ -1,14 +1,7 @@
const https = require('https');
const middleware = require('./_common/middleware');
exports.handler = async (event, context) => {
const { url } = event.queryStringParameters;
if (!url) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'url query parameter is required' }),
};
}
const handler = async (url) => {
// First, get the size of the website's HTML
const getHtmlSize = (url) => new Promise((resolve, reject) => {
@ -49,14 +42,10 @@ exports.handler = async (event, context) => {
}
carbonData.scanUrl = url;
return {
statusCode: 200,
body: JSON.stringify(carbonData),
};
return carbonData;
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: `Error: ${error.message}` }),
};
throw new Error(`Error: ${error.message}`);
}
};
exports.handler = middleware(handler);

60
api/content-links.js
View file

@ -1,60 +0,0 @@
const axios = require('axios');
const cheerio = require('cheerio');
const urlLib = require('url');
exports.handler = async (event, context) => {
let url = event.queryStringParameters.url;
// Check if url includes protocol
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = 'http://' + url;
}
try {
const response = await axios.get(url);
const html = response.data;
const $ = cheerio.load(html);
const internalLinksMap = new Map();
const externalLinksMap = new Map();
$('a[href]').each((i, link) => {
const href = $(link).attr('href');
const absoluteUrl = urlLib.resolve(url, href);
if (absoluteUrl.startsWith(url)) {
const count = internalLinksMap.get(absoluteUrl) || 0;
internalLinksMap.set(absoluteUrl, count + 1);
} else if (href.startsWith('http://') || href.startsWith('https://')) {
const count = externalLinksMap.get(absoluteUrl) || 0;
externalLinksMap.set(absoluteUrl, count + 1);
}
});
// Convert maps to sorted arrays
const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
if (internalLinks.length === 0 && externalLinks.length === 0) {
return {
statusCode: 400,
body: JSON.stringify({
skipped: 'No internal or external links found. '
+ 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
+ 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
+ 'You can rectify this by using a headless browser to render the page instead.',
}),
};
}
return {
statusCode: 200,
body: JSON.stringify({ internal: internalLinks, external: externalLinks }),
};
} catch (error) {
console.log(error);
return {
statusCode: 500,
body: JSON.stringify({ error: 'Failed fetching data' }),
};
}
};

14
api/cookies.js Normal file
View file

@ -0,0 +1,14 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
const handler = async (url, event, context) => {
try {
const response = await axios.get(url, {withCredentials: true});
const cookies = response.headers['set-cookie'];
return { cookies };
} catch (error) {
throw new Error(error.message);
}
};
exports.handler = middleware(handler);

26
api/dns-server.js
View file

@ -1,11 +1,11 @@
const dns = require('dns');
const dnsPromises = dns.promises;
// const https = require('https');
const axios = require('axios');
const commonMiddleware = require('./_common/middleware');
exports.handler = async (event) => {
const domain = event.queryStringParameters.url.replace(/^(?:https?:\/\/)?/i, "");
const handler = async (url) => {
try {
const domain = url.replace(/^(?:https?:\/\/)?/i, "");
const addresses = await dnsPromises.resolve4(domain);
const results = await Promise.all(addresses.map(async (address) => {
const hostname = await dnsPromises.reverse(address).catch(() => null);
@ -22,6 +22,7 @@ exports.handler = async (event) => {
dohDirectSupports,
};
}));
// let dohMozillaSupport = false;
// try {
// const mozillaList = await axios.get('https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records');
@ -29,20 +30,15 @@ exports.handler = async (event) => {
// } catch (error) {
// console.error(error);
// }
return {
statusCode: 200,
body: JSON.stringify({
domain,
dns: results,
// dohMozillaSupport,
}),
domain,
dns: results,
// dohMozillaSupport,
};
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({
error: `An error occurred while resolving DNS. ${error.message}`,
}),
};
throw new Error(`An error occurred while resolving DNS. ${error.message}`); // This will be caught and handled by the commonMiddleware
}
};
exports.handler = commonMiddleware(handler);

35
api/get-dns.js → api/dns.js
View file

@ -1,8 +1,9 @@
const dns = require('dns');
const util = require('util');
const middleware = require('./_common/middleware');
exports.handler = async function(event, context) {
let hostname = event.queryStringParameters.url;
const handler = async (url) => {
let hostname = url;
// Handle URLs by extracting hostname
if (hostname.startsWith('http://') || hostname.startsWith('https://')) {
@ -35,25 +36,19 @@ exports.handler = async function(event, context) {
]);
return {
statusCode: 200,
body: JSON.stringify({
A: a,
AAAA: aaaa,
MX: mx,
TXT: txt,
NS: ns,
CNAME: cname,
SOA: soa,
SRV: srv,
PTR: ptr
})
A: a,
AAAA: aaaa,
MX: mx,
TXT: txt,
NS: ns,
CNAME: cname,
SOA: soa,
SRV: srv,
PTR: ptr
};
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({
error: error.message
})
};
throw new Error(error.message);
}
};
exports.handler = middleware(handler);

29
api/dns-sec.js → api/dnssec.js
View file

@ -1,16 +1,7 @@
const https = require('https');
const commonMiddleware = require('./_common/middleware'); // Make sure this path is correct
exports.handler = async function(event, context) {
let { url } = event.queryStringParameters;
if (!url) {
return errorResponse('URL query parameter is required.');
}
// Extract hostname from URL
const parsedUrl = new URL(url);
const domain = parsedUrl.hostname;
const fetchDNSRecords = async (domain, event, context) => {
const dnsTypes = ['DNSKEY', 'DS', 'RRSIG'];
const records = {};
@ -48,22 +39,14 @@ exports.handler = async function(event, context) {
if (dnsResponse.Answer) {
records[type] = { isFound: true, answer: dnsResponse.Answer, response: dnsResponse.Answer };
} else {
records[type] = { isFound: false, answer: null, response: dnsResponse};
records[type] = { isFound: false, answer: null, response: dnsResponse };
}
} catch (error) {
return errorResponse(`Error fetching ${type} record: ${error.message}`);
throw new Error(`Error fetching ${type} record: ${error.message}`); // This will be caught and handled by the commonMiddleware
}
}
return {
statusCode: 200,
body: JSON.stringify(records),
};
return records;
};
const errorResponse = (message, statusCode = 444) => {
return {
statusCode: statusCode,
body: JSON.stringify({ error: message }),
};
};
exports.handler = commonMiddleware(fetchDNSRecords);

24
api/site-features.js → api/features.js
View file

@ -1,22 +1,15 @@
const https = require('https');
const middleware = require('./_common/middleware');
exports.handler = async function (event, context) {
const { url } = event.queryStringParameters;
const builtWithHandler = async (url) => {
const apiKey = process.env.BUILT_WITH_API_KEY;
const errorResponse = (message, statusCode = 500) => {
return {
statusCode: statusCode,
body: JSON.stringify({ error: message }),
};
};
if (!url) {
return errorResponse('URL query parameter is required', 400);
throw new Error('URL query parameter is required');
}
if (!apiKey) {
return errorResponse('Missing BuiltWith API key in environment variables', 500);
throw new Error('Missing BuiltWith API key in environment variables');
}
const apiUrl = `https://api.builtwith.com/free1/api.json?KEY=${apiKey}&LOOKUP=${encodeURIComponent(url)}`;
@ -46,11 +39,10 @@ exports.handler = async function (event, context) {
req.end();
});
return {
statusCode: 200,
body: response,
};
return response;
} catch (error) {
return errorResponse(`Error making request: ${error.message}`);
throw new Error(`Error making request: ${error.message}`);
}
};
exports.handler = middleware(builtWithHandler);

33
api/find-url-ip.js
View file

@ -1,33 +0,0 @@
const dns = require('dns');
/* Lambda function to fetch the IP address of a given URL */
exports.handler = function (event, context, callback) {
const addressParam = event.queryStringParameters.url;
if (!addressParam) {
callback(null, errorResponse('Address parameter is missing.'));
return;
}
const address = decodeURIComponent(addressParam)
.replaceAll('https://', '')
.replaceAll('http://', '');
dns.lookup(address, (err, ip, family) => {
if (err) {
callback(null, errorResponse(err.message));
} else {
callback(null, {
statusCode: 200,
body: JSON.stringify({ ip, family }),
});
}
});
};
const errorResponse = (message, statusCode = 444) => {
return {
statusCode: statusCode,
body: JSON.stringify({ error: message }),
};
};

35
api/follow-redirects.js
View file

@ -1,35 +0,0 @@
exports.handler = async (event) => {
const { url } = event.queryStringParameters;
const redirects = [url];
try {
const got = await import('got');
await got.default(url, {
followRedirect: true,
maxRedirects: 12,
hooks: {
beforeRedirect: [
(options, response) => {
redirects.push(response.headers.location);
},
],
},
});
return {
statusCode: 200,
body: JSON.stringify({
redirects: redirects,
}),
};
} catch (error) {
return errorResponse(`Error: ${error.message}`);
}
};
const errorResponse = (message, statusCode = 444) => {
return {
statusCode: statusCode,
body: JSON.stringify({ error: message }),
};
};

26
api/get-cookies.js
View file

@ -1,26 +0,0 @@
const axios = require('axios');
exports.handler = async function(event, context) {
const { url } = event.queryStringParameters;
if (!url) {
return {
statusCode: 400,
body: JSON.stringify({ message: 'url query string parameter is required' }),
};
}
try {
const response = await axios.get(url, {withCredentials: true});
const cookies = response.headers['set-cookie'];
return {
statusCode: 200,
body: JSON.stringify({ cookies }),
};
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
}
};

31
api/get-headers.js
View file

@ -1,31 +0,0 @@
const axios = require('axios');
exports.handler = async function(event, context) {
const { url } = event.queryStringParameters;
if (!url) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'url query string parameter is required' }),
};
}
try {
const response = await axios.get(url, {
validateStatus: function (status) {
return status >= 200 && status < 600; // Resolve only if the status code is less than 600
},
});
return {
statusCode: 200,
body: JSON.stringify(response.headers),
};
} catch (error) {
console.log(error);
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
}
};

21
api/get-ip.js Normal file
View file

@ -0,0 +1,21 @@
const dns = require('dns');
const middleware = require('./_common/middleware');
const lookupAsync = (address) => {
return new Promise((resolve, reject) => {
dns.lookup(address, (err, ip, family) => {
if (err) {
reject(err);
} else {
resolve({ ip, family });
}
});
});
};
const handler = async (url) => {
const address = url.replaceAll('https://', '').replaceAll('http://', '');
return await lookupAsync(address);
};
exports.handler = middleware(handler);

18
api/headers.js Normal file
View file

@ -0,0 +1,18 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
const handler = async (url, event, context) => {
try {
const response = await axios.get(url, {
validateStatus: function (status) {
return status >= 200 && status < 600; // Resolve only if the status code is less than 600
},
});
return response.headers;
} catch (error) {
throw new Error(error.message);
}
};
exports.handler = middleware(handler);

16
api/check-hsts.js → api/hsts.js
View file

@ -1,8 +1,7 @@
const https = require('https');
const middleware = require('./_common/middleware');
exports.handler = async function(event, context) {
const siteURL = event.queryStringParameters.url;
exports.handler = middleware(async (url, event, context) => {
const errorResponse = (message, statusCode = 500) => {
return {
statusCode: statusCode,
@ -16,15 +15,9 @@ exports.handler = async function(event, context) {
};
};
if (!siteURL) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'URL parameter is missing!' }),
};
}
return new Promise((resolve, reject) => {
const req = https.request(siteURL, res => {
const req = https.request(url, res => {
const headers = res.headers;
const hstsHeader = headers['strict-transport-security'];
@ -60,4 +53,5 @@ exports.handler = async function(event, context) {
req.end();
});
};
});

40
api/lighthouse-report.js
View file

@ -1,40 +0,0 @@
const axios = require('axios');
exports.handler = function(event, context, callback) {
const { url } = event.queryStringParameters;
if (!url) {
callback(null, {
statusCode: 400,
body: JSON.stringify({ error: 'URL param is required'}),
});
}
const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
if (!apiKey) {
callback(null, {
statusCode: 500,
body: JSON.stringify({ error: 'API key (GOOGLE_CLOUD_API_KEY) not set'}),
});
}
const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
axios.get(endpoint)
.then(
(response) => {
callback(null, {
statusCode: 200,
body: JSON.stringify(response.data),
});
}
).catch(
() => {
callback(null, {
statusCode: 500,
body: JSON.stringify({ error: 'Error running Lighthouse'}),
});
}
);
};

48
api/linked-pages.js Normal file
View file

@ -0,0 +1,48 @@
const axios = require('axios');
const cheerio = require('cheerio');
const urlLib = require('url');
const commonMiddleware = require('./_common/middleware');
const handler = async (url) => {
const response = await axios.get(url);
const html = response.data;
const $ = cheerio.load(html);
const internalLinksMap = new Map();
const externalLinksMap = new Map();
// Get all links on the page
$('a[href]').each((i, link) => {
const href = $(link).attr('href');
const absoluteUrl = urlLib.resolve(url, href);
// Check if absolute / relative, append to appropriate map or increment occurrence count
if (absoluteUrl.startsWith(url)) {
const count = internalLinksMap.get(absoluteUrl) || 0;
internalLinksMap.set(absoluteUrl, count + 1);
} else if (href.startsWith('http://') || href.startsWith('https://')) {
const count = externalLinksMap.get(absoluteUrl) || 0;
externalLinksMap.set(absoluteUrl, count + 1);
}
});
// Sort by most occurrences, remove supplicates, and convert to array
const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
// If there were no links, then mark as skipped and show reasons
if (internalLinks.length === 0 && externalLinks.length === 0) {
return {
statusCode: 400,
body: JSON.stringify({
skipped: 'No internal or external links found. '
+ 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
+ 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
+ 'You can rectify this by using a headless browser to render the page instead.',
}),
};
}
return { internal: internalLinks, external: externalLinks };
};
exports.handler = commonMiddleware(handler);

20
api/mail-config.js
View file

@ -1,11 +1,11 @@
const commonMiddleware = require('./_common/middleware');
const dns = require('dns').promises;
const URL = require('url-parse');
exports.handler = async (event, context) => {
const handler = async (url, event, context) => {
try {
let domain = event.queryStringParameters.url;
const parsedUrl = new URL(domain);
domain = parsedUrl.hostname || parsedUrl.pathname;
const domain = new URL(url).hostname || new URL(url).pathname;
// Get MX records
const mxRecords = await dns.resolveMx(domain);
@ -56,19 +56,13 @@ exports.handler = async (event, context) => {
}
return {
statusCode: 200,
body: JSON.stringify({
mxRecords,
txtRecords: emailTxtRecords,
mailServices,
}),
};
};
} catch (error) {
if (error.code === 'ENOTFOUND' || error.code === 'ENODATA') {
return {
statusCode: 200,
body: JSON.stringify({ skipped: 'No mail server in use on this domain' }),
};
return { skipped: 'No mail server in use on this domain' };
} else {
return {
statusCode: 500,
@ -77,3 +71,5 @@ exports.handler = async (event, context) => {
}
}
};
module.exports.handler = commonMiddleware(handler);

13
api/check-ports.js → api/ports.js
View file

@ -1,4 +1,5 @@
const net = require('net');
const middleware = require('./_common/middleware');
// A list of commonly used ports.
const PORTS = [
@ -12,7 +13,7 @@ async function checkPort(port, domain) {
return new Promise((resolve, reject) => {
const socket = new net.Socket();
socket.setTimeout(1500); // you may want to adjust the timeout
socket.setTimeout(1500);
socket.once('connect', () => {
socket.destroy();
@ -33,13 +34,9 @@ async function checkPort(port, domain) {
});
}
exports.handler = async (event, context) => {
const domain = event.queryStringParameters.url;
const handler = async (url, event, context) => {
const domain = url.replace(/(^\w+:|^)\/\//, '');
if (!domain) {
return errorResponse('Missing domain parameter.');
}
const delay = ms => new Promise(res => setTimeout(res, ms));
const timeout = delay(9000);
@ -88,3 +85,5 @@ const errorResponse = (message, statusCode = 444) => {
body: JSON.stringify({ error: message }),
};
};
exports.handler = middleware(handler);

22
api/quality.js Normal file
View file

@ -0,0 +1,22 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
const handler = async (url, event, context) => {
const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
if (!url) {
throw new Error('URL param is required');
}
if (!apiKey) {
throw new Error('API key (GOOGLE_CLOUD_API_KEY) not set');
}
const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
const response = await axios.get(endpoint);
return response.data;
};
exports.handler = middleware(handler);

45
api/read-robots-txt.js
View file

@ -1,45 +0,0 @@
const axios = require('axios');
exports.handler = async function(event, context) {
const siteURL = event.queryStringParameters.url;
if (!siteURL) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Missing url query parameter' }),
};
}
let parsedURL;
try {
parsedURL = new URL(siteURL);
} catch (error) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Invalid url query parameter' }),
};
}
const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
try {
const response = await axios.get(robotsURL);
if (response.status === 200) {
return {
statusCode: 200,
body: response.data,
};
} else {
return {
statusCode: response.status,
body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
};
}
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
};
}
};

27
api/redirects.js Normal file
View file

@ -0,0 +1,27 @@
const handler = async (url) => {
const redirects = [url];
const got = await import('got');
try {
await got.default(url, {
followRedirect: true,
maxRedirects: 12,
hooks: {
beforeRedirect: [
(options, response) => {
redirects.push(response.headers.location);
},
],
},
});
return {
redirects: redirects,
};
} catch (error) {
throw new Error(`Error: ${error.message}`);
}
};
const middleware = require('./_common/middleware');
exports.handler = middleware(handler);

66
api/robots-txt.js Normal file
View file

@ -0,0 +1,66 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
const parseRobotsTxt = (content) => {
const lines = content.split('\n');
const rules = [];
lines.forEach(line => {
line = line.trim(); // This removes trailing and leading whitespaces
let match = line.match(/^(Allow|Disallow):\s*(\S*)$/i);
if (match) {
const rule = {
lbl: match[1],
val: match[2],
};
rules.push(rule);
} else {
match = line.match(/^(User-agent):\s*(\S*)$/i);
if (match) {
const rule = {
lbl: match[1],
val: match[2],
};
rules.push(rule);
}
}
});
return { robots: rules };
}
const handler = async function(url) {
let parsedURL;
try {
parsedURL = new URL(url);
} catch (error) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Invalid url query parameter' }),
};
}
const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
try {
const response = await axios.get(robotsURL);
if (response.status === 200) {
return parseRobotsTxt(response.data);
} else {
return {
statusCode: response.status,
body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
};
}
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
};
}
};
exports.handler = middleware(handler);

35
api/screenshot.js
View file

@ -1,16 +1,11 @@
const puppeteer = require('puppeteer-core');
const chromium = require('chrome-aws-lambda');
const middleware = require('./_common/middleware');
exports.handler = async (event, context, callback) => {
let browser = null;
let targetUrl = event.queryStringParameters.url;
const screenshotHandler = async (targetUrl) => {
if (!targetUrl) {
callback(null, {
statusCode: 400,
body: JSON.stringify({ error: 'URL is missing from queryStringParameters' }),
});
return;
throw new Error('URL is missing from queryStringParameters');
}
if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
@ -20,13 +15,10 @@ exports.handler = async (event, context, callback) => {
try {
new URL(targetUrl);
} catch (error) {
callback(null, {
statusCode: 400,
body: JSON.stringify({ error: 'URL provided is invalid' }),
});
return;
throw new Error('URL provided is invalid');
}
let browser = null;
try {
browser = await puppeteer.launch({
args: chromium.args,
@ -40,9 +32,7 @@ exports.handler = async (event, context, callback) => {
let page = await browser.newPage();
await page.emulateMediaFeatures([{ name: 'prefers-color-scheme', value: 'dark' }]);
page.setDefaultNavigationTimeout(8000);
await page.goto(targetUrl, { waitUntil: 'domcontentloaded' });
await page.evaluate(() => {
@ -57,24 +47,15 @@ exports.handler = async (event, context, callback) => {
});
const screenshotBuffer = await page.screenshot();
const base64Screenshot = screenshotBuffer.toString('base64');
const response = {
statusCode: 200,
body: JSON.stringify({ image: base64Screenshot }),
};
return { image: base64Screenshot };
callback(null, response);
} catch (error) {
console.log(error);
callback(null, {
statusCode: 500,
body: JSON.stringify({ error: `An error occurred: ${error.message}` }),
});
} finally {
if (browser !== null) {
await browser.close();
}
}
};
exports.handler = middleware(screenshotHandler);

45
api/security-txt.js
View file

@ -1,5 +1,6 @@
const { https } = require('follow-redirects');
const { URL } = require('url');
const middleware = require('./_common/middleware');
const SECURITY_TXT_PATHS = [
'/security.txt',
@ -37,59 +38,39 @@ const isPgpSigned = (result) => {
return false;
};
exports.handler = async (event, context) => {
const urlParam = event.queryStringParameters.url;
if (!urlParam) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Missing url parameter' })
};
}
const securityTxtHandler = async (urlParam) => {
let url;
try {
url = new URL(urlParam.includes('://') ? urlParam : 'https://' + urlParam);
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: 'Invalid URL format' }),
};
throw new Error('Invalid URL format');
}
url.pathname = '';
for (let path of SECURITY_TXT_PATHS) {
try {
const result = await fetchSecurityTxt(url, path);
if (result && result.includes('<html')) return {
statusCode: 200,
body: JSON.stringify({ isPresent: false }),
};
if (result && result.includes('<html')) return { isPresent: false };
if (result) {
return {
statusCode: 200,
body: JSON.stringify({
isPresent: true,
foundIn: path,
content: result,
isPgpSigned: isPgpSigned(result),
fields: parseResult(result),
}),
isPresent: true,
foundIn: path,
content: result,
isPgpSigned: isPgpSigned(result),
fields: parseResult(result),
};
}
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
throw new Error(error.message);
}
}
return {
statusCode: 404,
body: JSON.stringify({ isPresent: false }),
};
return { isPresent: false };
};
exports.handler = middleware(securityTxtHandler);
async function fetchSecurityTxt(baseURL, path) {
return new Promise((resolve, reject) => {
const url = new URL(path, baseURL);

7
api/sitemap.js
View file

@ -1,8 +1,9 @@
const commonMiddleware = require('./_common/middleware');
const axios = require('axios');
const xml2js = require('xml2js');
exports.handler = async (event) => {
const url = event.queryStringParameters.url;
const handler = async (url) => {
let sitemapUrl = `${url}/sitemap.xml`;
try {
@ -59,3 +60,5 @@ exports.handler = async (event) => {
}
}
};
exports.handler = commonMiddleware(handler);

7
api/social-tags.js
View file

@ -1,8 +1,9 @@
const commonMiddleware = require('./_common/middleware');
const axios = require('axios');
const cheerio = require('cheerio');
exports.handler = async (event, context) => {
let url = event.queryStringParameters.url;
const handler = async (url) => {
// Check if url includes protocol
if (!url.startsWith('http://') && !url.startsWith('https://')) {
@ -66,3 +67,5 @@ exports.handler = async (event, context) => {
};
}
};
exports.handler = commonMiddleware(handler);

50
api/ssl-check.js
View file

@ -1,50 +0,0 @@
const https = require('https');
exports.handler = async function (event, context) {
const { url } = event.queryStringParameters;
const errorResponse = (message, statusCode = 500) => {
return {
statusCode: statusCode,
body: JSON.stringify({ error: message }),
};
};
if (!url) {
return errorResponse('URL query parameter is required', 400);
}
try {
const response = await new Promise((resolve, reject) => {
const req = https.request(url, res => {
// Check if the SSL handshake was authorized
if (!res.socket.authorized) {
resolve(errorResponse(`SSL handshake not authorized. Reason: ${res.socket.authorizationError}`));
} else {
let cert = res.socket.getPeerCertificate(true);
if (!cert || Object.keys(cert).length === 0) {
resolve(errorResponse("No certificate presented by the server."));
} else {
// omit the raw and issuerCertificate fields
const { raw, issuerCertificate, ...certWithoutRaw } = cert;
resolve({
statusCode: 200,
body: JSON.stringify(certWithoutRaw),
});
}
}
});
req.on('error', error => {
resolve(errorResponse(`Error fetching site certificate: ${error.message}`));
});
req.end();
});
return response;
} catch (error) {
return errorResponse(`Unexpected error occurred: ${error.message}`);
}
};

37
api/ssl.js Normal file
View file

@ -0,0 +1,37 @@
const https = require('https');
const middleware = require('./_common/middleware');
const fetchSiteCertificateHandler = async (url) => {
try {
const response = await new Promise((resolve, reject) => {
const req = https.request(url, res => {
// Check if the SSL handshake was authorized
if (!res.socket.authorized) {
reject(new Error(`SSL handshake not authorized. Reason: ${res.socket.authorizationError}`));
} else {
let cert = res.socket.getPeerCertificate(true);
if (!cert || Object.keys(cert).length === 0) {
reject(new Error("No certificate presented by the server."));
} else {
// omit the raw and issuerCertificate fields
const { raw, issuerCertificate, ...certWithoutRaw } = cert;
resolve(certWithoutRaw);
}
}
});
req.on('error', error => {
reject(new Error(`Error fetching site certificate: ${error.message}`));
});
req.end();
});
return response;
} catch (error) {
throw new Error(error.message);
}
};
exports.handler = middleware(fetchSiteCertificateHandler);

27
api/server-status.js → api/status.js
View file

@ -1,14 +1,10 @@
const https = require('https');
const { performance, PerformanceObserver } = require('perf_hooks');
const middleware = require('./_common/middleware');
exports.handler = async function(event, context) {
const { url } = event.queryStringParameters;
const checkURLHandler = async (url) => {
if (!url) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'You must provide a URL query parameter!' }),
};
throw new Error('You must provide a URL query parameter!');
}
let dnsLookupTime;
@ -43,10 +39,7 @@ exports.handler = async function(event, context) {
});
if (responseCode < 200 || responseCode >= 400) {
return {
statusCode: 200,
body: JSON.stringify({ error: `Received non-success response code: ${responseCode}` }),
};
throw new Error(`Received non-success response code: ${responseCode}`);
}
performance.mark('B');
@ -54,16 +47,12 @@ exports.handler = async function(event, context) {
let responseTime = performance.now() - startTime;
obs.disconnect();
return {
statusCode: 200,
body: JSON.stringify({ isUp: true, dnsLookupTime, responseTime, responseCode }),
};
return { isUp: true, dnsLookupTime, responseTime, responseCode };
} catch (error) {
obs.disconnect();
return {
statusCode: 200,
body: JSON.stringify({ error: `Error during operation: ${error.message}` }),
};
throw error;
}
};
exports.handler = middleware(checkURLHandler);

79
api/tech-stack.js
View file

@ -1,69 +1,30 @@
const Wappalyzer = require('wappalyzer');
const middleware = require('./_common/middleware');
const analyze = async (url) => {
const analyzeSiteTechnologies = async (url) => {
const options = {};
const wappalyzer = new Wappalyzer(options);
return (async function() {
try {
await wappalyzer.init()
const headers = {}
const storage = {
local: {},
session: {},
}
const site = await wappalyzer.open(url, headers, storage)
const results = await site.analyze()
return results;
} catch (error) {
return error;
} finally {
await wappalyzer.destroy()
}
})();
}
exports.handler = async (event, context, callback) => {
// Validate URL parameter
if (!event.queryStringParameters || !event.queryStringParameters.url) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Missing url parameter' }),
};
}
// Get URL from param
let url = event.queryStringParameters.url;
if (!/^https?:\/\//i.test(url)) {
url = 'http://' + url;
}
try {
return analyze(url).then(
(results) => {
if (!results.technologies || results.technologies.length === 0) {
return {
statusCode: 200,
body: JSON.stringify({ error: 'Unable to find any technologies for site' }),
};
}
return {
statusCode: 200,
body: JSON.stringify(results),
}
}
)
.catch((error) => {
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
});
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
await wappalyzer.init();
const headers = {};
const storage = {
local: {},
session: {},
};
const site = await wappalyzer.open(url, headers, storage);
const results = await site.analyze();
if (!results.technologies || results.technologies.length === 0) {
throw new Error('Unable to find any technologies for site');
}
return results;
} catch (error) {
throw new Error(error.message);
} finally {
await wappalyzer.destroy();
}
};
exports.handler = middleware(analyzeSiteTechnologies);

68
api/trace-route.js
View file

@ -1,55 +1,31 @@
const traceroute = require('traceroute');
const url = require('url');
const middleware = require('./_common/middleware');
exports.handler = async function(event, context) {
const urlString = event.queryStringParameters.url;
const executeTraceroute = async (urlString, context) => {
// Parse the URL and get the hostname
const urlObject = url.parse(urlString);
const host = urlObject.hostname;
try {
if (!urlString) {
throw new Error('URL parameter is missing!');
}
if (!host) {
throw new Error('Invalid URL provided');
}
// Parse the URL and get the hostname
const urlObject = url.parse(urlString);
const host = urlObject.hostname;
if (!host) {
throw new Error('Invalid URL provided');
}
// Traceroute with callback
const result = await new Promise((resolve, reject) => {
traceroute.trace(host, (err, hops) => {
if (err || !hops) {
reject(err || new Error('No hops found'));
} else {
resolve(hops);
}
});
// Check if remaining time is less than 8.8 seconds, then reject promise
if (context.getRemainingTimeInMillis() < 8800) {
reject(new Error('Lambda is about to timeout'));
// Traceroute with callback
const result = await new Promise((resolve, reject) => {
traceroute.trace(host, (err, hops) => {
if (err || !hops) {
reject(err || new Error('No hops found'));
} else {
resolve(hops);
}
});
});
return {
statusCode: 200,
body: JSON.stringify({
message: "Traceroute completed!",
result,
}),
};
} catch (err) {
const message = err.code === 'ENOENT'
? 'Traceroute command is not installed on the host.'
: err.message;
return {
statusCode: 500,
body: JSON.stringify({
error: message,
}),
};
}
return {
message: "Traceroute completed!",
result,
};
};
exports.handler = middleware(executeTraceroute);

26
api/get-txt.js → api/txt-records.js
View file

@ -1,9 +1,11 @@
const dns = require('dns').promises;
const middleware = require('./_common/middleware');
exports.handler = async (event) => {
const url = new URL(event.queryStringParameters.url);
const handler = async (url, event, context) => {
try {
const txtRecords = await dns.resolveTxt(url.hostname);
const parsedUrl = new URL(url);
const txtRecords = await dns.resolveTxt(parsedUrl.hostname);
// Parsing and formatting TXT records into a single object
const readableTxtRecords = txtRecords.reduce((acc, recordArray) => {
@ -16,15 +18,15 @@ exports.handler = async (event) => {
return { ...acc, ...recordObject };
}, {});
return {
statusCode: 200,
body: JSON.stringify(readableTxtRecords),
};
return readableTxtRecords;
} catch (error) {
console.error('Error:', error);
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
if (error.code === 'ERR_INVALID_URL') {
throw new Error(`Invalid URL ${error}`);
} else {
throw error;
}
}
};
exports.handler = middleware(handler);

96
api/whois-lookup.js → api/whois.js
View file

@ -1,14 +1,6 @@
const net = require('net');
const psl = require('psl');
// const { URL } = require('url');
const errorResponse = (message, statusCode = 444) => {
return {
statusCode: statusCode,
body: JSON.stringify({ error: message }),
};
};
const middleware = require('./_common/middleware');
const getBaseDomain = (url) => {
let protocol = '';
@ -22,55 +14,7 @@ const getBaseDomain = (url) => {
return protocol + parsed.domain;
};
exports.handler = async function(event, context) {
let url = event.queryStringParameters.url;
if (!url) {
return errorResponse('URL query parameter is required.', 400);
}
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = 'http://' + url;
}
let hostname;
try {
hostname = getBaseDomain(new URL(url).hostname);
} catch (error) {
return errorResponse(`Unable to parse URL: ${error}`, 400);
}
return new Promise((resolve, reject) => {
const client = net.createConnection({ port: 43, host: 'whois.internic.net' }, () => {
client.write(hostname + '\r\n');
});
let data = '';
client.on('data', (chunk) => {
data += chunk;
});
client.on('end', () => {
try {
const parsedData = parseWhoisData(data);
resolve({
statusCode: 200,
body: JSON.stringify(parsedData),
});
} catch (error) {
resolve(errorResponse(error.message));
}
});
client.on('error', (err) => {
resolve(errorResponse(err.message, 500));
});
});
};
const parseWhoisData = (data) => {
if (data.includes('No match for')) {
return { error: 'No matches found for domain in internic database'};
}
@ -100,3 +44,41 @@ const parseWhoisData = (data) => {
return parsedData;
};
const fetchWhoisData = async (url) => {
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = 'http://' + url;
}
let hostname;
try {
hostname = getBaseDomain(new URL(url).hostname);
} catch (error) {
throw new Error(`Unable to parse URL: ${error}`);
}
return new Promise((resolve, reject) => {
const client = net.createConnection({ port: 43, host: 'whois.internic.net' }, () => {
client.write(hostname + '\r\n');
});
let data = '';
client.on('data', (chunk) => {
data += chunk;
});
client.on('end', () => {
try {
const parsedData = parseWhoisData(data);
resolve(parsedData);
} catch (error) {
reject(error);
}
});
client.on('error', (err) => {
reject(err);
});
});
};
exports.handler = middleware(fetchWhoisData);

19
package.json
View file

@ -17,7 +17,7 @@
"scripts": {
"dev": "netlify dev",
"serve": "netlify serve --offline",
"start": "react-scripts start",
"start": "node server",
"build": "react-scripts build",
"test": "react-scripts test",
"eject": "react-scripts eject"
@ -35,9 +35,13 @@
"@types/react-router-dom": "^5.3.3",
"@types/react-simple-maps": "^3.0.0",
"@types/styled-components": "^5.1.26",
"aws-serverless-express": "^3.4.0",
"axios": "^1.4.0",
"cheerio": "^1.0.0-rc.12",
"chrome-aws-lambda": "^10.1.0",
"chromium": "^3.0.3",
"connect-history-api-fallback": "^2.0.0",
"dotenv": "^16.3.1",
"flatted": "^3.2.7",
"follow-redirects": "^1.15.2",
"got": "^13.0.0",
@ -46,7 +50,7 @@
"perf_hooks": "^0.0.1",
"psl": "^1.9.0",
"puppeteer": "^20.9.0",
"puppeteer-core": "^20.9.0",
"puppeteer-core": "^21.0.3",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-masonry-css": "^1.0.16",
@ -57,9 +61,9 @@
"styled-components": "^6.0.5",
"traceroute": "^1.0.0",
"typescript": "^5.1.6",
"wappalyzer": "^6.10.63",
"wappalyzer": "^6.10.65",
"web-vitals": "^3.4.0",
"xml2js": "^0.6.0"
"xml2js": "^0.6.2"
},
"eslintConfig": {
"extends": [
@ -82,5 +86,12 @@
"compilerOptions": {
"allowJs": true,
"outDir": "./dist"
},
"devDependencies": {
"serverless-domain-manager": "^7.1.1",
"serverless-offline": "^12.0.4",
"serverless-webpack": "^5.13.0",
"webpack": "^5.88.2",
"webpack-node-externals": "^3.0.0"
}
}

116
server.js Normal file
View file

@ -0,0 +1,116 @@
const express = require('express');
const awsServerlessExpress = require('aws-serverless-express');
const fs = require('fs');
const path = require('path');
const historyApiFallback = require('connect-history-api-fallback');
require('dotenv').config();
const app = express();
const API_DIR = '/api'; // Name of the dir containing the lambda functions
const dirPath = path.join(__dirname, API_DIR); // Path to the lambda functions dir
const guiPath = path.join(__dirname, 'build');
// Execute the lambda function
const executeHandler = async (handler, req) => {
return new Promise((resolve, reject) => {
const callback = (err, response) => err ? reject(err) : resolve(response);
const promise = handler(req, {}, callback);
if (promise && typeof promise.then === 'function') {
promise.then(resolve).catch(reject);
}
});
};
// Array of all the lambda function file names
const fileNames = fs.readdirSync(dirPath, { withFileTypes: true })
.filter(dirent => dirent.isFile() && dirent.name.endsWith('.js'))
.map(dirent => dirent.name);
const handlers = {};
fileNames.forEach(file => {
const routeName = file.split('.')[0];
const route = `${API_DIR}/${routeName}`;
const handler = require(path.join(dirPath, file)).handler;
handlers[route] = handler;
app.get(route, async (req, res) => {
try {
const { statusCode = 200, body = '' } = await executeHandler(handler, req);
res.status(statusCode).json(JSON.parse(body));
} catch (err) {
res.status(500).json({ error: err.message });
}
});
});
const timeout = (ms, jobName = null) => {
return new Promise((_, reject) => {
setTimeout(() => {
reject(new Error(`Timed out after the ${ms/1000} second limit${jobName ? `, when executing the ${jobName} task` : ''}`));
}, ms);
});
}
app.get('/api', async (req, res) => {
const results = {};
const { url } = req.query;
const maxExecutionTime = process.env.API_TIMEOUT_LIMIT || 15000;
const handlerPromises = Object.entries(handlers).map(async ([route, handler]) => {
const routeName = route.replace(`${API_DIR}/`, '');
try {
const result = await Promise.race([
executeHandler(handler, { query: { url } }),
timeout(maxExecutionTime, routeName)
]);
results[routeName] = JSON.parse((result || {}).body);
} catch (err) {
results[routeName] = { error: err.message };
}
});
await Promise.all(handlerPromises);
res.json(results);
});
// Handle SPA routing
app.use(historyApiFallback({
rewrites: [
{ from: /^\/api\/.*$/, to: function(context) { return context.parsedUrl.path; } },
]
}));
// Serve up the GUI - if build dir exists, and GUI feature enabled
if (process.env.DISABLE_GUI && process.env.DISABLE_GUI !== 'false') {
app.get('*', (req, res) => {
res.status(500).send(
'Welcome to Web-Check!<br />Access the API endpoints at '
+'<a href="/api"><code>/api</code></a>'
);
});
} else if (!fs.existsSync(guiPath)) {
app.get('*', (req, res) => {
res.status(500).send(
'Welcome to Web-Check!<br />Looks like the GUI app has not yet been compiled, '
+'run <code>yarn build</code> to continue, then restart the server.'
);
});
} else { // GUI enabled, and build files present, let's go!!
app.use(express.static(guiPath));
}
// Create serverless express server
const port = process.env.PORT || 3000;
const server = awsServerlessExpress.createServer(app).listen(port, () => {
console.log(`Server is running on port ${port}`);
});
exports.handler = (event, context) => {
awsServerlessExpress.proxy(server, event, context);
};

179
serverless.yml Normal file
View file

@ -0,0 +1,179 @@
service: web-check-api
provider:
name: aws
runtime: nodejs14.x
region: us-east-1
functions:
dnssec:
handler: api/dnssec.handler
events:
- http:
path: api/dnssec
method: get
linkedPages:
handler: api/linked-pages.handler
events:
- http:
path: api/linked-pages
method: get
robotsTxt:
handler: api/robots-txt.handler
events:
- http:
path: api/robots-txt
method: get
ssl:
handler: api/ssl.handler
events:
- http:
path: api/ssl
method: get
whois:
handler: api/whois.handler
events:
- http:
path: api/whois
method: get
carbon:
handler: api/carbon.handler
events:
- http:
path: api/carbon
method: get
features:
handler: api/features.handler
events:
- http:
path: api/features
method: get
mailConfig:
handler: api/mail-config.handler
events:
- http:
path: api/mail-config
method: get
screenshot:
handler: api/screenshot.handler
events:
- http:
path: api/screenshot
method: get
status:
handler: api/status.handler
events:
- http:
path: api/status
method: get
cookies:
handler: api/cookies.handler
events:
- http:
path: api/cookies
method: get
getIp:
handler: api/get-ip.handler
events:
- http:
path: api/get-ip
method: get
ports:
handler: api/ports.handler
events:
- http:
path: api/ports
method: get
securityTxt:
handler: api/security-txt.handler
events:
- http:
path: api/security-txt
method: get
techStack:
handler: api/tech-stack.handler
events:
- http:
path: api/tech-stack
method: get
dnsServer:
handler: api/dns-server.handler
events:
- http:
path: api/dns-server
method: get
headers:
handler: api/headers.handler
events:
- http:
path: api/headers
method: get
quality:
handler: api/quality.handler
events:
- http:
path: api/quality
method: get
sitemap:
handler: api/sitemap.handler
events:
- http:
path: api/sitemap
method: get
traceRoute:
handler: api/trace-route.handler
events:
- http:
path: api/trace-route
method: get
dns:
handler: api/dns.handler
events:
- http:
path: api/dns
method: get
hsts:
handler: api/hsts.handler
events:
- http:
path: api/hsts
method: get
redirects:
handler: api/redirects.handler
events:
- http:
path: api/redirects
method: get
socialTags:
handler: api/social-tags.handler
events:
- http:
path: api/social-tags
method: get
txtRecords:
handler: api/txt-records.handler
events:
- http:
path: api/txt-records
method: get
plugins:
# - serverless-webpack
- serverless-domain-manager
- serverless-offline
custom:
webpack:
webpackConfig: 'api/_common/aws-webpack.config.js'
includeModules: true
customDomain:
domainName: example.com
basePath: 'api'
stage: ${self:provider.stage}
createRoute53Record: true
serverless-offline:
prefix: ''
httpPort: 3000

4
src/pages/NotFound.tsx
View file

@ -20,6 +20,10 @@ const AboutContainer = styled.div`
color: ${colors.primary};
}
.im-drink { font-size: 6rem; }
header {
width: auto;
margin: 1rem;
}
`;
const HeaderLinkContainer = styled.nav`

39
src/pages/Results.tsx
View file

@ -149,14 +149,14 @@ const Results = (): JSX.Element => {
const urlTypeOnly = ['url'] as AddressType[]; // Many jobs only run with these address types
const api = '/api';
const api = process.env.REACT_APP_API_ENDPOINT || '/api'; // Where is the API hosted?
// Fetch and parse IP address for given URL
const [ipAddress, setIpAddress] = useMotherHook({
jobId: 'get-ip',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/find-url-ip?url=${address}`)
fetchRequest: () => fetch(`${api}/get-ip?url=${address}`)
.then(res => parseJson(res))
.then(res => res.ip),
});
@ -175,7 +175,7 @@ const Results = (): JSX.Element => {
jobId: 'ssl',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/ssl-check?url=${address}`).then((res) => parseJson(res)),
fetchRequest: () => fetch(`${api}/ssl?url=${address}`).then((res) => parseJson(res)),
});
// Fetch and parse cookies info
@ -183,7 +183,7 @@ const Results = (): JSX.Element => {
jobId: 'cookies',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/get-cookies?url=${address}`)
fetchRequest: () => fetch(`${api}/cookies?url=${address}`)
.then(res => parseJson(res))
.then(res => parseCookies(res.cookies)),
});
@ -193,9 +193,8 @@ const Results = (): JSX.Element => {
jobId: 'robots-txt',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/read-robots-txt?url=${address}`)
.then(res => res.text())
.then(res => parseRobotsTxt(res)),
fetchRequest: () => fetch(`${api}/robots-txt?url=${address}`)
.then(res => parseJson(res)),
});
// Fetch and parse headers
@ -203,7 +202,7 @@ const Results = (): JSX.Element => {
jobId: 'headers',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/get-headers?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/headers?url=${address}`).then(res => parseJson(res)),
});
// Fetch and parse DNS records
@ -211,7 +210,7 @@ const Results = (): JSX.Element => {
jobId: 'dns',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/get-dns?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/dns?url=${address}`).then(res => parseJson(res)),
});
// Fetch and parse Lighthouse performance data
@ -219,7 +218,7 @@ const Results = (): JSX.Element => {
jobId: 'quality',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/lighthouse-report?url=${address}`)
fetchRequest: () => fetch(`${api}/quality?url=${address}`)
.then(res => parseJson(res))
.then(res => res?.lighthouseResult || { error: 'No Data'}),
});
@ -249,7 +248,7 @@ const Results = (): JSX.Element => {
jobId: 'ports',
updateLoadingJobs,
addressInfo: { address: ipAddress, addressType: 'ipV4', expectedAddressTypes: ['ipV4', 'ipV6'] },
fetchRequest: () => fetch(`${api}/check-ports?url=${ipAddress}`)
fetchRequest: () => fetch(`${api}/ports?url=${ipAddress}`)
.then(res => parseJson(res)),
});
@ -268,7 +267,7 @@ const Results = (): JSX.Element => {
jobId: 'txt-records',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/get-txt?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/txt-records?url=${address}`).then(res => parseJson(res)),
});
// Fetches URL redirects
@ -276,7 +275,7 @@ const Results = (): JSX.Element => {
jobId: 'redirects',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/follow-redirects?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/redirects?url=${address}`).then(res => parseJson(res)),
});
// Get current status and response time of server
@ -284,7 +283,7 @@ const Results = (): JSX.Element => {
jobId: 'status',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/server-status?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/status?url=${address}`).then(res => parseJson(res)),
});
// Get current status and response time of server
@ -308,7 +307,7 @@ const Results = (): JSX.Element => {
jobId: 'carbon',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/get-carbon?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/carbon?url=${address}`).then(res => parseJson(res)),
});
// Check if a site is on the HSTS preload list
@ -316,7 +315,7 @@ const Results = (): JSX.Element => {
jobId: 'hsts',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/check-hsts?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/hsts?url=${address}`).then(res => parseJson(res)),
});
// Get a websites listed pages, from sitemap
@ -356,7 +355,7 @@ const Results = (): JSX.Element => {
jobId: 'features',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/site-features?url=${address}`)
fetchRequest: () => fetch(`${api}/features?url=${address}`)
.then(res => parseJson(res))
.then(res => {
if (res.Errors && res.Errors.length > 0) {
@ -371,7 +370,7 @@ const Results = (): JSX.Element => {
jobId: 'dnssec',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/dns-sec?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/dnssec?url=${address}`).then(res => parseJson(res)),
});
// Run a manual whois lookup on the domain
@ -379,7 +378,7 @@ const Results = (): JSX.Element => {
jobId: 'domain',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/whois-lookup?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/whois?url=${address}`).then(res => parseJson(res)),
});
// Get the DNS server(s) for a domain, and test DoH/DoT support
@ -395,7 +394,7 @@ const Results = (): JSX.Element => {
jobId: 'linked-pages',
updateLoadingJobs,
addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
fetchRequest: () => fetch(`${api}/content-links?url=${address}`).then(res => parseJson(res)),
fetchRequest: () => fetch(`${api}/linked-pages?url=${address}`).then(res => parseJson(res)),
});
// Get mail config for server, based on DNS records

2997
yarn.lock
View file

File diff suppressed because it is too large Load diff