dev1/backend/server2.js

750 lines
25 KiB
JavaScript
Executable File

import express from 'express';
import axios from 'axios';
import cors from 'cors';
import helmet from 'helmet'; // Import helmet for HTTP security headers
import dotenv from 'dotenv';
import xlsx from 'xlsx'; // Keep for CIP->SOC mapping only
import path from 'path';
import { fileURLToPath } from 'url';
import { open } from 'sqlite';
import sqlite3 from 'sqlite3';
import fs from 'fs';
import readline from 'readline';
// --- Basic file init ---
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootPath = path.resolve(__dirname, '..'); // Up one level
const env = process.env.NODE_ENV?.trim() || 'development';
const envPath = path.resolve(rootPath, `.env.${env}`);
dotenv.config({ path: envPath }); // Load .env
// Whitelist CORS
const allowedOrigins = [
'http://localhost:3000',
'http://34.16.120.118:3000',
'https://dev1.aptivaai.com'
];
// CIP->SOC mapping file
const mappingFilePath = '/home/jcoakley/aptiva-dev1-app/public/CIP_to_ONET_SOC.xlsx';
// Institution data
const institutionFilePath = path.resolve(rootPath, 'public/Institution_data.json');
// Create Express app
const app = express();
const PORT = process.env.PORT || 5001;
/**************************************************
* DB connections
**************************************************/
let db;
const initDB = async () => {
try {
db = await open({
filename: '/home/jcoakley/aptiva-dev1-app/salary_info.db',
driver: sqlite3.Database,
});
console.log('Connected to SQLite database.');
} catch (error) {
console.error('Error connecting to database:', error);
}
};
initDB();
let userProfileDb;
const initUserProfileDb = async () => {
try {
userProfileDb = await open({
filename: '/home/jcoakley/aptiva-dev1-app/user_profile.db',
driver: sqlite3.Database
});
console.log('Connected to user_profile.db.');
} catch (error) {
console.error('Error connecting to user_profile.db:', error);
}
};
initUserProfileDb();
/**************************************************
* Security, CORS, JSON Body
**************************************************/
app.use(
helmet({
contentSecurityPolicy: false,
crossOriginEmbedderPolicy: false,
})
);
app.use((req, res, next) => {
const origin = req.headers.origin;
if (origin && allowedOrigins.includes(origin)) {
res.setHeader('Access-Control-Allow-Origin', origin);
res.setHeader('Access-Control-Allow-Credentials', 'true');
res.setHeader(
'Access-Control-Allow-Headers',
'Authorization, Content-Type, Accept, Origin, X-Requested-With, Access-Control-Allow-Methods'
);
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
} else if (req.path.includes('Institution_data')) {
// For that JSON
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.setHeader(
'Access-Control-Allow-Headers',
'Content-Type, Accept, Origin, X-Requested-With'
);
} else {
// default
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.setHeader(
'Access-Control-Allow-Headers',
'Authorization, Content-Type, Accept, Origin, X-Requested-With'
);
}
if (req.method === 'OPTIONS') {
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Authorization, Content-Type');
return res.status(204).end();
}
next();
});
app.use(express.json());
app.use(express.static(path.join(__dirname, 'public')));
// For completeness
app.use((req, res, next) => {
next();
});
/**************************************************
* Load CIP->SOC mapping
**************************************************/
function loadMapping() {
try {
const workbook = xlsx.readFile(mappingFilePath);
const sheet = workbook.Sheets[workbook.SheetNames[0]];
return xlsx.utils.sheet_to_json(sheet);
} catch (error) {
console.error('Error reading CIP_to_ONET_SOC:', error);
return [];
}
}
const socToCipMapping = loadMapping();
if (socToCipMapping.length === 0) {
console.error("SOC to CIP mapping data is empty.");
}
/**************************************************
* Load single JSON with all states + US
* Replaces old GA-only approach
**************************************************/
// const projectionsFilePath = path.resolve(__dirname, '..', 'public', 'occprj.xlsx');
// const workbook = xlsx.readFile(projectionsFilePath);
// const sheet = workbook.Sheets['GAOccProj 2022-2032'];
// const projectionsData = xlsx.utils.sheet_to_json(sheet, { header: 1 });
const singleProjFile = path.resolve(__dirname, '..', 'public', 'economicproj.json');
let allProjections = [];
try {
const raw = fs.readFileSync(singleProjFile, 'utf8');
allProjections = JSON.parse(raw);
console.log(`Loaded ${allProjections.length} rows from economicproj.json`);
} catch (err) {
console.error('Error reading economicproj.json:', err);
}
/**************************************************
* ONet routes, CIP routes, distance routes, etc.
**************************************************/
// O*Net interest questions
app.get('/api/onet/questions', async (req, res) => {
const { start, end } = req.query;
if (!start || !end) {
return res.status(400).json({ error: 'Start and end parameters are required' });
}
try {
const questions = [];
let currentStart = parseInt(start, 10);
let currentEnd = parseInt(end, 10);
while (currentStart <= currentEnd) {
const response = await axios.get(
`https://services.onetcenter.org/ws/mnm/interestprofiler/questions?start=${currentStart}&end=${Math.min(
currentEnd,
currentStart + 11
)}`,
{
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD,
},
}
);
if (response.data.question && Array.isArray(response.data.question)) {
questions.push(...response.data.question);
}
const nextLink = response.data.link?.find((link) => link.rel === 'next');
if (nextLink) {
const nextParams = new URLSearchParams(nextLink.href.split('?')[1]);
currentStart = parseInt(nextParams.get('start'), 10);
currentEnd = parseInt(nextParams.get('end'), 10);
} else {
break;
}
}
res.status(200).json({ questions });
} catch (error) {
console.error('Error fetching O*Net questions:', error.message);
res.status(500).json({ error: 'Failed to fetch O*Net questions' });
}
});
// geocode
async function geocodeZipCode(zipCode) {
const apiKey = process.env.GOOGLE_MAPS_API_KEY;
if (!apiKey) {
console.error('Google Maps API Key missing');
}
try {
const geocodeUrl = `https://maps.googleapis.com/maps/api/geocode/json?address=${encodeURIComponent(
zipCode
)}&components=country:US&key=${apiKey}`;
const response = await axios.get(geocodeUrl);
if (response.data.status === 'OK' && response.data.results.length > 0) {
return response.data.results[0].geometry.location;
} else {
throw new Error('Geocoding failed');
}
} catch (error) {
console.error('Error geocoding ZIP code:', error.message);
return null;
}
}
// Distance
app.post('/api/maps/distance', async (req, res) => {
const { userZipcode, destinations } = req.body;
if (!userZipcode || !destinations) {
console.error('Missing required parameters:', { userZipcode, destinations });
return res.status(400).json({ error: 'User ZIP code and destination are required.' });
}
try {
const googleMapsApiKey = process.env.GOOGLE_MAPS_API_KEY;
const userLocation = await geocodeZipCode(userZipcode);
if (!userLocation) {
return res.status(400).json({ error: 'Unable to geocode user ZIP code.' });
}
const origins = `${userLocation.lat},${userLocation.lng}`;
const distanceUrl = `https://maps.googleapis.com/maps/api/distancematrix/json?origins=${origins}&destinations=${encodeURIComponent(
destinations
)}&units=imperial&key=${googleMapsApiKey}`;
const distanceResponse = await axios.get(distanceUrl);
if (distanceResponse.data.status !== 'OK') {
return res.status(500).json({ error: 'Error fetching distance from Google Maps API' });
}
const { distance, duration } = distanceResponse.data.rows[0].elements[0];
res.json({ distance: distance.text, duration: duration.text });
} catch (error) {
console.error('Error during distance calculation:', error);
res.status(500).json({ error: 'Internal server error', details: error.message });
}
});
// ONet submission
app.post('/api/onet/submit_answers', async (req, res) => {
console.log('POST /api/onet/submit_answers hit');
const { answers } = req.body;
if (!answers || answers.length !== 60) {
console.error('Invalid answers:', answers);
return res.status(400).json({ error: 'Answers must be 60 chars long.' });
}
try {
const careerUrl = `https://services.onetcenter.org/ws/mnm/interestprofiler/careers?answers=${answers}&start=1&end=1000`;
const resultsUrl = `https://services.onetcenter.org/ws/mnm/interestprofiler/results?answers=${answers}`;
// career suggestions
const careerResponse = await axios.get(careerUrl, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD,
},
headers: { Accept: 'application/json' },
});
// RIASEC
const resultsResponse = await axios.get(resultsUrl, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD,
},
headers: { Accept: 'application/json' },
});
const careerSuggestions = careerResponse.data.career || [];
const riaSecScores = resultsResponse.data.result || [];
// filter out lower ed
const filtered = filterHigherEducationCareers(careerSuggestions);
res.status(200).json({
careers: filtered,
riaSecScores,
});
} catch (error) {
console.error('Error in ONet API:', error.response?.data || error.message);
res.status(500).json({
error: 'Failed to fetch data from ONet',
details: error.response?.data || error.message
});
}
});
function filterHigherEducationCareers(careers) {
return careers
.map((c) => {
const edu = c.education;
if (!['No formal education', 'High school', 'Some college, no degree'].includes(edu)) {
return {
href: c.href,
fit: c.fit,
code: c.code,
title: c.title,
tags: c.tags
};
}
return null;
})
.filter(Boolean);
}
// ONet career details
app.get('/api/onet/career-details/:socCode', async (req, res) => {
const { socCode } = req.params;
if (!socCode) {
return res.status(400).json({ error: 'SOC code is required' });
}
try {
const response = await axios.get(`https://services.onetcenter.org/ws/mnm/careers/${socCode}`, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD
},
headers: { Accept: 'application/json' },
});
res.status(200).json(response.data);
} catch (err) {
console.error('Error fetching career details:', err);
res.status(500).json({ error: 'Failed to fetch career details' });
}
});
// ONet career description
app.get('/api/onet/career-description/:socCode', async (req, res) => {
const { socCode } = req.params;
if (!socCode) {
return res.status(400).json({ error: 'SOC Code is required' });
}
try {
const response = await axios.get(`https://services.onetcenter.org/ws/mnm/careers/${socCode}`, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD
},
headers: { Accept: 'application/json' },
});
if (response.data && response.data.title) {
const { what_they_do, on_the_job } = response.data;
const tasks = on_the_job?.task || [];
return res.json({
description: what_they_do || 'No description available',
tasks: tasks.length ? tasks : ['No tasks available']
});
}
return res.status(404).json({ error: 'Career not found for SOC code' });
} catch (error) {
console.error('Error in career-description route:', error);
res.status(500).json({ error: 'Failed to fetch career description' });
}
});
// CIP route
app.get('/api/cip/:socCode', (req, res) => {
const { socCode } = req.params;
console.log(`Received SOC Code: ${socCode.trim()}`);
for (let row of socToCipMapping) {
const mappedSOC = row['O*NET-SOC 2019 Code']?.trim();
if (mappedSOC === socCode.trim()) {
console.log('Found CIP code:', row['2020 CIP Code']);
return res.json({ cipCode: row['2020 CIP Code'] });
}
}
console.error('SOC code not found in mapping:', socCode);
res.status(404).json({ error: 'CIP code not found' });
});
/**************************************************
* Single schools / tuition / etc. routes
**************************************************/
app.get('/api/schools', (req, res) => {
// 1) Read `cipCodes` from query (comma-separated string)
const { cipCodes } = req.query;
if (!cipCodes ) {
return res.status(400).json({ error: 'cipCodes (comma-separated) and state are required.' });
}
try {
// 2) Convert `cipCodes` to array => e.g. "1101,1103,1104" => ["1101","1103","1104"]
const cipArray = cipCodes.split(',').map((c) => c.trim()).filter(Boolean);
if (cipArray.length === 0) {
return res.status(400).json({ error: 'No valid CIP codes were provided.' });
}
// 3) Load your raw schools data
let schoolsData = [];
try {
const rawData = fs.readFileSync(institutionFilePath, 'utf8');
schoolsData = JSON.parse(rawData);
} catch (err) {
console.error('Error parsing institution data:', err.message);
return res.status(500).json({ error: 'Failed to load schools data.' });
}
// 4) Filter any school whose CIP code matches ANY of the CIP codes in the array
// Convert the school's CIP code the same way you do in your old logic (remove dot, slice, etc.)
const filtered = schoolsData.filter((s) => {
const scip = s['CIPCODE']?.toString().replace('.', '').slice(0, 4);
return cipArray.some((cip) => scip.startsWith(cip));
});
// 5) (Optional) Deduplicate if you suspect overlaps among CIP codes.
// E.g. by a “UNITID” or unique property:
const uniqueMap = new Map();
for (const school of filtered) {
const key = school.UNITID || school.INSTNM; // pick your unique field
if (!uniqueMap.has(key)) {
uniqueMap.set(key, school);
}
}
const deduped = Array.from(uniqueMap.values());
console.log('Unique schools found:', deduped.length);
res.json(deduped);
} catch (err) {
console.error('Error reading Institution data:', err.message);
res.status(500).json({ error: 'Failed to load schools data.' });
}
});
// tuition
app.get('/api/tuition', (req, res) => {
const { cipCodes, state } = req.query;
if (!cipCodes || !state) {
return res.status(400).json({ error: 'cipCodes and state are required.' });
}
try {
const raw = fs.readFileSync(institutionFilePath, 'utf8');
const schoolsData = JSON.parse(raw);
const cipArray = cipCodes.split(',').map((c) => c.trim()).filter(Boolean);
if (!cipArray.length) {
return res.status(400).json({ error: 'No valid CIP codes.' });
}
// Filter logic
const filtered = schoolsData.filter((school) => {
const cval = school['CIPCODE']?.toString().replace(/\./g, '').slice(0, 4);
const sVal = school['State']?.toUpperCase().trim();
// Check if cval starts with ANY CIP in cipArray
const matchesCip = cipArray.some((cip) => cval.startsWith(cip));
const matchesState = sVal === state.toUpperCase().trim();
return matchesCip && matchesState;
});
// Optionally deduplicate by UNITID
const uniqueMap = new Map();
for (const school of filtered) {
const key = school.UNITID || school.INSTNM; // or something else unique
if (!uniqueMap.has(key)) {
uniqueMap.set(key, school);
}
}
const deduped = Array.from(uniqueMap.values());
console.log('Filtered Tuition Data Count:', deduped.length);
res.json(deduped);
} catch (err) {
console.error('Error reading tuition data:', err.message);
res.status(500).json({ error: 'Failed to load tuition data.' });
}
});
/**************************************************
* SINGLE route for projections from economicproj.json
**************************************************/
// Remove old GA Excel approach; unify with single JSON approach
app.get('/api/projections/:socCode', (req, res) => {
const { socCode } = req.params;
const { state } = req.query;
console.log('Projections request for', socCode, ' state=', state);
if (!socCode) {
return res.status(400).json({ error: 'SOC Code is required.' });
}
// If no ?state=, default to "United States"
const areaName = state ? state.trim() : 'United States';
// Find the row for the requested area
const rowForState = allProjections.find(
(row) =>
row['Occupation Code'] === socCode.trim() &&
row['Area Name']?.toLowerCase() === areaName.toLowerCase()
);
// Also find the row for "United States"
const rowForUS = allProjections.find(
(row) =>
row['Occupation Code'] === socCode.trim() &&
row['Area Name']?.toLowerCase() === 'united states'
);
if (!rowForState && !rowForUS) {
return res.status(404).json({ error: 'No projections found for this SOC + area.' });
}
function formatRow(r) {
if (!r) return null;
return {
area: r['Area Name'],
baseYear: r['Base Year'],
base: r['Base'],
projectedYear: r['Projected Year'],
projection: r['Projection'],
change: r['Change'],
percentChange: r['Percent Change'],
annualOpenings: r['Average Annual Openings'],
occupationName: r['Occupation Name']
};
}
const result = {
state: formatRow(rowForState),
national: formatRow(rowForUS)
};
return res.json(result);
});
/**************************************************
* Salary route
**************************************************/
app.get('/api/salary', async (req, res) => {
const { socCode, area } = req.query;
console.log('Received /api/salary request:', { socCode, area });
if (!socCode) {
return res.status(400).json({ error: 'SOC Code is required' });
}
// Query for regional
const regionalQuery = `
SELECT A_PCT10 AS regional_PCT10, A_PCT25 AS regional_PCT25,
A_MEDIAN AS regional_MEDIAN, A_PCT75 AS regional_PCT75,
A_PCT90 AS regional_PCT90
FROM salary_data
WHERE OCC_CODE = ? AND AREA_TITLE = ?
`;
// Query for national
const nationalQuery = `
SELECT A_PCT10 AS national_PCT10, A_PCT25 AS national_PCT25,
A_MEDIAN AS national_MEDIAN, A_PCT75 AS national_PCT75,
A_PCT90 AS national_PCT90
FROM salary_data
WHERE OCC_CODE = ? AND AREA_TITLE = 'U.S.'
`;
try {
let regionalRow = null;
let nationalRow = null;
if (area) {
regionalRow = await db.get(regionalQuery, [socCode, area]);
}
nationalRow = await db.get(nationalQuery, [socCode]);
if (!regionalRow && !nationalRow) {
console.log('No salary data found for:', { socCode, area });
return res.status(404).json({ error: 'No salary data found' });
}
const salaryData = {
regional: regionalRow || {},
national: nationalRow || {}
};
console.log('Salary data retrieved:', salaryData);
res.json(salaryData);
} catch (error) {
console.error('Error executing salary query:', error.message);
res.status(500).json({ error: 'Failed to fetch salary data' });
}
});
/**************************************************
* job-zones route
**************************************************/
app.post('/api/job-zones', async (req, res) => {
const { socCodes } = req.body;
if (!socCodes || !Array.isArray(socCodes) || socCodes.length === 0) {
return res.status(400).json({ error: "SOC Codes are required." });
}
try {
// Format them
const formattedSocCodes = socCodes.map((code) => {
let cleaned = code.trim().replace(/\./g, '');
if (!cleaned.includes('-') && cleaned.length === 6) {
cleaned = cleaned.slice(0, 2) + '-' + cleaned.slice(2);
}
return cleaned.slice(0, 7);
});
const placeholders = formattedSocCodes.map(() => '?').join(',');
const q = `
SELECT OCC_CODE, JOB_ZONE,
A_MEDIAN, A_PCT10, A_PCT25, A_PCT75
FROM salary_data
WHERE OCC_CODE IN (${placeholders})
`;
const rows = await db.all(q, formattedSocCodes);
console.log("Salary Data Query Results:", rows);
const jobZoneMapping = rows.reduce((acc, row) => {
const isMissing = [row.A_MEDIAN, row.A_PCT10, row.A_PCT25, row.A_PCT75]
.some((v) => !v || v === '#' || v === '*');
acc[row.OCC_CODE] = {
job_zone: row.JOB_ZONE,
limited_data: isMissing ? 1 : 0
};
return acc;
}, {});
console.log("Job Zone & Limited Data:", jobZoneMapping);
res.json(jobZoneMapping);
} catch (error) {
console.error("Error fetching job zones:", error);
res.status(500).json({ error: "Failed to fetch job zones." });
}
});
/**************************************************
* O*NET Skills route
**************************************************/
app.get('/api/skills/:socCode', async (req, res) => {
const { socCode } = req.params;
if (!socCode) {
return res.status(400).json({ error: 'SOC code is required' });
}
console.log(`Fetching O*NET skills for SOC Code: ${socCode}`);
try {
// 1) Build the O*NET API URL
const onetUrl = `https://services.onetcenter.org/ws/mnm/careers/${socCode}/skills`;
// 2) Call O*NET with Basic Auth
const response = await axios.get(onetUrl, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD
},
headers: { Accept: 'application/json' }
});
const data = response.data || {};
// 3) O*NET returns:
// {
// "code": "17-1011.00",
// "group": [
// {
// "title": { "id": "2.A", "name": "Basic Skills" },
// "element": [
// { "id": "2.A.1.a", "name": "reading work related information" },
// ...
// ]
// },
// ...
// ]
// }
// Instead of data.characteristic, parse data.group
const groups = data.group || [];
// 4) Flatten out the group[].element[] into a single skills array
const skills = [];
groups.forEach((groupItem) => {
const groupName = groupItem?.title?.name || 'Unknown Group';
if (Array.isArray(groupItem.element)) {
groupItem.element.forEach((elem) => {
// Each "element" is a skill with an id and a name
skills.push({
groupName,
skillId: elem.id,
skillName: elem.name
});
});
}
});
res.json({ skills });
} catch (error) {
console.error('Error fetching O*NET skills:', error.message);
if (error.response) {
console.error('O*NET error status:', error.response.status);
console.error('O*NET error data:', error.response.data);
} else if (error.request) {
// The request was made but no response was received
console.error('No response received from O*NET. Possibly a network or credentials error.');
console.error('Axios error.request:', error.request);
} else {
// Something else happened in setting up the request
console.error('Request setup error:', error.message);
}
return res.status(500).json({ error: 'Failed to fetch O*NET skills' });
}
});
/**************************************************
* user-profile by ID route
**************************************************/
app.get('/api/user-profile/:id', (req, res) => {
const { id } = req.params;
if (!id) return res.status(400).json({ error: 'Profile ID is required' });
const query = `SELECT area, zipcode FROM user_profile WHERE id = ?`;
db.get(query, [id], (err, row) => {
if (err) {
console.error('Error fetching user profile:', err.message);
return res.status(500).json({ error: 'Failed to fetch user profile' });
}
if (!row) {
return res.status(404).json({ error: 'Profile not found' });
}
res.json({ area: row.area, zipcode: row.zipcode });
});
});
/**************************************************
* Start the Express server
**************************************************/
app.listen(PORT, () => {
console.log(`Server running on https://34.16.120.118:${PORT}`);
});