jcoakley/dev1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrated server2 back to SQLite

Browse Source
This commit is contained in:
Josh 2025-05-20 15:43:41 +00:00
parent fc4e9da50b
commit 883f0762a3
8 changed files with 3915799 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91417
Abilities.txt Normal file
View File

File diff suppressed because it is too large Load Diff

58015
Knowledge.txt Normal file
View File

File diff suppressed because it is too large Load Diff

56653
Skills.txt Normal file
View File

File diff suppressed because it is too large Load Diff

186
backend/server2.js
View File

@ -1,20 +1,20 @@
/**************************************************
* server2.js - MySQL version
* server2.js - Reverted to SQLite version
**************************************************/
import express from 'express';
import axios from 'axios';
import cors from 'cors';
import helmet from 'helmet'; // For HTTP security headers
import dotenv from 'dotenv';
import xlsx from 'xlsx'; // For CIP->SOC mapping only
import xlsx from 'xlsx'; // Keep for CIP->SOC mapping only
import path from 'path';
import { fileURLToPath } from 'url';
import { open } from 'sqlite';
import sqlite3 from 'sqlite3';
import fs from 'fs';
import readline from 'readline';
// ********** NEW: use mysql2/promise for async/await queries **********
import mysql from 'mysql2/promise';
// --- Basic file init ---
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@ -28,40 +28,50 @@ dotenv.config({ path: envPath }); // Load .env
const allowedOrigins = [
'http://localhost:3000',
'http://34.16.120.118:3000',
'https://dev1.aptivaai.com'
'https://dev1.aptivaai.com',
];
// CIP->SOC mapping file
const mappingFilePath = '/home/jcoakley/aptiva-dev1-app/public/CIP_to_ONET_SOC.xlsx';
// Institution data
const institutionFilePath = path.resolve(rootPath, 'public/Institution_data.json');
// ********** CREATE TWO POOLS FOR TWO DATABASES **********
// salary_data_db => poolSalary
// user_profile_db => poolProfile
const poolSalary = mysql.createPool({
host: process.env.DB_HOST,
port: process.env.DB_PORT || 3306,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: 'salary_data_db',
connectionLimit: 10
});
const poolProfile = mysql.createPool({
host: process.env.DB_HOST,
port: process.env.DB_PORT || 3306,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: 'user_profile_db',
connectionLimit: 10
});
const institutionFilePath = path.resolve(rootPath, 'public', 'Institution_data.json');
// Create Express app
const app = express();
const PORT = process.env.PORT || 5001;
/**************************************************
* DB connections
**************************************************/
let db;
const initDB = async () => {
try {
db = await open({
filename: '/home/jcoakley/aptiva-dev1-app/salary_info.db',
driver: sqlite3.Database,
});
console.log('Connected to SQLite salary_info.db');
} catch (error) {
console.error('Error connecting to salary_info.db:', error);
}
};
initDB();
let userProfileDb;
const initUserProfileDb = async () => {
try {
userProfileDb = await open({
filename: '/home/jcoakley/aptiva-dev1-app/user_profile.db',
driver: sqlite3.Database,
});
console.log('Connected to user_profile.db.');
} catch (error) {
console.error('Error connecting to user_profile.db:', error);
}
};
initUserProfileDb();
/**************************************************
* Security, CORS, JSON Body
**************************************************/
@ -130,11 +140,12 @@ function loadMapping() {
}
const socToCipMapping = loadMapping();
if (socToCipMapping.length === 0) {
console.error("SOC to CIP mapping data is empty.");
console.error('SOC to CIP mapping data is empty.');
}
/**************************************************
* Load single JSON with all states + US
* Replaces old GA-only approach
**************************************************/
const singleProjFile = path.resolve(__dirname, '..', 'public', 'economicproj.json');
let allProjections = [];
@ -147,7 +158,7 @@ try {
}
/**************************************************
* ONet routes, CIP routes, distance routes, etc.
* O*Net routes, CIP routes, distance routes, etc.
**************************************************/
// O*Net interest questions
@ -193,7 +204,7 @@ app.get('/api/onet/questions', async (req, res) => {
}
});
// Geocode
// geocode
async function geocodeZipCode(zipCode) {
const apiKey = process.env.GOOGLE_MAPS_API_KEY;
if (!apiKey) {
@ -287,7 +298,7 @@ app.post('/api/onet/submit_answers', async (req, res) => {
console.error('Error in ONet API:', error.response?.data || error.message);
res.status(500).json({
error: 'Failed to fetch data from ONet',
details: error.response?.data || error.message
details: error.response?.data || error.message,
});
}
});
@ -301,7 +312,7 @@ function filterHigherEducationCareers(careers) {
fit: c.fit,
code: c.code,
title: c.title,
tags: c.tags
tags: c.tags,
};
}
return null;
@ -318,8 +329,8 @@ app.get('/api/onet/career-details/:socCode', async (req, res) => {
try {
const response = await axios.get(`https://services.onetcenter.org/ws/mnm/careers/${socCode}`, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD
username: process.env.ONet_USERNAME,
password: process.env.ONET_PASSWORD,
},
headers: { Accept: 'application/json' },
});
@ -340,7 +351,7 @@ app.get('/api/onet/career-description/:socCode', async (req, res) => {
const response = await axios.get(`https://services.onetcenter.org/ws/mnm/careers/${socCode}`, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD
password: process.env.ONET_PASSWORD,
},
headers: { Accept: 'application/json' },
});
@ -349,7 +360,7 @@ app.get('/api/onet/career-description/:socCode', async (req, res) => {
const tasks = on_the_job?.task || [];
return res.json({
description: what_they_do || 'No description available',
tasks: tasks.length ? tasks : ['No tasks available']
tasks: tasks.length ? tasks : ['No tasks available'],
});
}
return res.status(404).json({ error: 'Career not found for SOC code' });
@ -383,12 +394,17 @@ app.get('/api/schools', (req, res) => {
const { cipCodes } = req.query;
if (!cipCodes) {
return res.status(400).json({ error: 'cipCodes (comma-separated) and state are required.' });
return res
.status(400)
.json({ error: 'cipCodes (comma-separated) and state are required.' });
}
try {
// 2) Convert `cipCodes` to array => e.g. "1101,1103,1104" => ["1101","1103","1104"]
const cipArray = cipCodes.split(',').map((c) => c.trim()).filter(Boolean);
const cipArray = cipCodes
.split(',')
.map((c) => c.trim())
.filter(Boolean);
if (cipArray.length === 0) {
return res.status(400).json({ error: 'No valid CIP codes were provided.' });
}
@ -404,13 +420,12 @@ app.get('/api/schools', (req, res) => {
}
// 4) Filter any school whose CIP code matches ANY of the CIP codes in the array
// Convert the school's CIP code the same way you do in your old logic (remove dot, slice, etc.)
const filtered = schoolsData.filter((s) => {
const scip = s['CIPCODE']?.toString().replace('.', '').slice(0, 4);
return cipArray.some((cip) => scip.startsWith(cip));
});
// 5) (Optional) Deduplicate if you suspect overlaps among CIP codes.
// 5) (Optional) Deduplicate
const uniqueMap = new Map();
for (const school of filtered) {
const key = school.UNITID || school.INSTNM; // pick your unique field
@ -439,7 +454,10 @@ app.get('/api/tuition', (req, res) => {
const raw = fs.readFileSync(institutionFilePath, 'utf8');
const schoolsData = JSON.parse(raw);
const cipArray = cipCodes.split(',').map((c) => c.trim()).filter(Boolean);
const cipArray = cipCodes
.split(',')
.map((c) => c.trim())
.filter(Boolean);
if (!cipArray.length) {
return res.status(400).json({ error: 'No valid CIP codes.' });
}
@ -505,7 +523,9 @@ app.get('/api/projections/:socCode', (req, res) => {
);
if (!rowForState && !rowForUS) {
return res.status(404).json({ error: 'No projections found for this SOC + area.' });
return res
.status(404)
.json({ error: 'No projections found for this SOC + area.' });
}
function formatRow(r) {
@ -519,20 +539,20 @@ app.get('/api/projections/:socCode', (req, res) => {
change: r['Change'],
percentChange: r['Percent Change'],
annualOpenings: r['Average Annual Openings'],
occupationName: r['Occupation Name']
occupationName: r['Occupation Name'],
};
}
const result = {
state: formatRow(rowForState),
national: formatRow(rowForUS)
national: formatRow(rowForUS),
};
return res.json(result);
});
/**************************************************
* Salary route (uses poolSalary)
* Salary route
**************************************************/
app.get('/api/salary', async (req, res) => {
const { socCode, area } = req.query;
@ -541,35 +561,33 @@ app.get('/api/salary', async (req, res) => {
return res.status(400).json({ error: 'SOC Code is required' });
}
// Query for regional
const regionalQuery = `
SELECT A_PCT10 AS regional_PCT10, A_PCT25 AS regional_PCT25,
A_MEDIAN AS regional_MEDIAN, A_PCT75 AS regional_PCT75,
SELECT A_PCT10 AS regional_PCT10,
A_PCT25 AS regional_PCT25,
A_MEDIAN AS regional_MEDIAN,
A_PCT75 AS regional_PCT75,
A_PCT90 AS regional_PCT90
FROM salary_data
WHERE OCC_CODE = ? AND AREA_TITLE = ?
`;
// Query for national
const nationalQuery = `
SELECT A_PCT10 AS national_PCT10, A_PCT25 AS national_PCT25,
A_MEDIAN AS national_MEDIAN, A_PCT75 AS national_PCT75,
SELECT A_PCT10 AS national_PCT10,
A_PCT25 AS national_PCT25,
A_MEDIAN AS national_MEDIAN,
A_PCT75 AS national_PCT75,
A_PCT90 AS national_PCT90
FROM salary_data
WHERE OCC_CODE = ? AND AREA_TITLE = 'U.S.'
`;
try {
let regionalRow = null;
let nationalRow = null;
// If area is provided, fetch regional
if (area) {
const [regRows] = await poolSalary.query(regionalQuery, [socCode, area]);
regionalRow = regRows.length ? regRows[0] : null;
regionalRow = await db.get(regionalQuery, [socCode, area]);
}
// Always fetch national
const [natRows] = await poolSalary.query(nationalQuery, [socCode]);
nationalRow = natRows.length ? natRows[0] : null;
nationalRow = await db.get(nationalQuery, [socCode]);
if (!regionalRow && !nationalRow) {
console.log('No salary data found for:', { socCode, area });
@ -577,7 +595,7 @@ app.get('/api/salary', async (req, res) => {
}
const salaryData = {
regional: regionalRow || {},
national: nationalRow || {}
national: nationalRow || {},
};
console.log('Salary data retrieved:', salaryData);
res.json(salaryData);
@ -588,7 +606,7 @@ app.get('/api/salary', async (req, res) => {
});
/**************************************************
* job-zones route (uses poolSalary)
* job-zones route
**************************************************/
app.post('/api/job-zones', async (req, res) => {
const { socCodes } = req.body;
@ -605,16 +623,17 @@ app.post('/api/job-zones', async (req, res) => {
return cleaned.slice(0, 7);
});
const placeholders = formattedSocCodes.map(() => '?').join(',');
const q = `
SELECT OCC_CODE, JOB_ZONE,
A_MEDIAN, A_PCT10, A_PCT25, A_PCT75
SELECT OCC_CODE,
JOB_ZONE,
A_MEDIAN,
A_PCT10,
A_PCT25,
A_PCT75
FROM salary_data
WHERE OCC_CODE IN (${placeholders})
`;
// Use spread operator for the array
const [rows] = await poolSalary.query(q, [...formattedSocCodes]);
const rows = await db.all(q, formattedSocCodes);
console.log('Salary Data Query Results:', rows);
const jobZoneMapping = rows.reduce((acc, row) => {
@ -623,11 +642,10 @@ app.post('/api/job-zones', async (req, res) => {
);
acc[row.OCC_CODE] = {
job_zone: row.JOB_ZONE,
limited_data: isMissing ? 1 : 0
limited_data: isMissing ? 1 : 0,
};
return acc;
}, {});
console.log('Job Zone & Limited Data:', jobZoneMapping);
res.json(jobZoneMapping);
} catch (error) {
@ -655,9 +673,9 @@ app.get('/api/skills/:socCode', async (req, res) => {
const response = await axios.get(onetUrl, {
auth: {
username: process.env.ONET_USERNAME,
password: process.env.ONET_PASSWORD
password: process.env.ONET_PASSWORD,
},
headers: { Accept: 'application/json' }
headers: { Accept: 'application/json' },
});
const data = response.data || {};
@ -668,14 +686,12 @@ app.get('/api/skills/:socCode', async (req, res) => {
// Flatten out the group[].element[] into a single skills array
groups.forEach((groupItem) => {
const groupName = groupItem?.title?.name || 'Unknown Group';
if (Array.isArray(groupItem.element)) {
groupItem.element.forEach((elem) => {
// Each "element" is a skill with an id and a name
skills.push({
groupName,
skillId: elem.id,
skillName: elem.name
skillName: elem.name,
});
});
}
@ -700,25 +716,23 @@ app.get('/api/skills/:socCode', async (req, res) => {
});
/**************************************************
* user-profile by ID route (uses poolProfile)
* user-profile by ID route
**************************************************/
app.get('/api/user-profile/:id', async (req, res) => {
app.get('/api/user-profile/:id', (req, res) => {
const { id } = req.params;
if (!id) return res.status(400).json({ error: 'Profile ID is required' });
const query = `SELECT area, zipcode FROM user_profile WHERE id = ?`;
try {
const [rows] = await poolProfile.query(query, [id]);
if (!rows.length) {
userProfileDb.get(query, [id], (err, row) => {
if (err) {
console.error('Error fetching user profile:', err.message);
return res.status(500).json({ error: 'Failed to fetch user profile' });
}
if (!row) {
return res.status(404).json({ error: 'Profile not found' });
}
const row = rows[0];
res.json({ area: row.area, zipcode: row.zipcode });
} catch (err) {
console.error('Error fetching user profile:', err.message);
return res.status(500).json({ error: 'Failed to fetch user profile' });
}
});
});
/**************************************************

37
merge_ksa.js Normal file
View File

@ -0,0 +1,37 @@
// merge_ksa.js
import fs from 'fs';
import path from 'path';
import parseLine from './parseLine.js'; // your parseLine from above
function parseTextFile(filepath, ksaType) {
const raw = fs.readFileSync(filepath, 'utf8');
const lines = raw.split('\n').map(line => line.trim()).filter(Boolean);
// convert each line to an object with 15 columns
// then attach ksa_type
return lines.map((line) => {
const parsed = parseLine(line);
return {
...parsed,
ksa_type: ksaType
};
});
}
function main() {
// Adjust to your real file paths
const knowledgeFile = path.resolve('Knowledge.txt');
const skillsFile = path.resolve('Skills.txt');
const abilitiesFile = path.resolve('Abilities.txt');
const knowledgeData = parseTextFile(knowledgeFile, 'Knowledge');
const skillsData = parseTextFile(skillsFile, 'Skill');
const abilitiesData = parseTextFile(abilitiesFile, 'Ability');
const merged = [...knowledgeData, ...skillsData, ...abilitiesData];
fs.writeFileSync('ksa_data.json', JSON.stringify(merged, null, 2), 'utf8');
console.log(`Wrote ${merged.length} lines (incl. headers) to ksa_data.json`);
}
main();

44
parseLine.js Normal file
View File

@ -0,0 +1,44 @@
// parseLine.js
function parseLine(line) {
// Split on tabs
const cols = line.split(/\t/).map((c) => c.trim());
// We expect 15 columns, but we won't skip lines if some are missing or extra.
// We'll fill them with "" if not present, to preserve all data
const col0 = cols[0] || ""; // O*NET-SOC Code
const col1 = cols[1] || ""; // Title
const col2 = cols[2] || ""; // Element ID
const col3 = cols[3] || ""; // Element Name
const col4 = cols[4] || ""; // Scale ID
const col5 = cols[5] || ""; // Scale Name
const col6 = cols[6] || ""; // Data Value
const col7 = cols[7] || ""; // N
const col8 = cols[8] || ""; // Standard Error
const col9 = cols[9] || ""; // Lower CI Bound
const col10 = cols[10] || ""; // Upper CI Bound
const col11 = cols[11] || ""; // Recommend Suppress
const col12 = cols[12] || ""; // Not Relevant
const col13 = cols[13] || ""; // Date
const col14 = cols[14] || ""; // Domain Source
// Return an object with keys matching your definitions
return {
onetSocCode: col0,
title: col1,
elementID: col2,
elementName: col3,
scaleID: col4,
scaleName: col5,
dataValue: col6,
n: col7,
standardError: col8,
lowerCI: col9,
upperCI: col10,
recommendSuppress: col11,
notRelevant: col12,
date: col13,
domainSource: col14
};
}
export default parseLine;

3709532
public/ksa_data.json Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/components/CareerExplorer.js
View File

@ -629,7 +629,7 @@ function CareerExplorer() {
className="bg-green-600 text-white"
onClick={() => handleSelectForEducation(career)}
>
Search for Education
Plan your Education/Skills
</Button>
</td>
</tr>