Added GPT=produced KSA when not found.

backend/server3.js

@@ -15,6 +15,7 @@ import { v4 as uuidv4 } from 'uuid';
 import pkg from 'pdfjs-dist';
 import mysql from 'mysql2/promise'; // <-- MySQL instead of SQLite
 import OpenAI from 'openai';
+import Fuse from 'fuse.js';
 
 // Basic file init
 const __filename = fileURLToPath(import.meta.url);
@@ -2052,6 +2053,247 @@ app.delete('/api/premium/milestone-impacts/:impactId', authenticatePremiumUser,
   }
 });
 
+let onetKsaData = [];      // entire array from ksa_data.json
+let allKsaNames = [];      // an array of unique KSA names (for fuzzy matching)
+
+(async function loadKsaJson() {
+  try {
+    const filePath = path.join(__dirname, '..', 'public', 'ksa_data.json');
+    const raw = await fs.readFile(filePath, 'utf8');
+    onetKsaData = JSON.parse(raw);
+
+    // Build a set of unique KSA names for fuzzy search
+    const nameSet = new Set();
+    for (const row of onetKsaData) {
+      nameSet.add(row.elementName);
+    }
+    allKsaNames = Array.from(nameSet);
+    console.log(`Loaded ksa_data.json with ${onetKsaData.length} rows; ${allKsaNames.length} unique KSA names.`);
+  } catch (err) {
+    console.error('Error loading ksa_data.json:', err);
+  }
+})();
+
+// 2) Create fuzzy search index
+let fuse = null;
+function initFuzzySearch() {
+  if (!fuse) {
+    fuse = new Fuse(allKsaNames, {
+      includeScore: true,
+      threshold: 0.3, // adjust to your preference
+    });
+  }
+}
+
+function fuzzyMatchKsaName(name) {
+  if (!fuse) initFuzzySearch();
+  const results = fuse.search(name);
+  if (!results.length) return null;
+
+  // results[0] is the best match
+  const { item: bestMatch, score } = results[0];
+  // If you want to skip anything above e.g. 0.5 score, do:
+  if (score > 0.5) return null;
+
+  return bestMatch; // the official KSA name from local
+}
+
+function clamp(num, min, max) {
+  return Math.max(min, Math.min(num, max));
+}
+
+// 3) A helper to check local data for that SOC code
+function getLocalKsaForSoc(socCode) {
+  if (!onetKsaData.length) return [];
+  return onetKsaData.filter((r) => r.onetSocCode === socCode);
+}
+
+// 4) ChatGPT call
+async function fetchKsaFromOpenAI(socCode, careerTitle) {
+  const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+
+  // 1. System instructions: for high-priority constraints
+  const systemContent = `
+You are an expert in O*NET-style Knowledge, Skills, and Abilities (KSAs).
+Always produce a thorough KSA list for the career described.
+Carefully follow instructions about minimum counts per category.
+No additional commentary or disclaimers.
+`;
+
+  // 2. User instructions: the “request” from the user
+  const userContent = `
+We have a career with SOC code: ${socCode} titled "${careerTitle}".
+We need 3 arrays in JSON: "knowledge", "skills", "abilities".
+
+**Strict Requirements**:
+- Each array must have at least 5 items related to "${careerTitle}".
+- Each item: { "elementName": "...", "importanceValue": (1–5), "levelValue": (0–7) }.
+- Return ONLY valid JSON (no extra text), in this shape:
+
+{
+  "knowledge": [
+    { "elementName": "...", "importanceValue": 3, "levelValue": 5 },
+    ...
+  ],
+  "skills": [...],
+  "abilities": [...]
+}
+
+No extra commentary. Exactly 3 arrays, each with at least 5 items.
+Make sure to include relevant domain-specific knowledge (e.g. “Programming,” “Computer Systems,” etc.).
+`;
+
+  // 3. Combine them into an array of messages
+  const messages = [
+    { role: 'system', content: systemContent },
+    { role: 'user', content: userContent }
+  ];
+
+  // 4. Make the GPT-4 call
+  const completion = await openai.chat.completions.create({
+    model: 'gpt-4',
+    messages: messages,
+    temperature: 0.2,
+    max_tokens: 600
+  });
+
+  // 5. Attempt to parse the JSON
+  const rawText = completion?.choices?.[0]?.message?.content?.trim() || '';
+  let parsed = { knowledge: [], skills: [], abilities: [] };
+  try {
+    parsed = JSON.parse(rawText);
+  } catch (err) {
+    console.error('Error parsing GPT-4 JSON:', err, rawText);
+  }
+
+  return parsed; // e.g. { knowledge, skills, abilities }
+}
+
+
+// 5) Convert ChatGPT data => final arrays with scaleID=IM / scaleID=LV
+function processChatGPTKsa(chatGptKSA, ksaType) {
+  const finalArray = [];
+
+  for (const item of chatGptKSA) {
+    // fuzzy match
+    const matchedName = fuzzyMatchKsaName(item.elementName);
+    if (!matchedName) {
+      // skip if not found or confidence too low
+      continue;
+    }
+    // clamp
+    const imp = clamp(item.importanceValue, 1, 5);
+    const lvl = clamp(item.levelValue, 0, 7);
+
+    // produce 2 records: IM + LV
+    finalArray.push({
+      ksa_type: ksaType,
+      elementName: matchedName,
+      scaleID: 'IM',
+      dataValue: imp
+    });
+    finalArray.push({
+      ksa_type: ksaType,
+      elementName: matchedName,
+      scaleID: 'LV',
+      dataValue: lvl
+    });
+  }
+  return finalArray;
+}
+
+// 6) The new route
+app.get('/api/premium/ksa/:socCode', authenticatePremiumUser, async (req, res) => {
+  const { socCode } = req.params;
+  const { careerTitle = '' } = req.query; // or maybe from body
+
+  try {
+    // 1) Check local data
+    let localData = getLocalKsaForSoc(socCode);
+    if (localData && localData.length > 0) {
+      return res.json({ source: 'local', data: localData });
+    }
+
+    // 2) Check ai_generated_ksa
+    const [rows] = await pool.query(
+        'SELECT * FROM ai_generated_ksa WHERE soc_code = ? LIMIT 1',
+        [socCode]
+      );
+    if (rows && rows.length > 0) {
+  const row = rows[0];
+  const knowledge = JSON.parse(row.knowledge_json || '[]');
+  const skills = JSON.parse(row.skills_json || '[]');
+  const abilities = JSON.parse(row.abilities_json || '[]');
+
+  // Check if they are truly empty
+  const isAllEmpty = !knowledge.length && !skills.length && !abilities.length;
+  if (!isAllEmpty) {
+    // We have real data
+    return res.json({
+      source: 'db',
+      data: { knowledge, skills, abilities }
+    });
+  }
+  console.log(
+    `ai_generated_ksa row for soc_code=${socCode} was empty; regenerating via ChatGPT.`
+  );
+}
+
+    // 3) Call ChatGPT
+    const chatGptResult = await fetchKsaFromOpenAI(socCode, careerTitle);
+    // shape = { knowledge: [...], skills: [...], abilities: [...] }
+
+    // 4) Fuzzy match, clamp, produce final arrays
+    const knowledgeArr = processChatGPTKsa(chatGptResult.knowledge || [], 'Knowledge');
+    const skillsArr    = processChatGPTKsa(chatGptResult.skills || [], 'Skill');
+    const abilitiesArr = processChatGPTKsa(chatGptResult.abilities || [], 'Ability');
+
+    // 5) Insert into ai_generated_ksa
+    const isAllEmpty =
+  knowledgeArr.length === 0 &&
+  skillsArr.length === 0 &&
+  abilitiesArr.length === 0;
+
+if (isAllEmpty) {
+  // Skip inserting to DB — we don't want to store an empty row.
+  return res.status(500).json({
+    error: 'ChatGPT returned no KSA data. Please try again later.',
+    data: { knowledge: [], skills: [], abilities: [] }
+  });
+}
+
+// Otherwise, insert into DB as normal:
+await pool.query(`
+  INSERT INTO ai_generated_ksa (
+    soc_code,
+    career_title,
+    knowledge_json,
+    skills_json,
+    abilities_json
+  )
+  VALUES (?, ?, ?, ?, ?)
+`, [
+  socCode,
+  careerTitle,
+  JSON.stringify(knowledgeArr),
+  JSON.stringify(skillsArr),
+  JSON.stringify(abilitiesArr)
+]);
+
+return res.json({
+  source: 'chatgpt',
+  data: {
+    knowledge: knowledgeArr,
+    skills: skillsArr,
+    abilities: abilitiesArr
+  }
+    });
+  } catch (err) {
+    console.error('Error retrieving KSA fallback data:', err);
+    return res.status(500).json({ error: err.message || 'Failed to fetch KSA data.' });
+  }
+});
+
 /* ------------------------------------------------------------------
    RESUME OPTIMIZATION ENDPOINT
    ------------------------------------------------------------------ */

package-lock.json

@@ -25,6 +25,7 @@
         "docx": "^9.5.0",
         "dotenv": "^16.4.7",
         "file-saver": "^2.0.5",
+        "fuse.js": "^7.1.0",
         "helmet": "^8.0.0",
         "jsonwebtoken": "^9.0.2",
         "jwt-decode": "^4.0.0",
@@ -9685,6 +9686,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/fuse.js": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz",
+      "integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/gauge": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",

package.json

@@ -20,6 +20,7 @@
     "docx": "^9.5.0",
     "dotenv": "^16.4.7",
     "file-saver": "^2.0.5",
+    "fuse.js": "^7.1.0",
     "helmet": "^8.0.0",
     "jsonwebtoken": "^9.0.2",
     "jwt-decode": "^4.0.0",

src/components/EducationalProgramsPage.js

@@ -37,7 +37,7 @@ function ensureHttp(urlString) {
   if (/^https?:\/\//i.test(urlString)) {
     return urlString;
   }
-  // Otherwise prepend 'https://' (or 'http://').
+  // Otherwise prepend 'https://'.
   return `https://${urlString}`;
 }
 
@@ -58,6 +58,8 @@ function renderLevel(val) {
   return `${filled}${empty}`;
 }
 
+
+
 function EducationalProgramsPage() {
   const location = useLocation();
   const navigate = useNavigate();
@@ -103,7 +105,7 @@ function EducationalProgramsPage() {
     );
     if (proceed) {
       navigate('/milestone-tracker', { state: { selectedSchool: school } });
-    };
+    }
   };
 
   function getSearchLinks(ksaName, careerTitle) {
@@ -145,24 +147,86 @@ function EducationalProgramsPage() {
     loadKsaData();
   }, []);
 
-  // Filter: only IM >=3, then combine IM+LV
+  async function fetchAiKsaFallback(socCode, careerTitle) {
-  useEffect(() => {
+  // Optionally show a “loading” indicator
-    if (!socCode || !allKsaData.length) {
+  setLoadingKsa(true);
-      setKsaForCareer([]);
+  setKsaError(null);
-      return;
+
+  try {
+    const token = localStorage.getItem('token');
+    if (!token) {
+      throw new Error('No auth token found; cannot fetch AI-based KSAs.');
     }
-    let filtered = allKsaData.filter((r) => r.onetSocCode === socCode);
-    filtered = filtered.filter((r) => r.recommendSuppress !== 'Y');
-    filtered = filtered.filter((r) => ['IM', 'LV'].includes(r.scaleID));
 
-    let combined = combineIMandLV(filtered);
+    // Call the new endpoint in server3.js
-    combined = combined.filter((item) => {
+    const resp = await fetch(
-      return item.importanceValue !== null && item.importanceValue >= 3;
+      `/api/premium/ksa/${socCode}?careerTitle=${encodeURIComponent(careerTitle)}`,
-    });
+      {
-    combined.sort((a, b) => (b.importanceValue || 0) - (a.importanceValue || 0));
+        headers: {
+          Authorization: `Bearer ${token}`
+        }
+      }
+    );
 
+    if (!resp.ok) {
+      throw new Error(`AI KSA endpoint returned status ${resp.status}`);
+    }
+
+    const json = await resp.json();
+    // Expect shape: { source: 'chatgpt' | 'db' | 'local', data: { knowledge, skills, abilities } }
+
+    // The arrays from server may already be in the “IM/LV” format
+    // so we can combine them into one array for display:
+    const finalKsa = [...json.data.knowledge, ...json.data.skills, ...json.data.abilities];
+      finalKsa.forEach(item => {
+        item.onetSocCode = socCode;
+      });
+    const combined = combineIMandLV(finalKsa);
     setKsaForCareer(combined);
-  }, [socCode, allKsaData]);
+  } catch (err) {
+    console.error('Error fetching AI-based KSAs:', err);
+    setKsaError('Could not load AI-based KSAs. Please try again later.');
+    setKsaForCareer([]);
+  } finally {
+    setLoadingKsa(false);
+  }
+}
+ 
+  // Filter: only IM >=3, then combine IM+LV
+useEffect(() => {
+  if (!socCode) {
+    // no career => no KSA
+    setKsaForCareer([]);
+    return;
+  }
+
+  if (!allKsaData.length) {
+    // We haven't loaded local data yet (or it failed to load).
+    // We can either wait, or directly try fallback now.
+    // For example:
+    fetchAiKsaFallback(socCode, careerTitle);
+    return;
+  }
+
+  // Otherwise, we have local data loaded:
+  let filtered = allKsaData.filter((r) => r.onetSocCode === socCode);
+  filtered = filtered.filter((r) => r.recommendSuppress !== 'Y');
+  filtered = filtered.filter((r) => ['IM', 'LV'].includes(r.scaleID));
+
+  let combined = combineIMandLV(filtered);
+  combined = combined.filter((item) => {
+    return item.importanceValue !== null && item.importanceValue >= 3;
+  });
+  combined.sort((a, b) => (b.importanceValue || 0) - (a.importanceValue || 0));
+
+  if (combined.length === 0) {
+    // We found ZERO local KSA records for this socCode => fallback
+    fetchAiKsaFallback(socCode, careerTitle);
+  } else {
+    // We found local KSA data => just use it
+    setKsaForCareer(combined);
+  }
+}, [socCode, allKsaData, careerTitle]);
 
   // Load user profile
   useEffect(() => {
@@ -236,7 +300,6 @@ function EducationalProgramsPage() {
 
   // Sort schools in useMemo
   const filteredAndSortedSchools = useMemo(() => {
-    
     if (!schools) return [];
     let result = [...schools];
 
@@ -291,7 +354,6 @@ function EducationalProgramsPage() {
     const links = !isAbility ? getSearchLinks(elementName, careerTitle) : null;
     const definition = ONET_DEFINITIONS[elementName] || 'No definition available';
 
-
     return (
       <tr key={idx} className="border-b text-sm">
         <td className="p-2 font-medium text-gray-800">
@@ -427,7 +489,7 @@ function EducationalProgramsPage() {
         </div>
       </div>
     );
-  } // <-- MAKE SURE WE DO NOT HAVE EXTRA BRACKETS HERE
+  }
 
   // If no CIP codes => fallback
   if (!cipCodes.length) {
@@ -513,8 +575,8 @@ function EducationalProgramsPage() {
         )}
       </div>
 
-    <div className="grid gap-4 grid-cols-[repeat(auto-fit,minmax(250px,1fr))]">
+      <div className="grid gap-4 grid-cols-[repeat(auto-fit,minmax(250px,1fr))]">
-      {filteredAndSortedSchools.map((school, idx) => {
+        {filteredAndSortedSchools.map((school, idx) => {
           // 1) Ensure the website has a protocol:
           const displayWebsite = ensureHttp(school['Website']);
 
@@ -548,8 +610,8 @@ function EducationalProgramsPage() {
             </div>
           );
         })}
+      </div>
     </div>
-   </div>
   );
 }