dev1/backend/server2.js

import express from 'express';
import axios from 'axios';
import cors from 'cors';
import helmet from 'helmet'; // Import helmet for HTTP security headers
import dotenv from 'dotenv';
import xlsx from 'xlsx'; // Import xlsx to read the Excel file
import path from 'path';
import { fileURLToPath } from 'url'; // Import fileURLToPath to handle the current file's URL
import { open } from 'sqlite';  // Use the open method directly from sqlite package
import sqlite3 from 'sqlite3';
import fs from 'fs';
import readline from 'readline';

// Correct the order of variable initialization
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const rootPath = path.resolve(__dirname, '..'); // Go one level up to the root folder
const env = process.env.NODE_ENV?.trim() || 'development'; // Default to 'development'
const envPath = path.resolve(rootPath, `.env.${env}`); // Use root directory for .env files

// Load environment variables as soon as the server starts
dotenv.config({ path: envPath });  // Ensure this is called at the very top to load all environment variables

// Logging environment variables for debugging
console.log(`Loaded environment variables from: ${envPath}`);
console.log('ONET_USERNAME:', process.env.ONET_USERNAME);
console.log('ONET_PASSWORD:', process.env.ONET_PASSWORD);
console.log('Google Maps API Key:', process.env.GOOGLE_MAPS_API_KEY);


const allowedOrigins = ['http://localhost:3000', 'http://34.16.120.118:3000', 'https://dev1.aptivaai.com'];
const mappingFilePath = '/home/jcoakley/aptiva-dev1-app/public/CIP_to_ONET_SOC.xlsx';
const institutionFilePath = path.resolve(rootPath, 'public/Institution_data.json');

const app = express();
const PORT = process.env.PORT || 5001;

// Initialize database connection
let db;
const initDB = async () => {
  try {
    // Opening SQLite connection using sqlite's open function and sqlite3 as the driver
    db = await open({
      filename: '/home/jcoakley/aptiva-dev1-app/salary_info.db', // Path to SQLite DB file
      driver: sqlite3.Database, // Use sqlite3's Database driver
    });
    console.log('Connected to SQLite database.');
  } catch (error) {
    console.error('Error connecting to database:', error);
  }
};

// Initialize database before starting the server
initDB();

let userProfileDb;
const initUserProfileDb = async () => {
  try {
    userProfileDb = await open({
      filename: '/home/jcoakley/aptiva-dev1-app/user_profile.db', // Path to user_profile.db
      driver: sqlite3.Database
    });
    console.log('Connected to user_profile.db.');
  } catch (error) {
    console.error('Error connecting to user_profile.db:', error);
  }
};

// Initialize user_profile.db before starting the server
initUserProfileDb();

// Add security headers using helmet
app.use(
  helmet({
    contentSecurityPolicy: false, // Disable CSP for now; enable as needed later
    crossOriginEmbedderPolicy: false,
  })
);

// Updated CORS Middleware for dynamic handling of origins and static files
app.use((req, res, next) => {
  const origin = req.headers.origin;

  if (origin && allowedOrigins.includes(origin)) {
    // Allow requests from whitelisted frontend origins
    res.setHeader('Access-Control-Allow-Origin', origin);
    res.setHeader('Access-Control-Allow-Credentials', 'true');
    res.setHeader(
      'Access-Control-Allow-Headers',
      'Authorization, Content-Type, Accept, Origin, X-Requested-With, Access-Control-Allow-Methods'
    );
    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
  } else if (req.path.includes('Institution_data')) {
    // Handle static JSON file CORS
    res.setHeader('Access-Control-Allow-Origin', '*'); // Allow all origins for static file
    res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
    res.setHeader(
      'Access-Control-Allow-Headers',
      'Content-Type, Accept, Origin, X-Requested-With'
    );
  } else {
    // Default headers for other unhandled cases
    res.setHeader('Access-Control-Allow-Origin', '*');
    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
    res.setHeader(
      'Access-Control-Allow-Headers',
      'Authorization, Content-Type, Accept, Origin, X-Requested-With'
    );
  }

  if (req.method === 'OPTIONS') {
    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
    res.setHeader('Access-Control-Allow-Headers', 'Authorization, Content-Type');
    return res.status(204).end(); // Use 204 for no content
  }
  

  next();
});

// Middleware to parse JSON bodies
app.use(express.json());
app.use(express.static(path.join(__dirname, 'public')));

app.use((req, res, next) => {
  console.log(`Path: ${req.path}, Method: ${req.method}, Body:`, req.body);
  next();
});


// Route to fetch O*Net Interest Inventory questions with pagination
app.get('/api/onet/questions', async (req, res) => {
  const { start, end } = req.query;

  if (!start || !end) {
    return res.status(400).json({ error: 'Start and end parameters are required' });
  }

  try {
    const questions = [];
    let currentStart = parseInt(start, 10);
    let currentEnd = parseInt(end, 10);

    while (currentStart <= currentEnd) {
      // Fetch questions from O*Net API for the current range
      const response = await axios.get(
        `https://services.onetcenter.org/ws/mnm/interestprofiler/questions?start=${currentStart}&end=${Math.min(currentEnd, currentStart + 11)}`,
        {
          auth: {
            username: process.env.ONET_USERNAME,
            password: process.env.ONET_PASSWORD,
          },
        }
      );
      console.log('O*Net Response:', response.data);

      // Add questions to the result set
      if (response.data.question && Array.isArray(response.data.question)) {
        questions.push(...response.data.question);
      }

      // Check if there's a next page
      const nextLink = response.data.link?.find((link) => link.rel === 'next');
      if (nextLink) {
        // Update start and end based on the "next" link
        const nextParams = new URLSearchParams(nextLink.href.split('?')[1]);
        currentStart = parseInt(nextParams.get('start'), 10);
        currentEnd = parseInt(nextParams.get('end'), 10);
      } else {
        break; // Stop if there are no more pages
      }
    }

    res.status(200).json({ questions });
  } catch (error) {
    console.error('Error fetching O*Net questions:', error.message);
    res.status(500).json({ error: 'Failed to fetch O*Net questions' });
  }
});

// Helper function to geocode an address or zip code
const geocodeZipCode = async (zipCode) => {
  const apiKey = process.env.GOOGLE_MAPS_API_KEY; 
  if (!apiKey) {
    console.error('Google Maps API Key is missing.');
  } else {
    console.log('Google Maps API Key loaded:', apiKey);
  }
  try {
    const geocodeUrl = `https://maps.googleapis.com/maps/api/geocode/json?address=${encodeURIComponent(zipCode)}&components=country:US&key=${apiKey}`;
    console.log('Constructed Geocode URL:', geocodeUrl);  // Log the geocoding URL for debugging

    const response = await axios.get(geocodeUrl);

    if (response.data.status === 'OK' && response.data.results.length > 0) {
      const location = response.data.results[0].geometry.location;
      return location;  // Return the latitude and longitude
    } else {
      throw new Error('Geocoding failed');
    }
  } catch (error) {
    console.error('Error geocoding ZIP code:', error.message);
    return null;
  }
};


app.post('/api/maps/distance', async (req, res) => {
  const { userZipcode, destinations } = req.body;

  if (!userZipcode || !destinations) {
    console.error('Missing required parameters:', { userZipcode, destinations });
    return res.status(400).json({ error: 'User ZIP code and destination address are required.' });
  }

  try {
    const googleMapsApiKey = process.env.GOOGLE_MAPS_API_KEY; // Get the API key

    // Geocode the user's ZIP code
    const userLocation = await geocodeZipCode(userZipcode);
    if (!userLocation) {
      return res.status(400).json({ error: 'Unable to geocode user ZIP code.' });
    }

    const origins = `${userLocation.lat},${userLocation.lng}`;  // User's location as lat/lng

    console.log('Request Payload:', { userZipcode, destinations });  // Log the parameters

    // Call the Distance Matrix API using the geocoded user location and school address
    const distanceUrl = `https://maps.googleapis.com/maps/api/distancematrix/json?origins=${origins}&destinations=${encodeURIComponent(destinations)}&units=imperial&key=${googleMapsApiKey}`;
    const distanceResponse = await axios.get(distanceUrl);
    console.log('Distance API Request URL:', distanceUrl); // Log the final request URL

    if (distanceResponse.data.status !== 'OK') {
      return res.status(500).json({ error: 'Error fetching distance from Google Maps API' });
    }

    const { distance, duration } = distanceResponse.data.rows[0].elements[0];
    res.json({ distance: distance.text, duration: duration.text });

  } catch (error) {
    console.error('Error during distance calculation:', error);
    res.status(500).json({ error: 'Internal server error', details: error.message });
  }
});


// Load the economic projections data from the Excel file
const projectionsFilePath = path.resolve(__dirname, '..', 'public', 'occprj.xlsx'); // Adjusted path
const workbook = xlsx.readFile(projectionsFilePath);
const sheet = workbook.Sheets['GAOccProj 2022-2032']; // The sheet with your data
const projectionsData = xlsx.utils.sheet_to_json(sheet, { header: 1 }); // Convert the sheet to JSON

const loadMapping = () => {
  try {
    const workbook = xlsx.readFile(mappingFilePath); // Read the Excel file
    const sheet = workbook.Sheets[workbook.SheetNames[0]]; // Assuming the first sheet
    const data = xlsx.utils.sheet_to_json(sheet); // Convert the sheet to JSON
    return data; // Return the data from the Excel sheet
  } catch (error) {
    console.error('Error reading the CIP to ONET SOC mapping file:', error);
    return []; // Return an empty array if there's an issue loading the mapping
  }
};

// Load the mapping data from the Excel file
const socToCipMapping = loadMapping();
if (socToCipMapping.length === 0) {
  console.error("SOC to CIP mapping data is empty.");
}

const filterHigherEducationCareers = (careers) => {
  return careers
    .map((career) => {
      const educationLevel = career.education;
      if (
        ![
          "No formal education",
          "High school",
          "Some college, no degree",
        ].includes(educationLevel)
      ) {
        return {
          href: career.href,
          fit: career.fit,
          code: career.code,
          title: career.title,
          tags: career.tags,
        };
      }
    })
    .filter((career) => career); // Remove undefined values
};

// Route to handle submission of answers to O*Net API for career suggestions and RIASEC scores
app.post('/api/onet/submit_answers', async (req, res) => {
  console.log('POST /api/onet/submit_answers hit');
  const { answers } = req.body; // Get answers string from the request body

  if (!answers || answers.length !== 60) {
    console.error('Invalid answers provided:', answers);
    return res.status(400).json({ error: 'Answers parameter must be a 60-character string.' });
  }

  try {
    // URLs for career suggestions and RIASEC scores
    const careerUrl = `https://services.onetcenter.org/ws/mnm/interestprofiler/careers?answers=${answers}&start=1&end=1000`;
    const resultsUrl = `https://services.onetcenter.org/ws/mnm/interestprofiler/results?answers=${answers}`;

    // Fetch career suggestions
    console.log('Fetching career suggestions from:', careerUrl);
    const careerResponse = await axios.get(careerUrl, {
      auth: {
        username: process.env.ONET_USERNAME,
        password: process.env.ONET_PASSWORD
      },
      headers: {
        'Accept': 'application/json'
      }
    });

    console.log('Career API Response:', JSON.stringify(careerResponse.data, null, 2));

    // Fetch RIASEC scores
    console.log('Fetching RIASEC scores from:', resultsUrl);
    const resultsResponse = await axios.get(resultsUrl, {
      auth: {
        username: process.env.ONET_USERNAME,
        password: process.env.ONET_PASSWORD
      },
      headers: {
        'Accept': 'application/json'
      }
    });

    // Extract career suggestions and RIASEC scores
    const careerSuggestions = careerResponse.data.career || []; // Initialize the array
    const riaSecScores = resultsResponse.data.result || [];

    // Apply filtering based on education requirements INSIDE the try block
    const filteredCareers = filterHigherEducationCareers(careerSuggestions);

    // Logging for debugging
    console.log('Raw Career Suggestions:', careerSuggestions);
    console.log('RIASEC Scores:', riaSecScores);
    console.log('Filtered Careers:', filteredCareers);

    // Send the combined data to the frontend
    res.status(200).json({
      careers: filteredCareers,
      riaSecScores: riaSecScores
    });
  } catch (error) {
    console.error('Error fetching data from O*Net API:', error.response?.data || error.message);
    res.status(500).json({
      error: 'Failed to fetch data from O*Net API',
      details: error.response?.data || error.message
    });
  }
});


app.get('/api/onet/career-details/:socCode', async (req, res) => {
  const { socCode } = req.params;

  if (!socCode) {
    return res.status(400).json({ error: 'SOC Code is required.' });
  }

  try {
    const response = await axios.get(
      `https://services.onetcenter.org/ws/mnm/careers/${socCode}`,
      {
        auth: {
          username: process.env.ONET_USERNAME,
          password: process.env.ONET_PASSWORD,
        },
        headers: {
          Accept: 'application/json',
        },
      }
    );

    res.status(200).json(response.data); // Forward the API response to the frontend
  } catch (error) {
    console.error('Error fetching career details:', error.message);
    res.status(500).json({ error: 'Failed to fetch career details from O*NET API.' });
  }
});

// Endpoint to fetch career description and tasks from O*Net
app.get('/api/onet/career-description/:socCode', async (req, res) => {
  const { socCode } = req.params;

  if (!socCode) {
    return res.status(400).json({ error: 'SOC Code is required.' });
  }

  try {
    // Fetch career details using the O*Net API for the given SOC code
    const response = await axios.get(
      `https://services.onetcenter.org/ws/mnm/careers/${socCode}`,
      {
        auth: {
          username: process.env.ONET_USERNAME,
          password: process.env.ONET_PASSWORD,
        },
        headers: {
          'Accept': 'application/json',
        },
      }
    );

    // Check if the response contains the necessary data
    if (response.data && response.data.title) {
      const { title, what_they_do, on_the_job } = response.data;

      // Extract the tasks from the 'on_the_job' field
      const tasks = on_the_job?.task || [];

      // Prepare the data for the frontend
      const careerOverview = {
        description: what_they_do || 'No description available',
        tasks: tasks.length ? tasks : ['No tasks available'],
      };

      res.status(200).json(careerOverview);
    } else {
      res.status(404).json({ error: 'Career not found for the provided SOC code.' });
    }
  } catch (error) {
    console.error('Error fetching career description and tasks:', error.message);
    res.status(500).json({ error: 'Failed to fetch career description and tasks from O*Net.' });
  }
});
    
// Route to handle fetching CIP code based on SOC code
app.get('/api/cip/:socCode', (req, res) => {
  const { socCode } = req.params;
  console.log(`Received SOC Code: ${socCode.trim()}`);

  for (let row of socToCipMapping) {
    const mappedSOC = row['O*NET-SOC 2019 Code']?.trim(); // Trim spaces
    if (mappedSOC === socCode.trim()) {
      console.log('Found matching CIP Code:', row['2020 CIP Code']); 
      return res.json({ cipCode: row['2020 CIP Code'] });
    }
  }

  console.error('SOC code not found in mapping:', socCode);
  res.status(404).json({ error: 'CIP code not found for this SOC code' });
});

// Filtered schools endpoint
app.get('/api/schools', (req, res) => {
  const { cipCode, state } = req.query;
  console.log('Query Params:', { cipCode, state });

  // Validate required parameters
  if (!cipCode || !state) {
    return res.status(400).json({ error: 'CIP Code and State are required.' });
  }

  try {
    // Normalize CIP Code
    const matchedCIP = cipCode.replace('.', '').slice(0, 4);

    let schoolsData = [];
try {
  const rawData = fs.readFileSync(institutionFilePath, 'utf8');
  schoolsData = JSON.parse(rawData); // Parse entire JSON array
} catch (error) {
  console.error('Error parsing institution data:', error.message);
  return res.status(500).json({ error: 'Failed to load schools data.' });
}


    // Filter based on CIP code and state
    const filteredSchools = schoolsData.filter((school) => {
      const schoolCIP = school['CIPCODE']?.toString().replace('.', '').slice(0, 4);
      const schoolState = school['State']?.toUpperCase().trim();

      return (
        schoolCIP.startsWith(matchedCIP) &&
        schoolState === state.toUpperCase().trim()
      );
    });

    console.log('Filtered Schools Count:', filteredSchools.length);
    res.json(filteredSchools); // Return filtered results
  } catch (error) {
    console.error('Error reading Institution data:', error.message);
    res.status(500).json({ error: 'Failed to load schools data.' });
  }
});


// Route to handle fetching tuition data using CIP code and state as query parameters
app.get('/api/tuition', (req, res) => {
  const { cipCode, state } = req.query; // Extract query parameters
  console.log(`Received CIP Code: ${cipCode}, State: ${state}`);

  // Validate required parameters
  if (!cipCode || !state) {
    return res.status(400).json({ error: 'CIP Code and State are required.' });
  }

  try {
    const schoolsData = JSON.parse(fs.readFileSync(institutionFilePath, 'utf8'));

    console.log('Loaded Tuition Data:', schoolsData.length);

    // Filter data by CIP Code and State
    const filteredData = schoolsData.filter((school) => {
      const cipCodeValue = school['CIPCODE']?.toString().replace(/[^0-9]/g, ''); // Normalize CIP Code
      const stateValue = school['State']?.toUpperCase().trim(); // Normalize State

      return (
        cipCodeValue.startsWith(cipCode) && // Partial match CIP Code
        stateValue === state.toUpperCase().trim() // Exact match State
      );
    });

    console.log('Filtered Tuition Data Count:', filteredData.length);
    res.json(filteredData); // Return filtered results
  } catch (error) {
    console.error('Error reading tuition data:', error.message);
    res.status(500).json({ error: 'Failed to load tuition data.' });
  }
});


// Route to handle fetching economic projections for SOC code
app.get('/api/projections/:socCode', (req, res) => {
  const { socCode } = req.params;
  console.log('Received SOC Code:', socCode);

  const socRow = projectionsData.find(row => row[0] === socCode);

  if (socRow) {
    const projections = {
      "SOC Code": socRow[0],
      "Occupation": socRow[2],
      "2022 Employment": socRow[3],
      "2032 Employment": socRow[4],
      "Total Change": socRow[5],
      "Annual Openings": socRow[7],
      "Labor Force Exits": socRow[8],
      "Projected Growth": socRow[9]
    };

    res.status(200).json(projections);
  } else {
    res.status(404).json({ error: 'SOC Code not found' });
  }
});

app.get('/api/salary', async (req, res) => {
  const { socCode, area } = req.query;

  console.log('Received /api/salary request:', { socCode, area });

  if (!socCode || !area) {
    console.error('Missing required parameters:', { socCode, area });
    return res.status(400).json({ error: 'SOC Code and Area are required' });
  }

  const query = `
    SELECT A_PCT10, A_PCT25, A_MEDIAN, A_PCT75, A_PCT90
    FROM salary_data
    WHERE OCC_CODE = ? AND AREA_TITLE = ?
  `;

  try {
    if (process.env.DEBUG === 'true') {
      console.log('Executing query:', query, 'with params:', [socCode, area]);
    }
    
    // Use async/await for better error handling
    const row = await db.get(query, [socCode, area]);

    if (!row) {
      console.log('No salary data found for:', { socCode, area });
      return res.status(404).json({ error: 'No salary data found for the given SOC Code and Area' });
    }

    console.log('Salary data retrieved:', row);
    res.json(row);
  } catch (error) {
    console.error('Error executing query:', error.message);
    res.status(500).json({ error: 'Failed to fetch salary data' });
  }
});

// Route to fetch job zones and check for missing salary data
app.post('/api/job-zones', async (req, res) => {
  const { socCodes } = req.body;

  if (!socCodes || !Array.isArray(socCodes) || socCodes.length === 0) {
    return res.status(400).json({ error: "SOC Codes are required." });
  }

  try {
    // Ensure SOC codes are formatted correctly (no decimals)
    const formattedSocCodes = socCodes.map(code => {
      let cleanedCode = code.trim().replace(/\./g, ""); // Remove periods
      if (!cleanedCode.includes("-") && cleanedCode.length === 6) {
        cleanedCode = cleanedCode.slice(0, 2) + "-" + cleanedCode.slice(2, 6);
      }
      return cleanedCode.slice(0, 7); // Keep first 7 characters
    });

    const placeholders = formattedSocCodes.map(() => "?").join(",");
    const query = `
  SELECT OCC_CODE, JOB_ZONE, 
         A_MEDIAN, A_PCT10, A_PCT25, A_PCT75
  FROM salary_data 
  WHERE OCC_CODE IN (${placeholders})
`;

const rows = await db.all(query, formattedSocCodes);

// Log what is being retrieved from the database
console.log("Salary Data Query Results:", rows);

// Now process `limited_data` flag
const jobZoneMapping = rows.reduce((acc, row) => {
  // Convert empty fields or NULL to a falsy value
  const isMissingData = [row.A_MEDIAN, row.A_PCT10, row.A_PCT25, row.A_PCT75]
    .some(value => value === null || value === '' || value === '#' || value === '*');

  acc[row.OCC_CODE] = {
    job_zone: row.JOB_ZONE,
    limited_data: isMissingData ? 1 : 0  // Set limited_data flag correctly
  };

  return acc;
}, {});

console.log("Job Zone & Limited Data Mapping:", jobZoneMapping);
res.json(jobZoneMapping);

  } catch (error) {
    console.error("Error fetching job zones:", error);
    res.status(500).json({ error: "Failed to fetch job zones." });
  }
});


// Route to fetch user profile by ID
app.get('/api/user-profile/:id', (req, res) => {
  const { id } = req.params;

  if (!id) {
    return res.status(400).json({ error: 'Profile ID is required' });
  }

  const query = `SELECT area, zipcode FROM user_profile WHERE id = ?`;
  db.get(query, [id], (err, row) => {
    if (err) {
      console.error('Error fetching user profile:', err.message);
      return res.status(500).json({ error: 'Failed to fetch user profile' });
    }

    if (!row) {
      return res.status(404).json({ error: 'Profile not found' });
    }

    res.json({ area: row.area, zipcode: row.zipcode });
  });
});


// Start the Express server
app.listen(PORT, () => {
  console.log(`Server running on https://34.16.120.118:${PORT}`);
});